From b9b543b23f0a35f55d8e3598abc948567b46f899 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Nov 2023 16:58:14 +0200 Subject: [PATCH 01/54] Make `warpSync` a self-contained syncing strategy --- .../client/network/sync/src/chain_sync.rs | 5 +- substrate/client/network/sync/src/warp.rs | 620 ++++++++++++++---- 2 files changed, 486 insertions(+), 139 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index 3825cfa33f73b..50e07e6c2a36f 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -1873,8 +1873,7 @@ where /// A batch of blocks have been processed, with or without errors. /// /// Call this when a batch of blocks have been processed by the import - /// queue, with or without errors. If an error is returned, the pending response - /// from the peer must be dropped. + /// queue, with or without errors. pub fn on_blocks_processed( &mut self, imported: usize, @@ -2324,7 +2323,7 @@ where /// Returns the number of the first block in the sequence. /// /// It is expected that `blocks` are in ascending order. -fn validate_blocks( +pub fn validate_blocks( blocks: &Vec>, peer_id: &PeerId, request: Option>, diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 169b3de35aa1b..a2317926c804c 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -21,23 +21,31 @@ pub use sp_consensus_grandpa::{AuthorityList, SetId}; use crate::{ - schema::v1::{StateRequest, StateResponse}, + chain_sync::validate_blocks, + schema::v1::StateResponse, state::{ImportResult, StateSync}, + types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse}, }; use codec::{Decode, Encode}; use futures::channel::oneshot; -use log::error; +use libp2p::PeerId; +use log::{debug, error, info, trace}; use sc_client_api::ProofProvider; +use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::message::{ BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; use sp_blockchain::HeaderBackend; +use sp_consensus::BlockOrigin; use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; -use std::{fmt, sync::Arc}; +use std::{collections::HashMap, fmt, sync::Arc}; /// Log target for this file. const LOG_TARGET: &'static str = "sync"; +/// Number of peers that need to be connected before warp sync is started. +const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; + /// Scale-encoded warp sync proof response. pub struct EncodedProof(pub Vec); @@ -76,6 +84,31 @@ pub trait WarpSyncProvider: Send + Sync { fn current_authorities(&self) -> AuthorityList; } +mod rep { + use sc_network::ReputationChange as Rep; + + /// Unexpected response received form a peer + pub const UNEXPECTED_RESPONSE: Rep = Rep::new(-(1 << 29), "Unexpected response"); + + /// Peer provided invalid warp proof data + pub const BAD_WARP_PROOF: Rep = Rep::new(-(1 << 29), "Bad warp proof"); + + /// Peer did not provide us with advertised block data. + pub const NO_BLOCK: Rep = Rep::new(-(1 << 29), "No requested block data"); + + /// Reputation change for peers which send us non-requested block data. + pub const NOT_REQUESTED: Rep = Rep::new(-(1 << 29), "Not requested block data"); + + /// Reputation change for peers which send us a block which we fail to verify. + pub const VERIFICATION_FAIL: Rep = Rep::new(-(1 << 29), "Block verification failed"); + + /// Peer response data does not have requested bits. + pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); + + /// Reputation change for peers which send us a known bad state. + pub const BAD_STATE: Rep = Rep::new(-(1 << 29), "Bad state"); +} + /// Reported warp sync phase. #[derive(Clone, Eq, PartialEq, Debug)] pub enum WarpSyncPhase { @@ -93,6 +126,8 @@ pub enum WarpSyncPhase { ImportingState, /// Downloading block history. DownloadingBlocks(NumberFor), + /// Warp sync is complete. + Complete, } impl fmt::Display for WarpSyncPhase { @@ -106,6 +141,7 @@ impl fmt::Display for WarpSyncPhase { Self::DownloadingState => write!(f, "Downloading state"), Self::ImportingState => write!(f, "Importing state"), Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), + Self::Complete => write!(f, "Warp sync is complete"), } } } @@ -154,6 +190,8 @@ impl WarpSyncParams { /// Warp sync phase. enum Phase { + /// Waiting for enough peers to connect. + WaitingForPeers { warp_sync_provider: Arc> }, /// Downloading warp proofs. WarpProof { set_id: SetId, @@ -166,8 +204,10 @@ enum Phase { PendingTargetBlock, /// Downloading target block. TargetBlock(B::Header), - /// Downloading state. + /// Downloading and importing state. State(StateSync), + /// Warp sync is complete. + Complete, } /// Import warp proof result. @@ -186,11 +226,47 @@ pub enum TargetBlockImportResult { BadResponse, } +enum PeerState { + Available, + DownloadingProofs, + DownloadingTargetBlock, + DownloadingState, +} + +impl PeerState { + fn is_available(&self) -> bool { + matches!(self, PeerState::Available) + } +} + +struct Peer { + best_number: NumberFor, + state: PeerState, +} + +enum WarpSyncAction { + /// Send warp proof request to peer. + SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, + /// Send block request to peer. Always implies dropping a stale block request to the same peer. + SendBlockRequest { peer_id: PeerId, request: BlockRequest }, + /// Send state request to peer. + SendStateRequest { peer_id: PeerId, request: OpaqueStateRequest }, + /// Disconnect and report peer. + DropPeer(BadPeer), + /// Import blocks. + ImportBlocks { origin: BlockOrigin, blocks: Vec> }, + /// Warp sync has finished. + Finished, +} + /// Warp sync state machine. Accumulates warp proofs and state. pub struct WarpSync { phase: Phase, client: Arc, total_proof_bytes: u64, + total_state_bytes: u64, + peers: HashMap>, + actions: Vec>, } impl WarpSync @@ -202,19 +278,19 @@ where /// authorities. Alternatively we can pass a target block when we want to skip downloading /// proofs, in this case we will continue polling until the target block is known. pub fn new(client: Arc, warp_sync_config: WarpSyncConfig) -> Self { - let last_hash = client.hash(Zero::zero()).unwrap().expect("Genesis header always exists"); - match warp_sync_config { - WarpSyncConfig::WithProvider(warp_sync_provider) => { - let phase = Phase::WarpProof { - set_id: 0, - authorities: warp_sync_provider.current_authorities(), - last_hash, - warp_sync_provider: warp_sync_provider.clone(), - }; - Self { client, phase, total_proof_bytes: 0 } - }, - WarpSyncConfig::WaitForTarget => - Self { client, phase: Phase::PendingTargetBlock, total_proof_bytes: 0 }, + let phase = match warp_sync_config { + WarpSyncConfig::WithProvider(warp_sync_provider) => + Phase::WaitingForPeers { warp_sync_provider }, + WarpSyncConfig::WaitForTarget => Phase::PendingTargetBlock, + }; + + Self { + client, + phase, + total_proof_bytes: 0, + total_state_bytes: 0, + peers: HashMap::new(), + actions: Vec::new(), } } @@ -232,154 +308,398 @@ where self.phase = Phase::TargetBlock(header); } - /// Validate and import a state response. - pub fn import_state(&mut self, response: StateResponse) -> ImportResult { - match &mut self.phase { - Phase::WarpProof { .. } | Phase::TargetBlock(_) | Phase::PendingTargetBlock { .. } => { - log::debug!(target: "sync", "Unexpected state response"); - ImportResult::BadResponse - }, - Phase::State(sync) => sync.import(response), + pub fn new_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { + self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); + + self.try_to_start_warp_sync(); + } + + pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + self.peers.remove(peer_id); + } + + fn try_to_start_warp_sync(&mut self) { + let Phase::WaitingForPeers { warp_sync_provider } = self.phase else { return }; + + if self.peers.len() < MIN_PEERS_TO_START_WARP_SYNC { + return } + + let genesis_hash = + self.client.hash(Zero::zero()).unwrap().expect("Genesis hash always exists"); + self.phase = Phase::WarpProof { + set_id: 0, + authorities: warp_sync_provider.current_authorities(), + last_hash: genesis_hash, + warp_sync_provider, + }; + trace!(target: LOG_TARGET, "Started warp sync with {} peers.", self.peers.len()); } - /// Validate and import a warp proof response. - pub fn import_warp_proof(&mut self, response: EncodedProof) -> WarpProofImportResult { - match &mut self.phase { - Phase::State(_) | Phase::TargetBlock(_) | Phase::PendingTargetBlock { .. } => { - log::debug!(target: "sync", "Unexpected warp proof response"); - WarpProofImportResult::BadResponse + /// Process warp proof response. + pub fn on_warp_proof_response(&mut self, peer_id: &PeerId, response: EncodedProof) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.state = PeerState::Available; + } + + let Phase::WarpProof { set_id, authorities, last_hash, warp_sync_provider } = + &mut self.phase + else { + debug!(target: LOG_TARGET, "Unexpected warp proof response"); + self.actions + .push(WarpSyncAction::DropPeer(BadPeer(*peer_id, rep::UNEXPECTED_RESPONSE))); + return + }; + + match warp_sync_provider.verify(&response, *set_id, authorities.clone()) { + Err(e) => { + debug!(target: LOG_TARGET, "Bad warp proof response: {}", e); + self.actions + .push(WarpSyncAction::DropPeer(BadPeer(*peer_id, rep::BAD_WARP_PROOF))) + }, + Ok(VerificationResult::Partial(new_set_id, new_authorities, new_last_hash)) => { + log::debug!(target: LOG_TARGET, "Verified partial proof, set_id={:?}", new_set_id); + *set_id = new_set_id; + *authorities = new_authorities; + *last_hash = new_last_hash; + self.total_proof_bytes += response.0.len() as u64; + }, + Ok(VerificationResult::Complete(new_set_id, _, header)) => { + log::debug!( + target: LOG_TARGET, + "Verified complete proof, set_id={:?}. Continuing with target block download: {} ({}).", + new_set_id, + header.hash(), + header.number(), + ); + self.total_proof_bytes += response.0.len() as u64; + self.phase = Phase::TargetBlock(header); }, - Phase::WarpProof { set_id, authorities, last_hash, warp_sync_provider } => - match warp_sync_provider.verify(&response, *set_id, authorities.clone()) { - Err(e) => { - log::debug!(target: "sync", "Bad warp proof response: {}", e); - WarpProofImportResult::BadResponse - }, - Ok(VerificationResult::Partial(new_set_id, new_authorities, new_last_hash)) => { - log::debug!(target: "sync", "Verified partial proof, set_id={:?}", new_set_id); - *set_id = new_set_id; - *authorities = new_authorities; - *last_hash = new_last_hash; - self.total_proof_bytes += response.0.len() as u64; - WarpProofImportResult::Success - }, - Ok(VerificationResult::Complete(new_set_id, _, header)) => { - log::debug!(target: "sync", "Verified complete proof, set_id={:?}", new_set_id); - self.total_proof_bytes += response.0.len() as u64; - self.phase = Phase::TargetBlock(header); - WarpProofImportResult::Success - }, - }, } } - /// Import the target block body. - pub fn import_target_block(&mut self, block: BlockData) -> TargetBlockImportResult { - match &mut self.phase { - Phase::WarpProof { .. } | Phase::State(_) | Phase::PendingTargetBlock { .. } => { - log::debug!(target: "sync", "Unexpected target block response"); - TargetBlockImportResult::BadResponse - }, - Phase::TargetBlock(header) => - if let Some(block_header) = &block.header { - if block_header == header { - if block.body.is_some() { - let state_sync = StateSync::new( - self.client.clone(), - header.clone(), - block.body, - block.justifications, - false, - ); - self.phase = Phase::State(state_sync); - TargetBlockImportResult::Success - } else { - log::debug!( - target: "sync", - "Importing target block failed: missing body.", - ); - TargetBlockImportResult::BadResponse - } - } else { - log::debug!( - target: "sync", - "Importing target block failed: different header.", - ); - TargetBlockImportResult::BadResponse - } - } else { - log::debug!(target: "sync", "Importing target block failed: missing header."); - TargetBlockImportResult::BadResponse - }, + /// Process (target) block response. + pub fn on_block_response( + &mut self, + peer_id: PeerId, + request: BlockRequest, + blocks: Vec>, + ) { + if let Err(bad_peer) = self.on_block_response_inner(peer_id, request, blocks) { + self.actions.push(WarpSyncAction::DropPeer(bad_peer)); } } - /// Produce next state request. - pub fn next_state_request(&self) -> Option { - match &self.phase { - Phase::WarpProof { .. } | Phase::TargetBlock(_) | Phase::PendingTargetBlock { .. } => - None, - Phase::State(sync) => Some(sync.next_request()), + fn on_block_response_inner( + &mut self, + peer_id: PeerId, + request: BlockRequest, + blocks: Vec>, + ) -> Result<(), BadPeer> { + if let Some(peer) = self.peers.get_mut(&peer_id) { + peer.state = PeerState::Available; + } + + let Phase::TargetBlock(header) = &mut self.phase else { + debug!(target: "sync", "Unexpected target block response from {peer_id}"); + return Err(BadPeer(peer_id, rep::UNEXPECTED_RESPONSE)) + }; + + if blocks.is_empty() { + debug!( + target: LOG_TARGET, + "Downloading target block failed: empty block response from {peer_id}", + ); + return Err(BadPeer(peer_id, rep::NO_BLOCK)) } + + if blocks.len() > 1 { + debug!( + target: LOG_TARGET, + "Too many blocks ({}) in warp target block response from {peer_id}", + blocks.len(), + ); + return Err(BadPeer(peer_id, rep::NOT_REQUESTED)) + } + + validate_blocks::(&blocks, &peer_id, Some(request))?; + + let block = blocks.pop().expect("`blocks` len checked above; qed"); + + let Some(block_header) = &block.header else { + debug!( + target: "sync", + "Downloading target block failed: missing header in response from {peer_id}.", + ); + return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) + }; + + if block_header != header { + debug!( + target: "sync", + "Downloading target block failed: different header in response from {peer_id}.", + ); + return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) + } + + if block.body.is_none() { + debug!( + target: "sync", + "Downloading target block failed: missing body in response from {peer_id}.", + ); + return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) + } + + debug!( + target: LOG_TARGET, + "Downloaded target block {} ({}), continuing with state sync.", + header.hash(), + header.number(), + ); + let state_sync = StateSync::new( + self.client.clone(), + header.clone(), + block.body, + block.justifications, + false, + ); + self.phase = Phase::State(state_sync); + Ok(()) } - /// Produce next warp proof request. - pub fn next_warp_proof_request(&self) -> Option> { - match &self.phase { - Phase::WarpProof { last_hash, .. } => Some(WarpProofRequest { begin: *last_hash }), - Phase::TargetBlock(_) | Phase::State(_) | Phase::PendingTargetBlock { .. } => None, + // Process state response. + pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { + if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { + self.actions.push(WarpSyncAction::DropPeer(bad_peer)); } } - /// Produce next target block request. - pub fn next_target_block_request(&self) -> Option<(NumberFor, BlockRequest)> { - match &self.phase { - Phase::WarpProof { .. } | Phase::State(_) | Phase::PendingTargetBlock { .. } => None, - Phase::TargetBlock(header) => { - let request = BlockRequest:: { - id: 0, - fields: BlockAttributes::HEADER | - BlockAttributes::BODY | BlockAttributes::JUSTIFICATION, - from: FromBlock::Hash(header.hash()), - direction: Direction::Ascending, - max: Some(1), + fn on_state_response_inner( + &mut self, + peer_id: PeerId, + response: OpaqueStateResponse, + ) -> Result<(), BadPeer> { + if let Some(peer) = self.peers.get_mut(&peer_id) { + peer.state = PeerState::Available; + } + + let Phase::State(state_sync) = &mut self.phase else { + debug!(target: "sync", "Unexpected state response"); + return Err(BadPeer(peer_id, rep::UNEXPECTED_RESPONSE)) + }; + + let response: Box = response.0.downcast().map_err(|_error| { + error!( + target: LOG_TARGET, + "Failed to downcast opaque state response, this is an implementation bug." + ); + + BadPeer(peer_id, rep::BAD_RESPONSE) + })?; + + debug!( + target: LOG_TARGET, + "Importing state data from {} with {} keys, {} proof nodes.", + peer_id, + response.entries.len(), + response.proof.len(), + ); + + let import_result = state_sync.import(*response); + self.total_state_bytes = state_sync.progress().size; + + match import_result { + ImportResult::Import(hash, header, state, body, justifications) => { + let origin = BlockOrigin::NetworkInitialSync; + let block = IncomingBlock { + hash, + header: Some(header), + body, + indexed_body: None, + justifications, + origin: None, + allow_missing_state: true, + import_existing: true, + skip_execution: true, + state: Some(state), }; - Some((*header.number(), request)) + debug!(target: LOG_TARGET, "State download is complete. Import is queued"); + self.actions.push(WarpSyncAction::ImportBlocks { origin, blocks: vec![block] }); + Ok(()) + }, + ImportResult::Continue => Ok(()), + ImportResult::BadResponse => { + debug!(target: LOG_TARGET, "Bad state data received from {peer_id}"); + Err(BadPeer(peer_id, rep::BAD_STATE)) }, } } - /// Return target block hash if it is known. - pub fn target_block_hash(&self) -> Option { - match &self.phase { - Phase::WarpProof { .. } | Phase::TargetBlock(_) | Phase::PendingTargetBlock { .. } => - None, - Phase::State(s) => Some(s.target()), + /// A batch of blocks have been processed, with or without errors. + /// + /// Normally this should be called when target block with state is imported. + pub fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, B::Hash)>, + ) { + let Phase::State(state_sync) = &self.phase else { + debug!(target: LOG_TARGET, "Unexpected block import of {imported} of {count}."); + return + }; + + trace!(target: LOG_TARGET, "Warp sync: imported {imported} of {count}."); + + for (result, hash) in results { + if hash != state_sync.target() { + debug!( + target: LOG_TARGET, + "Unexpected block processed: {hash} with result {result:?}.", + ); + continue + } + + if let Err(e) = result { + error!( + target: LOG_TARGET, + "Warp sync failed. Failed to import target block with state: {e:?}." + ); + } + + let total_mib = (self.total_proof_bytes + state_sync.progress().size) / (1024 * 1024); + info!( + target: LOG_TARGET, + "Warp sync is complete ({total_mib} MiB), continuing with block sync.", + ); + + self.phase = Phase::Complete; + self.actions.push(WarpSyncAction::Finished); } } - /// Return target block number if it is known. - pub fn target_block_number(&self) -> Option> { - match &self.phase { - Phase::WarpProof { .. } | Phase::PendingTargetBlock { .. } => None, - Phase::TargetBlock(header) => Some(*header.number()), - Phase::State(s) => Some(s.target_block_num()), + /// Get candidate for warp/block request. + fn select_synced_available_peer( + &self, + min_best_number: Option>, + ) -> Option<(&PeerId, &Peer)> { + let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); + // Find a random peer that is synced as much as peer majority and is above + // `best_number_at_least`. + for (peer_id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + return Some((peer_id, peer)) + } + } } + + None } - /// Check if the state is complete. - pub fn is_complete(&self) -> bool { - match &self.phase { - Phase::WarpProof { .. } | Phase::TargetBlock(_) | Phase::PendingTargetBlock { .. } => - false, - Phase::State(sync) => sync.is_complete(), + /// Produce next warp proof request. + fn warp_proof_request(&self) -> Option<(PeerId, WarpProofRequest)> { + let Phase::WarpProof { last_hash, .. } = &self.phase else { return None }; + + if self + .peers + .iter() + .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingProofs)) + { + // Only one warp proof request at a time is possible. + return None + } + + let Some((peer_id, peer)) = self.select_synced_available_peer(None) else { return None }; + + trace!(target: LOG_TARGET, "New WarpProofRequest to {peer_id}, begin hash: {last_hash}."); + peer.state = PeerState::DownloadingProofs; + + Some((*peer_id, WarpProofRequest { begin: *last_hash })) + } + + /// Produce next target block request. + fn target_block_request(&self) -> Option<(PeerId, BlockRequest)> { + let Phase::TargetBlock(target_header) = &self.phase else { return None }; + + if self + .peers + .iter() + .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingTargetBlock)) + { + // Only one target block request at a time is possible. + return None + } + + let Some((peer_id, peer)) = + self.select_synced_available_peer(Some(*target_header.number())) + else { + return None + }; + + trace!( + target: LOG_TARGET, + "New target block request to {peer_id}, target: {} ({}).", + target_header.hash(), + target_header.number(), + ); + peer.state = PeerState::DownloadingTargetBlock; + + Some(( + *peer_id, + BlockRequest:: { + id: 0, + fields: BlockAttributes::HEADER | + BlockAttributes::BODY | + BlockAttributes::JUSTIFICATION, + from: FromBlock::Hash(target_header.hash()), + direction: Direction::Ascending, + max: Some(1), + }, + )) + } + + /// Produce next state request. + fn state_request(&self) -> Option<(PeerId, OpaqueStateRequest)> { + let Phase::State(state_sync) = &self.phase else { return None }; + + if self + .peers + .iter() + .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingState)) + { + // Only one state request at a time is possible. + return None } + + let Some((peer_id, peer)) = + self.select_synced_available_peer(Some(state_sync.target_block_num())) + else { + return None + }; + + peer.state = PeerState::DownloadingTargetBlock; + let request = state_sync.next_request(); + trace!( + target: LOG_TARGET, + "New state request to {peer_id}: {request:?}.", + ); + + Some((*peer_id, OpaqueStateRequest(Box::new(request)))) } - /// Returns state sync estimated progress (percentage, bytes) + /// Returns state sync estimated progress (stage, bytes received). pub fn progress(&self) -> WarpSyncProgress { match &self.phase { + Phase::WaitingForPeers { .. } => WarpSyncProgress { + phase: WarpSyncPhase::AwaitingPeers { + required_peers: MIN_PEERS_TO_START_WARP_SYNC, + }, + total_bytes: self.total_proof_bytes, + }, Phase::WarpProof { .. } => WarpSyncProgress { phase: WarpSyncPhase::DownloadingWarpProofs, total_bytes: self.total_proof_bytes, @@ -392,14 +712,42 @@ where phase: WarpSyncPhase::AwaitingTargetBlock, total_bytes: self.total_proof_bytes, }, - Phase::State(sync) => WarpSyncProgress { - phase: if self.is_complete() { + Phase::State(state_sync) => WarpSyncProgress { + phase: if state_sync.is_complete() { WarpSyncPhase::ImportingState } else { WarpSyncPhase::DownloadingState }, - total_bytes: self.total_proof_bytes + sync.progress().size, + total_bytes: self.total_proof_bytes + state_sync.progress().size, + }, + Phase::Complete => WarpSyncProgress { + phase: WarpSyncPhase::Complete, + total_bytes: self.total_proof_bytes + self.total_state_bytes, }, } } + + /// Get actions that should be performed by the owner on [`WarpSync`]'s behalf + #[must_use] + pub fn actions(&mut self) -> impl Iterator> { + let warp_proof_request = self + .warp_proof_request() + .into_iter() + .map(|(peer_id, request)| WarpSyncAction::SendWarpProofRequest { peer_id, request }); + self.actions.extend(warp_proof_request); + + let target_block_request = self + .target_block_request() + .into_iter() + .map(|(peer_id, request)| WarpSyncAction::SendBlockRequest { peer_id, request }); + self.actions.extend(target_block_request); + + let state_request = self + .state_request() + .into_iter() + .map(|(peer_id, request)| WarpSyncAction::SendStateRequest { peer_id, request }); + self.actions.extend(state_request); + + std::mem::take(&mut self.actions).into_iter() + } } From 4b4c880c5656fe71efdffa7d697fc866ac03788c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Nov 2023 15:17:00 +0200 Subject: [PATCH 02/54] Cleanup warp sync leftovers in `ChainSync` --- .../client/network/sync/src/chain_sync.rs | 295 ++---------------- .../network/sync/src/chain_sync/test.rs | 18 +- substrate/client/network/sync/src/types.rs | 3 - substrate/client/network/sync/src/warp.rs | 17 +- 4 files changed, 55 insertions(+), 278 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index 50e07e6c2a36f..39247a3449c86 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -34,12 +34,7 @@ use crate::{ schema::v1::StateResponse, state::{ImportResult, StateSync}, types::{ - BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, SyncMode, SyncState, - SyncStatus, - }, - warp::{ - self, EncodedProof, WarpProofImportResult, WarpProofRequest, WarpSync, WarpSyncConfig, - WarpSyncPhase, WarpSyncProgress, + BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, SyncState, SyncStatus, }, }; @@ -193,8 +188,6 @@ pub enum ChainSyncAction { CancelBlockRequest { peer_id: PeerId }, /// Send state request to peer. SendStateRequest { peer_id: PeerId, request: OpaqueStateRequest }, - /// Send warp proof request to peer. - SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, /// Peer misbehaved. Disconnect, report it and cancel the block request to it. DropPeer(BadPeer), /// Import blocks. @@ -208,6 +201,20 @@ pub enum ChainSyncAction { }, } +/// Sync operation mode. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum ChainSyncMode { + /// Full block download and verification. + Full, + /// Download blocks and the latest state. + LightState { + /// Skip state proof download and verification. + skip_proofs: bool, + /// Download indexed transactions for recent blocks. + storage_chain_mode: bool, + }, +} + /// The main data structure which contains all the state for a chains /// active syncing strategy. pub struct ChainSync { @@ -222,7 +229,7 @@ pub struct ChainSync { /// The best block hash in our queue of blocks to import best_queued_hash: B::Hash, /// Current mode (full/light) - mode: SyncMode, + mode: ChainSyncMode, /// Any extra justification requests. extra_justifications: ExtraRequests, /// A set of hashes of blocks that are being downloaded or have been @@ -240,14 +247,6 @@ pub struct ChainSync { downloaded_blocks: usize, /// State sync in progress, if any. state_sync: Option>, - /// Warp sync in progress, if any. - warp_sync: Option>, - /// Warp sync configuration. - /// - /// Will be `None` after `self.warp_sync` is `Some(_)`. - warp_sync_config: Option>, - /// A temporary storage for warp sync target block until warp sync is initialized. - warp_sync_target_block_header: Option, /// Enable importing existing blocks. This is used used after the state download to /// catch up to the latest state while re-importing blocks. import_existing: bool, @@ -316,10 +315,6 @@ pub(crate) enum PeerSyncState { DownloadingJustification(B::Hash), /// Downloading state. DownloadingState, - /// Downloading warp proof. - DownloadingWarpProof, - /// Downloading warp sync target block. - DownloadingWarpTargetBlock, /// Actively downloading block history after warp sync. DownloadingGap(NumberFor), } @@ -343,11 +338,10 @@ where { /// Create a new instance. pub fn new( - mode: SyncMode, + mode: ChainSyncMode, client: Arc, max_parallel_downloads: u32, max_blocks_per_request: u32, - warp_sync_config: Option>, ) -> Result { let mut sync = Self { client, @@ -364,11 +358,8 @@ where max_blocks_per_request, downloaded_blocks: 0, state_sync: None, - warp_sync: None, import_existing: false, gap_sync: None, - warp_sync_config, - warp_sync_target_block_header: None, actions: Vec::new(), }; @@ -407,21 +398,6 @@ where SyncState::Idle }; - let warp_sync_progress = match (&self.warp_sync, &self.mode, &self.gap_sync) { - (_, _, Some(gap_sync)) => Some(WarpSyncProgress { - phase: WarpSyncPhase::DownloadingBlocks(gap_sync.best_queued_number), - total_bytes: 0, - }), - (None, SyncMode::Warp, _) => Some(WarpSyncProgress { - phase: WarpSyncPhase::AwaitingPeers { - required_peers: MIN_PEERS_TO_START_WARP_SYNC, - }, - total_bytes: 0, - }), - (Some(sync), _, _) => Some(sync.progress()), - _ => None, - }; - SyncStatus { state: sync_state, best_seen_block, @@ -429,7 +405,6 @@ where num_connected_peers: 0u32, queued_blocks: self.queue_blocks.len() as u32, state_sync: self.state_sync.as_ref().map(|s| s.progress()), - warp_sync: warp_sync_progress, } } @@ -554,20 +529,6 @@ where }, ); - if let SyncMode::Warp = self.mode { - if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() - { - log::debug!(target: LOG_TARGET, "Starting warp state sync."); - - if let Some(config) = self.warp_sync_config.take() { - let mut warp_sync = WarpSync::new(self.client.clone(), config); - if let Some(header) = self.warp_sync_target_block_header.take() { - warp_sync.set_target_block(header); - } - self.warp_sync = Some(warp_sync); - } - } - } Ok(req) }, Ok(BlockStatus::Queued) | @@ -886,43 +847,9 @@ where return Ok(()) } }, - PeerSyncState::DownloadingWarpTargetBlock => { - peer.state = PeerSyncState::Available; - if let Some(warp_sync) = &mut self.warp_sync { - if blocks.len() == 1 { - validate_blocks::(&blocks, peer_id, Some(request))?; - match warp_sync.import_target_block( - blocks.pop().expect("`blocks` len checked above."), - ) { - warp::TargetBlockImportResult::Success => return Ok(()), - warp::TargetBlockImportResult::BadResponse => - return Err(BadPeer(*peer_id, rep::VERIFICATION_FAIL)), - } - } else if blocks.is_empty() { - debug!(target: LOG_TARGET, "Empty block response from {peer_id}"); - return Err(BadPeer(*peer_id, rep::NO_BLOCK)) - } else { - debug!( - target: LOG_TARGET, - "Too many blocks ({}) in warp target block response from {}", - blocks.len(), - peer_id, - ); - return Err(BadPeer(*peer_id, rep::NOT_REQUESTED)) - } - } else { - debug!( - target: LOG_TARGET, - "Logic error: we think we are downloading warp target block from {}, but no warp sync is happening.", - peer_id, - ); - return Ok(()) - } - }, PeerSyncState::Available | PeerSyncState::DownloadingJustification(..) | - PeerSyncState::DownloadingState | - PeerSyncState::DownloadingWarpProof => Vec::new(), + PeerSyncState::DownloadingState => Vec::new(), } } else { // When request.is_none() this is a block announcement. Just accept blocks. @@ -1037,7 +964,7 @@ where is_descendent_of(&**client, base, block) }); - if let SyncMode::LightState { skip_proofs, .. } = &self.mode { + if let ChainSyncMode::LightState { skip_proofs, .. } = &self.mode { if self.state_sync.is_none() && !self.peers.is_empty() && self.queue_blocks.is_empty() { // Finalized a recent block. let mut heads: Vec<_> = self.peers.values().map(|peer| peer.best_number).collect(); @@ -1202,11 +1129,11 @@ where fn required_block_attributes(&self) -> BlockAttributes { match self.mode { - SyncMode::Full => + ChainSyncMode::Full => BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::LightState { storage_chain_mode: false, .. } | SyncMode::Warp => + ChainSyncMode::LightState { storage_chain_mode: false, .. } => BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::LightState { storage_chain_mode: true, .. } => + ChainSyncMode::LightState { storage_chain_mode: true, .. } => BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::INDEXED_BODY, @@ -1215,9 +1142,8 @@ where fn skip_execution(&self) -> bool { match self.mode { - SyncMode::Full => false, - SyncMode::LightState { .. } => true, - SyncMode::Warp => true, + ChainSyncMode::Full => false, + ChainSyncMode::LightState { .. } => true, } } @@ -1353,25 +1279,19 @@ where /// state for. fn reset_sync_start_point(&mut self) -> Result<(), ClientError> { let info = self.client.info(); - if matches!(self.mode, SyncMode::LightState { .. }) && info.finalized_state.is_some() { + if matches!(self.mode, ChainSyncMode::LightState { .. }) && info.finalized_state.is_some() { warn!( target: LOG_TARGET, "Can't use fast sync mode with a partially synced database. Reverting to full sync mode." ); - self.mode = SyncMode::Full; - } - if matches!(self.mode, SyncMode::Warp) && info.finalized_state.is_some() { - warn!( - target: LOG_TARGET, - "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." - ); - self.mode = SyncMode::Full; + self.mode = ChainSyncMode::Full; } + self.import_existing = false; self.best_queued_hash = info.best_hash; self.best_queued_number = info.best_number; - if self.mode == SyncMode::Full && + if self.mode == ChainSyncMode::Full && self.client.block_status(info.best_hash)? != BlockStatus::InChainWithState { self.import_existing = true; @@ -1450,44 +1370,6 @@ where .collect() } - /// Set the warp sync target block externally in case we skip warp proofs downloading. - pub fn set_warp_sync_target_block(&mut self, header: B::Header) { - if let Some(ref mut warp_sync) = self.warp_sync { - warp_sync.set_target_block(header); - } else { - self.warp_sync_target_block_header = Some(header); - } - } - - /// Generate block request for downloading of the target block body during warp sync. - fn warp_target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { - let sync = &self.warp_sync.as_ref()?; - - if self.allowed_requests.is_empty() || - sync.is_complete() || - self.peers - .iter() - .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpTargetBlock) - { - // Only one pending warp target block request is allowed. - return None - } - - if let Some((target_number, request)) = sync.next_target_block_request() { - // Find a random peer that has a block with the target number. - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= target_number { - trace!(target: LOG_TARGET, "New warp target block request for {id}"); - peer.state = PeerSyncState::DownloadingWarpTargetBlock; - self.allowed_requests.clear(); - return Some((*id, request)) - } - } - } - - None - } - /// Submit blocks received in a response. pub fn on_block_response( &mut self, @@ -1564,12 +1446,6 @@ where /// Get block requests scheduled by sync to be sent out. fn block_requests(&mut self) -> Vec<(PeerId, BlockRequest)> { - if self.mode == SyncMode::Warp { - return self - .warp_target_block_request() - .map_or_else(|| Vec::new(), |req| Vec::from([req])) - } - if self.allowed_requests.is_empty() || self.state_sync.is_some() { return Vec::new() } @@ -1694,7 +1570,7 @@ where if self.allowed_requests.is_empty() { return None } - if (self.state_sync.is_some() || self.warp_sync.is_some()) && + if self.state_sync.is_some() && self.peers.iter().any(|(_, peer)| peer.state == PeerSyncState::DownloadingState) { // Only one pending state request is allowed. @@ -1715,55 +1591,6 @@ where } } } - if let Some(sync) = &self.warp_sync { - if sync.is_complete() { - return None - } - if let (Some(request), Some(target)) = - (sync.next_state_request(), sync.target_block_number()) - { - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= target { - trace!(target: LOG_TARGET, "New StateRequest for {id}: {request:?}"); - peer.state = PeerSyncState::DownloadingState; - self.allowed_requests.clear(); - return Some((*id, OpaqueStateRequest(Box::new(request)))) - } - } - } - } - None - } - - /// Get a warp proof request scheduled by sync to be sent out (if any). - fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { - if let Some(sync) = &self.warp_sync { - if self.allowed_requests.is_empty() || - sync.is_complete() || - self.peers - .iter() - .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpProof) - { - // Only one pending state request is allowed. - return None - } - if let Some(request) = sync.next_warp_proof_request() { - let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - // Find a random peer that is synced as much as peer majority. - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= median { - trace!(target: LOG_TARGET, "New WarpProofRequest for {id}"); - peer.state = PeerSyncState::DownloadingWarpProof; - self.allowed_requests.clear(); - return Some((*id, request)) - } - } - } - } - } None } @@ -1797,15 +1624,6 @@ where response.proof.len(), ); sync.import(*response) - } else if let Some(sync) = &mut self.warp_sync { - debug!( - target: LOG_TARGET, - "Importing state data from {} with {} keys, {} proof nodes.", - peer_id, - response.entries.len(), - response.proof.len(), - ); - sync.import_state(*response) } else { debug!(target: LOG_TARGET, "Ignored obsolete state response from {peer_id}"); return Err(BadPeer(*peer_id, rep::NOT_REQUESTED)) @@ -1838,38 +1656,6 @@ where } } - /// Submit a warp proof response received. - pub fn on_warp_sync_response(&mut self, peer_id: &PeerId, response: EncodedProof) { - if let Some(peer) = self.peers.get_mut(peer_id) { - if let PeerSyncState::DownloadingWarpProof = peer.state { - peer.state = PeerSyncState::Available; - self.allowed_requests.set_all(); - } - } - let import_result = if let Some(sync) = &mut self.warp_sync { - debug!( - target: LOG_TARGET, - "Importing warp proof data from {}, {} bytes.", - peer_id, - response.0.len(), - ); - sync.import_warp_proof(response) - } else { - debug!(target: LOG_TARGET, "Ignored obsolete warp sync response from {peer_id}"); - self.actions - .push(ChainSyncAction::DropPeer(BadPeer(*peer_id, rep::NOT_REQUESTED))); - return - }; - - match import_result { - WarpProofImportResult::Success => {}, - WarpProofImportResult::BadResponse => { - debug!(target: LOG_TARGET, "Bad proof data received from {peer_id}"); - self.actions.push(ChainSyncAction::DropPeer(BadPeer(*peer_id, rep::BAD_BLOCK))); - }, - } - } - /// A batch of blocks have been processed, with or without errors. /// /// Call this when a batch of blocks have been processed by the import @@ -1941,21 +1727,7 @@ where self.state_sync.as_ref().map_or(0, |s| s.progress().size / (1024 * 1024)), ); self.state_sync = None; - self.mode = SyncMode::Full; - self.restart(); - } - let warp_sync_complete = self - .warp_sync - .as_ref() - .map_or(false, |s| s.target_block_hash() == Some(hash)); - if warp_sync_complete { - info!( - target: LOG_TARGET, - "Warp sync is complete ({} MiB), restarting block sync.", - self.warp_sync.as_ref().map_or(0, |s| s.progress().total_bytes / (1024 * 1024)), - ); - self.warp_sync = None; - self.mode = SyncMode::Full; + self.mode = ChainSyncMode::Full; self.restart(); } let gap_sync_complete = @@ -2011,7 +1783,6 @@ where e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { warn!(target: LOG_TARGET, "💔 Error importing block {hash:?}: {}", e.unwrap_err()); self.state_sync = None; - self.warp_sync = None; self.restart(); }, Err(BlockImportError::Cancelled) => {}, @@ -2042,12 +1813,6 @@ where .map(|(peer_id, request)| ChainSyncAction::SendStateRequest { peer_id, request }); self.actions.extend(state_request); - let warp_proof_request = self - .warp_sync_request() - .into_iter() - .map(|(peer_id, request)| ChainSyncAction::SendWarpProofRequest { peer_id, request }); - self.actions.extend(warp_proof_request); - std::mem::take(&mut self.actions).into_iter() } diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/chain_sync/test.rs index 15b2a95a07c87..1cb00cbbc5758 100644 --- a/substrate/client/network/sync/src/chain_sync/test.rs +++ b/substrate/client/network/sync/src/chain_sync/test.rs @@ -38,7 +38,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { let client = Arc::new(TestClientBuilder::new().build()); let peer_id = PeerId::random(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); let (a1_hash, a1_number) = { let a1 = BlockBuilderBuilder::new(&*client) @@ -91,7 +91,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { fn restart_doesnt_affect_peers_downloading_finality_data() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let peer_id2 = PeerId::random(); @@ -275,7 +275,7 @@ fn do_ancestor_search_when_common_block_to_best_qeued_gap_is_to_big() { let mut client = Arc::new(TestClientBuilder::new().build()); let info = client.info(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64).unwrap(); let peer_id1 = PeerId::random(); let peer_id2 = PeerId::random(); @@ -421,7 +421,7 @@ fn can_sync_huge_fork() { let info = client.info(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64).unwrap(); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); let just = (*b"TEST", Vec::new()); @@ -554,7 +554,7 @@ fn syncs_fork_without_duplicate_requests() { let info = client.info(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64).unwrap(); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); let just = (*b"TEST", Vec::new()); @@ -689,7 +689,7 @@ fn removes_target_fork_on_disconnect() { let mut client = Arc::new(TestClientBuilder::new().build()); let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let common_block = blocks[1].clone(); @@ -714,7 +714,7 @@ fn can_import_response_with_missing_blocks() { let empty_client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, empty_client.clone(), 1, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, empty_client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let best_block = blocks[3].clone(); @@ -745,7 +745,7 @@ fn ancestor_search_repeat() { #[test] fn sync_restart_removes_block_but_not_justification_requests() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); let peers = vec![PeerId::random(), PeerId::random()]; @@ -887,7 +887,7 @@ fn request_across_forks() { fork_blocks }; - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64, None).unwrap(); + let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64).unwrap(); // Add the peers, all at the common ancestor 100. let common_block = blocks.last().unwrap(); diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index 5931cf47b28a4..6ecbc8825c4aa 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -23,7 +23,6 @@ use sc_network_common::{role::Roles, types::ReputationChange}; use libp2p::PeerId; -use crate::warp::WarpSyncProgress; use sc_network_common::sync::message::BlockRequest; use sp_runtime::traits::{Block as BlockT, NumberFor}; @@ -93,8 +92,6 @@ pub struct SyncStatus { pub queued_blocks: u32, /// State sync status in progress, if any. pub state_sync: Option, - /// Warp sync in progress, if any. - pub warp_sync: Option>, } /// A peer did not behave as expected and should be reported. diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index a2317926c804c..d5d383b4a2d00 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -29,7 +29,7 @@ use crate::{ use codec::{Decode, Encode}; use futures::channel::oneshot; use libp2p::PeerId; -use log::{debug, error, info, trace}; +use log::{debug, error, info, trace, warn}; use sc_client_api::ProofProvider; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::message::{ @@ -278,6 +278,21 @@ where /// authorities. Alternatively we can pass a target block when we want to skip downloading /// proofs, in this case we will continue polling until the target block is known. pub fn new(client: Arc, warp_sync_config: WarpSyncConfig) -> Self { + if client.info().finalized_state.is_some() { + warn!( + target: LOG_TARGET, + "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." + ); + return Self { + client, + phase: Phase::Complete, + total_proof_bytes: 0, + total_state_bytes: 0, + peers: HashMap::new(), + actions: vec![WarpSyncAction::Finished], + } + } + let phase = match warp_sync_config { WarpSyncConfig::WithProvider(warp_sync_provider) => Phase::WaitingForPeers { warp_sync_provider }, From 1c45b93ba06bed2e8834d942acc619f5d25eaebd Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 21 Nov 2023 17:34:02 +0200 Subject: [PATCH 03/54] Introduce `SyncingStrategy` proxy enum --- substrate/client/network/sync/src/engine.rs | 9 +- substrate/client/network/sync/src/lib.rs | 1 + substrate/client/network/sync/src/strategy.rs | 304 ++++++++++++++++++ substrate/client/network/sync/src/warp.rs | 4 +- 4 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 substrate/client/network/sync/src/strategy.rs diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 2cb8eab22f7ac..8e4d2bf75ac6e 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -452,13 +452,8 @@ where ); let block_announce_protocol_name = block_announce_config.notifications_protocol.clone(); - let chain_sync = ChainSync::new( - mode, - client.clone(), - max_parallel_downloads, - max_blocks_per_request, - warp_sync_config, - )?; + let chain_sync = + ChainSync::new(mode, client.clone(), max_parallel_downloads, max_blocks_per_request)?; let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index 1a7e773c95f7a..e22eaafa52502 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -28,6 +28,7 @@ mod futures_stream; mod pending_responses; mod request_metrics; mod schema; +mod strategy; mod types; pub mod block_relay_protocol; diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs new file mode 100644 index 0000000000000..b0cbda58b896c --- /dev/null +++ b/substrate/client/network/sync/src/strategy.rs @@ -0,0 +1,304 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! [`SyncingStrategy`] is a proxy between [`crate::engine::SyncingEngine`] +//! and specific syncing algorithms. + +use crate::{ + chain_sync::{ChainSync, ChainSyncAction}, + types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, + warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction}, +}; +use libp2p::PeerId; +use sc_client_api::{BlockBackend, ProofProvider}; +use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sc_network_common::sync::message::{BlockAnnounce, BlockData, BlockRequest}; +use sp_blockchain::{HeaderBackend, HeaderMetadata}; +use sp_consensus::BlockOrigin; +use sp_runtime::{ + traits::{Block as BlockT, Header, NumberFor}, + Justifications, +}; + +#[derive(Debug)] +pub enum SyncingAction { + /// Send block request to peer. Always implies dropping a stale block request to the same peer. + SendBlockRequest { peer_id: PeerId, request: BlockRequest }, + /// Drop stale block request. + CancelBlockRequest { peer_id: PeerId }, + /// Send state request to peer. + SendStateRequest { peer_id: PeerId, request: OpaqueStateRequest }, + /// Send warp proof request to peer. + SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, + /// Peer misbehaved. Disconnect, report it and cancel any requests to it. + DropPeer(BadPeer), + /// Import blocks. + ImportBlocks { origin: BlockOrigin, blocks: Vec> }, + /// Import justifications. + ImportJustifications { + peer_id: PeerId, + hash: B::Hash, + number: NumberFor, + justifications: Justifications, + }, + /// Syncing strategy has finished. + Finished, +} + +/// Proxy to specific syncing strategies. +pub enum SyncingStrategy { + WarpSyncStrategy(WarpSync), + ChainSyncStrategy(ChainSync), +} + +impl SyncingStrategy +where + B: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, +{ + pub fn new_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.new_peer(peer_id, best_hash, best_number), + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.new_peer(peer_id, best_hash, best_number), + } + } + + pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), + } + } + + /// Submit a validated block announcement. + /// + /// Returns new best hash & best number of the peer if they are updated. + pub fn on_validated_block_announce( + &mut self, + is_best: bool, + peer_id: PeerId, + announce: &BlockAnnounce, + ) -> Option<(B::Hash, NumberFor)> { + match self { + SyncingStrategy::WarpSyncStrategy(_) => + Some((announce.header.hash(), *announce.header.number())), + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_validated_block_announce(is_best, peer_id, announce), + } + } + + /// Configure an explicit fork sync request in case external code has detected that there is a + /// stale fork missing. + pub fn set_sync_fork_request( + &mut self, + peers: Vec, + hash: &B::Hash, + number: NumberFor, + ) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.set_sync_fork_request(peers, hash, number), + } + } + + /// Request extra justification. + pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.request_justification(hash, number), + } + } + + /// Clear extra justification requests. + pub fn clear_justification_requests(&mut self) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.clear_justification_requests(), + } + } + + /// Report a justification import (successful or not). + pub fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_justification_import(hash, number, success), + } + } + + /// Process block response. + pub fn on_block_response( + &mut self, + peer_id: PeerId, + request: BlockRequest, + blocks: Vec>, + ) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.on_block_response(peer_id, request, blocks), + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_block_response(peer_id, request, blocks), + } + } + + /// Process state response. + pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.on_state_response(peer_id, response), + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_state_response(peer_id, response), + } + } + + /// Process warp proof response. + pub fn on_warp_proof_response(&mut self, peer_id: &PeerId, response: EncodedProof) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.on_warp_proof_response(peer_id, response), + SyncingStrategy::ChainSyncStrategy(_) => {}, + } + } + + /// A batch of blocks have been processed, with or without errors. + pub fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, B::Hash)>, + ) { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.on_blocks_processed(imported, count, results), + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_blocks_processed(imported, count, results), + } + } + + /// Notify a syncing strategy that a block has been finalized. + pub fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.on_block_finalized(hash, number), + } + } + + /// Inform sync about a new best imported block. + pub fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.update_chain_info(best_hash, best_number), + } + } + + // Are we in major sync mode? + pub fn is_major_syncing(&self) -> bool { + match self { + SyncingStrategy::WarpSyncStrategy(_) => false, + SyncingStrategy::ChainSyncStrategy(strategy) => + strategy.status().state.is_major_syncing(), + } + } + + /// Get the number of peers known to the syncing strategy. + pub fn num_peers(&self) -> usize { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => strategy.num_peers(), + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_peers(), + } + } + + /// Returns the current sync status. + pub fn status(&self) -> SyncStatus { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => strategy.status(), + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.status(), + } + } + + /// Get the total number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + match self { + SyncingStrategy::WarpSyncStrategy(_) => 0, + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_downloaded_blocks(), + } + } + + /// Get an estimate of the number of parallel sync requests. + pub fn num_sync_requests(&self) -> usize { + match self { + SyncingStrategy::WarpSyncStrategy(_) => 0, + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_sync_requests(), + } + } + + /// Get actions that should be performed by the owner on the strategy's behalf + #[must_use] + pub fn actions(&mut self) -> Box>> { + match self { + SyncingStrategy::WarpSyncStrategy(strategy) => + Box::new(strategy.actions().map(|action| match action { + WarpSyncAction::SendWarpProofRequest { peer_id, request } => + SyncingAction::SendWarpProofRequest { peer_id, request }, + WarpSyncAction::SendBlockRequest { peer_id, request } => + SyncingAction::SendBlockRequest { peer_id, request }, + WarpSyncAction::SendStateRequest { peer_id, request } => + SyncingAction::SendStateRequest { peer_id, request }, + WarpSyncAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), + WarpSyncAction::ImportBlocks { origin, blocks } => + SyncingAction::ImportBlocks { origin, blocks }, + WarpSyncAction::Finished => SyncingAction::Finished, + })), + SyncingStrategy::ChainSyncStrategy(strategy) => + Box::new(strategy.actions().map(|action| match action { + ChainSyncAction::SendBlockRequest { peer_id, request } => + SyncingAction::SendBlockRequest { peer_id, request }, + ChainSyncAction::CancelBlockRequest { peer_id } => + SyncingAction::CancelBlockRequest { peer_id }, + ChainSyncAction::SendStateRequest { peer_id, request } => + SyncingAction::SendStateRequest { peer_id, request }, + ChainSyncAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), + ChainSyncAction::ImportBlocks { origin, blocks } => + SyncingAction::ImportBlocks { origin, blocks }, + ChainSyncAction::ImportJustifications { + peer_id, + hash, + number, + justifications, + } => SyncingAction::ImportJustifications { + peer_id, + hash, + number, + justifications, + }, + })), + } + } +} diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index d5d383b4a2d00..63c6b6a225a0c 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -244,7 +244,7 @@ struct Peer { state: PeerState, } -enum WarpSyncAction { +pub enum WarpSyncAction { /// Send warp proof request to peer. SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, /// Send block request to peer. Always implies dropping a stale block request to the same peer. @@ -482,7 +482,7 @@ where Ok(()) } - // Process state response. + /// Process state response. pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { self.actions.push(WarpSyncAction::DropPeer(bad_peer)); From efd05115cce0dc8553e1ea03d09ddc6d8f3e3285 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Nov 2023 17:56:53 +0200 Subject: [PATCH 04/54] Plug `SyncingStrategy` into `SyncingEngine` --- .../client/network/sync/src/chain_sync.rs | 23 +- substrate/client/network/sync/src/engine.rs | 264 ++++++++++++------ substrate/client/network/sync/src/warp.rs | 47 +++- 3 files changed, 232 insertions(+), 102 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index 39247a3449c86..d9cd0c14aea92 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -998,12 +998,15 @@ where } /// Submit a validated block announcement. + /// + /// Returns new best hash & best number of the peer if they are updated. + #[must_use] pub fn on_validated_block_announce( &mut self, is_best: bool, peer_id: PeerId, announce: &BlockAnnounce, - ) { + ) -> Option<(B::Hash, NumberFor)> { let number = *announce.header.number(); let hash = announce.header.hash(); let parent_status = @@ -1016,19 +1019,23 @@ where peer } else { error!(target: LOG_TARGET, "💔 Called `on_validated_block_announce` with a bad peer ID"); - return + return Some((hash, number)) }; if let PeerSyncState::AncestorSearch { .. } = peer.state { trace!(target: LOG_TARGET, "Peer {} is in the ancestor search state.", peer_id); - return + return None } - if is_best { + let new_peer_info = if is_best { // update their best block peer.best_number = number; peer.best_hash = hash; - } + + Some((hash, number)) + } else { + None + }; // If the announced block is the best they have and is not ahead of us, our common number // is either one further ahead or it's the one they just announced, if we know about it. @@ -1049,7 +1056,7 @@ where if let Some(target) = self.fork_targets.get_mut(&hash) { target.peers.insert(peer_id); } - return + return new_peer_info } if ancient_parent { @@ -1060,7 +1067,7 @@ where hash, announce.header, ); - return + return new_peer_info } if self.status().state == SyncState::Idle { @@ -1081,6 +1088,8 @@ where .peers .insert(peer_id); } + + new_peer_info } /// Notify that a sync peer has disconnected. diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 8e4d2bf75ac6e..4046716f67f8f 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -25,17 +25,18 @@ use crate::{ }, block_relay_protocol::{BlockDownloader, BlockResponseError}, block_request_handler::MAX_BLOCKS_IN_RESPONSE, - chain_sync::{ChainSync, ChainSyncAction}, + chain_sync::{ChainSync, ChainSyncMode}, pending_responses::{PendingResponses, ResponseEvent}, schema::v1::{StateRequest, StateResponse}, service::{ self, syncing_service::{SyncingService, ToServiceCommand}, }, + strategy::{SyncingAction, SyncingStrategy}, types::{ BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, }, - warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, + warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncParams}, }; use codec::{Decode, Encode}; @@ -45,7 +46,7 @@ use futures::{ FutureExt, StreamExt, }; use libp2p::{request_response::OutboundFailure, PeerId}; -use log::{debug, trace}; +use log::{debug, error, trace}; use prometheus_endpoint::{ register, Counter, Gauge, GaugeVec, MetricSource, Opts, PrometheusError, Registry, SourcedGauge, U64, @@ -67,7 +68,10 @@ use sc_network::{ }; use sc_network_common::{ role::Roles, - sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, + sync::{ + message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, + SyncMode, + }, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::{Error as ClientError, HeaderMetadata}; @@ -220,6 +224,15 @@ impl MetricSource for MajorSyncingGauge { } } +fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { + match sync_mode { + SyncMode::Full => ChainSyncMode::Full, + SyncMode::LightState { skip_proofs, storage_chain_mode } => + ChainSyncMode::LightState { skip_proofs, storage_chain_mode }, + SyncMode::Warp => ChainSyncMode::Full, + } +} + /// Peer information #[derive(Debug)] pub struct Peer { @@ -233,9 +246,17 @@ pub struct Peer { } pub struct SyncingEngine { - /// State machine that handles the list of in-progress requests. Only full node peers are - /// registered. - chain_sync: ChainSync, + // Syncing strategy. + strategy: SyncingStrategy, + + /// Chain sync mode + chain_sync_mode: ChainSyncMode, + + /// The number of parallel downloads to guard against slow peers. + max_parallel_downloads: u32, + + /// Maximum number of blocks to request. + max_blocks_per_request: u32, /// Blockchain client. client: Arc, @@ -425,19 +446,6 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; - // Split warp sync params into warp sync config and a channel to retreive target block - // header. - let (warp_sync_config, warp_sync_target_block_header_rx) = - warp_sync_params.map_or((None, None), |params| { - let (config, target_block_rx) = params.split(); - (Some(config), target_block_rx) - }); - - // Make sure polling of the target block channel is a no-op if there is no block to - // retrieve. - let warp_sync_target_block_header_rx_fused = warp_sync_target_block_header_rx - .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); - let block_announce_config = Self::get_block_announce_proto_config( protocol_id, fork_id, @@ -452,8 +460,34 @@ where ); let block_announce_protocol_name = block_announce_config.notifications_protocol.clone(); - let chain_sync = - ChainSync::new(mode, client.clone(), max_parallel_downloads, max_blocks_per_request)?; + // Split warp sync params into warp sync config and a channel to retreive target block + // header. + let (warp_sync_config, warp_sync_target_block_header_rx) = + warp_sync_params.map_or((None, None), |params| { + let (config, target_block_rx) = params.split(); + (Some(config), target_block_rx) + }); + + // Make sure polling of the target block channel is a no-op if there is no block to + // retrieve. + let warp_sync_target_block_header_rx_fused = warp_sync_target_block_header_rx + .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); + + let chain_sync_mode = chain_sync_mode(mode); + + // Initialize syncing strategy. + let strategy = if let SyncMode::Warp = mode { + let warp_sync_config = warp_sync_config + .expect("Warp sync configuration must be supplied in warp sync mode."); + SyncingStrategy::WarpSyncStrategy(WarpSync::new(client.clone(), warp_sync_config)) + } else { + SyncingStrategy::ChainSyncStrategy(ChainSync::new( + chain_sync_mode, + client.clone(), + max_parallel_downloads, + max_blocks_per_request, + )?) + }; let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); @@ -480,7 +514,10 @@ where Self { roles, client, - chain_sync, + strategy, + chain_sync_mode, + max_parallel_downloads, + max_blocks_per_request, network_service, peers: HashMap::new(), block_announce_data_cache: LruMap::new(ByLength::new(cache_capacity)), @@ -534,36 +571,41 @@ where let n = u64::try_from(self.peers.len()).unwrap_or(std::u64::MAX); metrics.peers.set(n); - let m = self.chain_sync.metrics(); - - metrics.fork_targets.set(m.fork_targets.into()); - metrics.queued_blocks.set(m.queued_blocks.into()); - - metrics - .justifications - .with_label_values(&["pending"]) - .set(m.justifications.pending_requests.into()); - metrics - .justifications - .with_label_values(&["active"]) - .set(m.justifications.active_requests.into()); - metrics - .justifications - .with_label_values(&["failed"]) - .set(m.justifications.failed_requests.into()); - metrics - .justifications - .with_label_values(&["importing"]) - .set(m.justifications.importing_requests.into()); + if let SyncingStrategy::ChainSyncStrategy(chain_sync) = &self.strategy { + let m = chain_sync.metrics(); + + metrics.fork_targets.set(m.fork_targets.into()); + metrics.queued_blocks.set(m.queued_blocks.into()); + + metrics + .justifications + .with_label_values(&["pending"]) + .set(m.justifications.pending_requests.into()); + metrics + .justifications + .with_label_values(&["active"]) + .set(m.justifications.active_requests.into()); + metrics + .justifications + .with_label_values(&["failed"]) + .set(m.justifications.failed_requests.into()); + metrics + .justifications + .with_label_values(&["importing"]) + .set(m.justifications.importing_requests.into()); + } } } - fn update_peer_info(&mut self, peer_id: &PeerId) { - if let Some(info) = self.chain_sync.peer_info(peer_id) { - if let Some(ref mut peer) = self.peers.get_mut(peer_id) { - peer.info.best_hash = info.best_hash; - peer.info.best_number = info.best_number; - } + fn update_peer_info( + &mut self, + peer_id: &PeerId, + best_hash: B::Hash, + best_number: NumberFor, + ) { + if let Some(ref mut peer) = self.peers.get_mut(peer_id) { + peer.info.best_hash = best_hash; + peer.info.best_number = best_number; } } @@ -575,9 +617,11 @@ where match validation_result { BlockAnnounceValidationResult::Skip { peer_id: _ } => {}, BlockAnnounceValidationResult::Process { is_new_best, peer_id, announce } => { - self.chain_sync.on_validated_block_announce(is_new_best, peer_id, &announce); - - self.update_peer_info(&peer_id); + if let Some((best_hash, best_number)) = + self.strategy.on_validated_block_announce(is_new_best, peer_id, &announce) + { + self.update_peer_info(&peer_id, best_hash, best_number); + } if let Some(data) = announce.data { if !data.is_empty() { @@ -677,7 +721,7 @@ where pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { log::debug!(target: LOG_TARGET, "New best block imported {hash:?}/#{number}"); - self.chain_sync.update_chain_info(&hash, number); + self.strategy.update_chain_info(&hash, number); self.network_service.set_notification_handshake( self.block_announce_protocol_name.clone(), BlockAnnouncesHandshake::::build(self.roles, number, hash, self.genesis_hash) @@ -705,19 +749,20 @@ where // Update atomic variables self.num_connected.store(self.peers.len(), Ordering::Relaxed); - self.is_major_syncing - .store(self.chain_sync.status().state.is_major_syncing(), Ordering::Relaxed); + self.is_major_syncing.store(self.strategy.is_major_syncing(), Ordering::Relaxed); // Process actions requested by `ChainSync`. - self.process_chain_sync_actions(); + self.process_strategy_actions(); } } - fn process_chain_sync_actions(&mut self) { - self.chain_sync.actions().for_each(|action| match action { - ChainSyncAction::SendBlockRequest { peer_id, request } => { + fn process_strategy_actions(&mut self) { + let mut strategy_finished = false; + + self.strategy.actions().for_each(|action| match action { + SyncingAction::SendBlockRequest { peer_id, request } => { // Sending block request implies dropping obsolete pending response as we are not - // interested in it anymore (see [`ChainSyncAction::SendBlockRequest`]). + // interested in it anymore (see [`SyncingAction::SendBlockRequest`]). // Furthermore, only one request at a time is allowed to any peer. let removed = self.pending_responses.remove(&peer_id); self.send_block_request(peer_id, request.clone()); @@ -730,12 +775,12 @@ where removed, ) }, - ChainSyncAction::CancelBlockRequest { peer_id } => { + SyncingAction::CancelBlockRequest { peer_id } => { let removed = self.pending_responses.remove(&peer_id); trace!(target: LOG_TARGET, "Processed {action:?}, response removed: {removed}."); }, - ChainSyncAction::SendStateRequest { peer_id, request } => { + SyncingAction::SendStateRequest { peer_id, request } => { self.send_state_request(peer_id, request); trace!( @@ -743,7 +788,7 @@ where "Processed `ChainSyncAction::SendBlockRequest` to {peer_id}.", ); }, - ChainSyncAction::SendWarpProofRequest { peer_id, request } => { + SyncingAction::SendWarpProofRequest { peer_id, request } => { self.send_warp_proof_request(peer_id, request.clone()); trace!( @@ -753,7 +798,7 @@ where request, ); }, - ChainSyncAction::DropPeer(BadPeer(peer_id, rep)) => { + SyncingAction::DropPeer(BadPeer(peer_id, rep)) => { self.pending_responses.remove(&peer_id); self.network_service .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); @@ -761,7 +806,7 @@ where trace!(target: LOG_TARGET, "Processed {action:?}."); }, - ChainSyncAction::ImportBlocks { origin, blocks } => { + SyncingAction::ImportBlocks { origin, blocks } => { let count = blocks.len(); self.import_blocks(origin, blocks); @@ -770,7 +815,7 @@ where "Processed `ChainSyncAction::ImportBlocks` with {count} blocks.", ); }, - ChainSyncAction::ImportJustifications { peer_id, hash, number, justifications } => { + SyncingAction::ImportJustifications { peer_id, hash, number, justifications } => { self.import_justifications(peer_id, hash, number, justifications); trace!( @@ -781,7 +826,34 @@ where number, ) }, + SyncingAction::Finished => { + strategy_finished = true; + }, }); + + if strategy_finished { + if let SyncingStrategy::WarpSyncStrategy(_) = &self.strategy { + let chain_sync = match ChainSync::new( + self.chain_sync_mode, + self.client.clone(), + self.max_parallel_downloads, + self.max_blocks_per_request, + ) { + Ok(chain_sync) => chain_sync, + Err(e) => { + error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); + panic!("Failed to start `ChainSync` due to error: {e}."); + }, + }; + self.strategy = SyncingStrategy::ChainSyncStrategy(chain_sync); + } else { + error!( + target: LOG_TARGET, + "Syncing unexpectedly finished. Only warp sync strategy can finish.", + ); + debug_assert!(false); + } + } } fn perform_periodic_actions(&mut self) { @@ -824,18 +896,18 @@ where fn process_service_command(&mut self, command: ToServiceCommand) { match command { ToServiceCommand::SetSyncForkRequest(peers, hash, number) => { - self.chain_sync.set_sync_fork_request(peers, &hash, number); + self.strategy.set_sync_fork_request(peers, &hash, number); }, ToServiceCommand::EventStream(tx) => self.event_streams.push(tx), ToServiceCommand::RequestJustification(hash, number) => - self.chain_sync.request_justification(&hash, number), + self.strategy.request_justification(&hash, number), ToServiceCommand::ClearJustificationRequests => - self.chain_sync.clear_justification_requests(), + self.strategy.clear_justification_requests(), ToServiceCommand::BlocksProcessed(imported, count, results) => { - self.chain_sync.on_blocks_processed(imported, count, results); + self.strategy.on_blocks_processed(imported, count, results); }, ToServiceCommand::JustificationImported(peer_id, hash, number, success) => { - self.chain_sync.on_justification_import(hash, number, success); + self.strategy.on_justification_import(hash, number, success); if !success { log::info!( target: LOG_TARGET, @@ -851,7 +923,7 @@ where ToServiceCommand::NewBestBlockImported(hash, number) => self.new_best_block_imported(hash, number), ToServiceCommand::Status(tx) => { - let mut status = self.chain_sync.status(); + let mut status = self.strategy.status(); status.num_connected_peers = self.peers.len() as u32; let _ = tx.send(status); }, @@ -859,22 +931,22 @@ where let _ = tx.send(self.num_active_peers()); }, ToServiceCommand::SyncState(tx) => { - let _ = tx.send(self.chain_sync.status()); + let _ = tx.send(self.strategy.status()); }, ToServiceCommand::BestSeenBlock(tx) => { - let _ = tx.send(self.chain_sync.status().best_seen_block); + let _ = tx.send(self.strategy.status().best_seen_block); }, ToServiceCommand::NumSyncPeers(tx) => { - let _ = tx.send(self.chain_sync.status().num_peers); + let _ = tx.send(self.strategy.status().num_peers); }, ToServiceCommand::NumQueuedBlocks(tx) => { - let _ = tx.send(self.chain_sync.status().queued_blocks); + let _ = tx.send(self.strategy.status().queued_blocks); }, ToServiceCommand::NumDownloadedBlocks(tx) => { - let _ = tx.send(self.chain_sync.num_downloaded_blocks()); + let _ = tx.send(self.strategy.num_downloaded_blocks()); }, ToServiceCommand::NumSyncRequests(tx) => { - let _ = tx.send(self.chain_sync.num_sync_requests()); + let _ = tx.send(self.strategy.num_sync_requests()); }, ToServiceCommand::PeersInfo(tx) => { let peers_info = self @@ -885,7 +957,7 @@ where let _ = tx.send(peers_info); }, ToServiceCommand::OnBlockFinalized(hash, header) => - self.chain_sync.on_block_finalized(&hash, *header.number()), + self.strategy.on_block_finalized(&hash, *header.number()), } } @@ -945,11 +1017,18 @@ where fn pass_warp_sync_target_block_header(&mut self, header: Result) { match header { - Ok(header) => { - self.chain_sync.set_warp_sync_target_block(header); - }, + Ok(header) => + if let SyncingStrategy::WarpSyncStrategy(warp_sync) = &mut self.strategy { + warp_sync.set_target_block(header); + } else { + error!( + target: LOG_TARGET, + "Cannot set warp sync target block: no warp sync strategy is active." + ); + debug_assert!(false); + }, Err(err) => { - log::error!( + error!( target: LOG_TARGET, "Failed to get target block for warp sync. Error: {err:?}", ); @@ -985,7 +1064,7 @@ where } } - self.chain_sync.peer_disconnected(&peer_id); + self.strategy.peer_disconnected(&peer_id); self.pending_responses.remove(&peer_id); self.event_streams.retain(|stream| { @@ -1066,7 +1145,7 @@ where } if status.roles.is_full() && - self.chain_sync.num_peers() >= + self.strategy.num_peers() >= self.default_peers_set_num_full + self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot @@ -1076,7 +1155,7 @@ where } if status.roles.is_light() && - (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light + (self.peers.len() - self.strategy.num_peers()) >= self.default_peers_set_num_light { // Make sure that not all slots are occupied by light clients. log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); @@ -1096,7 +1175,10 @@ where inbound, }; - self.chain_sync.new_peer(peer_id, peer.info.best_hash, peer.info.best_number); + // Only forward full peers to syncing strategy. + if status.roles.is_full() { + self.strategy.new_peer(peer_id, peer.info.best_hash, peer.info.best_number); + } log::debug!(target: LOG_TARGET, "Connected {peer_id}"); @@ -1213,7 +1295,7 @@ where PeerRequest::Block(req) => { match self.block_downloader.block_response_into_blocks(&req, resp) { Ok(blocks) => { - self.chain_sync.on_block_response(peer_id, req, blocks); + self.strategy.on_block_response(peer_id, req, blocks); }, Err(BlockResponseError::DecodeFailed(e)) => { debug!( @@ -1258,10 +1340,10 @@ where }, }; - self.chain_sync.on_state_response(peer_id, response); + self.strategy.on_state_response(peer_id, response); }, PeerRequest::WarpProof => { - self.chain_sync.on_warp_sync_response(&peer_id, EncodedProof(resp)); + self.strategy.on_warp_proof_response(&peer_id, EncodedProof(resp)); }, }, Ok(Err(e)) => { diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 63c6b6a225a0c..85f72e5a9ca93 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -24,7 +24,7 @@ use crate::{ chain_sync::validate_blocks, schema::v1::StateResponse, state::{ImportResult, StateSync}, - types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse}, + types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, }; use codec::{Decode, Encode}; use futures::channel::oneshot; @@ -37,7 +37,10 @@ use sc_network_common::sync::message::{ }; use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; -use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; +use sp_runtime::{ + traits::{Block as BlockT, Header, NumberFor, Zero}, + SaturatedConversion, +}; use std::{collections::HashMap, fmt, sync::Arc}; /// Log target for this file. @@ -409,7 +412,7 @@ where &mut self, peer_id: PeerId, request: BlockRequest, - blocks: Vec>, + mut blocks: Vec>, ) -> Result<(), BadPeer> { if let Some(peer) = self.peers.get_mut(&peer_id) { peer.state = PeerState::Available; @@ -597,7 +600,7 @@ where fn select_synced_available_peer( &self, min_best_number: Option>, - ) -> Option<(&PeerId, &Peer)> { + ) -> Option<(&PeerId, &mut Peer)> { let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); if !targets.is_empty() { targets.sort(); @@ -742,6 +745,42 @@ where } } + /// Get the number of peers known to syncing. + pub fn num_peers(&self) -> usize { + self.peers.len() + } + + /// Returns the current sync status. + pub fn status(&self) -> SyncStatus { + SyncStatus { + state: match &self.phase { + Phase::WaitingForPeers { .. } => SyncState::Downloading { target: Zero::zero() }, + Phase::WarpProof { .. } => SyncState::Downloading { target: Zero::zero() }, + Phase::PendingTargetBlock => SyncState::Downloading { target: Zero::zero() }, + Phase::TargetBlock(header) => SyncState::Downloading { target: *header.number() }, + Phase::State(state_sync) => + SyncState::Downloading { target: state_sync.target_block_num() }, + Phase::Complete => SyncState::Idle, + }, + best_seen_block: match &self.phase { + Phase::WaitingForPeers { .. } => None, + Phase::WarpProof { .. } => None, + Phase::PendingTargetBlock => None, + Phase::TargetBlock(header) => Some(*header.number()), + Phase::State(state_sync) => Some(state_sync.target_block_num()), + Phase::Complete => None, + }, + num_peers: self.peers.len().saturated_into(), + num_connected_peers: self.peers.len().saturated_into(), + queued_blocks: 0, + state_sync: if let Phase::State(state_sync) = &self.phase { + Some(state_sync.progress()) + } else { + None + }, + } + } + /// Get actions that should be performed by the owner on [`WarpSync`]'s behalf #[must_use] pub fn actions(&mut self) -> impl Iterator> { From fc5f074712382e369eddc7a30d4c061d41aefcc5 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 11:46:49 +0200 Subject: [PATCH 05/54] Resolve borrowing issues in `WarpSync` --- .../client/network/sync/src/chain_sync.rs | 14 +-- substrate/client/network/sync/src/warp.rs | 112 +++++++++++------- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index d9cd0c14aea92..be17737123b45 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -33,9 +33,7 @@ use crate::{ extra_requests::ExtraRequests, schema::v1::StateResponse, state::{ImportResult, StateSync}, - types::{ - BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, SyncState, SyncStatus, - }, + types::{BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, }; use codec::Encode; @@ -90,9 +88,6 @@ const STATE_SYNC_FINALITY_THRESHOLD: u32 = 8; /// so far behind. const MAJOR_SYNC_BLOCKS: u8 = 5; -/// Number of peers that need to be connected before warp sync is started. -const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; - mod rep { use sc_network::ReputationChange as Rep; /// Reputation change when a peer sent us a message that led to a @@ -367,13 +362,6 @@ where Ok(sync) } - /// Get peer's best hash & number. - pub fn peer_info(&self, peer_id: &PeerId) -> Option> { - self.peers - .get(peer_id) - .map(|p| PeerInfo { best_hash: p.best_hash, best_number: p.best_number }) - } - /// Returns the current sync status. pub fn status(&self) -> SyncStatus { let median_seen = self.median_seen(); diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 85f72e5a9ca93..91bc39ee51ed9 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -337,7 +337,7 @@ where } fn try_to_start_warp_sync(&mut self) { - let Phase::WaitingForPeers { warp_sync_provider } = self.phase else { return }; + let Phase::WaitingForPeers { warp_sync_provider } = &self.phase else { return }; if self.peers.len() < MIN_PEERS_TO_START_WARP_SYNC { return @@ -349,7 +349,7 @@ where set_id: 0, authorities: warp_sync_provider.current_authorities(), last_hash: genesis_hash, - warp_sync_provider, + warp_sync_provider: Arc::clone(warp_sync_provider), }; trace!(target: LOG_TARGET, "Started warp sync with {} peers.", self.peers.len()); } @@ -569,23 +569,31 @@ where trace!(target: LOG_TARGET, "Warp sync: imported {imported} of {count}."); + let mut complete = false; + for (result, hash) in results { - if hash != state_sync.target() { + if hash == state_sync.target() { + match result { + Ok(_) => { + complete = true; + }, + Err(e) => { + error!( + target: LOG_TARGET, + "Warp sync failed. Failed to import target block with state: {e:?}." + ); + }, + } + } else { debug!( target: LOG_TARGET, "Unexpected block processed: {hash} with result {result:?}.", ); - continue - } - - if let Err(e) = result { - error!( - target: LOG_TARGET, - "Warp sync failed. Failed to import target block with state: {e:?}." - ); } + } - let total_mib = (self.total_proof_bytes + state_sync.progress().size) / (1024 * 1024); + if complete { + let total_mib = (self.total_proof_bytes + self.total_state_bytes) / (1024 * 1024); info!( target: LOG_TARGET, "Warp sync is complete ({total_mib} MiB), continuing with block sync.", @@ -596,32 +604,13 @@ where } } - /// Get candidate for warp/block request. - fn select_synced_available_peer( - &self, - min_best_number: Option>, - ) -> Option<(&PeerId, &mut Peer)> { - let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); - // Find a random peer that is synced as much as peer majority and is above - // `best_number_at_least`. - for (peer_id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= threshold { - return Some((peer_id, peer)) - } - } - } - - None - } - /// Produce next warp proof request. - fn warp_proof_request(&self) -> Option<(PeerId, WarpProofRequest)> { + fn warp_proof_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { let Phase::WarpProof { last_hash, .. } = &self.phase else { return None }; + // Copy `last_hash` early to cut the borrowing tie. + let begin = *last_hash; + if self .peers .iter() @@ -631,16 +620,18 @@ where return None } - let Some((peer_id, peer)) = self.select_synced_available_peer(None) else { return None }; + let Some((peer_id, peer)) = select_synced_available_peer(&mut self.peers, None) else { + return None + }; - trace!(target: LOG_TARGET, "New WarpProofRequest to {peer_id}, begin hash: {last_hash}."); + trace!(target: LOG_TARGET, "New WarpProofRequest to {peer_id}, begin hash: {begin}."); peer.state = PeerState::DownloadingProofs; - Some((*peer_id, WarpProofRequest { begin: *last_hash })) + Some((*peer_id, WarpProofRequest { begin })) } /// Produce next target block request. - fn target_block_request(&self) -> Option<(PeerId, BlockRequest)> { + fn target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { let Phase::TargetBlock(target_header) = &self.phase else { return None }; if self @@ -652,8 +643,12 @@ where return None } + // Cut the borrowing tie. + let target_hash = target_header.hash(); + let target_number = *target_header.number(); + let Some((peer_id, peer)) = - self.select_synced_available_peer(Some(*target_header.number())) + select_synced_available_peer(&mut self.peers, Some(target_number)) else { return None }; @@ -661,9 +656,10 @@ where trace!( target: LOG_TARGET, "New target block request to {peer_id}, target: {} ({}).", - target_header.hash(), - target_header.number(), + target_hash, + target_number, ); + peer.state = PeerState::DownloadingTargetBlock; Some(( @@ -673,7 +669,7 @@ where fields: BlockAttributes::HEADER | BlockAttributes::BODY | BlockAttributes::JUSTIFICATION, - from: FromBlock::Hash(target_header.hash()), + from: FromBlock::Hash(target_hash), direction: Direction::Ascending, max: Some(1), }, @@ -681,7 +677,7 @@ where } /// Produce next state request. - fn state_request(&self) -> Option<(PeerId, OpaqueStateRequest)> { + fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { let Phase::State(state_sync) = &self.phase else { return None }; if self @@ -694,12 +690,12 @@ where } let Some((peer_id, peer)) = - self.select_synced_available_peer(Some(state_sync.target_block_num())) + select_synced_available_peer(&mut self.peers, Some(state_sync.target_block_num())) else { return None }; - peer.state = PeerState::DownloadingTargetBlock; + peer.state = PeerState::DownloadingState; let request = state_sync.next_request(); trace!( target: LOG_TARGET, @@ -805,3 +801,27 @@ where std::mem::take(&mut self.actions).into_iter() } } + +/// Get candidate for warp/block request. +/// +/// Due to borrowing issues this is a free-standing function accepting a reference to `peers`. +fn select_synced_available_peer( + peers: &mut HashMap>, + min_best_number: Option>, +) -> Option<(&PeerId, &mut Peer)> { + let mut targets: Vec<_> = peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); + // Find a random peer that is synced as much as peer majority and is above + // `best_number_at_least`. + for (peer_id, peer) in peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + return Some((peer_id, peer)) + } + } + } + + None +} From abff7203da6d8b19a4128282a573ac28e4c0b821 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 12:12:13 +0200 Subject: [PATCH 06/54] Report `WarpSyncProgress` during gap sync --- .../client/network/sync/src/chain_sync.rs | 7 +++++++ .../network/sync/src/chain_sync/test.rs | 20 +++++++++---------- substrate/client/network/sync/src/types.rs | 4 ++++ substrate/client/network/sync/src/warp.rs | 1 + 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index be17737123b45..f211754287311 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -34,6 +34,7 @@ use crate::{ schema::v1::StateResponse, state::{ImportResult, StateSync}, types::{BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + warp::{WarpSyncPhase, WarpSyncProgress}, }; use codec::Encode; @@ -386,6 +387,11 @@ where SyncState::Idle }; + let warp_sync_progress = self.gap_sync.as_ref().map(|gap_sync| WarpSyncProgress { + phase: WarpSyncPhase::DownloadingBlocks(gap_sync.best_queued_number), + total_bytes: 0, + }); + SyncStatus { state: sync_state, best_seen_block, @@ -393,6 +399,7 @@ where num_connected_peers: 0u32, queued_blocks: self.queue_blocks.len() as u32, state_sync: self.state_sync.as_ref().map(|s| s.progress()), + warp_sync: warp_sync_progress, } } diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/chain_sync/test.rs index 1cb00cbbc5758..1a4d2b032f154 100644 --- a/substrate/client/network/sync/src/chain_sync/test.rs +++ b/substrate/client/network/sync/src/chain_sync/test.rs @@ -38,7 +38,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { let client = Arc::new(TestClientBuilder::new().build()); let peer_id = PeerId::random(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); let (a1_hash, a1_number) = { let a1 = BlockBuilderBuilder::new(&*client) @@ -91,7 +91,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { fn restart_doesnt_affect_peers_downloading_finality_data() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let peer_id2 = PeerId::random(); @@ -181,7 +181,7 @@ fn send_block_announce(header: Header, peer_id: PeerId, sync: &mut ChainSync>(); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let common_block = blocks[1].clone(); @@ -714,7 +714,7 @@ fn can_import_response_with_missing_blocks() { let empty_client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, empty_client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, empty_client.clone(), 1, 64).unwrap(); let peer_id1 = PeerId::random(); let best_block = blocks[3].clone(); @@ -745,7 +745,7 @@ fn ancestor_search_repeat() { #[test] fn sync_restart_removes_block_but_not_justification_requests() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); let peers = vec![PeerId::random(), PeerId::random()]; @@ -887,7 +887,7 @@ fn request_across_forks() { fork_blocks }; - let mut sync = ChainSync::new(SyncMode::Full, client.clone(), 5, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64).unwrap(); // Add the peers, all at the common ancestor 100. let common_block = blocks.last().unwrap(); diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index 6ecbc8825c4aa..b93f0250ab0c5 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -30,6 +30,8 @@ use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; pub use sc_network_common::sync::SyncMode; +use crate::warp::WarpSyncProgress; + /// The sync status of a peer we are trying to sync with #[derive(Debug)] pub struct PeerInfo { @@ -92,6 +94,8 @@ pub struct SyncStatus { pub queued_blocks: u32, /// State sync status in progress, if any. pub state_sync: Option, + /// Warp sync in progress, if any. + pub warp_sync: Option>, } /// A peer did not behave as expected and should be reported. diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 91bc39ee51ed9..c77b7d9ab34b4 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -774,6 +774,7 @@ where } else { None }, + warp_sync: Some(self.progress()), } } From f913b652d106fa6883beb0874cd8f0a8266b53eb Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 12:22:18 +0200 Subject: [PATCH 07/54] Actualize docs --- substrate/client/network/sync/src/strategy.rs | 2 ++ substrate/client/network/sync/src/warp.rs | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index b0cbda58b896c..a2d9519b143f4 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -77,6 +77,7 @@ where + Sync + 'static, { + /// Notify that a new peer has connected. pub fn new_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { match self { SyncingStrategy::WarpSyncStrategy(strategy) => @@ -86,6 +87,7 @@ where } } + /// Notify that a peer has disconnected. pub fn peer_disconnected(&mut self, peer_id: &PeerId) { match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index c77b7d9ab34b4..a455cce6eec55 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Warp sync support. +//! Warp syncing strategy. Bootstraps chain by downloading warp proofs and state. pub use sp_consensus_grandpa::{AuthorityList, SetId}; @@ -191,7 +191,7 @@ impl WarpSyncParams { } } -/// Warp sync phase. +/// Warp sync phase used by warp sync state machine. enum Phase { /// Waiting for enough peers to connect. WaitingForPeers { warp_sync_provider: Arc> }, @@ -247,6 +247,7 @@ struct Peer { state: PeerState, } +/// Action that should be performed on [`WarpSync`]'s behalf. pub enum WarpSyncAction { /// Send warp proof request to peer. SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, @@ -326,16 +327,19 @@ where self.phase = Phase::TargetBlock(header); } + /// Notify that a new peer has connected. pub fn new_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); self.try_to_start_warp_sync(); } + /// Notify that a peer has disconnected. pub fn peer_disconnected(&mut self, peer_id: &PeerId) { self.peers.remove(peer_id); } + /// Start warp sync as soon as we have enough peers. fn try_to_start_warp_sync(&mut self) { let Phase::WaitingForPeers { warp_sync_provider } = &self.phase else { return }; @@ -354,7 +358,7 @@ where trace!(target: LOG_TARGET, "Started warp sync with {} peers.", self.peers.len()); } - /// Process warp proof response. + /// Process warp proof response. pub fn on_warp_proof_response(&mut self, peer_id: &PeerId, response: EncodedProof) { if let Some(peer) = self.peers.get_mut(peer_id) { peer.state = PeerState::Available; @@ -604,7 +608,7 @@ where } } - /// Produce next warp proof request. + /// Produce warp proof request. fn warp_proof_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { let Phase::WarpProof { last_hash, .. } = &self.phase else { return None }; @@ -630,7 +634,7 @@ where Some((*peer_id, WarpProofRequest { begin })) } - /// Produce next target block request. + /// Produce target block request. fn target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { let Phase::TargetBlock(target_header) = &self.phase else { return None }; @@ -676,7 +680,7 @@ where )) } - /// Produce next state request. + /// Produce state request. fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { let Phase::State(state_sync) = &self.phase else { return None }; From 9c5884de27da44b9c713807effd8e3c9c6aa2f58 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 13:12:17 +0200 Subject: [PATCH 08/54] Fix: stop state requests once state sync is over --- substrate/client/network/sync/src/warp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index a455cce6eec55..f90a9ec9bc6b0 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -684,6 +684,10 @@ where fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { let Phase::State(state_sync) = &self.phase else { return None }; + if state_sync.is_complete() { + return None + } + if self .peers .iter() From 1ace10f0e6e04eca4ef0ab4cdc8163d18bfbc6ae Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 14:16:27 +0200 Subject: [PATCH 09/54] Pass connected peers to `ChainSync` upon construction --- substrate/client/network/sync/src/engine.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 4046716f67f8f..56c4a2921d89c 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -833,7 +833,8 @@ where if strategy_finished { if let SyncingStrategy::WarpSyncStrategy(_) = &self.strategy { - let chain_sync = match ChainSync::new( + // `ChainSyncStrategy` continues `WarpSyncStrategy`. + let mut chain_sync = match ChainSync::new( self.chain_sync_mode, self.client.clone(), self.max_parallel_downloads, @@ -845,6 +846,12 @@ where panic!("Failed to start `ChainSync` due to error: {e}."); }, }; + // Let `ChainSync` know about connected peers. + for (peer_id, peer) in &self.peers { + if peer.info.roles.is_full() { + chain_sync.new_peer(*peer_id, peer.info.best_hash, peer.info.best_number); + } + } self.strategy = SyncingStrategy::ChainSyncStrategy(chain_sync); } else { error!( From 124ea31850712f4d370263beefbfe97a6435ac1f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 23 Nov 2023 14:51:56 +0200 Subject: [PATCH 10/54] Fix: warp sync should be considered major syncing --- substrate/client/network/sync/src/strategy.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index a2d9519b143f4..4fa12d905c838 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -223,7 +223,8 @@ where // Are we in major sync mode? pub fn is_major_syncing(&self) -> bool { match self { - SyncingStrategy::WarpSyncStrategy(_) => false, + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.status().state.is_major_syncing(), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.status().state.is_major_syncing(), } From c7be20bbe041a16d987820b7f3cdcf99f6915d44 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Nov 2023 17:51:50 +0200 Subject: [PATCH 11/54] Fall back to full sync if warp sync has failed --- substrate/client/network/sync/src/warp.rs | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index f90a9ec9bc6b0..82fd3740cfe06 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -574,18 +574,19 @@ where trace!(target: LOG_TARGET, "Warp sync: imported {imported} of {count}."); let mut complete = false; + let mut success = false; for (result, hash) in results { if hash == state_sync.target() { - match result { - Ok(_) => { - complete = true; - }, + complete = true; + success |= match result { + Ok(_) => true, Err(e) => { error!( target: LOG_TARGET, - "Warp sync failed. Failed to import target block with state: {e:?}." + "Failed to import target block with state: {e:?}." ); + false }, } } else { @@ -598,10 +599,18 @@ where if complete { let total_mib = (self.total_proof_bytes + self.total_state_bytes) / (1024 * 1024); - info!( - target: LOG_TARGET, - "Warp sync is complete ({total_mib} MiB), continuing with block sync.", - ); + + if success { + info!( + target: LOG_TARGET, + "Warp sync is complete ({total_mib} MiB), continuing with block sync.", + ); + } else { + error!( + target: LOG_TARGET, + "Warp sync failed. Falling back to full sync.", + ); + } self.phase = Phase::Complete; self.actions.push(WarpSyncAction::Finished); From cf46bceba35542d9ec069d81d9ad2d5230632db7 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 27 Nov 2023 18:43:30 +0200 Subject: [PATCH 12/54] Incapsulate strategy switching in `SyncingStrategy` --- substrate/client/network/sync/src/engine.rs | 95 ++++--------------- substrate/client/network/sync/src/strategy.rs | 91 +++++++++++++++++- 2 files changed, 107 insertions(+), 79 deletions(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 56c4a2921d89c..b217f42af267a 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -25,18 +25,17 @@ use crate::{ }, block_relay_protocol::{BlockDownloader, BlockResponseError}, block_request_handler::MAX_BLOCKS_IN_RESPONSE, - chain_sync::{ChainSync, ChainSyncMode}, pending_responses::{PendingResponses, ResponseEvent}, schema::v1::{StateRequest, StateResponse}, service::{ self, syncing_service::{SyncingService, ToServiceCommand}, }, - strategy::{SyncingAction, SyncingStrategy}, + strategy::{SyncingAction, SyncingConfig, SyncingStrategy}, types::{ BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, }, - warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncParams}, + warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, }; use codec::{Decode, Encode}; @@ -68,10 +67,7 @@ use sc_network::{ }; use sc_network_common::{ role::Roles, - sync::{ - message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, - SyncMode, - }, + sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::{Error as ClientError, HeaderMetadata}; @@ -224,15 +220,6 @@ impl MetricSource for MajorSyncingGauge { } } -fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { - match sync_mode { - SyncMode::Full => ChainSyncMode::Full, - SyncMode::LightState { skip_proofs, storage_chain_mode } => - ChainSyncMode::LightState { skip_proofs, storage_chain_mode }, - SyncMode::Warp => ChainSyncMode::Full, - } -} - /// Peer information #[derive(Debug)] pub struct Peer { @@ -246,17 +233,11 @@ pub struct Peer { } pub struct SyncingEngine { - // Syncing strategy. + /// Syncing strategy. strategy: SyncingStrategy, - /// Chain sync mode - chain_sync_mode: ChainSyncMode, - - /// The number of parallel downloads to guard against slow peers. - max_parallel_downloads: u32, - - /// Maximum number of blocks to request. - max_blocks_per_request: u32, + /// Syncing configuration for startegies. + syncing_config: SyncingConfig, /// Blockchain client. client: Arc, @@ -398,6 +379,7 @@ where } else { net_config.network_config.max_blocks_per_request }; + let syncing_config = SyncingConfig { mode, max_parallel_downloads, max_blocks_per_request }; let cache_capacity = (net_config.network_config.default_peers_set.in_peers + net_config.network_config.default_peers_set.out_peers) .max(1); @@ -473,21 +455,8 @@ where let warp_sync_target_block_header_rx_fused = warp_sync_target_block_header_rx .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); - let chain_sync_mode = chain_sync_mode(mode); - // Initialize syncing strategy. - let strategy = if let SyncMode::Warp = mode { - let warp_sync_config = warp_sync_config - .expect("Warp sync configuration must be supplied in warp sync mode."); - SyncingStrategy::WarpSyncStrategy(WarpSync::new(client.clone(), warp_sync_config)) - } else { - SyncingStrategy::ChainSyncStrategy(ChainSync::new( - chain_sync_mode, - client.clone(), - max_parallel_downloads, - max_blocks_per_request, - )?) - }; + let strategy = SyncingStrategy::new(&syncing_config, client.clone(), warp_sync_config)?; let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); @@ -515,9 +484,7 @@ where roles, client, strategy, - chain_sync_mode, - max_parallel_downloads, - max_blocks_per_request, + syncing_config, network_service, peers: HashMap::new(), block_announce_data_cache: LruMap::new(ByLength::new(cache_capacity)), @@ -757,8 +724,6 @@ where } fn process_strategy_actions(&mut self) { - let mut strategy_finished = false; - self.strategy.actions().for_each(|action| match action { SyncingAction::SendBlockRequest { peer_id, request } => { // Sending block request implies dropping obsolete pending response as we are not @@ -827,40 +792,20 @@ where ) }, SyncingAction::Finished => { - strategy_finished = true; - }, - }); - - if strategy_finished { - if let SyncingStrategy::WarpSyncStrategy(_) = &self.strategy { - // `ChainSyncStrategy` continues `WarpSyncStrategy`. - let mut chain_sync = match ChainSync::new( - self.chain_sync_mode, - self.client.clone(), - self.max_parallel_downloads, - self.max_blocks_per_request, - ) { - Ok(chain_sync) => chain_sync, - Err(e) => { - error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); - panic!("Failed to start `ChainSync` due to error: {e}."); - }, - }; - // Let `ChainSync` know about connected peers. - for (peer_id, peer) in &self.peers { + let connected_peers = self.peers.iter().filter_map(|(peer_id, peer)| { if peer.info.roles.is_full() { - chain_sync.new_peer(*peer_id, peer.info.best_hash, peer.info.best_number); + Some((*peer_id, peer.info.best_hash, peer.info.best_number)) + } else { + None } - } - self.strategy = SyncingStrategy::ChainSyncStrategy(chain_sync); - } else { - error!( - target: LOG_TARGET, - "Syncing unexpectedly finished. Only warp sync strategy can finish.", + }); + self.strategy.switch_to_next( + &self.syncing_config, + self.client.clone(), + connected_peers, ); - debug_assert!(false); - } - } + }, + }); } fn perform_periodic_actions(&mut self) { diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 4fa12d905c838..6eaa9956910f8 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -20,20 +20,48 @@ //! and specific syncing algorithms. use crate::{ - chain_sync::{ChainSync, ChainSyncAction}, + chain_sync::{ChainSync, ChainSyncAction, ChainSyncMode}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, - warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction}, + warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}, }; use libp2p::PeerId; +use log::error; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; -use sc_network_common::sync::message::{BlockAnnounce, BlockData, BlockRequest}; -use sp_blockchain::{HeaderBackend, HeaderMetadata}; +use sc_network_common::sync::{ + message::{BlockAnnounce, BlockData, BlockRequest}, + SyncMode, +}; +use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; use sp_consensus::BlockOrigin; use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor}, Justifications, }; +use std::sync::Arc; + +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + +/// Corresponding `ChainSync` mode. +fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { + match sync_mode { + SyncMode::Full => ChainSyncMode::Full, + SyncMode::LightState { skip_proofs, storage_chain_mode } => + ChainSyncMode::LightState { skip_proofs, storage_chain_mode }, + SyncMode::Warp => ChainSyncMode::Full, + } +} + +/// Syncing configuration containing data for all strategies. +pub struct SyncingConfig { + /// Syncing mode. + pub mode: SyncMode, + /// The number of parallel downloads to guard against slow peers. + pub max_parallel_downloads: u32, + /// Maximum number of blocks to request. + pub max_blocks_per_request: u32, +} #[derive(Debug)] pub enum SyncingAction { @@ -77,6 +105,26 @@ where + Sync + 'static, { + /// Initialize a new syncing startegy. + pub fn new( + config: &SyncingConfig, + client: Arc, + warp_sync_config: Option>, + ) -> Result { + if let SyncMode::Warp = config.mode { + let warp_sync_config = warp_sync_config + .expect("Warp sync configuration must be supplied in warp sync mode."); + Ok(Self::WarpSyncStrategy(WarpSync::new(client.clone(), warp_sync_config))) + } else { + Ok(Self::ChainSyncStrategy(ChainSync::new( + chain_sync_mode(config.mode), + client.clone(), + config.max_parallel_downloads, + config.max_blocks_per_request, + )?)) + } + } + /// Notify that a new peer has connected. pub fn new_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { match self { @@ -304,4 +352,39 @@ where })), } } + + pub fn switch_to_next( + &mut self, + config: &SyncingConfig, + client: Arc, + connected_peers: impl Iterator)>, + ) { + match self { + Self::WarpSyncStrategy(_) => { + // `ChainSyncStrategy` continues `WarpSyncStrategy`. + let mut chain_sync = match ChainSync::new( + chain_sync_mode(config.mode), + client, + config.max_parallel_downloads, + config.max_blocks_per_request, + ) { + Ok(chain_sync) => chain_sync, + Err(e) => { + error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); + panic!("Failed to start `ChainSync` due to error: {e}."); + }, + }; + // Let `ChainSync` know about connected peers. + connected_peers.into_iter().for_each(|(peer_id, best_hash, best_number)| { + chain_sync.new_peer(peer_id, best_hash, best_number) + }); + + *self = Self::ChainSyncStrategy(chain_sync); + }, + Self::ChainSyncStrategy(_) => { + error!(target: LOG_TARGET, "`ChainSyncStrategy` is final startegy, cannot switch to next."); + debug_assert!(false); + }, + } + } } From 982069d935460a56c57bb692e374150194ba28ff Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 28 Nov 2023 13:24:39 +0200 Subject: [PATCH 13/54] minor: remove unused structs in `WarpSync` --- substrate/client/network/sync/src/warp.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 82fd3740cfe06..bb5cdf7db31b3 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -213,22 +213,6 @@ enum Phase { Complete, } -/// Import warp proof result. -pub enum WarpProofImportResult { - /// Import was successful. - Success, - /// Bad proof. - BadResponse, -} - -/// Import target block result. -pub enum TargetBlockImportResult { - /// Import was successful. - Success, - /// Invalid block. - BadResponse, -} - enum PeerState { Available, DownloadingProofs, From 54607ef58e26f429c4b63a40908f4dd162b7b934 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 28 Nov 2023 17:31:37 +0200 Subject: [PATCH 14/54] Extract state sync strategy from warp sync strategy --- substrate/client/network/sync/src/lib.rs | 1 + .../client/network/sync/src/state_strategy.rs | 330 ++++++++++++++++++ substrate/client/network/sync/src/strategy.rs | 95 ++++- substrate/client/network/sync/src/warp.rs | 247 ++----------- 4 files changed, 442 insertions(+), 231 deletions(-) create mode 100644 substrate/client/network/sync/src/state_strategy.rs diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index e22eaafa52502..07b8749dfe45c 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -28,6 +28,7 @@ mod futures_stream; mod pending_responses; mod request_metrics; mod schema; +mod state_strategy; mod strategy; mod types; diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs new file mode 100644 index 0000000000000..c747fd29201c4 --- /dev/null +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -0,0 +1,330 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! State sync strategy. + +use crate::{ + schema::v1::StateResponse, + state::{ImportResult, StateSync}, + types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, +}; +use libp2p::PeerId; +use log::{debug, error, info, trace}; +use sc_client_api::ProofProvider; +use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sp_consensus::BlockOrigin; +use sp_runtime::{ + traits::{Block as BlockT, NumberFor}, + Justifications, SaturatedConversion, +}; +use std::{collections::HashMap, sync::Arc}; + +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + +mod rep { + use sc_network::ReputationChange as Rep; + + /// Peer response data does not have requested bits. + pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); + + /// Reputation change for peers which send us a known bad state. + pub const BAD_STATE: Rep = Rep::new(-(1 << 29), "Bad state"); +} + +/// Action that should be performed on [`StateStrategy`]'s behalf. +pub enum StateStrategyAction { + /// Send state request to peer. + SendStateRequest { peer_id: PeerId, request: OpaqueStateRequest }, + /// Disconnect and report peer. + DropPeer(BadPeer), + /// Import blocks. + ImportBlocks { origin: BlockOrigin, blocks: Vec> }, + /// State sync has finished. + Finished, +} + +enum PeerState { + Available, + DownloadingState, +} + +impl PeerState { + fn is_available(&self) -> bool { + matches!(self, PeerState::Available) + } +} + +struct Peer { + best_number: NumberFor, + state: PeerState, +} + +pub struct StateStrategy { + state_sync: StateSync, + peers: HashMap>, + actions: Vec>, +} + +impl StateStrategy +where + B: BlockT, + Client: ProofProvider + Send + Sync + 'static, +{ + // Create a new instance. + pub fn new( + client: Arc, + target_header: B::Header, + target_body: Option>, + target_justifications: Option, + skip_proof: bool, + initial_peers: impl Iterator)>, + ) -> Self { + let peers = initial_peers + .map(|(peer_id, best_number)| { + (peer_id, Peer { best_number, state: PeerState::Available }) + }) + .collect(); + Self { + state_sync: StateSync::new( + client, + target_header, + target_body, + target_justifications, + skip_proof, + ), + peers, + actions: Vec::new(), + } + } + + /// Notify that a new peer has connected. + pub fn new_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { + self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); + } + + /// Notify that a peer has disconnected. + pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + self.peers.remove(peer_id); + } + + /// Process state response. + pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { + if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { + self.actions.push(StateStrategyAction::DropPeer(bad_peer)); + } + } + + fn on_state_response_inner( + &mut self, + peer_id: PeerId, + response: OpaqueStateResponse, + ) -> Result<(), BadPeer> { + if let Some(peer) = self.peers.get_mut(&peer_id) { + peer.state = PeerState::Available; + } + + let response: Box = response.0.downcast().map_err(|_error| { + error!( + target: LOG_TARGET, + "Failed to downcast opaque state response, this is an implementation bug." + ); + + BadPeer(peer_id, rep::BAD_RESPONSE) + })?; + + debug!( + target: LOG_TARGET, + "Importing state data from {} with {} keys, {} proof nodes.", + peer_id, + response.entries.len(), + response.proof.len(), + ); + + let import_result = self.state_sync.import(*response); + + match import_result { + ImportResult::Import(hash, header, state, body, justifications) => { + let origin = BlockOrigin::NetworkInitialSync; + let block = IncomingBlock { + hash, + header: Some(header), + body, + indexed_body: None, + justifications, + origin: None, + allow_missing_state: true, + import_existing: true, + skip_execution: true, + state: Some(state), + }; + debug!(target: LOG_TARGET, "State download is complete. Import is queued"); + self.actions + .push(StateStrategyAction::ImportBlocks { origin, blocks: vec![block] }); + Ok(()) + }, + ImportResult::Continue => Ok(()), + ImportResult::BadResponse => { + debug!(target: LOG_TARGET, "Bad state data received from {peer_id}"); + Err(BadPeer(peer_id, rep::BAD_STATE)) + }, + } + } + + /// A batch of blocks have been processed, with or without errors. + /// + /// Normally this should be called when target block with state is imported. + pub fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, B::Hash)>, + ) { + trace!(target: LOG_TARGET, "State sync: imported {imported} of {count}."); + + let mut complete = false; + let mut success = false; + + for (result, hash) in results { + if hash == self.state_sync.target() { + complete = true; + success |= match result { + Ok(_) => true, + Err(e) => { + error!( + target: LOG_TARGET, + "Failed to import target block with state: {e:?}." + ); + false + }, + } + } else { + debug!( + target: LOG_TARGET, + "Unexpected block processed: {hash} with result {result:?}.", + ); + } + } + + if complete { + if success { + info!( + target: LOG_TARGET, + "State sync is complete ({} MiB), continuing with block sync.", + self.state_sync.progress().size / (1024 * 1024), + ); + } else { + error!( + target: LOG_TARGET, + "State sync failed. Falling back to full sync.", + ); + } + + self.actions.push(StateStrategyAction::Finished); + } + } + + /// Produce state request. + fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { + if self.state_sync.is_complete() { + return None + } + + if self + .peers + .iter() + .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingState)) + { + // Only one state request at a time is possible. + return None + } + + let Some((peer_id, peer)) = + select_synced_available_peer(&mut self.peers, self.state_sync.target_block_num()) + else { + return None + }; + + peer.state = PeerState::DownloadingState; + let request = self.state_sync.next_request(); + trace!( + target: LOG_TARGET, + "New state request to {peer_id}: {request:?}.", + ); + + Some((*peer_id, OpaqueStateRequest(Box::new(request)))) + } + + /// Returns the current sync status. + pub fn status(&self) -> SyncStatus { + SyncStatus { + state: if self.state_sync.is_complete() { + SyncState::Idle + } else { + SyncState::Downloading { target: self.state_sync.target_block_num() } + }, + best_seen_block: Some(self.state_sync.target_block_num()), + num_peers: self.peers.len().saturated_into(), + num_connected_peers: self.peers.len().saturated_into(), + queued_blocks: 0, + state_sync: Some(self.state_sync.progress()), + warp_sync: None, + } + } + + /// Get the number of peers known to syncing. + pub fn num_peers(&self) -> usize { + self.peers.len() + } + + /// Get actions that should be performed by the owner on [`WarpSync`]'s behalf + #[must_use] + pub fn actions(&mut self) -> impl Iterator> { + let state_request = self + .state_request() + .into_iter() + .map(|(peer_id, request)| StateStrategyAction::SendStateRequest { peer_id, request }); + self.actions.extend(state_request); + + std::mem::take(&mut self.actions).into_iter() + } +} + +/// Get peer for state request. +/// +/// Due to borrowing issues this is a free-standing function accepting a reference to `peers`. +fn select_synced_available_peer( + peers: &mut HashMap>, + min_best_number: NumberFor, +) -> Option<(&PeerId, &mut Peer)> { + let mut targets: Vec<_> = peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number); + // Find a random peer that is synced as much as peer majority and is above + // `best_number_at_least`. + for (peer_id, peer) in peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + return Some((peer_id, peer)) + } + } + } + + None +} diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 6eaa9956910f8..cd7e2cacbb8ba 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -21,11 +21,12 @@ use crate::{ chain_sync::{ChainSync, ChainSyncAction, ChainSyncMode}, + state_strategy::{StateStrategy, StateStrategyAction}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}, }; use libp2p::PeerId; -use log::error; +use log::{error, info}; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::{ @@ -91,6 +92,7 @@ pub enum SyncingAction { /// Proxy to specific syncing strategies. pub enum SyncingStrategy { WarpSyncStrategy(WarpSync), + StateSyncStrategy(StateStrategy), ChainSyncStrategy(ChainSync), } @@ -130,6 +132,8 @@ where match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.new_peer(peer_id, best_hash, best_number), + SyncingStrategy::StateSyncStrategy(strategy) => + strategy.new_peer(peer_id, best_hash, best_number), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.new_peer(peer_id, best_hash, best_number), } @@ -139,6 +143,7 @@ where pub fn peer_disconnected(&mut self, peer_id: &PeerId) { match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), + SyncingStrategy::StateSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), } } @@ -155,6 +160,8 @@ where match self { SyncingStrategy::WarpSyncStrategy(_) => Some((announce.header.hash(), *announce.header.number())), + SyncingStrategy::StateSyncStrategy(_) => + Some((announce.header.hash(), *announce.header.number())), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_validated_block_announce(is_best, peer_id, announce), } @@ -170,6 +177,7 @@ where ) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.set_sync_fork_request(peers, hash, number), } @@ -179,6 +187,7 @@ where pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.request_justification(hash, number), } @@ -188,6 +197,7 @@ where pub fn clear_justification_requests(&mut self) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.clear_justification_requests(), } } @@ -196,6 +206,7 @@ where pub fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_justification_import(hash, number, success), } @@ -211,6 +222,7 @@ where match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.on_block_response(peer_id, request, blocks), + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_block_response(peer_id, request, blocks), } @@ -219,7 +231,8 @@ where /// Process state response. pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { match self { - SyncingStrategy::WarpSyncStrategy(strategy) => + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(strategy) => strategy.on_state_response(peer_id, response), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_state_response(peer_id, response), @@ -231,6 +244,7 @@ where match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.on_warp_proof_response(peer_id, response), + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(_) => {}, } } @@ -243,7 +257,8 @@ where results: Vec<(Result>, BlockImportError>, B::Hash)>, ) { match self { - SyncingStrategy::WarpSyncStrategy(strategy) => + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(strategy) => strategy.on_blocks_processed(imported, count, results), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_blocks_processed(imported, count, results), @@ -254,6 +269,7 @@ where pub fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_block_finalized(hash, number), } @@ -263,6 +279,7 @@ where pub fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { match self { SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.update_chain_info(best_hash, best_number), } @@ -271,8 +288,8 @@ where // Are we in major sync mode? pub fn is_major_syncing(&self) -> bool { match self { - SyncingStrategy::WarpSyncStrategy(strategy) => - strategy.status().state.is_major_syncing(), + SyncingStrategy::WarpSyncStrategy(_) => true, + SyncingStrategy::StateSyncStrategy(_) => true, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.status().state.is_major_syncing(), } @@ -282,6 +299,7 @@ where pub fn num_peers(&self) -> usize { match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.num_peers(), + SyncingStrategy::StateSyncStrategy(strategy) => strategy.num_peers(), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_peers(), } } @@ -290,6 +308,7 @@ where pub fn status(&self) -> SyncStatus { match self { SyncingStrategy::WarpSyncStrategy(strategy) => strategy.status(), + SyncingStrategy::StateSyncStrategy(strategy) => strategy.status(), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.status(), } } @@ -298,6 +317,7 @@ where pub fn num_downloaded_blocks(&self) -> usize { match self { SyncingStrategy::WarpSyncStrategy(_) => 0, + SyncingStrategy::StateSyncStrategy(_) => 0, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_downloaded_blocks(), } } @@ -306,6 +326,7 @@ where pub fn num_sync_requests(&self) -> usize { match self { SyncingStrategy::WarpSyncStrategy(_) => 0, + SyncingStrategy::StateSyncStrategy(_) => 0, SyncingStrategy::ChainSyncStrategy(strategy) => strategy.num_sync_requests(), } } @@ -320,13 +341,18 @@ where SyncingAction::SendWarpProofRequest { peer_id, request }, WarpSyncAction::SendBlockRequest { peer_id, request } => SyncingAction::SendBlockRequest { peer_id, request }, - WarpSyncAction::SendStateRequest { peer_id, request } => - SyncingAction::SendStateRequest { peer_id, request }, WarpSyncAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), - WarpSyncAction::ImportBlocks { origin, blocks } => - SyncingAction::ImportBlocks { origin, blocks }, WarpSyncAction::Finished => SyncingAction::Finished, })), + SyncingStrategy::StateSyncStrategy(strategy) => + Box::new(strategy.actions().map(|action| match action { + StateStrategyAction::SendStateRequest { peer_id, request } => + SyncingAction::SendStateRequest { peer_id, request }, + StateStrategyAction::DropPeer(bad_peer) => SyncingAction::DropPeer(bad_peer), + StateStrategyAction::ImportBlocks { origin, blocks } => + SyncingAction::ImportBlocks { origin, blocks }, + StateStrategyAction::Finished => SyncingAction::Finished, + })), SyncingStrategy::ChainSyncStrategy(strategy) => Box::new(strategy.actions().map(|action| match action { ChainSyncAction::SendBlockRequest { peer_id, request } => @@ -360,8 +386,55 @@ where connected_peers: impl Iterator)>, ) { match self { - Self::WarpSyncStrategy(_) => { - // `ChainSyncStrategy` continues `WarpSyncStrategy`. + Self::WarpSyncStrategy(warp_sync) => { + match warp_sync.take_result() { + Some(res) => { + info!( + target: LOG_TARGET, + "Warp sync finished, continuing with state sync." + ); + let state_sync = StateStrategy::new( + client, + res.target_header, + res.target_body, + res.target_justifications, + false, + connected_peers + .map(|(peer_id, _best_hash, best_number)| (peer_id, best_number)), + ); + + *self = Self::StateSyncStrategy(state_sync); + }, + None => { + error!( + target: LOG_TARGET, + "Warp sync failed. Continuing with full sync." + ); + let mut chain_sync = match ChainSync::new( + chain_sync_mode(config.mode), + client, + config.max_parallel_downloads, + config.max_blocks_per_request, + ) { + Ok(chain_sync) => chain_sync, + Err(e) => { + error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); + panic!("Failed to start `ChainSync` due to error: {e}."); + }, + }; + // Let `ChainSync` know about connected peers. + connected_peers.into_iter().for_each( + |(peer_id, best_hash, best_number)| { + chain_sync.new_peer(peer_id, best_hash, best_number) + }, + ); + + *self = Self::ChainSyncStrategy(chain_sync); + }, + } + }, + Self::StateSyncStrategy(_) => { + info!(target: LOG_TARGET, "State sync finished, continuing with block sync."); let mut chain_sync = match ChainSync::new( chain_sync_mode(config.mode), client, diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index bb5cdf7db31b3..8ae624a1cb40b 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -22,24 +22,20 @@ pub use sp_consensus_grandpa::{AuthorityList, SetId}; use crate::{ chain_sync::validate_blocks, - schema::v1::StateResponse, - state::{ImportResult, StateSync}, - types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + types::{BadPeer, SyncState, SyncStatus}, }; use codec::{Decode, Encode}; use futures::channel::oneshot; use libp2p::PeerId; -use log::{debug, error, info, trace, warn}; +use log::{debug, error, trace, warn}; use sc_client_api::ProofProvider; -use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::message::{ BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; use sp_blockchain::HeaderBackend; -use sp_consensus::BlockOrigin; use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor, Zero}, - SaturatedConversion, + Justifications, SaturatedConversion, }; use std::{collections::HashMap, fmt, sync::Arc}; @@ -104,12 +100,6 @@ mod rep { /// Reputation change for peers which send us a block which we fail to verify. pub const VERIFICATION_FAIL: Rep = Rep::new(-(1 << 29), "Block verification failed"); - - /// Peer response data does not have requested bits. - pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); - - /// Reputation change for peers which send us a known bad state. - pub const BAD_STATE: Rep = Rep::new(-(1 << 29), "Bad state"); } /// Reported warp sync phase. @@ -192,7 +182,7 @@ impl WarpSyncParams { } /// Warp sync phase used by warp sync state machine. -enum Phase { +enum Phase { /// Waiting for enough peers to connect. WaitingForPeers { warp_sync_provider: Arc> }, /// Downloading warp proofs. @@ -207,8 +197,6 @@ enum Phase { PendingTargetBlock, /// Downloading target block. TargetBlock(B::Header), - /// Downloading and importing state. - State(StateSync), /// Warp sync is complete. Complete, } @@ -217,7 +205,6 @@ enum PeerState { Available, DownloadingProofs, DownloadingTargetBlock, - DownloadingState, } impl PeerState { @@ -237,24 +224,27 @@ pub enum WarpSyncAction { SendWarpProofRequest { peer_id: PeerId, request: WarpProofRequest }, /// Send block request to peer. Always implies dropping a stale block request to the same peer. SendBlockRequest { peer_id: PeerId, request: BlockRequest }, - /// Send state request to peer. - SendStateRequest { peer_id: PeerId, request: OpaqueStateRequest }, /// Disconnect and report peer. DropPeer(BadPeer), - /// Import blocks. - ImportBlocks { origin: BlockOrigin, blocks: Vec> }, /// Warp sync has finished. Finished, } +pub struct WarpSyncResult { + pub target_header: B::Header, + pub target_body: Option>, + pub target_justifications: Option, +} + /// Warp sync state machine. Accumulates warp proofs and state. pub struct WarpSync { - phase: Phase, + phase: Phase, client: Arc, total_proof_bytes: u64, total_state_bytes: u64, peers: HashMap>, actions: Vec>, + result: Option>, } impl WarpSync @@ -278,6 +268,7 @@ where total_state_bytes: 0, peers: HashMap::new(), actions: vec![WarpSyncAction::Finished], + result: None, } } @@ -294,6 +285,7 @@ where total_state_bytes: 0, peers: HashMap::new(), actions: Vec::new(), + result: None, } } @@ -456,151 +448,16 @@ where return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) } - debug!( - target: LOG_TARGET, - "Downloaded target block {} ({}), continuing with state sync.", - header.hash(), - header.number(), - ); - let state_sync = StateSync::new( - self.client.clone(), - header.clone(), - block.body, - block.justifications, - false, - ); - self.phase = Phase::State(state_sync); + self.result = Some(WarpSyncResult { + target_header: header.clone(), + target_body: block.body, + target_justifications: block.justifications, + }); + self.phase = Phase::Complete; + self.actions.push(WarpSyncAction::Finished); Ok(()) } - /// Process state response. - pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { - if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { - self.actions.push(WarpSyncAction::DropPeer(bad_peer)); - } - } - - fn on_state_response_inner( - &mut self, - peer_id: PeerId, - response: OpaqueStateResponse, - ) -> Result<(), BadPeer> { - if let Some(peer) = self.peers.get_mut(&peer_id) { - peer.state = PeerState::Available; - } - - let Phase::State(state_sync) = &mut self.phase else { - debug!(target: "sync", "Unexpected state response"); - return Err(BadPeer(peer_id, rep::UNEXPECTED_RESPONSE)) - }; - - let response: Box = response.0.downcast().map_err(|_error| { - error!( - target: LOG_TARGET, - "Failed to downcast opaque state response, this is an implementation bug." - ); - - BadPeer(peer_id, rep::BAD_RESPONSE) - })?; - - debug!( - target: LOG_TARGET, - "Importing state data from {} with {} keys, {} proof nodes.", - peer_id, - response.entries.len(), - response.proof.len(), - ); - - let import_result = state_sync.import(*response); - self.total_state_bytes = state_sync.progress().size; - - match import_result { - ImportResult::Import(hash, header, state, body, justifications) => { - let origin = BlockOrigin::NetworkInitialSync; - let block = IncomingBlock { - hash, - header: Some(header), - body, - indexed_body: None, - justifications, - origin: None, - allow_missing_state: true, - import_existing: true, - skip_execution: true, - state: Some(state), - }; - debug!(target: LOG_TARGET, "State download is complete. Import is queued"); - self.actions.push(WarpSyncAction::ImportBlocks { origin, blocks: vec![block] }); - Ok(()) - }, - ImportResult::Continue => Ok(()), - ImportResult::BadResponse => { - debug!(target: LOG_TARGET, "Bad state data received from {peer_id}"); - Err(BadPeer(peer_id, rep::BAD_STATE)) - }, - } - } - - /// A batch of blocks have been processed, with or without errors. - /// - /// Normally this should be called when target block with state is imported. - pub fn on_blocks_processed( - &mut self, - imported: usize, - count: usize, - results: Vec<(Result>, BlockImportError>, B::Hash)>, - ) { - let Phase::State(state_sync) = &self.phase else { - debug!(target: LOG_TARGET, "Unexpected block import of {imported} of {count}."); - return - }; - - trace!(target: LOG_TARGET, "Warp sync: imported {imported} of {count}."); - - let mut complete = false; - let mut success = false; - - for (result, hash) in results { - if hash == state_sync.target() { - complete = true; - success |= match result { - Ok(_) => true, - Err(e) => { - error!( - target: LOG_TARGET, - "Failed to import target block with state: {e:?}." - ); - false - }, - } - } else { - debug!( - target: LOG_TARGET, - "Unexpected block processed: {hash} with result {result:?}.", - ); - } - } - - if complete { - let total_mib = (self.total_proof_bytes + self.total_state_bytes) / (1024 * 1024); - - if success { - info!( - target: LOG_TARGET, - "Warp sync is complete ({total_mib} MiB), continuing with block sync.", - ); - } else { - error!( - target: LOG_TARGET, - "Warp sync failed. Falling back to full sync.", - ); - } - - self.phase = Phase::Complete; - self.actions.push(WarpSyncAction::Finished); - } - } - /// Produce warp proof request. fn warp_proof_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { let Phase::WarpProof { last_hash, .. } = &self.phase else { return None }; @@ -673,40 +530,7 @@ where )) } - /// Produce state request. - fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { - let Phase::State(state_sync) = &self.phase else { return None }; - - if state_sync.is_complete() { - return None - } - - if self - .peers - .iter() - .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingState)) - { - // Only one state request at a time is possible. - return None - } - - let Some((peer_id, peer)) = - select_synced_available_peer(&mut self.peers, Some(state_sync.target_block_num())) - else { - return None - }; - - peer.state = PeerState::DownloadingState; - let request = state_sync.next_request(); - trace!( - target: LOG_TARGET, - "New state request to {peer_id}: {request:?}.", - ); - - Some((*peer_id, OpaqueStateRequest(Box::new(request)))) - } - - /// Returns state sync estimated progress (stage, bytes received). + /// Returns warp sync estimated progress (stage, bytes received). pub fn progress(&self) -> WarpSyncProgress { match &self.phase { Phase::WaitingForPeers { .. } => WarpSyncProgress { @@ -727,14 +551,6 @@ where phase: WarpSyncPhase::AwaitingTargetBlock, total_bytes: self.total_proof_bytes, }, - Phase::State(state_sync) => WarpSyncProgress { - phase: if state_sync.is_complete() { - WarpSyncPhase::ImportingState - } else { - WarpSyncPhase::DownloadingState - }, - total_bytes: self.total_proof_bytes + state_sync.progress().size, - }, Phase::Complete => WarpSyncProgress { phase: WarpSyncPhase::Complete, total_bytes: self.total_proof_bytes + self.total_state_bytes, @@ -755,8 +571,6 @@ where Phase::WarpProof { .. } => SyncState::Downloading { target: Zero::zero() }, Phase::PendingTargetBlock => SyncState::Downloading { target: Zero::zero() }, Phase::TargetBlock(header) => SyncState::Downloading { target: *header.number() }, - Phase::State(state_sync) => - SyncState::Downloading { target: state_sync.target_block_num() }, Phase::Complete => SyncState::Idle, }, best_seen_block: match &self.phase { @@ -764,17 +578,12 @@ where Phase::WarpProof { .. } => None, Phase::PendingTargetBlock => None, Phase::TargetBlock(header) => Some(*header.number()), - Phase::State(state_sync) => Some(state_sync.target_block_num()), Phase::Complete => None, }, num_peers: self.peers.len().saturated_into(), num_connected_peers: self.peers.len().saturated_into(), queued_blocks: 0, - state_sync: if let Phase::State(state_sync) = &self.phase { - Some(state_sync.progress()) - } else { - None - }, + state_sync: None, warp_sync: Some(self.progress()), } } @@ -794,14 +603,12 @@ where .map(|(peer_id, request)| WarpSyncAction::SendBlockRequest { peer_id, request }); self.actions.extend(target_block_request); - let state_request = self - .state_request() - .into_iter() - .map(|(peer_id, request)| WarpSyncAction::SendStateRequest { peer_id, request }); - self.actions.extend(state_request); - std::mem::take(&mut self.actions).into_iter() } + + pub fn take_result(&mut self) -> Option> { + self.result.take() + } } /// Get candidate for warp/block request. From 3fcfa13bf1138fd991980ae2171868685180ce47 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 28 Nov 2023 17:35:24 +0200 Subject: [PATCH 15/54] minor: revert useless change in `types.rs` --- substrate/client/network/sync/src/types.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index b93f0250ab0c5..5931cf47b28a4 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -23,6 +23,7 @@ use sc_network_common::{role::Roles, types::ReputationChange}; use libp2p::PeerId; +use crate::warp::WarpSyncProgress; use sc_network_common::sync::message::BlockRequest; use sp_runtime::traits::{Block as BlockT, NumberFor}; @@ -30,8 +31,6 @@ use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; pub use sc_network_common::sync::SyncMode; -use crate::warp::WarpSyncProgress; - /// The sync status of a peer we are trying to sync with #[derive(Debug)] pub struct PeerInfo { From a61401b379da9620270cb29af9843a0e1d31ef0f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 29 Nov 2023 13:28:10 +0200 Subject: [PATCH 16/54] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- substrate/client/network/sync/src/engine.rs | 10 +++++----- substrate/client/network/sync/src/warp.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index b217f42af267a..78c694e30776f 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -793,11 +793,11 @@ where }, SyncingAction::Finished => { let connected_peers = self.peers.iter().filter_map(|(peer_id, peer)| { - if peer.info.roles.is_full() { - Some((*peer_id, peer.info.best_hash, peer.info.best_number)) - } else { - None - } + peer.info.roles.is_full().then_some(( + *peer_id, + peer.info.best_hash, + peer.info.best_number, + )) }); self.strategy.switch_to_next( &self.syncing_config, diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 8ae624a1cb40b..9426d2a237652 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -399,7 +399,7 @@ where } let Phase::TargetBlock(header) = &mut self.phase else { - debug!(target: "sync", "Unexpected target block response from {peer_id}"); + debug!(target: LOG_TARGET, "Unexpected target block response from {peer_id}"); return Err(BadPeer(peer_id, rep::UNEXPECTED_RESPONSE)) }; @@ -426,7 +426,7 @@ where let Some(block_header) = &block.header else { debug!( - target: "sync", + target: LOG_TARGET, "Downloading target block failed: missing header in response from {peer_id}.", ); return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) @@ -434,7 +434,7 @@ where if block_header != header { debug!( - target: "sync", + target: LOG_TARGET, "Downloading target block failed: different header in response from {peer_id}.", ); return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) @@ -442,7 +442,7 @@ where if block.body.is_none() { debug!( - target: "sync", + target: LOG_TARGET, "Downloading target block failed: missing body in response from {peer_id}.", ); return Err(BadPeer(peer_id, rep::VERIFICATION_FAIL)) @@ -558,7 +558,7 @@ where } } - /// Get the number of peers known to syncing. + /// Get the number of peers known to warp sync. pub fn num_peers(&self) -> usize { self.peers.len() } From 0b07b3645774b9f7ec6128647c7b487a9085d9c3 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 29 Nov 2023 17:12:41 +0200 Subject: [PATCH 17/54] Apply review suggestions --- .../client/network/sync/src/chain_sync.rs | 24 +++++++++---------- substrate/client/network/sync/src/engine.rs | 4 ++-- .../client/network/sync/src/state_strategy.rs | 5 ++-- substrate/client/network/sync/src/strategy.rs | 20 ++++++++-------- substrate/client/network/sync/src/warp.rs | 4 ++-- 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index f211754287311..768dd5ea43083 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -422,8 +422,8 @@ where } /// Notify syncing state machine that a new sync peer has connected. - pub fn new_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { - match self.new_peer_inner(peer_id, best_hash, best_number) { + pub fn add_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { + match self.add_peer_inner(peer_id, best_hash, best_number) { Ok(Some(request)) => self.actions.push(ChainSyncAction::SendBlockRequest { peer_id, request }), Ok(None) => {}, @@ -432,7 +432,7 @@ where } #[must_use] - fn new_peer_inner( + fn add_peer_inner( &mut self, peer_id: PeerId, best_hash: B::Hash, @@ -1022,15 +1022,13 @@ where return None } - let new_peer_info = if is_best { + let peer_info = is_best.then_some({ // update their best block peer.best_number = number; peer.best_hash = hash; - Some((hash, number)) - } else { - None - }; + (hash, number) + }); // If the announced block is the best they have and is not ahead of us, our common number // is either one further ahead or it's the one they just announced, if we know about it. @@ -1051,7 +1049,7 @@ where if let Some(target) = self.fork_targets.get_mut(&hash) { target.peers.insert(peer_id); } - return new_peer_info + return peer_info } if ancient_parent { @@ -1062,7 +1060,7 @@ where hash, announce.header, ); - return new_peer_info + return peer_info } if self.status().state == SyncState::Idle { @@ -1084,11 +1082,11 @@ where .insert(peer_id); } - new_peer_info + peer_info } /// Notify that a sync peer has disconnected. - pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + pub fn remove_peer(&mut self, peer_id: &PeerId) { self.blocks.clear_peer_download(peer_id); if let Some(gap_sync) = &mut self.gap_sync { gap_sync.blocks.clear_peer_download(peer_id) @@ -1266,7 +1264,7 @@ where } // handle peers that were in other states. - let action = match self.new_peer_inner(peer_id, p.best_hash, p.best_number) { + let action = match self.add_peer_inner(peer_id, p.best_hash, p.best_number) { // since the request is not a justification, remove it from pending responses Ok(None) => ChainSyncAction::CancelBlockRequest { peer_id }, // update the request if the new one is available diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 6362dfefc2926..f6bf56d4647cf 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -1031,7 +1031,7 @@ where } } - self.strategy.peer_disconnected(&peer_id); + self.strategy.remove_peer(&peer_id); self.pending_responses.remove(&peer_id); self.event_streams .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer_id)).is_ok()); @@ -1177,7 +1177,7 @@ where // Only forward full peers to syncing strategy. if status.roles.is_full() { - self.strategy.new_peer(peer_id, peer.info.best_hash, peer.info.best_number); + self.strategy.add_peer(peer_id, peer.info.best_hash, peer.info.best_number); } log::debug!(target: LOG_TARGET, "Connected {peer_id}"); diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index c747fd29201c4..1df0f6bbf703d 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -75,6 +75,7 @@ struct Peer { state: PeerState, } +/// Syncing strategy that downloads and imports a recent state directly. pub struct StateStrategy { state_sync: StateSync, peers: HashMap>, @@ -114,12 +115,12 @@ where } /// Notify that a new peer has connected. - pub fn new_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { + pub fn add_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); } /// Notify that a peer has disconnected. - pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + pub fn remove_peer(&mut self, peer_id: &PeerId) { self.peers.remove(peer_id); } diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index cd7e2cacbb8ba..1fd7ce095c554 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -128,23 +128,23 @@ where } /// Notify that a new peer has connected. - pub fn new_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { + pub fn add_peer(&mut self, peer_id: PeerId, best_hash: B::Hash, best_number: NumberFor) { match self { SyncingStrategy::WarpSyncStrategy(strategy) => - strategy.new_peer(peer_id, best_hash, best_number), + strategy.add_peer(peer_id, best_hash, best_number), SyncingStrategy::StateSyncStrategy(strategy) => - strategy.new_peer(peer_id, best_hash, best_number), + strategy.add_peer(peer_id, best_hash, best_number), SyncingStrategy::ChainSyncStrategy(strategy) => - strategy.new_peer(peer_id, best_hash, best_number), + strategy.add_peer(peer_id, best_hash, best_number), } } /// Notify that a peer has disconnected. - pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + pub fn remove_peer(&mut self, peer_id: &PeerId) { match self { - SyncingStrategy::WarpSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), - SyncingStrategy::StateSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), - SyncingStrategy::ChainSyncStrategy(strategy) => strategy.peer_disconnected(peer_id), + SyncingStrategy::WarpSyncStrategy(strategy) => strategy.remove_peer(peer_id), + SyncingStrategy::StateSyncStrategy(strategy) => strategy.remove_peer(peer_id), + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.remove_peer(peer_id), } } @@ -425,7 +425,7 @@ where // Let `ChainSync` know about connected peers. connected_peers.into_iter().for_each( |(peer_id, best_hash, best_number)| { - chain_sync.new_peer(peer_id, best_hash, best_number) + chain_sync.add_peer(peer_id, best_hash, best_number) }, ); @@ -449,7 +449,7 @@ where }; // Let `ChainSync` know about connected peers. connected_peers.into_iter().for_each(|(peer_id, best_hash, best_number)| { - chain_sync.new_peer(peer_id, best_hash, best_number) + chain_sync.add_peer(peer_id, best_hash, best_number) }); *self = Self::ChainSyncStrategy(chain_sync); diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 9426d2a237652..535260bea3969 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -304,14 +304,14 @@ where } /// Notify that a new peer has connected. - pub fn new_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { + pub fn add_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); self.try_to_start_warp_sync(); } /// Notify that a peer has disconnected. - pub fn peer_disconnected(&mut self, peer_id: &PeerId) { + pub fn remove_peer(&mut self, peer_id: &PeerId) { self.peers.remove(peer_id); } From fab9cd2eb419836a8ebff420feb3b1651d8c4ac5 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 29 Nov 2023 17:21:25 +0200 Subject: [PATCH 18/54] Apply more review suggestions --- substrate/client/network/sync/src/state_strategy.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index 1df0f6bbf703d..1fc7f0f9f267f 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -157,9 +157,7 @@ where response.proof.len(), ); - let import_result = self.state_sync.import(*response); - - match import_result { + match self.state_sync.import(*response) { ImportResult::Import(hash, header, state, body, justifications) => { let origin = BlockOrigin::NetworkInitialSync; let block = IncomingBlock { From cfde1302ee41a636245976f223dbebb106095bbc Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 29 Nov 2023 17:35:29 +0200 Subject: [PATCH 19/54] Define `LOG_TARGET` in crate root and get rid of hardcoded targets --- .../sync/src/block_announce_validator.rs | 5 +---- .../network/sync/src/block_request_handler.rs | 2 +- substrate/client/network/sync/src/blocks.rs | 13 +++++++------ substrate/client/network/sync/src/chain_sync.rs | 10 ++++------ substrate/client/network/sync/src/engine.rs | 6 ++---- .../client/network/sync/src/extra_requests.rs | 13 +++++++------ substrate/client/network/sync/src/lib.rs | 3 +++ .../network/sync/src/pending_responses.rs | 5 +---- substrate/client/network/sync/src/state.rs | 17 +++++++++-------- .../network/sync/src/state_request_handler.rs | 6 ++++-- .../client/network/sync/src/state_strategy.rs | 4 +--- substrate/client/network/sync/src/strategy.rs | 4 +--- substrate/client/network/sync/src/warp.rs | 4 +--- .../network/sync/src/warp_request_handler.rs | 9 ++++++--- 14 files changed, 48 insertions(+), 53 deletions(-) diff --git a/substrate/client/network/sync/src/block_announce_validator.rs b/substrate/client/network/sync/src/block_announce_validator.rs index 961b581cddcef..62c0d1c16e213 100644 --- a/substrate/client/network/sync/src/block_announce_validator.rs +++ b/substrate/client/network/sync/src/block_announce_validator.rs @@ -19,7 +19,7 @@ //! [`BlockAnnounceValidator`] is responsible for async validation of block announcements. //! [`Stream`] implemented by [`BlockAnnounceValidator`] never terminates. -use crate::futures_stream::FuturesStream; +use crate::{futures_stream::FuturesStream, LOG_TARGET}; use futures::{stream::FusedStream, Future, FutureExt, Stream, StreamExt}; use libp2p::PeerId; use log::{debug, error, trace, warn}; @@ -33,9 +33,6 @@ use std::{ task::{Context, Poll}, }; -/// Log target for this file. -const LOG_TARGET: &str = "sync"; - /// Maximum number of concurrent block announce validations. /// /// If the queue reaches the maximum, we drop any new block diff --git a/substrate/client/network/sync/src/block_request_handler.rs b/substrate/client/network/sync/src/block_request_handler.rs index f363dda3a2d18..1a055570b2738 100644 --- a/substrate/client/network/sync/src/block_request_handler.rs +++ b/substrate/client/network/sync/src/block_request_handler.rs @@ -24,6 +24,7 @@ use crate::{ BlockResponse as BlockResponseSchema, BlockResponse, Direction, }, service::network::NetworkServiceHandle, + LOG_TARGET, }; use codec::{Decode, DecodeAll, Encode}; @@ -56,7 +57,6 @@ use std::{ /// Maximum blocks per response. pub(crate) const MAX_BLOCKS_IN_RESPONSE: usize = 128; -const LOG_TARGET: &str = "sync"; const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER: usize = 2; diff --git a/substrate/client/network/sync/src/blocks.rs b/substrate/client/network/sync/src/blocks.rs index 539a8a5d612cb..4988045a47867 100644 --- a/substrate/client/network/sync/src/blocks.rs +++ b/substrate/client/network/sync/src/blocks.rs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use crate::LOG_TARGET; use libp2p::PeerId; use log::trace; use sc_network_common::sync::message; @@ -87,10 +88,10 @@ impl BlockCollection { match self.blocks.get(&start) { Some(&BlockRangeState::Downloading { .. }) => { - trace!(target: "sync", "Inserting block data still marked as being downloaded: {}", start); + trace!(target: LOG_TARGET, "Inserting block data still marked as being downloaded: {}", start); }, Some(BlockRangeState::Complete(existing)) if existing.len() >= blocks.len() => { - trace!(target: "sync", "Ignored block data already downloaded: {}", start); + trace!(target: LOG_TARGET, "Ignored block data already downloaded: {}", start); return }, _ => (), @@ -162,7 +163,7 @@ impl BlockCollection { }; // crop to peers best if range.start > peer_best { - trace!(target: "sync", "Out of range for peer {} ({} vs {})", who, range.start, peer_best); + trace!(target: LOG_TARGET, "Out of range for peer {} ({} vs {})", who, range.start, peer_best); return None } range.end = cmp::min(peer_best + One::one(), range.end); @@ -173,7 +174,7 @@ impl BlockCollection { .next() .map_or(false, |(n, _)| range.start > *n + max_ahead.into()) { - trace!(target: "sync", "Too far ahead for peer {} ({})", who, range.start); + trace!(target: LOG_TARGET, "Too far ahead for peer {} ({})", who, range.start); return None } @@ -224,7 +225,7 @@ impl BlockCollection { }; *range_data = BlockRangeState::Queued { len }; } - trace!(target: "sync", "{} blocks ready for import", ready.len()); + trace!(target: LOG_TARGET, "{} blocks ready for import", ready.len()); ready } @@ -235,7 +236,7 @@ impl BlockCollection { self.blocks.remove(&block_num); block_num += One::one(); } - trace!(target: "sync", "Cleared blocks from {:?} to {:?}", from, to); + trace!(target: LOG_TARGET, "Cleared blocks from {:?} to {:?}", from, to); } } diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index 768dd5ea43083..cab2669778671 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -35,6 +35,7 @@ use crate::{ state::{ImportResult, StateSync}, types::{BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, warp::{WarpSyncPhase, WarpSyncProgress}, + LOG_TARGET, }; use codec::Encode; @@ -66,9 +67,6 @@ use std::{ #[cfg(test)] mod test; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - /// Maximum blocks to store in the import queue. const MAX_IMPORTING_BLOCKS: usize = 2048; @@ -1045,7 +1043,7 @@ where // known block case if known || self.is_already_downloading(&hash) { - trace!(target: "sync", "Known block announce from {}: {}", peer_id, hash); + trace!(target: LOG_TARGET, "Known block announce from {}: {}", peer_id, hash); if let Some(target) = self.fork_targets.get_mut(&hash) { target.peers.insert(peer_id); } @@ -1054,7 +1052,7 @@ where if ancient_parent { trace!( - target: "sync", + target: LOG_TARGET, "Ignored ancient block announced from {}: {} {:?}", peer_id, hash, @@ -1065,7 +1063,7 @@ where if self.status().state == SyncState::Idle { trace!( - target: "sync", + target: LOG_TARGET, "Added sync target for block announced from {}: {} {:?}", peer_id, hash, diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index f6bf56d4647cf..aec28fe4bb897 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -36,6 +36,7 @@ use crate::{ BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, }, warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, + LOG_TARGET, }; use codec::{Decode, DecodeAll, Encode}; @@ -97,9 +98,6 @@ const TICK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1100) /// Maximum number of known block hashes to keep for a peer. const MAX_KNOWN_BLOCKS: usize = 1024; // ~32kb per peer + LruHashSet overhead -/// Logging target for the file. -const LOG_TARGET: &str = "sync"; - /// If the block announces stream to peer has been inactive for 30 seconds meaning local node /// has not sent or received block announcements to/from the peer, report the node for inactivity, /// disconnect it and attempt to establish connection to some other peer. @@ -868,7 +866,7 @@ where }, ToServiceCommand::AnnounceBlock(hash, data) => self.announce_block(hash, data), ToServiceCommand::NewBestBlockImported(hash, number) => { - log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); + log::debug!(target: LOG_TARGET, "New best block imported {:?}/#{}", hash, number); self.strategy.update_chain_info(&hash, number); let _ = self.notification_service.try_set_handshake( diff --git a/substrate/client/network/sync/src/extra_requests.rs b/substrate/client/network/sync/src/extra_requests.rs index 8edd1a772e26b..296a48635a049 100644 --- a/substrate/client/network/sync/src/extra_requests.rs +++ b/substrate/client/network/sync/src/extra_requests.rs @@ -19,6 +19,7 @@ use crate::{ chain_sync::{PeerSync, PeerSyncState}, request_metrics::Metrics, + LOG_TARGET, }; use fork_tree::ForkTree; use libp2p::PeerId; @@ -102,7 +103,7 @@ impl ExtraRequests { // ignore the `Revert` error. }, Err(err) => { - debug!(target: "sync", "Failed to insert request {:?} into tree: {}", request, err); + debug!(target: LOG_TARGET, "Failed to insert request {:?} into tree: {}", request, err); }, _ => (), } @@ -126,7 +127,7 @@ impl ExtraRequests { // messages to chain sync. if let Some(request) = self.active_requests.remove(&who) { if let Some(r) = resp { - trace!(target: "sync", + trace!(target: LOG_TARGET, "Queuing import of {} from {:?} for {:?}", self.request_type_name, who, request, ); @@ -134,7 +135,7 @@ impl ExtraRequests { self.importing_requests.insert(request); return Some((who, request.0, request.1, r)) } else { - trace!(target: "sync", + trace!(target: LOG_TARGET, "Empty {} response from {:?} for {:?}", self.request_type_name, who, request, ); @@ -142,7 +143,7 @@ impl ExtraRequests { self.failed_requests.entry(request).or_default().push((who, Instant::now())); self.pending_requests.push_front(request); } else { - trace!(target: "sync", + trace!(target: LOG_TARGET, "No active {} request to {:?}", self.request_type_name, who, ); @@ -217,7 +218,7 @@ impl ExtraRequests { }; if self.tree.finalize_root(&finalized_hash).is_none() { - warn!(target: "sync", + warn!(target: LOG_TARGET, "‼️ Imported {:?} {:?} which isn't a root in the tree: {:?}", finalized_hash, finalized_number, self.tree.roots().collect::>() ); @@ -322,7 +323,7 @@ impl<'a, B: BlockT> Matcher<'a, B> { } self.extras.active_requests.insert(*peer, request); - trace!(target: "sync", + trace!(target: LOG_TARGET, "Sending {} request to {:?} for {:?}", self.extras.request_type_name, peer, request, ); diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index 07b8749dfe45c..cc362636b5d16 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -42,3 +42,6 @@ pub mod state; pub mod state_request_handler; pub mod warp; pub mod warp_request_handler; + +/// Log target for this crate. +const LOG_TARGET: &str = "sync"; diff --git a/substrate/client/network/sync/src/pending_responses.rs b/substrate/client/network/sync/src/pending_responses.rs index 55308dfc1ea90..1e2f6c2575c2e 100644 --- a/substrate/client/network/sync/src/pending_responses.rs +++ b/substrate/client/network/sync/src/pending_responses.rs @@ -19,7 +19,7 @@ //! [`PendingResponses`] is responsible for keeping track of pending responses and //! polling them. [`Stream`] implemented by [`PendingResponses`] never terminates. -use crate::types::PeerRequest; +use crate::{types::PeerRequest, LOG_TARGET}; use futures::{ channel::oneshot, future::BoxFuture, @@ -33,9 +33,6 @@ use sp_runtime::traits::Block as BlockT; use std::task::{Context, Poll, Waker}; use tokio_stream::StreamMap; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - /// Response result. type ResponseResult = Result, RequestFailure>, oneshot::Canceled>; diff --git a/substrate/client/network/sync/src/state.rs b/substrate/client/network/sync/src/state.rs index 5d34613d1c5e3..f7cfbc8ac3d95 100644 --- a/substrate/client/network/sync/src/state.rs +++ b/substrate/client/network/sync/src/state.rs @@ -21,6 +21,7 @@ use crate::{ schema::v1::{StateEntry, StateRequest, StateResponse}, types::StateDownloadProgress, + LOG_TARGET, }; use codec::{Decode, Encode}; use log::debug; @@ -91,20 +92,20 @@ where /// Validate and import a state response. pub fn import(&mut self, response: StateResponse) -> ImportResult { if response.entries.is_empty() && response.proof.is_empty() { - debug!(target: "sync", "Bad state response"); + debug!(target: LOG_TARGET, "Bad state response"); return ImportResult::BadResponse } if !self.skip_proof && response.proof.is_empty() { - debug!(target: "sync", "Missing proof"); + debug!(target: LOG_TARGET, "Missing proof"); return ImportResult::BadResponse } let complete = if !self.skip_proof { - debug!(target: "sync", "Importing state from {} trie nodes", response.proof.len()); + debug!(target: LOG_TARGET, "Importing state from {} trie nodes", response.proof.len()); let proof_size = response.proof.len() as u64; let proof = match CompactProof::decode(&mut response.proof.as_ref()) { Ok(proof) => proof, Err(e) => { - debug!(target: "sync", "Error decoding proof: {:?}", e); + debug!(target: LOG_TARGET, "Error decoding proof: {:?}", e); return ImportResult::BadResponse }, }; @@ -115,7 +116,7 @@ where ) { Err(e) => { debug!( - target: "sync", + target: LOG_TARGET, "StateResponse failed proof verification: {}", e, ); @@ -123,11 +124,11 @@ where }, Ok(values) => values, }; - debug!(target: "sync", "Imported with {} keys", values.len()); + debug!(target: LOG_TARGET, "Imported with {} keys", values.len()); let complete = completed == 0; if !complete && !values.update_last_key(completed, &mut self.last_key) { - debug!(target: "sync", "Error updating key cursor, depth: {}", completed); + debug!(target: LOG_TARGET, "Error updating key cursor, depth: {}", completed); }; for values in values.0 { @@ -185,7 +186,7 @@ where } for state in response.entries { debug!( - target: "sync", + target: LOG_TARGET, "Importing state from {:?} to {:?}", state.entries.last().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), state.entries.first().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), diff --git a/substrate/client/network/sync/src/state_request_handler.rs b/substrate/client/network/sync/src/state_request_handler.rs index f78fadccc2d5f..6bd2389fb5d1b 100644 --- a/substrate/client/network/sync/src/state_request_handler.rs +++ b/substrate/client/network/sync/src/state_request_handler.rs @@ -17,7 +17,10 @@ //! Helper for handling (i.e. answering) state requests from a remote peer via the //! `crate::request_responses::RequestResponsesBehaviour`. -use crate::schema::v1::{KeyValueStateEntry, StateEntry, StateRequest, StateResponse}; +use crate::{ + schema::v1::{KeyValueStateEntry, StateEntry, StateRequest, StateResponse}, + LOG_TARGET, +}; use codec::{Decode, Encode}; use futures::{channel::oneshot, stream::StreamExt}; @@ -39,7 +42,6 @@ use std::{ time::Duration, }; -const LOG_TARGET: &str = "sync"; const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; // Actual reponse may be bigger. const MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER: usize = 2; diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index 1fc7f0f9f267f..4e9cba0fa777a 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -22,6 +22,7 @@ use crate::{ schema::v1::StateResponse, state::{ImportResult, StateSync}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + LOG_TARGET, }; use libp2p::PeerId; use log::{debug, error, info, trace}; @@ -34,9 +35,6 @@ use sp_runtime::{ }; use std::{collections::HashMap, sync::Arc}; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - mod rep { use sc_network::ReputationChange as Rep; diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 1fd7ce095c554..3c9db740a9ea7 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -24,6 +24,7 @@ use crate::{ state_strategy::{StateStrategy, StateStrategyAction}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}, + LOG_TARGET, }; use libp2p::PeerId; use log::{error, info}; @@ -41,9 +42,6 @@ use sp_runtime::{ }; use std::sync::Arc; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - /// Corresponding `ChainSync` mode. fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { match sync_mode { diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 535260bea3969..9b4c0f998e1c9 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -23,6 +23,7 @@ pub use sp_consensus_grandpa::{AuthorityList, SetId}; use crate::{ chain_sync::validate_blocks, types::{BadPeer, SyncState, SyncStatus}, + LOG_TARGET, }; use codec::{Decode, Encode}; use futures::channel::oneshot; @@ -39,9 +40,6 @@ use sp_runtime::{ }; use std::{collections::HashMap, fmt, sync::Arc}; -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - /// Number of peers that need to be connected before warp sync is started. const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; diff --git a/substrate/client/network/sync/src/warp_request_handler.rs b/substrate/client/network/sync/src/warp_request_handler.rs index b23f30c50dd24..9f3af24743b3f 100644 --- a/substrate/client/network/sync/src/warp_request_handler.rs +++ b/substrate/client/network/sync/src/warp_request_handler.rs @@ -20,7 +20,10 @@ use codec::Decode; use futures::{channel::oneshot, stream::StreamExt}; use log::debug; -use crate::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}; +use crate::{ + warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}, + LOG_TARGET, +}; use sc_network::{ config::ProtocolId, request_responses::{ @@ -120,10 +123,10 @@ impl RequestHandler { match self.handle_request(payload, pending_response) { Ok(()) => { - debug!(target: "sync", "Handled grandpa warp sync request from {}.", peer) + debug!(target: LOG_TARGET, "Handled grandpa warp sync request from {}.", peer) }, Err(e) => debug!( - target: "sync", + target: LOG_TARGET, "Failed to handle grandpa warp sync request from {}: {}", peer, e, ), From 8e704e122ef9302acad0afc79f8156fa1cf1bf9c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 29 Nov 2023 17:40:21 +0200 Subject: [PATCH 20/54] Fix tests compilation --- .../network/sync/src/chain_sync/test.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/chain_sync/test.rs index 1a4d2b032f154..90f73f73ad923 100644 --- a/substrate/client/network/sync/src/chain_sync/test.rs +++ b/substrate/client/network/sync/src/chain_sync/test.rs @@ -53,7 +53,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { }; // add a new peer with the same best block - sync.new_peer(peer_id, a1_hash, a1_number); + sync.add_peer(peer_id, a1_hash, a1_number); // and request a justification for the block sync.request_justification(&a1_hash, a1_number); @@ -117,8 +117,8 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { let (b1_hash, b1_number) = new_blocks(50); // add 2 peers at blocks that we don't have locally - sync.new_peer(peer_id1, Hash::random(), 42); - sync.new_peer(peer_id2, Hash::random(), 10); + sync.add_peer(peer_id1, Hash::random(), 42); + sync.add_peer(peer_id2, Hash::random(), 10); // we wil send block requests to these peers // for these blocks we don't know about @@ -128,7 +128,7 @@ fn restart_doesnt_affect_peers_downloading_finality_data() { .all(|(p, _)| { p == peer_id1 || p == peer_id2 })); // add a new peer at a known block - sync.new_peer(peer_id3, b1_hash, b1_number); + sync.add_peer(peer_id3, b1_hash, b1_number); // we request a justification for a block we have locally sync.request_justification(&b1_hash, b1_number); @@ -283,8 +283,8 @@ fn do_ancestor_search_when_common_block_to_best_qeued_gap_is_to_big() { let best_block = blocks.last().unwrap().clone(); let max_blocks_to_request = sync.max_blocks_per_request; // Connect the node we will sync from - sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()); - sync.new_peer(peer_id2, info.best_hash, 0); + sync.add_peer(peer_id1, best_block.hash(), *best_block.header().number()); + sync.add_peer(peer_id2, info.best_hash, 0); let mut best_block_num = 0; while best_block_num < MAX_DOWNLOAD_AHEAD { @@ -432,7 +432,7 @@ fn can_sync_huge_fork() { let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()); + sync.add_peer(peer_id1, common_block.hash(), *common_block.header().number()); send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); @@ -565,7 +565,7 @@ fn syncs_fork_without_duplicate_requests() { let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()); + sync.add_peer(peer_id1, common_block.hash(), *common_block.header().number()); send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); @@ -694,7 +694,7 @@ fn removes_target_fork_on_disconnect() { let peer_id1 = PeerId::random(); let common_block = blocks[1].clone(); // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()); + sync.add_peer(peer_id1, common_block.hash(), *common_block.header().number()); // Create a "new" header and announce it let mut header = blocks[0].header().clone(); @@ -702,7 +702,7 @@ fn removes_target_fork_on_disconnect() { send_block_announce(header, peer_id1, &mut sync); assert!(sync.fork_targets.len() == 1); - let _ = sync.peer_disconnected(&peer_id1); + let _ = sync.remove_peer(&peer_id1); assert!(sync.fork_targets.len() == 0); } @@ -718,7 +718,7 @@ fn can_import_response_with_missing_blocks() { let peer_id1 = PeerId::random(); let best_block = blocks[3].clone(); - sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()); + sync.add_peer(peer_id1, best_block.hash(), *best_block.header().number()); sync.peers.get_mut(&peer_id1).unwrap().state = PeerSyncState::Available; sync.peers.get_mut(&peer_id1).unwrap().common_number = 0; @@ -769,7 +769,7 @@ fn sync_restart_removes_block_but_not_justification_requests() { let (b1_hash, b1_number) = new_blocks(50); // add new peer and request blocks from them - sync.new_peer(peers[0], Hash::random(), 42); + sync.add_peer(peers[0], Hash::random(), 42); // we don't actually perform any requests, just keep track of peers waiting for a response let mut pending_responses = HashSet::new(); @@ -782,7 +782,7 @@ fn sync_restart_removes_block_but_not_justification_requests() { } // add a new peer at a known block - sync.new_peer(peers[1], b1_hash, b1_number); + sync.add_peer(peers[1], b1_hash, b1_number); // we request a justification for a block we have locally sync.request_justification(&b1_hash, b1_number); @@ -837,7 +837,7 @@ fn sync_restart_removes_block_but_not_justification_requests() { sync.peers.get(&peers[1]).unwrap().state, PeerSyncState::DownloadingJustification(b1_hash), ); - let _ = sync.peer_disconnected(&peers[1]); + let _ = sync.remove_peer(&peers[1]); pending_responses.remove(&peers[1]); assert_eq!(pending_responses.len(), 0); } @@ -892,9 +892,9 @@ fn request_across_forks() { // Add the peers, all at the common ancestor 100. let common_block = blocks.last().unwrap(); let peer_id1 = PeerId::random(); - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()); + sync.add_peer(peer_id1, common_block.hash(), *common_block.header().number()); let peer_id2 = PeerId::random(); - sync.new_peer(peer_id2, common_block.hash(), *common_block.header().number()); + sync.add_peer(peer_id2, common_block.hash(), *common_block.header().number()); // Peer 1 announces 107 from fork 1, 100-107 get downloaded. { From 37c79aeaf5d2c610c06fd26c58d985e2e6bde3f9 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 30 Nov 2023 14:41:44 +0200 Subject: [PATCH 21/54] Fix best block update in `ChainSync::on_validated_block_announce` --- substrate/client/network/sync/src/chain_sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index cab2669778671..43fa22a3a54ff 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -1020,7 +1020,7 @@ where return None } - let peer_info = is_best.then_some({ + let peer_info = is_best.then(|| { // update their best block peer.best_number = number; peer.best_hash = hash; From 8bd7e39b26f4ba7648cda4f009886260b58f3c82 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 30 Nov 2023 17:17:54 +0200 Subject: [PATCH 22/54] Fix zombienet warp sync tests --- .../zombienet/0001-basic-warp-sync/test-warp-sync.zndsl | 4 +++- .../test-validators-warp-sync.zndsl | 7 +++++-- .../test-block-building-warp-sync.zndsl | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl b/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl index dc84804b70b02..3d0e3440ad6c0 100644 --- a/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl +++ b/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl @@ -22,7 +22,9 @@ alice: reports block height is at least 12133 within 60 seconds bob: reports block height is at least 12133 within 60 seconds charlie: reports block height is at least 12133 within 60 seconds -dave: log line matches "Warp sync is complete" within 60 seconds +dave: log line matches "Warp sync finished" within 60 seconds +# State sync is logically part of warp sync +dave: log line matches "State sync finished" within 60 seconds # workaround for: https://github.com/paritytech/zombienet/issues/580 dave: count of log lines containing "Block history download is complete" is 1 within 10 seconds diff --git a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl index 05c458fbf4b79..dcd5d3fd93e77 100644 --- a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl +++ b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl @@ -19,8 +19,11 @@ charlie: reports block height is at least 12133 within 60 seconds dave: reports block height is at least 12133 within 60 seconds eve: reports block height is at least 12133 within 60 seconds -alice: log line matches "Warp sync is complete" within 60 seconds -bob: log line matches "Warp sync is complete" within 60 seconds +alice: log line matches "Warp sync finished" within 60 seconds +bob: log line matches "Warp sync finished" within 60 seconds +# State sync is logically part of warp sync +alice: log line matches "State sync finished" within 60 seconds +bob: log line matches "State sync finished" within 60 seconds # workaround for: https://github.com/paritytech/zombienet/issues/580 alice: count of log lines containing "Block history download is complete" is 1 within 60 seconds diff --git a/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl b/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl index a4ba46017a3f7..8405ed5b993bc 100644 --- a/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl +++ b/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl @@ -22,7 +22,9 @@ alice: reports block height is at least 12133 within 60 seconds bob: reports block height is at least 12133 within 60 seconds charlie: reports block height is at least 12133 within 60 seconds -dave: log line matches "Warp sync is complete" within 60 seconds +dave: log line matches "Warp sync finished" within 60 seconds +# State sync is logically part of warp sync +dave: log line matches "State sync finished" within 60 seconds # workaround for: https://github.com/paritytech/zombienet/issues/580 dave: count of log lines containing "Block history download is complete" is 1 within 10 seconds From de33eca313e330322390926749a7b263f173fc1f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 30 Nov 2023 18:20:38 +0200 Subject: [PATCH 23/54] Move `ChainSync` metrics reporting to `ChainSync` --- .../client/network/sync/src/chain_sync.rs | 84 +++++++++++++++++-- .../network/sync/src/chain_sync/test.rs | 18 ++-- substrate/client/network/sync/src/engine.rs | 62 +++----------- substrate/client/network/sync/src/strategy.rs | 20 ++++- substrate/client/network/sync/src/types.rs | 7 -- 5 files changed, 114 insertions(+), 77 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index 43fa22a3a54ff..c49d98b4286d1 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -33,7 +33,7 @@ use crate::{ extra_requests::ExtraRequests, schema::v1::StateResponse, state::{ImportResult, StateSync}, - types::{BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, + types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, warp::{WarpSyncPhase, WarpSyncProgress}, LOG_TARGET, }; @@ -41,7 +41,7 @@ use crate::{ use codec::Encode; use libp2p::PeerId; use log::{debug, error, info, trace, warn}; - +use prometheus_endpoint::{register, Gauge, GaugeVec, Opts, PrometheusError, Registry, U64}; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::message::{ @@ -122,6 +122,38 @@ mod rep { pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); } +struct Metrics { + queued_blocks: Gauge, + fork_targets: Gauge, + justifications: GaugeVec, +} + +impl Metrics { + fn register(r: &Registry) -> Result { + Ok(Self { + queued_blocks: { + let g = + Gauge::new("substrate_sync_queued_blocks", "Number of blocks in import queue")?; + register(g, r)? + }, + fork_targets: { + let g = Gauge::new("substrate_sync_fork_targets", "Number of fork sync targets")?; + register(g, r)? + }, + justifications: { + let g = GaugeVec::new( + Opts::new( + "substrate_sync_extra_justifications", + "Number of extra justifications requests", + ), + &["status"], + )?; + register(g, r)? + }, + }) + } +} + enum AllowedRequests { Some(HashSet), All, @@ -248,6 +280,8 @@ pub struct ChainSync { gap_sync: Option>, /// Pending actions. actions: Vec>, + /// Prometheus metrics. + metrics: Option, } /// All the data we have about a Peer that we are trying to sync with @@ -336,6 +370,7 @@ where client: Arc, max_parallel_downloads: u32, max_blocks_per_request: u32, + metrics_registry: Option, ) -> Result { let mut sync = Self { client, @@ -355,6 +390,18 @@ where import_existing: false, gap_sync: None, actions: Vec::new(), + metrics: metrics_registry + .map(|r| match Metrics::register(&r) { + Ok(metrics) => Some(metrics), + Err(err) => { + log::error!( + target: LOG_TARGET, + "Failed to register `ChainSync` metrics {err:?}", + ); + None + }, + }) + .flatten(), }; sync.reset_sync_start_point()?; @@ -1104,12 +1151,33 @@ where } } - /// Get prometheus metrics. - pub fn metrics(&self) -> Metrics { - Metrics { - queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), - fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), - justifications: self.extra_justifications.metrics(), + /// Report prometheus metrics. + pub fn report_metrics(&self) { + if let Some(metrics) = &self.metrics { + metrics + .fork_targets + .set(self.fork_targets.len().try_into().unwrap_or(std::u64::MAX)); + metrics + .queued_blocks + .set(self.queue_blocks.len().try_into().unwrap_or(std::u64::MAX)); + + let justifications_metrics = self.extra_justifications.metrics(); + metrics + .justifications + .with_label_values(&["pending"]) + .set(justifications_metrics.pending_requests.into()); + metrics + .justifications + .with_label_values(&["active"]) + .set(justifications_metrics.active_requests.into()); + metrics + .justifications + .with_label_values(&["failed"]) + .set(justifications_metrics.failed_requests.into()); + metrics + .justifications + .with_label_values(&["importing"]) + .set(justifications_metrics.importing_requests.into()); } } diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/chain_sync/test.rs index 90f73f73ad923..c89096bc6c904 100644 --- a/substrate/client/network/sync/src/chain_sync/test.rs +++ b/substrate/client/network/sync/src/chain_sync/test.rs @@ -38,7 +38,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { let client = Arc::new(TestClientBuilder::new().build()); let peer_id = PeerId::random(); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64, None).unwrap(); let (a1_hash, a1_number) = { let a1 = BlockBuilderBuilder::new(&*client) @@ -91,7 +91,7 @@ fn processes_empty_response_on_justification_request_for_unknown_block() { fn restart_doesnt_affect_peers_downloading_finality_data() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64, None).unwrap(); let peer_id1 = PeerId::random(); let peer_id2 = PeerId::random(); @@ -275,7 +275,7 @@ fn do_ancestor_search_when_common_block_to_best_qeued_gap_is_to_big() { let mut client = Arc::new(TestClientBuilder::new().build()); let info = client.info(); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64, None).unwrap(); let peer_id1 = PeerId::random(); let peer_id2 = PeerId::random(); @@ -421,7 +421,7 @@ fn can_sync_huge_fork() { let info = client.info(); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64, None).unwrap(); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); let just = (*b"TEST", Vec::new()); @@ -554,7 +554,7 @@ fn syncs_fork_without_duplicate_requests() { let info = client.info(); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64, None).unwrap(); let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); let just = (*b"TEST", Vec::new()); @@ -689,7 +689,7 @@ fn removes_target_fork_on_disconnect() { let mut client = Arc::new(TestClientBuilder::new().build()); let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64, None).unwrap(); let peer_id1 = PeerId::random(); let common_block = blocks[1].clone(); @@ -714,7 +714,7 @@ fn can_import_response_with_missing_blocks() { let empty_client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(ChainSyncMode::Full, empty_client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, empty_client.clone(), 1, 64, None).unwrap(); let peer_id1 = PeerId::random(); let best_block = blocks[3].clone(); @@ -745,7 +745,7 @@ fn ancestor_search_repeat() { #[test] fn sync_restart_removes_block_but_not_justification_requests() { let mut client = Arc::new(TestClientBuilder::new().build()); - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 1, 64, None).unwrap(); let peers = vec![PeerId::random(), PeerId::random()]; @@ -887,7 +887,7 @@ fn request_across_forks() { fork_blocks }; - let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64).unwrap(); + let mut sync = ChainSync::new(ChainSyncMode::Full, client.clone(), 5, 64, None).unwrap(); // Add the peers, all at the common ancestor 100. let common_block = blocks.last().unwrap(); diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index aec28fe4bb897..883a637698f63 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -48,8 +48,7 @@ use futures::{ use libp2p::{request_response::OutboundFailure, PeerId}; use log::{debug, error, trace}; use prometheus_endpoint::{ - register, Counter, Gauge, GaugeVec, MetricSource, Opts, PrometheusError, Registry, - SourcedGauge, U64, + register, Counter, Gauge, MetricSource, Opts, PrometheusError, Registry, SourcedGauge, U64, }; use prost::Message; use schnellru::{ByLength, LruMap}; @@ -138,9 +137,6 @@ mod rep { struct Metrics { peers: Gauge, - queued_blocks: Gauge, - fork_targets: Gauge, - justifications: GaugeVec, import_queue_blocks_submitted: Counter, import_queue_justifications_submitted: Counter, } @@ -153,25 +149,6 @@ impl Metrics { let g = Gauge::new("substrate_sync_peers", "Number of peers we sync with")?; register(g, r)? }, - queued_blocks: { - let g = - Gauge::new("substrate_sync_queued_blocks", "Number of blocks in import queue")?; - register(g, r)? - }, - fork_targets: { - let g = Gauge::new("substrate_sync_fork_targets", "Number of fork sync targets")?; - register(g, r)? - }, - justifications: { - let g = GaugeVec::new( - Opts::new( - "substrate_sync_extra_justifications", - "Number of extra justifications requests", - ), - &["status"], - )?; - register(g, r)? - }, import_queue_blocks_submitted: { let c = Counter::new( "substrate_sync_import_queue_blocks_submitted", @@ -381,7 +358,12 @@ where } else { net_config.network_config.max_blocks_per_request }; - let syncing_config = SyncingConfig { mode, max_parallel_downloads, max_blocks_per_request }; + let syncing_config = SyncingConfig { + mode, + max_parallel_downloads, + max_blocks_per_request, + metrics_registry: metrics_registry.cloned(), + }; let cache_capacity = (net_config.network_config.default_peers_set.in_peers + net_config.network_config.default_peers_set.out_peers) .max(1); @@ -457,7 +439,8 @@ where .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); // Initialize syncing strategy. - let strategy = SyncingStrategy::new(&syncing_config, client.clone(), warp_sync_config)?; + let strategy = + SyncingStrategy::new(syncing_config.clone(), client.clone(), warp_sync_config)?; let block_announce_protocol_name = block_announce_config.protocol_name().clone(); let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); @@ -540,31 +523,8 @@ where if let Some(metrics) = &self.metrics { let n = u64::try_from(self.peers.len()).unwrap_or(std::u64::MAX); metrics.peers.set(n); - - if let SyncingStrategy::ChainSyncStrategy(chain_sync) = &self.strategy { - let m = chain_sync.metrics(); - - metrics.fork_targets.set(m.fork_targets.into()); - metrics.queued_blocks.set(m.queued_blocks.into()); - - metrics - .justifications - .with_label_values(&["pending"]) - .set(m.justifications.pending_requests.into()); - metrics - .justifications - .with_label_values(&["active"]) - .set(m.justifications.active_requests.into()); - metrics - .justifications - .with_label_values(&["failed"]) - .set(m.justifications.failed_requests.into()); - metrics - .justifications - .with_label_values(&["importing"]) - .set(m.justifications.importing_requests.into()); - } } + self.strategy.report_metrics(); } fn update_peer_info( @@ -793,7 +753,7 @@ where )) }); self.strategy.switch_to_next( - &self.syncing_config, + self.syncing_config.clone(), self.client.clone(), connected_peers, ); diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 3c9db740a9ea7..ae71f79d30bfc 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -28,6 +28,7 @@ use crate::{ }; use libp2p::PeerId; use log::{error, info}; +use prometheus_endpoint::Registry; use sc_client_api::{BlockBackend, ProofProvider}; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::{ @@ -53,6 +54,7 @@ fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { } /// Syncing configuration containing data for all strategies. +#[derive(Clone, Debug)] pub struct SyncingConfig { /// Syncing mode. pub mode: SyncMode, @@ -60,6 +62,8 @@ pub struct SyncingConfig { pub max_parallel_downloads: u32, /// Maximum number of blocks to request. pub max_blocks_per_request: u32, + /// Prometheus metrics registry. + pub metrics_registry: Option, } #[derive(Debug)] @@ -107,7 +111,7 @@ where { /// Initialize a new syncing startegy. pub fn new( - config: &SyncingConfig, + config: SyncingConfig, client: Arc, warp_sync_config: Option>, ) -> Result { @@ -121,6 +125,7 @@ where client.clone(), config.max_parallel_downloads, config.max_blocks_per_request, + config.metrics_registry, )?)) } } @@ -329,6 +334,15 @@ where } } + /// Report Prometheus metrics + pub fn report_metrics(&self) { + match self { + SyncingStrategy::WarpSyncStrategy(_) => {}, + SyncingStrategy::StateSyncStrategy(_) => {}, + SyncingStrategy::ChainSyncStrategy(strategy) => strategy.report_metrics(), + } + } + /// Get actions that should be performed by the owner on the strategy's behalf #[must_use] pub fn actions(&mut self) -> Box>> { @@ -379,7 +393,7 @@ where pub fn switch_to_next( &mut self, - config: &SyncingConfig, + config: SyncingConfig, client: Arc, connected_peers: impl Iterator)>, ) { @@ -413,6 +427,7 @@ where client, config.max_parallel_downloads, config.max_blocks_per_request, + config.metrics_registry, ) { Ok(chain_sync) => chain_sync, Err(e) => { @@ -438,6 +453,7 @@ where client, config.max_parallel_downloads, config.max_blocks_per_request, + config.metrics_registry, ) { Ok(chain_sync) => chain_sync, Err(e) => { diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index 5931cf47b28a4..cbe26a73694cc 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -109,13 +109,6 @@ impl fmt::Display for BadPeer { impl std::error::Error for BadPeer {} -#[derive(Debug)] -pub struct Metrics { - pub queued_blocks: u32, - pub fork_targets: u32, - pub justifications: crate::request_metrics::Metrics, -} - #[derive(Debug)] pub enum PeerRequest { Block(BlockRequest), From 4d0d13c8c93bb015299e33db22d866463598c80d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 30 Nov 2023 19:36:42 +0200 Subject: [PATCH 24/54] Cleanup next peer selection --- .../client/network/sync/src/chain_sync.rs | 22 +++---- .../client/network/sync/src/state_strategy.rs | 56 +++++++--------- substrate/client/network/sync/src/warp.rs | 66 ++++++++----------- 3 files changed, 62 insertions(+), 82 deletions(-) diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs index c49d98b4286d1..330c92384b0f7 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -390,18 +390,16 @@ where import_existing: false, gap_sync: None, actions: Vec::new(), - metrics: metrics_registry - .map(|r| match Metrics::register(&r) { - Ok(metrics) => Some(metrics), - Err(err) => { - log::error!( - target: LOG_TARGET, - "Failed to register `ChainSync` metrics {err:?}", - ); - None - }, - }) - .flatten(), + metrics: metrics_registry.and_then(|r| match Metrics::register(&r) { + Ok(metrics) => Some(metrics), + Err(err) => { + log::error!( + target: LOG_TARGET, + "Failed to register `ChainSync` metrics {err:?}", + ); + None + }, + }), }; sync.reset_sync_start_point()?; diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index 4e9cba0fa777a..13973761f80a5 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -251,20 +251,36 @@ where return None } - let Some((peer_id, peer)) = - select_synced_available_peer(&mut self.peers, self.state_sync.target_block_num()) - else { - return None - }; - - peer.state = PeerState::DownloadingState; + let peer_id = self + .schedule_next_peer(PeerState::DownloadingState, self.state_sync.target_block_num())?; let request = self.state_sync.next_request(); trace!( target: LOG_TARGET, "New state request to {peer_id}: {request:?}.", ); + Some((peer_id, OpaqueStateRequest(Box::new(request)))) + } - Some((*peer_id, OpaqueStateRequest(Box::new(request)))) + fn schedule_next_peer( + &mut self, + new_state: PeerState, + min_best_number: NumberFor, + ) -> Option { + let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number); + // Find a random peer that is synced as much as peer majority and is above + // `min_best_number`. + for (peer_id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + peer.state = new_state; + return Some(*peer_id) + } + } + } + None } /// Returns the current sync status. @@ -301,27 +317,3 @@ where std::mem::take(&mut self.actions).into_iter() } } - -/// Get peer for state request. -/// -/// Due to borrowing issues this is a free-standing function accepting a reference to `peers`. -fn select_synced_available_peer( - peers: &mut HashMap>, - min_best_number: NumberFor, -) -> Option<(&PeerId, &mut Peer)> { - let mut targets: Vec<_> = peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - let threshold = std::cmp::max(median, min_best_number); - // Find a random peer that is synced as much as peer majority and is above - // `best_number_at_least`. - for (peer_id, peer) in peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= threshold { - return Some((peer_id, peer)) - } - } - } - - None -} diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 9b4c0f998e1c9..a8e771e4bb75c 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -456,6 +456,29 @@ where Ok(()) } + /// Reserve a peer for a request assigning `new_state`. + fn schedule_next_peer( + &mut self, + new_state: PeerState, + min_best_number: Option>, + ) -> Option { + let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); + // Find a random peer that is synced as much as peer majority and is above + // `min_best_number`. + for (peer_id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + peer.state = new_state; + return Some(*peer_id) + } + } + } + None + } + /// Produce warp proof request. fn warp_proof_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { let Phase::WarpProof { last_hash, .. } = &self.phase else { return None }; @@ -472,14 +495,10 @@ where return None } - let Some((peer_id, peer)) = select_synced_available_peer(&mut self.peers, None) else { - return None - }; - + let peer_id = self.schedule_next_peer(PeerState::DownloadingProofs, None)?; trace!(target: LOG_TARGET, "New WarpProofRequest to {peer_id}, begin hash: {begin}."); - peer.state = PeerState::DownloadingProofs; - Some((*peer_id, WarpProofRequest { begin })) + Some((peer_id, WarpProofRequest { begin })) } /// Produce target block request. @@ -499,11 +518,8 @@ where let target_hash = target_header.hash(); let target_number = *target_header.number(); - let Some((peer_id, peer)) = - select_synced_available_peer(&mut self.peers, Some(target_number)) - else { - return None - }; + let peer_id = + self.schedule_next_peer(PeerState::DownloadingTargetBlock, Some(target_number))?; trace!( target: LOG_TARGET, @@ -512,10 +528,8 @@ where target_number, ); - peer.state = PeerState::DownloadingTargetBlock; - Some(( - *peer_id, + peer_id, BlockRequest:: { id: 0, fields: BlockAttributes::HEADER | @@ -608,27 +622,3 @@ where self.result.take() } } - -/// Get candidate for warp/block request. -/// -/// Due to borrowing issues this is a free-standing function accepting a reference to `peers`. -fn select_synced_available_peer( - peers: &mut HashMap>, - min_best_number: Option>, -) -> Option<(&PeerId, &mut Peer)> { - let mut targets: Vec<_> = peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); - // Find a random peer that is synced as much as peer majority and is above - // `best_number_at_least`. - for (peer_id, peer) in peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= threshold { - return Some((peer_id, peer)) - } - } - } - - None -} From 5db9500019c6e9ed5d408146529f24fc265b8efa Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 1 Dec 2023 16:03:40 +0200 Subject: [PATCH 25/54] Simplify target block processing in `StateStrategy` --- .../client/network/sync/src/state_strategy.rs | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index 13973761f80a5..9256658b6594a 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -194,44 +194,46 @@ where ) { trace!(target: LOG_TARGET, "State sync: imported {imported} of {count}."); - let mut complete = false; - let mut success = false; - - for (result, hash) in results { - if hash == self.state_sync.target() { - complete = true; - success |= match result { - Ok(_) => true, - Err(e) => { - error!( - target: LOG_TARGET, - "Failed to import target block with state: {e:?}." - ); - false - }, + let results = results + .into_iter() + .filter_map(|(result, hash)| { + if hash == self.state_sync.target() { + Some(result) + } else { + debug!( + target: LOG_TARGET, + "Unexpected block processed: {hash} with result {result:?}.", + ); + None } - } else { - debug!( - target: LOG_TARGET, - "Unexpected block processed: {hash} with result {result:?}.", - ); - } - } + }) + .collect::>(); - if complete { - if success { - info!( - target: LOG_TARGET, - "State sync is complete ({} MiB), continuing with block sync.", - self.state_sync.progress().size / (1024 * 1024), - ); - } else { + if !results.is_empty() { + // We processed the target block + results.iter().filter_map(|result| result.as_ref().err()).for_each(|e| { error!( target: LOG_TARGET, - "State sync failed. Falling back to full sync.", + "Failed to import target block with state: {e:?}." ); + }); + match results.into_iter().any(|result| result.is_ok()) { + true => { + info!( + target: LOG_TARGET, + "State sync is complete ({} MiB), continuing with block sync.", + self.state_sync.progress().size / (1024 * 1024), + ); + }, + false => { + error!( + target: LOG_TARGET, + "State sync failed. Falling back to full sync.", + ); + // TODO: Test this scenario. + // Make sure fail-over doesn't render useless warp sync tests. + }, } - self.actions.push(StateStrategyAction::Finished); } } From e37c0008bd4d48df45884ca6bc499af2c4805ba9 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 1 Dec 2023 16:21:22 +0200 Subject: [PATCH 26/54] Shutdown greacefully in case of `ChainSync` initialization failure --- substrate/client/network/sync/src/engine.rs | 164 ++++++++++-------- substrate/client/network/sync/src/strategy.rs | 12 +- 2 files changed, 94 insertions(+), 82 deletions(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 883a637698f63..14b0e2ac8b2eb 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -672,93 +672,103 @@ where self.is_major_syncing.store(self.strategy.is_major_syncing(), Ordering::Relaxed); // Process actions requested by `ChainSync`. - self.process_strategy_actions(); + if let Err(e) = self.process_strategy_actions() { + error!("Terminating `SyncingEngine` due to fatal error: {e:?}"); + return + } } } - fn process_strategy_actions(&mut self) { - self.strategy.actions().for_each(|action| match action { - SyncingAction::SendBlockRequest { peer_id, request } => { - // Sending block request implies dropping obsolete pending response as we are not - // interested in it anymore (see [`SyncingAction::SendBlockRequest`]). - // Furthermore, only one request at a time is allowed to any peer. - let removed = self.pending_responses.remove(&peer_id); - self.send_block_request(peer_id, request.clone()); + fn process_strategy_actions(&mut self) -> Result<(), ClientError> { + for action in self.strategy.actions() { + match action { + SyncingAction::SendBlockRequest { peer_id, request } => { + // Sending block request implies dropping obsolete pending response as we are + // not interested in it anymore (see [`SyncingAction::SendBlockRequest`]). + // Furthermore, only one request at a time is allowed to any peer. + let removed = self.pending_responses.remove(&peer_id); + self.send_block_request(peer_id, request.clone()); + + trace!( + target: LOG_TARGET, + "Processed `ChainSyncAction::SendBlockRequest` to {} with {:?}, stale response removed: {}.", + peer_id, + request, + removed, + ) + }, + SyncingAction::CancelBlockRequest { peer_id } => { + let removed = self.pending_responses.remove(&peer_id); - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendBlockRequest` to {} with {:?}, stale response removed: {}.", - peer_id, - request, - removed, - ) - }, - SyncingAction::CancelBlockRequest { peer_id } => { - let removed = self.pending_responses.remove(&peer_id); + trace!( + target: LOG_TARGET, + "Processed {action:?}, response removed: {removed}.", + ); + }, + SyncingAction::SendStateRequest { peer_id, request } => { + self.send_state_request(peer_id, request); - trace!(target: LOG_TARGET, "Processed {action:?}, response removed: {removed}."); - }, - SyncingAction::SendStateRequest { peer_id, request } => { - self.send_state_request(peer_id, request); + trace!( + target: LOG_TARGET, + "Processed `ChainSyncAction::SendBlockRequest` to {peer_id}.", + ); + }, + SyncingAction::SendWarpProofRequest { peer_id, request } => { + self.send_warp_proof_request(peer_id, request.clone()); - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendBlockRequest` to {peer_id}.", - ); - }, - SyncingAction::SendWarpProofRequest { peer_id, request } => { - self.send_warp_proof_request(peer_id, request.clone()); + trace!( + target: LOG_TARGET, + "Processed `ChainSyncAction::SendWarpProofRequest` to {}, request: {:?}.", + peer_id, + request, + ); + }, + SyncingAction::DropPeer(BadPeer(peer_id, rep)) => { + self.pending_responses.remove(&peer_id); + self.network_service + .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); + self.network_service.report_peer(peer_id, rep); - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::SendWarpProofRequest` to {}, request: {:?}.", - peer_id, - request, - ); - }, - SyncingAction::DropPeer(BadPeer(peer_id, rep)) => { - self.pending_responses.remove(&peer_id); - self.network_service - .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); - self.network_service.report_peer(peer_id, rep); + trace!(target: LOG_TARGET, "Processed {action:?}."); + }, + SyncingAction::ImportBlocks { origin, blocks } => { + let count = blocks.len(); + self.import_blocks(origin, blocks); - trace!(target: LOG_TARGET, "Processed {action:?}."); - }, - SyncingAction::ImportBlocks { origin, blocks } => { - let count = blocks.len(); - self.import_blocks(origin, blocks); + trace!( + target: LOG_TARGET, + "Processed `ChainSyncAction::ImportBlocks` with {count} blocks.", + ); + }, + SyncingAction::ImportJustifications { peer_id, hash, number, justifications } => { + self.import_justifications(peer_id, hash, number, justifications); - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::ImportBlocks` with {count} blocks.", - ); - }, - SyncingAction::ImportJustifications { peer_id, hash, number, justifications } => { - self.import_justifications(peer_id, hash, number, justifications); + trace!( + target: LOG_TARGET, + "Processed `ChainSyncAction::ImportJustifications` from peer {} for block {} ({}).", + peer_id, + hash, + number, + ) + }, + SyncingAction::Finished => { + let connected_peers = self.peers.iter().filter_map(|(peer_id, peer)| { + peer.info.roles.is_full().then_some(( + *peer_id, + peer.info.best_hash, + peer.info.best_number, + )) + }); + self.strategy.switch_to_next( + self.syncing_config.clone(), + self.client.clone(), + connected_peers, + )?; + }, + } + } - trace!( - target: LOG_TARGET, - "Processed `ChainSyncAction::ImportJustifications` from peer {} for block {} ({}).", - peer_id, - hash, - number, - ) - }, - SyncingAction::Finished => { - let connected_peers = self.peers.iter().filter_map(|(peer_id, peer)| { - peer.info.roles.is_full().then_some(( - *peer_id, - peer.info.best_hash, - peer.info.best_number, - )) - }); - self.strategy.switch_to_next( - self.syncing_config.clone(), - self.client.clone(), - connected_peers, - ); - }, - }); + Ok(()) } fn perform_periodic_actions(&mut self) { diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index ae71f79d30bfc..8ef2400231da0 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -391,12 +391,13 @@ where } } + /// Switch to next strategy if the active one finished. pub fn switch_to_next( &mut self, config: SyncingConfig, client: Arc, connected_peers: impl Iterator)>, - ) { + ) -> Result<(), ClientError> { match self { Self::WarpSyncStrategy(warp_sync) => { match warp_sync.take_result() { @@ -431,8 +432,8 @@ where ) { Ok(chain_sync) => chain_sync, Err(e) => { - error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); - panic!("Failed to start `ChainSync` due to error: {e}."); + error!(target: LOG_TARGET, "Failed to start `ChainSync`."); + return Err(e) }, }; // Let `ChainSync` know about connected peers. @@ -457,8 +458,8 @@ where ) { Ok(chain_sync) => chain_sync, Err(e) => { - error!(target: LOG_TARGET, "Failed to start `ChainSync` due to error: {e}."); - panic!("Failed to start `ChainSync` due to error: {e}."); + error!(target: LOG_TARGET, "Failed to start `ChainSync`."); + return Err(e); }, }; // Let `ChainSync` know about connected peers. @@ -473,5 +474,6 @@ where debug_assert!(false); }, } + Ok(()) } } From 36db215cc4c76a68670b9378890532434fb368ef Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 1 Dec 2023 16:28:19 +0200 Subject: [PATCH 27/54] minor: comment --- substrate/client/network/sync/src/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 14b0e2ac8b2eb..722a8f9698070 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -671,7 +671,7 @@ where self.num_connected.store(self.peers.len(), Ordering::Relaxed); self.is_major_syncing.store(self.strategy.is_major_syncing(), Ordering::Relaxed); - // Process actions requested by `ChainSync`. + // Process actions requested by a syncing strategy. if let Err(e) = self.process_strategy_actions() { error!("Terminating `SyncingEngine` due to fatal error: {e:?}"); return From 7f5a4075e552860be3f29d229d2dccfa84f013ad Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 4 Dec 2023 14:00:12 +0200 Subject: [PATCH 28/54] minor: update comments re warp sync test --- substrate/client/network/sync/src/state_strategy.rs | 1 - substrate/client/network/test/src/sync.rs | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/state_strategy.rs index 9256658b6594a..8431b5c0fe740 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/state_strategy.rs @@ -231,7 +231,6 @@ where "State sync failed. Falling back to full sync.", ); // TODO: Test this scenario. - // Make sure fail-over doesn't render useless warp sync tests. }, } self.actions.push(StateStrategyAction::Finished); diff --git a/substrate/client/network/test/src/sync.rs b/substrate/client/network/test/src/sync.rs index f2be662ada164..873da4b5d9993 100644 --- a/substrate/client/network/test/src/sync.rs +++ b/substrate/client/network/test/src/sync.rs @@ -1234,10 +1234,12 @@ async fn warp_sync() { net.peer(2).push_blocks(64, false); // Wait for peer 1 to sync state. net.run_until_sync().await; + // Make sure it was not a full sync. assert!(!net.peer(3).client().has_state_at(&BlockId::Number(1))); + // Make sure warp sync was successful. assert!(net.peer(3).client().has_state_at(&BlockId::Number(64))); - // Wait for peer 1 download block history + // Wait for peer 3 to download block history (gap sync). futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); if net.peer(3).has_body(gap_end) && net.peer(3).has_body(target) { From 3ad88a4628dfc53e68c0774d0ea09c31d210ce70 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 5 Dec 2023 16:45:56 +0200 Subject: [PATCH 29/54] Add test for warp sync failover to full sync --- substrate/client/network/sync/src/warp.rs | 4 +-- substrate/client/network/test/src/lib.rs | 6 ++++- substrate/client/network/test/src/sync.rs | 31 ++++++++++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index a8e771e4bb75c..982d8d4875295 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -28,7 +28,7 @@ use crate::{ use codec::{Decode, Encode}; use futures::channel::oneshot; use libp2p::PeerId; -use log::{debug, error, trace, warn}; +use log::{debug, error, trace}; use sc_client_api::ProofProvider; use sc_network_common::sync::message::{ BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, @@ -255,7 +255,7 @@ where /// proofs, in this case we will continue polling until the target block is known. pub fn new(client: Arc, warp_sync_config: WarpSyncConfig) -> Self { if client.info().finalized_state.is_some() { - warn!( + error!( target: LOG_TARGET, "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." ); diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 71f13b74a5328..659dba28b17c2 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -699,6 +699,8 @@ pub struct FullPeerConfig { pub storage_chain: bool, /// Optional target block header to sync to pub target_block: Option<::Header>, + /// Force genesis even in case of warp & light state sync. + pub force_genesis: bool, } #[async_trait::async_trait] @@ -758,7 +760,9 @@ pub trait TestNetFactory: Default + Sized + Send { *genesis_extra_storage = storage; } - if matches!(config.sync_mode, SyncMode::LightState { .. } | SyncMode::Warp) { + if !config.force_genesis && + matches!(config.sync_mode, SyncMode::LightState { .. } | SyncMode::Warp) + { test_client_builder = test_client_builder.set_no_genesis(); } let backend = test_client_builder.backend(); diff --git a/substrate/client/network/test/src/sync.rs b/substrate/client/network/test/src/sync.rs index 873da4b5d9993..c025a8262f0e5 100644 --- a/substrate/client/network/test/src/sync.rs +++ b/substrate/client/network/test/src/sync.rs @@ -1232,7 +1232,7 @@ async fn warp_sync() { let target = net.peer(0).push_blocks(1, false).pop().unwrap(); net.peer(1).push_blocks(64, false); net.peer(2).push_blocks(64, false); - // Wait for peer 1 to sync state. + // Wait for peer 3 to sync state. net.run_until_sync().await; // Make sure it was not a full sync. assert!(!net.peer(3).client().has_state_at(&BlockId::Number(1))); @@ -1251,6 +1251,35 @@ async fn warp_sync() { .await; } +/// If there is a finalized state in the DB, warp sync falls back to full sync. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn warp_sync_failover_to_full_sync() { + sp_tracing::try_init_simple(); + let mut net = TestNet::new(0); + // Create 3 synced peers and 1 peer trying to warp sync. + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(FullPeerConfig { + sync_mode: SyncMode::Warp, + // We want some finalized state in the DB to make warp sync impossible. + force_genesis: true, + ..Default::default() + }); + net.peer(0).push_blocks(64, false); + net.peer(1).push_blocks(64, false); + net.peer(2).push_blocks(64, false); + // Even though we requested peer 3 to warp sync, it'll fall back to full sync if there is + // a finalized state in the DB. + assert!(net.peer(3).client().info().finalized_state.is_some()); + // Wait for peer 3 to sync. + net.run_until_sync().await; + // Make sure it was a full sync (peer 3 has state for all blocks). + (1..65) + .into_iter() + .for_each(|i| assert!(net.peer(3).client().has_state_at(&BlockId::Number(i as u64)))); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn warp_sync_to_target_block() { sp_tracing::try_init_simple(); From 55e53db5b0cabe65c28335c3cfec1fee13b0a28f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 5 Dec 2023 17:36:35 +0200 Subject: [PATCH 30/54] Rework directory structure --- polkadot/node/service/src/lib.rs | 2 +- substrate/bin/node/cli/src/service.rs | 2 +- substrate/client/consensus/grandpa/src/warp_proof.rs | 2 +- substrate/client/informant/src/display.rs | 5 +---- substrate/client/network/sync/src/engine.rs | 6 ++++-- substrate/client/network/sync/src/extra_requests.rs | 4 ++-- substrate/client/network/sync/src/lib.rs | 7 ++----- .../network/sync/src/{ => strategy}/chain_sync.rs | 6 ++++-- .../sync/src/{ => strategy}/chain_sync/test.rs | 0 .../network/sync/src/{strategy.rs => strategy/mod.rs} | 11 ++++++++--- .../sync/src/{state_strategy.rs => strategy/state.rs} | 2 +- .../sync/src/{state.rs => strategy/state_sync.rs} | 0 .../client/network/sync/src/{ => strategy}/warp.rs | 2 +- substrate/client/network/sync/src/types.rs | 2 +- .../client/network/sync/src/warp_request_handler.rs | 2 +- substrate/client/network/test/src/lib.rs | 2 +- substrate/client/service/src/builder.rs | 4 ++-- substrate/client/service/src/lib.rs | 2 +- 18 files changed, 32 insertions(+), 29 deletions(-) rename substrate/client/network/sync/src/{ => strategy}/chain_sync.rs (99%) rename substrate/client/network/sync/src/{ => strategy}/chain_sync/test.rs (100%) rename substrate/client/network/sync/src/{strategy.rs => strategy/mod.rs} (98%) rename substrate/client/network/sync/src/{state_strategy.rs => strategy/state.rs} (99%) rename substrate/client/network/sync/src/{state.rs => strategy/state_sync.rs} (100%) rename substrate/client/network/sync/src/{ => strategy}/warp.rs (99%) diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 70159301fc412..629f8dae63c28 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -731,7 +731,7 @@ pub fn new_full( }: NewFullParams, ) -> Result { use polkadot_node_network_protocol::request_response::IncomingRequest; - use sc_network_sync::warp::WarpSyncParams; + use sc_network_sync::WarpSyncParams; let is_offchain_indexing_enabled = config.offchain_worker.indexing_enabled; let role = config.role.clone(); diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index 4f8c6198cdce7..cbedcdf67bd22 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -30,7 +30,7 @@ use node_primitives::Block; use sc_client_api::{Backend, BlockBackend}; use sc_consensus_babe::{self, SlotProportion}; use sc_network::{event::Event, NetworkEventStream, NetworkService}; -use sc_network_sync::{warp::WarpSyncParams, SyncingService}; +use sc_network_sync::{strategy::warp::WarpSyncParams, SyncingService}; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; use sc_statement_store::Store as StatementStore; use sc_telemetry::{Telemetry, TelemetryWorker}; diff --git a/substrate/client/consensus/grandpa/src/warp_proof.rs b/substrate/client/consensus/grandpa/src/warp_proof.rs index a0fae6998f5a7..29111712ec382 100644 --- a/substrate/client/consensus/grandpa/src/warp_proof.rs +++ b/substrate/client/consensus/grandpa/src/warp_proof.rs @@ -23,7 +23,7 @@ use crate::{ BlockNumberOps, GrandpaJustification, SharedAuthoritySet, }; use sc_client_api::Backend as ClientBackend; -use sc_network_sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider}; +use sc_network_sync::strategy::warp::{EncodedProof, VerificationResult, WarpSyncProvider}; use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend}; use sp_consensus_grandpa::{AuthorityList, SetId, GRANDPA_ENGINE_ID}; use sp_runtime::{ diff --git a/substrate/client/informant/src/display.rs b/substrate/client/informant/src/display.rs index 64ddb71d572e8..e72fd381b4097 100644 --- a/substrate/client/informant/src/display.rs +++ b/substrate/client/informant/src/display.rs @@ -21,10 +21,7 @@ use ansi_term::Colour; use log::info; use sc_client_api::ClientInfo; use sc_network::NetworkStatus; -use sc_network_sync::{ - warp::{WarpSyncPhase, WarpSyncProgress}, - SyncState, SyncStatus, -}; +use sc_network_sync::{SyncState, SyncStatus, WarpSyncPhase, WarpSyncProgress}; use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Saturating, Zero}; use std::{fmt, time::Instant}; diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 722a8f9698070..3a313aa00734a 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -31,11 +31,13 @@ use crate::{ self, syncing_service::{SyncingService, ToServiceCommand}, }, - strategy::{SyncingAction, SyncingConfig, SyncingStrategy}, + strategy::{ + warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, + SyncingAction, SyncingConfig, SyncingStrategy, + }, types::{ BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, }, - warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, LOG_TARGET, }; diff --git a/substrate/client/network/sync/src/extra_requests.rs b/substrate/client/network/sync/src/extra_requests.rs index 296a48635a049..cd3008d270b1f 100644 --- a/substrate/client/network/sync/src/extra_requests.rs +++ b/substrate/client/network/sync/src/extra_requests.rs @@ -17,8 +17,8 @@ // along with this program. If not, see . use crate::{ - chain_sync::{PeerSync, PeerSyncState}, request_metrics::Metrics, + strategy::chain_sync::{PeerSync, PeerSyncState}, LOG_TARGET, }; use fork_tree::ForkTree; @@ -346,7 +346,7 @@ impl<'a, B: BlockT> Matcher<'a, B> { #[cfg(test)] mod tests { use super::*; - use crate::chain_sync::PeerSync; + use crate::strategy::chain_sync::PeerSync; use quickcheck::{Arbitrary, Gen, QuickCheck}; use sp_blockchain::Error as ClientError; use sp_test_primitives::{Block, BlockNumber, Hash}; diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index cc362636b5d16..494e3b87aa955 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -19,17 +19,15 @@ //! Blockchain syncing implementation in Substrate. pub use service::syncing_service::SyncingService; +pub use strategy::warp::{WarpSyncParams, WarpSyncPhase, WarpSyncProgress}; pub use types::{SyncEvent, SyncEventStream, SyncState, SyncStatus, SyncStatusProvider}; mod block_announce_validator; -mod chain_sync; mod extra_requests; mod futures_stream; mod pending_responses; mod request_metrics; mod schema; -mod state_strategy; -mod strategy; mod types; pub mod block_relay_protocol; @@ -38,9 +36,8 @@ pub mod blocks; pub mod engine; pub mod mock; pub mod service; -pub mod state; pub mod state_request_handler; -pub mod warp; +pub mod strategy; pub mod warp_request_handler; /// Log target for this crate. diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/strategy/chain_sync.rs similarity index 99% rename from substrate/client/network/sync/src/chain_sync.rs rename to substrate/client/network/sync/src/strategy/chain_sync.rs index 330c92384b0f7..e5ab9d411eb87 100644 --- a/substrate/client/network/sync/src/chain_sync.rs +++ b/substrate/client/network/sync/src/strategy/chain_sync.rs @@ -32,9 +32,11 @@ use crate::{ blocks::BlockCollection, extra_requests::ExtraRequests, schema::v1::StateResponse, - state::{ImportResult, StateSync}, + strategy::{ + state_sync::{ImportResult, StateSync}, + warp::{WarpSyncPhase, WarpSyncProgress}, + }, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, - warp::{WarpSyncPhase, WarpSyncProgress}, LOG_TARGET, }; diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/strategy/chain_sync/test.rs similarity index 100% rename from substrate/client/network/sync/src/chain_sync/test.rs rename to substrate/client/network/sync/src/strategy/chain_sync/test.rs diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy/mod.rs similarity index 98% rename from substrate/client/network/sync/src/strategy.rs rename to substrate/client/network/sync/src/strategy/mod.rs index 8ef2400231da0..a43eb819c2406 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy/mod.rs @@ -19,13 +19,16 @@ //! [`SyncingStrategy`] is a proxy between [`crate::engine::SyncingEngine`] //! and specific syncing algorithms. +pub mod chain_sync; +mod state; +mod state_sync; +pub mod warp; + use crate::{ - chain_sync::{ChainSync, ChainSyncAction, ChainSyncMode}, - state_strategy::{StateStrategy, StateStrategyAction}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncStatus}, - warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}, LOG_TARGET, }; +use chain_sync::{ChainSync, ChainSyncAction, ChainSyncMode}; use libp2p::PeerId; use log::{error, info}; use prometheus_endpoint::Registry; @@ -41,7 +44,9 @@ use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor}, Justifications, }; +use state::{StateStrategy, StateStrategyAction}; use std::sync::Arc; +use warp::{EncodedProof, WarpProofRequest, WarpSync, WarpSyncAction, WarpSyncConfig}; /// Corresponding `ChainSync` mode. fn chain_sync_mode(sync_mode: SyncMode) -> ChainSyncMode { diff --git a/substrate/client/network/sync/src/state_strategy.rs b/substrate/client/network/sync/src/strategy/state.rs similarity index 99% rename from substrate/client/network/sync/src/state_strategy.rs rename to substrate/client/network/sync/src/strategy/state.rs index 8431b5c0fe740..ea231c7a4c82a 100644 --- a/substrate/client/network/sync/src/state_strategy.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -20,7 +20,7 @@ use crate::{ schema::v1::StateResponse, - state::{ImportResult, StateSync}, + strategy::state_sync::{ImportResult, StateSync}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, LOG_TARGET, }; diff --git a/substrate/client/network/sync/src/state.rs b/substrate/client/network/sync/src/strategy/state_sync.rs similarity index 100% rename from substrate/client/network/sync/src/state.rs rename to substrate/client/network/sync/src/strategy/state_sync.rs diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs similarity index 99% rename from substrate/client/network/sync/src/warp.rs rename to substrate/client/network/sync/src/strategy/warp.rs index 982d8d4875295..2d0c3b5557faa 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -21,7 +21,7 @@ pub use sp_consensus_grandpa::{AuthorityList, SetId}; use crate::{ - chain_sync::validate_blocks, + strategy::chain_sync::validate_blocks, types::{BadPeer, SyncState, SyncStatus}, LOG_TARGET, }; diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index cbe26a73694cc..d4b087413780a 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -23,7 +23,7 @@ use sc_network_common::{role::Roles, types::ReputationChange}; use libp2p::PeerId; -use crate::warp::WarpSyncProgress; +use crate::strategy::warp::WarpSyncProgress; use sc_network_common::sync::message::BlockRequest; use sp_runtime::traits::{Block as BlockT, NumberFor}; diff --git a/substrate/client/network/sync/src/warp_request_handler.rs b/substrate/client/network/sync/src/warp_request_handler.rs index 9f3af24743b3f..39cf1c5d8067e 100644 --- a/substrate/client/network/sync/src/warp_request_handler.rs +++ b/substrate/client/network/sync/src/warp_request_handler.rs @@ -21,7 +21,7 @@ use futures::{channel::oneshot, stream::StreamExt}; use log::debug; use crate::{ - warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}, + strategy::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}, LOG_TARGET, }; use sc_network::{ diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 659dba28b17c2..aeed2985ace48 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -66,7 +66,7 @@ use sc_network_sync::{ block_request_handler::BlockRequestHandler, service::{network::NetworkServiceProvider, syncing_service::SyncingService}, state_request_handler::StateRequestHandler, - warp::{ + strategy::warp::{ AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncParams, WarpSyncProvider, }, warp_request_handler, diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index d078f44f198fa..f6273d2cd06c2 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -52,8 +52,8 @@ use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ block_relay_protocol::BlockRelayParams, block_request_handler::BlockRequestHandler, engine::SyncingEngine, service::network::NetworkServiceProvider, - state_request_handler::StateRequestHandler, warp::WarpSyncParams, - warp_request_handler::RequestHandler as WarpSyncRequestHandler, SyncingService, + state_request_handler::StateRequestHandler, + warp_request_handler::RequestHandler as WarpSyncRequestHandler, SyncingService, WarpSyncParams, }; use sc_rpc::{ author::AuthorApiServer, diff --git a/substrate/client/service/src/lib.rs b/substrate/client/service/src/lib.rs index ff9eb982b862f..9d62b7a6b3b99 100644 --- a/substrate/client/service/src/lib.rs +++ b/substrate/client/service/src/lib.rs @@ -76,7 +76,7 @@ pub use sc_chain_spec::{ pub use sc_consensus::ImportQueue; pub use sc_executor::NativeExecutionDispatch; -pub use sc_network_sync::warp::WarpSyncParams; +pub use sc_network_sync::WarpSyncParams; #[doc(hidden)] pub use sc_network_transactions::config::{TransactionImport, TransactionImportFuture}; pub use sc_rpc::{ From 17895cf7b1e625461bc53c0e9bdcbfaebd3759ec Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 6 Dec 2023 12:00:22 +0200 Subject: [PATCH 31/54] Remove unneded trait bound of `Client` in `WarpSync` --- substrate/client/network/sync/src/strategy/warp.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 2d0c3b5557faa..64ed7f985a779 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -29,7 +29,6 @@ use codec::{Decode, Encode}; use futures::channel::oneshot; use libp2p::PeerId; use log::{debug, error, trace}; -use sc_client_api::ProofProvider; use sc_network_common::sync::message::{ BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; @@ -248,7 +247,7 @@ pub struct WarpSync { impl WarpSync where B: BlockT, - Client: HeaderBackend + ProofProvider + 'static, + Client: HeaderBackend + 'static, { /// Create a new instance. When passing a warp sync provider we will be checking for proof and /// authorities. Alternatively we can pass a target block when we want to skip downloading @@ -321,12 +320,10 @@ where return } - let genesis_hash = - self.client.hash(Zero::zero()).unwrap().expect("Genesis hash always exists"); self.phase = Phase::WarpProof { set_id: 0, authorities: warp_sync_provider.current_authorities(), - last_hash: genesis_hash, + last_hash: self.client.info().genesis_hash, warp_sync_provider: Arc::clone(warp_sync_provider), }; trace!(target: LOG_TARGET, "Started warp sync with {} peers.", self.peers.len()); From 1795194eff0e7340fd980332ea7f58a72c363d21 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 6 Dec 2023 18:18:27 +0200 Subject: [PATCH 32/54] Introduce some warp sync tests --- substrate/client/network/sync/Cargo.toml | 2 +- .../client/network/sync/src/strategy/warp.rs | 132 ++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/substrate/client/network/sync/Cargo.toml b/substrate/client/network/sync/Cargo.toml index a9b8ec577e3f8..2999c7b2017f5 100644 --- a/substrate/client/network/sync/Cargo.toml +++ b/substrate/client/network/sync/Cargo.toml @@ -46,7 +46,7 @@ sp-consensus-grandpa = { path = "../../../primitives/consensus/grandpa" } sp-runtime = { path = "../../../primitives/runtime" } [dev-dependencies] -tokio = { version = "1.22.0", features = ["macros"] } +mockall = "0.11.3" quickcheck = { version = "1.0.3", default-features = false } sc-block-builder = { path = "../../block-builder" } sp-test-primitives = { path = "../../../primitives/test-primitives" } diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 64ed7f985a779..5af6c5602b972 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -619,3 +619,135 @@ where self.result.take() } } + +#[cfg(test)] +mod test { + use super::{EncodedProof, VerificationResult, WarpSync, WarpSyncAction, WarpSyncConfig}; + use sp_blockchain::{BlockStatus, Error as BlockchainError, HeaderBackend, Info}; + use sp_consensus_grandpa::{AuthorityList, SetId}; + use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; + use std::sync::Arc; + use substrate_test_runtime_client::runtime::{Block, Hash}; + + mockall::mock! { + pub Client {} + + impl HeaderBackend for Client { + fn header(&self, hash: B::Hash) -> Result, BlockchainError>; + fn info(&self) -> Info; + fn status(&self, hash: B::Hash) -> Result; + fn number( + &self, + hash: B::Hash, + ) -> Result::Header as HeaderT>::Number>, BlockchainError>; + fn hash(&self, number: NumberFor) -> Result, BlockchainError>; + } + } + + mockall::mock! { + pub WarpSyncProvider {} + + impl super::WarpSyncProvider for WarpSyncProvider { + fn generate( + &self, + start: B::Hash, + ) -> Result>; + fn verify( + &self, + proof: &EncodedProof, + set_id: SetId, + authorities: AuthorityList, + ) -> Result, Box>; + fn current_authorities(&self) -> AuthorityList; + } + } + + fn mock_client_with_state() -> MockClient { + let mut client = MockClient::::new(); + let genesis_hash = Hash::random(); + client.expect_info().return_once(move || Info { + best_hash: genesis_hash, + best_number: 0, + genesis_hash, + finalized_hash: genesis_hash, + finalized_number: 0, + // We need some finalized state to render warp sync impossible. + finalized_state: Some((genesis_hash, 0)), + number_leaves: 0, + block_gap: None, + }); + + client + } + + fn mock_client_without_state() -> MockClient { + let mut client = MockClient::::new(); + let genesis_hash = Hash::random(); + client.expect_info().return_once(move || Info { + best_hash: genesis_hash, + best_number: 0, + genesis_hash, + finalized_hash: genesis_hash, + finalized_number: 0, + finalized_state: None, + number_leaves: 0, + block_gap: None, + }); + + client + } + + #[test] + fn warp_sync_with_provider_for_db_with_finalized_state_is_noop() { + let client = mock_client_with_state(); + let provider = MockWarpSyncProvider::::new(); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Warp sync instantly finishes + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0], WarpSyncAction::Finished)); + + // ... with no result. + assert!(warp_sync.take_result().is_none()); + } + + #[test] + fn warp_sync_to_target_for_db_with_finalized_state_is_noop() { + let client = mock_client_with_state(); + let config = WarpSyncConfig::WaitForTarget; + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Warp sync instantly finishes + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0], WarpSyncAction::Finished)); + + // ... with no result. + assert!(warp_sync.take_result().is_none()); + } + + #[test] + fn warp_sync_with_provider_for_empty_db_doesnt_finish_instantly() { + let client = mock_client_without_state(); + let provider = MockWarpSyncProvider::::new(); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // No actions are emitted. + assert_eq!(warp_sync.actions().count(), 0) + } + + #[test] + fn warp_sync_to_target_for_empty_db_doesnt_finish_instantly() { + let client = mock_client_without_state(); + let config = WarpSyncConfig::WaitForTarget; + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // No actions are emitted. + assert_eq!(warp_sync.actions().count(), 0) + } + + // TODO +} From 9b164b81aac31c98125c80c6026ab408c4e76b72 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 7 Dec 2023 12:57:15 +0200 Subject: [PATCH 33/54] Add more warp sync startegy tests --- .../client/network/sync/src/strategy/warp.rs | 164 +++++++++++++++++- 1 file changed, 162 insertions(+), 2 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 5af6c5602b972..08c1ffe372271 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -622,7 +622,7 @@ where #[cfg(test)] mod test { - use super::{EncodedProof, VerificationResult, WarpSync, WarpSyncAction, WarpSyncConfig}; + use super::*; use sp_blockchain::{BlockStatus, Error as BlockchainError, HeaderBackend, Info}; use sp_consensus_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; @@ -683,7 +683,7 @@ mod test { fn mock_client_without_state() -> MockClient { let mut client = MockClient::::new(); let genesis_hash = Hash::random(); - client.expect_info().return_once(move || Info { + client.expect_info().returning(move || Info { best_hash: genesis_hash, best_number: 0, genesis_hash, @@ -749,5 +749,165 @@ mod test { assert_eq!(warp_sync.actions().count(), 0) } + #[test] + fn warp_sync_is_started_only_when_there_is_enough_peers() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Warp sync is not started when there is not enough peers. + for _ in 0..(MIN_PEERS_TO_START_WARP_SYNC - 1) { + warp_sync.add_peer(PeerId::random(), Hash::random(), 10); + assert!(matches!(warp_sync.phase, Phase::WaitingForPeers { .. })) + } + + // Now we have enough peers and warp sync is started. + warp_sync.add_peer(PeerId::random(), Hash::random(), 10); + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })) + } + + #[test] + fn no_peer_is_scheduled_if_no_peers_connected() { + let client = mock_client_without_state(); + let provider = MockWarpSyncProvider::::new(); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + assert!(warp_sync.schedule_next_peer(PeerState::DownloadingProofs, None).is_none()); + } + + #[test] + fn enough_peers_are_used_in_tests() { + // Tests below use 10 peers. Fail early if it's less than a threshold for warp sync. + assert!( + 10 >= MIN_PEERS_TO_START_WARP_SYNC, + "Tests must be updated to use that many initial peers.", + ); + } + + #[test] + fn at_least_median_synced_peer_is_scheduled() { + for _ in 0..100 { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + let peer_id = warp_sync.schedule_next_peer(PeerState::DownloadingProofs, None); + assert!(warp_sync.peers.get(&peer_id.unwrap()).unwrap().best_number >= 6); + } + } + + #[test] + fn min_best_number_peer_is_scheduled() { + for _ in 0..10 { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + let peer_id = warp_sync.schedule_next_peer(PeerState::DownloadingProofs, Some(10)); + assert!(warp_sync.peers.get(&peer_id.unwrap()).unwrap().best_number == 10); + } + } + + #[test] + fn no_warp_proof_request_if_another_phase() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set to another phase. + warp_sync.phase = Phase::PendingTargetBlock; + + // No request is made. + assert!(warp_sync.warp_proof_request().is_none()); + } + + #[test] + fn warp_proof_request_starts_at_last_hash() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + let known_last_hash = Hash::random(); + + // Manually set last hash to known value. + match &mut warp_sync.phase { + Phase::WarpProof { last_hash, .. } => { + *last_hash = known_last_hash; + }, + _ => panic!("Invalid phase."), + } + + let (_peer_id, request) = warp_sync.warp_proof_request().unwrap(); + assert_eq!(request.begin, known_last_hash); + } + + #[test] + fn no_parallel_warp_proof_requests() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make requests. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + // First request is made. + assert!(warp_sync.warp_proof_request().is_some()); + // Second request is not made. + assert!(warp_sync.warp_proof_request().is_none()); + } + // TODO } From 0bab8272cbe0f0845aa493cdedf76603625e160c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 7 Dec 2023 17:48:21 +0200 Subject: [PATCH 34/54] Add more tests --- .../client/network/sync/src/strategy/warp.rs | 147 +++++++++++++++++- 1 file changed, 141 insertions(+), 6 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 08c1ffe372271..d465149b387e0 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -623,11 +623,15 @@ where #[cfg(test)] mod test { use super::*; + use sc_block_builder::BlockBuilderBuilder; use sp_blockchain::{BlockStatus, Error as BlockchainError, HeaderBackend, Info}; use sp_consensus_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; - use std::sync::Arc; - use substrate_test_runtime_client::runtime::{Block, Hash}; + use std::{io::ErrorKind, sync::Arc}; + use substrate_test_runtime_client::{ + runtime::{Block, Hash}, + DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, + }; mockall::mock! { pub Client {} @@ -697,6 +701,18 @@ mod test { client } + fn single_action(warp_sync: &mut WarpSync) -> WarpSyncAction + where + B: BlockT, + Client: HeaderBackend + 'static, + { + let mut actions = warp_sync.actions().collect::>(); + if actions.len() != 1 { + panic!("Got {} actions (1 expected).", actions.len()); + } + actions.pop().unwrap() + } + #[test] fn warp_sync_with_provider_for_db_with_finalized_state_is_noop() { let client = mock_client_with_state(); @@ -720,9 +736,7 @@ mod test { let mut warp_sync = WarpSync::new(Arc::new(client), config); // Warp sync instantly finishes - let actions = warp_sync.actions().collect::>(); - assert_eq!(actions.len(), 1); - assert!(matches!(actions[0], WarpSyncAction::Finished)); + assert!(matches!(single_action(&mut warp_sync), WarpSyncAction::Finished)); // ... with no result. assert!(warp_sync.take_result().is_none()); @@ -833,7 +847,7 @@ mod test { } #[test] - fn no_warp_proof_request_if_another_phase() { + fn no_warp_proof_request_in_another_phase() { let client = mock_client_without_state(); let mut provider = MockWarpSyncProvider::::new(); provider @@ -909,5 +923,126 @@ mod test { assert!(warp_sync.warp_proof_request().is_none()); } + #[test] + fn bad_warp_proof_response_drops_peer() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + // Warp proof verification fails. + provider.expect_verify().return_once(|_proof, _set_id, _authorities| { + Err(Box::new(std::io::Error::new(ErrorKind::Other, "test-verification-failure"))) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + // Consume `SendWarpProofRequest` action. + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] + else { + panic!("Invalid action"); + }; + + warp_sync.on_warp_proof_response(&request_peer_id, EncodedProof(Vec::new())); + + // The first action is a "drop peer" action, the second is the next warp proof request + // we are not interested in. + let actions = warp_sync.actions().collect::>(); + assert!(matches!( + actions[0], + WarpSyncAction::DropPeer(BadPeer(peer_id, _rep)) if peer_id == request_peer_id + )); + } + + #[test] + fn partial_warp_proof_doesnt_advance_phase() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + // Warp proof is partial. + provider.expect_verify().return_once(|_proof, set_id, authorities| { + Ok(VerificationResult::Partial(set_id, authorities, Hash::random())) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + // Consume `SendWarpProofRequest` action. + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] + else { + panic!("Invalid action"); + }; + + warp_sync.on_warp_proof_response(&request_peer_id, EncodedProof(Vec::new())); + + assert!(warp_sync.actions.is_empty(), "No extra actions generated"); + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + } + + #[test] + fn complete_warp_proof_advances_phase() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + // Consume `SendWarpProofRequest` action. + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] + else { + panic!("Invalid action"); + }; + + warp_sync.on_warp_proof_response(&request_peer_id, EncodedProof(Vec::new())); + + assert!(warp_sync.actions.is_empty(), "No extra actions generated"); + assert!( + matches!(warp_sync.phase, Phase::TargetBlock(header) if header == *target_block.header()) + ); + } + // TODO } From dfa0168c6658b4f86a47120b9fbf3908c76e2e49 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 8 Dec 2023 19:02:03 +0200 Subject: [PATCH 35/54] Add more warp sync tests --- .../client/network/sync/src/strategy/warp.rs | 395 +++++++++++++++++- 1 file changed, 388 insertions(+), 7 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index d465149b387e0..2cc6163797712 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -630,7 +630,7 @@ mod test { use std::{io::ErrorKind, sync::Arc}; use substrate_test_runtime_client::{ runtime::{Block, Hash}, - DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, + BlockBuilderExt, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; mockall::mock! { @@ -954,13 +954,14 @@ mod test { warp_sync.on_warp_proof_response(&request_peer_id, EncodedProof(Vec::new())); - // The first action is a "drop peer" action, the second is the next warp proof request - // we are not interested in. - let actions = warp_sync.actions().collect::>(); + // We only interested in alredy generated actions, not new requests. + let actions = std::mem::take(&mut warp_sync.actions); + assert_eq!(actions.len(), 1); assert!(matches!( actions[0], WarpSyncAction::DropPeer(BadPeer(peer_id, _rep)) if peer_id == request_peer_id )); + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); } #[test] @@ -1033,16 +1034,396 @@ mod test { assert_eq!(actions.len(), 1); let WarpSyncAction::SendWarpProofRequest { peer_id: request_peer_id, .. } = actions[0] else { - panic!("Invalid action"); + panic!("Invalid action."); }; warp_sync.on_warp_proof_response(&request_peer_id, EncodedProof(Vec::new())); - assert!(warp_sync.actions.is_empty(), "No extra actions generated"); + assert!(warp_sync.actions.is_empty(), "No extra actions generated."); assert!( matches!(warp_sync.phase, Phase::TargetBlock(header) if header == *target_block.header()) ); } - // TODO + #[test] + fn no_target_block_requests_in_another_phase() { + let client = mock_client_without_state(); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(Arc::new(client), config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + // We are not in `Phase::TargetBlock` + assert!(matches!(warp_sync.phase, Phase::WarpProof { .. })); + + // No request is made. + assert!(warp_sync.target_block_request().is_none()); + } + + #[test] + fn target_block_request_is_correct() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + let (_peer_id, request) = warp_sync.target_block_request().unwrap(); + assert_eq!(request.from, FromBlock::Hash(target_block.header().hash())); + assert_eq!( + request.fields, + BlockAttributes::HEADER | BlockAttributes::BODY | BlockAttributes::JUSTIFICATION + ); + assert_eq!(request.max, Some(1)); + } + + #[test] + fn externally_set_target_block_is_requested() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + let config = WarpSyncConfig::WaitForTarget; + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // No actions generated so far. + assert_eq!(warp_sync.actions().count(), 0); + + warp_sync.set_target_block(target_header); + assert!(matches!(warp_sync.phase, Phase::TargetBlock(_))); + + let (_peer_id, request) = warp_sync.target_block_request().unwrap(); + assert_eq!(request.from, FromBlock::Hash(target_block.header().hash())); + assert_eq!( + request.fields, + BlockAttributes::HEADER | BlockAttributes::BODY | BlockAttributes::JUSTIFICATION + ); + assert_eq!(request.max, Some(1)); + } + + #[test] + fn no_parallel_target_block_requests() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + // First target block request is made. + assert!(warp_sync.target_block_request().is_some()); + // No parallel request is made. + assert!(warp_sync.target_block_request().is_none()); + } + + #[test] + fn target_block_response_with_no_blocks_drops_peer() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + let (peer_id, request) = warp_sync.target_block_request().unwrap(); + + // Empty block response received. + let response = Vec::new(); + // Peer is dropped. + assert!(matches!( + warp_sync.on_block_response_inner(peer_id, request, response), + Err(BadPeer(id, _rep)) if id == peer_id, + )); + } + + #[test] + fn target_block_response_with_extra_blocks_drops_peer() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + let mut extra_block_builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap(); + extra_block_builder + .push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])) + .unwrap(); + let extra_block = extra_block_builder.build().unwrap().block; + + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + let (peer_id, request) = warp_sync.target_block_request().unwrap(); + + // Block response with extra blocks received. + let response = vec![ + BlockData:: { + hash: target_block.header().hash(), + header: Some(target_block.header().clone()), + body: Some(target_block.extrinsics().iter().cloned().collect::>()), + indexed_body: None, + receipt: None, + message_queue: None, + justification: None, + justifications: None, + }, + BlockData:: { + hash: extra_block.header().hash(), + header: Some(extra_block.header().clone()), + body: Some(extra_block.extrinsics().iter().cloned().collect::>()), + indexed_body: None, + receipt: None, + message_queue: None, + justification: None, + justifications: None, + }, + ]; + // Peer is dropped. + assert!(matches!( + warp_sync.on_block_response_inner(peer_id, request, response), + Err(BadPeer(id, _rep)) if id == peer_id, + )); + } + + #[test] + fn target_block_response_with_wrong_block_drops_peer() { + sp_tracing::try_init_simple(); + + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + let mut wrong_block_builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap(); + wrong_block_builder + .push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])) + .unwrap(); + let wrong_block = wrong_block_builder.build().unwrap().block; + + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + let (peer_id, request) = warp_sync.target_block_request().unwrap(); + + // Wrong block received. + let response = vec![BlockData:: { + hash: wrong_block.header().hash(), + header: Some(wrong_block.header().clone()), + body: Some(wrong_block.extrinsics().iter().cloned().collect::>()), + indexed_body: None, + receipt: None, + message_queue: None, + justification: None, + justifications: None, + }]; + // Peer is dropped. + assert!(matches!( + warp_sync.on_block_response_inner(peer_id, request, response), + Err(BadPeer(id, _rep)) if id == peer_id, + )); + } + + #[test] + fn correct_target_block_response_sets_strategy_result() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut provider = MockWarpSyncProvider::::new(); + provider + .expect_current_authorities() + .once() + .return_const(AuthorityList::default()); + let mut target_block_builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap(); + target_block_builder + .push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])) + .unwrap(); + let target_block = target_block_builder.build().unwrap().block; + let target_header = target_block.header().clone(); + // Warp proof is complete. + provider.expect_verify().return_once(move |_proof, set_id, authorities| { + Ok(VerificationResult::Complete(set_id, authorities, target_header)) + }); + let config = WarpSyncConfig::WithProvider(Arc::new(provider)); + let mut warp_sync = WarpSync::new(client, config); + + // Make sure we have enough peers to make a request. + for best_number in 1..11 { + warp_sync.add_peer(PeerId::random(), Hash::random(), best_number); + } + + // Manually set `TargetBlock` phase. + warp_sync.phase = Phase::TargetBlock(target_block.header().clone()); + + let (peer_id, request) = warp_sync.target_block_request().unwrap(); + + // Correct block received. + let body = Some(target_block.extrinsics().iter().cloned().collect::>()); + let justifications = Some(Justifications::from((*b"FRNK", Vec::new()))); + let response = vec![BlockData:: { + hash: target_block.header().hash(), + header: Some(target_block.header().clone()), + body: body.clone(), + indexed_body: None, + receipt: None, + message_queue: None, + justification: None, + justifications: justifications.clone(), + }]; + + assert!(warp_sync.on_block_response_inner(peer_id, request, response).is_ok()); + + // Strategy finishes. + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0], WarpSyncAction::Finished)); + + // With correct result. + let result = warp_sync.take_result().unwrap(); + assert_eq!(result.target_header, *target_block.header()); + assert_eq!(result.target_body, body); + assert_eq!(result.target_justifications, justifications); + } } From 74e3179859fbf6cf0105a13801051530de37428f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 11 Dec 2023 17:14:25 +0200 Subject: [PATCH 36/54] Initial `StateStrategy` tests --- .../client/network/sync/src/strategy/state.rs | 98 ++++++++++++++++++- .../client/network/sync/src/strategy/warp.rs | 16 +-- 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index ea231c7a4c82a..8b72ee21c516f 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -230,7 +230,6 @@ where target: LOG_TARGET, "State sync failed. Falling back to full sync.", ); - // TODO: Test this scenario. }, } self.actions.push(StateStrategyAction::Finished); @@ -318,3 +317,100 @@ where std::mem::take(&mut self.actions).into_iter() } } + +#[cfg(test)] +mod test { + use super::*; + use sc_block_builder::BlockBuilderBuilder; + use sp_runtime::traits::Zero; + use substrate_test_runtime_client::{ + runtime::{Block, Hash}, + BlockBuilderExt, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, + }; + + #[test] + fn no_peer_is_scheduled_if_no_peers_connected() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + let target_header = target_block.header().clone(); + + let mut state_strategy = + StateStrategy::new(client, target_header, None, None, false, std::iter::empty()); + + assert!(state_strategy + .schedule_next_peer(PeerState::DownloadingState, Zero::zero()) + .is_none()); + } + + #[test] + fn at_least_median_synced_peer_is_scheduled() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + for _ in 0..100 { + let peers = (1..=10) + .map(|best_number| (PeerId::random(), best_number)) + .collect::>(); + let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + + let mut state_strategy = StateStrategy::new( + client.clone(), + target_block.header().clone(), + None, + None, + false, + initial_peers, + ); + + let peer_id = + state_strategy.schedule_next_peer(PeerState::DownloadingState, Zero::zero()); + assert!(*peers.get(&peer_id.unwrap()).unwrap() >= 6); + } + } + + #[test] + fn min_best_number_peer_is_scheduled() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + for _ in 0..10 { + let peers = (1..=10) + .map(|best_number| (PeerId::random(), best_number)) + .collect::>(); + let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + + let mut state_strategy = StateStrategy::new( + client.clone(), + target_block.header().clone(), + None, + None, + false, + initial_peers, + ); + + let peer_id = state_strategy.schedule_next_peer(PeerState::DownloadingState, 10); + assert!(*peers.get(&peer_id.unwrap()).unwrap() == 10); + } + } +} diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index 2cc6163797712..daa1e46928e19 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -701,18 +701,6 @@ mod test { client } - fn single_action(warp_sync: &mut WarpSync) -> WarpSyncAction - where - B: BlockT, - Client: HeaderBackend + 'static, - { - let mut actions = warp_sync.actions().collect::>(); - if actions.len() != 1 { - panic!("Got {} actions (1 expected).", actions.len()); - } - actions.pop().unwrap() - } - #[test] fn warp_sync_with_provider_for_db_with_finalized_state_is_noop() { let client = mock_client_with_state(); @@ -736,7 +724,9 @@ mod test { let mut warp_sync = WarpSync::new(Arc::new(client), config); // Warp sync instantly finishes - assert!(matches!(single_action(&mut warp_sync), WarpSyncAction::Finished)); + let actions = warp_sync.actions().collect::>(); + assert_eq!(actions.len(), 1); + assert!(matches!(actions[0], WarpSyncAction::Finished)); // ... with no result. assert!(warp_sync.take_result().is_none()); From edd28b210cbdcd872a3ac77659a9db36a63f1b94 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 11 Dec 2023 17:51:10 +0200 Subject: [PATCH 37/54] Add more `StateStrategy` tests --- .../client/network/sync/src/strategy/state.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 8b72ee21c516f..9c61417d1d792 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -321,6 +321,8 @@ where #[cfg(test)] mod test { use super::*; + use crate::schema::v1::StateRequest; + use codec::{Decode, Encode}; use sc_block_builder::BlockBuilderBuilder; use sp_runtime::traits::Zero; use substrate_test_runtime_client::{ @@ -413,4 +415,70 @@ mod test { assert!(*peers.get(&peer_id.unwrap()).unwrap() == 10); } } + + #[test] + fn state_request_contains_correct_hash() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + let peers = (1..=10) + .map(|best_number| (PeerId::random(), best_number)) + .collect::>(); + let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + + let mut state_strategy = StateStrategy::new( + client.clone(), + target_block.header().clone(), + None, + None, + false, + initial_peers, + ); + + let (_peer_id, mut opaque_request) = state_strategy.state_request().unwrap(); + let request: &mut StateRequest = opaque_request.0.downcast_mut().unwrap(); + let hash = Hash::decode(&mut &*request.block).unwrap(); + + assert_eq!(hash, target_block.header().hash()); + } + + #[test] + fn no_parallel_state_requests() { + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let target_block = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap() + .build() + .unwrap() + .block; + + let peers = (1..=10) + .map(|best_number| (PeerId::random(), best_number)) + .collect::>(); + let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + + let mut state_strategy = StateStrategy::new( + client.clone(), + target_block.header().clone(), + None, + None, + false, + initial_peers, + ); + + // First request is sent. + assert!(state_strategy.state_request().is_some()); + + // No parallel request is sent. + assert!(state_strategy.state_request().is_none()); + } } From 242f09f34dd244a43c2b65b93924c25dbc0c7c0f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 11 Dec 2023 17:56:16 +0200 Subject: [PATCH 38/54] minor: make clippy happy --- substrate/client/network/sync/src/strategy/state.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 9c61417d1d792..8780846a71486 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -367,7 +367,7 @@ mod test { let peers = (1..=10) .map(|best_number| (PeerId::random(), best_number)) .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); let mut state_strategy = StateStrategy::new( client.clone(), @@ -400,7 +400,7 @@ mod test { let peers = (1..=10) .map(|best_number| (PeerId::random(), best_number)) .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); let mut state_strategy = StateStrategy::new( client.clone(), @@ -431,7 +431,7 @@ mod test { let peers = (1..=10) .map(|best_number| (PeerId::random(), best_number)) .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); let mut state_strategy = StateStrategy::new( client.clone(), @@ -464,7 +464,7 @@ mod test { let peers = (1..=10) .map(|best_number| (PeerId::random(), best_number)) .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (p.clone(), n.clone())); + let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); let mut state_strategy = StateStrategy::new( client.clone(), From 465cd524e0b66adbaaa470784b8d063e742c63dc Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 12 Dec 2023 13:00:38 +0200 Subject: [PATCH 39/54] Introduce `StateSyncProvider` trait --- .../network/sync/src/strategy/chain_sync.rs | 6 +-- .../client/network/sync/src/strategy/mod.rs | 2 +- .../client/network/sync/src/strategy/state.rs | 53 +++++++++++------- .../network/sync/src/strategy/state_sync.rs | 54 +++++++++++++------ 4 files changed, 77 insertions(+), 38 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/chain_sync.rs b/substrate/client/network/sync/src/strategy/chain_sync.rs index e5ab9d411eb87..7441f006cab47 100644 --- a/substrate/client/network/sync/src/strategy/chain_sync.rs +++ b/substrate/client/network/sync/src/strategy/chain_sync.rs @@ -33,7 +33,7 @@ use crate::{ extra_requests::ExtraRequests, schema::v1::StateResponse, strategy::{ - state_sync::{ImportResult, StateSync}, + state_sync::{ImportResult, StateSync, StateSyncProvider}, warp::{WarpSyncPhase, WarpSyncProgress}, }, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, @@ -1650,7 +1650,7 @@ where } for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.common_number >= sync.target_block_num() { + if peer.state.is_available() && peer.common_number >= sync.target_number() { peer.state = PeerSyncState::DownloadingState; let request = sync.next_request(); trace!(target: LOG_TARGET, "New StateRequest for {}: {:?}", id, request); @@ -1787,7 +1787,7 @@ where self.update_peer_common_number(&peer, number); } let state_sync_complete = - self.state_sync.as_ref().map_or(false, |s| s.target() == hash); + self.state_sync.as_ref().map_or(false, |s| s.target_hash() == hash); if state_sync_complete { info!( target: LOG_TARGET, diff --git a/substrate/client/network/sync/src/strategy/mod.rs b/substrate/client/network/sync/src/strategy/mod.rs index a43eb819c2406..c1d4125e3aeef 100644 --- a/substrate/client/network/sync/src/strategy/mod.rs +++ b/substrate/client/network/sync/src/strategy/mod.rs @@ -99,7 +99,7 @@ pub enum SyncingAction { /// Proxy to specific syncing strategies. pub enum SyncingStrategy { WarpSyncStrategy(WarpSync), - StateSyncStrategy(StateStrategy), + StateSyncStrategy(StateStrategy), ChainSyncStrategy(ChainSync), } diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 8780846a71486..80ee3f159201b 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -20,7 +20,7 @@ use crate::{ schema::v1::StateResponse, - strategy::state_sync::{ImportResult, StateSync}, + strategy::state_sync::{ImportResult, StateSync, StateSyncProvider}, types::{BadPeer, OpaqueStateRequest, OpaqueStateResponse, SyncState, SyncStatus}, LOG_TARGET, }; @@ -74,44 +74,61 @@ struct Peer { } /// Syncing strategy that downloads and imports a recent state directly. -pub struct StateStrategy { - state_sync: StateSync, +pub struct StateStrategy { + state_sync: Box>, peers: HashMap>, actions: Vec>, } -impl StateStrategy -where - B: BlockT, - Client: ProofProvider + Send + Sync + 'static, -{ - // Create a new instance. - pub fn new( +impl StateStrategy { + /// Create a new instance. + pub fn new( client: Arc, target_header: B::Header, target_body: Option>, target_justifications: Option, skip_proof: bool, initial_peers: impl Iterator)>, - ) -> Self { + ) -> Self + where + Client: ProofProvider + Send + Sync + 'static, + { let peers = initial_peers .map(|(peer_id, best_number)| { (peer_id, Peer { best_number, state: PeerState::Available }) }) .collect(); Self { - state_sync: StateSync::new( + state_sync: Box::new(StateSync::new( client, target_header, target_body, target_justifications, skip_proof, - ), + )), peers, actions: Vec::new(), } } + // Create a new instance with a custom state sync provider. + // Used in tests. + #[cfg(test)] + fn new_with_provider( + state_sync_provider: Box>, + initial_peers: impl Iterator)>, + ) -> Self { + Self { + state_sync: state_sync_provider, + peers: initial_peers + .map(|(peer_id, best_number)| { + (peer_id, Peer { best_number, state: PeerState::Available }) + }) + .collect(), + actions: Vec::new(), + } + } + /// Notify that a new peer has connected. pub fn add_peer(&mut self, peer_id: PeerId, _best_hash: B::Hash, best_number: NumberFor) { self.peers.insert(peer_id, Peer { best_number, state: PeerState::Available }); @@ -197,7 +214,7 @@ where let results = results .into_iter() .filter_map(|(result, hash)| { - if hash == self.state_sync.target() { + if hash == self.state_sync.target_hash() { Some(result) } else { debug!( @@ -251,8 +268,8 @@ where return None } - let peer_id = self - .schedule_next_peer(PeerState::DownloadingState, self.state_sync.target_block_num())?; + let peer_id = + self.schedule_next_peer(PeerState::DownloadingState, self.state_sync.target_number())?; let request = self.state_sync.next_request(); trace!( target: LOG_TARGET, @@ -289,9 +306,9 @@ where state: if self.state_sync.is_complete() { SyncState::Idle } else { - SyncState::Downloading { target: self.state_sync.target_block_num() } + SyncState::Downloading { target: self.state_sync.target_number() } }, - best_seen_block: Some(self.state_sync.target_block_num()), + best_seen_block: Some(self.state_sync.target_number()), num_peers: self.peers.len().saturated_into(), num_connected_peers: self.peers.len().saturated_into(), queued_blocks: 0, diff --git a/substrate/client/network/sync/src/strategy/state_sync.rs b/substrate/client/network/sync/src/strategy/state_sync.rs index f7cfbc8ac3d95..4d3aa9d3034b7 100644 --- a/substrate/client/network/sync/src/strategy/state_sync.rs +++ b/substrate/client/network/sync/src/strategy/state_sync.rs @@ -35,6 +35,32 @@ use sp_runtime::{ }; use std::{collections::HashMap, sync::Arc}; +/// Generic state sync provider. Used for mocking in tests. +pub trait StateSyncProvider: Send + Sync { + /// Validate and import a state response. + fn import(&mut self, response: StateResponse) -> ImportResult; + /// Produce next state request. + fn next_request(&self) -> StateRequest; + /// Check if the state is complete. + fn is_complete(&self) -> bool; + /// Returns target block number. + fn target_number(&self) -> NumberFor; + /// Returns target block hash. + fn target_hash(&self) -> B::Hash; + /// Returns state sync estimated progress. + fn progress(&self) -> StateDownloadProgress; +} + +/// Import state chunk result. +pub enum ImportResult { + /// State is complete and ready for import. + Import(B::Hash, B::Header, ImportedState, Option>, Option), + /// Continue downloading. + Continue, + /// Bad state chunk. + BadResponse, +} + /// State sync state machine. Accumulates partial state data until it /// is ready to be imported. pub struct StateSync { @@ -51,16 +77,6 @@ pub struct StateSync { skip_proof: bool, } -/// Import state chunk result. -pub enum ImportResult { - /// State is complete and ready for import. - Import(B::Hash, B::Header, ImportedState, Option>, Option), - /// Continue downloading. - Continue, - /// Bad state chunk. - BadResponse, -} - impl StateSync where B: BlockT, @@ -88,9 +104,15 @@ where skip_proof, } } +} +impl StateSyncProvider for StateSync +where + B: BlockT, + Client: ProofProvider + Send + Sync + 'static, +{ /// Validate and import a state response. - pub fn import(&mut self, response: StateResponse) -> ImportResult { + fn import(&mut self, response: StateResponse) -> ImportResult { if response.entries.is_empty() && response.proof.is_empty() { debug!(target: LOG_TARGET, "Bad state response"); return ImportResult::BadResponse @@ -238,7 +260,7 @@ where } /// Produce next state request. - pub fn next_request(&self) -> StateRequest { + fn next_request(&self) -> StateRequest { StateRequest { block: self.target_block.encode(), start: self.last_key.clone().into_vec(), @@ -247,22 +269,22 @@ where } /// Check if the state is complete. - pub fn is_complete(&self) -> bool { + fn is_complete(&self) -> bool { self.complete } /// Returns target block number. - pub fn target_block_num(&self) -> NumberFor { + fn target_number(&self) -> NumberFor { *self.target_header.number() } /// Returns target block hash. - pub fn target(&self) -> B::Hash { + fn target_hash(&self) -> B::Hash { self.target_block } /// Returns state sync estimated progress. - pub fn progress(&self) -> StateDownloadProgress { + fn progress(&self) -> StateDownloadProgress { let cursor = *self.last_key.get(0).and_then(|last| last.get(0)).unwrap_or(&0u8); let percent_done = cursor as u32 * 100 / 256; StateDownloadProgress { percentage: percent_done, size: self.imported_bytes } From 6a082bf2542dcbb01a86a756d84bf13c0fd6d437 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 12 Dec 2023 17:00:14 +0200 Subject: [PATCH 40/54] Add more `StateStrategy` tests --- .../client/network/sync/src/strategy/state.rs | 231 +++++++++++++++++- 1 file changed, 221 insertions(+), 10 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 80ee3f159201b..6413f8d56b071 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -338,15 +338,34 @@ impl StateStrategy { #[cfg(test)] mod test { use super::*; - use crate::schema::v1::StateRequest; - use codec::{Decode, Encode}; + use crate::{ + schema::v1::{StateRequest, StateResponse}, + strategy::state_sync::{ImportResult, StateSyncProvider}, + types::StateDownloadProgress, + }; + use codec::Decode; use sc_block_builder::BlockBuilderBuilder; + use sc_client_api::KeyValueStates; + use sc_consensus::{ImportedAux, ImportedState}; use sp_runtime::traits::Zero; use substrate_test_runtime_client::{ runtime::{Block, Hash}, BlockBuilderExt, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; + mockall::mock! { + pub StateSync {} + + impl StateSyncProvider for StateSync { + fn import(&mut self, response: StateResponse) -> ImportResult; + fn next_request(&self) -> StateRequest; + fn is_complete(&self) -> bool; + fn target_number(&self) -> NumberFor; + fn target_hash(&self) -> B::Hash; + fn progress(&self) -> StateDownloadProgress; + } + } + #[test] fn no_peer_is_scheduled_if_no_peers_connected() { let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); @@ -445,10 +464,7 @@ mod test { .unwrap() .block; - let peers = (1..=10) - .map(|best_number| (PeerId::random(), best_number)) - .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); + let initial_peers = (1..=10).map(|best_number| (PeerId::random(), best_number)); let mut state_strategy = StateStrategy::new( client.clone(), @@ -478,10 +494,7 @@ mod test { .unwrap() .block; - let peers = (1..=10) - .map(|best_number| (PeerId::random(), best_number)) - .collect::>(); - let initial_peers = peers.iter().map(|(p, n)| (*p, *n)); + let initial_peers = (1..=10).map(|best_number| (PeerId::random(), best_number)); let mut state_strategy = StateStrategy::new( client.clone(), @@ -498,4 +511,202 @@ mod test { // No parallel request is sent. assert!(state_strategy.state_request().is_none()); } + + #[test] + fn received_state_response_makes_peer_available_again() { + let mut state_sync_provider = MockStateSync::::new(); + state_sync_provider.expect_import().return_once(|_| ImportResult::Continue); + let peer_id = PeerId::random(); + let initial_peers = std::iter::once((peer_id, 10)); + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), initial_peers); + // Manually set the peer's state. + state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; + + let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); + state_strategy.on_state_response(peer_id, dummy_response); + + assert!(state_strategy.peers.get(&peer_id).unwrap().state.is_available()); + } + + #[test] + fn bad_state_response_drops_peer() { + let mut state_sync_provider = MockStateSync::::new(); + // Provider says that state response is bad. + state_sync_provider.expect_import().return_once(|_| ImportResult::BadResponse); + let peer_id = PeerId::random(); + let initial_peers = std::iter::once((peer_id, 10)); + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), initial_peers); + // Manually set the peer's state. + state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; + let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); + // Receiving response drops the peer. + assert!(matches!( + state_strategy.on_state_response_inner(peer_id, dummy_response), + Err(BadPeer(id, _rep)) if id == peer_id, + )); + } + + #[test] + fn partial_state_response_doesnt_generate_actions() { + let mut state_sync_provider = MockStateSync::::new(); + // Sync provider says that the response is partial. + state_sync_provider.expect_import().return_once(|_| ImportResult::Continue); + let peer_id = PeerId::random(); + let initial_peers = std::iter::once((peer_id, 10)); + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), initial_peers); + // Manually set the peer's state . + state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; + + let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); + state_strategy.on_state_response(peer_id, dummy_response); + + // No actions generated. + assert_eq!(state_strategy.actions.len(), 0) + } + + #[test] + fn complete_state_response_leads_to_block_import() { + // Build block to use for checks. + let client = Arc::new(TestClientBuilder::new().set_no_genesis().build()); + let mut block_builder = BlockBuilderBuilder::new(&*client) + .on_parent_block(client.chain_info().best_hash) + .with_parent_block_number(client.chain_info().best_number) + .build() + .unwrap(); + block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); + let block = block_builder.build().unwrap().block; + let header = block.header().clone(); + let hash = header.hash().clone(); + let body = Some(block.extrinsics().iter().cloned().collect::>()); + let state = ImportedState { block: hash, state: KeyValueStates(Vec::new()) }; + let justifications = Some(Justifications::from((*b"FRNK", Vec::new()))); + + // Prepare `StateSync` + let mut state_sync_provider = MockStateSync::::new(); + let import = ImportResult::Import( + hash.clone(), + header.clone(), + state.clone(), + body.clone(), + justifications.clone(), + ); + state_sync_provider.expect_import().return_once(move |_| import); + + // Reference values to check against. + let expected_origin = BlockOrigin::NetworkInitialSync; + let expected_block = IncomingBlock { + hash, + header: Some(header), + body, + indexed_body: None, + justifications, + origin: None, + allow_missing_state: true, + import_existing: true, + skip_execution: true, + state: Some(state), + }; + let expected_blocks = vec![expected_block]; + + // Prepare `StateStrategy`. + let peer_id = PeerId::random(); + let initial_peers = std::iter::once((peer_id, 10)); + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), initial_peers); + // Manually set the peer's state . + state_strategy.peers.get_mut(&peer_id).unwrap().state = PeerState::DownloadingState; + + // Receive response. + let dummy_response = OpaqueStateResponse(Box::new(StateResponse::default())); + state_strategy.on_state_response(peer_id, dummy_response); + + assert_eq!(state_strategy.actions.len(), 1); + assert!(matches!( + &state_strategy.actions[0], + StateStrategyAction::ImportBlocks { origin, blocks } + if *origin == expected_origin && *blocks == expected_blocks, + )); + } + + #[test] + fn importing_unknown_block_doesnt_finish_strategy() { + let target_hash = Hash::random(); + let unknown_hash = Hash::random(); + let mut state_sync_provider = MockStateSync::::new(); + state_sync_provider.expect_target_hash().return_const(target_hash); + + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), std::iter::empty()); + + // Unknown block imported. + state_strategy.on_blocks_processed( + 1, + 1, + vec![( + Ok(BlockImportStatus::ImportedUnknown(1, ImportedAux::default(), None)), + unknown_hash, + )], + ); + + // No actions generated. + assert_eq!(state_strategy.actions.len(), 0); + } + + #[test] + fn importing_target_block_finishes_strategy() { + let target_hash = Hash::random(); + let mut state_sync_provider = MockStateSync::::new(); + state_sync_provider.expect_target_hash().return_const(target_hash.clone()); + + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), std::iter::empty()); + + // Target block imported. + state_strategy.on_blocks_processed( + 1, + 1, + vec![( + Ok(BlockImportStatus::ImportedUnknown(1, ImportedAux::default(), None)), + target_hash, + )], + ); + + // Strategy finishes. + assert_eq!(state_strategy.actions.len(), 1); + assert!(matches!(&state_strategy.actions[0], StateStrategyAction::Finished)); + } + + #[test] + fn finished_strategy_doesnt_generate_more_actions() { + let target_hash = Hash::random(); + let mut state_sync_provider = MockStateSync::::new(); + state_sync_provider.expect_target_hash().return_const(target_hash.clone()); + state_sync_provider.expect_is_complete().return_const(true); + + // Get enough peers for possible spurious requests. + let initial_peers = (1..=10).map(|best_number| (PeerId::random(), best_number)); + + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), initial_peers); + + state_strategy.on_blocks_processed( + 1, + 1, + vec![( + Ok(BlockImportStatus::ImportedUnknown(1, ImportedAux::default(), None)), + target_hash, + )], + ); + + // Strategy finishes. + let actions = state_strategy.actions().collect::>(); + assert_eq!(actions.len(), 1); + assert!(matches!(&actions[0], StateStrategyAction::Finished)); + + // No more actions generated. + assert_eq!(state_strategy.actions().count(), 0); + } } From 20dd76d23882cb2f01577034031e2a1c4bfe0061 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 12 Dec 2023 17:21:30 +0200 Subject: [PATCH 41/54] Add PRdoc --- prdoc/pr_2467.prdoc | 15 +++++++++++++++ .../client/network/sync/src/strategy/state.rs | 8 ++++---- 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 prdoc/pr_2467.prdoc diff --git a/prdoc/pr_2467.prdoc b/prdoc/pr_2467.prdoc new file mode 100644 index 0000000000000..b19a5af581214 --- /dev/null +++ b/prdoc/pr_2467.prdoc @@ -0,0 +1,15 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Extract warp sync strategy from `ChainSync` + +doc: + - audience: Node Dev + description: | + `WarpSync` (and `StateSync` as the logical part of warp sync) is extracted from `ChainSync` + as independent syncing strategy. `SyncingStrategy` enum is introduced as a proxy between + `SyncingEngine` and specific syncing strategies. `SyncingStrategy` may be replaced by a trait + in a follow-up PRs. + +crates: + - name: sc-network-sync diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 6413f8d56b071..88f63a0db33b4 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -579,7 +579,7 @@ mod test { block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); let block = block_builder.build().unwrap().block; let header = block.header().clone(); - let hash = header.hash().clone(); + let hash = header.hash(); let body = Some(block.extrinsics().iter().cloned().collect::>()); let state = ImportedState { block: hash, state: KeyValueStates(Vec::new()) }; let justifications = Some(Justifications::from((*b"FRNK", Vec::new()))); @@ -587,7 +587,7 @@ mod test { // Prepare `StateSync` let mut state_sync_provider = MockStateSync::::new(); let import = ImportResult::Import( - hash.clone(), + hash, header.clone(), state.clone(), body.clone(), @@ -659,7 +659,7 @@ mod test { fn importing_target_block_finishes_strategy() { let target_hash = Hash::random(); let mut state_sync_provider = MockStateSync::::new(); - state_sync_provider.expect_target_hash().return_const(target_hash.clone()); + state_sync_provider.expect_target_hash().return_const(target_hash); let mut state_strategy = StateStrategy::new_with_provider(Box::new(state_sync_provider), std::iter::empty()); @@ -683,7 +683,7 @@ mod test { fn finished_strategy_doesnt_generate_more_actions() { let target_hash = Hash::random(); let mut state_sync_provider = MockStateSync::::new(); - state_sync_provider.expect_target_hash().return_const(target_hash.clone()); + state_sync_provider.expect_target_hash().return_const(target_hash); state_sync_provider.expect_is_complete().return_const(true); // Get enough peers for possible spurious requests. From 14fdbb88c70ad4d5a8ecebf10cfc501d62f5e606 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 12 Dec 2023 17:24:52 +0200 Subject: [PATCH 42/54] Improve PRdoc --- prdoc/pr_2467.prdoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prdoc/pr_2467.prdoc b/prdoc/pr_2467.prdoc index b19a5af581214..db88ff1fa579a 100644 --- a/prdoc/pr_2467.prdoc +++ b/prdoc/pr_2467.prdoc @@ -6,10 +6,10 @@ title: Extract warp sync strategy from `ChainSync` doc: - audience: Node Dev description: | - `WarpSync` (and `StateSync` as the logical part of warp sync) is extracted from `ChainSync` - as independent syncing strategy. `SyncingStrategy` enum is introduced as a proxy between - `SyncingEngine` and specific syncing strategies. `SyncingStrategy` may be replaced by a trait - in a follow-up PRs. + `WarpSync`, and `StateSync` as the logical part of warp sync, are extracted from `ChainSync` + as independent syncing strategies. `SyncingStrategy` enum is introduced as a proxy between + `SyncingEngine` and specific strategies. `SyncingStrategy` may be replaced by a trait in a + follow-up PRs. crates: - name: sc-network-sync From c0115cfefa7ecb35b2e919c803f3775871af128d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 13 Dec 2023 17:00:15 +0200 Subject: [PATCH 43/54] Add test for failed block import in `StateStrategy` --- .../client/network/sync/src/strategy/state.rs | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 88f63a0db33b4..4d2af27a768dd 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -656,7 +656,7 @@ mod test { } #[test] - fn importing_target_block_finishes_strategy() { + fn succesfully_importing_target_block_finishes_strategy() { let target_hash = Hash::random(); let mut state_sync_provider = MockStateSync::::new(); state_sync_provider.expect_target_hash().return_const(target_hash); @@ -679,6 +679,30 @@ mod test { assert!(matches!(&state_strategy.actions[0], StateStrategyAction::Finished)); } + #[test] + fn failure_to_import_target_block_finishes_strategy() { + let target_hash = Hash::random(); + let mut state_sync_provider = MockStateSync::::new(); + state_sync_provider.expect_target_hash().return_const(target_hash); + + let mut state_strategy = + StateStrategy::new_with_provider(Box::new(state_sync_provider), std::iter::empty()); + + // Target block imported. + state_strategy.on_blocks_processed( + 1, + 1, + vec![( + Err(BlockImportError::VerificationFailed(None, String::from("test-error"))), + target_hash, + )], + ); + + // Strategy finishes. + assert_eq!(state_strategy.actions.len(), 1); + assert!(matches!(&state_strategy.actions[0], StateStrategyAction::Finished)); + } + #[test] fn finished_strategy_doesnt_generate_more_actions() { let target_hash = Hash::random(); From fc3da77611928c3466d3d541387ae60e4fdc7415 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 13 Dec 2023 17:02:11 +0200 Subject: [PATCH 44/54] minor: comment --- substrate/client/network/sync/src/strategy/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 4d2af27a768dd..0fdefabcb0d04 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -688,7 +688,7 @@ mod test { let mut state_strategy = StateStrategy::new_with_provider(Box::new(state_sync_provider), std::iter::empty()); - // Target block imported. + // Target block import failed. state_strategy.on_blocks_processed( 1, 1, From aded4c5d3be7eb3fde02f6af4aedc969a0a29799 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 13 Dec 2023 18:34:48 +0200 Subject: [PATCH 45/54] minor: remove unused import --- substrate/client/network/sync/src/types.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index d4b087413780a..7412107a8ec7b 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -29,8 +29,6 @@ use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; -pub use sc_network_common::sync::SyncMode; - /// The sync status of a peer we are trying to sync with #[derive(Debug)] pub struct PeerInfo { From 43454d7d84fbadbd2f603b571848b30acabac732 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 18 Dec 2023 16:13:43 +0200 Subject: [PATCH 46/54] minor: put strategy module into `strategy.rs` --- .../client/network/sync/src/{strategy/mod.rs => strategy.rs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename substrate/client/network/sync/src/{strategy/mod.rs => strategy.rs} (99%) diff --git a/substrate/client/network/sync/src/strategy/mod.rs b/substrate/client/network/sync/src/strategy.rs similarity index 99% rename from substrate/client/network/sync/src/strategy/mod.rs rename to substrate/client/network/sync/src/strategy.rs index c1d4125e3aeef..3f31cd8b4f2b3 100644 --- a/substrate/client/network/sync/src/strategy/mod.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -247,7 +247,7 @@ where } } - /// Process warp proof response. + /// Process warp proof response. pub fn on_warp_proof_response(&mut self, peer_id: &PeerId, response: EncodedProof) { match self { SyncingStrategy::WarpSyncStrategy(strategy) => From 7a242a4657ecd01cb953766bf055d1f6b7b02726 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 20 Dec 2023 12:47:29 +0200 Subject: [PATCH 47/54] minor: formatting --- .../network/sync/src/strategy/chain_sync.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/chain_sync.rs b/substrate/client/network/sync/src/strategy/chain_sync.rs index 7441f006cab47..62c260d582b5a 100644 --- a/substrate/client/network/sync/src/strategy/chain_sync.rs +++ b/substrate/client/network/sync/src/strategy/chain_sync.rs @@ -486,7 +486,7 @@ where // There is nothing sync can get from the node that has no blockchain data. match self.block_status(&best_hash) { Err(e) => { - debug!(target:LOG_TARGET, "Error reading blockchain: {e}"); + debug!(target: LOG_TARGET, "Error reading blockchain: {e}"); Err(BadPeer(peer_id, rep::BLOCKCHAIN_READ_ERROR)) }, Ok(BlockStatus::KnownBad) => { @@ -509,7 +509,7 @@ where // an ancestor search, which is what we do in the next match case below. if self.queue_blocks.len() > MAJOR_SYNC_BLOCKS.into() { debug!( - target:LOG_TARGET, + target: LOG_TARGET, "New peer {} with unknown best hash {} ({}), assuming common block.", peer_id, self.best_queued_hash, @@ -531,7 +531,7 @@ where // If we are at genesis, just start downloading. let (state, req) = if self.best_queued_number.is_zero() { debug!( - target:LOG_TARGET, + target: LOG_TARGET, "New peer {peer_id} with best hash {best_hash} ({best_number}).", ); @@ -540,7 +540,7 @@ where let common_best = std::cmp::min(self.best_queued_number, best_number); debug!( - target:LOG_TARGET, + target: LOG_TARGET, "New peer {} with unknown best hash {} ({}), searching for common ancestor.", peer_id, best_hash, @@ -832,7 +832,7 @@ where } if matching_hash.is_none() && current.is_zero() { trace!( - target:LOG_TARGET, + target: LOG_TARGET, "Ancestry search: genesis mismatch for peer {peer_id}", ); return Err(BadPeer(*peer_id, rep::GENESIS_MISMATCH)) @@ -1237,7 +1237,7 @@ where .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) { trace!( - target:LOG_TARGET, + target: LOG_TARGET, "Accepted {} blocks ({:?}) with origin {:?}", new_blocks.len(), h, @@ -2221,7 +2221,7 @@ pub fn validate_blocks( let hash = header.hash(); if hash != b.hash { debug!( - target:LOG_TARGET, + target: LOG_TARGET, "Bad header received from {}. Expected hash {:?}, got {:?}", peer_id, b.hash, @@ -2238,7 +2238,7 @@ pub fn validate_blocks( ); if expected != got { debug!( - target:LOG_TARGET, + target: LOG_TARGET, "Bad extrinsic root for a block {} received from {}. Expected {:?}, got {:?}", b.hash, peer_id, From dd70e6c966f447b08af6b3c9801a875e12566f0d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 Jan 2024 12:35:14 +0200 Subject: [PATCH 48/54] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- substrate/client/network/sync/src/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 3a313aa00734a..47d2c7af714c7 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -731,7 +731,7 @@ where .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); self.network_service.report_peer(peer_id, rep); - trace!(target: LOG_TARGET, "Processed {action:?}."); + trace!(target: LOG_TARGET, "{peer_id:?} dropped: {rep:?}."); }, SyncingAction::ImportBlocks { origin, blocks } => { let count = blocks.len(); From 0c9d09c11ba32e5b5cfada6c950041820f7bed4f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 Jan 2024 12:54:39 +0200 Subject: [PATCH 49/54] Add comment re skipping proofs in state sync --- substrate/client/network/sync/src/strategy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 3f31cd8b4f2b3..a64864b532ed3 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -416,6 +416,7 @@ where res.target_header, res.target_body, res.target_justifications, + // skip proofs, only set to `true` in `FastUnsafe` sync mode false, connected_peers .map(|(peer_id, _best_hash, best_number)| (peer_id, best_number)), From eeb14c3334efdd72bb9313873bd52a7a19955049 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 Jan 2024 18:11:30 +0200 Subject: [PATCH 50/54] Fix issue with slow target block download during warp sync --- substrate/client/network/sync/src/strategy.rs | 10 ++++---- .../client/network/sync/src/strategy/state.rs | 24 ++++++++++++++++++- .../client/network/sync/src/strategy/warp.rs | 23 +++++++++++++++++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index a64864b532ed3..466f8cd17fe8f 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -41,7 +41,7 @@ use sc_network_common::sync::{ use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; use sp_consensus::BlockOrigin; use sp_runtime::{ - traits::{Block as BlockT, Header, NumberFor}, + traits::{Block as BlockT, NumberFor}, Justifications, }; use state::{StateStrategy, StateStrategyAction}; @@ -166,10 +166,10 @@ where announce: &BlockAnnounce, ) -> Option<(B::Hash, NumberFor)> { match self { - SyncingStrategy::WarpSyncStrategy(_) => - Some((announce.header.hash(), *announce.header.number())), - SyncingStrategy::StateSyncStrategy(_) => - Some((announce.header.hash(), *announce.header.number())), + SyncingStrategy::WarpSyncStrategy(strategy) => + strategy.on_validated_block_announce(is_best, peer_id, announce), + SyncingStrategy::StateSyncStrategy(strategy) => + strategy.on_validated_block_announce(is_best, peer_id, announce), SyncingStrategy::ChainSyncStrategy(strategy) => strategy.on_validated_block_announce(is_best, peer_id, announce), } diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 0fdefabcb0d04..84b576496b0c0 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -28,9 +28,10 @@ use libp2p::PeerId; use log::{debug, error, info, trace}; use sc_client_api::ProofProvider; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sc_network_common::sync::message::BlockAnnounce; use sp_consensus::BlockOrigin; use sp_runtime::{ - traits::{Block as BlockT, NumberFor}, + traits::{Block as BlockT, Header, NumberFor}, Justifications, SaturatedConversion, }; use std::{collections::HashMap, sync::Arc}; @@ -139,6 +140,27 @@ impl StateStrategy { self.peers.remove(peer_id); } + /// Submit a validated block announcement. + /// + /// Returns new best hash & best number of the peer if they are updated. + #[must_use] + pub fn on_validated_block_announce( + &mut self, + is_best: bool, + peer_id: PeerId, + announce: &BlockAnnounce, + ) -> Option<(B::Hash, NumberFor)> { + is_best.then_some({ + let best_number = *announce.header.number(); + let best_hash = announce.header.hash(); + if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { + peer.best_number = best_number; + } + // Let `SyncingEngine` know that we should update the peer info. + (best_hash, best_number) + }) + } + /// Process state response. pub fn on_state_response(&mut self, peer_id: PeerId, response: OpaqueStateResponse) { if let Err(bad_peer) = self.on_state_response_inner(peer_id, response) { diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index daa1e46928e19..bf009603bb6e0 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -30,7 +30,7 @@ use futures::channel::oneshot; use libp2p::PeerId; use log::{debug, error, trace}; use sc_network_common::sync::message::{ - BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, + BlockAnnounce, BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; use sp_blockchain::HeaderBackend; use sp_runtime::{ @@ -312,6 +312,27 @@ where self.peers.remove(peer_id); } + /// Submit a validated block announcement. + /// + /// Returns new best hash & best number of the peer if they are updated. + #[must_use] + pub fn on_validated_block_announce( + &mut self, + is_best: bool, + peer_id: PeerId, + announce: &BlockAnnounce, + ) -> Option<(B::Hash, NumberFor)> { + is_best.then_some({ + let best_number = *announce.header.number(); + let best_hash = announce.header.hash(); + if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { + peer.best_number = best_number; + } + // Let `SyncingEngine` know that we should update the peer info. + (best_hash, best_number) + }) + } + /// Start warp sync as soon as we have enough peers. fn try_to_start_warp_sync(&mut self) { let Phase::WaitingForPeers { warp_sync_provider } = &self.phase else { return }; From 3d50db24670cfc1df418ac977d6de1b4dd6f32ff Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 11 Jan 2024 18:12:12 +0200 Subject: [PATCH 51/54] Apply review suggestions --- .../client/network/sync/src/strategy/state.rs | 28 ++++++++-------- .../client/network/sync/src/strategy/warp.rs | 33 ++++++++++--------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 84b576496b0c0..f6f7ad86009c8 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -182,6 +182,7 @@ impl StateStrategy { target: LOG_TARGET, "Failed to downcast opaque state response, this is an implementation bug." ); + debug_assert!(false); BadPeer(peer_id, rep::BAD_RESPONSE) })?; @@ -283,8 +284,8 @@ impl StateStrategy { if self .peers - .iter() - .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingState)) + .values() + .any(|peer| matches!(peer.state, PeerState::DownloadingState)) { // Only one state request at a time is possible. return None @@ -306,17 +307,18 @@ impl StateStrategy { min_best_number: NumberFor, ) -> Option { let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - let threshold = std::cmp::max(median, min_best_number); - // Find a random peer that is synced as much as peer majority and is above - // `min_best_number`. - for (peer_id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= threshold { - peer.state = new_state; - return Some(*peer_id) - } + if targets.is_empty() { + return None + } + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number); + // Find a random peer that is synced as much as peer majority and is above + // `min_best_number`. + for (peer_id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + peer.state = new_state; + return Some(*peer_id) } } None diff --git a/substrate/client/network/sync/src/strategy/warp.rs b/substrate/client/network/sync/src/strategy/warp.rs index bf009603bb6e0..7935b5f29b68d 100644 --- a/substrate/client/network/sync/src/strategy/warp.rs +++ b/substrate/client/network/sync/src/strategy/warp.rs @@ -481,17 +481,18 @@ where min_best_number: Option>, ) -> Option { let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); - // Find a random peer that is synced as much as peer majority and is above - // `min_best_number`. - for (peer_id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= threshold { - peer.state = new_state; - return Some(*peer_id) - } + if targets.is_empty() { + return None + } + targets.sort(); + let median = targets[targets.len() / 2]; + let threshold = std::cmp::max(median, min_best_number.unwrap_or(Zero::zero())); + // Find a random peer that is synced as much as peer majority and is above + // `min_best_number`. + for (peer_id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= threshold { + peer.state = new_state; + return Some(*peer_id) } } None @@ -506,8 +507,8 @@ where if self .peers - .iter() - .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingProofs)) + .values() + .any(|peer| matches!(peer.state, PeerState::DownloadingProofs)) { // Only one warp proof request at a time is possible. return None @@ -525,8 +526,8 @@ where if self .peers - .iter() - .any(|(_, peer)| matches!(peer.state, PeerState::DownloadingTargetBlock)) + .values() + .any(|peer| matches!(peer.state, PeerState::DownloadingTargetBlock)) { // Only one target block request at a time is possible. return None @@ -636,6 +637,8 @@ where std::mem::take(&mut self.actions).into_iter() } + /// Take the result of finished warp sync, returning `None` if the sync was unsuccessful. + #[must_use] pub fn take_result(&mut self) -> Option> { self.result.take() } From 180e5826013ff3ca9f21d5a4c5e5bd91fe99e81c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 Jan 2024 12:50:56 +0200 Subject: [PATCH 52/54] Don't log state sync result twice --- substrate/client/network/sync/src/strategy.rs | 12 ++++++--- .../client/network/sync/src/strategy/state.rs | 27 ++++++++----------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 466f8cd17fe8f..71b5962ef19bc 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -409,7 +409,7 @@ where Some(res) => { info!( target: LOG_TARGET, - "Warp sync finished, continuing with state sync." + "Warp sync is complete, continuing with state sync." ); let state_sync = StateStrategy::new( client, @@ -427,7 +427,7 @@ where None => { error!( target: LOG_TARGET, - "Warp sync failed. Continuing with full sync." + "Warp sync failed. Falling back to full sync." ); let mut chain_sync = match ChainSync::new( chain_sync_mode(config.mode), @@ -453,8 +453,12 @@ where }, } }, - Self::StateSyncStrategy(_) => { - info!(target: LOG_TARGET, "State sync finished, continuing with block sync."); + Self::StateSyncStrategy(state_sync) => { + if state_sync.is_succeded() { + info!(target: LOG_TARGET, "State sync is complete, continuing with block sync."); + } else { + error!(target: LOG_TARGET, "State sync failed. Falling back to full sync."); + } let mut chain_sync = match ChainSync::new( chain_sync_mode(config.mode), client, diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index f6f7ad86009c8..800feb67bd744 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -25,7 +25,7 @@ use crate::{ LOG_TARGET, }; use libp2p::PeerId; -use log::{debug, error, info, trace}; +use log::{debug, error, trace}; use sc_client_api::ProofProvider; use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; use sc_network_common::sync::message::BlockAnnounce; @@ -79,6 +79,7 @@ pub struct StateStrategy { state_sync: Box>, peers: HashMap>, actions: Vec>, + succeded: bool, } impl StateStrategy { @@ -109,6 +110,7 @@ impl StateStrategy { )), peers, actions: Vec::new(), + succeded: false, } } @@ -127,6 +129,7 @@ impl StateStrategy { }) .collect(), actions: Vec::new(), + succeded: false, } } @@ -257,21 +260,7 @@ impl StateStrategy { "Failed to import target block with state: {e:?}." ); }); - match results.into_iter().any(|result| result.is_ok()) { - true => { - info!( - target: LOG_TARGET, - "State sync is complete ({} MiB), continuing with block sync.", - self.state_sync.progress().size / (1024 * 1024), - ); - }, - false => { - error!( - target: LOG_TARGET, - "State sync failed. Falling back to full sync.", - ); - }, - } + self.succeded |= results.into_iter().any(|result| result.is_ok()); self.actions.push(StateStrategyAction::Finished); } } @@ -357,6 +346,12 @@ impl StateStrategy { std::mem::take(&mut self.actions).into_iter() } + + /// Check if state sync has succeded. + #[must_use] + pub fn is_succeded(&self) -> bool { + self.succeded + } } #[cfg(test)] From e72c559fc7cf3a20b69d614a6ec77d023bae72e4 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 Jan 2024 13:35:20 +0200 Subject: [PATCH 53/54] Print nicer informant messages during state sync --- substrate/client/informant/src/display.rs | 5 +- substrate/client/network/sync/src/strategy.rs | 2 +- .../client/network/sync/src/strategy/state.rs | 5 +- .../network/sync/src/strategy/state_sync.rs | 46 +++++++++++++++++-- substrate/client/network/sync/src/types.rs | 13 +----- 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/substrate/client/informant/src/display.rs b/substrate/client/informant/src/display.rs index e72fd381b4097..bcf3794032fe8 100644 --- a/substrate/client/informant/src/display.rs +++ b/substrate/client/informant/src/display.rs @@ -127,9 +127,10 @@ impl InformantDisplay { ), (_, Some(state), _) => ( "⚙️ ", - "Downloading state".into(), + "State sync".into(), format!( - ", {}%, {:.2} Mib", + ", {}, {}%, {:.2} Mib", + state.phase, state.percentage, (state.size as f32) / (1024f32 * 1024f32) ), diff --git a/substrate/client/network/sync/src/strategy.rs b/substrate/client/network/sync/src/strategy.rs index 71b5962ef19bc..ee99252fc9117 100644 --- a/substrate/client/network/sync/src/strategy.rs +++ b/substrate/client/network/sync/src/strategy.rs @@ -21,7 +21,7 @@ pub mod chain_sync; mod state; -mod state_sync; +pub mod state_sync; pub mod warp; use crate::{ diff --git a/substrate/client/network/sync/src/strategy/state.rs b/substrate/client/network/sync/src/strategy/state.rs index 800feb67bd744..ae3f7b6005594 100644 --- a/substrate/client/network/sync/src/strategy/state.rs +++ b/substrate/client/network/sync/src/strategy/state.rs @@ -359,8 +359,7 @@ mod test { use super::*; use crate::{ schema::v1::{StateRequest, StateResponse}, - strategy::state_sync::{ImportResult, StateSyncProvider}, - types::StateDownloadProgress, + strategy::state_sync::{ImportResult, StateSyncProgress, StateSyncProvider}, }; use codec::Decode; use sc_block_builder::BlockBuilderBuilder; @@ -381,7 +380,7 @@ mod test { fn is_complete(&self) -> bool; fn target_number(&self) -> NumberFor; fn target_hash(&self) -> B::Hash; - fn progress(&self) -> StateDownloadProgress; + fn progress(&self) -> StateSyncProgress; } } diff --git a/substrate/client/network/sync/src/strategy/state_sync.rs b/substrate/client/network/sync/src/strategy/state_sync.rs index 4d3aa9d3034b7..1ed1de7c8efaa 100644 --- a/substrate/client/network/sync/src/strategy/state_sync.rs +++ b/substrate/client/network/sync/src/strategy/state_sync.rs @@ -20,7 +20,6 @@ use crate::{ schema::v1::{StateEntry, StateRequest, StateResponse}, - types::StateDownloadProgress, LOG_TARGET, }; use codec::{Decode, Encode}; @@ -33,7 +32,7 @@ use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor}, Justifications, }; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, fmt, sync::Arc}; /// Generic state sync provider. Used for mocking in tests. pub trait StateSyncProvider: Send + Sync { @@ -48,7 +47,36 @@ pub trait StateSyncProvider: Send + Sync { /// Returns target block hash. fn target_hash(&self) -> B::Hash; /// Returns state sync estimated progress. - fn progress(&self) -> StateDownloadProgress; + fn progress(&self) -> StateSyncProgress; +} + +// Reported state sync phase. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum StateSyncPhase { + // State download in progress. + DownloadingState, + // Download is complete, state is being imported. + ImportingState, +} + +impl fmt::Display for StateSyncPhase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::DownloadingState => write!(f, "Downloading state"), + Self::ImportingState => write!(f, "Importing state"), + } + } +} + +/// Reported state download progress. +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct StateSyncProgress { + /// Estimated download percentage. + pub percentage: u32, + /// Total state size in bytes downloaded so far. + pub size: u64, + /// Current state sync phase. + pub phase: StateSyncPhase, } /// Import state chunk result. @@ -284,9 +312,17 @@ where } /// Returns state sync estimated progress. - fn progress(&self) -> StateDownloadProgress { + fn progress(&self) -> StateSyncProgress { let cursor = *self.last_key.get(0).and_then(|last| last.get(0)).unwrap_or(&0u8); let percent_done = cursor as u32 * 100 / 256; - StateDownloadProgress { percentage: percent_done, size: self.imported_bytes } + StateSyncProgress { + percentage: percent_done, + size: self.imported_bytes, + phase: if self.complete { + StateSyncPhase::ImportingState + } else { + StateSyncPhase::DownloadingState + }, + } } } diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs index 7412107a8ec7b..4074b33eee1a9 100644 --- a/substrate/client/network/sync/src/types.rs +++ b/substrate/client/network/sync/src/types.rs @@ -23,7 +23,7 @@ use sc_network_common::{role::Roles, types::ReputationChange}; use libp2p::PeerId; -use crate::strategy::warp::WarpSyncProgress; +use crate::strategy::{state_sync::StateSyncProgress, warp::WarpSyncProgress}; use sc_network_common::sync::message::BlockRequest; use sp_runtime::traits::{Block as BlockT, NumberFor}; @@ -67,15 +67,6 @@ impl SyncState { } } -/// Reported state download progress. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct StateDownloadProgress { - /// Estimated download percentage. - pub percentage: u32, - /// Total state size in bytes downloaded so far. - pub size: u64, -} - /// Syncing status and statistics. #[derive(Debug, Clone)] pub struct SyncStatus { @@ -90,7 +81,7 @@ pub struct SyncStatus { /// Number of blocks queued for import pub queued_blocks: u32, /// State sync status in progress, if any. - pub state_sync: Option, + pub state_sync: Option, /// Warp sync in progress, if any. pub warp_sync: Option>, } From 398c8b6b45df34fdc3501d55b804c43f31d44c58 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 Jan 2024 14:37:12 +0200 Subject: [PATCH 54/54] Update warp sync zombienet tests --- .../zombienet/0001-basic-warp-sync/test-warp-sync.zndsl | 4 ++-- .../test-validators-warp-sync.zndsl | 8 ++++---- .../test-block-building-warp-sync.zndsl | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl b/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl index 75aefea1082d7..c5644797321e1 100644 --- a/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl +++ b/substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl @@ -21,9 +21,9 @@ charlie: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds dave: reports block height is at least 1 within 60 seconds dave: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds -dave: log line matches "Warp sync finished" within 60 seconds +dave: log line matches "Warp sync is complete" within 60 seconds # State sync is logically part of warp sync -dave: log line matches "State sync finished" within 60 seconds +dave: log line matches "State sync is complete" within 60 seconds dave: log line matches "Block history download is complete" within 10 seconds dave: count of log lines containing "error" is 0 within 10 seconds diff --git a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl index da4f16ab42060..6e733580d6deb 100644 --- a/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl +++ b/substrate/zombienet/0002-validators-warp-sync/test-validators-warp-sync.zndsl @@ -19,11 +19,11 @@ charlie: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds dave: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds eve: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds -alice: log line matches "Warp sync finished" within 60 seconds -bob: log line matches "Warp sync finished" within 60 seconds +alice: log line matches "Warp sync is complete" within 60 seconds +bob: log line matches "Warp sync is complete" within 60 seconds # State sync is logically part of warp sync -alice: log line matches "State sync finished" within 60 seconds -bob: log line matches "State sync finished" within 60 seconds +alice: log line matches "State sync is complete" within 60 seconds +bob: log line matches "State sync is complete" within 60 seconds alice: log line matches "Block history download is complete" within 120 seconds bob: log line matches "Block history download is complete" within 120 seconds diff --git a/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl b/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl index 447598da6638d..a9a0ce442f160 100644 --- a/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl +++ b/substrate/zombienet/0003-block-building-warp-sync/test-block-building-warp-sync.zndsl @@ -26,9 +26,9 @@ dave: reports block height is at least 1 within 60 seconds dave: reports block height is at least {{DB_BLOCK_HEIGHT}} within 60 seconds dave: reports block height is greater than {{DB_BLOCK_HEIGHT}} within 60 seconds -dave: log line matches "Warp sync finished" within 60 seconds +dave: log line matches "Warp sync is complete" within 60 seconds # State sync is logically part of warp sync -dave: log line matches "State sync finished" within 60 seconds +dave: log line matches "State sync is complete" within 60 seconds dave: log line matches "Block history download is complete" within 10 seconds dave: reports substrate_beefy_best_block is at least {{DB_BLOCK_HEIGHT}} within 180 seconds