diff --git a/Cargo.lock b/Cargo.lock index 14cf92d9bccbe..08da4064c838e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11920,6 +11920,26 @@ dependencies = [ "sc-cli", ] +[[package]] +name = "substrate" +version = "0.0.0" +dependencies = [ + "aquamarine", + "chain-spec-builder", + "frame-support", + "node-cli", + "sc-cli", + "sc-consensus-aura", + "sc-consensus-babe", + "sc-consensus-beefy", + "sc-consensus-grandpa", + "sc-consensus-manual-seal", + "sc-consensus-pow", + "sc-service", + "sp-runtime", + "subkey", +] + [[package]] name = "substrate-bip39" version = "0.4.4" diff --git a/Cargo.toml b/Cargo.toml index 95acf0b9702cb..5671238e76116 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,42 @@ +[package] +name = "substrate" +description = "Next-generation framework for blockchain innovation" +license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +authors = ["Parity Technologies "] +edition = "2021" +version = "0.0.0" + +# This list of dependencies is for documentation purposes only. +[dependencies] +aquamarine = "0.3.2" + +subkey = { path = "bin/utils/subkey" } +chain-spec-builder = { path = "bin/utils/chain-spec-builder" } + +sc-service = { path = "client/service" } +sc-cli = { path = "client/cli" } +sc-consensus-aura = { path = "client/consensus/aura" } +sc-consensus-babe = { path = "client/consensus/babe" } +sc-consensus-grandpa = { path = "client/consensus/grandpa" } +sc-consensus-beefy = { path = "client/consensus/beefy" } +sc-consensus-manual-seal = { path = "client/consensus/manual-seal" } +sc-consensus-pow = { path = "client/consensus/pow" } + +sp-runtime = { path = "primitives/runtime" } +frame-support = { path = "frame/support" } + +node-cli = { path = "bin/node/cli" } + +# Exists here to be backwards compatible and to support `cargo run` in the workspace. +# +# Just uses the `node-cli` main binary. `node-cli` itself also again exposes the node as +# `substrate-node`. Using `node-cli` directly you can also enable features. +[[bin]] +name = "substrate" +path = "bin/node/cli/bin/main.rs" + [workspace] resolver = "2" diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 032ba271f3c81..00b6775a50d0f 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -6,7 +6,7 @@ description = "Generic Substrate node implementation in Rust." build = "build.rs" edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -default-run = "substrate" +default-run = "substrate-node" homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -25,8 +25,10 @@ maintenance = { status = "actively-developed" } is-it-maintained-issue-resolution = { repository = "paritytech/substrate" } is-it-maintained-open-issues = { repository = "paritytech/substrate" } +# The same node binary as the `substrate` (defined in the workspace `Cargo.toml`) binary, +# but just exposed by this crate here. [[bin]] -name = "substrate" +name = "substrate-node" path = "bin/main.rs" required-features = ["cli"] diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 19a074081c550..11a1c57a713f0 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -33,7 +33,7 @@ async fn benchmark_block_works() { common::run_node_for_a_while(base_dir.path(), &["--dev", "--no-hardware-benchmarks"]).await; // Invoke `benchmark block` with all options to make sure that they are valid. - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(["benchmark", "block", "--dev"]) .arg("-d") .arg(base_dir.path()) diff --git a/bin/node/cli/tests/benchmark_extrinsic_works.rs b/bin/node/cli/tests/benchmark_extrinsic_works.rs index 8e6363b7d0665..f7addd883b41f 100644 --- a/bin/node/cli/tests/benchmark_extrinsic_works.rs +++ b/bin/node/cli/tests/benchmark_extrinsic_works.rs @@ -32,7 +32,7 @@ fn benchmark_extrinsic_works() { fn benchmark_extrinsic(pallet: &str, extrinsic: &str) { let base_dir = tempdir().expect("could not create a temp dir"); - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(&["benchmark", "extrinsic", "--dev"]) .arg("-d") .arg(base_dir.path()) diff --git a/bin/node/cli/tests/benchmark_machine_works.rs b/bin/node/cli/tests/benchmark_machine_works.rs index 2cdadb64603ec..b3e3f9c78dea5 100644 --- a/bin/node/cli/tests/benchmark_machine_works.rs +++ b/bin/node/cli/tests/benchmark_machine_works.rs @@ -22,7 +22,7 @@ use std::process::Command; /// Tests that the `benchmark machine` command works for the substrate dev runtime. #[test] fn benchmark_machine_works() { - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(["benchmark", "machine", "--dev"]) .args([ "--verify-duration", @@ -48,7 +48,7 @@ fn benchmark_machine_works() { #[test] #[cfg(debug_assertions)] fn benchmark_machine_fails_with_slow_hardware() { - let output = Command::new(cargo_bin("substrate")) + let output = Command::new(cargo_bin("substrate-node")) .args(["benchmark", "machine", "--dev"]) .args([ "--verify-duration", diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs index 3273e6f868112..b246167f2c447 100644 --- a/bin/node/cli/tests/benchmark_overhead_works.rs +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -28,7 +28,7 @@ fn benchmark_overhead_works() { // Only put 10 extrinsics into the block otherwise it takes forever to build it // especially for a non-release build. - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(&["benchmark", "overhead", "--dev", "-d"]) .arg(base_path) .arg("--weight-path") diff --git a/bin/node/cli/tests/benchmark_pallet_works.rs b/bin/node/cli/tests/benchmark_pallet_works.rs index 9c3e7e2995391..8441333429bea 100644 --- a/bin/node/cli/tests/benchmark_pallet_works.rs +++ b/bin/node/cli/tests/benchmark_pallet_works.rs @@ -34,7 +34,7 @@ fn benchmark_pallet_works() { } fn benchmark_pallet(steps: u32, repeat: u32, should_work: bool) { - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(["benchmark", "pallet", "--dev"]) // Use the `addition` benchmark since is the fastest. .args(["--pallet", "frame-benchmarking", "--extrinsic", "addition"]) diff --git a/bin/node/cli/tests/benchmark_storage_works.rs b/bin/node/cli/tests/benchmark_storage_works.rs index 953c07ca7f0db..e4566f4f9b18d 100644 --- a/bin/node/cli/tests/benchmark_storage_works.rs +++ b/bin/node/cli/tests/benchmark_storage_works.rs @@ -40,7 +40,7 @@ fn benchmark_storage_works() { } fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus { - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .args(&["benchmark", "storage", "--dev"]) .arg("--db") .arg(db) diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index dc5d36184f0c6..ce97dea6f6c84 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -24,7 +24,7 @@ use tempfile::tempdir; fn build_spec_works() { let base_path = tempdir().expect("could not create a temp dir"); - let output = Command::new(cargo_bin("substrate")) + let output = Command::new(cargo_bin("substrate-node")) .args(&["build-spec", "--dev", "-d"]) .arg(base_path.path()) .output() diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index 67bc5e6031ea0..083a79c477bab 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -30,7 +30,7 @@ async fn check_block_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(&["check-block", "--dev", "-d"]) .arg(base_path.path()) .arg("1") diff --git a/bin/node/cli/tests/export_import_flow.rs b/bin/node/cli/tests/export_import_flow.rs index b5785f99ea81f..0dc001ac43011 100644 --- a/bin/node/cli/tests/export_import_flow.rs +++ b/bin/node/cli/tests/export_import_flow.rs @@ -96,7 +96,7 @@ impl<'a> ExportImportRevertExecutor<'a> { }; // Running the command and capturing the output. - let output = Command::new(cargo_bin("substrate")) + let output = Command::new(cargo_bin("substrate-node")) .args(&arguments) .arg(&base_path) .arg(&self.exported_blocks_file) @@ -160,7 +160,7 @@ impl<'a> ExportImportRevertExecutor<'a> { /// Runs the `revert` command. fn run_revert(&self) { - let output = Command::new(cargo_bin("substrate")) + let output = Command::new(cargo_bin("substrate-node")) .args(&["revert", "--dev", "-d"]) .arg(&self.base_path.path()) .output() diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs index 3695c318a8df2..10b0e518e9e87 100644 --- a/bin/node/cli/tests/inspect_works.rs +++ b/bin/node/cli/tests/inspect_works.rs @@ -30,7 +30,7 @@ async fn inspect_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(&["inspect", "--dev", "-d"]) .arg(base_path.path()) .args(&["block", "1"]) diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index 77421f865a0d9..58c4f474521f4 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -29,7 +29,7 @@ async fn purge_chain_works() { common::run_node_for_a_while(base_path.path(), &["--dev", "--no-hardware-benchmarks"]).await; - let status = Command::new(cargo_bin("substrate")) + let status = Command::new(cargo_bin("substrate-node")) .args(&["purge-chain", "--dev", "-d"]) .arg(base_path.path()) .arg("-y") diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index 1308067da0256..f10ea6a055b49 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -33,7 +33,7 @@ async fn running_the_node_works_and_can_be_interrupted() { async fn run_command_and_kill(signal: Signal) { let base_path = tempdir().expect("could not create a temp dir"); let mut cmd = common::KillChildOnDrop( - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(&["--dev", "-d"]) diff --git a/bin/node/cli/tests/telemetry.rs b/bin/node/cli/tests/telemetry.rs index 176d2e81ad06b..2321f56c473d6 100644 --- a/bin/node/cli/tests/telemetry.rs +++ b/bin/node/cli/tests/telemetry.rs @@ -67,7 +67,7 @@ async fn telemetry_works() { } }); - let mut substrate = process::Command::new(cargo_bin("substrate")); + let mut substrate = process::Command::new(cargo_bin("substrate-node")); let mut substrate = KillChildOnDrop( substrate diff --git a/bin/node/cli/tests/temp_base_path_works.rs b/bin/node/cli/tests/temp_base_path_works.rs index fdcd9e23dde5a..42f493afad256 100644 --- a/bin/node/cli/tests/temp_base_path_works.rs +++ b/bin/node/cli/tests/temp_base_path_works.rs @@ -31,7 +31,7 @@ use substrate_cli_test_utils as common; //#[tokio::test] async fn temp_base_path_works() { common::run_with_timeout(Duration::from_secs(60 * 10), async move { - let mut cmd = Command::new(cargo_bin("substrate")); + let mut cmd = Command::new(cargo_bin("substrate-node")); let mut child = common::KillChildOnDrop( cmd.args(&["--dev", "--tmp", "--no-hardware-benchmarks"]) .stdout(Stdio::piped()) diff --git a/bin/node/cli/tests/version.rs b/bin/node/cli/tests/version.rs index e239277c9b0ed..ac1a6b79682ec 100644 --- a/bin/node/cli/tests/version.rs +++ b/bin/node/cli/tests/version.rs @@ -21,17 +21,17 @@ use regex::Regex; use std::process::Command; fn expected_regex() -> Regex { - Regex::new(r"^substrate (.+)-([a-f\d]+)$").unwrap() + Regex::new(r"^substrate-node (.+)-([a-f\d]+)$").unwrap() } #[test] fn version_is_full() { let expected = expected_regex(); - let output = Command::new(cargo_bin("substrate")).args(&["--version"]).output().unwrap(); + let output = Command::new(cargo_bin("substrate-node")).args(&["--version"]).output().unwrap(); assert!(output.status.success(), "command returned with non-success exit code"); - let output = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + let output = dbg!(String::from_utf8_lossy(&output.stdout).trim().to_owned()); let captures = expected.captures(output.as_str()).expect("could not parse version in output"); assert_eq!(&captures[1], env!("CARGO_PKG_VERSION")); @@ -41,11 +41,11 @@ fn version_is_full() { fn test_regex_matches_properly() { let expected = expected_regex(); - let captures = expected.captures("substrate 2.0.0-da487d19d").unwrap(); + let captures = expected.captures("substrate-node 2.0.0-da487d19d").unwrap(); assert_eq!(&captures[1], "2.0.0"); assert_eq!(&captures[2], "da487d19d"); - let captures = expected.captures("substrate 2.0.0-alpha.5-da487d19d").unwrap(); + let captures = expected.captures("substrate-node 2.0.0-alpha.5-da487d19d").unwrap(); assert_eq!(&captures[1], "2.0.0-alpha.5"); assert_eq!(&captures[2], "da487d19d"); } diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 412b41009340c..c592f8ac226c3 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -13,6 +13,13 @@ publish = false [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +[[bin]] +path = "bin/main.rs" +name = "chain-spec-builder" + +[lib] +crate-type = ["rlib"] + [dependencies] ansi_term = "0.12.1" clap = { version = "4.2.5", features = ["derive"] } diff --git a/bin/utils/chain-spec-builder/README.md b/bin/utils/chain-spec-builder/README.md deleted file mode 100644 index 3e9ac0bddbdc1..0000000000000 --- a/bin/utils/chain-spec-builder/README.md +++ /dev/null @@ -1 +0,0 @@ -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file diff --git a/bin/utils/chain-spec-builder/bin/main.rs b/bin/utils/chain-spec-builder/bin/main.rs new file mode 100644 index 0000000000000..53e11abbf6282 --- /dev/null +++ b/bin/utils/chain-spec-builder/bin/main.rs @@ -0,0 +1,89 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use chain_spec_builder::{ + generate_authority_keys_and_store, generate_chain_spec, print_seeds, ChainSpecBuilder, +}; +use clap::Parser; +use node_cli::chain_spec; +use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; +use sp_core::{crypto::Ss58Codec, sr25519}; +use std::fs; + +fn main() -> Result<(), String> { + #[cfg(build_type = "debug")] + println!( + "The chain spec builder builds a chain specification that includes a Substrate runtime \ + compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ + the chain spec builder binary in `--release` mode.\n", + ); + + let builder = ChainSpecBuilder::parse(); + let chain_spec_path = builder.chain_spec_path().to_path_buf(); + + let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder { + ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => { + let authorities = authorities.max(1); + let rand_str = || -> String { + OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() + }; + + let authority_seeds = (0..authorities).map(|_| rand_str()).collect::>(); + let nominator_seeds = (0..nominators).map(|_| rand_str()).collect::>(); + let endowed_seeds = (0..endowed).map(|_| rand_str()).collect::>(); + let sudo_seed = rand_str(); + + print_seeds(&authority_seeds, &nominator_seeds, &endowed_seeds, &sudo_seed); + + if let Some(keystore_path) = keystore_path { + generate_authority_keys_and_store(&authority_seeds, &keystore_path)?; + } + + let nominator_accounts = nominator_seeds + .into_iter() + .map(|seed| { + chain_spec::get_account_id_from_seed::(&seed).to_ss58check() + }) + .collect(); + + let endowed_accounts = endowed_seeds + .into_iter() + .map(|seed| { + chain_spec::get_account_id_from_seed::(&seed).to_ss58check() + }) + .collect(); + + let sudo_account = + chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); + + (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) + }, + ChainSpecBuilder::New { + authority_seeds, + nominator_accounts, + endowed_accounts, + sudo_account, + .. + } => (authority_seeds, nominator_accounts, endowed_accounts, sudo_account), + }; + + let json = + generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account)?; + + fs::write(chain_spec_path, json).map_err(|err| err.to_string()) +} diff --git a/bin/utils/chain-spec-builder/src/lib.rs b/bin/utils/chain-spec-builder/src/lib.rs new file mode 100644 index 0000000000000..fa3c6835c799e --- /dev/null +++ b/bin/utils/chain-spec-builder/src/lib.rs @@ -0,0 +1,234 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Substrate's chain spec builder utility. +//! +//! A chain-spec is short for `chain-configuration`. See the [`sc-chain-spec`] for more information. +//! +//! See [`ChainSpecBuilder`] for a list of available commands. + +use std::path::{Path, PathBuf}; + +use ansi_term::Style; +use clap::Parser; + +use node_cli::chain_spec::{self, AccountId}; +use sc_keystore::LocalKeystore; +use sp_core::crypto::{ByteArray, Ss58Codec}; +use sp_keystore::KeystorePtr; + +/// A utility to easily create a testnet chain spec definition with a given set +/// of authorities and endowed accounts and/or generate random accounts. +#[derive(Parser)] +#[command(rename_all = "kebab-case")] +pub enum ChainSpecBuilder { + /// Create a new chain spec with the given authorities, endowed and sudo + /// accounts. + New { + /// Authority key seed. + #[arg(long, short, required = true)] + authority_seeds: Vec, + /// Active nominators (SS58 format), each backing a random subset of the aforementioned + /// authorities. + #[arg(long, short, default_value = "0")] + nominator_accounts: Vec, + /// Endowed account address (SS58 format). + #[arg(long, short)] + endowed_accounts: Vec, + /// Sudo account address (SS58 format). + #[arg(long, short)] + sudo_account: String, + /// The path where the chain spec should be saved. + #[arg(long, short, default_value = "./chain_spec.json")] + chain_spec_path: PathBuf, + }, + /// Create a new chain spec with the given number of authorities and endowed + /// accounts. Random keys will be generated as required. + Generate { + /// The number of authorities. + #[arg(long, short)] + authorities: usize, + /// The number of nominators backing the aforementioned authorities. + /// + /// Will nominate a random subset of `authorities`. + #[arg(long, short, default_value_t = 0)] + nominators: usize, + /// The number of endowed accounts. + #[arg(long, short, default_value_t = 0)] + endowed: usize, + /// The path where the chain spec should be saved. + #[arg(long, short, default_value = "./chain_spec.json")] + chain_spec_path: PathBuf, + /// Path to use when saving generated keystores for each authority. + /// + /// At this path, a new folder will be created for each authority's + /// keystore named `auth-$i` where `i` is the authority index, i.e. + /// `auth-0`, `auth-1`, etc. + #[arg(long, short)] + keystore_path: Option, + }, +} + +impl ChainSpecBuilder { + /// Returns the path where the chain spec should be saved. + pub fn chain_spec_path(&self) -> &Path { + match self { + ChainSpecBuilder::New { chain_spec_path, .. } => chain_spec_path.as_path(), + ChainSpecBuilder::Generate { chain_spec_path, .. } => chain_spec_path.as_path(), + } + } +} + +fn genesis_constructor( + authority_seeds: &[String], + nominator_accounts: &[AccountId], + endowed_accounts: &[AccountId], + sudo_account: &AccountId, +) -> chain_spec::RuntimeGenesisConfig { + let authorities = authority_seeds + .iter() + .map(AsRef::as_ref) + .map(chain_spec::authority_keys_from_seed) + .collect::>(); + + chain_spec::testnet_genesis( + authorities, + nominator_accounts.to_vec(), + sudo_account.clone(), + Some(endowed_accounts.to_vec()), + ) +} + +/// Generate the chain spec using the given seeds and accounts. +pub fn generate_chain_spec( + authority_seeds: Vec, + nominator_accounts: Vec, + endowed_accounts: Vec, + sudo_account: String, +) -> Result { + let parse_account = |address: String| { + AccountId::from_string(&address) + .map_err(|err| format!("Failed to parse account address: {:?}", err)) + }; + + let nominator_accounts = nominator_accounts + .into_iter() + .map(parse_account) + .collect::, String>>()?; + + let endowed_accounts = endowed_accounts + .into_iter() + .map(parse_account) + .collect::, String>>()?; + + let sudo_account = parse_account(sudo_account)?; + + let chain_spec = chain_spec::ChainSpec::from_genesis( + "Custom", + "custom", + sc_chain_spec::ChainType::Live, + move || { + genesis_constructor( + &authority_seeds, + &nominator_accounts, + &endowed_accounts, + &sudo_account, + ) + }, + vec![], + None, + None, + None, + None, + Default::default(), + ); + + chain_spec.as_json(false) +} + +/// Generate the authority keys and store them in the given `keystore_path`. +pub fn generate_authority_keys_and_store( + seeds: &[String], + keystore_path: &Path, +) -> Result<(), String> { + for (n, seed) in seeds.iter().enumerate() { + let keystore: KeystorePtr = + LocalKeystore::open(keystore_path.join(format!("auth-{}", n)), None) + .map_err(|err| err.to_string())? + .into(); + + let (_, _, grandpa, babe, im_online, authority_discovery) = + chain_spec::authority_keys_from_seed(seed); + + let insert_key = |key_type, public| { + keystore + .insert(key_type, &format!("//{}", seed), public) + .map_err(|_| format!("Failed to insert key: {}", grandpa)) + }; + + insert_key(sp_core::crypto::key_types::BABE, babe.as_slice())?; + + insert_key(sp_core::crypto::key_types::GRANDPA, grandpa.as_slice())?; + + insert_key(sp_core::crypto::key_types::IM_ONLINE, im_online.as_slice())?; + + insert_key( + sp_core::crypto::key_types::AUTHORITY_DISCOVERY, + authority_discovery.as_slice(), + )?; + } + + Ok(()) +} + +/// Print the given seeds +pub fn print_seeds( + authority_seeds: &[String], + nominator_seeds: &[String], + endowed_seeds: &[String], + sudo_seed: &str, +) { + let header = Style::new().bold().underline(); + let entry = Style::new().bold(); + + println!("{}", header.paint("Authority seeds")); + + for (n, seed) in authority_seeds.iter().enumerate() { + println!("{} //{}", entry.paint(format!("auth-{}:", n)), seed); + } + + println!("{}", header.paint("Nominator seeds")); + + for (n, seed) in nominator_seeds.iter().enumerate() { + println!("{} //{}", entry.paint(format!("nom-{}:", n)), seed); + } + + println!(); + + if !endowed_seeds.is_empty() { + println!("{}", header.paint("Endowed seeds")); + for (n, seed) in endowed_seeds.iter().enumerate() { + println!("{} //{}", entry.paint(format!("endowed-{}:", n)), seed); + } + + println!(); + } + + println!("{}", header.paint("Sudo seed")); + println!("//{}", sudo_seed); +} diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index deb191a67cfbe..88b18d1110b55 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -16,6 +16,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! Substrate's chain spec builder utility. +//! +//! A chain-spec is short for `chain-configuration`. See the [`sc-chain-spec`] for more information. +//! +//! See [`ChainSpecBuilder`] for a list of available commands. + use std::{ fs, path::{Path, PathBuf}, @@ -37,7 +43,7 @@ use sp_keystore::KeystorePtr; /// of authorities and endowed accounts and/or generate random accounts. #[derive(Parser)] #[command(rename_all = "kebab-case")] -enum ChainSpecBuilder { +pub enum ChainSpecBuilder { /// Create a new chain spec with the given authorities, endowed and sudo /// accounts. New { diff --git a/bin/utils/subkey/src/lib.rs b/bin/utils/subkey/src/lib.rs index 201d4d25f84ab..f3023acde4047 100644 --- a/bin/utils/subkey/src/lib.rs +++ b/bin/utils/subkey/src/lib.rs @@ -16,6 +16,298 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! # Subkey +//! +//! Subkey is a commandline utility included with Substrate. It allows generating and restoring keys +//! for Substrate based chains such as Polkadot, Kusama and a growing number of parachains and +//! Substrate based projects. + +//! `subkey` provides a few sub-commands to generate keys, check keys, sign messages, verify +//! messages, etc... +//! +//! You can see the full list of commands with `subkey --help`. Most commands have additional help +//! available with for instance `subkey generate --help` for the `generate` command. +//! +//! ## Safety first +//! +//! `subkey` does not need an internet connection to work. Indeed, for the best security, you should +//! be using `subkey` on a machine that is **not connected** to the internet. +//! +//! `subkey` deals with **seeds** and **private keys**. Make sure to use `subkey` in a safe +//! environment (ie. no one looking over your shoulder) and on a safe computer (ie. no one able to +//! check your command history). +//! +//! If you save any output of `subkey` into a file, make sure to apply proper permissions and/or +//! delete the file as soon as possible. +//! +//! ## Usage +//! +//! The following guide explains *some* of the `subkey` commands. For the full list and the most up +//! to date documentation, make sure to check the integrated help with `subkey --help`. +//! +//! ### Install with Cargo +//! +//! You will need to have the Substrate build dependencies to install Subkey. Use the following two +//! commands to install the dependencies and Subkey, respectively: +//! +//! Command: +//! +//! ```bash +//! # Install only `subkey`, at a specific version of the subkey crate +//! cargo install --force subkey --git https://github.com/paritytech/substrate --version --locked +//! # If you run into issues building, you likely are missing deps defined in https://docs.substrate.io/install/ +//! ``` +//! +//! ### Run in a container +//! +//! ```bash +//! # Use `--pull=always` with the `latest` tag, or specify a version in a tag +//! docker run -it --pull=always docker.io/parity/subkey:latest +//! ``` +//! +//! ### Generate a random account +//! +//! Generating a new key is as simple as running: +//! +//! ```bash +//! subkey generate +//! ``` +//! +//! The output looks similar to: +//! +//! ```text +//! Secret phrase `hotel forest jar hover kite book view eight stuff angle legend defense` is account: +//! Secret seed: 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +//! Public key (hex): 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +//! Account ID: 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +//! SS58 Address: 5Hpm9fq3W3dQgwWpAwDS2ZHKAdnk86QRCu7iX4GnmDxycrte +//! ``` +//! +//! --- +//! ☠️ DO NT RE-USE ANY OF THE SEEDS AND SECRETS FROM THIS PAGE ☠️. +//! +//! You can read more about security and risks in [SECURITY.md](./SECURITY.md) and in the [Polkadot Wiki](https://wiki.polkadot.network/docs/learn-account-generation). +//! +//! --- +//! +//! The output above shows a **secret phrase** (also called **mnemonic phrase**) and the **secret +//! seed** (also called **Private Key**). Those 2 secrets are the pieces of information you MUST +//! keep safe and secret. All the other information below can be derived from those secrets. +//! +//! The output above also show the **public key** and the **Account ID**. Those are the independant +//! from the network where you will use the key. +//! +//! The **SS58 address** (or **Public Address**) of a new account is a reprensentation of the public +//! keys of an account for a given network (for instance Kusama or Polkadot). +//! +//! You can read more about the [SS58 format in the Substrate Docs](https://docs.substrate.io/reference/address-formats/) and see the list of reserved prefixes in the [SS58 Registry](https://github.com/paritytech/ss58-registry). +//! +//! For instance, considering the previous seed +//! `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` the SS58 addresses are: +//! +//! - Polkadot: `16m4J167Mptt8UXL8aGSAi7U2FnPpPxZHPrCgMG9KJzVoFqM` +//! - Kusama: `JLNozAv8QeLSbLFwe2UvWeKKE4yvmDbfGxTuiYkF2BUMx4M` +//! +//! ### Json output +//! +//! `subkey` can calso generate the output as *json*. This is useful for automation. +//! +//! command: +//! +//! ```bash +//! subkey generate --output-type json +//! ``` +//! +//! output: +//! +//! ```json +//! { +//! "accountId": "0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515", +//! "publicKey": "0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515", +//! "secretPhrase": "hotel forest jar hover kite book view eight stuff angle legend defense", +//! "secretSeed": "0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d", +//! "ss58Address": "5Hpm9fq3W3dQgwWpAwDS2ZHKAdnk86QRCu7iX4GnmDxycrte" +//! } +//! ``` +//! +//! So if you only want to get the `secretSeed` for instance, you can use: +//! +//! command: +//! +//! ```bash +//! subkey generate --output-type json | jq -r .secretSeed +//! ``` +//! +//! output: +//! +//! ```text +//! 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +//! ``` +//! +//! ### Additional user-defined password +//! +//! `subkey` supports an additional user-defined secret that will be appended to the seed. Let's see +//! the following example: +//! +//! ```bash +//! subkey generate --password extra_secret +//! ``` +//! +//! output: +//! +//! ```text +//! Secret phrase `soup lyrics media market way crouch elevator put moon useful question wide` is account: +//! Secret seed: 0xe7cfd179d6537a676cb94bac3b5c5c9cb1550e846ac4541040d077dfbac2e7fd +//! Public key (hex): 0xf6a233c3e1de1a2ae0486100b460b3ce3d7231ddfe9dadabbd35ab968c70905d +//! Account ID: 0xf6a233c3e1de1a2ae0486100b460b3ce3d7231ddfe9dadabbd35ab968c70905d +//! SS58 Address: 5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC +//! ``` +//! +//! Using the `inspect` command (see more details below), we see that knowning only the **secret +//! seed** is no longer sufficient to recover the account: +//! +//! ```bash +//! subkey inspect "soup lyrics media market way crouch elevator put moon useful question wide" +//! ``` +//! +//! which recovers the account `5Fe4sqj2K4fRuzEGvToi4KATqZfiDU7TqynjXG6PZE2dxwyh` and not +//! `5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC` as we expected. The additional user-defined +//! **password** (`extra_secret` in our example) is now required to fully recover the account. Let's +//! inspect the the previous mnemonic, this time passing also the required `password` as shown +//! below: +//! +//! ```bash +//! subkey inspect --password extra_secret "soup lyrics media market way crouch elevator put moon useful question wide" +//! ``` +//! +//! This time, we properly recovered `5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC`. +//! +//! ### Inspecting a key +//! +//! If you have *some data* about a key, `subkey inpsect` will help you discover more information +//! about it. +//! +//! If you have **secrets** that you would like to verify for instance, you can use: +//! +//! ```bash +//! subkey inspect < mnemonic | seed > +//! ``` +//! +//! If you have only **public data**, you can see a subset of the information: +//! +//! ```bash +//! subkey inspect --public < pubkey | address > +//! ``` +//! +//! **NOTE**: While you will be able to recover the secret seed from the mnemonic, the opposite is +//! not possible. +//! +//! **NOTE**: For obvious reasons, the **secrets** cannot be recovered from passing **public data** +//! such as `pubkey` or `address` as input. +//! +//! command: +//! +//! ```bash +//! subkey inspect 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +//! ``` +//! +//! output: +//! +//! ```text +//! Secret Key URI `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` is account: +//! Secret seed: 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +//! Public key (hex): 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +//! Account ID: 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +//! SS58 Address: 5Hpm9fq3W3dQgwWpAwDS2ZHKAdnk86QRCu7iX4GnmDxycrte +//! ``` +//! +//! ### Signing +//! +//! `subkey` allows using a **secret key** to sign a random message. The signature can then be +//! verified by anyone using your **public key**: +//! +//! ```bash +//! echo -n | subkey sign --suri +//! ``` +//! +//! example: +//! +//! ```text +//! MESSAGE=hello +//! SURI=0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +//! echo -n $MESSAGE | subkey sign --suri $SURI +//! ``` +//! +//! output: +//! +//! ```text +//! 9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c +//! ``` +//! +//! **NOTE**: Each run of the `sign` command will yield a different output. While each signature is +//! different, they are all valid. +//! +//! ### Verifying a signature +//! +//! Given a message, a signature and an address, `subkey` can verify whether the **message** has +//! been digitally signed by the holder (or one of the holders) of the **private key** for the given +//! **address**: +//! +//! ```bash +//! echo -n | subkey verify
+//! ``` +//! +//! example: +//! +//! ```bash +//! MESSAGE=hello +//! URI=0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +//! SIGNATURE=9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c +//! echo -n $MESSAGE | subkey verify $SIGNATURE $URI +//! ``` +//! +//! output: +//! +//! ```text +//! Signature verifies correctly. +//! ``` +//! +//! A failure looks like: +//! +//! ```text +//! Error: SignatureInvalid +//! ``` +//! +//! ### Using the vanity generator +//! +//! You can use the included vanity generator to find a seed that provides an address which includes +//! the desired pattern. Be warned, depending on your hardware this may take a while. +//! +//! command: +//! +//! ```bash +//! subkey vanity --network polkadot --pattern bob +//! ``` +//! +//! output: +//! +//! ```text +//! Generating key containing pattern 'bob' +//! best: 190 == top: 189 +//! Secret Key URI `0x8c9a73097f235b84021a446bc2826a00c690ea0be3e0d81a84931cb4146d6691` is account: +//! Secret seed: 0x8c9a73097f235b84021a446bc2826a00c690ea0be3e0d81a84931cb4146d6691 +//! Public key (hex): 0x1a8b32e95c1f571118ea0b84801264c3c70f823e320d099e5de31b9b1f18f843 +//! Account ID: 0x1a8b32e95c1f571118ea0b84801264c3c70f823e320d099e5de31b9b1f18f843 +//! SS58 Address: 1bobYxBPjZWRPbVo35aSwci1u5Zmq8P6J2jpa4kkudBZMqE +//! ``` +//! +//! `Bob` now got a nice address starting with their name: +//! 1**bob**YxBPjZWRPbVo35aSwci1u5Zmq8P6J2jpa4kkudBZMqE. +//! +//! **Note**: While `Bob`, having a short name (3 chars), got a result rather quickly, it will take +//! much longer for `Alice` who has a much longer name, thus the chances to generate a random +//! address that contains the chain `alice` will be much smaller. + use clap::Parser; use sc_cli::{ Error, GenerateCmd, GenerateNodeKeyCmd, InspectKeyCmd, InspectNodeKeyCmd, SignCmd, VanityCmd, diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index d004fc1beb097..9d48d2bdf644f 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -15,6 +15,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + +//! Various subcommands that can be included in a substrate-based chain's CLI. + mod build_spec_cmd; mod chain_info_cmd; mod check_block_cmd; diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index 0ef2ee7b1828c..104e8ec8b798e 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -17,6 +17,8 @@ // along with this program. If not, see . //! Substrate CLI library. +//! +//! To see a full list of commands available, see [`commands`]. #![warn(missing_docs)] #![warn(unused_extern_crates)] @@ -26,7 +28,7 @@ use clap::{CommandFactory, FromArgMatches, Parser}; use sc_service::Configuration; pub mod arg_enums; -mod commands; +pub mod commands; mod config; mod error; mod params; diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index bd20ee0c917aa..0290335d43d8b 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -15,19 +15,67 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! I/O host interface for substrate runtime. +//! # Substrate Primitives: IO +//! +//! This crate contains interfaces for the runtime to communicate with the outside world, ergo `io`. +//! In other context, such interfaces are referred to as "**host functions**". +//! +//! Each set of host functions are defined with an instance of the +//! [`sp_runtime_interface::runtime_interface`] macro. +//! +//! Most notably, this crate contains host functions for: +//! +//! - [`hashing`] +//! - [`crypto`] +//! - [`trie`] +//! - [`offchain`] +//! - [`storage`] +//! - [`allocator`] +//! - [`logging`] +//! +//! All of the default host functions provided by this crate, and by default contained in all +//! substrate-based clients are amalgamated in [`SubstrateHostFunctions`]. +//! +//! ## Externalities +//! +//! Host functions go hand in hand with the concept of externalities. Externalities are an +//! environment in which host functions are provided, and thus can be accessed. Some host functions +//! are only accessible in an externality environment that provides it. +//! +//! A typical error for substrate developers is the following: +//! +//! ```should_panic +//! use sp_io::storage::get; +//! # fn main() { +//! let data = get(b"hello world"); +//! # } +//! ``` +//! +//! This code will panic with the following error: +//! +//! ```no_compile +//! thread 'main' panicked at '`get_version_1` called outside of an Externalities-provided environment.' +//! ``` +//! +//! Such error messages should always be interpreted as "code accessing host functions accessed +//! outside of externalities". +//! +//! An externality is any type that implements [`sp_externalities::Externalities`]. A simple example +//! of which is [`TestExternalities`], which is commonly used in tests and is exported from this +//! crate. +//! +//! ``` +//! use sp_io::{storage::get, TestExternalities}; +//! # fn main() { +//! TestExternalities::default().execute_with(|| { +//! let data = get(b"hello world"); +//! }); +//! # } +//! ``` #![warn(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(enable_alloc_error_handler, feature(alloc_error_handler))] -#![cfg_attr( - feature = "std", - doc = "Substrate runtime standard library as compiled when linked with Rust's standard library." -)] -#![cfg_attr( - not(feature = "std"), - doc = "Substrate's runtime standard library as compiled without Rust's standard library." -)] use sp_std::vec::Vec; diff --git a/scripts/ci/deny.toml b/scripts/ci/deny.toml index 408a9e55bc40b..5297d07143c22 100644 --- a/scripts/ci/deny.toml +++ b/scripts/ci/deny.toml @@ -96,6 +96,7 @@ exceptions = [ { allow = ["GPL-3.0 WITH Classpath-exception-2.0"], name = "sc-transaction-pool" }, { allow = ["GPL-3.0 WITH Classpath-exception-2.0"], name = "sc-transaction-pool-api" }, { allow = ["GPL-3.0 WITH Classpath-exception-2.0"], name = "subkey" }, + { allow = ["GPL-3.0 WITH Classpath-exception-2.0"], name = "substrate" }, ] # Some crates don't have (easily) machine readable licensing information, diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 0c04abfd5aa30..4669f705faceb 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -77,8 +77,8 @@ build-linux-substrate: - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" script: - rusty-cachier snapshot create - - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release --verbose - - mv $CARGO_TARGET_DIR/release/substrate ./artifacts/substrate/. + - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p node-cli --verbose + - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 0a73e0b4114bb..69d53012f79e7 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -162,7 +162,7 @@ cargo-check-try-runtime-and-experimental: - .test-refs script: - rusty-cachier snapshot create - - time cargo check --locked --features try-runtime,experimental + - time cargo check --workspace --locked --features try-runtime,experimental - rusty-cachier cache upload test-deterministic-wasm: @@ -305,7 +305,7 @@ quick-benchmarks: WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" script: - rusty-cachier snapshot create - - time cargo run --locked --release --features runtime-benchmarks -- benchmark pallet --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - time cargo run --locked --release -p node-cli --features runtime-benchmarks -- benchmark pallet --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - rusty-cachier cache upload test-frame-examples-compile-to-wasm: diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000000..eb313ddadf76d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,262 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! # Substrate +//! +//! Substrate is a Rust framework for building blockchains in a modular and extensible way. While in +//! itself un-opinionated, it is the main engine behind the Polkadot ecosystem. +//! +//! [![github]](https://github.com/paritytech/substrate/) - [![polkadot]](https://polkadot.network) +//! +//! This crate in itself does not contain any code and is just meant ot be a documentation hub for +//! substrate-based crates. +//! +//! ## Overview +//! +//! Substrate approaches blockchain development with an acknowledgement of a few self-evident +//! truths: +//! +//! 1. Society and technology evolves. +//! 2. Humans are fallible. +//! +//! This, specifically, makes the task of designing a correct, safe and long-lasting blockchain +//! system hard. +//! +//! Nonetheless, in order to achieve this goal, substrate embraces the following: +//! +//! 1. Use of **Rust** as a modern, and safe programming language, which limits human error through +//! various means, most notably memory safety. +//! 2. Substrate is written from the ground-up with a generic, modular and extensible design. This +//! ensures that software components can be easily swapped and upgraded. Examples of this is +//! multiple consensus mechanisms provided by Substrate, as listed below. +//! 3. Lastly, the final blockchain system created with the above properties needs to be +//! upgradeable. In order to achieve this, Substrate is designed as a meta-protocol, whereby the +//! application logic of the blockchain (called "Runtime") is encoded as a Wasm blob, and is +//! stored onchain. The rest of the system (called "Client") acts as the executor of the Wasm +//! blob. +//! +//! In essence, the meta-protocol of all Substrate based chains is the "Runtime as Wasm blob" +//! accord. This enables the Runtime to become inherently upgradeable (without forks). The upgrade +//! is merely a matter of the Wasm blob being changed in the chain state, which is, in principle, +//! same as updating an account's balance. +//! +//! To learn more about the substrate architecture using some visuals, see [`substrate_diagram`]. +//! +//! `FRAME`, Substrate's default runtime development library takes the above even further by +//! embracing a declarative programming model whereby correctness is enhanced and the system is +//! highly configurable through parameterization. +//! +//! All in all, this design enables all substrate-based chains to achieve forkless, self-enacting +//! upgrades out of the box. Combined with governance abilities that are shipped with `FRAME`, this +//! enables a chain to survive the test of time. +//! +//! ## How to Get Stared +//! +//! Most developers want to leave the client side code as-is, and focus on the runtime. To do so, +//! look into the [`frame_support`] crate, which is the entry point crate into runtime development +//! with FRAME. +//! +//! > Side note, it is entirely possible to craft a substrate-based runtime without FRAME, an +//! > example of which can be found [here](https://github.com/JoshOrndorff/frameless-node-template). +//! +//! In more broad terms, the following avenues exist into developing with substrate: +//! +//! * **Templates**: A number of substrate-based templates exist and they can be used for various +//! purposes, with zero to little additional code needed. All of these templates contain runtimes +//! that are highly configurable and are likely suitable for basic needs. +//! * `FRAME`: If need, one can customize that runtime even further, by using `FRAME` and developing +//! custom modules. +//! * **Core**: To the contrary, some developers may want to customize the client side software to +//! achieve novel goals such as a new consensus engine, or a new database backend. While +//! Substrate's main configurability is in the runtime, the client is also highly generic and can +//! be customized to a great extent. +//! +//! ## Structure +//! +//! Substrate is a massive cargo workspace with hundreds of crates, therefore it is useful to know +//! how to navigate its crates. +//! +//! In broad terms, it is divided into three categories: +//! +//! * `sc-*` (short for *substrate-client*) crates, located under `./client` folder. These are all +//! the client crates. Notable examples are crates such as [`sc-network`], various consensus +//! crates, [`sc-rpc-api`] and [`sc-client-db`], all of which are expected to reside in the client +//! side. +//! * `sp-*` (short for *substrate-primitives*) crates, located under `./primitives` folder. These +//! are the traits that glue the client and runtime together, but are not opinionated about what +//! framework is using for building the runtime. Notable examples are [`sp-api`] and [`sp-io`], +//! which form the communication bridge between the client and runtime, as explained in +//! [`substrate_diagram`]. +//! * `pallet-*` and `frame-*` crates, located under `./frame` folder. These are the crates related +//! to FRAME. See [`frame_support`] for more information. +//! +//! ### Binaries +//! +//! Multiple binaries are shipped with substrate, the most important of which are located in the +//! `./bin` folder. +//! +//! * [`node`] is an extensive substrate node that contains the superset of all runtime and client +//! side features. The corresponding runtime, called [`kitchensink_runtime`] contains all of the +//! modules that are provided with `FRAME`. This node and runtime is only used for testing. +//! * [`node-template`]: a template node that contains a minimal set of features and can act as a +//! starting point of a project. +//! * [`subkey`]: Substrate's key management utility. +//! * [`chain-spec-builder`]: Substrate's utility to build *chain specifications*. Such +//! specifications can then be used with `--chain` argument of a typical substrate node's CLI. +//! +//! ## Parachain? +//! +//! As noted above, Substrate is the main engine behind the Polkadot ecosystem. One of the ways +//! through which Polkadot can be utilized is by building "parachains", blockchains that are +//! connected to Polkadot's shared security. +//! +//! To build a parachain, one could use [`Cumulus`](https://github.com/paritytech/cumulus/), the +//! library on top of Substrate, empowering any substrate-based chain to be a Polkadot parachain. +//! +//! ## Where To Go Next? +//! +//! Additional noteworthy crates within substrate: +//! +//! - RPC APIs of a Substrate node: [`sc-rpc-api`] +//! - CLI Options of a Substrate node: [`sc-cli`] +//! - All of the consensus related crates provided by Substrate: +//! - [`sc-consensus-aura`] +//! - [`sc-consensus-babe`] +//! - [`sc-consensus-grandpa`] +//! - [`sc-consensus-beefy`] +//! - [`sc-consensus-manual-seal`] +//! - [`sc-consensus-pow`] +//! +//! Additional noteworthy external resources: +//! +//! - [Substrate Developer Hub](https://substrate.dev) +//! - [Parity Tech's Documentation Hub](https://paritytech.github.io/) +//! - [Frontier: Substrate's Ethereum Compatibility Library](https://paritytech.github.io/frontier/) +//! - [Polkadot Wiki](https://wiki.polkadot.network/en/) +//! +//! Notable upstream crates: +//! +//! - [`parity-db`](https://github.com/paritytech/parity-db) +//! - [`trie`](https://github.com/paritytech/trie) +//! - [`parity-common`](https://github.com/paritytech/parity-common) +//! +//! Templates: +//! +//! - classic [`substrate-node-template`](https://github.com/substrate-developer-hub/substrate-node-template) +//! - classic [cumulus-parachain-template](https://github.com/substrate-developer-hub/substrate-parachain-template) +//! - [`extended-parachain-template`](https://github.com/paritytech/extended-parachain-template) +//! - [`frontier-parachain-template`](https://github.com/paritytech/frontier-parachain-template) +//! +//! [polkadot]: +//! https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white +//! [github]: +//! https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [`sp-io`]: ../sp_io/index.html +//! [`sp-api`]: ../sp_api/index.html +//! [`sp-api`]: ../sp_api/index.html +//! [`sc-client-db`]: ../sc_client_db/index.html +//! [`sc-network`]: ../sc_network/index.html +//! [`sc-rpc-api`]: ../sc_rpc_api/index.html +//! [`sc-cli`]: ../sc_cli/index.html +//! [`sc-consensus-aura`]: ../sc_consensus_aura/index.html +//! [`sc-consensus-babe`]: ../sc_consensus_babe/index.html +//! [`sc-consensus-grandpa`]: ../sc_consensus_grandpa/index.html +//! [`sc-consensus-beefy`]: ../sc_consensus_beefy/index.html +//! [`sc-consensus-manual-seal`]: ../sc_consensus_manual_seal/index.html +//! [`sc-consensus-pow`]: ../sc_consensus_pow/index.html +//! [`node`]: ../node_cli/index.html +//! [`node-template`]: ../node_template/index.html +//! [`kitchensink_runtime`]: ../kitchensink_runtime/index.html +//! [`subkey`]: ..//subkey/index.html +//! [`chian-spec-builder`]: ../chain_spec_builder/index.html + +#![deny(rustdoc::broken_intra_doc_links)] +#![deny(rustdoc::private_intra_doc_links)] + +#[cfg_attr(doc, aquamarine::aquamarine)] +/// In this module, we explore substrate at a more depth. First, let's establish substrate being +/// divided into a client and runtime. +/// +/// ```mermaid +/// graph TB +/// subgraph Substrate +/// direction LR +/// subgraph Client +/// end +/// subgraph Runtime +/// end +/// end +/// ``` +/// +/// The client and the runtime of course need to communicate. This is done through two concepts: +/// +/// 1. Host functions: a way for the (Wasm) runtime to talk to the client. All host functions are +/// defined in [`sp-io`]. For example, [`sp-io::storage`] are the set of host functions that +/// allow the runtime to read and write data to the on-chain state. +/// 2. Runtime APIs: a way for the client to talk to the Wasm runtime. Runtime APIs are defined +/// using macros and utilities in [`sp-api`]. For example, [`sp-api::Core`] is the most basic +/// runtime API that any blockchain must implement in order to be able to (re) execute blocks. +/// +/// ```mermaid +/// graph TB +/// subgraph Substrate +/// direction LR +/// subgraph Client +/// end +/// subgraph Runtime +/// end +/// Client --runtime-api--> Runtime +/// Runtime --host-functions--> Client +/// end +/// ``` +/// +/// Finally, let's expand the diagram a bit further and look at the internals of each component: +/// +/// ```mermaid +/// graph TB +/// subgraph Substrate +/// direction LR +/// subgraph Client +/// Database +/// Networking +/// Consensus +/// end +/// subgraph Runtime +/// subgraph FRAME +/// direction LR +/// Governance +/// Currency +/// Staking +/// Identity +/// end +/// end +/// Client --runtime-api--> Runtime +/// Runtime --host-functions--> Client +/// end +/// ``` +/// +/// As noted the runtime contains all of the application specific logic of the blockchain. This is +/// usually written with `FRAME`. The client, on the other hand, contains reusable and generic +/// components that are not specific to one single blockchain, such as networking, database, and the +/// consensus engine. +/// +/// [`sp-io`]: ../../sp_io/index.html +/// [`sp-api`]: ../../sp_api/index.html +/// [`sp-io::storage`]: ../../sp_io/storage/index.html +/// [`sp-api::Core`]: ../../sp_api/trait.Core.html +pub mod substrate_diagram {} diff --git a/test-utils/cli/build.rs b/test-utils/cli/build.rs new file mode 100644 index 0000000000000..a68cb706e8fbd --- /dev/null +++ b/test-utils/cli/build.rs @@ -0,0 +1,25 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::env; + +fn main() { + if let Ok(profile) = env::var("PROFILE") { + println!("cargo:rustc-cfg=build_type=\"{}\"", profile); + } +} diff --git a/test-utils/cli/src/lib.rs b/test-utils/cli/src/lib.rs index b33c0e9b77c71..7e704b70f8096 100644 --- a/test-utils/cli/src/lib.rs +++ b/test-utils/cli/src/lib.rs @@ -27,7 +27,6 @@ use node_primitives::{Hash, Header}; use regex::Regex; use sp_rpc::{list::ListOrValue, number::NumberOrHex}; use std::{ - env, io::{BufRead, BufReader, Read}, ops::{Deref, DerefMut}, path::{Path, PathBuf}, @@ -62,7 +61,7 @@ use tokio::io::{AsyncBufReadExt, AsyncRead}; /// /// [`Child`]: std::process::Child pub fn start_node() -> Child { - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(&["--dev", "--tmp", "--rpc-port=45789", "--no-hardware-benchmarks"]) @@ -99,15 +98,19 @@ pub fn start_node() -> Child { /// build_substrate(&["--features=try-runtime"]); /// ``` pub fn build_substrate(args: &[&str]) { + let is_release_build = !cfg!(build_type = "debug"); + // Get the root workspace directory from the CARGO_MANIFEST_DIR environment variable - let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); - let root_dir = std::path::Path::new(&manifest_dir) - .parent() - .expect("Failed to find root workspace directory"); - let output = Command::new("cargo") - .arg("build") + let mut cmd = Command::new("cargo"); + + cmd.arg("build").arg("-p=node-cli"); + + if is_release_build { + cmd.arg("--release"); + } + + let output = cmd .args(args) - .current_dir(root_dir) .output() .expect(format!("Failed to execute 'cargo b' with args {:?}'", args).as_str()); @@ -196,7 +199,7 @@ pub async fn wait_n_finalized_blocks(n: usize, url: &str) { /// Run the node for a while (3 blocks) pub async fn run_node_for_a_while(base_path: &Path, args: &[&str]) { run_with_timeout(Duration::from_secs(60 * 10), async move { - let mut cmd = Command::new(cargo_bin("substrate")) + let mut cmd = Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(args) diff --git a/utils/frame/try-runtime/cli/tests/create_snapshot.rs b/utils/frame/try-runtime/cli/tests/create_snapshot.rs index 59a36fd702d72..c524e5af48ad3 100644 --- a/utils/frame/try-runtime/cli/tests/create_snapshot.rs +++ b/utils/frame/try-runtime/cli/tests/create_snapshot.rs @@ -47,7 +47,7 @@ async fn create_snapshot_works() { common::run_with_timeout(Duration::from_secs(60), async move { fn create_snapshot(ws_url: &str, snap_file: &PathBuf, at: Hash) -> Child { - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(&["try-runtime", "--runtime=existing"]) diff --git a/utils/frame/try-runtime/cli/tests/execute_block.rs b/utils/frame/try-runtime/cli/tests/execute_block.rs index 025b753be7784..9ad49e0ceaf9a 100644 --- a/utils/frame/try-runtime/cli/tests/execute_block.rs +++ b/utils/frame/try-runtime/cli/tests/execute_block.rs @@ -33,7 +33,7 @@ async fn block_execution_works() { common::run_with_timeout(Duration::from_secs(60), async move { fn execute_block(ws_url: &str, at: Hash) -> Child { - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(&["try-runtime", "--runtime=existing"]) diff --git a/utils/frame/try-runtime/cli/tests/follow_chain.rs b/utils/frame/try-runtime/cli/tests/follow_chain.rs index 63b8b4c386b0d..d7ff7b09cbb72 100644 --- a/utils/frame/try-runtime/cli/tests/follow_chain.rs +++ b/utils/frame/try-runtime/cli/tests/follow_chain.rs @@ -35,7 +35,7 @@ async fn follow_chain_works() { common::run_with_timeout(Duration::from_secs(60), async move { fn start_follow(ws_url: &str) -> Child { - Command::new(cargo_bin("substrate")) + Command::new(cargo_bin("substrate-node")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) .args(&["try-runtime", "--runtime=existing"])