Improve stake-meta-generator usability (#134)

dev/Dockerfile

@@ -34,4 +34,4 @@ RUN mkdir -p docker-output
 RUN --mount=type=cache,mode=0777,target=/solana/target \
     --mount=type=cache,mode=0777,target=/usr/local/cargo/registry \
     --mount=type=cache,mode=0777,target=/usr/local/cargo/git \
-    cargo build --verbose --release && cp target/release/solana* ./docker-output
+    cargo build --verbose --release && cp target/release/solana* ./docker-output

ledger/src/bank_forks_utils.rs

@@ -180,7 +180,7 @@ pub fn load_bank_forks(
 }
 
 #[allow(clippy::too_many_arguments)]
-fn bank_forks_from_snapshot(
+pub fn bank_forks_from_snapshot(
     genesis_config: &GenesisConfig,
     account_paths: Vec<PathBuf>,
     shrink_paths: Option<Vec<PathBuf>>,

tip-distributor/Cargo.toml

@@ -30,9 +30,9 @@ tip-distribution = { path = "../jito-programs/tip-payment/programs/tip-distribut
 tokio = { version = "1.12.0", features = ["rt-multi-thread", "macros", "sync", "time", "full"] }
 
 [[bin]]
-name = "stake-meta-generator"
+name = "solana-stake-meta-generator"
 path = "src/bin/stake-meta-generator.rs"
 
 [[bin]]
-name = "merkle-root-generator"
+name = "solana-merkle-root-generator"
 path = "src/bin/merkle-root-generator.rs"

tip-distributor/README.md

@@ -36,7 +36,7 @@ In order to use this library as the merkle root creator one must follow the foll
 2. Download the snapshot onto your worker machine (where this script will run).
 3. Run `solana-ledger-tool -l ${PATH_TO_LEDGER} create-snapshot ${YOUR_SLOT} ${WHERE_TO_CREATE_SNAPSHOT}`
    1. The snapshot created at `${WHERE_TO_CREATE_SNAPSHOT}` will have the highest slot of `${YOUR_SLOT}`, assuming you downloaded the correct snapshot.
-4. Run `stake-meta-generator --ledger-path ${WHERE_TO_CREATE_SNAPSHOT} --tip-distribution-program-id ${PUBKEY} --out-path ${JSON_OUT_PATH} --snapshot-bank-hash ${HASH} --snapshot-slot ${SLOT} --rpc-url ${URL}`
+4. Run `stake-meta-generator --ledger-path ${WHERE_TO_CREATE_SNAPSHOT} --tip-distribution-program-id ${PUBKEY} --out-path ${JSON_OUT_PATH} --snapshot-slot ${SLOT} --rpc-url ${URL}`
    1. Note: `${WHERE_TO_CREATE_SNAPSHOT}` must be the same in steps 3 & 4.
 5. Run `merkle-root-generator --path-to-my-keypair ${KEYPAIR_PATH} --stake-meta-coll-path ${STAKE_META_COLLECTION_JSON} --rpc-url ${URL} --upload-roots ${BOOL} --force-upload-root ${BOOL}`
 

tip-distributor/src/bin/stake-meta-generator.rs

@@ -30,10 +30,6 @@ struct Args {
     #[clap(long, env)]
     out_path: String,
 
-    /// The expected base58 encoded snapshot bank hash.
-    #[clap(long, env)]
-    snapshot_bank_hash: String,
-
     /// The expected snapshot slot.
     #[clap(long, env)]
     snapshot_slot: Slot,
@@ -62,7 +58,6 @@ fn main() {
 
     run_workflow(
         &args.ledger_path,
-        args.snapshot_bank_hash,
         args.snapshot_slot,
         args.tip_distribution_program_id,
         args.out_path,

tip-distributor/src/stake_meta_generator_workflow.rs

@@ -9,33 +9,29 @@ use {
     solana_client::{client_error::ClientError, rpc_client::RpcClient},
     solana_ledger::{
         bank_forks_utils,
-        blockstore::{Blockstore, BlockstoreError},
-        blockstore_options::{AccessType, BlockstoreOptions, BlockstoreRecoveryMode},
+        blockstore::BlockstoreError,
         blockstore_processor::{BlockstoreProcessorError, ProcessOptions},
     },
     solana_runtime::{
         bank::Bank,
-        bank_forks::BankForks,
         hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
         snapshot_config::SnapshotConfig,
-        snapshot_utils,
         stakes::StakeAccount,
         vote_account::VoteAccount,
     },
     solana_sdk::{
         account::ReadableAccount,
         bs58,
         clock::{Epoch, Slot},
-        genesis_config::GenesisConfig,
         pubkey::Pubkey,
     },
     std::{
         collections::HashMap,
         fmt::{Debug, Display, Formatter},
-        fs::{self, File},
+        fs::File,
         io::{BufWriter, Write},
-        path::Path,
-        sync::{Arc, RwLock},
+        path::{Path, PathBuf},
+        sync::Arc,
     },
     thiserror::Error as ThisError,
 };
@@ -75,14 +71,13 @@ impl Display for Error {
 /// to a JSON file.
 pub fn run_workflow(
     ledger_path: &Path,
-    snapshot_bank_hash: String,
     snapshot_slot: Slot,
     tip_distribution_program_id: Pubkey,
     out_path: String,
     rpc_client: RpcClient,
 ) -> Result<(), Error> {
     info!("Creating bank from ledger path...");
-    let bank = create_bank_from_snapshot(ledger_path, snapshot_bank_hash, snapshot_slot)?;
+    let bank = create_bank_from_snapshot(ledger_path, snapshot_slot)?;
 
     info!("Generating stake_meta_collection object...");
     let stake_meta_coll =
@@ -94,25 +89,26 @@ pub fn run_workflow(
     Ok(())
 }
 
-fn create_bank_from_snapshot(
-    ledger_path: &Path,
-    expected_bank_hash: String,
-    snapshot_slot: Slot,
-) -> Result<Arc<Bank>, Error> {
+fn create_bank_from_snapshot(ledger_path: &Path, snapshot_slot: Slot) -> Result<Arc<Bank>, Error> {
     let genesis_config = open_genesis_config(ledger_path, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE);
-    let blockstore = open_blockstore(ledger_path, AccessType::Secondary, None)?;
-
-    let bank_forks = load_bank_forks(&blockstore, &genesis_config, snapshot_slot)?;
+    let snapshot_config = SnapshotConfig {
+        full_snapshot_archive_interval_slots: Slot::MAX,
+        incremental_snapshot_archive_interval_slots: Slot::MAX,
+        full_snapshot_archives_dir: PathBuf::from(ledger_path),
+        incremental_snapshot_archives_dir: PathBuf::from(ledger_path),
+        bank_snapshots_dir: PathBuf::from(ledger_path),
+        ..SnapshotConfig::default()
+    };
+    let (bank_forks, _snapshot_hashes) = bank_forks_utils::bank_forks_from_snapshot(
+        &genesis_config,
+        vec![PathBuf::from(ledger_path).join(Path::new("stake-meta.accounts"))],
+        None,
+        &snapshot_config,
+        &ProcessOptions::default(),
+        None,
+    );
 
     let working_bank = bank_forks.read().unwrap().working_bank();
-    assert_eq!(
-        working_bank.hash().to_string(),
-        expected_bank_hash,
-        "expected working bank hash {}, found {} at slot {}",
-        expected_bank_hash,
-        working_bank.hash(),
-        snapshot_slot
-    );
     assert_eq!(
         working_bank.slot(),
         snapshot_slot,
@@ -124,105 +120,6 @@ fn create_bank_from_snapshot(
     Ok(working_bank)
 }
 
-fn load_bank_forks(
-    blockstore: &Blockstore,
-    genesis_config: &GenesisConfig,
-    snapshot_slot: Slot,
-) -> Result<Arc<RwLock<BankForks>>, Error> {
-    let bank_snapshots_dir = blockstore
-        .ledger_path()
-        .join(if blockstore.is_primary_access() {
-            "snapshot"
-        } else {
-            "snapshot.ledger-tool"
-        });
-
-    let full_snapshot_archives_dir = blockstore.ledger_path().to_path_buf();
-    let full_snapshot_slot = snapshot_utils::get_highest_full_snapshot_archive_slot(
-        &full_snapshot_archives_dir,
-        Some(snapshot_slot),
-    )
-    .ok_or(Error::SnapshotSlotNotFound)?;
-    assert_eq!(full_snapshot_slot, snapshot_slot);
-    if full_snapshot_slot != snapshot_slot {
-        assert_eq!(full_snapshot_slot, snapshot_slot, "The expected snapshot was not found, try moving your snapshot to a different directory than the ledger directory if you haven't already. [actual_highest_snapshot={}, expected_highest_snapshot={}]", full_snapshot_slot, snapshot_slot);
-    }
-
-    let incremental_snapshot_archives_dir = blockstore.ledger_path().to_path_buf();
-
-    let snapshot_config = SnapshotConfig {
-        full_snapshot_archive_interval_slots: Slot::MAX,
-        incremental_snapshot_archive_interval_slots: Slot::MAX,
-        full_snapshot_archives_dir,
-        incremental_snapshot_archives_dir,
-        bank_snapshots_dir,
-        ..SnapshotConfig::default()
-    };
-
-    let account_paths = if blockstore.is_primary_access() {
-        vec![blockstore.ledger_path().join("accounts")]
-    } else {
-        let non_primary_accounts_path = blockstore.ledger_path().join("accounts.ledger-tool");
-        info!(
-            "Default accounts path is switched aligning with Blockstore's secondary access: {:?}",
-            non_primary_accounts_path
-        );
-
-        if non_primary_accounts_path.exists() {
-            info!("Clearing {:?}", non_primary_accounts_path);
-            if let Err(err) = fs::remove_dir_all(&non_primary_accounts_path) {
-                error!(
-                    "error deleting accounts path {:?}: {}",
-                    non_primary_accounts_path, err
-                );
-                return Err(err.into());
-            }
-        }
-
-        vec![non_primary_accounts_path]
-    };
-
-    Ok(bank_forks_utils::load(
-        genesis_config,
-        blockstore,
-        account_paths,
-        None,
-        Some(&snapshot_config),
-        ProcessOptions {
-            new_hard_forks: None,
-            halt_at_slot: Some(snapshot_slot),
-            poh_verify: false,
-            ..ProcessOptions::default()
-        },
-        None,
-        None,
-        None,
-    )
-    .map(|(bank_forks, ..)| bank_forks)?)
-}
-
-fn open_blockstore(
-    ledger_path: &Path,
-    access_type: AccessType,
-    wal_recovery_mode: Option<BlockstoreRecoveryMode>,
-) -> Result<Blockstore, Error> {
-    match Blockstore::open_with_options(
-        ledger_path,
-        BlockstoreOptions {
-            access_type,
-            recovery_mode: wal_recovery_mode,
-            enforce_ulimit_nofile: true,
-            ..BlockstoreOptions::default()
-        },
-    ) {
-        Ok(blockstore) => Ok(blockstore),
-        Err(e) => {
-            error!("Failed to open ledger at {:?}: {:?}", ledger_path, e);
-            Err(e.into())
-        }
-    }
-}
-
 fn write_to_json_file(
     stake_meta_coll: &StakeMetaCollection,
     out_path: String,