Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Revert "Add run parent directory for accounts files (#29794)" #29899

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 1 addition & 6 deletions core/src/snapshot_packager_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,7 @@ mod tests {
snapshot_hash::SnapshotHash,
snapshot_package::{SnapshotPackage, SnapshotType},
snapshot_utils::{
self, create_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion,
SNAPSHOT_STATUS_CACHE_FILENAME,
self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILENAME,
},
},
solana_sdk::hash::Hash,
Expand Down Expand Up @@ -268,10 +267,6 @@ mod tests {

fn create_and_verify_snapshot(temp_dir: &Path) {
let accounts_dir = temp_dir.join("accounts");
let accounts_dir = create_accounts_run_and_snapshot_dirs(accounts_dir)
.unwrap()
.0;

let snapshots_dir = temp_dir.join("snapshots");
let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives");
let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives");
Expand Down
5 changes: 2 additions & 3 deletions core/tests/epoch_accounts_hash.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#![allow(clippy::integer_arithmetic)]
use {
crate::snapshot_utils::create_tmp_accounts_dir_for_tests,
log::*,
solana_core::{
accounts_hash_verifier::AccountsHashVerifier,
Expand Down Expand Up @@ -443,9 +442,9 @@ fn test_snapshots_have_expected_epoch_accounts_hash() {
std::thread::sleep(Duration::from_secs(1));
};

let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests();
let accounts_dir = TempDir::new().unwrap();
let deserialized_bank = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir.as_path().to_path_buf()],
&[accounts_dir.path().to_path_buf()],
&snapshot_config.bank_snapshots_dir,
&full_snapshot_archive_info,
None,
Expand Down
23 changes: 10 additions & 13 deletions core/tests/snapshots.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#![allow(clippy::integer_arithmetic)]

use {
crate::snapshot_utils::create_tmp_accounts_dir_for_tests,
bincode::serialize_into,
crossbeam_channel::unbounded,
fs_extra::dir::CopyOptions,
Expand Down Expand Up @@ -75,8 +74,7 @@ struct SnapshotTestConfig {
incremental_snapshot_archives_dir: TempDir,
full_snapshot_archives_dir: TempDir,
bank_snapshots_dir: TempDir,
accounts_dir: PathBuf,
_accounts_tmp_dir: TempDir,
accounts_dir: TempDir,
}

impl SnapshotTestConfig {
Expand All @@ -87,7 +85,7 @@ impl SnapshotTestConfig {
full_snapshot_archive_interval_slots: Slot,
incremental_snapshot_archive_interval_slots: Slot,
) -> SnapshotTestConfig {
let (_accounts_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests();
let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap();
let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
Expand All @@ -104,7 +102,7 @@ impl SnapshotTestConfig {
let bank0 = Bank::new_with_paths_for_tests(
&genesis_config_info.genesis_config,
Arc::<RuntimeConfig>::default(),
vec![accounts_dir.clone()],
vec![accounts_dir.path().to_path_buf()],
AccountSecondaryIndexes::default(),
accounts_db::AccountShrinkThreshold::default(),
);
Expand Down Expand Up @@ -133,7 +131,6 @@ impl SnapshotTestConfig {
full_snapshot_archives_dir,
bank_snapshots_dir,
accounts_dir,
_accounts_tmp_dir,
}
}
}
Expand Down Expand Up @@ -299,8 +296,8 @@ fn run_bank_forks_snapshot_n<F>(
.unwrap();

// Restore bank from snapshot
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests();
let account_paths = &[temporary_accounts_dir];
let temporary_accounts_dir = TempDir::new().unwrap();
let account_paths = &[temporary_accounts_dir.path().to_path_buf()];
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);

Expand Down Expand Up @@ -729,7 +726,7 @@ fn test_bank_forks_incremental_snapshot(
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
);
trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}",
snapshot_test_config.accounts_dir.display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());
snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());

let bank_forks = &mut snapshot_test_config.bank_forks;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
Expand Down Expand Up @@ -824,11 +821,11 @@ fn test_bank_forks_incremental_snapshot(
// Accounts directory needs to be separate from the active accounts directory
// so that dropping append vecs in the active accounts directory doesn't
// delete the unpacked appendvecs in the snapshot
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests();
let temporary_accounts_dir = TempDir::new().unwrap();
restore_from_snapshots_and_check_banks_are_equal(
&bank,
&snapshot_test_config.snapshot_config,
temporary_accounts_dir,
temporary_accounts_dir.path().to_path_buf(),
&snapshot_test_config.genesis_config_info.genesis_config,
)
.unwrap();
Expand Down Expand Up @@ -1123,7 +1120,7 @@ fn test_snapshots_with_background_services(
}

// Load the snapshot and ensure it matches what's in BankForks
let (_tmp_dir, temporary_accounts_dir) = create_tmp_accounts_dir_for_tests();
let temporary_accounts_dir = TempDir::new().unwrap();
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_test_config.snapshot_config.bank_snapshots_dir,
&snapshot_test_config
Expand All @@ -1132,7 +1129,7 @@ fn test_snapshots_with_background_services(
&snapshot_test_config
.snapshot_config
.incremental_snapshot_archives_dir,
&[temporary_accounts_dir],
&[temporary_accounts_dir.as_ref().to_path_buf()],
&snapshot_test_config.genesis_config_info.genesis_config,
&RuntimeConfig::default(),
None,
Expand Down
20 changes: 2 additions & 18 deletions ledger-tool/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ use {
snapshot_hash::StartingSnapshotHashes,
snapshot_minimizer::SnapshotMinimizer,
snapshot_utils::{
self, create_accounts_run_and_snapshot_dirs, move_and_async_delete_path, ArchiveFormat,
SnapshotVersion, DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
self, move_and_async_delete_path, ArchiveFormat, SnapshotVersion,
DEFAULT_ARCHIVE_COMPRESSION, SUPPORTED_ARCHIVE_COMPRESSION,
},
},
solana_sdk::{
Expand Down Expand Up @@ -1099,22 +1099,6 @@ fn load_bank_forks(
);
vec![non_primary_accounts_path]
};

// For all account_paths, set up the run/ and snapshot/ sub directories.
let account_run_paths: Vec<PathBuf> = account_paths.into_iter().map(
|account_path| {
match create_accounts_run_and_snapshot_dirs(&account_path) {
Ok((account_run_path, _account_snapshot_path)) => account_run_path,
Err(err) => {
eprintln!("Unable to create account run and snapshot sub directories: {}, err: {err:?}", account_path.display());
exit(1);
}
}
}).collect();

// From now on, use run/ paths in the same way as the previous account_paths.
let account_paths = account_run_paths;

info!("Cleaning contents of account paths: {:?}", account_paths);
let mut measure = Measure::start("clean_accounts_paths");
account_paths.iter().for_each(|path| {
Expand Down
7 changes: 1 addition & 6 deletions local-cluster/src/local_cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ use {
ValidatorVoteKeypairs,
},
snapshot_config::SnapshotConfig,
snapshot_utils::create_accounts_run_and_snapshot_dirs,
},
solana_sdk::{
account::{Account, AccountSharedData},
Expand Down Expand Up @@ -148,11 +147,7 @@ impl LocalCluster {
config: &mut ValidatorConfig,
ledger_path: &Path,
) {
config.account_paths = vec![
create_accounts_run_and_snapshot_dirs(ledger_path.join("accounts"))
.unwrap()
.0,
];
config.account_paths = vec![ledger_path.join("accounts")];
config.tower_storage = Arc::new(FileTowerStorage::new(ledger_path.to_path_buf()));

let snapshot_config = &mut config.snapshot_config;
Expand Down
6 changes: 2 additions & 4 deletions local-cluster/tests/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ use {
validator_configs::*,
},
solana_rpc_client::rpc_client::RpcClient,
solana_runtime::{
snapshot_config::SnapshotConfig, snapshot_utils::create_accounts_run_and_snapshot_dirs,
},
solana_runtime::snapshot_config::SnapshotConfig,
solana_sdk::{
account::AccountSharedData,
clock::{self, Slot, DEFAULT_MS_PER_SLOT, DEFAULT_TICKS_PER_SLOT},
Expand Down Expand Up @@ -438,7 +436,7 @@ pub fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<Pa
.collect();
let account_storage_paths: Vec<_> = account_storage_dirs
.iter()
.map(|a| create_accounts_run_and_snapshot_dirs(a.path()).unwrap().0)
.map(|a| a.path().to_path_buf())
.collect();
(account_storage_dirs, account_storage_paths)
}
Expand Down
10 changes: 2 additions & 8 deletions local-cluster/tests/local_cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ use {
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_config::SnapshotConfig,
snapshot_package::SnapshotType,
snapshot_utils::{
self, create_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion,
},
snapshot_utils::{self, ArchiveFormat, SnapshotVersion},
},
solana_sdk::{
account::AccountSharedData,
Expand Down Expand Up @@ -2154,11 +2152,7 @@ fn create_snapshot_to_hard_fork(
let (bank_forks, ..) = bank_forks_utils::load(
&genesis_config,
blockstore,
vec![
create_accounts_run_and_snapshot_dirs(ledger_path.join("accounts"))
.unwrap()
.0,
],
vec![ledger_path.join("accounts")],
None,
snapshot_config.as_ref(),
process_options,
Expand Down
10 changes: 1 addition & 9 deletions runtime/src/accounts_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ use {
read_only_accounts_cache::ReadOnlyAccountsCache,
rent_collector::RentCollector,
rent_paying_accounts_by_partition::RentPayingAccountsByPartition,
snapshot_utils::create_accounts_run_and_snapshot_dirs,
sorted_storages::SortedStorages,
storable_accounts::StorableAccounts,
verify_accounts_hash_in_background::VerifyAccountsHashInBackground,
Expand Down Expand Up @@ -1114,14 +1113,7 @@ impl AccountStorageEntry {
pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec<TempDir>, Vec<PathBuf>)> {
let temp_dirs: IoResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
let temp_dirs = temp_dirs?;

let paths: IoResult<Vec<_>> = temp_dirs
.iter()
.map(|temp_dir| {
create_accounts_run_and_snapshot_dirs(temp_dir).map(|(run_dir, _snapshot_dir)| run_dir)
})
.collect();
let paths = paths?;
let paths: Vec<PathBuf> = temp_dirs.iter().map(|t| t.path().to_path_buf()).collect();
Ok((temp_dirs, paths))
}

Expand Down
12 changes: 1 addition & 11 deletions runtime/src/hardened_unpack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,6 @@ where
}

let parts: Vec<_> = parts.map(|p| p.unwrap()).collect();
let account_filename =
(parts.len() == 2 && parts[0] == "accounts").then(|| PathBuf::from(parts[1]));
let unpack_dir = match entry_checker(parts.as_slice(), kind) {
UnpackPath::Invalid => {
return Err(UnpackError::Archive(format!(
Expand Down Expand Up @@ -177,16 +175,8 @@ where
let entry_path_buf = unpack_dir.join(entry.path()?);
set_perms(&entry_path_buf, mode)?;

let entry_path = if let Some(account_filename) = account_filename {
let stripped_path = unpack_dir.join(account_filename); // strip away "accounts"
fs::rename(&entry_path_buf, &stripped_path)?;
stripped_path
} else {
entry_path_buf
};

// Process entry after setting permissions
entry_processor(entry_path);
entry_processor(entry_path_buf);

total_entries += 1;
let now = Instant::now();
Expand Down
8 changes: 3 additions & 5 deletions runtime/src/serde_snapshot/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ use {
bank::{Bank, BankTestConfig},
epoch_accounts_hash,
genesis_utils::{self, activate_all_features, activate_feature},
snapshot_utils::{
create_tmp_accounts_dir_for_tests, get_storages_to_serialize, ArchiveFormat,
},
snapshot_utils::{get_storages_to_serialize, ArchiveFormat},
status_cache::StatusCache,
},
bincode::serialize_into,
Expand Down Expand Up @@ -577,7 +575,7 @@ fn test_extra_fields_full_snapshot_archive() {
// Set extra field
bank.fee_rate_governor.lamports_per_signature = 7000;

let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests();
let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap();
let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
Expand All @@ -597,7 +595,7 @@ fn test_extra_fields_full_snapshot_archive() {

// Deserialize
let (dbank, _) = snapshot_utils::bank_from_snapshot_archives(
&[accounts_dir],
&[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(),
&snapshot_archive_info,
None,
Expand Down
Loading