Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Do not shell out for tar #19043

Merged
merged 1 commit into from
Aug 4, 2021
Merged

Do not shell out for tar #19043

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 39 additions & 62 deletions runtime/src/snapshot_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ use {
collections::HashSet,
fmt,
fs::{self, File},
io::{self, BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write},
io::{BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, Write},
path::{Path, PathBuf},
process::{self, ExitStatus},
process::ExitStatus,
str::FromStr,
sync::Arc,
},
tar::Archive,
tar::{self, Archive},
tempfile::TempDir,
thiserror::Error,
};
Expand Down Expand Up @@ -580,69 +580,46 @@ pub fn archive_snapshot_package(
let file_ext = get_archive_ext(snapshot_package.archive_format);

// Tar the staging directory into the archive at `archive_path`
//
// system `tar` program is used for -S (sparse file support)
let archive_path = tar_dir.join(format!(
"{}{}.{}",
TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
));

let mut tar = process::Command::new("tar")
.args(&[
"chS",
"-C",
staging_dir.path().to_str().unwrap(),
"accounts",
"snapshots",
"version",
])
.stdin(process::Stdio::null())
.stdout(process::Stdio::piped())
.stderr(process::Stdio::inherit())
.spawn()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process spawn"))?;

match &mut tar.stdout {
None => {
return Err(SnapshotError::Io(IoError::new(
ErrorKind::Other,
"tar stdout unavailable".to_string(),
)));
}
Some(tar_output) => {
let mut archive_file = fs::File::create(&archive_path)?;

match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => {
let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
ArchiveFormat::TarGzip => {
let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
ArchiveFormat::Tar => {
io::copy(tar_output, &mut archive_file)?;
}
ArchiveFormat::TarZstd => {
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
};
}
}
{
let mut archive_file = fs::File::create(&archive_path)?;

let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
let mut archive = tar::Builder::new(encoder);
for dir in ["accounts", "snapshots"] {
archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
}
archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
archive.into_inner()?;
Ok(())
};

let tar_exit_status = tar
.wait()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process wait"))?;
if !tar_exit_status.success() {
warn!("tar command failed with exit code: {}", tar_exit_status);
return Err(SnapshotError::ArchiveGenerationFailure(tar_exit_status));
match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => {
let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::TarGzip => {
let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::TarZstd => {
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::Tar => {
do_archive_files(&mut archive_file)?;
}
};
}

// Atomically move the archive into position for other validators to find
Expand Down Expand Up @@ -2671,7 +2648,7 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar;
let snapshot_archive_format = ArchiveFormat::TarGzip;

let full_snapshot_archive_path = bank_to_full_snapshot_archive(
snapshots_dir.path(),
Expand Down Expand Up @@ -2746,7 +2723,7 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar;
let snapshot_archive_format = ArchiveFormat::TarZstd;

let full_snapshot_slot = slot;
let full_snapshot_archive_path = bank_to_full_snapshot_archive(
Expand Down