Skip to content

Commit

Permalink
Merge pull request rust-lang#1291 from bjorn3/build_system_rework2
Browse files Browse the repository at this point in the history 
Move downloaded test project to downloads/
  • Loading branch information
@bjorn3
bjorn3 authored Oct 13, 2022
2 parents e1a7791 + 24198ce commit 019040d
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 86 deletions.
7 changes: 1 addition & 6 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,4 @@ perf.data.old
/build_sysroot/compiler-builtins
/build_sysroot/rustc_version
/rust
/rand
/regex
/simple-raytracer
/portable-simd
/abi-cafe
/abi-checker
/download
6 changes: 3 additions & 3 deletions build_system/abi_cafe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::path::Path;

use super::build_sysroot;
use super::config;
use super::prepare;
use super::utils::{cargo_command, spawn_and_wait};
use super::SysrootKind;

Expand Down Expand Up @@ -35,9 +36,8 @@ pub(crate) fn run(
);

eprintln!("Running abi-cafe");
let mut abi_cafe_path = env::current_dir().unwrap();
abi_cafe_path.push("abi-cafe");
env::set_current_dir(&abi_cafe_path.clone()).unwrap();
let abi_cafe_path = prepare::ABI_CAFE.source_dir();
env::set_current_dir(abi_cafe_path.clone()).unwrap();

let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];

Expand Down
157 changes: 103 additions & 54 deletions build_system/prepare.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,56 +7,61 @@ use std::process::Command;
use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
use super::utils::{cargo_command, copy_dir_recursively, spawn_and_wait};

pub(crate) const ABI_CAFE: GitRepo = GitRepo::github(
"Gankra",
"abi-cafe",
"4c6dc8c9c687e2b3a760ff2176ce236872b37212",
"abi-cafe",
);

pub(crate) const RAND: GitRepo =
GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");

pub(crate) const REGEX: GitRepo =
GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");

pub(crate) const PORTABLE_SIMD: GitRepo = GitRepo::github(
"rust-lang",
"portable-simd",
"d5cd4a8112d958bd3a252327e0d069a6363249bd",
"portable-simd",
);

pub(crate) const SIMPLE_RAYTRACER: GitRepo = GitRepo::github(
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
"<none>",
);

pub(crate) fn prepare() {
if Path::new("download").exists() {
std::fs::remove_dir_all(Path::new("download")).unwrap();
}
std::fs::create_dir_all(Path::new("download")).unwrap();

prepare_sysroot();

// FIXME maybe install this only locally?
eprintln!("[INSTALL] hyperfine");
Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();

clone_repo_shallow_github(
"abi-cafe",
"Gankra",
"abi-cafe",
"4c6dc8c9c687e2b3a760ff2176ce236872b37212",
);
apply_patches("abi-cafe", Path::new("abi-cafe"));

clone_repo_shallow_github(
"rand",
"rust-random",
"rand",
"0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
);
apply_patches("rand", Path::new("rand"));

clone_repo_shallow_github(
"regex",
"rust-lang",
"regex",
"341f207c1071f7290e3f228c710817c280c8dca1",
);

clone_repo_shallow_github(
"portable-simd",
"rust-lang",
"portable-simd",
"d5cd4a8112d958bd3a252327e0d069a6363249bd",
);
apply_patches("portable-simd", Path::new("portable-simd"));

clone_repo_shallow_github(
"simple-raytracer",
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
);
ABI_CAFE.fetch();
RAND.fetch();
REGEX.fetch();
PORTABLE_SIMD.fetch();
SIMPLE_RAYTRACER.fetch();

eprintln!("[LLVM BUILD] simple-raytracer");
let build_cmd = cargo_command("cargo", "build", None, Path::new("simple-raytracer"));
let build_cmd = cargo_command("cargo", "build", None, &SIMPLE_RAYTRACER.source_dir());
spawn_and_wait(build_cmd);
fs::copy(
Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
Path::new("simple-raytracer").join(get_file_name("raytracer_cg_llvm", "bin")),
SIMPLE_RAYTRACER
.source_dir()
.join("target")
.join("debug")
.join(get_file_name("main", "bin")),
SIMPLE_RAYTRACER.source_dir().join(get_file_name("raytracer_cg_llvm", "bin")),
)
.unwrap();
}
Expand Down Expand Up @@ -88,38 +93,78 @@ fn prepare_sysroot() {
apply_patches("sysroot", &sysroot_src);
}

pub(crate) struct GitRepo {
url: GitRepoUrl,
rev: &'static str,
patch_name: &'static str,
}

enum GitRepoUrl {
Github { user: &'static str, repo: &'static str },
}

impl GitRepo {
const fn github(
user: &'static str,
repo: &'static str,
rev: &'static str,
patch_name: &'static str,
) -> GitRepo {
GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name }
}

pub(crate) fn source_dir(&self) -> PathBuf {
match self.url {
GitRepoUrl::Github { user: _, repo } => {
std::env::current_dir().unwrap().join("download").join(repo)
}
}
}

fn fetch(&self) {
match self.url {
GitRepoUrl::Github { user, repo } => {
clone_repo_shallow_github(&self.source_dir(), user, repo, self.rev);
}
}
apply_patches(self.patch_name, &self.source_dir());
}
}

#[allow(dead_code)]
fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
fn clone_repo(download_dir: &Path, repo: &str, rev: &str) {
eprintln!("[CLONE] {}", repo);
// Ignore exit code as the repo may already have been checked out
Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
Command::new("git").arg("clone").arg(repo).arg(&download_dir).spawn().unwrap().wait().unwrap();

let mut clean_cmd = Command::new("git");
clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
clean_cmd.arg("checkout").arg("--").arg(".").current_dir(&download_dir);
spawn_and_wait(clean_cmd);

let mut checkout_cmd = Command::new("git");
checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(download_dir);
spawn_and_wait(checkout_cmd);
}

fn clone_repo_shallow_github(target_dir: &str, username: &str, repo: &str, rev: &str) {
fn clone_repo_shallow_github(download_dir: &Path, user: &str, repo: &str, rev: &str) {
if cfg!(windows) {
// Older windows doesn't have tar or curl by default. Fall back to using git.
clone_repo(target_dir, &format!("https://github.com/{}/{}.git", username, repo), rev);
clone_repo(download_dir, &format!("https://github.com/{}/{}.git", user, repo), rev);
return;
}

let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", username, repo, rev);
let archive_file = format!("{}.tar.gz", rev);
let archive_dir = format!("{}-{}", repo, rev);
let downloads_dir = std::env::current_dir().unwrap().join("download");

eprintln!("[DOWNLOAD] {}/{} from {}", username, repo, archive_url);
let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", user, repo, rev);
let archive_file = downloads_dir.join(format!("{}.tar.gz", rev));
let archive_dir = downloads_dir.join(format!("{}-{}", repo, rev));

eprintln!("[DOWNLOAD] {}/{} from {}", user, repo, archive_url);

// Remove previous results if they exists
let _ = std::fs::remove_file(&archive_file);
let _ = std::fs::remove_dir_all(&archive_dir);
let _ = std::fs::remove_dir_all(target_dir);
let _ = std::fs::remove_dir_all(&download_dir);

// Download zip archive
let mut download_cmd = Command::new("curl");
Expand All @@ -128,13 +173,13 @@ fn clone_repo_shallow_github(target_dir: &str, username: &str, repo: &str, rev:

// Unpack tar archive
let mut unpack_cmd = Command::new("tar");
unpack_cmd.arg("xf").arg(&archive_file);
unpack_cmd.arg("xf").arg(&archive_file).current_dir(downloads_dir);
spawn_and_wait(unpack_cmd);

// Rename unpacked dir to the expected name
std::fs::rename(archive_dir, target_dir).unwrap();
std::fs::rename(archive_dir, &download_dir).unwrap();

init_git_repo(Path::new(target_dir));
init_git_repo(&download_dir);

// Cleanup
std::fs::remove_file(archive_file).unwrap();
Expand Down Expand Up @@ -175,6 +220,10 @@ fn get_patches(source_dir: &Path, crate_name: &str) -> Vec<PathBuf> {
}

fn apply_patches(crate_name: &str, target_dir: &Path) {
if crate_name == "<none>" {
return;
}

for patch in get_patches(&std::env::current_dir().unwrap(), crate_name) {
eprintln!(
"[PATCH] {:?} <- {:?}",
Expand Down
48 changes: 25 additions & 23 deletions build_system/tests.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use super::build_sysroot;
use super::config;
use super::prepare;
use super::rustc_info::get_wrapper_file_name;
use super::utils::{cargo_command, hyperfine_command, spawn_and_wait, spawn_and_wait_with_input};
use build_system::SysrootKind;
Expand Down Expand Up @@ -217,7 +218,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[

const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
TestCase::new("test.rust-random/rand", &|runner| {
runner.in_dir(["rand"], |runner| {
runner.in_dir(prepare::RAND.source_dir(), |runner| {
runner.run_cargo("clean", []);

if runner.host_triple == runner.target_triple {
Expand All @@ -230,7 +231,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
});
}),
TestCase::new("bench.simple-raytracer", &|runner| {
runner.in_dir(["simple-raytracer"], |runner| {
runner.in_dir(prepare::SIMPLE_RAYTRACER.source_dir(), |runner| {
let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string()).parse().unwrap();

if runner.host_triple == runner.target_triple {
Expand Down Expand Up @@ -273,19 +274,28 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
});
}),
TestCase::new("test.libcore", &|runner| {
runner.in_dir(["build_sysroot", "sysroot_src", "library", "core", "tests"], |runner| {
runner.run_cargo("clean", []);
runner.in_dir(
std::env::current_dir()
.unwrap()
.join("build_sysroot")
.join("sysroot_src")
.join("library")
.join("core")
.join("tests"),
|runner| {
runner.run_cargo("clean", []);

if runner.host_triple == runner.target_triple {
runner.run_cargo("test", []);
} else {
eprintln!("Cross-Compiling: Not running tests");
runner.run_cargo("build", ["--tests"]);
}
});
if runner.host_triple == runner.target_triple {
runner.run_cargo("test", []);
} else {
eprintln!("Cross-Compiling: Not running tests");
runner.run_cargo("build", ["--tests"]);
}
},
);
}),
TestCase::new("test.regex-shootout-regex-dna", &|runner| {
runner.in_dir(["regex"], |runner| {
runner.in_dir(prepare::REGEX.source_dir(), |runner| {
runner.run_cargo("clean", []);

// newer aho_corasick versions throw a deprecation warning
Expand Down Expand Up @@ -336,7 +346,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
});
}),
TestCase::new("test.regex", &|runner| {
runner.in_dir(["regex"], |runner| {
runner.in_dir(prepare::REGEX.source_dir(), |runner| {
runner.run_cargo("clean", []);

// newer aho_corasick versions throw a deprecation warning
Expand Down Expand Up @@ -367,7 +377,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
});
}),
TestCase::new("test.portable-simd", &|runner| {
runner.in_dir(["portable-simd"], |runner| {
runner.in_dir(prepare::PORTABLE_SIMD.source_dir(), |runner| {
runner.run_cargo("clean", []);
runner.run_cargo("build", ["--all-targets", "--target", &runner.target_triple]);

Expand Down Expand Up @@ -506,16 +516,8 @@ impl TestRunner {
}
}

fn in_dir<'a, I, F>(&self, dir: I, callback: F)
where
I: IntoIterator<Item = &'a str>,
F: FnOnce(&TestRunner),
{
fn in_dir(&self, new: impl AsRef<Path>, callback: impl FnOnce(&TestRunner)) {
let current = env::current_dir().unwrap();
let mut new = current.clone();
for d in dir {
new.push(d);
}

env::set_current_dir(new).unwrap();
callback(self);
Expand Down
4 changes: 4 additions & 0 deletions clean_all.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,8 @@ set -e

rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
rm -rf target/ build/ perf.data{,.old} y.bin
rm -rf download/

# Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
# FIXME remove at some point in the future
rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/

0 comments on commit 019040d

Please sign in to comment.