WIP: move the 'record all toolstates' list into rustbuild

src/bootstrap/builder.rs

@@ -383,6 +383,7 @@ impl<'a> Builder<'a> {
                 describe!(check::Std, check::Rustc, check::Rustdoc, check::Clippy)
             }
             Kind::Test => describe!(
+                crate::toolstate::ToolStateRecord,
                 crate::toolstate::ToolStateCheck,
                 test::ExpandYamlAnchors,
                 test::Tidy,
@@ -563,6 +564,10 @@ impl<'a> Builder<'a> {
         self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths);
     }
 
+    pub fn execute_cli_for_paths(&self, paths: &[PathBuf]) {
+        self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), paths);
+    }
+
     pub fn default_doc(&self, paths: Option<&[PathBuf]>) {
         let paths = paths.unwrap_or(&[]);
         self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths);

src/bootstrap/toolstate.rs

@@ -85,12 +85,40 @@ static STABLE_TOOLS: &[(&str, &str)] = &[
 // We do require that we checked whether they build or not on the tools builder,
 // though, as otherwise we will be unable to file an issue if they start
 // failing.
+//
+// FIXME: deduplicate this information with "stable=true/false" in `tool.rs`.
 static NIGHTLY_TOOLS: &[(&str, &str)] = &[
     ("miri", "src/tools/miri"),
     ("embedded-book", "src/doc/embedded-book"),
     // ("rustc-dev-guide", "src/doc/rustc-dev-guide"),
 ];
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ToolStateRecord;
+
+impl Step for ToolStateRecord {
+    type Output = ();
+
+    /// Builds (and tests, with `x.py test`) all toolstate-tracked tools.
+    fn run(self, builder: &Builder<'_>) {
+        let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable");
+        let paths: Vec<PathBuf> = STABLE_TOOLS
+            .iter()
+            .chain(if is_nightly { NIGHTLY_TOOLS.iter() } else { [].iter() })
+            .map(|(_tool, path)| PathBuf::from(path))
+            .collect();
+        builder.execute_cli_for_paths(&paths);
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("record-toolstate")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self);
+    }
+}
+
 fn print_error(tool: &str, submodule: &str) {
     eprintln!();
     eprintln!("We detected that this PR updated '{}', but its tests failed.", tool);
@@ -144,12 +172,11 @@ impl Step for ToolStateCheck {
 
     /// Checks tool state status.
     ///
-    /// This is intended to be used in the `checktools.sh` script. To use
-    /// this, set `save-toolstates` in `config.toml` so that tool status will
-    /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all
-    /// of the tools to populate the JSON file. After that is done, this
-    /// command can be run to check for any status failures, and exits with an
-    /// error if there are any.
+    /// This is intended to be used in the `checktools.sh` script. To use this,
+    /// set `save-toolstates` in `config.toml` so that tool status will be saved
+    /// to a JSON file. Then, run `x.py test --no-fail-fast record-toolstate` to
+    /// populate the JSON file. After that is done, this command can be run to
+    /// check for any status failures, and exits with an error if there are any.
     ///
     /// This also handles publishing the results to the `history` directory of
     /// the toolstate repo https://github.com/rust-lang-nursery/rust-toolstate
@@ -242,11 +269,11 @@ impl Step for ToolStateCheck {
     }
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("check-tools")
+        run.path("check-toolstate")
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(ToolStateCheck);
+        run.builder.ensure(Self);
     }
 }
 
@@ -281,6 +308,7 @@ impl Builder<'_> {
         // Toolstate isn't tracked for clippy, but since most tools do, we avoid
         // checking in all the places we could save toolstate and just do so
         // here.
+        // FIXME: make this a parameter of `tool::ToolBuild` instead.
         if tool == "clippy-driver" {
             return;
         }

src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh

@@ -5,22 +5,14 @@ set -eu
 X_PY="$1"
 
 # Try to test all the tools and store the build/test success in the TOOLSTATE_FILE
-
 set +e
-python3 "$X_PY" test --no-fail-fast \
-    src/doc/book \
-    src/doc/nomicon \
-    src/doc/reference \
-    src/doc/rust-by-example \
-    src/doc/embedded-book \
-    src/doc/edition-guide \
-    src/tools/rls \
-    src/tools/rustfmt \
-    src/tools/miri \
-
+python3 "$X_PY" test --no-fail-fast record-toolstate
 set -e
-
 # debugging: print out the saved toolstates
 cat /tmp/toolstate/toolstates.json
+# Check the JSON to ensure that there are no unexpected tool failures.
 python3 "$X_PY" test check-tools
+
+# Clippy is not part of the toolstate system;
+# just test it the regular way.
 python3 "$X_PY" test src/tools/clippy