Auto merge of #58897 - Mark-Simulacrum:tool-rework, r=alexcrichton

Rework how bootstrap tools are built

This makes bootstrap tools buildable and testable in stage 0 with the downloaded bootstrap compiler, futhermore, it makes it such that they cannot be built in any other stage.

Notably, this will also mean that compiletest may need to wait a cycle before it can use changes to `libtest`, as it no longer depends on the in-tree libtest.

Author: bors

src/bootstrap/builder.rs

@@ -670,28 +670,27 @@ impl<'a> Builder<'a> {
             .map(|entry| entry.path())
     }
 
-    pub fn rustdoc(&self, host: Interned<String>) -> PathBuf {
-        self.ensure(tool::Rustdoc { host })
+    pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
+        self.ensure(tool::Rustdoc { compiler })
     }
 
-    pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
+    pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
         let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
-        let compiler = self.compiler(self.top_stage, host);
         cmd.env("RUSTC_STAGE", compiler.stage.to_string())
             .env("RUSTC_SYSROOT", self.sysroot(compiler))
             // Note that this is *not* the sysroot_libdir because rustdoc must be linked
             // equivalently to rustc.
             .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler))
             .env("CFG_RELEASE_CHANNEL", &self.config.channel)
-            .env("RUSTDOC_REAL", self.rustdoc(host))
+            .env("RUSTDOC_REAL", self.rustdoc(compiler))
             .env("RUSTDOC_CRATE_VERSION", self.rust_version())
             .env("RUSTC_BOOTSTRAP", "1");
 
         // Remove make-related flags that can cause jobserver problems.
         cmd.env_remove("MAKEFLAGS");
         cmd.env_remove("MFLAGS");
 
-        if let Some(linker) = self.linker(host) {
+        if let Some(linker) = self.linker(compiler.host) {
             cmd.env("RUSTC_TARGET_LINKER", linker);
         }
         cmd
@@ -753,7 +752,7 @@ impl<'a> Builder<'a> {
                 // This is the intended out directory for compiler documentation.
                 my_out = self.compiler_doc_out(target);
             }
-            let rustdoc = self.rustdoc(compiler.host);
+            let rustdoc = self.rustdoc(compiler);
             self.clear_if_dirty(&my_out, &rustdoc);
         } else if cmd != "test" {
             match mode {
@@ -910,7 +909,7 @@ impl<'a> Builder<'a> {
             .env(
                 "RUSTDOC_REAL",
                 if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
-                    self.rustdoc(compiler.host)
+                    self.rustdoc(compiler)
                 } else {
                     PathBuf::from("/path/to/nowhere/rustdoc/not/required")
                 },

src/bootstrap/dist.rs

@@ -479,7 +479,7 @@ impl Step for Rustc {
             t!(fs::create_dir_all(image.join("bin")));
             builder.cp_r(&src.join("bin"), &image.join("bin"));
 
-            builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
+            builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 
             // Copy runtime DLLs needed by the compiler
             if libdir != "bin" {

src/bootstrap/doc.rs

@@ -335,7 +335,7 @@ fn invoke_rustdoc(
     let footer = builder.src.join("src/doc/footer.inc");
     let version_info = out.join("version_info.html");
 
-    let mut cmd = builder.rustdoc_cmd(compiler.host);
+    let mut cmd = builder.rustdoc_cmd(compiler);
 
     let out = out.join("book");
 
@@ -415,7 +415,7 @@ impl Step for Standalone {
             }
 
             let html = out.join(filename).with_extension("html");
-            let rustdoc = builder.rustdoc(compiler.host);
+            let rustdoc = builder.rustdoc(compiler);
             if up_to_date(&path, &html) &&
                up_to_date(&footer, &html) &&
                up_to_date(&favicon, &html) &&
@@ -425,7 +425,7 @@ impl Step for Standalone {
                 continue
             }
 
-            let mut cmd = builder.rustdoc_cmd(compiler.host);
+            let mut cmd = builder.rustdoc_cmd(compiler);
             cmd.arg("--html-after-content").arg(&footer)
                .arg("--html-before-content").arg(&version_info)
                .arg("--html-in-header").arg(&favicon)
@@ -824,7 +824,7 @@ impl Step for Rustdoc {
         builder.ensure(Rustc { stage, target });
 
         // Build rustdoc.
-        builder.ensure(tool::Rustdoc { host: compiler.host });
+        builder.ensure(tool::Rustdoc { compiler: compiler });
 
         // Symlink compiler docs to the output directory of rustdoc documentation.
         let out_dir = builder.stage_out(compiler, Mode::ToolRustc)
@@ -883,7 +883,11 @@ impl Step for ErrorIndex {
         builder.info(&format!("Documenting error index ({})", target));
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
-        let mut index = builder.tool_cmd(Tool::ErrorIndex);
+        let compiler = builder.compiler(2, builder.config.build);
+        let mut index = tool::ErrorIndex::command(
+            builder,
+            compiler,
+        );
         index.arg("html");
         index.arg(out.join("error-index.html"));
 

src/bootstrap/test.rs

@@ -177,7 +177,7 @@ impl Step for Cargotest {
             cmd.arg(&builder.initial_cargo)
                 .arg(&out_dir)
                 .env("RUSTC", builder.rustc(compiler))
-                .env("RUSTDOC", builder.rustdoc(compiler.host)),
+                .env("RUSTDOC", builder.rustdoc(compiler)),
         );
     }
 }
@@ -414,7 +414,6 @@ impl Step for Miri {
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct CompiletestTest {
-    stage: u32,
     host: Interned<String>,
 }
 
@@ -427,16 +426,14 @@ impl Step for CompiletestTest {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(CompiletestTest {
-            stage: run.builder.top_stage,
             host: run.target,
         });
     }
 
     /// Runs `cargo test` for compiletest.
     fn run(self, builder: &Builder<'_>) {
-        let stage = self.stage;
         let host = self.host;
-        let compiler = builder.compiler(stage, host);
+        let compiler = builder.compiler(0, host);
 
         let mut cargo = tool::prepare_tool_cargo(builder,
                                                  compiler,
@@ -563,7 +560,7 @@ impl Step for RustdocTheme {
                 builder.sysroot_libdir(self.compiler, self.compiler.host),
             )
             .env("CFG_RELEASE_CHANNEL", &builder.config.channel)
-            .env("RUSTDOC_REAL", builder.rustdoc(self.compiler.host))
+            .env("RUSTDOC_REAL", builder.rustdoc(self.compiler))
             .env("RUSTDOC_CRATE_VERSION", builder.rust_version())
             .env("RUSTC_BOOTSTRAP", "1");
         if let Some(linker) = builder.linker(self.compiler.host) {
@@ -1042,7 +1039,7 @@ impl Step for Compiletest {
             || mode == "js-doc-test"
         {
             cmd.arg("--rustdoc-path")
-                .arg(builder.rustdoc(compiler.host));
+                .arg(builder.rustdoc(compiler));
         }
 
         cmd.arg("--src-base")
@@ -1464,7 +1461,10 @@ impl Step for ErrorIndex {
         t!(fs::create_dir_all(&dir));
         let output = dir.join("error-index.md");
 
-        let mut tool = builder.tool_cmd(Tool::ErrorIndex);
+        let mut tool = tool::ErrorIndex::command(
+            builder,
+            builder.compiler(compiler.stage, builder.config.build),
+        );
         tool.arg("markdown")
             .arg(&output)
             .env("CFG_BUILD", &builder.config.build)
@@ -1489,7 +1489,7 @@ fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) ->
     }
 
     builder.info(&format!("doc tests for: {}", markdown.display()));
-    let mut cmd = builder.rustdoc_cmd(compiler.host);
+    let mut cmd = builder.rustdoc_cmd(compiler);
     builder.add_rust_test_threads(&mut cmd);
     cmd.arg("--test");
     cmd.arg(markdown);

src/bootstrap/tool.rs

@@ -251,9 +251,9 @@ pub fn prepare_tool_cargo(
     cargo
 }
 
-macro_rules! tool {
+macro_rules! bootstrap_tool {
     ($(
-        $name:ident, $path:expr, $tool_name:expr, $mode:expr
+        $name:ident, $path:expr, $tool_name:expr
         $(,llvm_tools = $llvm:expr)*
         $(,is_external_tool = $external:expr)*
         ;
@@ -267,10 +267,7 @@ macro_rules! tool {
 
         impl Tool {
             pub fn get_mode(&self) -> Mode {
-                let mode = match self {
-                    $(Tool::$name => $mode,)+
-                };
-                mode
+                Mode::ToolBootstrap
             }
 
             /// Whether this tool requires LLVM to run
@@ -283,27 +280,15 @@ macro_rules! tool {
 
         impl<'a> Builder<'a> {
             pub fn tool_exe(&self, tool: Tool) -> PathBuf {
-                let stage = self.tool_default_stage(tool);
                 match tool {
                     $(Tool::$name =>
                         self.ensure($name {
-                            compiler: self.compiler(stage, self.config.build),
+                            compiler: self.compiler(0, self.config.build),
                             target: self.config.build,
                         }),
                     )+
                 }
             }
-
-            pub fn tool_default_stage(&self, tool: Tool) -> u32 {
-                // Compile the error-index in the same stage as rustdoc to avoid
-                // recompiling rustdoc twice if we can. Otherwise compile
-                // everything else in stage0 as there's no need to rebootstrap
-                // everything.
-                match tool {
-                    Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
-                    _ => 0,
-                }
-            }
         }
 
         $(
@@ -322,7 +307,8 @@ macro_rules! tool {
 
             fn make_run(run: RunConfig<'_>) {
                 run.builder.ensure($name {
-                    compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+                    // snapshot compiler
+                    compiler: run.builder.compiler(0, run.builder.config.build),
                     target: run.target,
                 });
             }
@@ -332,7 +318,7 @@ macro_rules! tool {
                     compiler: self.compiler,
                     target: self.target,
                     tool: $tool_name,
-                    mode: $mode,
+                    mode: Mode::ToolBootstrap,
                     path: $path,
                     is_optional_tool: false,
                     source_type: if false $(|| $external)* {
@@ -348,21 +334,67 @@ macro_rules! tool {
     }
 }
 
-tool!(
-    Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolBootstrap;
-    ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;
-    UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen", Mode::ToolBootstrap;
-    Tidy, "src/tools/tidy", "tidy", Mode::ToolBootstrap;
-    Linkchecker, "src/tools/linkchecker", "linkchecker", Mode::ToolBootstrap;
-    CargoTest, "src/tools/cargotest", "cargotest", Mode::ToolBootstrap;
-    Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolBootstrap, llvm_tools = true;
-    BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolBootstrap;
-    RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolBootstrap;
-    RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolBootstrap,
-        is_external_tool = true;
-    RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolBootstrap;
+bootstrap_tool!(
+    Rustbook, "src/tools/rustbook", "rustbook";
+    UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen";
+    Tidy, "src/tools/tidy", "tidy";
+    Linkchecker, "src/tools/linkchecker", "linkchecker";
+    CargoTest, "src/tools/cargotest", "cargotest";
+    Compiletest, "src/tools/compiletest", "compiletest", llvm_tools = true;
+    BuildManifest, "src/tools/build-manifest", "build-manifest";
+    RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
+    RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
+    RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
 );
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct ErrorIndex {
+    pub compiler: Compiler,
+}
+
+impl ErrorIndex {
+    pub fn command(builder: &Builder<'_>, compiler: Compiler) -> Command {
+        let mut cmd = Command::new(builder.ensure(ErrorIndex {
+            compiler
+        }));
+        add_lib_path(
+            vec![PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))],
+            &mut cmd,
+        );
+        cmd
+    }
+}
+
+impl Step for ErrorIndex {
+    type Output = PathBuf;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/error_index_generator")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        // Compile the error-index in the same stage as rustdoc to avoid
+        // recompiling rustdoc twice if we can.
+        let stage = if run.builder.top_stage >= 2 { run.builder.top_stage } else { 0 };
+        run.builder.ensure(ErrorIndex {
+            compiler: run.builder.compiler(stage, run.builder.config.build),
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        builder.ensure(ToolBuild {
+            compiler: self.compiler,
+            target: self.compiler.host,
+            tool: "error_index_generator",
+            mode: Mode::ToolRustc,
+            path: "src/tools/error_index_generator",
+            is_optional_tool: false,
+            source_type: SourceType::InTree,
+            extra_features: Vec::new(),
+        }).expect("expected to build -- essential tool")
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct RemoteTestServer {
     pub compiler: Compiler,
@@ -399,7 +431,9 @@ impl Step for RemoteTestServer {
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustdoc {
-    pub host: Interned<String>,
+    /// This should only ever be 0 or 2.
+    /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
+    pub compiler: Compiler,
 }
 
 impl Step for Rustdoc {
@@ -413,12 +447,12 @@ impl Step for Rustdoc {
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Rustdoc {
-            host: run.host,
+            compiler: run.builder.compiler(run.builder.top_stage, run.host),
         });
     }
 
     fn run(self, builder: &Builder<'_>) -> PathBuf {
-        let target_compiler = builder.compiler(builder.top_stage, self.host);
+        let target_compiler = self.compiler;
         if target_compiler.stage == 0 {
             if !target_compiler.is_snapshot(builder) {
                 panic!("rustdoc in stage 0 must be snapshot rustdoc");
@@ -626,7 +660,7 @@ impl<'a> Builder<'a> {
     /// `host`.
     pub fn tool_cmd(&self, tool: Tool) -> Command {
         let mut cmd = Command::new(self.tool_exe(tool));
-        let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
+        let compiler = self.compiler(0, self.config.build);
         self.prepare_tool_cmd(compiler, tool, &mut cmd);
         cmd
     }
@@ -638,7 +672,7 @@ impl<'a> Builder<'a> {
     fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) {
         let host = &compiler.host;
         let mut lib_paths: Vec<PathBuf> = vec![
-            if compiler.stage == 0 && tool != Tool::ErrorIndex {
+            if compiler.stage == 0 {
                 self.build.rustc_snapshot_libdir()
             } else {
                 PathBuf::from(&self.sysroot_libdir(compiler, compiler.host))