Allow testing a cranelift backend built into rustc itself

This avoids building cranelift twice in rust's CI and is a lot easier
than trying to make building of codegen backends work from within a
cargo invocation done by rust's build system.

Author: bjorn3

build_system/abi_cafe.rs

@@ -1,10 +1,8 @@
-use std::path::Path;
-
 use super::build_sysroot;
 use super::path::Dirs;
 use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
-use super::SysrootKind;
+use super::{CodegenBackend, SysrootKind};
 
 static ABI_CAFE_REPO: GitRepo =
     GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
@@ -15,7 +13,7 @@ pub(crate) fn run(
     channel: &str,
     sysroot_kind: SysrootKind,
     dirs: &Dirs,
-    cg_clif_dylib: &Path,
+    cg_clif_dylib: &CodegenBackend,
     rustup_toolchain_name: Option<&str>,
     bootstrap_host_compiler: &Compiler,
 ) {
@@ -41,7 +39,14 @@ pub(crate) fn run(
     cmd.arg("--pairs");
     cmd.args(pairs);
     cmd.arg("--add-rustc-codegen-backend");
-    cmd.arg(format!("cgclif:{}", cg_clif_dylib.display()));
+    match cg_clif_dylib {
+        CodegenBackend::Local(path) => {
+            cmd.arg(format!("cgclif:{}", path.display()));
+        }
+        CodegenBackend::Builtin(name) => {
+            cmd.arg(format!("cgclif:{name}"));
+        }
+    }
     cmd.current_dir(ABI_CAFE.source_dir(dirs));
 
     spawn_and_wait(cmd);

build_system/build_sysroot.rs

@@ -5,7 +5,7 @@ use std::process::{self, Command};
 use super::path::{Dirs, RelPath};
 use super::rustc_info::{get_file_name, get_rustc_version};
 use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
-use super::SysrootKind;
+use super::{CodegenBackend, SysrootKind};
 
 static DIST_DIR: RelPath = RelPath::DIST;
 static BIN_DIR: RelPath = RelPath::DIST.join("bin");
@@ -15,7 +15,7 @@ pub(crate) fn build_sysroot(
     dirs: &Dirs,
     channel: &str,
     sysroot_kind: SysrootKind,
-    cg_clif_dylib_src: &Path,
+    cg_clif_dylib_src: &CodegenBackend,
     bootstrap_host_compiler: &Compiler,
     rustup_toolchain_name: Option<&str>,
     target_triple: String,
@@ -28,17 +28,23 @@ pub(crate) fn build_sysroot(
 
     let is_native = bootstrap_host_compiler.triple == target_triple;
 
-    // Copy the backend
-    let cg_clif_dylib_path = if cfg!(windows) {
-        // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
-        // binaries.
-        BIN_DIR
-    } else {
-        LIB_DIR
-    }
-    .to_path(dirs)
-    .join(cg_clif_dylib_src.file_name().unwrap());
-    try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
+    let cg_clif_dylib_path = match cg_clif_dylib_src {
+        CodegenBackend::Local(src_path) => {
+            // Copy the backend
+            let cg_clif_dylib_path = if cfg!(windows) {
+                // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
+                // binaries.
+                BIN_DIR
+            } else {
+                LIB_DIR
+            }
+            .to_path(dirs)
+            .join(src_path.file_name().unwrap());
+            try_hard_link(src_path, &cg_clif_dylib_path);
+            CodegenBackend::Local(cg_clif_dylib_path)
+        }
+        CodegenBackend::Builtin(name) => CodegenBackend::Builtin(name.clone()),
+    };
 
     // Build and copy rustc and cargo wrappers
     let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin");
@@ -65,6 +71,9 @@ pub(crate) fn build_sysroot(
                 .env("RUSTC", &bootstrap_host_compiler.rustc)
                 .env("RUSTDOC", &bootstrap_host_compiler.rustdoc);
         }
+        if let CodegenBackend::Builtin(name) = cg_clif_dylib_src {
+            build_cargo_wrapper_cmd.env("BUILTIN_BACKEND", name);
+        }
         spawn_and_wait(build_cargo_wrapper_cmd);
         try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
     }
@@ -159,15 +168,15 @@ fn build_sysroot_for_triple(
     dirs: &Dirs,
     channel: &str,
     compiler: Compiler,
-    cg_clif_dylib_path: &Path,
+    cg_clif_dylib_path: &CodegenBackend,
     sysroot_kind: SysrootKind,
 ) -> SysrootTarget {
     match sysroot_kind {
         SysrootKind::None => build_rtstartup(dirs, &compiler)
             .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
         SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
         SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
+            build_clif_sysroot_for_triple(dirs, channel, compiler, cg_clif_dylib_path)
         }
     }
 }
@@ -211,7 +220,7 @@ fn build_clif_sysroot_for_triple(
     dirs: &Dirs,
     channel: &str,
     mut compiler: Compiler,
-    cg_clif_dylib_path: &Path,
+    cg_clif_dylib_path: &CodegenBackend,
 ) -> SysrootTarget {
     match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
         Err(e) => {
@@ -249,7 +258,14 @@ fn build_clif_sysroot_for_triple(
 
     // Build sysroot
     let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
-    rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
+    match cg_clif_dylib_path {
+        CodegenBackend::Local(path) => {
+            rustflags.push_str(&format!(" -Zcodegen-backend={}", path.to_str().unwrap()));
+        }
+        CodegenBackend::Builtin(name) => {
+            rustflags.push_str(&format!(" -Zcodegen-backend={name}"));
+        }
+    };
     // Necessary for MinGW to find rsbegin.o and rsend.o
     rustflags
         .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));

build_system/mod.rs

@@ -43,6 +43,12 @@ pub(crate) enum SysrootKind {
     Llvm,
 }
 
+#[derive(Clone, Debug)]
+pub(crate) enum CodegenBackend {
+    Local(PathBuf),
+    Builtin(String),
+}
+
 pub(crate) fn main() {
     if env::var("RUST_BACKTRACE").is_err() {
         env::set_var("RUST_BACKTRACE", "1");
@@ -79,6 +85,7 @@ pub(crate) fn main() {
     let mut sysroot_kind = SysrootKind::Clif;
     let mut use_unstable_features = true;
     let mut frozen = false;
+    let mut use_backend = None;
     while let Some(arg) = args.next().as_deref() {
         match arg {
             "--out-dir" => {
@@ -98,6 +105,12 @@ pub(crate) fn main() {
             }
             "--no-unstable-features" => use_unstable_features = false,
             "--frozen" => frozen = true,
+            "--use-backend" => {
+                use_backend = Some(match args.next() {
+                    Some(name) => name,
+                    None => arg_error!("--use-backend requires argument"),
+                });
+            }
             flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
             arg => arg_error!("Unexpected argument {}", arg),
         }
@@ -164,12 +177,16 @@ pub(crate) fn main() {
     env::set_var("RUSTC", "rustc_should_be_set_explicitly");
     env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly");
 
-    let cg_clif_dylib = build_backend::build_backend(
-        &dirs,
-        channel,
-        &bootstrap_host_compiler,
-        use_unstable_features,
-    );
+    let cg_clif_dylib = if let Some(name) = use_backend {
+        CodegenBackend::Builtin(name)
+    } else {
+        CodegenBackend::Local(build_backend::build_backend(
+            &dirs,
+            channel,
+            &bootstrap_host_compiler,
+            use_unstable_features,
+        ))
+    };
     match command {
         Command::Prepare => {
             // Handled above

build_system/tests.rs

@@ -3,11 +3,10 @@ use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
-use super::SysrootKind;
+use super::{CodegenBackend, SysrootKind};
 use std::env;
 use std::ffi::OsStr;
 use std::fs;
-use std::path::Path;
 use std::process::Command;
 
 static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
@@ -215,7 +214,7 @@ pub(crate) fn run_tests(
     dirs: &Dirs,
     channel: &str,
     sysroot_kind: SysrootKind,
-    cg_clif_dylib: &Path,
+    cg_clif_dylib: &CodegenBackend,
     bootstrap_host_compiler: &Compiler,
     rustup_toolchain_name: Option<&str>,
     target_triple: String,

build_system/usage.txt

@@ -29,6 +29,10 @@ OPTIONS:
     --frozen
             Require Cargo.lock and cache are up to date
 
+    --use-backend NAME
+            Use the existing Cranelift (or other) backend of the rustc with which we built.
+            Warning: This is meant for use in rust's CI only!
+
 REQUIREMENTS:
     * Rustup: By default rustup is used to install the right nightly version. If you don't want to
       use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and

scripts/cargo-clif.rs

@@ -12,17 +12,21 @@ fn main() {
 
     let mut rustflags = String::new();
     rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
-    rustflags.push_str(
-        sysroot
-            .join(if cfg!(windows) { "bin" } else { "lib" })
-            .join(
-                env::consts::DLL_PREFIX.to_string()
-                    + "rustc_codegen_cranelift"
-                    + env::consts::DLL_SUFFIX,
-            )
-            .to_str()
-            .unwrap(),
-    );
+    if let Some(name) = option_env!("BUILTIN_BACKEND") {
+        rustflags.push_str(name);
+    } else {
+        rustflags.push_str(
+            sysroot
+                .join(if cfg!(windows) { "bin" } else { "lib" })
+                .join(
+                    env::consts::DLL_PREFIX.to_string()
+                        + "rustc_codegen_cranelift"
+                        + env::consts::DLL_SUFFIX,
+                )
+                .to_str()
+                .unwrap(),
+        );
+    }
     rustflags.push_str(" --sysroot ");
     rustflags.push_str(sysroot.to_str().unwrap());
     env::set_var("RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + &rustflags);

scripts/rustc-clif.rs

@@ -19,9 +19,13 @@ fn main() {
     let mut args = vec![];
     args.push(OsString::from("-Cpanic=abort"));
     args.push(OsString::from("-Zpanic-abort-tests"));
-    let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
-    codegen_backend_arg.push(cg_clif_dylib_path);
-    args.push(codegen_backend_arg);
+    if let Some(name) = option_env!("BUILTIN_BACKEND") {
+        args.push(OsString::from(format!("-Zcodegen-backend={name}")))
+    } else {
+        let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
+        codegen_backend_arg.push(cg_clif_dylib_path);
+        args.push(codegen_backend_arg);
+    }
     if !passed_args.iter().any(|arg| {
         arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true)
     }) {

scripts/rustdoc-clif.rs

@@ -19,9 +19,13 @@ fn main() {
     let mut args = vec![];
     args.push(OsString::from("-Cpanic=abort"));
     args.push(OsString::from("-Zpanic-abort-tests"));
-    let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
-    codegen_backend_arg.push(cg_clif_dylib_path);
-    args.push(codegen_backend_arg);
+    if let Some(name) = option_env!("BUILTIN_BACKEND") {
+        args.push(OsString::from(format!("-Zcodegen-backend={name}")))
+    } else {
+        let mut codegen_backend_arg = OsString::from("-Zcodegen-backend=");
+        codegen_backend_arg.push(cg_clif_dylib_path);
+        args.push(codegen_backend_arg);
+    }
     if !passed_args.iter().any(|arg| {
         arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true)
     }) {