Merge pull request #1762 from Kobzol/codegen-backend-clif

Add support for benchmarking Cranelift codegen backend

Author: Kobzol

collector/src/bin/collector.rs

@@ -667,6 +667,7 @@ fn main_result() -> anyhow::Result<i32> {
             let get_suite = |rustc: &str, id: &str| {
                 let toolchain = get_local_toolchain(
                     &[Profile::Opt],
+                    &[CodegenBackend::Llvm],
                     rustc,
                     ToolchainConfig::default(),
                     id,
@@ -723,6 +724,7 @@ fn main_result() -> anyhow::Result<i32> {
             let get_toolchain = |rustc: &str, id: &str| {
                 let toolchain = get_local_toolchain(
                     &[Profile::Opt],
+                    &[CodegenBackend::Llvm],
                     rustc,
                     ToolchainConfig::default(),
                     id,
@@ -761,6 +763,7 @@ fn main_result() -> anyhow::Result<i32> {
 
             let toolchain = get_local_toolchain(
                 &profiles,
+                &backends,
                 &local.rustc,
                 *ToolchainConfig::default()
                     .rustdoc(opts.rustdoc.as_deref())
@@ -960,6 +963,7 @@ fn main_result() -> anyhow::Result<i32> {
                 |rustc: &str, suffix: &str| -> anyhow::Result<String> {
                     let toolchain = get_local_toolchain(
                         profiles,
+                        &[CodegenBackend::Llvm],
                         rustc,
                         *ToolchainConfig::default()
                             .rustdoc(opts.rustdoc.as_deref())
@@ -1085,6 +1089,7 @@ fn get_local_toolchain_for_runtime_benchmarks(
 ) -> anyhow::Result<Toolchain> {
     get_local_toolchain(
         &[Profile::Opt],
+        &[CodegenBackend::Llvm],
         &local.rustc,
         *ToolchainConfig::default()
             .cargo(local.cargo.as_deref())

collector/src/compile/benchmark/codegen_backend.rs

@@ -2,10 +2,11 @@
 #[value(rename_all = "PascalCase")]
 pub enum CodegenBackend {
     Llvm,
+    Cranelift,
 }
 
 impl CodegenBackend {
     pub fn all() -> Vec<CodegenBackend> {
-        vec![CodegenBackend::Llvm]
+        vec![CodegenBackend::Llvm, CodegenBackend::Cranelift]
     }
 }

collector/src/compile/benchmark/mod.rs

@@ -263,10 +263,12 @@ impl Benchmark {
         }
 
         eprintln!("Preparing {}", self.name);
-        let profile_dirs = profiles
-            .iter()
-            .map(|profile| Ok((*profile, self.make_temp_dir(&self.path)?)))
-            .collect::<anyhow::Result<Vec<_>>>()?;
+        let mut target_dirs: Vec<((CodegenBackend, Profile), TempDir)> = vec![];
+        for backend in backends {
+            for profile in &profiles {
+                target_dirs.push(((*backend, *profile), self.make_temp_dir(&self.path)?));
+            }
+        }
 
         // In parallel (but with a limit to the number of CPUs), prepare all
         // profiles. This is done in parallel vs. sequentially because:
@@ -290,22 +292,22 @@ impl Benchmark {
         // to do this in Cargo today. We would also ideally build in the same
         // target directory, but that's also not possible, as Cargo takes a
         // target-directory global lock during compilation.
+        //
+        // To avoid potential problems with recompilations, artifacts compiled by
+        // different codegen backends are stored in separate directories.
         let preparation_start = std::time::Instant::now();
         std::thread::scope::<_, anyhow::Result<()>>(|s| {
             let server = jobserver::Client::new(num_cpus::get()).context("jobserver::new")?;
-            let mut threads = Vec::with_capacity(profile_dirs.len());
-            for (profile, prep_dir) in &profile_dirs {
+            let mut threads = Vec::with_capacity(target_dirs.len());
+            for ((backend, profile), prep_dir) in &target_dirs {
                 let server = server.clone();
                 let thread = s.spawn::<_, anyhow::Result<()>>(move || {
                     wait_for_future(async move {
-                        // Prepare all backend artifacts into the same target directory
-                        for backend in backends {
-                            let server = server.clone();
-                            self.mk_cargo_process(toolchain, prep_dir.path(), *profile, *backend)
-                                .jobserver(server)
-                                .run_rustc(false)
-                                .await?;
-                        }
+                        let server = server.clone();
+                        self.mk_cargo_process(toolchain, prep_dir.path(), *profile, *backend)
+                            .jobserver(server)
+                            .run_rustc(false)
+                            .await?;
                         Ok::<(), anyhow::Error>(())
                     })?;
                     Ok(())
@@ -331,88 +333,82 @@ impl Benchmark {
         );
 
         let benchmark_start = std::time::Instant::now();
-        for &backend in backends {
-            for (profile, prep_dir) in &profile_dirs {
-                let profile = *profile;
-                eprintln!(
-                    "Running {}: {:?} + {:?} + {:?}",
-                    self.name, profile, scenarios, backend
-                );
+        for ((backend, profile), prep_dir) in &target_dirs {
+            let backend = *backend;
+            let profile = *profile;
+            eprintln!(
+                "Running {}: {:?} + {:?} + {:?}",
+                self.name, profile, scenarios, backend
+            );
+
+            // We want at least two runs for all benchmarks (since we run
+            // self-profile separately).
+            processor.start_first_collection();
+            for i in 0..std::cmp::max(iterations, 2) {
+                if i == 1 {
+                    let different = processor.finished_first_collection();
+                    if iterations == 1 && !different {
+                        // Don't run twice if this processor doesn't need it and
+                        // we've only been asked to run once.
+                        break;
+                    }
+                }
+                log::debug!("Benchmark iteration {}/{}", i + 1, iterations);
+                // Don't delete the directory on error.
+                let timing_dir = ManuallyDrop::new(self.make_temp_dir(prep_dir.path())?);
+                let cwd = timing_dir.path();
+
+                // A full non-incremental build.
+                if scenarios.contains(&Scenario::Full) {
+                    self.mk_cargo_process(toolchain, cwd, profile, backend)
+                        .processor(processor, Scenario::Full, "Full", None)
+                        .run_rustc(true)
+                        .await?;
+                }
 
-                // We want at least two runs for all benchmarks (since we run
-                // self-profile separately).
-                processor.start_first_collection();
-                for i in 0..std::cmp::max(iterations, 2) {
-                    if i == 1 {
-                        let different = processor.finished_first_collection();
-                        if iterations == 1 && !different {
-                            // Don't run twice if this processor doesn't need it and
-                            // we've only been asked to run once.
-                            break;
-                        }
+                // Rustdoc does not support incremental compilation
+                if profile != Profile::Doc {
+                    // An incremental  from scratch (slowest incremental case).
+                    // This is required for any subsequent incremental builds.
+                    if scenarios.iter().any(|s| s.is_incr()) {
+                        self.mk_cargo_process(toolchain, cwd, profile, backend)
+                            .incremental(true)
+                            .processor(processor, Scenario::IncrFull, "IncrFull", None)
+                            .run_rustc(true)
+                            .await?;
                     }
-                    log::debug!("Benchmark iteration {}/{}", i + 1, iterations);
-                    // Don't delete the directory on error.
-                    let timing_dir = ManuallyDrop::new(self.make_temp_dir(prep_dir.path())?);
-                    let cwd = timing_dir.path();
 
-                    // A full non-incremental build.
-                    if scenarios.contains(&Scenario::Full) {
+                    // An incremental build with no changes (fastest incremental case).
+                    if scenarios.contains(&Scenario::IncrUnchanged) {
                         self.mk_cargo_process(toolchain, cwd, profile, backend)
-                            .processor(processor, Scenario::Full, "Full", None)
+                            .incremental(true)
+                            .processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None)
                             .run_rustc(true)
                             .await?;
                     }
 
-                    // Rustdoc does not support incremental compilation
-                    if profile != Profile::Doc {
-                        // An incremental  from scratch (slowest incremental case).
-                        // This is required for any subsequent incremental builds.
-                        if scenarios.iter().any(|s| s.is_incr()) {
-                            self.mk_cargo_process(toolchain, cwd, profile, backend)
-                                .incremental(true)
-                                .processor(processor, Scenario::IncrFull, "IncrFull", None)
-                                .run_rustc(true)
-                                .await?;
-                        }
+                    if scenarios.contains(&Scenario::IncrPatched) {
+                        for (i, patch) in self.patches.iter().enumerate() {
+                            log::debug!("applying patch {}", patch.name);
+                            patch.apply(cwd).map_err(|s| anyhow::anyhow!("{}", s))?;
 
-                        // An incremental build with no changes (fastest incremental case).
-                        if scenarios.contains(&Scenario::IncrUnchanged) {
+                            // An incremental build with some changes (realistic
+                            // incremental case).
+                            let scenario_str = format!("IncrPatched{}", i);
                             self.mk_cargo_process(toolchain, cwd, profile, backend)
                                 .incremental(true)
                                 .processor(
                                     processor,
-                                    Scenario::IncrUnchanged,
-                                    "IncrUnchanged",
-                                    None,
+                                    Scenario::IncrPatched,
+                                    &scenario_str,
+                                    Some(patch),
                                 )
                                 .run_rustc(true)
                                 .await?;
                         }
-
-                        if scenarios.contains(&Scenario::IncrPatched) {
-                            for (i, patch) in self.patches.iter().enumerate() {
-                                log::debug!("applying patch {}", patch.name);
-                                patch.apply(cwd).map_err(|s| anyhow::anyhow!("{}", s))?;
-
-                                // An incremental build with some changes (realistic
-                                // incremental case).
-                                let scenario_str = format!("IncrPatched{}", i);
-                                self.mk_cargo_process(toolchain, cwd, profile, backend)
-                                    .incremental(true)
-                                    .processor(
-                                        processor,
-                                        Scenario::IncrPatched,
-                                        &scenario_str,
-                                        Some(patch),
-                                    )
-                                    .run_rustc(true)
-                                    .await?;
-                            }
-                        }
                     }
-                    drop(ManuallyDrop::into_inner(timing_dir));
                 }
+                drop(ManuallyDrop::into_inner(timing_dir));
             }
         }
         log::trace!(

collector/src/compile/execute/bencher.rs

@@ -94,6 +94,7 @@ impl<'a> BenchProcessor<'a> {
 
         let backend = match backend {
             CodegenBackend::Llvm => database::CodegenBackend::Llvm,
+            CodegenBackend::Cranelift => database::CodegenBackend::Cranelift,
         };
 
         if let Some(files) = stats.2 {

collector/src/compile/execute/mod.rs

@@ -267,6 +267,14 @@ impl<'a> CargoProcess<'a> {
                 cmd.arg("-Ztimings");
             }
             cmd.arg("--");
+
+            match self.backend {
+                CodegenBackend::Llvm => {}
+                CodegenBackend::Cranelift => {
+                    cmd.arg("-Zcodegen-backend=cranelift");
+                }
+            }
+
             // --wrap-rustc-with is not a valid rustc flag. But rustc-fake
             // recognizes it, strips it (and its argument) out, and uses it as an
             // indicator that the rustc invocation should be profiled. This works

collector/src/toolchain.rs

@@ -1,3 +1,4 @@
+use crate::compile::benchmark::codegen_backend::CodegenBackend;
 use crate::compile::benchmark::profile::Profile;
 use anyhow::{anyhow, Context};
 use log::debug;
@@ -330,6 +331,7 @@ impl<'a> ToolchainConfig<'a> {
 ///   for the nightly Cargo via `rustup`.
 pub fn get_local_toolchain(
     profiles: &[Profile],
+    codegen_backends: &[CodegenBackend],
     rustc: &str,
     toolchain_config: ToolchainConfig<'_>,
     id_suffix: &str,
@@ -357,8 +359,18 @@ pub fn get_local_toolchain(
                 anyhow::bail!("rustup-toolchain-install-master is not installed but must be");
             }
 
-            if !Command::new("rustup-toolchain-install-master")
-                .arg(toolchain)
+            let mut additional_components = vec![];
+            if codegen_backends.contains(&CodegenBackend::Cranelift) {
+                additional_components.push("rustc-codegen-cranelift");
+            }
+
+            let mut cmd = Command::new("rustup-toolchain-install-master");
+            cmd.arg(toolchain);
+            for component in additional_components {
+                cmd.arg("-c").arg(component);
+            }
+
+            if !cmd
                 .status()
                 .context("failed to run `rustup-toolchain-install-master`")?
                 .success()

database/src/lib.rs

@@ -367,12 +367,15 @@ impl PartialOrd for Scenario {
 pub enum CodegenBackend {
     /// The default LLVM backend
     Llvm,
+    /// Cranelift codegen backend
+    Cranelift,
 }
 
 impl CodegenBackend {
     pub fn as_str(self) -> &'static str {
         match self {
             CodegenBackend::Llvm => "llvm",
+            CodegenBackend::Cranelift => "cranelift",
         }
     }
 }
@@ -382,6 +385,7 @@ impl FromStr for CodegenBackend {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s.to_ascii_lowercase().as_str() {
             "llvm" => CodegenBackend::Llvm,
+            "cranelift" => CodegenBackend::Cranelift,
             _ => return Err(format!("{} is not a codegen backend", s)),
         })
     }

site/frontend/src/pages/compare/compile/common.ts

@@ -51,6 +51,7 @@ export const defaultCompileFilter: CompileBenchmarkFilter = {
 };
 
 export type Profile = "check" | "debug" | "opt" | "doc";
+export type CodegenBackend = "llvm" | "cranelift";
 export type Category = "primary" | "secondary";
 
 export type CompileBenchmarkMap = Dict<CompileBenchmarkMetadata>;
@@ -74,13 +75,15 @@ export interface CompileBenchmarkComparison {
   benchmark: string;
   profile: Profile;
   scenario: string;
+  backend: CodegenBackend;
   comparison: StatComparison;
 }
 
 export interface CompileTestCase {
   benchmark: string;
   profile: Profile;
   scenario: string;
+  backend: CodegenBackend;
   category: Category;
 }
 
@@ -156,6 +159,7 @@ export function computeCompileComparisonsWithNonRelevant(
           benchmark: c.benchmark,
           profile: c.profile,
           scenario: c.scenario,
+          backend: c.backend,
           category: (benchmarkMap[c.benchmark] || {}).category || "secondary",
         };
         return calculateComparison(c.comparison, testCase);

site/frontend/src/pages/compare/compile/table/comparisons-table.vue

@@ -55,6 +55,7 @@ const {toggleExpanded, isExpanded} = useExpandedStore();
           <th>Benchmark</th>
           <th>Profile</th>
           <th>Scenario</th>
+          <th>Backend</th>
           <th>% Change</th>
           <th class="narrow">
             Significance Threshold
@@ -98,6 +99,7 @@ const {toggleExpanded, isExpanded} = useExpandedStore();
               {{ comparison.testCase.profile }}
             </td>
             <td>{{ comparison.testCase.scenario }}</td>
+            <td>{{ comparison.testCase.backend }}</td>
             <td>
               <div class="numeric-aligned">
                 <span v-bind:class="percentClass(comparison.percent)">

site/src/api.rs

@@ -271,6 +271,7 @@ pub mod comparison {
         pub benchmark: String,
         pub profile: String,
         pub scenario: String,
+        pub backend: String,
         pub comparison: StatComparison,
     }
 

site/src/comparison.rs

@@ -152,6 +152,7 @@ pub async fn handle_compare(
             benchmark: comparison.benchmark.to_string(),
             profile: comparison.profile.to_string(),
             scenario: comparison.scenario.to_string(),
+            backend: comparison.backend.to_string(),
             comparison: comparison.comparison.into(),
         })
         .collect();