Add CI workflow for benchmarking Cranelift backend

.github/workflows/ci.yml

@@ -90,7 +90,6 @@ jobs:
       - name: Run unit tests
         run: cargo test --all
 
-
   test_benchmarks:
     strategy:
       matrix:
@@ -149,6 +148,40 @@ jobs:
           PROFILES: ${{ matrix.PROFILES }}
           SHELL: "/bin/bash"
 
+  test_backends:
+    name: Test codegen backend benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Install latest beta
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: beta
+          override: true
+
+      - name: Configure environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`
+          echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build collector
+        run: cargo build -p collector
+
+      - name: Check compile benchmarks
+        run: sh -x -c "ci/check-compile-benchmarks.sh"
+        env:
+          BENCH_INCLUDE_EXCLUDE_OPTS: "--include helloworld --exclude helloworld-tiny"
+          PROFILES: Debug,Opt
+          BACKENDS: Cranelift
+          SHELL: "/bin/bash"
+
   test_runtime_benchmarks:
     name: Test runtime benchmarks
     runs-on: ubuntu-latest

ci/check-compile-benchmarks.sh

@@ -6,9 +6,11 @@ bash -c "while true; do sleep 30; echo \$(date) - running ...; done" &
 PING_LOOP_PID=$!
 trap 'kill $PING_LOOP_PID' ERR 1 2 3 6
 
+BACKENDS=${BACKENDS:-Llvm}
+
 # Install a toolchain.
 RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \
-    bindir=`cargo run -p collector --bin collector install_next`
+    bindir=`cargo run -p collector --bin collector install_next --backends ${BACKENDS}`
 
 # Do some benchmarking.
 RUST_BACKTRACE=1 \
@@ -21,6 +23,7 @@ RUST_BACKTRACE=1 \
         --profiles $PROFILES \
         --cargo $bindir/cargo \
         --scenarios All \
+        --backends $BACKENDS \
         --rustdoc $bindir/rustdoc \
         $BENCH_INCLUDE_EXCLUDE_OPTS
 

collector/src/bin/collector.rs

@@ -539,7 +539,11 @@ enum Commands {
     },
 
     /// Installs the next commit for perf.rust-lang.org
-    InstallNext,
+    InstallNext {
+        /// Install additional components to enable benchmarking of the given backends.
+        #[arg(long = "backends", value_parser = EnumArgParser::<CodegenBackend>::default(), default_value = "Llvm")]
+        codegen_backends: MultiEnumValue<CodegenBackend>,
+    },
 
     /// Download a crate into collector/benchmarks.
     Download(DownloadCommand),
@@ -854,8 +858,12 @@ fn main_result() -> anyhow::Result<i32> {
                     runs,
                 } => {
                     let sha = commit.sha.to_string();
-                    let sysroot = Sysroot::install(sha.clone(), &target_triple)
-                        .with_context(|| format!("failed to install sysroot for {:?}", commit))?;
+                    let sysroot = Sysroot::install(
+                        sha.clone(),
+                        &target_triple,
+                        vec![CodegenBackend::Llvm],
+                    )
+                    .with_context(|| format!("failed to install sysroot for {:?}", commit))?;
 
                     let mut benchmarks = get_compile_benchmarks(
                         &compile_benchmark_dir,
@@ -1030,7 +1038,7 @@ fn main_result() -> anyhow::Result<i32> {
             Ok(0)
         }
 
-        Commands::InstallNext => {
+        Commands::InstallNext { codegen_backends } => {
             let last_sha = Command::new("git")
                 .arg("ls-remote")
                 .arg("https://github.com/rust-lang/rust.git")
@@ -1040,7 +1048,7 @@ fn main_result() -> anyhow::Result<i32> {
             let last_sha = String::from_utf8(last_sha.stdout).expect("utf8");
             let last_sha = last_sha.split_whitespace().next().expect(&last_sha);
             let commit = get_commit_or_fake_it(last_sha).expect("success");
-            let mut sysroot = Sysroot::install(commit.sha, &target_triple)?;
+            let mut sysroot = Sysroot::install(commit.sha, &target_triple, codegen_backends.0)?;
             sysroot.preserve(); // don't delete it
 
             // Print the directory containing the toolchain.

collector/src/toolchain.rs

@@ -20,7 +20,11 @@ pub struct Sysroot {
 }
 
 impl Sysroot {
-    pub fn install(sha: String, triple: &str) -> anyhow::Result<Self> {
+    pub fn install(
+        sha: String,
+        triple: &str,
+        backends: Vec<CodegenBackend>,
+    ) -> anyhow::Result<Self> {
         let unpack_into = "cache";
 
         fs::create_dir_all(unpack_into)?;
@@ -31,10 +35,13 @@ impl Sysroot {
             triple: triple.to_owned(),
         };
 
-        download.get_and_extract(ModuleVariant::Rustc)?;
-        download.get_and_extract(ModuleVariant::Std)?;
-        download.get_and_extract(ModuleVariant::Cargo)?;
-        download.get_and_extract(ModuleVariant::RustSrc)?;
+        download.get_and_extract(Component::Rustc)?;
+        download.get_and_extract(Component::Std)?;
+        download.get_and_extract(Component::Cargo)?;
+        download.get_and_extract(Component::RustSrc)?;
+        if backends.contains(&CodegenBackend::Cranelift) {
+            download.get_and_extract(Component::Cranelift)?;
+        }
 
         let sysroot = download.into_sysroot()?;
 
@@ -70,29 +77,30 @@ struct SysrootDownload {
 
 const BASE_URL: &str = "https://ci-artifacts.rust-lang.org/rustc-builds";
 
-// FIXME(eddyb) rename to just `Component`.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum ModuleVariant {
+enum Component {
     Cargo,
     Rustc,
     Std,
     RustSrc,
+    Cranelift,
 }
 
-impl fmt::Display for ModuleVariant {
+impl fmt::Display for Component {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            ModuleVariant::Cargo => write!(f, "cargo"),
-            ModuleVariant::Rustc => write!(f, "rustc"),
-            ModuleVariant::Std => write!(f, "rust-std"),
-            ModuleVariant::RustSrc => write!(f, "rust-src"),
+            Component::Cargo => write!(f, "cargo"),
+            Component::Rustc => write!(f, "rustc"),
+            Component::Std => write!(f, "rust-std"),
+            Component::RustSrc => write!(f, "rust-src"),
+            Component::Cranelift => write!(f, "rustc-codegen-cranelift"),
         }
     }
 }
 
-impl ModuleVariant {
+impl Component {
     fn url(&self, channel: &str, sysroot: &SysrootDownload, triple: &str) -> String {
-        let suffix = if *self == ModuleVariant::RustSrc {
+        let suffix = if *self == Component::RustSrc {
             String::new()
         } else {
             format!("-{}", triple)
@@ -137,15 +145,15 @@ impl SysrootDownload {
         })
     }
 
-    fn get_and_extract(&self, variant: ModuleVariant) -> anyhow::Result<()> {
+    fn get_and_extract(&self, component: Component) -> anyhow::Result<()> {
         let archive_path = self.directory.join(format!(
             "{}-{}-{}.tar.xz",
-            self.rust_sha, self.triple, variant,
+            self.rust_sha, self.triple, component,
         ));
         if archive_path.exists() {
             let reader = BufReader::new(File::open(&archive_path)?);
             let decompress = XzDecoder::new(reader);
-            let extract = self.extract(variant, decompress);
+            let extract = self.extract(component, decompress);
             match extract {
                 Ok(()) => return Ok(()),
                 Err(err) => {
@@ -158,17 +166,17 @@ impl SysrootDownload {
         // We usually have nightlies but we want to avoid breaking down if we
         // accidentally end up with a beta or stable commit.
         let urls = [
-            variant.url("nightly", self, &self.triple),
-            variant.url("beta", self, &self.triple),
-            variant.url("stable", self, &self.triple),
+            component.url("nightly", self, &self.triple),
+            component.url("beta", self, &self.triple),
+            component.url("stable", self, &self.triple),
         ];
         for url in &urls {
             log::debug!("requesting: {}", url);
             let resp = reqwest::blocking::get(url)?;
             log::debug!("{}", resp.status());
             if resp.status().is_success() {
                 let reader = XzDecoder::new(BufReader::new(resp));
-                match self.extract(variant, reader) {
+                match self.extract(component, reader) {
                     Ok(()) => return Ok(()),
                     Err(err) => {
                         log::warn!("extracting {} failed: {:?}", url, err);
@@ -181,17 +189,17 @@ impl SysrootDownload {
             "unable to download sha {} triple {} module {} from any of {:?}",
             self.rust_sha,
             self.triple,
-            variant,
+            component,
             urls
         ))
     }
 
-    fn extract<T: Read>(&self, variant: ModuleVariant, reader: T) -> anyhow::Result<()> {
+    fn extract<T: Read>(&self, component: Component, reader: T) -> anyhow::Result<()> {
         let mut archive = Archive::new(reader);
-        let prefix = if variant == ModuleVariant::Std {
-            format!("rust-std-{}", self.triple)
-        } else {
-            variant.to_string()
+        let prefix = match component {
+            Component::Std => format!("rust-std-{}", self.triple),
+            Component::Cranelift => format!("{component}-preview"),
+            _ => component.to_string(),
         };
 
         let unpack_into = self.directory.join(&self.rust_sha);