Don't use hyperfine during testing

A new command ./y.rs bench is introduced for benchmarking. This change
allows skipping hyperfine installation in ./y.rs prepare and thus avoids
writing to ~/.cargo/bin.

Author: bjorn3

.cirrus.yml

@@ -6,8 +6,6 @@ task:
     - pkg install -y curl git bash
     - curl https://sh.rustup.rs -sSf --output rustup.sh
     - sh rustup.sh --default-toolchain none -y --profile=minimal
-  cargo_bin_cache:
-    folder: ~/.cargo/bin
   target_cache:
     folder: target
   prepare_script:

.github/workflows/main.yml

@@ -62,12 +62,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:

.github/workflows/nightly-cranelift.yml

@@ -13,12 +13,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ubuntu-latest-cargo-installed-crates
-
     - name: Prepare dependencies
       run: |
         git config --global user.email "[email protected]"

.github/workflows/rustc.yml

@@ -10,12 +10,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:
@@ -44,12 +38,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:

.vscode/settings.json

@@ -30,7 +30,7 @@
             ]
         },
         {
-            "sysroot_src": "./build_sysroot/sysroot_src/library",
+            "sysroot_src": "./download/sysroot/sysroot_src/library",
             "crates": [
                 {
                     "root_module": "./example/std_example.rs",

build_system/bench.rs

@@ -0,0 +1,79 @@
+use std::env;
+use std::fs;
+use std::path::Path;
+
+use super::path::{Dirs, RelPath};
+use super::prepare::GitRepo;
+use super::rustc_info::{get_file_name, get_wrapper_file_name};
+use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject};
+
+pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
+    "ebobby",
+    "simple-raytracer",
+    "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "<none>",
+);
+
+pub(crate) static SIMPLE_RAYTRACER: CargoProject =
+    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
+
+pub(crate) fn benchmark(dirs: &Dirs) {
+    benchmark_simple_raytracer(dirs);
+}
+
+fn benchmark_simple_raytracer(dirs: &Dirs) {
+    if std::process::Command::new("hyperfine").output().is_err() {
+        eprintln!("Hyperfine not installed");
+        eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
+        std::process::exit(1);
+    }
+
+    let run_runs = env::var("RUN_RUNS")
+        .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
+        .parse()
+        .unwrap();
+
+    eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
+    let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin"));
+    let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
+    let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
+
+    let clean_cmd = format!(
+        "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let llvm_build_cmd = format!(
+        "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let clif_build_cmd = format!(
+        "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
+        cargo_clif = cargo_clif.display(),
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+
+    let bench_compile =
+        hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
+
+    spawn_and_wait(bench_compile);
+
+    eprintln!("[BENCH RUN] ebobby/simple-raytracer");
+    fs::copy(
+        target_dir.join("debug").join(get_file_name("main", "bin")),
+        RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
+    )
+    .unwrap();
+
+    let mut bench_run = hyperfine_command(
+        0,
+        run_runs,
+        None,
+        Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
+        Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
+    );
+    bench_run.current_dir(RelPath::BUILD.to_path(dirs));
+    spawn_and_wait(bench_run);
+}

build_system/mod.rs

@@ -5,6 +5,7 @@ use std::process;
 use self::utils::is_ci;
 
 mod abi_cafe;
+mod bench;
 mod build_backend;
 mod build_sysroot;
 mod config;
@@ -20,6 +21,7 @@ USAGE:
     ./y.rs prepare [--out-dir DIR]
     ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
     ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+    ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
 
 OPTIONS:
     --sysroot none|clif|llvm
@@ -54,6 +56,7 @@ enum Command {
     Prepare,
     Build,
     Test,
+    Bench,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -67,7 +70,7 @@ pub fn main() {
     if env::var("RUST_BACKTRACE").is_err() {
         env::set_var("RUST_BACKTRACE", "1");
     }
-    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
+    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); // FIXME disable this by default
     env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
 
     if is_ci() {
@@ -83,6 +86,7 @@ pub fn main() {
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
         Some("test") => Command::Test,
+        Some("bench") => Command::Bench,
         Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
         Some(command) => arg_error!("Unknown command {}", command),
         None => {
@@ -198,5 +202,16 @@ pub fn main() {
                 &target_triple,
             );
         }
+        Command::Bench => {
+            build_sysroot::build_sysroot(
+                &dirs,
+                channel,
+                sysroot_kind,
+                &cg_clif_dylib,
+                &host_triple,
+                &target_triple,
+            );
+            bench::benchmark(&dirs);
+        }
     }
 }

build_system/prepare.rs

@@ -33,14 +33,14 @@ pub(crate) fn prepare(dirs: &Dirs) {
     super::tests::RAND_REPO.fetch(dirs);
     super::tests::REGEX_REPO.fetch(dirs);
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs);
+    super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
 
     eprintln!("[LLVM BUILD] simple-raytracer");
     let host_compiler = Compiler::host();
-    let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
+    let build_cmd = super::bench::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
     spawn_and_wait(build_cmd);
     fs::copy(
-        super::tests::SIMPLE_RAYTRACER
+        super::bench::SIMPLE_RAYTRACER
             .target_dir(dirs)
             .join(&host_compiler.triple)
             .join("debug")

build_system/tests.rs

@@ -1,13 +1,9 @@
-use crate::build_system::build_sysroot::SYSROOT_SRC;
-
-use super::build_sysroot;
+use super::bench::SIMPLE_RAYTRACER;
+use super::build_sysroot::{self, SYSROOT_SRC};
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
-use super::rustc_info::{get_file_name, get_wrapper_file_name};
-use super::utils::{
-    hyperfine_command, is_ci, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
-};
+use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
 use super::SysrootKind;
 use std::env;
 use std::ffi::OsStr;
@@ -253,16 +249,6 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
 static PORTABLE_SIMD: CargoProject =
     CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
 
-pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
-    "ebobby",
-    "simple-raytracer",
-    "804a7a21b9e673a482797aa289a18ed480e4d813",
-    "<none>",
-);
-
-pub(crate) static SIMPLE_RAYTRACER: CargoProject =
-    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
-
 static LIBCORE_TESTS: CargoProject =
     CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
 
@@ -282,67 +268,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::new("bench.simple-raytracer", &|runner| {
-        let run_runs = env::var("RUN_RUNS")
-            .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
-            .parse()
-            .unwrap();
-
-        if runner.is_native {
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
-            let cargo_clif = RelPath::DIST
-                .to_path(&runner.dirs)
-                .join(get_wrapper_file_name("cargo-clif", "bin"));
-            let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs);
-            let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs);
-
-            let clean_cmd = format!(
-                "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let llvm_build_cmd = format!(
-                "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let clif_build_cmd = format!(
-                "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
-                cargo_clif = cargo_clif.display(),
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-
-            let bench_compile =
-                hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
-
-            spawn_and_wait(bench_compile);
-
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer");
-            fs::copy(
-                target_dir.join("debug").join(get_file_name("main", "bin")),
-                RelPath::BUILD
-                    .to_path(&runner.dirs)
-                    .join(get_file_name("raytracer_cg_clif", "bin")),
-            )
-            .unwrap();
-
-            let mut bench_run = hyperfine_command(
-                0,
-                run_runs,
-                None,
-                Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
-                Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
-            );
-            bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
-            spawn_and_wait(bench_run);
-        } else {
-            spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs));
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
-            eprintln!("[COMPILE] ebobby/simple-raytracer");
-            spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
-        }
+    TestCase::new("test.simple-raytracer", &|runner| {
+        spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.host_compiler.cargo, &runner.dirs));
+        spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
     }),
     TestCase::new("test.libcore", &|runner| {
         spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));

config.txt

@@ -44,7 +44,7 @@ aot.issue-72793
 
 testsuite.extended_sysroot
 test.rust-random/rand
-bench.simple-raytracer
+test.simple-raytracer
 test.libcore
 test.regex-shootout-regex-dna
 test.regex