Add command for profiling and comparing toolchains

Current implementation allows using an arbitrary profiler, but the
intended use case is profiling with cachegrind. When cachegrind is
selected, the command will automatically compare results with `cg_diff`
and post process output with `cg_annotate`.

Author: tmiasko

collector/src/main.rs

@@ -11,7 +11,7 @@ use std::fs;
 use std::io::{stderr, Write};
 use std::path::{Path, PathBuf};
 use std::process;
-use std::process::Command;
+use std::process::{Command, Stdio};
 use std::{str, time::Instant};
 use tokio::runtime::Runtime;
 
@@ -486,6 +486,117 @@ fn get_local_toolchain(
     Ok((rustc, rustdoc, cargo))
 }
 
+fn generate_cachegrind_diffs(
+    id1: &str,
+    id2: &str,
+    out_dir: &Path,
+    benchmarks: &[Benchmark],
+    build_kinds: &[BuildKind],
+    scenario_kinds: &[ScenarioKind],
+    errors: &mut BenchmarkErrors,
+) {
+    for benchmark in benchmarks {
+        for &build_kind in build_kinds {
+            for &scenario_kind in scenario_kinds {
+                if let ScenarioKind::IncrPatched = scenario_kind {
+                    continue;
+                }
+                let filename = |prefix, id| {
+                    format!(
+                        "{}-{}-{}-{:?}-{:?}",
+                        prefix, id, benchmark.name, build_kind, scenario_kind
+                    )
+                };
+                let id_diff = format!("{}-{}", id1, id2);
+                let cgout1 = out_dir.join(filename("cgout", id1));
+                let cgout2 = out_dir.join(filename("cgout", id2));
+                let cgdiff = out_dir.join(filename("cgdiff", &id_diff));
+                let cgann = out_dir.join(filename("cgann", &id_diff));
+
+                if let Err(e) = cg_diff(&cgout1, &cgout2, &cgdiff) {
+                    errors.incr();
+                    eprintln!("collector error: {:?}", e);
+                    continue;
+                }
+                if let Err(e) = cg_annotate(&cgdiff, &cgann) {
+                    errors.incr();
+                    eprintln!("collector error: {:?}", e);
+                    continue;
+                }
+            }
+        }
+    }
+}
+
+/// Compares two Cachegrind output files using cg_diff and writes result to path.
+fn cg_diff(cgout1: &Path, cgout2: &Path, path: &Path) -> anyhow::Result<()> {
+    let output = Command::new("cg_diff")
+        .arg("--mod-filename=s#/rustc/[^/]*/##")
+        .arg("--mod-funcname=s/[.]llvm[.].*//")
+        .arg(cgout1)
+        .arg(cgout2)
+        .stderr(Stdio::inherit())
+        .output()
+        .context("failed to run `cg_diff`")?;
+
+    if !output.status.success() {
+        anyhow::bail!("failed to generate cachegrind diff");
+    }
+
+    fs::write(path, output.stdout).context("failed to write `cg_diff` output")?;
+
+    Ok(())
+}
+
+/// Post process Cachegrind output file and writes resutl to path.
+fn cg_annotate(cgout: &Path, path: &Path) -> anyhow::Result<()> {
+    let output = Command::new("cg_annotate")
+        .arg("--show-percs=no")
+        .arg(cgout)
+        .stderr(Stdio::inherit())
+        .output()
+        .context("failed to run `cg_annotate`")?;
+
+    if !output.status.success() {
+        anyhow::bail!("failed to annotate cachegrind output");
+    }
+
+    fs::write(path, output.stdout).context("failed to write `cg_annotate` output")?;
+
+    Ok(())
+}
+
+fn profile(
+    compiler: Compiler,
+    id: &str,
+    profiler: Profiler,
+    out_dir: &Path,
+    benchmarks: &[Benchmark],
+    build_kinds: &[BuildKind],
+    scenario_kinds: &[ScenarioKind],
+    errors: &mut BenchmarkErrors,
+) {
+    eprintln!("Profiling {} with {:?}", id, profiler);
+    for (i, benchmark) in benchmarks.iter().enumerate() {
+        eprintln!("{}", n_benchmarks_remaining(benchmarks.len() - i));
+        let mut processor = execute::ProfileProcessor::new(profiler, out_dir, id);
+        let result = benchmark.measure(
+            &mut processor,
+            &build_kinds,
+            &scenario_kinds,
+            compiler,
+            Some(1),
+        );
+        if let Err(ref s) = result {
+            errors.incr();
+            eprintln!(
+                "collector error: Failed to profile '{}' with {:?}, recorded: {:?}",
+                benchmark.name, profiler, s
+            );
+        }
+    }
+}
+
 fn main() {
     match main_result() {
         Ok(code) => process::exit(code),
@@ -585,6 +696,35 @@ fn main_result() -> anyhow::Result<i32> {
             (@arg RUSTDOC: --rustdoc +takes_value "The path to the local rustdoc to benchmark")
         )
 
+        (@subcommand diff_local =>
+            (about: "Profiles and compares two toolchains with one of several profilers")
+
+            // Mandatory arguments
+            (@arg PROFILER: +required +takes_value
+             "One of: 'self-profile', 'time-passes', 'perf-record',\n\
+             'oprofile', 'cachegrind', 'callgrind', 'dhat', 'massif',\n\
+             'eprintln', 'llvm-lines'")
+            (@arg RUSTC_BEFORE: +required +takes_value "The path to the local rustc to benchmark")
+            (@arg RUSTC_AFTER:  +required +takes_value "The path to the local rustc to benchmark")
+
+             // Options
+            (@arg BUILDS: --builds       +takes_value
+             "One or more (comma-separated) of: 'Check', \n\
+             'Debug', 'Doc', 'Opt', 'All'")
+            (@arg CARGO:   --cargo       +takes_value "The path to the local Cargo to use")
+            (@arg EXCLUDE: --exclude     +takes_value
+             "Exclude all benchmarks matching anything in\n\
+             this comma-separated list of patterns")
+            (@arg INCLUDE: --include     +takes_value
+             "Include only benchmarks matching something in\n\
+             this comma-separated list of patterns")
+            (@arg OUT_DIR: --("out-dir") +takes_value "Output directory")
+            (@arg RUNS:    --runs        +takes_value
+             "One or more (comma-separated) of: 'Full',\n\
+             'IncrFull', 'IncrUnchanged', 'IncrPatched', 'All'")
+            (@arg RUSTDOC: --rustdoc +takes_value "The path to the local rustdoc to benchmark")
+        )
+
         (@subcommand install_next =>
             (about: "Installs the next commit for perf.rust-lang.org")
 
@@ -796,38 +936,87 @@ fn main_result() -> anyhow::Result<i32> {
             let rustdoc = sub_m.value_of("RUSTDOC");
 
             let (rustc, rustdoc, cargo) = get_local_toolchain(&build_kinds, rustc, rustdoc, cargo)?;
-
             let compiler = Compiler {
                 rustc: &rustc,
                 rustdoc: rustdoc.as_deref(),
                 cargo: &cargo,
                 triple: &target_triple,
                 is_nightly: true,
             };
-
             let benchmarks = get_benchmarks(&benchmark_dir, include, exclude)?;
+            let mut errors = BenchmarkErrors::new();
+            profile(
+                compiler,
+                id,
+                profiler,
+                &out_dir,
+                &benchmarks,
+                &build_kinds,
+                &scenario_kinds,
+                &mut errors,
+            );
+            errors.fail_if_nonzero()?;
+            Ok(0)
+        }
+
+        ("diff_local", Some(sub_m)) => {
+            // Mandatory arguments
+            let profiler = Profiler::from_name(sub_m.value_of("PROFILER").unwrap())?;
+            let rustc1 = sub_m.value_of("RUSTC_BEFORE").unwrap();
+            let rustc2 = sub_m.value_of("RUSTC_AFTER").unwrap();
+
+            // Options
+            let build_kinds = build_kinds_from_arg(&sub_m.value_of("BUILDS"))?;
+            let cargo = sub_m.value_of("CARGO");
+            let exclude = sub_m.value_of("EXCLUDE");
+            let include = sub_m.value_of("INCLUDE");
+            let out_dir = PathBuf::from(sub_m.value_of_os("OUT_DIR").unwrap_or(default_out_dir));
+            let scenario_kinds = scenario_kinds_from_arg(sub_m.value_of("RUNS"))?;
+            let rustdoc = sub_m.value_of("RUSTDOC");
 
-            eprintln!("Profiling with {:?}", profiler);
+            let id1 = rustc1.strip_prefix('+').unwrap_or("before");
+            let id2 = rustc2.strip_prefix('+').unwrap_or("after");
+            let mut toolchains = Vec::new();
+            for (id, rustc) in [(id1, rustc1), (id2, rustc2)] {
+                let (rustc, rustdoc, cargo) =
+                    get_local_toolchain(&build_kinds, rustc, rustdoc, cargo)?;
+                toolchains.push((id.to_owned(), rustc, rustdoc, cargo));
+            }
 
+            let benchmarks = get_benchmarks(&benchmark_dir, include, exclude)?;
             let mut errors = BenchmarkErrors::new();
-            for (i, benchmark) in benchmarks.iter().enumerate() {
-                eprintln!("{}", n_benchmarks_remaining(benchmarks.len() - i));
-                let mut processor = execute::ProfileProcessor::new(profiler, &out_dir, &id);
-                let result = benchmark.measure(
-                    &mut processor,
+            for (id, rustc, rustdoc, cargo) in &toolchains {
+                let compiler = Compiler {
+                    rustc: &rustc,
+                    rustdoc: rustdoc.as_deref(),
+                    cargo: &cargo,
+                    triple: &target_triple,
+                    is_nightly: true,
+                };
+                profile(
+                    compiler,
+                    id,
+                    profiler,
+                    &out_dir,
+                    &benchmarks,
                     &build_kinds,
                     &scenario_kinds,
-                    compiler,
-                    Some(1),
+                    &mut errors,
                 );
-                if let Err(ref s) = result {
-                    errors.incr();
-                    eprintln!(
-                        "collector error: Failed to profile '{}' with {:?}, recorded: {:?}",
-                        benchmark.name, profiler, s
-                    );
-                }
             }
+
+            if let Profiler::Cachegrind = profiler {
+                generate_cachegrind_diffs(
+                    id1,
+                    id2,
+                    &out_dir,
+                    &benchmarks,
+                    &build_kinds,
+                    &scenario_kinds,
+                    &mut errors,
+                );
+            }
+
             errors.fail_if_nonzero()?;
             Ok(0)
         }