create bench_local_diff command

Author: s7tya

Cargo.lock

@@ -44,6 +44,7 @@ regex = "1.7.1"
 analyzeme = "12.0.0"
 
 benchlib = { path = "benchlib" }
+itertools = "0.13.0"
 
 [target.'cfg(windows)'.dependencies]
 miow = "0.3"

collector/Cargo.toml

@@ -19,10 +19,12 @@ use anyhow::Context;
 use clap::builder::TypedValueParser;
 use clap::{Arg, Parser};
 use humansize::{format_size, BINARY};
+use itertools::Itertools;
 use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
 use tabled::builder::Builder;
 use tabled::settings::object::{Columns, Rows};
 use tabled::settings::{Alignment, Border, Color, Modify};
+use tabled::{Table, Tabled};
 use tokio::runtime::Runtime;
 
 use collector::api::next_artifact::NextArtifact;
@@ -52,7 +54,7 @@ use collector::toolchain::{
 use collector::utils::cachegrind::cachegrind_diff;
 use collector::utils::{is_installed, wait_for_future};
 use collector::{utils, CollectorCtx, CollectorStepBuilder};
-use database::{ArtifactId, ArtifactIdNumber, Commit, CommitType, Connection, Pool};
+use database::{ArtifactId, ArtifactIdNumber, Commit, CommitType, Connection, Lookup, Pool};
 
 fn n_normal_benchmarks_remaining(n: usize) -> String {
     let suffix = if n == 1 { "" } else { "s" };
@@ -628,6 +630,18 @@ enum Commands {
         #[command(flatten)]
         db: DbOption,
     },
+
+    /// Displays diff between two local bench result.
+    BenchLocalDiff {
+        #[command(flatten)]
+        db: DbOption,
+
+        #[arg(long)]
+        a_id: String,
+
+        #[arg(long)]
+        b_id: String,
+    },
 }
 
 #[derive(Debug, clap::Parser)]
@@ -1187,6 +1201,113 @@ Make sure to modify `{dir}/perf-config.json` if the category/artifact don't matc
             println!("Data of artifact {name} were removed");
             Ok(0)
         }
+        Commands::BenchLocalDiff { db, a_id, b_id } => {
+            let pool = Pool::open(&db.db);
+            let rt = build_async_runtime();
+            let mut conn = rt.block_on(pool.connection());
+            let index = rt.block_on(database::Index::load(&mut *conn));
+
+            let sids = index
+                .compile_statistic_descriptions()
+                .map(|(_, sid)| sid)
+                .collect::<Vec<_>>();
+
+            let a_id_number = rt
+                .block_on(conn.artifact_by_name(&a_id))
+                .expect("Cannot find specified artifact")
+                .lookup(&index)
+                .unwrap();
+            let b_id_number = rt
+                .block_on(conn.artifact_by_name(&b_id))
+                .expect("Cannot find specified artifact")
+                .lookup(&index)
+                .unwrap();
+
+            let pstats =
+                rt.block_on(conn.get_pstats(&sids, &[Some(a_id_number), Some(b_id_number)]));
+            let tuple_pstats = pstats
+                .into_iter()
+                .map(|row| row.into_iter().collect_tuple::<(_, _)>().unwrap())
+                .collect::<Vec<(Option<f64>, Option<f64>)>>();
+
+            #[derive(Tabled)]
+            struct Regression {
+                count: usize,
+                #[tabled(display_with = "display_range")]
+                range: (f64, f64),
+                #[tabled(display_with = "display_mean")]
+                mean: f64,
+            }
+
+            fn display_range(value: &(f64, f64)) -> String {
+                format!("[{:+.2}%, {:+.2}%]", value.0, value.1)
+            }
+
+            fn display_mean(value: &f64) -> String {
+                format!("{:+.2}%", value)
+            }
+
+            impl From<&Vec<f64>> for Regression {
+                fn from(value: &Vec<f64>) -> Self {
+                    let min = *value.iter().min_by(|a, b| a.total_cmp(b)).unwrap();
+                    let max = *value.iter().max_by(|a, b| a.total_cmp(b)).unwrap();
+                    let count = value.len();
+
+                    Regression {
+                        range: (min, max),
+                        count,
+                        mean: (value.iter().sum::<f64>() / count as f64),
+                    }
+                }
+            }
+
+            const DEFAULT_SIGNIFICANCE_THRESHOLD: f64 = 0.002;
+            let change = tuple_pstats
+                .iter()
+                .filter_map(|&(a, b)| match (a, b) {
+                    (Some(a), Some(b)) => {
+                        if a == 0.0 {
+                            None
+                        } else {
+                            Some((b - a) / a)
+                        }
+                    }
+                    (_, _) => None,
+                })
+                .filter(|c| c.abs() >= DEFAULT_SIGNIFICANCE_THRESHOLD * 100.0)
+                .collect::<Vec<_>>();
+            let negative_change = change
+                .iter()
+                .copied()
+                .filter(|&c| c < 0.0)
+                .collect::<Vec<_>>();
+            let positive_change = change
+                .iter()
+                .copied()
+                .filter(|&c| c > 0.0)
+                .collect::<Vec<_>>();
+
+            #[derive(Tabled)]
+            struct NamedRegression {
+                name: String,
+                #[tabled(inline)]
+                regression: Regression,
+            }
+
+            let regressions = [negative_change, positive_change, change]
+                .into_iter()
+                .map(|c| Regression::from(&c))
+                .zip(["❌", "✅", "✅, ❌"])
+                .map(|(c, label)| NamedRegression {
+                    name: label.to_string(),
+                    regression: c,
+                })
+                .collect::<Vec<_>>();
+
+            println!("{}", Table::new(regressions));
+
+            Ok(0)
+        }
     }
 }
 
@@ -1736,7 +1857,6 @@ fn bench_compile(
                 category,
             ));
             print_intro();
-
             let mut processor = BenchProcessor::new(
                 tx.conn(),
                 benchmark_name,