Skip to content

Commit f403e19

Browse files
KobzolKobzol
committed
Store compile benchmark codegen backend into the database
1 parent 1b56523 commit f403e19

File tree

10 files changed

+127
-35
lines changed

10 files changed

+127
-35
lines changed

‎collector/src/compile/execute/bencher.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,14 @@ impl<'a> BenchProcessor<'a> {
9292
Profile::Clippy => database::Profile::Clippy,
9393
};
9494

95+
let backend = match backend {
96+
CodegenBackend::LLVM => database::CodegenBackend::LLVM,
97+
};
98+
9599
if let Some(files) = stats.2 {
96100
if env::var_os("RUSTC_PERF_UPLOAD_TO_S3").is_some() {
101+
// FIXME: Record codegen backend in the self profile name
102+
97103
// We can afford to have the uploads run concurrently with
98104
// rustc. Generally speaking, they take up almost no CPU time
99105
// (just copying data into the network). Plus, during
@@ -131,6 +137,7 @@ impl<'a> BenchProcessor<'a> {
131137
self.benchmark.0.as_str(),
132138
profile,
133139
scenario,
140+
backend,
134141
stat,
135142
value,
136143
));

‎database/schema.md

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,16 @@ Here is the diagram for compile-time benchmarks:
2626
│ └────────────┘ └───────────────┘ │└────────────┘ │
2727
│ │ │
2828
│ │ │
29-
│ ┌───────────────┐ ┌──────────┐ | │
29+
│ ┌───────────────┐ ┌──────────┐ │ │
3030
│ │ pstat_series │ │ pstat │ │ │
3131
│ ├───────────────┤ ├──────────┤ │ │
3232
│ │ id * │◄┐│ id * │ │ │
3333
└─┤ crate │ └┤ series │ │ │
3434
│ profile │ │ aid ├───┼───────────────┘
35-
│ cache │ │ cid ├───┘
36-
│ statistic │ │ value │
37-
└───────────────┘ └──────────┘
35+
│ scenario │ │ cid │ │
36+
│ backend │ │ value ├───┘
37+
│ metric │ └──────────┘
38+
└───────────────┘
3839
```
3940

4041
For runtime benchmarks the schema very similar, but there are different table names:
@@ -140,19 +141,20 @@ of a crate, profile, scenario and the metric being collected.
140141

141142
* crate (aka `benchmark`): the benchmarked crate which might be a crate from crates.io or a crate made specifically to stress some part of the compiler.
142143
* profile: what type of compilation is happening - check build, optimized build (a.k.a. release build), debug build, or doc build.
143-
* cache (aka `scenario`): describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build.
144-
* statistic (aka `metric`): the type of metric being collected
144+
* scenario: describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build.
145+
* backend: codegen backend used for compilation.
146+
* metric: the type of metric being collected.
145147

146148
This corresponds to a [`statistic description`](../docs/glossary.md).
147149

148-
There is a separate table for this collection to avoid duplicating crates, prfiles, scenarios etc.
150+
There is a separate table for this collection to avoid duplicating crates, profiles, scenarios etc.
149151
many times in the `pstat` table.
150152

151153
```
152154
sqlite> select * from pstat_series limit 1;
153-
id crate profile cache statistic
154-
---------- ---------- ---------- ---------- ------------
155-
1 helloworld check full task-clock:u
155+
id crate profile scenario backend metric
156+
---------- ---------- ---------- ---------- ------- ------------
157+
1 helloworld check full llvm task-clock:u
156158
```
157159

158160
### pstat

‎database/src/bin/import-sqlite.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ async fn main() {
4545
let sqlite_aid = sqlite_conn.artifact_id(&aid).await;
4646
let postgres_aid = postgres_conn.artifact_id(&aid).await;
4747

48-
for (&(benchmark, profile, scenario, metric), id) in
48+
for (&(benchmark, profile, scenario, backend, metric), id) in
4949
sqlite_idx.compile_statistic_descriptions()
5050
{
5151
if benchmarks.insert(benchmark) {
@@ -73,6 +73,7 @@ async fn main() {
7373
&benchmark,
7474
profile,
7575
scenario,
76+
backend,
7677
metric.as_str(),
7778
stat,
7879
)

‎database/src/lib.rs

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,39 @@ impl PartialOrd for Scenario {
360360
}
361361
}
362362

363+
/// The codegen backend used for compilation.
364+
#[derive(
365+
Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
366+
)]
367+
pub enum CodegenBackend {
368+
/// The default LLVM backend
369+
LLVM,
370+
}
371+
372+
impl CodegenBackend {
373+
pub fn as_str(self) -> &'static str {
374+
match self {
375+
CodegenBackend::LLVM => "llvm",
376+
}
377+
}
378+
}
379+
380+
impl FromStr for CodegenBackend {
381+
type Err = String;
382+
fn from_str(s: &str) -> Result<Self, Self::Err> {
383+
Ok(match s.to_ascii_lowercase().as_str() {
384+
"llvm" => CodegenBackend::LLVM,
385+
_ => return Err(format!("{} is not a codegen backend", s)),
386+
})
387+
}
388+
}
389+
390+
impl fmt::Display for CodegenBackend {
391+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
392+
write!(f, "{}", self.as_str())
393+
}
394+
}
395+
363396
/// An identifier for a built version of the compiler
364397
#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
365398
pub enum ArtifactId {
@@ -436,7 +469,7 @@ pub struct Index {
436469
artifacts: Indexed<Box<str>>,
437470
/// Id lookup of compile stat description ids
438471
/// For legacy reasons called `pstat_series` in the database, and so the name is kept here.
439-
pstat_series: Indexed<(Benchmark, Profile, Scenario, Metric)>,
472+
pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Metric)>,
440473
/// Id lookup of runtime stat description ids
441474
runtime_pstat_series: Indexed<(Benchmark, Metric)>,
442475
}
@@ -556,6 +589,7 @@ pub enum DbLabel {
556589
benchmark: Benchmark,
557590
profile: Profile,
558591
scenario: Scenario,
592+
backend: CodegenBackend,
559593
metric: Metric,
560594
},
561595
}
@@ -573,10 +607,11 @@ impl Lookup for DbLabel {
573607
benchmark,
574608
profile,
575609
scenario,
610+
backend,
576611
metric,
577612
} => index
578613
.pstat_series
579-
.get(&(*benchmark, *profile, *scenario, *metric)),
614+
.get(&(*benchmark, *profile, *scenario, *backend, *metric)),
580615
}
581616
}
582617
}
@@ -626,7 +661,7 @@ impl Index {
626661
self.pstat_series
627662
.map
628663
.keys()
629-
.map(|(_, _, _, metric)| metric)
664+
.map(|(_, _, _, _, metric)| metric)
630665
.collect::<std::collections::HashSet<_>>()
631666
.into_iter()
632667
.map(|s| s.to_string())
@@ -652,7 +687,7 @@ impl Index {
652687
&self,
653688
) -> impl Iterator<
654689
Item = (
655-
&(Benchmark, Profile, Scenario, Metric),
690+
&(Benchmark, Profile, Scenario, CodegenBackend, Metric),
656691
StatisticalDescriptionId,
657692
),
658693
> + '_ {

‎database/src/pool.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::{ArtifactCollection, ArtifactId, ArtifactIdNumber, CompileBenchmark};
1+
use crate::{ArtifactCollection, ArtifactId, ArtifactIdNumber, CodegenBackend, CompileBenchmark};
22
use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step};
33
use chrono::{DateTime, Utc};
44
use hashbrown::HashMap;
@@ -44,6 +44,7 @@ pub trait Connection: Send + Sync {
4444
benchmark: &str,
4545
profile: Profile,
4646
scenario: Scenario,
47+
backend: CodegenBackend,
4748
metric: &str,
4849
value: f64,
4950
);

‎database/src/pool/postgres.rs

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
22
use crate::{
3-
ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, CollectionId, Commit, CommitType,
4-
CompileBenchmark, Date, Index, Profile, QueuedCommit, Scenario,
3+
ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, CodegenBackend, CollectionId,
4+
Commit, CommitType, CompileBenchmark, Date, Index, Profile, QueuedCommit, Scenario,
55
};
66
use anyhow::Context as _;
77
use chrono::{DateTime, TimeZone, Utc};
@@ -254,6 +254,13 @@ static MIGRATIONS: &[&str] = &[
254254
UNIQUE(aid, component)
255255
);
256256
"#,
257+
r#"
258+
alter table pstat_series rename column cache to scenario;
259+
alter table pstat_series rename column statistic to metric;
260+
alter table pstat_series add backend text not null default 'llvm';
261+
alter table pstat_series drop constraint pstat_series_crate_profile_cache_statistic_key;
262+
alter table pstat_series add constraint test_case UNIQUE(crate, profile, scenario, backend, metric);
263+
"#,
257264
];
258265

259266
#[async_trait::async_trait]
@@ -461,8 +468,8 @@ impl PostgresConnection {
461468
get_error: conn.prepare("select benchmark, error from error where aid = $1").await.unwrap(),
462469
select_self_query_series: conn.prepare("select id from self_profile_query_series where crate = $1 and profile = $2 and cache = $3 and query = $4").await.unwrap(),
463470
insert_self_query_series: conn.prepare("insert into self_profile_query_series (crate, profile, cache, query) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
464-
insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, cache, statistic) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
465-
select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and cache = $3 and statistic = $4").await.unwrap(),
471+
insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, metric) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
472+
select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and metric = $5").await.unwrap(),
466473
collection_id: conn.prepare("insert into collection (perf_commit) VALUES ($1) returning id").await.unwrap(),
467474
record_duration: conn.prepare("
468475
insert into artifact_collection_duration (
@@ -612,7 +619,7 @@ where
612619
pstat_series: self
613620
.conn()
614621
.query(
615-
"select id, crate, profile, cache, statistic from pstat_series;",
622+
"select id, crate, profile, scenario, backend, metric from pstat_series;",
616623
&[],
617624
)
618625
.await
@@ -625,7 +632,8 @@ where
625632
Benchmark::from(row.get::<_, String>(1).as_str()),
626633
Profile::from_str(row.get::<_, String>(2).as_str()).unwrap(),
627634
row.get::<_, String>(3).as_str().parse().unwrap(),
628-
row.get::<_, String>(4).as_str().into(),
635+
CodegenBackend::from_str(row.get::<_, String>(4).as_str()).unwrap(),
636+
row.get::<_, String>(5).as_str().into(),
629637
),
630638
)
631639
})
@@ -819,16 +827,18 @@ where
819827
benchmark: &str,
820828
profile: Profile,
821829
scenario: Scenario,
830+
backend: CodegenBackend,
822831
metric: &str,
823832
stat: f64,
824833
) {
825834
let profile = profile.to_string();
826835
let scenario = scenario.to_string();
836+
let backend = backend.to_string();
827837
let sid = self
828838
.conn()
829839
.query_opt(
830840
&self.statements().select_pstat_series,
831-
&[&benchmark, &profile, &scenario, &metric],
841+
&[&benchmark, &profile, &scenario, &backend, &metric],
832842
)
833843
.await
834844
.unwrap();
@@ -838,14 +848,14 @@ where
838848
self.conn()
839849
.query_opt(
840850
&self.statements().insert_pstat_series,
841-
&[&benchmark, &profile, &scenario, &metric],
851+
&[&benchmark, &profile, &scenario, &backend, &metric],
842852
)
843853
.await
844854
.unwrap();
845855
self.conn()
846856
.query_one(
847857
&self.statements().select_pstat_series,
848-
&[&benchmark, &profile, &scenario, &metric],
858+
&[&benchmark, &profile, &scenario, &backend, &metric],
849859
)
850860
.await
851861
.unwrap()

‎database/src/pool/sqlite.rs

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
22
use crate::{
3-
ArtifactCollection, ArtifactId, Benchmark, CollectionId, Commit, CommitType, CompileBenchmark,
4-
Date, Profile,
3+
ArtifactCollection, ArtifactId, Benchmark, CodegenBackend, CollectionId, Commit, CommitType,
4+
CompileBenchmark, Date, Profile,
55
};
66
use crate::{ArtifactIdNumber, Index, QueryDatum, QueuedCommit};
77
use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
@@ -368,6 +368,24 @@ static MIGRATIONS: &[Migration] = &[
368368
);
369369
"#,
370370
),
371+
// Add codegen backend column and add it to the unique constraint.
372+
// Also rename cache to scenario and statistic to metric, while we're at it.
373+
Migration::without_foreign_key_constraints(
374+
r#"
375+
create table pstat_series_new(
376+
id integer primary key not null,
377+
crate text not null references benchmark(name) on delete cascade on update cascade,
378+
profile text not null,
379+
scenario text not null,
380+
backend text not null,
381+
metric text not null,
382+
UNIQUE(crate, profile, scenario, backend, metric)
383+
);
384+
insert into pstat_series_new select id, crate, profile, cache, 'llvm', statistic from pstat_series;
385+
drop table pstat_series;
386+
alter table pstat_series_new rename to pstat_series;
387+
"#,
388+
),
371389
];
372390

373391
#[async_trait::async_trait]
@@ -480,7 +498,7 @@ impl Connection for SqliteConnection {
480498
.collect();
481499
let pstat_series = self
482500
.raw()
483-
.prepare("select id, crate, profile, cache, statistic from pstat_series;")
501+
.prepare("select id, crate, profile, scenario, backend, metric from pstat_series;")
484502
.unwrap()
485503
.query_map(params![], |row| {
486504
Ok((
@@ -489,7 +507,8 @@ impl Connection for SqliteConnection {
489507
Benchmark::from(row.get::<_, String>(1)?.as_str()),
490508
Profile::from_str(row.get::<_, String>(2)?.as_str()).unwrap(),
491509
row.get::<_, String>(3)?.as_str().parse().unwrap(),
492-
row.get::<_, String>(4)?.as_str().into(),
510+
CodegenBackend::from_str(row.get::<_, String>(4)?.as_str()).unwrap(),
511+
row.get::<_, String>(5)?.as_str().into(),
493512
),
494513
))
495514
})
@@ -649,21 +668,25 @@ impl Connection for SqliteConnection {
649668
benchmark: &str,
650669
profile: Profile,
651670
scenario: crate::Scenario,
671+
backend: CodegenBackend,
652672
metric: &str,
653673
value: f64,
654674
) {
655675
let profile = profile.to_string();
656676
let scenario = scenario.to_string();
657-
self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, cache, statistic) VALUES (?, ?, ?, ?)", params![
677+
let backend = backend.to_string();
678+
self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, metric) VALUES (?, ?, ?, ?, ?)", params![
658679
&benchmark,
659680
&profile,
660681
&scenario,
682+
&backend,
661683
&metric,
662684
]).unwrap();
663-
let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and cache = ? and statistic = ?", params![
685+
let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and metric = ?", params![
664686
&benchmark,
665687
&profile,
666688
&scenario,
689+
&backend,
667690
&metric,
668691
], |r| r.get(0)).unwrap();
669692
self.raw_ref()

‎docs/glossary.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ The following is a glossary of domain specific terminology. Although benchmarks
2121
- `incr-full`: incremental compilation is used, with an empty incremental cache.
2222
- `incr-unchanged`: incremental compilation is used, with a full incremental cache and no code changes made.
2323
- `incr-patched`: incremental compilation is used, with a full incremental cache and some code changes made.
24+
* **backend**: the codegen backend used for compiling Rust code.
25+
- `llvm`: the default codegen backend
2426
* **category**: a high-level group of benchmarks. Currently, there are three categories, primary (mostly real-world crates), secondary (mostly stress tests), and stable (old real-world crates, only used for the dashboard).
2527
* **artifact type**: describes what kind of artifact does the benchmark build. Either `library` or `binary`.
2628

0 commit comments

Comments
 (0)