Merge branch 'master' into categorize-perf-runs

Author: rylev

Cargo.lock

@@ -6,28 +6,27 @@ edition = '2018'
 
 [dependencies]
 clap = "2.25"
-env_logger = "0.7"
+env_logger = "0.8"
 anyhow = "1"
 thiserror = "1"
 log = "0.4"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tempfile = "3"
 libc = "0.2"
-chrono = "0.4"
+chrono = { version = "0.4", features = ["serde"] }
 lazy_static = "1"
-semver = "0.9"
-reqwest = { version = "0.10", features = ["json"] }
+semver = "1.0"
+reqwest = { version = "0.11", features = ["json"] }
 xz2 = "0.1.3"
 tar = "0.4"
-tokio = { version = "0.2", features = ["rt-core"] }
-rustc-artifacts = "0.2"
+tokio = { version = "1.6", features = ["rt"] }
 database = { path = "../database" }
 intern = { path = "../intern" }
 futures = "0.3.5"
 num_cpus = "1.13"
 jobserver = "0.1.21"
-crossbeam-utils = "0.7"
+crossbeam-utils = "0.8"
 snap = "1"
 filetime = "0.2.14"
 walkdir = "2"

collector/Cargo.toml

@@ -224,3 +224,29 @@ pub fn command_output(cmd: &mut Command) -> anyhow::Result<process::Output> {
 
     Ok(output)
 }
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct MasterCommit {
+    pub sha: String,
+    pub parent_sha: String,
+    /// This is the pull request which this commit merged in.
+    #[serde(default)]
+    pub pr: Option<u32>,
+    pub time: chrono::DateTime<chrono::Utc>,
+}
+
+/// This provides the master-branch Rust commits which should have accompanying
+/// bors artifacts available.
+///
+/// The first commit returned (at index 0) is the most recent, the last is the
+/// oldest.
+///
+/// Specifically, this is the last 168 days of bors commits.
+///
+/// Note that this does not contain try commits today, so it should not be used
+/// to validate hashes or expand them generally speaking. This may also change
+/// in the future.
+pub async fn master_commits() -> Result<Vec<MasterCommit>, Box<dyn std::error::Error + Sync + Send>> {
+    let response = reqwest::get("https://triage.rust-lang.org/bors-commit-list").await?;
+    Ok(response.json().await?)
+}

collector/src/lib.rs

@@ -546,13 +546,12 @@ fn main_result() -> anyhow::Result<i32> {
 
     let benchmark_dir = PathBuf::from("collector/benchmarks");
 
-    let mut builder = tokio::runtime::Builder::new();
+    let mut builder = tokio::runtime::Builder::new_multi_thread();
     // We want to minimize noise from the runtime
     builder
-        .core_threads(1)
-        .max_threads(1)
-        .enable_io()
-        .basic_scheduler();
+        .worker_threads(1)
+        .max_blocking_threads(1)
+        .enable_io();
     let mut rt = builder.build().expect("built runtime");
 
     let default_db = "results.db";
@@ -810,9 +809,9 @@ fn main_result() -> anyhow::Result<i32> {
 }
 
 pub fn get_commit_or_fake_it(sha: &str) -> anyhow::Result<Commit> {
-    let mut rt = tokio::runtime::Runtime::new().unwrap();
+    let rt = tokio::runtime::Runtime::new().unwrap();
     Ok(rt
-        .block_on(rustc_artifacts::master_commits())
+        .block_on(collector::master_commits())
         .map_err(|e| anyhow::anyhow!("{:?}", e))
         .context("getting master commit list")?
         .into_iter()

collector/src/main.rs

@@ -7,21 +7,21 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-hashbrown = { version = "0.7", features = ["serde"] }
+hashbrown = { version = "0.11", features = ["serde"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-rusqlite = { version = "0.23", features = ["bundled"] }
-tokio-postgres = { version = "0.5.4", features = ["with-serde_json-1", "with-chrono-0_4"] }
+rusqlite = { version = "0.25", features = ["bundled"] }
+tokio-postgres = { version = "0.7", features = ["with-serde_json-1", "with-chrono-0_4", "runtime"] }
 anyhow = "1"
 async-trait = "0.1"
-tokio = { version = "0.2.21", features = ["sync", "macros"] }
+tokio = { version = "1.6", features = ["sync", "macros"] }
 snap = "1"
 intern = { path = "../intern" }
 chrono = "0.4"
-reqwest = { version = "0.10.5", features = ["blocking"] }
-postgres-native-tls = "0.3"
+reqwest = { version = "0.11", features = ["blocking"] }
+postgres-native-tls = "0.5"
 native-tls = "0.2"
 lazy_static = "1"
-env_logger = "0.7"
+env_logger = "0.8"
 futures = "0.3.5"
 log = "0.4"

database/Cargo.toml

@@ -310,7 +310,7 @@ async fn copy<R: Table>(
                 R::trailer()
             )
             .as_str(),
-            vec![],
+            Vec::<u32>::new(),
         )
         .await
         .unwrap();

database/src/bin/export-to-sqlite.rs

@@ -289,14 +289,14 @@ impl Ingesting for Sqlite<'_> {
         let mut exact = series.chunks_exact(9);
         for series in exact.by_ref() {
             cached
-                .execute(series.iter().flat_map(|v| {
+                .execute(rusqlite::params_from_iter(series.iter().flat_map(|v| {
                     vec![
                         ToSqlOutput::Borrowed(ValueRef::Text(v.krate.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.profile.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.cache.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.statistic.as_bytes())),
                     ]
-                }))
+                })))
                 .unwrap();
         }
         for v in exact.remainder() {
@@ -357,14 +357,14 @@ impl Ingesting for Sqlite<'_> {
         let mut exact = series.chunks_exact(21);
         for series in exact.by_ref() {
             cached
-                .execute(series.iter().flat_map(|v| {
+                .execute(rusqlite::params_from_iter(series.iter().flat_map(|v| {
                     vec![
                         ToSqlOutput::Borrowed(ValueRef::Text(v.krate.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.profile.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.cache.as_bytes())),
                         ToSqlOutput::Borrowed(ValueRef::Text(v.query.as_bytes())),
                     ]
-                }))
+                })))
                 .unwrap();
         }
         for v in exact.remainder() {

database/src/bin/ingest-json.rs

@@ -210,7 +210,7 @@ where
     }
 
     async fn get(&self) -> ManagedConnection<T> {
-        let permit = self.permits.clone().acquire_owned().await;
+        let permit = self.permits.clone().acquire_owned().await.unwrap();
         let conn = {
             let mut slots = self.connections.lock().unwrap_or_else(|e| e.into_inner());
             slots.pop()

database/src/pool.rs

@@ -4,13 +4,11 @@ version = "0.1.0"
 authors = ["Mark Rousskov <[email protected]>"]
 edition = "2018"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 bumpalo = "3.2"
-parking_lot = "0.10"
-arc-swap = "0.4"
-hashbrown = "0.7.0"
+parking_lot = "0.11"
+arc-swap = "1.3"
+hashbrown = "0.11"
 lazy_static = "1"
 serde = "1"
 serde_derive = "1"

intern/Cargo.toml

@@ -5,45 +5,44 @@ version = "0.1.0"
 edition = '2018'
 
 [dependencies]
-env_logger = "0.7"
+env_logger = "0.8"
 anyhow = "1"
 thiserror = "1"
 futures = "0.3"
-tokio = { version = "0.2", features = ["macros", "time"] }
+tokio = { version = "1.6", features = ["macros", "time"] }
 log = "0.4"
 serde = { version = "1", features = ["rc"] }
 serde_derive = "1"
 serde_json = "1"
-hyper = "0.13"
+hyper = { version = "0.14", features = ["server", "stream"] }
 headers = "0.3"
 http = "0.2"
 home = "0.5"
 chrono = "0.4"
-rmp-serde = "0.14.2"
-semver = "0.9"
+rmp-serde = "0.15"
+semver = "1.0"
 ring = "0.16.10"
 hex = "0.4.2"
 regex = "1"
 lazy_static = "1"
-reqwest = { version = "0.10", features = ["json", "blocking"] }
+reqwest = { version = "0.11", features = ["json", "blocking"] }
 toml = "0.5"
 rust_team_data = { git = "https://github.com/rust-lang/team" }
-parking_lot = "0.10"
+parking_lot = "0.11"
 snap = "1"
-rustc-artifacts = "0.2.2"
-itertools = "0.9"
-hashbrown = "0.7"
-arc-swap = "0.4"
-rusqlite = { version = "0.23", features = ["bundled"] }
+itertools = "0.10"
+hashbrown = { version = "0.11", features = ["serde"] }
+arc-swap = "1.3"
+rusqlite = { version = "0.25", features = ["bundled"] }
 async-trait = "0.1"
 database = { path = "../database" }
-bytes = "0.5.6"
+bytes = "1.0"
 url = "2"
 analyzeme = { git = "https://github.com/rust-lang/measureme" }
 tar = "0.4"
 inferno = { version="0.10", default-features = false }
 mime = "0.3"
-prometheus = "0.11"
+prometheus = "0.12"
 
 [target.'cfg(unix)'.dependencies]
 jemallocator = "0.3"
@@ -54,4 +53,4 @@ path = "../collector"
 
 [dev-dependencies]
 lazy_static = "1"
-pretty_assertions = "0.6"
+pretty_assertions = "0.7"

site/Cargo.toml

@@ -24,7 +24,7 @@ pub async fn handle_triage(
     let start = body.start;
     let end = body.end;
     // Compare against self to get next
-    let master_commits = rustc_artifacts::master_commits().await?;
+    let master_commits = collector::master_commits().await?;
     let comparison = compare_given_commits(
         start.clone(),
         start.clone(),
@@ -76,7 +76,7 @@ pub async fn handle_compare(
     body: api::days::Request,
     data: &InputData,
 ) -> Result<api::days::Response, BoxedError> {
-    let master_commits = rustc_artifacts::master_commits().await?;
+    let master_commits = collector::master_commits().await?;
     let comparison =
         compare_given_commits(body.start, body.end, body.stat, data, &master_commits).await?;
 
@@ -213,7 +213,7 @@ pub async fn compare_given_commits(
     end: Bound,
     stat: String,
     data: &InputData,
-    master_commits: &[rustc_artifacts::Commit],
+    master_commits: &[collector::MasterCommit],
 ) -> Result<Comparison, BoxedError> {
     let a = data
         .data_for(true, start.clone())
@@ -257,7 +257,7 @@ impl DateData {
         conn: &dyn database::Connection,
         commit: ArtifactId,
         series: &mut [selector::SeriesResponse<T>],
-        master_commits: &[rustc_artifacts::Commit],
+        master_commits: &[collector::MasterCommit],
     ) -> Self
     where
         T: Iterator<Item = (db::ArtifactId, Option<f64>)>,
@@ -336,7 +336,7 @@ pub struct Comparison {
 
 impl Comparison {
     /// Gets the previous commit before `a`
-    pub fn prev(&self, master_commits: &[rustc_artifacts::Commit]) -> Option<String> {
+    pub fn prev(&self, master_commits: &[collector::MasterCommit]) -> Option<String> {
         match &self.a_id {
             ArtifactId::Commit(a) => master_commits
                 .iter()
@@ -350,7 +350,7 @@ impl Comparison {
     pub async fn is_contiguous(
         &self,
         conn: &dyn database::Connection,
-        master_commits: &[rustc_artifacts::Commit],
+        master_commits: &[collector::MasterCommit],
     ) -> bool {
         match (&self.a_id, &self.b_id) {
             (ArtifactId::Commit(a), ArtifactId::Commit(b)) => {
@@ -365,7 +365,7 @@ impl Comparison {
     }
 
     /// Gets the sha of the next commit after `b`
-    pub fn next(&self, master_commits: &[rustc_artifacts::Commit]) -> Option<String> {
+    pub fn next(&self, master_commits: &[collector::MasterCommit]) -> Option<String> {
         match &self.b_id {
             ArtifactId::Commit(b) => master_commits
                 .iter()

site/src/comparison.rs

@@ -188,7 +188,7 @@ r? @ghost",
     let rollup_merge_sha = rollup_merge_sha.to_owned();
     tokio::task::spawn(async move {
         // Give github time to create the merge commit reference
-        tokio::time::delay_for(Duration::from_secs(30)).await;
+        tokio::time::sleep(Duration::from_secs(30)).await;
         // This provides the master SHA so that we can check that we only queue
         // an appropriate try build. If there's ever a race condition, i.e.,
         // master was pushed while this command was running, the user will have to

site/src/github.rs

@@ -147,7 +147,7 @@ impl InputData {
     pub async fn missing_commits(&self) -> Vec<(Commit, MissingReason)> {
         let conn = self.conn().await;
         let (master_commits, queued_commits, in_progress_artifacts) = futures::join!(
-            rustc_artifacts::master_commits(),
+            collector::master_commits(),
             conn.queued_commits(),
             conn.in_progress_artifacts()
         );

site/src/load.rs

@@ -67,7 +67,7 @@ async fn main() {
     futures::pin_mut!(fut);
     loop {
         futures::select! {
-            s = server => {
+            _s = server => {
                 eprintln!("Server completed unexpectedly.");
                 return;
             }

site/src/main.rs

@@ -3,7 +3,7 @@
 
 use crate::load::InputData;
 use anyhow::Context;
-use bytes::buf::BufExt;
+use bytes::Buf;
 use hyper::StatusCode;
 use std::collections::HashMap;
 use std::fmt;

site/src/self_profile.rs

@@ -130,7 +130,7 @@ pub fn generate(pieces: super::Pieces, opt: Opt) -> anyhow::Result<Vec<u8>> {
 
     let mut seq = serializer.serialize_seq(None)?;
 
-    let data = ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events)
+    let data = ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events, None)
         .map_err(|e| anyhow::format_err!("{:?}", e))?;
 
     let thread_to_collapsed_thread =

site/src/self_profile/crox.rs

@@ -7,7 +7,7 @@ pub struct Opt {}
 
 pub fn generate(title: &str, pieces: super::Pieces, _: Opt) -> anyhow::Result<Vec<u8>> {
     let profiling_data =
-        ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events)
+        ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events, None)
             .map_err(|e| anyhow::format_err!("{:?}", e))?;
 
     let recorded_stacks = collapse_stacks(&profiling_data)

site/src/self_profile/flamegraph.rs

@@ -7,7 +7,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use bytes::buf::BufExt;
+use bytes::Buf;
 use parking_lot::Mutex;
 use std::cell::RefCell;
 use std::collections::HashMap;
@@ -1773,7 +1773,7 @@ async fn run_server(data: Arc<RwLock<Option<Arc<InputData>>>>, addr: SocketAddr)
             }))
         }
     });
-    let server = hyper::Server::bind(&addr).serve(svc);
+    let server = hyper::server::Server::bind(&addr).serve(svc);
     if let Err(e) = server.await {
         eprintln!("server error: {:?}", e);
     }