Merge pull request #645 from eddyb/build-std

Build the stdlib from rust-src sources.

Author: Mark-Simulacrum

.gitignore

@@ -1,2 +1,4 @@
 target
 *.pyc
+/cache/
+/rust.git/

collector/src/bin/rustc-perf-collector/sysroot.rs

@@ -1,11 +1,10 @@
 use anyhow::{anyhow, Context};
 use chrono::{DateTime, Utc};
 use collector::Sha;
-use std::ffi::OsStr;
 use std::fmt;
 use std::fs::{self, File};
 use std::io::{BufReader, Read};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use tar::Archive;
 use xz2::bufread::XzDecoder;
 
@@ -40,10 +39,116 @@ impl Sysroot {
         };
 
         download.get_and_extract(ModuleVariant::Rustc)?;
-        download.get_and_extract(ModuleVariant::Std)?;
+        // HACK(eddyb) commented out because we build our own stdlib
+        // (see `fn build_std` below).
+        // download.get_and_extract(ModuleVariant::Std)?;
         download.get_and_extract(ModuleVariant::Cargo)?;
+        download.get_and_extract(ModuleVariant::RustSrc)?;
 
-        download.into_sysroot()
+        let sysroot_dir = download.directory.join(&download.rust_sha);
+        let sysroot = download.into_sysroot()?;
+
+        // FIXME(eddyb) remove this once we no longer need to
+        // build our own stdlib (see `fn build_std` below).
+        sysroot.build_std(sysroot_dir)?;
+
+        Ok(sysroot)
+    }
+
+    /// Build `std`+`test`+`proc_macro` in a similar way to Cargo's `-Zbuild-std`
+    /// feature, but only once, and move the resulting libraries into the sysroot.
+    ///
+    /// We only need this until https://github.com/rust-lang/cargo/pull/8073
+    /// reaches beta, because then `rust-lang/rust` builds will have that
+    /// treatment. For now, we only have access to that Cargo change here,
+    /// using the newly built Cargo.
+    ///
+    /// For more background on why we need this, see this comment:
+    /// https://github.com/rust-lang/rust/issues/69060#issuecomment-604928032
+    /// (in short, Cargo used to include `rustc -vV` output, which contains
+    /// the commit hash, into `-Cmetadata`, producing different `std`s,
+    /// and making the perf runs incomparable, up to several % of difference).
+    fn build_std(&self, sysroot_dir: PathBuf) -> anyhow::Result<()> {
+        // Make sure everything below gets absolute directories.
+        let sysroot_dir = sysroot_dir.canonicalize()?;
+
+        let sysroot_rustlib_dir = sysroot_dir.join("lib/rustlib");
+        let rust_src_dir = sysroot_rustlib_dir.join("src/rust");
+
+        // HACK(eddyb) add a top-level `Cargo.toml` that has the necessary
+        // `patch.crates-io` entries for `rustc-std-workspace-{core,alloc,std}`.
+        // (maybe `rust-src` should include such a `Cargo.toml`?)
+        fs::write(
+            rust_src_dir.join("Cargo.toml"),
+            "\
+[workspace]
+members = ['src/libtest']
+
+[patch.crates-io]
+# See comments in `tools/rustc-std-workspace-core/README.md` for what's going on
+# here
+rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' }
+rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' }
+rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' }
+",
+        )?;
+
+        // HACK(eddyb) we need `std` to run the build scripts to build `std`.
+        let vanilla_sysroot_dir = {
+            let vanilla_download = SysrootDownload {
+                directory: sysroot_dir.join("vanilla-sysroot"),
+                rust_sha: self.sha.clone(),
+                triple: self.triple.clone(),
+            };
+            vanilla_download.get_and_extract(ModuleVariant::Std)?;
+            vanilla_download.directory.join(vanilla_download.rust_sha)
+        };
+
+        let rustflags = format!(
+            "--sysroot={sysroot} --remap-path-prefix={remap_from}={remap_to}",
+            sysroot = vanilla_sysroot_dir.display(),
+            remap_from = rust_src_dir.display(),
+            remap_to = "/rustc/REDACTED_SHA_HASH/"
+        );
+
+        // Run Cargo to produce `$local_build_target_dir/release/deps/lib*.rlib`.
+        let local_build_target_dir = sysroot_dir.join("build-std-target");
+        let cargo_status = std::process::Command::new(&self.cargo)
+            .env("RUSTC", &self.rustc)
+            .env("RUSTFLAGS", rustflags)
+            .env("__CARGO_DEFAULT_LIB_METADATA", "rustc-perf-std")
+            .args(&["build", "--release"])
+            .arg("--target-dir")
+            .arg(&local_build_target_dir)
+            .args(&["--features", "panic-unwind", "--features", "backtrace"])
+            .arg("--manifest-path")
+            .arg(rust_src_dir.join("src/libtest/Cargo.toml"))
+            .status()?;
+        if !cargo_status.success() {
+            return Err(anyhow!(
+                "unable to build stdlib for {} triple {}",
+                self.sha,
+                self.triple
+            ));
+        }
+
+        // Move all of the `rlib` files into the main sysroot.
+        let sysroot_target_lib_dir = sysroot_rustlib_dir.join(&self.triple).join("lib");
+        for entry in fs::read_dir(local_build_target_dir.join("release/deps"))? {
+            let entry = entry?;
+            let path = entry.path();
+            if let (Some(name), Some(ext)) = (path.file_name(), path.extension()) {
+                if ext == "rlib" {
+                    fs::rename(&path, sysroot_target_lib_dir.join(name))?;
+                }
+            }
+        }
+
+        // Clean up, to avoid accidental usage of these directories.
+        fs::remove_dir_all(vanilla_sysroot_dir)?;
+        fs::remove_dir_all(local_build_target_dir)?;
+
+        Ok(())
     }
 }
 
@@ -66,14 +171,15 @@ struct SysrootDownload {
     triple: String,
 }
 
-const MODULE_URL: &str =
-    "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/@SHA@/@MODULE@-nightly-@[email protected]";
+const BASE_URL: &str = "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds";
 
+// FIXME(eddyb) rename to just `Component`.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 enum ModuleVariant {
     Cargo,
     Rustc,
     Std,
+    RustSrc,
 }
 
 impl fmt::Display for ModuleVariant {
@@ -82,47 +188,45 @@ impl fmt::Display for ModuleVariant {
             ModuleVariant::Cargo => write!(f, "cargo"),
             ModuleVariant::Rustc => write!(f, "rustc"),
             ModuleVariant::Std => write!(f, "rust-std"),
+            ModuleVariant::RustSrc => write!(f, "rust-src"),
         }
     }
 }
 
 impl ModuleVariant {
     fn url(&self, sysroot: &SysrootDownload, triple: &str) -> String {
-        MODULE_URL
-            .replace("@MODULE@", &self.to_string())
-            .replace("@SHA@", &sysroot.rust_sha)
-            .replace("@TRIPLE@", triple)
+        let suffix = if *self == ModuleVariant::RustSrc {
+            String::new()
+        } else {
+            format!("-{}", triple)
+        };
+        format!(
+            "{base}/{sha}/{module}-nightly{suffix}.tar.xz",
+            base = BASE_URL,
+            module = self,
+            sha = sysroot.rust_sha,
+            suffix = suffix,
+        )
     }
 }
 
 impl SysrootDownload {
     fn into_sysroot(self) -> anyhow::Result<Sysroot> {
+        let sysroot_bin_dir = self.directory.join(&self.rust_sha).join("bin");
+        let sysroot_bin = |name| {
+            let path = sysroot_bin_dir.join(name);
+            path.canonicalize().with_context(|| {
+                format!(
+                    "failed to canonicalize {} path for {}: {:?}",
+                    name, self.rust_sha, path
+                )
+            })
+        };
+
         Ok(Sysroot {
-            rustc: self
-                .directory
-                .join(&self.rust_sha)
-                .join("rustc/bin/rustc")
-                .canonicalize()
-                .with_context(|| {
-                    format!("failed to canonicalize rustc path for {}", self.rust_sha)
-                })?,
-            rustdoc: self
-                .directory
-                .join(&self.rust_sha)
-                .join("rustc/bin/rustdoc")
-                .canonicalize()
-                .with_context(|| {
-                    format!("failed to canonicalize rustdoc path for {}", self.rust_sha)
-                })?,
-            cargo: {
-                let path = self.directory.join(&self.rust_sha).join("cargo/bin/cargo");
-                path.canonicalize().with_context(|| {
-                    format!(
-                        "failed to canonicalize cargo path for {}: {:?}",
-                        self.rust_sha, path
-                    )
-                })?
-            },
+            rustc: sysroot_bin("rustc")?,
+            rustdoc: sysroot_bin("rustdoc")?,
+            cargo: sysroot_bin("cargo")?,
             sha: self.rust_sha,
             triple: self.triple,
         })
@@ -161,19 +265,21 @@ impl SysrootDownload {
         }
 
         return Err(anyhow!(
-            "unable to download sha {} triple {} module {}",
+            "unable to download sha {} triple {} module {} from {}",
             self.rust_sha,
             self.triple,
-            variant
+            variant,
+            url
         ));
     }
 
     fn extract<T: Read>(&self, variant: ModuleVariant, reader: T) -> anyhow::Result<()> {
-        let is_std = variant == ModuleVariant::Std;
         let mut archive = Archive::new(reader);
-        let std_prefix = format!("rust-std-{}/lib/rustlib", self.triple);
-
-        let mut to_link = Vec::new();
+        let prefix = if variant == ModuleVariant::Std {
+            format!("rust-std-{}", self.triple)
+        } else {
+            variant.to_string()
+        };
 
         let unpack_into = self.directory.join(&self.rust_sha);
 
@@ -184,21 +290,11 @@ impl SysrootDownload {
             assert!(components.next().is_some(), "strip container directory");
             let path = components.as_path();
 
-            let path = if is_std {
-                if let Ok(path) = path.strip_prefix(&std_prefix) {
-                    if path.extension() == Some(OsStr::new("dylib")) {
-                        to_link.push(path.to_owned());
-                        continue;
-                    } else {
-                        Path::new("rustc/lib/rustlib").join(path)
-                    }
-                } else {
-                    continue;
-                }
+            let path = if let Ok(path) = path.strip_prefix(&prefix) {
+                unpack_into.join(path)
             } else {
-                path.into()
+                continue;
             };
-            let path = unpack_into.join(path);
             fs::create_dir_all(&path.parent().unwrap()).with_context(|| {
                 format!(
                     "could not create intermediate directories for {}",
@@ -208,24 +304,6 @@ impl SysrootDownload {
             entry.unpack(path)?;
         }
 
-        let link_dst_prefix = unpack_into.join(format!("rustc/lib/rustlib/{}/lib", self.triple));
-        let link_src_prefix = format!("{}/lib", self.triple);
-        for path in to_link {
-            let src = unpack_into.join("rustc/lib").join(
-                path.strip_prefix(&link_src_prefix)
-                    .with_context(|| format!("stripping prefix from: {:?}", path))?,
-            );
-            let dst = link_dst_prefix.join(&path);
-            fs::create_dir_all(&dst.parent().unwrap()).with_context(|| {
-                format!(
-                    "could not create intermediate directories for {}",
-                    dst.display()
-                )
-            })?;
-            log::trace!("linking {} to {}", src.display(), dst.display());
-            fs::hard_link(src, dst)?;
-        }
-
         Ok(())
     }
 }