Make "Assemble stage1 compiler" orders of magnitude faster (take 2)

Cargo.lock

@@ -225,6 +225,7 @@ dependencies = [
  "serde_json",
  "tar",
  "toml",
+ "walkdir",
  "winapi",
  "xz2",
 ]

src/bootstrap/Cargo.toml

@@ -48,6 +48,7 @@ ignore = "0.4.10"
 opener = "0.5"
 once_cell = "1.7.2"
 xz2 = "0.1"
+walkdir = "2"
 
 [target.'cfg(windows)'.dependencies.winapi]
 version = "0.3"

src/bootstrap/dist.rs

@@ -848,7 +848,12 @@ impl Step for PlainSourceTarball {
 
     /// Creates the plain source tarball
     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
-        let tarball = Tarball::new(builder, "rustc", "src");
+        // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
+        // means neither rustup nor rustup-toolchain-install-master know how to download it.
+        // It also contains symbolic links, unlike other any other dist tarball.
+        // It's used for distros building rustc from source in a pre-vendored environment.
+        let mut tarball = Tarball::new(builder, "rustc", "src");
+        tarball.permit_symlinks(true);
         let plain_dst_src = tarball.image_dir();
 
         // This is the set of root paths which will become part of the source package
@@ -1868,7 +1873,6 @@ fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
 
 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
 ///
-
 /// Returns whether the files were actually copied.
 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
     if let Some(config) = builder.config.target_config.get(&target) {
@@ -1978,6 +1982,8 @@ impl Step for LlvmTools {
             }
         }
 
+        builder.ensure(crate::native::Llvm { target });
+
         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
         tarball.set_overlay(OverlayKind::LLVM);
         tarball.is_preview(true);

src/bootstrap/lib.rs

@@ -1408,6 +1408,10 @@ impl Build {
 
     /// Copies a file from `src` to `dst`
     pub fn copy(&self, src: &Path, dst: &Path) {
+        self.copy_internal(src, dst, false);
+    }
+
+    fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) {
         if self.config.dry_run {
             return;
         }
@@ -1417,15 +1421,22 @@ impl Build {
         }
         let _ = fs::remove_file(&dst);
         let metadata = t!(src.symlink_metadata());
+        let mut src = src.to_path_buf();
         if metadata.file_type().is_symlink() {
-            let link = t!(fs::read_link(src));
-            t!(symlink_file(link, dst));
-        } else if let Ok(()) = fs::hard_link(src, dst) {
+            if dereference_symlinks {
+                src = t!(fs::canonicalize(src));
+            } else {
+                let link = t!(fs::read_link(src));
+                t!(symlink_file(link, dst));
+                return;
+            }
+        }
+        if let Ok(()) = fs::hard_link(&src, dst) {
             // Attempt to "easy copy" by creating a hard link
             // (symlinks don't work on windows), but if that fails
             // just fall back to a slow `copy` operation.
         } else {
-            if let Err(e) = fs::copy(src, dst) {
+            if let Err(e) = fs::copy(&src, dst) {
                 panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e)
             }
             t!(fs::set_permissions(dst, metadata.permissions()));
@@ -1497,20 +1508,10 @@ impl Build {
         let dst = dstdir.join(src.file_name().unwrap());
         self.verbose_than(1, &format!("Install {:?} to {:?}", src, dst));
         t!(fs::create_dir_all(dstdir));
-        drop(fs::remove_file(&dst));
-        {
-            if !src.exists() {
-                panic!("Error: File \"{}\" not found!", src.display());
-            }
-            let metadata = t!(src.symlink_metadata());
-            if let Err(e) = fs::copy(&src, &dst) {
-                panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e)
-            }
-            t!(fs::set_permissions(&dst, metadata.permissions()));
-            let atime = FileTime::from_last_access_time(&metadata);
-            let mtime = FileTime::from_last_modification_time(&metadata);
-            t!(filetime::set_file_times(&dst, atime, mtime));
+        if !src.exists() {
+            panic!("Error: File \"{}\" not found!", src.display());
         }
+        self.copy_internal(src, &dst, true);
         chmod(&dst, perms);
     }
 

src/bootstrap/tarball.rs

@@ -102,6 +102,7 @@ pub(crate) struct Tarball<'a> {
 
     include_target_in_component_name: bool,
     is_preview: bool,
+    permit_symlinks: bool,
 }
 
 impl<'a> Tarball<'a> {
@@ -141,6 +142,7 @@ impl<'a> Tarball<'a> {
 
             include_target_in_component_name: false,
             is_preview: false,
+            permit_symlinks: false,
         }
     }
 
@@ -160,6 +162,10 @@ impl<'a> Tarball<'a> {
         self.is_preview = is;
     }
 
+    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
+        self.permit_symlinks = flag;
+    }
+
     pub(crate) fn image_dir(&self) -> &Path {
         t!(std::fs::create_dir_all(&self.image_dir));
         &self.image_dir
@@ -316,6 +322,18 @@ impl<'a> Tarball<'a> {
         }
         self.builder.run(&mut cmd);
 
+        // Ensure there are no symbolic links in the tarball. In particular,
+        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
+        let decompressed_output = self.temp_dir.join(&package_name);
+        if !self.builder.config.dry_run && !self.permit_symlinks {
+            for entry in walkdir::WalkDir::new(&decompressed_output) {
+                let entry = t!(entry);
+                if entry.path_is_symlink() {
+                    panic!("generated a symlink in a tarball: {}", entry.path().display());
+                }
+            }
+        }
+
         // Use either the first compression format defined, or "gz" as the default.
         let ext = self
             .builder
@@ -328,7 +346,7 @@ impl<'a> Tarball<'a> {
 
         GeneratedTarball {
             path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)),
-            decompressed_output: self.temp_dir.join(package_name),
+            decompressed_output,
             work: self.temp_dir,
         }
     }