Auto merge of #41954 - Mark-Simulacrum:rollup, r=Mark-Simulacrum

Rollup of 15 pull requests

- Successful merges: #41820, #41843, #41847, #41860, #41896, #41912, #41916, #41918, #41919, #41921, #41923, #41934, #41940, #41942, #41943
- Failed merges: #41914

Author: bors

.gitmodules

@@ -13,7 +13,7 @@
 	path = src/jemalloc
 	url = https://github.com/rust-lang/jemalloc.git
 [submodule "src/rust-installer"]
-	path = src/rust-installer
+	path = src/tools/rust-installer
 	url = https://github.com/rust-lang/rust-installer.git
 [submodule "src/liblibc"]
 	path = src/liblibc

src/Cargo.lock

@@ -13,6 +13,7 @@ members = [
   "tools/build-manifest",
   "tools/remote-test-client",
   "tools/remote-test-server",
+  "tools/rust-installer",
 ]
 
 # These projects have their own Cargo.lock

src/Cargo.toml

@@ -194,6 +194,8 @@ fn main() {
                 // do that we pass a weird flag to the compiler to get it to do
                 // so. Note that this is definitely a hack, and we should likely
                 // flesh out rpath support more fully in the future.
+                //
+                // FIXME: remove condition after next stage0
                 if stage != "0" {
                     cmd.arg("-Z").arg("osx-rpath-install-name");
                 }
@@ -218,6 +220,17 @@ fn main() {
             cmd.arg("-Z").arg("unstable-options");
             cmd.arg("-C").arg("target-feature=+crt-static");
         }
+
+        // Force all crates compiled by this compiler to (a) be unstable and (b)
+        // allow the `rustc_private` feature to link to other unstable crates
+        // also in the sysroot.
+        //
+        // FIXME: remove condition after next stage0
+        if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
+            if stage != "0" {
+                cmd.arg("-Z").arg("force-unstable-if-unmarked");
+            }
+        }
     }
 
     if verbose > 1 {

src/bootstrap/bin/rustc.rs

@@ -264,7 +264,7 @@ impl Config {
             let table = match p.parse() {
                 Some(table) => table,
                 None => {
-                    println!("failed to parse TOML configuration:");
+                    println!("failed to parse TOML configuration '{}':", file.to_str().unwrap());
                     for err in p.errors.iter() {
                         let (loline, locol) = p.to_linecol(err.lo);
                         let (hiline, hicol) = p.to_linecol(err.hi);

src/bootstrap/config.rs

@@ -175,6 +175,9 @@
 [rust]
 
 # Whether or not to optimize the compiler and standard library
+# Note: the slowness of the non optimized compiler compiling itself usually
+#       outweighs the time gains in not doing optimizations, therefore a
+#       full bootstrap takes much more time with optimize set to false.
 #optimize = true
 
 # Number of codegen units to use for each compiler invocation. A value of 0

src/bootstrap/config.toml.example

@@ -26,12 +26,6 @@ use std::process::{Command, Stdio};
 
 use build_helper::output;
 
-#[cfg(not(target_os = "solaris"))]
-const SH_CMD: &'static str = "sh";
-// On Solaris, sh is the historical bourne shell, not a POSIX shell, or bash.
-#[cfg(target_os = "solaris")]
-const SH_CMD: &'static str = "bash";
-
 use {Build, Compiler, Mode};
 use channel;
 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, exe};
@@ -55,6 +49,10 @@ pub fn tmpdir(build: &Build) -> PathBuf {
     build.out.join("tmp/dist")
 }
 
+fn rust_installer(build: &Build) -> Command {
+    build.tool_cmd(&Compiler::new(0, &build.config.build), "rust-installer")
+}
+
 /// Builds the `rust-docs` installer component.
 ///
 /// Slurps up documentation from the `stage`'s `host`.
@@ -74,14 +72,14 @@ pub fn docs(build: &Build, stage: u32, host: &str) {
     let src = build.out.join(host).join("doc");
     cp_r(&src, &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust-Documentation")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-documentation-is-installed.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rust-docs")
        .arg("--legacy-manifest-dirs=rustlib,cargo")
@@ -124,14 +122,14 @@ pub fn mingw(build: &Build, host: &str) {
        .arg(host);
     build.run(&mut cmd);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust-MinGW")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-MinGW-is-installed.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rust-mingw")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -190,15 +188,15 @@ pub fn rustc(build: &Build, stage: u32, host: &str) {
     }
 
     // Finally, wrap everything up in a nice tarball!
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, host))
        .arg("--component-name=rustc")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -300,14 +298,14 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) {
     let src = build.sysroot(compiler).join("lib/rustlib");
     cp_r(&src.join(target), &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=std-is-standing-at-the-ready.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, target))
        .arg(format!("--component-name=rust-std-{}", target))
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -356,14 +354,14 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
     println!("image_src: {:?}, dst: {:?}", image_src, dst);
     cp_r(&image_src, &dst);
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=save-analysis-saved.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", name, target))
        .arg(format!("--component-name=rust-analysis-{}", target))
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -471,13 +469,17 @@ pub fn rust_src(build: &Build) {
     write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 
     // Create plain source tarball
-    let tarball = rust_src_location(build);
+    let mut tarball = rust_src_location(build);
+    tarball.set_extension(""); // strip .gz
+    tarball.set_extension(""); // strip .tar
     if let Some(dir) = tarball.parent() {
         t!(fs::create_dir_all(dir));
     }
-    let mut cmd = Command::new("tar");
-    cmd.arg("-czf").arg(sanitize_sh(&tarball))
-       .arg(&plain_name)
+    let mut cmd = rust_installer(build);
+    cmd.arg("tarball")
+       .arg("--input").arg(&plain_name)
+       .arg("--output").arg(&tarball)
+       .arg("--work-dir=.")
        .current_dir(tmpdir(build));
     build.run(&mut cmd);
 
@@ -521,14 +523,14 @@ pub fn rust_src(build: &Build) {
     }
 
     // Create source tarball in rust-installer format
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Awesome-Source.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}", name))
        .arg("--component-name=rust-src")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -622,15 +624,15 @@ pub fn cargo(build: &Build, stage: u32, target: &str) {
     t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 
     // Generate the installer tarball
-    let mut cmd = Command::new("sh");
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, target))
        .arg("--component-name=cargo")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -671,15 +673,15 @@ pub fn rls(build: &Build, stage: u32, target: &str) {
     t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 
     // Generate the installer tarball
-    let mut cmd = Command::new("sh");
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("generate")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=RLS-ready-to-serve.")
-       .arg(format!("--image-dir={}", sanitize_sh(&image)))
-       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg("--image-dir").arg(&image)
+       .arg("--work-dir").arg(&tmpdir(build))
+       .arg("--output-dir").arg(&distdir(build))
+       .arg("--non-installed-overlay").arg(&overlay)
        .arg(format!("--package-name={}-{}", name, target))
        .arg("--component-name=rls")
        .arg("--legacy-manifest-dirs=rustlib,cargo");
@@ -730,29 +732,28 @@ pub fn extended(build: &Build, stage: u32, target: &str) {
     // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
     // the std files during uninstall. To do this ensure that rustc comes
     // before rust-std in the list below.
-    let mut input_tarballs = format!("{},{},{},{},{},{}",
-                                     sanitize_sh(&rustc_installer),
-                                     sanitize_sh(&cargo_installer),
-                                     sanitize_sh(&rls_installer),
-                                     sanitize_sh(&analysis_installer),
-                                     sanitize_sh(&docs_installer),
-                                     sanitize_sh(&std_installer));
+    let mut tarballs = vec![rustc_installer, cargo_installer, rls_installer,
+                            analysis_installer, docs_installer, std_installer];
     if target.contains("pc-windows-gnu") {
-        input_tarballs.push_str(",");
-        input_tarballs.push_str(&sanitize_sh(&mingw_installer));
+        tarballs.push(mingw_installer);
+    }
+    let mut input_tarballs = tarballs[0].as_os_str().to_owned();
+    for tarball in &tarballs[1..] {
+        input_tarballs.push(",");
+        input_tarballs.push(tarball);
     }
 
-    let mut cmd = Command::new(SH_CMD);
-    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/combine-installers.sh")))
+    let mut cmd = rust_installer(build);
+    cmd.arg("combine")
        .arg("--product-name=Rust")
        .arg("--rel-manifest-dir=rustlib")
        .arg("--success-message=Rust-is-ready-to-roll.")
-       .arg(format!("--work-dir={}", sanitize_sh(&work)))
-       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg("--work-dir").arg(&work)
+       .arg("--output-dir").arg(&distdir(build))
        .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
        .arg("--legacy-manifest-dirs=rustlib,cargo")
-       .arg(format!("--input-tarballs={}", input_tarballs))
-       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)));
+       .arg("--input-tarballs").arg(input_tarballs)
+       .arg("--non-installed-overlay").arg(&overlay);
     build.run(&mut cmd);
 
     let mut license = String::new();

src/bootstrap/dist.rs

@@ -479,7 +479,8 @@ impl Build {
         // compiled with debuginfo.
         if mode != Mode::Tool {
              cargo.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
-                  .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string());
+                  .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string())
+                  .env("RUSTC_FORCE_UNSTABLE", "1");
         }
 
         // Enable usage of unstable features
@@ -524,7 +525,9 @@ impl Build {
         // the comipiler, libs, and tests are stable and we don't want to make
         // their deps unstable (since this would break the first invariant
         // above).
-        if mode != Mode::Tool {
+        //
+        // FIXME: remove this after next stage0
+        if mode != Mode::Tool && stage == 0 {
             cargo.env("RUSTBUILD_UNSTABLE", "1");
         }
 

src/bootstrap/lib.rs

@@ -309,11 +309,15 @@ pub fn openssl(build: &Build, target: &str) {
     configure.arg("no-ssl3");
 
     let os = match target {
+        "aarch64-linux-android" => "linux-aarch64",
         "aarch64-unknown-linux-gnu" => "linux-aarch64",
+        "arm-linux-androideabi" => "android",
         "arm-unknown-linux-gnueabi" => "linux-armv4",
         "arm-unknown-linux-gnueabihf" => "linux-armv4",
+        "armv7-linux-androideabi" => "android-armv7",
         "armv7-unknown-linux-gnueabihf" => "linux-armv4",
         "i686-apple-darwin" => "darwin-i386-cc",
+        "i686-linux-android" => "android-x86",
         "i686-unknown-freebsd" => "BSD-x86-elf",
         "i686-unknown-linux-gnu" => "linux-elf",
         "i686-unknown-linux-musl" => "linux-elf",
@@ -326,6 +330,7 @@ pub fn openssl(build: &Build, target: &str) {
         "powerpc64le-unknown-linux-gnu" => "linux-ppc64le",
         "s390x-unknown-linux-gnu" => "linux64-s390x",
         "x86_64-apple-darwin" => "darwin64-x86_64-cc",
+        "x86_64-linux-android" => "linux-x86_64",
         "x86_64-unknown-freebsd" => "BSD-x86_64",
         "x86_64-unknown-linux-gnu" => "linux-x86_64",
         "x86_64-unknown-linux-musl" => "linux-x86_64",
@@ -337,6 +342,18 @@ pub fn openssl(build: &Build, target: &str) {
     for flag in build.cflags(target) {
         configure.arg(flag);
     }
+    // There is no specific os target for android aarch64 or x86_64,
+    // so we need to pass some extra cflags
+    if target == "aarch64-linux-android" || target == "x86_64-linux-android" {
+        configure.arg("-mandroid");
+        configure.arg("-fomit-frame-pointer");
+    }
+    // Make PIE binaries
+    // Non-PIE linker support was removed in Lollipop
+    // https://source.android.com/security/enhancements/enhancements50
+    if target == "i686-linux-android" {
+        configure.arg("no-asm");
+    }
     configure.current_dir(&obj);
     println!("Configuring openssl for {}", target);
     build.run_quiet(&mut configure);