update Miri

Cargo.lock

@@ -2262,6 +2262,7 @@ dependencies = [
  "rand 0.8.5",
  "regex",
  "rustc-workspace-hack",
+ "rustc_version",
  "shell-escape",
  "smallvec",
  "ui_test",

src/bootstrap/test.rs

@@ -461,24 +461,30 @@ impl Step for RustDemangler {
 pub struct Miri {
     stage: u32,
     host: TargetSelection,
+    target: TargetSelection,
 }
 
 impl Step for Miri {
     type Output = ();
-    const ONLY_HOSTS: bool = true;
+    const ONLY_HOSTS: bool = false;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.path("src/tools/miri")
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(Miri { stage: run.builder.top_stage, host: run.target });
+        run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
+            host: run.build_triple(),
+            target: run.target,
+        });
     }
 
     /// Runs `cargo test` for miri.
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let host = self.host;
+        let target = self.target;
         let compiler = builder.compiler(stage, host);
         // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
         // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
@@ -495,7 +501,7 @@ impl Step for Miri {
         builder.ensure(compile::Std::new(compiler_std, host));
         let sysroot = builder.sysroot(compiler_std);
 
-        // # Run `cargo miri setup`.
+        // # Run `cargo miri setup` for the given target.
         let mut cargo = tool::prepare_tool_cargo(
             builder,
             compiler,
@@ -508,6 +514,7 @@ impl Step for Miri {
         );
         cargo.add_rustc_lib_path(builder, compiler);
         cargo.arg("--").arg("miri").arg("setup");
+        cargo.arg("--target").arg(target.rustc_target_arg());
 
         // Tell `cargo miri setup` where to find the sources.
         cargo.env("MIRI_LIB_SRC", builder.src.join("library"));
@@ -556,17 +563,54 @@ impl Step for Miri {
         cargo.add_rustc_lib_path(builder, compiler);
 
         // miri tests need to know about the stage sysroot
-        cargo.env("MIRI_SYSROOT", miri_sysroot);
+        cargo.env("MIRI_SYSROOT", &miri_sysroot);
         cargo.env("MIRI_HOST_SYSROOT", sysroot);
         cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
-        cargo.env("MIRI", miri);
+        cargo.env("MIRI", &miri);
         // propagate --bless
         if builder.config.cmd.bless() {
             cargo.env("MIRI_BLESS", "Gesundheit");
         }
 
+        // Set the target.
+        cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg());
+        // Forward test filters.
+        cargo.arg("--").args(builder.config.cmd.test_args());
+
+        let mut cargo = Command::from(cargo);
+        builder.run(&mut cargo);
+
+        // # Run `cargo miri test`.
+        // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures
+        // that we get the desired output), but that is sufficient to make sure that the libtest harness
+        // itself executes properly under Miri.
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            host,
+            "run",
+            "src/tools/miri/cargo-miri",
+            SourceType::Submodule,
+            &[],
+        );
+        cargo.add_rustc_lib_path(builder, compiler);
+        cargo.arg("--").arg("miri").arg("test");
+        cargo
+            .arg("--manifest-path")
+            .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml"));
+        cargo.arg("--target").arg(target.rustc_target_arg());
+        cargo.arg("--tests"); // don't run doctests, they are too confused by the staging
         cargo.arg("--").args(builder.config.cmd.test_args());
 
+        // Tell `cargo miri` where to find things.
+        cargo.env("MIRI_SYSROOT", &miri_sysroot);
+        cargo.env("MIRI_HOST_SYSROOT", sysroot);
+        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+        cargo.env("MIRI", &miri);
+        // Debug things.
+        cargo.env("RUST_BACKTRACE", "1");
+
         let mut cargo = Command::from(cargo);
         builder.run(&mut cargo);
     }

src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh

@@ -25,3 +25,8 @@ python3 "$X_PY" test --stage 2 check-tools
 python3 "$X_PY" test --stage 2 src/tools/clippy
 python3 "$X_PY" test --stage 2 src/tools/rustfmt
 python3 "$X_PY" test --stage 2 src/tools/miri
+# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc.
+# Also cover some other targets (on both of these hosts) via cross-testing.
+python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc
+#FIXME(https://github.com/rust-lang/rust/issues/103519): macOS testing is currently disabled
+# python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin

src/tools/miri/.github/workflows/ci.yml

@@ -22,6 +22,7 @@ jobs:
       RUST_BACKTRACE: 1
       HOST_TARGET: ${{ matrix.host_target }}
     strategy:
+      fail-fast: false
       matrix:
         build: [linux64, macos, win32]
         include:
@@ -61,7 +62,7 @@ jobs:
           restore-keys: ${{ runner.os }}-cargo
 
       - name: Install rustup-toolchain-install-master
-        if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
         shell: bash
         run: |
           cargo install -f rustup-toolchain-install-master
@@ -89,11 +90,46 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Install required toolchain
-        # We need a toolchain that can actually build Miri, just a nightly won't do.
+
+      # This is exactly duplicated from above. GHA is pretty terrible when it comes
+      # to avoiding code duplication.
+
+      # Cache the global cargo directory, but NOT the local `target` directory which
+      # we cannot reuse anyway when the nightly changes (and it grows quite large
+      # over time).
+      - name: Add cache for cargo
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            # Taken from <https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci>.
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            # contains package information of crates installed via `cargo install`.
+            ~/.cargo/.crates.toml
+            ~/.cargo/.crates2.json
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo
+
+      - name: Install rustup-toolchain-install-master
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        shell: bash
+        run: |
+          cargo install -f rustup-toolchain-install-master
+
+      - name: Install "master" toolchain
+        shell: bash
         run: |
-          cargo install rustup-toolchain-install-master # TODO: cache this?
           ./rustup-toolchain "" -c clippy
+
+      - name: Show Rust version
+        run: |
+          rustup show
+          rustc -Vv
+          cargo -V
+
       - name: rustfmt
         run: ./miri fmt --check
       - name: clippy

src/tools/miri/CONTRIBUTING.md

@@ -104,7 +104,7 @@ MIRI_LOG=rustc_mir::interpret=info,miri::stacked_borrows ./miri run tests/pass/v
 In addition, you can set `MIRI_BACKTRACE=1` to get a backtrace of where an
 evaluation error was originally raised.
 
-#### UI testing
+### UI testing
 
 We use ui-testing in Miri, meaning we generate `.stderr` and `.stdout` files for the output
 produced by Miri. You can use `./miri bless` to automatically (re)generate these files when
@@ -257,7 +257,7 @@ Note: When you are working with a locally built rustc or any other toolchain tha
 is not the same as the one in `rust-version`, you should not have `.auto-everything` or
 `.auto-toolchain` as that will keep resetting your toolchain.
 
-```
+```sh
 rm -f .auto-everything .auto-toolchain
 ```
 
@@ -275,3 +275,51 @@ see <https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html>.
 
 With this, you should now have a working development setup! See
 [above](#building-and-testing-miri) for how to proceed working on Miri.
+
+## Advanced topic: Syncing with the rustc repo
+
+We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit
+changes between the rustc and Miri repositories. For now, a fork of josh needs to be built
+from source. This downloads and runs josh:
+
+```sh
+git clone https://github.com/RalfJung/josh
+cd josh
+cargo run --release -p josh-proxy -- --local=$(pwd)/local --remote=https://github.com --no-background
+```
+
+### Importing changes from the rustc repo
+
+We assume we start on an up-to-date master branch in the Miri repo.
+
+```sh
+# Fetch rustc side of the history. Takes ca 5 min the first time.
+# Do NOT change that commit ID, it needs to be exactly this!
+git fetch http://localhost:8000/rust-lang/rust.git:at_commit=75dd959a3a40eb5b4574f8d2e23aa6efbeb33573[:prefix=src/tools/miri]:/src/tools/miri.git master
+# Include that history into ours.
+git merge FETCH_HEAD -m "merge rustc history"
+# Update toolchain reference and apply formatting.
+./rustup-toolchain HEAD && ./miri fmt
+git commit -am "rustup"
+```
+
+Now push this to a new branch in your Miri fork, and create a PR. It is worth
+running `./miri test` locally in parallel, since the test suite in the Miri repo
+is stricter than the one on the rustc side, so some small tweaks might be
+needed.
+
+### Exporting changes to the rustc repo
+
+We will use the josh proxy to push to your fork of rustc. You need to make sure
+that the master branch of your fork is up-to-date. Also make sure that there
+exists no branch called `miri` in your fork. Then run the following in the Miri
+repo, assuming we are on an up-to-date master branch:
+
+```sh
+# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
+# Do NOT change that commit ID, it needs to be exactly this!
+git push http://localhost:8000/YOUR_NAME/rust.git:at_commit=75dd959a3a40eb5b4574f8d2e23aa6efbeb33573[:prefix=src/tools/miri]:/src/tools/miri.git -o base=master HEAD:miri
+```
+
+This will create a new branch in your fork, and the output should include a link
+to create a rustc PR that will integrate those changes into the main repository.

src/tools/miri/Cargo.lock

@@ -320,9 +320,9 @@ dependencies = [
 
 [[package]]
 name = "libffi-sys"
-version = "2.0.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab4106b7f09d7b87d021334d5618fac1dfcfb824d4c5fe111ff0074dfd242e15"
+checksum = "84e78d02e5a8eae9c24c38ce6e6026f80e16dff76adcdae4bc5c6c52c2de4a60"
 dependencies = [
  "cc",
 ]
@@ -419,6 +419,7 @@ dependencies = [
  "rand",
  "regex",
  "rustc-workspace-hack",
+ "rustc_version",
  "shell-escape",
  "smallvec",
  "ui_test",

src/tools/miri/Cargo.toml

@@ -31,14 +31,17 @@ smallvec = "1.7"
 rustc-workspace-hack = "1.0.0"
 measureme = "10.0.0"
 
-[target."cfg(unix)".dependencies]
+[target.'cfg(unix)'.dependencies]
 libc = "0.2"
+
+[target.'cfg(target_os = "linux")'.dependencies]
 libffi = "3.0.0"
 libloading = "0.7"
 
 [dev-dependencies]
 colored = "2"
 ui_test = "0.3.1"
+rustc_version = "0.4"
 # Features chosen to match those required by env_logger, to avoid rebuilds
 regex = { version = "1.5.5", default-features = false, features = ["perf", "std"] }
 lazy_static = "1.4.0"

src/tools/miri/README.md

@@ -377,6 +377,11 @@ to Miri failing to detect cases of undefined behavior in a program.
 * `-Zmiri-retag-fields` changes Stacked Borrows retagging to recurse into fields.
   This means that references in fields of structs/enums/tuples/arrays/... are retagged,
   and in particular, they are protected when passed as function arguments.
+* `-Zmiri-retag-fields=<all|none|scalar>` controls when Stacked Borrows retagging recurses into
+  fields. `all` means it always recurses (like `-Zmiri-retag-fields`), `none` means it never
+  recurses (the default), `scalar` means it only recurses for types where we would also emit
+  `noalias` annotations in the generated LLVM IR (types passed as indivudal scalars or pairs of
+  scalars).
 * `-Zmiri-tag-gc=<blocks>` configures how often the pointer tag garbage collector runs. The default
   is to search for and remove unreachable tags once every `10000` basic blocks. Setting this to
   `0` disables the garbage collector, which causes some programs to have explosive memory usage
@@ -435,11 +440,10 @@ Moreover, Miri recognizes some environment variables:
   purpose.
 * `MIRI_NO_STD` (recognized by `cargo miri` and the test suite) makes sure that the target's
   sysroot is built without libstd. This allows testing and running no_std programs.
-* `MIRI_BLESS` (recognized by the test suite) overwrite all `stderr` and `stdout` files
-  instead of checking whether the output matches.
-* `MIRI_SKIP_UI_CHECKS` (recognized by the test suite) don't check whether the
-  `stderr` or `stdout` files match the actual output. Useful for the rustc test suite
-  which has subtle differences that we don't care about.
+* `MIRI_BLESS` (recognized by the test suite and `cargo-miri-test/run-test.py`): overwrite all
+  `stderr` and `stdout` files instead of checking whether the output matches.
+* `MIRI_SKIP_UI_CHECKS` (recognized by the test suite): don't check whether the
+  `stderr` or `stdout` files match the actual output.
 
 The following environment variables are *internal* and must not be used by
 anyone but Miri itself. They are used to communicate between different Miri
@@ -532,6 +536,27 @@ extern "Rust" {
     /// This is internal and unstable and should not be used; we give it here
     /// just to be complete.
     fn miri_start_panic(payload: *mut u8) -> !;
+
+    /// Miri-provided extern function to get the internal unique identifier for the allocation that a pointer
+    /// points to. This is only useful as an input to `miri_print_stacks`, and it is a separate call because
+    /// getting a pointer to an allocation at runtime can change the borrow stacks in the allocation.
+    fn miri_get_alloc_id(ptr: *const ()) -> u64;
+
+    /// Miri-provided extern function to print (from the interpreter, not the program) the contents of all
+    /// borrow stacks in an allocation. The format of what this emits is unstable and may change at any time.
+    /// In particular, users should be aware that Miri will periodically attempt to garbage collect the
+    /// contents of all stacks. Callers of this function may wish to pass `-Zmiri-tag-gc=0` to disable the GC.
+    fn miri_print_stacks(alloc_id: u64);
+
+    /// Miri-provided extern function to print (from the interpreter, not the
+    /// program) the contents of a section of program memory, as bytes. Bytes
+    /// written using this function will emerge from the interpreter's stdout.
+    fn miri_write_to_stdout(bytes: &[u8]);
+
+    /// Miri-provided extern function to print (from the interpreter, not the
+    /// program) the contents of a section of program memory, as bytes. Bytes
+    /// written using this function will emerge from the interpreter's stderr.
+    fn miri_write_to_stderr(bytes: &[u8]);
 }
 ```
 

src/tools/miri/bench-cargo-miri/serde2/src/main.rs

@@ -5,6 +5,7 @@ use serde::Deserialize;
 use std::thread;
 
 #[derive(Deserialize)]
+#[allow(unused)]
 struct DeriveStruct {
     buffer: Vec<i16>,
 }

src/tools/miri/rust-version

@@ -1 +1 @@
-acb8934fd57b3c2740c4abac0a5728c2c9b1423b
+b1ab3b738ac718da74cd4aa0bb7f362d0adbdf84

src/tools/miri/src/bin/miri.rs

@@ -32,7 +32,7 @@ use rustc_middle::{
 };
 use rustc_session::{config::CrateType, search_paths::PathKind, CtfeBacktrace};
 
-use miri::{BacktraceStyle, ProvenanceMode};
+use miri::{BacktraceStyle, ProvenanceMode, RetagFields};
 
 struct MiriCompilerCalls {
     miri_config: miri::MiriConfig,
@@ -426,7 +426,14 @@ fn main() {
         } else if arg == "-Zmiri-mute-stdout-stderr" {
             miri_config.mute_stdout_stderr = true;
         } else if arg == "-Zmiri-retag-fields" {
-            miri_config.retag_fields = true;
+            miri_config.retag_fields = RetagFields::Yes;
+        } else if let Some(retag_fields) = arg.strip_prefix("-Zmiri-retag-fields=") {
+            miri_config.retag_fields = match retag_fields {
+                "all" => RetagFields::Yes,
+                "none" => RetagFields::No,
+                "scalar" => RetagFields::OnlyScalar,
+                _ => show_error!("`-Zmiri-retag-fields` can only be `all`, `none`, or `scalar`"),
+            };
         } else if arg == "-Zmiri-track-raw-pointers" {
             eprintln!(
                 "WARNING: `-Zmiri-track-raw-pointers` has no effect; it is enabled by default"