Unrolled build for rust-lang#140251

Rollup merge of rust-lang#140251 - Zalathar:coverage-dump-path, r=clubby789

coverage-dump: Resolve global file IDs to filenames

The coverage-dump tool, used by coverage tests, currently includes “global file ID” numbers in its dump output.

This PR adds support for parsing coverage filename information from LLVM assembly `.ll` files, and resolving those file IDs to the corresponding filename, for inclusion in dump output.

This makes dump output more informative, especially for test cases involving multiple files, and will be important for testing expansion region support in the future.

---

The bootstrap changes don't necessarily have to land at the same time (e.g. they could be deferred to after the stage0 redesign if requested), but I would prefer to land them now if possible.

Author: rust-timer

Cargo.lock

@@ -777,6 +777,7 @@ name = "coverage-dump"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "itertools",
  "leb128",
  "md-5",
  "miniz_oxide 0.7.4",

src/bootstrap/src/core/build_steps/check.rs

@@ -527,3 +527,70 @@ tool_check_step!(Bootstrap { path: "src/bootstrap", default: false });
 // `run-make-support` will be built as part of suitable run-make compiletest test steps, but support
 // check to make it easier to work on.
 tool_check_step!(RunMakeSupport { path: "src/tools/run-make-support", default: false });
+
+/// Check step for the `coverage-dump` bootstrap tool. The coverage-dump tool
+/// is used internally by coverage tests.
+///
+/// FIXME(Zalathar): This is temporarily separate from the other tool check
+/// steps so that it can use the stage 0 compiler instead of `top_stage`,
+/// without introducing conflicts with the stage 0 redesign (#119899).
+///
+/// After the stage 0 redesign lands, we can look into using the stage 0
+/// compiler to check all bootstrap tools (#139170).
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(crate) struct CoverageDump;
+
+impl CoverageDump {
+    const PATH: &str = "src/tools/coverage-dump";
+}
+
+impl Step for CoverageDump {
+    type Output = ();
+
+    /// Most contributors won't care about coverage-dump, so don't make their
+    /// check builds slower unless they opt in and check it explicitly.
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path(Self::PATH)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self {});
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        // Make sure we haven't forgotten any fields, if there are any.
+        let Self {} = self;
+        let display_name = "coverage-dump";
+        let host = builder.config.build;
+        let target = host;
+        let mode = Mode::ToolBootstrap;
+
+        let compiler = builder.compiler(0, host);
+        let cargo = prepare_tool_cargo(
+            builder,
+            compiler,
+            mode,
+            target,
+            builder.kind,
+            Self::PATH,
+            SourceType::InTree,
+            &[],
+        );
+
+        let stamp = BuildStamp::new(&builder.cargo_out(compiler, mode, target))
+            .with_prefix(&format!("{display_name}-check"));
+
+        let _guard = builder.msg_tool(
+            builder.kind,
+            mode,
+            display_name,
+            compiler.stage,
+            &compiler.host,
+            &target,
+        );
+        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+    }
+}

src/bootstrap/src/core/build_steps/run.rs

@@ -392,3 +392,31 @@ impl Step for CyclicStep {
         builder.ensure(CyclicStep { n: self.n.saturating_sub(1) })
     }
 }
+
+/// Step to manually run the coverage-dump tool (`./x run coverage-dump`).
+///
+/// The coverage-dump tool is an internal detail of coverage tests, so this run
+/// step is only needed when testing coverage-dump manually.
+#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+pub struct CoverageDump;
+
+impl Step for CoverageDump {
+    type Output = ();
+
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/coverage-dump")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self {});
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        let mut cmd = builder.tool_cmd(Tool::CoverageDump);
+        cmd.args(&builder.config.free_args);
+        cmd.run(builder);
+    }
+}

src/bootstrap/src/core/build_steps/test.rs

@@ -54,6 +54,7 @@ impl Step for CrateBootstrap {
         run.path("src/tools/jsondoclint")
             .path("src/tools/suggest-tests")
             .path("src/tools/replace-version-placeholder")
+            .path("src/tools/coverage-dump")
             // We want `./x test tidy` to _run_ the tidy tool, not its tests.
             // So we need a separate alias to test the tidy tool itself.
             .alias("tidyselftest")

src/bootstrap/src/core/builder/mod.rs

@@ -961,6 +961,7 @@ impl<'a> Builder<'a> {
                 check::RunMakeSupport,
                 check::Compiletest,
                 check::FeaturesStatusDump,
+                check::CoverageDump,
             ),
             Kind::Test => describe!(
                 crate::core::build_steps::toolstate::ToolStateCheck,
@@ -1114,6 +1115,7 @@ impl<'a> Builder<'a> {
                 run::UnicodeTableGenerator,
                 run::FeaturesStatusDump,
                 run::CyclicStep,
+                run::CoverageDump,
             ),
             Kind::Setup => {
                 describe!(setup::Profile, setup::Hook, setup::Link, setup::Editor)

src/tools/coverage-dump/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.71"
+itertools = "0.12"
 leb128 = "0.2.5"
 md5 = { package = "md-5" , version = "0.10.5" }
 miniz_oxide = "0.7.1"

src/tools/coverage-dump/src/covfun.rs

@@ -1,23 +1,33 @@
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Write as _};
-use std::sync::OnceLock;
+use std::sync::LazyLock;
 
-use anyhow::{Context, anyhow};
+use anyhow::{Context, anyhow, bail, ensure};
+use itertools::Itertools;
 use regex::Regex;
 
-use crate::parser::{Parser, unescape_llvm_string_contents};
+use crate::covmap::FilenameTables;
+use crate::llvm_utils::unescape_llvm_string_contents;
+use crate::parser::Parser;
+
+#[cfg(test)]
+mod tests;
 
 pub(crate) fn dump_covfun_mappings(
     llvm_ir: &str,
+    filename_tables: &FilenameTables,
     function_names: &HashMap<u64, String>,
 ) -> anyhow::Result<()> {
     // Extract function coverage entries from the LLVM IR assembly, and associate
     // each entry with its (demangled) name.
     let mut covfun_entries = llvm_ir
         .lines()
-        .filter_map(covfun_line_data)
-        .map(|line_data| (function_names.get(&line_data.name_hash).map(String::as_str), line_data))
-        .collect::<Vec<_>>();
+        .filter(|line| is_covfun_line(line))
+        .map(parse_covfun_line)
+        .map_ok(|line_data| {
+            (function_names.get(&line_data.name_hash).map(String::as_str), line_data)
+        })
+        .collect::<Result<Vec<_>, _>>()?;
     covfun_entries.sort_by(|a, b| {
         // Sort entries primarily by name, to help make the order consistent
         // across platforms and relatively insensitive to changes.
@@ -41,8 +51,12 @@ pub(crate) fn dump_covfun_mappings(
         println!("Number of files: {num_files}");
 
         for i in 0..num_files {
-            let global_file_id = parser.read_uleb128_u32()?;
-            println!("- file {i} => global file {global_file_id}");
+            let global_file_id = parser.read_uleb128_usize()?;
+            let &CovfunLineData { filenames_hash, .. } = line_data;
+            let Some(filename) = filename_tables.lookup(filenames_hash, global_file_id) else {
+                bail!("couldn't resolve global file: {filenames_hash}, {global_file_id}");
+            };
+            println!("- file {i} => {filename}");
         }
 
         let num_expressions = parser.read_uleb128_u32()?;
@@ -107,36 +121,50 @@ pub(crate) fn dump_covfun_mappings(
     Ok(())
 }
 
+#[derive(Debug, PartialEq, Eq)]
 struct CovfunLineData {
-    name_hash: u64,
     is_used: bool,
+    name_hash: u64,
+    filenames_hash: u64,
     payload: Vec<u8>,
 }
 
-/// Checks a line of LLVM IR assembly to see if it contains an `__llvm_covfun`
-/// entry, and if so extracts relevant data in a `CovfunLineData`.
-fn covfun_line_data(line: &str) -> Option<CovfunLineData> {
-    let re = {
-        // We cheat a little bit and match variable names `@__covrec_[HASH]u`
-        // rather than the section name, because the section name is harder to
-        // extract and differs across Linux/Windows/macOS. We also extract the
-        // symbol name hash from the variable name rather than the data, since
-        // it's easier and both should match.
-        static RE: OnceLock<Regex> = OnceLock::new();
-        RE.get_or_init(|| {
-            Regex::new(
-                r#"^@__covrec_(?<name_hash>[0-9A-Z]+)(?<is_used>u)? = .*\[[0-9]+ x i8\] c"(?<payload>[^"]*)".*$"#,
-            )
-            .unwrap()
-        })
-    };
+fn is_covfun_line(line: &str) -> bool {
+    line.starts_with("@__covrec_")
+}
 
-    let captures = re.captures(line)?;
-    let name_hash = u64::from_str_radix(&captures["name_hash"], 16).unwrap();
+/// Given a line of LLVM IR assembly that should contain an `__llvm_covfun`
+/// entry, parses it to extract relevant data in a `CovfunLineData`.
+fn parse_covfun_line(line: &str) -> anyhow::Result<CovfunLineData> {
+    ensure!(is_covfun_line(line));
+
+    // We cheat a little bit and match variable names `@__covrec_[HASH]u`
+    // rather than the section name, because the section name is harder to
+    // extract and differs across Linux/Windows/macOS.
+    const RE_STRING: &str = r#"(?x)^
+        @__covrec_[0-9A-Z]+(?<is_used>u)?
+        \ = \ # (trailing space)
+        .*
+        <\{
+            \ i64 \ (?<name_hash> -? [0-9]+),
+            \ i32 \ -? [0-9]+, # (length of payload; currently unused)
+            \ i64 \ -? [0-9]+, # (source hash; currently unused)
+            \ i64 \ (?<filenames_hash> -? [0-9]+),
+            \ \[ [0-9]+ \ x \ i8 \] \ c"(?<payload>[^"]*)"
+            \ # (trailing space)
+        }>
+        .*$
+    "#;
+    static RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(RE_STRING).unwrap());
+
+    let captures =
+        RE.captures(line).with_context(|| format!("couldn't parse covfun line: {line:?}"))?;
     let is_used = captures.name("is_used").is_some();
+    let name_hash = i64::from_str_radix(&captures["name_hash"], 10).unwrap() as u64;
+    let filenames_hash = i64::from_str_radix(&captures["filenames_hash"], 10).unwrap() as u64;
     let payload = unescape_llvm_string_contents(&captures["payload"]);
 
-    Some(CovfunLineData { name_hash, is_used, payload })
+    Ok(CovfunLineData { is_used, name_hash, filenames_hash, payload })
 }
 
 // Extra parser methods only needed when parsing `covfun` payloads.

src/tools/coverage-dump/src/covfun/tests.rs

@@ -0,0 +1,53 @@
+use super::{CovfunLineData, parse_covfun_line};
+
+/// Integers in LLVM IR are not inherently signed/unsigned, and the text format tends
+/// to emit them in signed form, so this helper function converts `i64` to `u64`.
+fn as_u64(x: i64) -> u64 {
+    x as u64
+}
+
+#[test]
+fn parse_covfun_line_data() {
+    struct Case {
+        line: &'static str,
+        expected: CovfunLineData,
+    }
+    let cases = &[
+        // Copied from `trivial.ll`:
+        Case {
+            line: r#"@__covrec_49A9BAAE5F896E81u = linkonce_odr hidden constant <{ i64, i32, i64, i64, [9 x i8] }> <{ i64 5307978893922758273, i32 9, i64 445092354169400020, i64 6343436898695299756, [9 x i8] c"\01\01\00\01\01\03\01\00\0D" }>, section "__LLVM_COV,__llvm_covfun", align 8"#,
+            expected: CovfunLineData {
+                is_used: true,
+                name_hash: as_u64(5307978893922758273),
+                filenames_hash: as_u64(6343436898695299756),
+                payload: b"\x01\x01\x00\x01\x01\x03\x01\x00\x0D".to_vec(),
+            },
+        },
+        // Copied from `on-off-sandwich.ll`:
+        Case {
+            line: r#"@__covrec_D0CE53C5E64F319Au = linkonce_odr hidden constant <{ i64, i32, i64, i64, [14 x i8] }> <{ i64 -3400688559180533350, i32 14, i64 7307957714577672185, i64 892196767019953100, [14 x i8] c"\01\01\00\02\01\10\05\02\10\01\07\05\00\06" }>, section "__LLVM_COV,__llvm_covfun", align 8"#,
+            expected: CovfunLineData {
+                is_used: true,
+                name_hash: as_u64(-3400688559180533350),
+                filenames_hash: as_u64(892196767019953100),
+                payload: b"\x01\x01\x00\x02\x01\x10\x05\x02\x10\x01\x07\x05\x00\x06".to_vec(),
+            },
+        },
+        // Copied from `no-core.ll`:
+        Case {
+            line: r#"@__covrec_F8016FC82D46106u = linkonce_odr hidden constant <{ i64, i32, i64, i64, [9 x i8] }> <{ i64 1116917981370409222, i32 9, i64 -8857254680411629915, i64 -3625186110715410276, [9 x i8] c"\01\01\00\01\01\0C\01\00\0D" }>, section "__LLVM_COV,__llvm_covfun", align 8"#,
+            expected: CovfunLineData {
+                is_used: true,
+                name_hash: as_u64(1116917981370409222),
+                filenames_hash: as_u64(-3625186110715410276),
+                payload: b"\x01\x01\x00\x01\x01\x0C\x01\x00\x0D".to_vec(),
+            },
+        },
+    ];
+
+    for &Case { line, ref expected } in cases {
+        println!("- {line}");
+        let line_data = parse_covfun_line(line).map_err(|e| e.to_string());
+        assert_eq!(line_data.as_ref(), Ok(expected));
+    }
+}

src/tools/coverage-dump/src/covmap.rs

@@ -0,0 +1,75 @@
+use std::collections::HashMap;
+use std::sync::LazyLock;
+
+use anyhow::{Context, ensure};
+use regex::Regex;
+
+use crate::llvm_utils::{truncated_md5, unescape_llvm_string_contents};
+use crate::parser::Parser;
+
+#[derive(Debug, Default)]
+pub(crate) struct FilenameTables {
+    map: HashMap<u64, Vec<String>>,
+}
+
+impl FilenameTables {
+    pub(crate) fn lookup(&self, filenames_hash: u64, global_file_id: usize) -> Option<&str> {
+        let table = self.map.get(&filenames_hash)?;
+        let filename = table.get(global_file_id)?;
+        Some(filename)
+    }
+}
+
+struct CovmapLineData {
+    payload: Vec<u8>,
+}
+
+pub(crate) fn make_filename_tables(llvm_ir: &str) -> anyhow::Result<FilenameTables> {
+    let mut map = HashMap::default();
+
+    for line in llvm_ir.lines().filter(|line| is_covmap_line(line)) {
+        let CovmapLineData { payload } = parse_covmap_line(line)?;
+
+        let mut parser = Parser::new(&payload);
+        let n_filenames = parser.read_uleb128_usize()?;
+        let uncompressed_bytes = parser.read_chunk_to_uncompressed_bytes()?;
+        parser.ensure_empty()?;
+
+        let mut filenames_table = vec![];
+
+        let mut parser = Parser::new(&uncompressed_bytes);
+        for _ in 0..n_filenames {
+            let len = parser.read_uleb128_usize()?;
+            let bytes = parser.read_n_bytes(len)?;
+            let filename = str::from_utf8(bytes)?;
+            filenames_table.push(filename.to_owned());
+        }
+
+        let filenames_hash = truncated_md5(&payload);
+        map.insert(filenames_hash, filenames_table);
+    }
+
+    Ok(FilenameTables { map })
+}
+
+fn is_covmap_line(line: &str) -> bool {
+    line.starts_with("@__llvm_coverage_mapping ")
+}
+
+fn parse_covmap_line(line: &str) -> anyhow::Result<CovmapLineData> {
+    ensure!(is_covmap_line(line));
+
+    const RE_STRING: &str = r#"(?x)^
+        @__llvm_coverage_mapping \ =
+        .*
+        \[ [0-9]+ \ x \ i8 \] \ c"(?<payload>[^"]*)"
+        .*$
+    "#;
+    static RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(RE_STRING).unwrap());
+
+    let captures =
+        RE.captures(line).with_context(|| format!("couldn't parse covmap line: {line:?}"))?;
+    let payload = unescape_llvm_string_contents(&captures["payload"]);
+
+    Ok(CovmapLineData { payload })
+}