Source-code based coverage is failing to identify some executable lines

[wpbrown]

I tried to generate test code coverage with this source file using -Zinstrument-coverage and the LLVM tools.

Source File:
https://github.com/wpbrown/rust-codecov/blob/main/codecovsample/src/main.rs

I expected to see this happen: 76% line coverage

Instead, this happened: 83% line coverage.

cargo-kcov, -Zprofile+grcov, and tarpaulin agree on 76%.

In the output below lines 65-67, 116-118, 163-165 were not executable but should have been executable+uncovered. Line 169-170 should be executable+covered.

coverage output

#![allow(dead_code)]#![allow(dead_code)]
    2|       |
    3|       |use async_trait::async_trait;
    4|       |
    5|      1|fn main() {
    6|      1|    println!("codecovsample::main");
    7|      1|}
    8|       |
    9|       |enum Covered {
   10|       |    Variant1,
   11|       |    Variant2,
   12|       |}
   13|       |enum Uncovered {
   14|       |    Variant1,
   15|       |    Variant2,
   16|       |}
   17|       |enum PartiallyCovered {
   18|       |    Variant1,
   19|       |    Variant2,
   20|       |}
   21|       |
   22|      2|fn fn_covered_enum(input: Covered) {
   23|      2|    match input {
   24|      2|        Covered::Variant1 => { println!("Variant1"); }
                                           ^1
   25|      1|        Covered::Variant2 => { println!("Variant2"); }
   26|       |    }
   27|      2|}
   28|       |
   29|      0|fn fn_uncovered_enum(input: Uncovered) {
   30|      0|    match input {
   31|      0|        Uncovered::Variant1 => { println!("Variant1"); }
   32|      0|        Uncovered::Variant2 => { println!("Variant2"); }
   33|       |    }
   34|      0|}
   35|       |
   36|      1|fn fn_partially_covered_enum(input: PartiallyCovered) {
   37|      1|    match input {
   38|      1|        PartiallyCovered::Variant1 => { println!("Variant1"); }
   39|      0|        PartiallyCovered::Variant2 => { println!("Variant2"); }
   40|       |    }
   41|      1|}
   42|       |
   43|       |trait ATrait {
   44|       |    fn covered(&self);
   45|       |    fn uncovered(&self);
   46|       |    fn func_covered();    
   47|       |    fn func_uncovered();
   48|       |
   49|      2|    fn default_covered(&self) {
   50|      2|        println!("default_covered");
   51|      2|    }
  ------------------
  | <codecovsample::ATraitImplDirect as codecovsample::ATrait>::default_covered:
  |   49|      1|    fn default_covered(&self) {
  |   50|      1|        println!("default_covered");
  |   51|      1|    }
  ------------------
  | <codecovsample::ATraitImplGeneric as codecovsample::ATrait>::default_covered:
  |   49|      1|    fn default_covered(&self) {
  |   50|      1|        println!("default_covered");
  |   51|      1|    }
  ------------------
   52|       |    
   53|      0|    fn default_uncovered(&self) {
   54|      0|        println!("default_uncovered");
   55|      0|    }
   56|       |}
   57|       |trait BTrait {
   58|       |    fn covered(&self);
   59|       |    fn uncovered(&self);
   60|       |
   61|      1|    fn default_covered(&self) {
   62|      1|        println!("default_covered");
   63|      1|    }
   64|       |    
   65|       |    fn default_uncovered(&self) {
   66|       |        println!("default_uncovered");
   67|       |    }
   68|       |}
   69|       |
   70|       |struct ATraitImplDirect;
   71|       |
   72|       |impl ATrait for ATraitImplDirect {
   73|      1|    fn covered(&self) {
   74|      1|        println!("covered")
   75|      1|    }
   76|       |
   77|      0|    fn uncovered(&self) {
   78|      0|        println!("uncovered");
   79|      0|    }
   80|       |
   81|      1|    fn func_covered() {
   82|      1|        println!("func_covered");
   83|      1|    }
   84|       |
   85|      0|    fn func_uncovered() {
   86|      0|        println!("func_covered");
   87|      0|    }
   88|       |}
   89|       |
   90|       |struct ATraitImplGeneric;
   91|       |
   92|       |impl ATrait for ATraitImplGeneric {
   93|      1|    fn covered(&self) {
   94|      1|        println!("covered")
   95|      1|    }
   96|       |
   97|      0|    fn uncovered(&self) {
   98|      0|        println!("uncovered");
   99|      0|    }
  100|       |
  101|      1|    fn func_covered() {
  102|      1|        println!("func_covered");
  103|      1|    }
  104|       |
  105|      0|    fn func_uncovered() {
  106|      0|        println!("func_covered");
  107|      0|    }
  108|       |}
  109|       |struct BTraitImplBoxed;
  110|       |
  111|       |impl BTrait for BTraitImplBoxed {
  112|      1|    fn covered(&self) {
  113|      1|        println!("covered")
  114|      1|    }
  115|       |
  116|       |    fn uncovered(&self) {
  117|       |        println!("uncovered");
  118|       |    }
  119|       |}
  120|       |
  121|       |macro_rules! simple_rule {
  122|       |    () => {
  123|       |        println!("simple rule");
  124|       |    };
  125|       |}
  126|       |
  127|      1|fn call_simple_rule() {
  128|      1|    simple_rule!();
  129|      1|}
  130|       |
  131|      1|fn call_generic_atrait<T: ATrait>(input: T) {
  132|      1|    input.covered();
  133|      1|    input.default_covered();
  134|      1|    T::func_covered();
  135|      1|}
  136|       |
  137|      1|async fn async_func() {
  138|       |    println!("async_func");
  139|       |}
  140|       |
  141|      1|async fn async_func_anon() {
  142|      1|    let x = async {
  143|      1|        println!("async_func");
  144|      1|    };
  145|      1|    x.await;
  146|      1|}
  147|       |
  148|       |#[async_trait]
  149|       |trait AsyncTrait {
  150|       |    async fn covered(&self);
  151|       |    async fn uncovered(&self);
  152|       |}
  153|       |
  154|       |struct AsyncTraitImpl;
  155|       |
  156|       |#[async_trait]
  157|       |impl AsyncTrait for AsyncTraitImpl {
  158|      1|    async fn covered(&self) {
  159|      1|        println!("covered");
  160|      1|        async_func_from_trait_covered().await;
  161|      1|    }
  162|       |
  163|       |    async fn uncovered(&self) {
  164|       |        println!("uncovered");
  165|       |    }
  166|       |}
  167|       |
  168|      1|async fn async_func_from_trait_covered() {
  169|       |    println!("covered async func from trait");
  170|       |}
  171|       |
  172|       |#[cfg(test)]
  173|       |mod tests {
  174|       |    use futures::executor::block_on;
  175|       |
  176|       |    use super::*;
  177|       |
  178|       |    #[test]
  179|      1|    fn test_main() {
  ------------------
  | codecovsample::tests::test_main::{closure#0}:
  |  179|      1|    fn test_main() {
  ------------------
  180|      1|        main();
  181|      1|    }
  ------------------
  | codecovsample::tests::test_main:
  |  179|      1|    fn test_main() {
  |  180|      1|        main();
  |  181|      1|    }
  ------------------
  182|       |
  183|       |    #[test]
  184|      1|    fn cover_enum() {
  ------------------
  | codecovsample::tests::cover_enum::{closure#0}:
  |  184|      1|    fn cover_enum() {
  ------------------
  185|      1|        fn_covered_enum(Covered::Variant1);
  186|      1|        fn_covered_enum(Covered::Variant2);
  187|      1|    }
  ------------------
  | codecovsample::tests::cover_enum:
  |  184|      1|    fn cover_enum() {
  |  185|      1|        fn_covered_enum(Covered::Variant1);
  |  186|      1|        fn_covered_enum(Covered::Variant2);
  |  187|      1|    }
  ------------------
  188|       |
  189|       |    #[test]
  190|      1|    fn partially_cover_enum() {
  ------------------
  | codecovsample::tests::partially_cover_enum::{closure#0}:
  |  190|      1|    fn partially_cover_enum() {
  ------------------
  191|      1|        fn_partially_covered_enum(PartiallyCovered::Variant1);
  192|      1|    }
  ------------------
  | codecovsample::tests::partially_cover_enum:
  |  190|      1|    fn partially_cover_enum() {
  |  191|      1|        fn_partially_covered_enum(PartiallyCovered::Variant1);
  |  192|      1|    }
  ------------------
  193|       |
  194|       |    #[test]
  195|      1|    fn cover_atrait_direct() {
  ------------------
  | codecovsample::tests::cover_atrait_direct::{closure#0}:
  |  195|      1|    fn cover_atrait_direct() {
  ------------------
  196|      1|        let x = ATraitImplDirect;
  197|      1|        x.covered();
  198|      1|        x.default_covered();
  199|      1|        <ATraitImplDirect as ATrait>::func_covered();
  200|      1|    }
  ------------------
  | codecovsample::tests::cover_atrait_direct:
  |  195|      1|    fn cover_atrait_direct() {
  |  196|      1|        let x = ATraitImplDirect;
  |  197|      1|        x.covered();
  |  198|      1|        x.default_covered();
  |  199|      1|        <ATraitImplDirect as ATrait>::func_covered();
  |  200|      1|    }
  ------------------
  201|       |    #[test]
  202|      1|    fn cover_atrait_boxed() {
  ------------------
  | codecovsample::tests::cover_atrait_boxed::{closure#0}:
  |  202|      1|    fn cover_atrait_boxed() {
  ------------------
  203|      1|        let x: Box<dyn BTrait> = Box::new(BTraitImplBoxed);
  204|      1|        x.covered();
  205|      1|        x.default_covered();
  206|      1|    }
  ------------------
  | codecovsample::tests::cover_atrait_boxed:
  |  202|      1|    fn cover_atrait_boxed() {
  |  203|      1|        let x: Box<dyn BTrait> = Box::new(BTraitImplBoxed);
  |  204|      1|        x.covered();
  |  205|      1|        x.default_covered();
  |  206|      1|    }
  ------------------
  207|       |
  208|       |    #[test]
  209|      1|    fn cover_simple_rule() {
  ------------------
  | codecovsample::tests::cover_simple_rule::{closure#0}:
  |  209|      1|    fn cover_simple_rule() {
  ------------------
  210|      1|        call_simple_rule();
  211|      1|    }
  ------------------
  | codecovsample::tests::cover_simple_rule:
  |  209|      1|    fn cover_simple_rule() {
  |  210|      1|        call_simple_rule();
  |  211|      1|    }
  ------------------
  212|       |
  213|       |    #[test]
  214|      1|    fn cover_generic_atrait() {
  ------------------
  | codecovsample::tests::cover_generic_atrait::{closure#0}:
  |  214|      1|    fn cover_generic_atrait() {
  ------------------
  215|      1|        let x = ATraitImplGeneric;
  216|      1|        call_generic_atrait(x);
  217|      1|    }
  ------------------
  | codecovsample::tests::cover_generic_atrait:
  |  214|      1|    fn cover_generic_atrait() {
  |  215|      1|        let x = ATraitImplGeneric;
  |  216|      1|        call_generic_atrait(x);
  |  217|      1|    }
  ------------------
  218|       |
  219|       |    #[test]
  220|      1|    fn cover_async_funcs() {
  ------------------
  | codecovsample::tests::cover_async_funcs::{closure#0}:
  |  220|      1|    fn cover_async_funcs() {
  ------------------
  221|      1|        block_on(async {
  222|      1|            async_func().await;
  223|      1|            async_func_anon().await;
  224|      1|        });
  225|      1|    }
  ------------------
  | codecovsample::tests::cover_async_funcs:
  |  220|      1|    fn cover_async_funcs() {
  |  221|      1|        block_on(async {
  |  222|       |            async_func().await;
  |  223|       |            async_func_anon().await;
  |  224|      1|        });
  |  225|      1|    }
  ------------------
  226|       |
  227|       |    #[test]
  228|      1|    fn cover_async_trait() {
  ------------------
  | codecovsample::tests::cover_async_trait::{closure#0}:
  |  228|      1|    fn cover_async_trait() {
  ------------------
  229|      1|        block_on(async {
  230|      1|            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  231|      1|            x.covered().await;
  232|      1|        });
  233|      1|    }
  ------------------
  | codecovsample::tests::cover_async_trait:
  |  228|      1|    fn cover_async_trait() {
  |  229|      1|        block_on(async {
  |  230|       |            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  |  231|       |            x.covered().await;
  |  232|      1|        });
  |  233|      1|    }
  ------------------
  234|       |}

Meta

rustc --version --verbose:

rustc 1.53.0-nightly (07e0e2ec2 2021-03-24)
binary: rustc
commit-hash: 07e0e2ec268c140e607e1ac7f49f145612d0f597
commit-date: 2021-03-24
host: x86_64-unknown-linux-gnu
release: 1.53.0-nightly
LLVM version: 12.0.0

[eggyal]

@rustbot label: +A-code-coverage
cc @richkadel

[richkadel]

Thanks for the bug report @wpbrown !

There was a recent bug fix (PR #83774) that addresses most of the missing coverage in your example (except for the last one).

I just tested with rustc 1.53.0-nightly (c755ee4ce 2021-04-04) and I see the expected coverage now for the first three ranges you reported seeing no coverage. These three are now showing coverage (with 0 executions) as expected:

In the output below lines 65-67, 116-118, 163-165 were not executable but should have been executable+uncovered.

I was surprised to see no coverage for line 169, and I do see the println!() did print, so that does seem to be a bug.

So as of today, this issue is still a bug that I will need to look into further:

Line 169-170 should be executable+covered.

[richkadel]

cc: @tmandry @wesleywiser

Note, here is a snippet showing the missing coverage and code relevant to its codepath.

I haven't thoroughly investigated this. I should be able to reproduce this in a simplified test (without the async_trait and executor dependencies) that I can add to our unit tests.

I don't have a theory yet. It could either be the closure's MIR is not included in coverage (seems unlikely to me), or llvm-cov is silently dropping coverage because it doesn't like something. (This is what was happening for the other missing coverage reported in this issue, that has since been resolved.)

The test (shown at the bottom) calls and awaits the implementation of covered() below:

  148|       |#[async_trait]
  149|       |trait AsyncTrait {
  150|       |    async fn covered(&self);
  151|       |    async fn uncovered(&self);
  152|       |}
  153|       |
  154|       |struct AsyncTraitImpl;
  155|       |
  156|       |#[async_trait]
  157|       |impl AsyncTrait for AsyncTraitImpl {
  158|      1|    async fn covered(&self) {
  159|      1|        println!("covered");
  160|      1|        async_func_from_trait_covered().await;
  161|      1|    }
  162|       |
  163|      0|    async fn uncovered(&self) {
  164|      0|        println!("uncovered");
  165|      0|    }
  166|       |}
  167|       |

I do see the test output, including the println!() from covered() above, and from the awaited call to async_func_from_trait_covered(), below:

covered
covered async func from trait

The async closure seems to be missing coverage, as shown here. Note the wrapper function is covered and called (showing the coverage of 1 on line 168), but there should also be coverage for the awaited implicit closure (lines 168-170, I believe).

Since the program output proves this is called, the coverage should be showing 1 execution.

  168|      1|async fn async_func_from_trait_covered() {
  169|       |    println!("covered async func from trait");
  170|       |}

And the test that invokes it is covered:

  226|       |
  227|       |    #[test]
  228|      1|    fn cover_async_trait() {
  ------------------
  | codecovsample::tests::cover_async_trait::{closure#0}:
  |  228|      1|    fn cover_async_trait() {
  ------------------
  229|      1|        block_on(async {
  230|      1|            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  231|      1|            x.covered().await;
  232|      1|        });
  233|      1|    }
  ------------------
  | codecovsample::tests::cover_async_trait:
  |  228|      1|    fn cover_async_trait() {
  |  229|      1|        block_on(async {
  |  230|       |            let x: Box<dyn AsyncTrait> = Box::new(AsyncTraitImpl);
  |  231|       |            x.covered().await;
  |  232|      1|        });
  |  233|      1|    }
  ------------------
  234|       |}

[wpbrown]

Thank you. I see improvement in the latest nightly. I minified the test case to the async issues for anyone looking at this (but it still uses executor and async_trait.)

https://github.com/wpbrown/rust-codecov/blob/async_reduce/codecovsample/src/main.rs

Since this function is missing coverage too the issue isn't specific to async_trait thought.

    8|       |
    9|      1|async fn async_func() {
   10|       |    println!("async_func");
   11|       |}
   12|       |

[richkadel]

So it turns out that the internal (auto-generated) closure actually does get coverage, but for some reason the code region for the println!() macro, by itself, is not added or not counted. I've created a draft PR with a new test and current results, demonstrating this issue. Once I figure out how to fix it, I'll submit the PR with the updated results.

rust/src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.async2.txt

Lines 9 to 34 in 3ece606

	9| 1|fn non_async_func() {
	10| 1| println!("non_async_func was covered");
	11| 1| let b = true;
	12| 1| if b {
	13| 1| println!("non_async_func println in block");
	14| 1| }
	15| 1|}
	16| |
	17| |// FIXME(#83985): The auto-generated closure in an async function is failing to include
	18| |// the println!() and `let` assignment lines in the coverage code region(s), as it does in the
	19| |// non-async function above, unless the `println!()` is inside a covered block.
	20| 1|async fn async_func() {
	21| | println!("async_func was covered");
	22| | let b = true;
	23| 1| if b {
	24| 1| println!("async_func println in block");
	25| 1| }
	^0
	26| 1|}
	27| |
	28| |// FIXME(#83985): As above, this async function only has the `println!()` macro call, which is not
	29| |// showing coverage, so the entire async closure _appears_ uncovered; but this is not exactly true.
	30| |// It's only certain kinds of lines and/or their context that results in missing coverage.
	31| 1|async fn async_func_just_println() {
	32| | println!("async_func_just_println was covered");
	33| |}

[richkadel]

Update: #84323 fixes the issue.