Skip to content

Commit 64d1954

Browse files
committed
[LIT] Added an option to llvm-lit to emit the necessary test coverage data, divided per test case
This patch is the first part of https://llvm.org/OpenProjects.html#llvm_patch_coverage. We have first define a new variable LLVM_TEST_COVERAGE which when set, pass --per-test-coverage option to llvm-lit which will help in setting a unique value to LLVM_PROFILE_FILE for each RUN. So for example coverage data for test case llvm/test/Analysis/AliasSet/memtransfer.ll will be emitted as build/test/Analysis/AliasSet/memtransfer0.profraw Reviewed By: hnrklssn Differential Revision: https://reviews.llvm.org/D154280
1 parent 20c8f58 commit 64d1954

File tree

14 files changed

+123
-16
lines changed

14 files changed

+123
-16
lines changed

llvm/CMakeLists.txt

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,9 @@ set(LIT_ARGS_DEFAULT "-sv")
676676
if (MSVC_IDE OR XCODE)
677677
set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar")
678678
endif()
679+
if(LLVM_INDIVIDUAL_TEST_COVERAGE)
680+
set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --per-test-coverage")
681+
endif()
679682
set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit")
680683

681684
# On Win32 hosts, provide an option to specify the path to the GnuWin32 tools.
@@ -797,24 +800,25 @@ if (MSVC_IDE)
797800
option(LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION "Configure project to use Visual Studio native visualizers" TRUE)
798801
endif()
799802

800-
if (LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR
801-
LLVM_ENABLE_IR_PGO)
802-
if(NOT LLVM_PROFILE_MERGE_POOL_SIZE)
803-
# A pool size of 1-2 is probably sufficient on a SSD. 3-4 should be fine
804-
# for spining disks. Anything higher may only help on slower mediums.
805-
set(LLVM_PROFILE_MERGE_POOL_SIZE "4")
806-
endif()
807-
if(NOT LLVM_PROFILE_FILE_PATTERN)
808-
if(NOT LLVM_PROFILE_DATA_DIR)
809-
file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR)
803+
if(NOT LLVM_INDIVIDUAL_TEST_COVERAGE)
804+
if(LLVM_BUILD_INSTRUMENTED OR LLVM_BUILD_INSTRUMENTED_COVERAGE OR LLVM_ENABLE_IR_PGO)
805+
if(NOT LLVM_PROFILE_MERGE_POOL_SIZE)
806+
# A pool size of 1-2 is probably sufficient on an SSD. 3-4 should be fine
807+
# for spinning disks. Anything higher may only help on slower mediums.
808+
set(LLVM_PROFILE_MERGE_POOL_SIZE "4")
810809
endif()
811-
file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN)
812-
endif()
813-
if(NOT LLVM_CSPROFILE_FILE_PATTERN)
814-
if(NOT LLVM_CSPROFILE_DATA_DIR)
815-
file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR)
810+
if(NOT LLVM_PROFILE_FILE_PATTERN)
811+
if(NOT LLVM_PROFILE_DATA_DIR)
812+
file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/profiles" LLVM_PROFILE_DATA_DIR)
813+
endif()
814+
file(TO_NATIVE_PATH "${LLVM_PROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_PROFILE_FILE_PATTERN)
815+
endif()
816+
if(NOT LLVM_CSPROFILE_FILE_PATTERN)
817+
if(NOT LLVM_CSPROFILE_DATA_DIR)
818+
file(TO_NATIVE_PATH "${LLVM_BINARY_DIR}/csprofiles" LLVM_CSPROFILE_DATA_DIR)
819+
endif()
820+
file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN)
816821
endif()
817-
file(TO_NATIVE_PATH "${LLVM_CSPROFILE_DATA_DIR}/%${LLVM_PROFILE_MERGE_POOL_SIZE}m.profraw" LLVM_CSPROFILE_FILE_PATTERN)
818822
endif()
819823
endif()
820824

llvm/cmake/modules/HandleLLVMOptions.cmake

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1160,6 +1160,7 @@ if(LLVM_PROFDATA_FILE AND EXISTS ${LLVM_PROFDATA_FILE})
11601160
endif()
11611161

11621162
option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation" Off)
1163+
option(LLVM_INDIVIDUAL_TEST_COVERAGE "Emit individual coverage file for each test case." OFF)
11631164
mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE)
11641165
append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping"
11651166
CMAKE_CXX_FLAGS

llvm/docs/CMake.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,12 @@ enabled sub-projects. Nearly all of these variable names begin with
375375
will limit code coverage summaries to just the listed directories. If unset,
376376
coverage reports will include all sources identified by the tooling.
377377

378+
**LLVM_INDIVIDUAL_TEST_COVERAGE**: BOOL
379+
Enable individual test case coverage. When set to ON, code coverage data for
380+
each test case will be generated and stored in a separate directory under the
381+
config.test_exec_root path. This feature allows code coverage analysis of each
382+
individual test case. Defaults to OFF.
383+
378384
**LLVM_BUILD_LLVM_DYLIB**:BOOL
379385
If enabled, the target for building the libLLVM shared library is added.
380386
This library contains all of LLVM's components in a single shared library.

llvm/docs/CommandGuide/lit.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,12 @@ The timing data is stored in the `test_exec_root` in a file named
181181
Run the tests in a random order, not failing/slowest first. Deprecated,
182182
use :option:`--order` instead.
183183

184+
.. option:: --per-test-coverage
185+
186+
Emit the necessary test coverage data, divided per test case (involves
187+
setting a unique value to LLVM_PROFILE_FILE for each RUN). The coverage
188+
data files will be emitted in the directory specified by `config.test_exec_root`.
189+
184190
.. option:: --max-failures N
185191

186192
Stop execution after the given number ``N`` of failures.

llvm/utils/lit/lit/LitConfig.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ def __init__(
3737
maxIndividualTestTime=0,
3838
parallelism_groups={},
3939
echo_all_commands=False,
40+
per_test_coverage=False,
4041
):
4142
# The name of the test runner.
4243
self.progname = progname
@@ -87,6 +88,7 @@ def __init__(
8788
self.maxIndividualTestTime = maxIndividualTestTime
8889
self.parallelism_groups = parallelism_groups
8990
self.echo_all_commands = echo_all_commands
91+
self.per_test_coverage = per_test_coverage
9092

9193
@property
9294
def maxIndividualTestTime(self):
@@ -128,6 +130,22 @@ def maxIndividualTestTime(self, value):
128130
elif self.maxIndividualTestTime < 0:
129131
self.fatal("The timeout per test must be >= 0 seconds")
130132

133+
@property
134+
def per_test_coverage(self):
135+
"""
136+
Interface for getting the per_test_coverage value
137+
"""
138+
return self._per_test_coverage
139+
140+
@per_test_coverage.setter
141+
def per_test_coverage(self, value):
142+
"""
143+
Interface for setting the per_test_coverage value
144+
"""
145+
if not isinstance(value, bool):
146+
self.fatal("per_test_coverage must set to a value of type bool.")
147+
self._per_test_coverage = value
148+
131149
def load_config(self, config, path):
132150
"""load_config(config, path) - Load a config object from an alternate
133151
path."""

llvm/utils/lit/lit/TestRunner.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,10 +1067,25 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
10671067
def executeScript(test, litConfig, tmpBase, commands, cwd):
10681068
bashPath = litConfig.getBashPath()
10691069
isWin32CMDEXE = litConfig.isWindows and not bashPath
1070+
coverage_index = 0 # Counter for coverage file index
10701071
script = tmpBase + ".script"
10711072
if isWin32CMDEXE:
10721073
script += ".bat"
10731074

1075+
# Set unique LLVM_PROFILE_FILE for each run command
1076+
for j, ln in enumerate(commands):
1077+
match = re.match(kPdbgRegex, ln)
1078+
if match:
1079+
command = match.group(2)
1080+
commands[j] = match.expand(": '\\1'; \\2" if command else ": '\\1'")
1081+
if litConfig.per_test_coverage:
1082+
# Extract the test case name from the test object
1083+
test_case_name = test.path_in_suite[-1]
1084+
test_case_name = test_case_name.rsplit(".", 1)[0] # Remove the file extension
1085+
llvm_profile_file = f"{test_case_name}{coverage_index}.profraw"
1086+
commands[j] = f"export LLVM_PROFILE_FILE={llvm_profile_file} && {commands[j]}"
1087+
coverage_index += 1
1088+
10741089
# Write script file
10751090
mode = "w"
10761091
open_kwargs = {}

llvm/utils/lit/lit/cl_arguments.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,12 @@ def parse_args():
184184
help="Do not fail the run if all tests are filtered out",
185185
action="store_true",
186186
)
187+
execution_group.add_argument(
188+
"--per-test-coverage",
189+
dest="per_test_coverage",
190+
action="store_true",
191+
help="Enable individual test case coverage",
192+
)
187193
execution_group.add_argument(
188194
"--ignore-fail",
189195
dest="ignoreFail",

llvm/utils/lit/lit/main.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ def main(builtin_params={}):
4141
params=params,
4242
config_prefix=opts.configPrefix,
4343
echo_all_commands=opts.echoAllCommands,
44+
per_test_coverage=opts.per_test_coverage,
4445
)
4546

4647
discovered_tests = lit.discovery.find_tests_for_inputs(
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import lit.formats
2+
import os
3+
4+
config.name = "per-test-coverage-by-lit-cfg"
5+
config.suffixes = [".py"]
6+
config.test_format = lit.formats.ShTest(execute_external=True)
7+
lit_config.per_test_coverage = True
8+
config.substitutions.append(("%{python}", '"%s"' % (sys.executable)))
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Check that the environment variable is set correctly
2+
# RUN: %{python} %s | FileCheck %s
3+
4+
# Python script to read the environment variable
5+
# and print its value
6+
import os
7+
8+
llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE')
9+
print(llvm_profile_file)
10+
11+
# CHECK: per-test-coverage-by-lit-cfg0.profraw
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import lit.formats
2+
import os
3+
4+
config.name = "per-test-coverage"
5+
config.suffixes = [".py"]
6+
config.test_format = lit.formats.ShTest(execute_external=True)
7+
config.substitutions.append(("%{python}", '"%s"' % (sys.executable)))
8+
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Check that the environment variable is set correctly
2+
# RUN: %{python} %s | FileCheck %s
3+
4+
# Python script to read the environment variable
5+
# and print its value
6+
import os
7+
8+
llvm_profile_file = os.environ.get('LLVM_PROFILE_FILE')
9+
print(llvm_profile_file)
10+
11+
# CHECK: per-test-coverage0.profraw
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Test if lit_config.per_test_coverage in lit.cfg sets individual test case coverage.
2+
3+
# RUN: %{lit} -a -v %{inputs}/per-test-coverage-by-lit-cfg/per-test-coverage-by-lit-cfg.py \
4+
# RUN: | FileCheck -match-full-lines %s
5+
#
6+
# CHECK: PASS: per-test-coverage-by-lit-cfg :: per-test-coverage-by-lit-cfg.py ({{[^)]*}})
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Test LLVM_PROFILE_FILE is set when --per-test-coverage is passed to command line.
2+
3+
# RUN: %{lit} -a -v --per-test-coverage %{inputs}/per-test-coverage/per-test-coverage.py \
4+
# RUN: | FileCheck -match-full-lines %s
5+
#
6+
# CHECK: PASS: per-test-coverage :: per-test-coverage.py ({{[^)]*}})

0 commit comments

Comments
 (0)