[libc++][WIP] Move the libc++ test format to Lit #90803
Open
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
libc++
|
@llvm/pr-subscribers-libunwind @llvm/pr-subscribers-testing-tools Author: Louis Dionne (ldionne) ChangesThis allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++. With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient. Patch is 37.44 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/90803.diff 6 Files Affected:
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index 84b3270a8940ac..b220e5ebcafb17 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -18,7 +18,7 @@ import libcxx.test.format
# Basic configuration of the test suite
config.name = os.path.basename('@LIBCXX_TEST_CONFIG@')
config.test_source_root = os.path.join('@LIBCXX_SOURCE_DIR@', 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+config.test_format = libcxx.test.format.LibcxxTest()
config.recursiveExpansionLimit = 10
config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test')
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 387862ae6f496d..6177dc9dccd327 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -99,7 +99,7 @@ def _executeWithFakeConfig(test, commands):
order="smart",
params={},
)
- return libcxx.test.format._executeScriptInternal(test, litConfig, commands)
+ return lit.formats.standardlibrarytest._executeScriptInternal(test, litConfig, commands)
def _makeConfigTest(config):
@@ -121,12 +121,12 @@ def _makeConfigTest(config):
class TestWrapper(lit.Test.Test):
def __enter__(self):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
os.makedirs(testDir)
return self
def __exit__(self, *args):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
shutil.rmtree(testDir)
os.remove(tmp.name)
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 7e5281c0b74064..e74e4838f7dfe6 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -7,50 +7,8 @@
# ===----------------------------------------------------------------------===##
import lit
-import libcxx.test.config as config
import lit.formats
import os
-import re
-
-
-def _getTempPaths(test):
- """
- Return the values to use for the %T and %t substitutions, respectively.
-
- The difference between this and Lit's default behavior is that we guarantee
- that %T is a path unique to the test being run.
- """
- tmpDir, _ = lit.TestRunner.getTempPaths(test)
- _, testName = os.path.split(test.getExecPath())
- tmpDir = os.path.join(tmpDir, testName + ".dir")
- tmpBase = os.path.join(tmpDir, "t")
- return tmpDir, tmpBase
-
-
-def _checkBaseSubstitutions(substitutions):
- substitutions = [s for (s, _) in substitutions]
- for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
- assert s in substitutions, "Required substitution {} was not provided".format(s)
-
-def _executeScriptInternal(test, litConfig, commands):
- """
- Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
-
- TODO: This really should be easier to access from Lit itself
- """
- parsedCommands = parseScript(test, preamble=commands)
-
- _, tmpBase = _getTempPaths(test)
- execDir = os.path.dirname(test.getExecPath())
- try:
- res = lit.TestRunner.executeScriptInternal(
- test, litConfig, tmpBase, parsedCommands, execDir, debug=False
- )
- except lit.TestRunner.ScriptFatal as e:
- res = ("", str(e), 127, None)
- (out, err, exitCode, timeoutInfo) = res
-
- return (out, err, exitCode, timeoutInfo, parsedCommands)
def _validateModuleDependencies(modules):
@@ -61,334 +19,66 @@ def _validateModuleDependencies(modules):
)
-def parseScript(test, preamble):
- """
- Extract the script from a test, with substitutions applied.
-
- Returns a list of commands ready to be executed.
-
- - test
- The lit.Test to parse.
-
- - preamble
- A list of commands to perform before any command in the test.
- These commands can contain unexpanded substitutions, but they
- must not be of the form 'RUN:' -- they must be proper commands
- once substituted.
- """
- # Get the default substitutions
- tmpDir, tmpBase = _getTempPaths(test)
+def _buildModule(test, litConfig, command):
+ tmpDir, tmpBase = lit.formats.standardlibrarytest._getTempPaths(test)
+ execDir = os.path.dirname(test.getExecPath())
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
- # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions
- #
- # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as
- # errors, which doesn't make sense for clang-verify tests because we may want to check
- # for specific warning diagnostics.
- _checkBaseSubstitutions(substitutions)
- substitutions.append(
- ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe")
+ substituted = lit.TestRunner.applySubstitutions(
+ [command], substitutions, recursion_limit=test.config.recursiveExpansionLimit
)
- substitutions.append(
- (
- "%{verify}",
- "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0",
- )
- )
- substitutions.append(("%{run}", "%{exec} %t.exe"))
-
- # Parse the test file, including custom directives
- additionalCompileFlags = []
- fileDependencies = []
- modules = [] # The enabled modules
- moduleCompileFlags = [] # The compilation flags to use modules
- parsers = [
- lit.TestRunner.IntegratedTestKeywordParser(
- "FILE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.LIST,
- initial_value=fileDependencies,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "MODULE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=modules,
- ),
- ]
-
- # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first
- # class support for conditional keywords in Lit, which would allow evaluating arbitrary
- # Lit boolean expressions instead.
- for feature in test.config.available_features:
- parser = lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS({}):".format(feature),
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- )
- parsers.append(parser)
-
- scriptInTest = lit.TestRunner.parseIntegratedTestScript(
- test, additional_parsers=parsers, require_script=not preamble
- )
- if isinstance(scriptInTest, lit.Test.Result):
- return scriptInTest
-
- script = []
-
- # For each file dependency in FILE_DEPENDENCIES, inject a command to copy
- # that file to the execution directory. Execute the copy from %S to allow
- # relative paths from the test directory.
- for dep in fileDependencies:
- script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)]
- script += preamble
- script += scriptInTest
-
- # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
- # Modules need to be built with the same compilation flags as the
- # test. So add these flags before adding the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(additionalCompileFlags)
- )
-
- if modules:
- _validateModuleDependencies(modules)
-
- # The moduleCompileFlags are added to the %{compile_flags}, but
- # the modules need to be built without these flags. So expand the
- # %{compile_flags} eagerly and hardcode them in the build script.
- compileFlags = config._getSubstitution("%{compile_flags}", test.config)
-
- # Building the modules needs to happen before the other script
- # commands are executed. Therefore the commands are added to the
- # front of the list.
- if "std.compat" in modules:
- script.insert(
- 0,
- "%dbg(MODULE std.compat) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
- "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm",
- )
- moduleCompileFlags.extend(
- ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"]
- )
-
- # Make sure the std module is built before std.compat. Libc++'s
- # std.compat module depends on the std module. It is not
- # known whether the compiler expects the modules in the order of
- # their dependencies. However it's trivial to provide them in
- # that order.
- script.insert(
- 0,
- "%dbg(MODULE std) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm",
- )
- moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
-
- # Add compile flags required for the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
+ (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(test, litConfig, tmpBase, substituted, execDir)
+ if exitCode != 0:
+ return lit.Test.Result(
+ lit.Test.UNRESOLVED, "Failed to build module std for '{}':\n{}\n{}".format(test.getFilePath(), out, err)
)
- # Perform substitutions in the script itself.
- script = lit.TestRunner.applySubstitutions(
- script, substitutions, recursion_limit=test.config.recursiveExpansionLimit
- )
-
- return script
-
-
-class CxxStandardLibraryTest(lit.formats.FileBasedTest):
- """
- Lit test format for the C++ Standard Library conformance test suite.
-
- Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test.
- Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform
- the appropriate operations (compile/link/run). See
- https://libcxx.llvm.org/TestingLibcxx.html#test-names
- for a complete description of those semantics.
-
- Substitution requirements
- ===============================
- The test format operates by assuming that each test's configuration provides
- the following substitutions, which it will reuse in the shell scripts it
- constructs:
- %{cxx} - A command that can be used to invoke the compiler
- %{compile_flags} - Flags to use when compiling a test case
- %{link_flags} - Flags to use when linking a test case
- %{flags} - Flags to use either when compiling or linking a test case
- %{exec} - A command to prefix the execution of executables
-
- Note that when building an executable (as opposed to only compiling a source
- file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
- in the same command line. In other words, the test format doesn't perform
- separate compilation and linking steps in this case.
-
- Additional provided substitutions and features
- ==============================================
- The test format will define the following substitutions for use inside tests:
-
- %{build}
- Expands to a command-line that builds the current source
- file with the %{flags}, %{compile_flags} and %{link_flags}
- substitutions, and that produces an executable named %t.exe.
-
- %{verify}
- Expands to a command-line that builds the current source
- file with the %{flags} and %{compile_flags} substitutions
- and enables clang-verify. This can be used to write .sh.cpp
- tests that use clang-verify. Note that this substitution can
- only be used when the 'verify-support' feature is available.
-
- %{run}
- Equivalent to `%{exec} %t.exe`. This is intended to be used
- in conjunction with the %{build} substitution.
- """
-
- def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
- SUPPORTED_SUFFIXES = [
- "[.]pass[.]cpp$",
- "[.]pass[.]mm$",
- "[.]compile[.]pass[.]cpp$",
- "[.]compile[.]pass[.]mm$",
- "[.]compile[.]fail[.]cpp$",
- "[.]link[.]pass[.]cpp$",
- "[.]link[.]pass[.]mm$",
- "[.]link[.]fail[.]cpp$",
- "[.]sh[.][^.]+$",
- "[.]gen[.][^.]+$",
- "[.]verify[.]cpp$",
- ]
-
- sourcePath = testSuite.getSourcePath(pathInSuite)
- filename = os.path.basename(sourcePath)
-
- # Ignore dot files, excluded tests and tests with an unsupported suffix
- hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
- if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
- return
-
- # If this is a generated test, run the generation step and add
- # as many Lit tests as necessary.
- if re.search('[.]gen[.][^.]+$', filename):
- for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
- yield test
- else:
- yield lit.Test.Test(testSuite, pathInSuite, localConfig)
+class LibcxxTest(lit.formats.StandardLibraryTest):
def execute(self, test, litConfig):
- supportsVerify = "verify-support" in test.config.available_features
- filename = test.path_in_suite[-1]
-
- if re.search("[.]sh[.][^.]+$", filename):
- steps = [] # The steps are already in the script
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.pass.cpp") or filename.endswith(
- ".compile.pass.mm"
- ):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
- "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".verify.cpp"):
- if not supportsVerify:
- return lit.Test.Result(
- lit.Test.UNSUPPORTED,
- "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(
- test.getFullName()
- ),
- )
- steps = ["%dbg(COMPILED WITH) %{verify}"]
- return self._executeShTest(test, litConfig, steps)
- # Make sure to check these ones last, since they will match other
- # suffixes above too.
- elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
- "%dbg(EXECUTED AS) %{exec} %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- else:
- return lit.Test.Result(
- lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
+ # Parse any MODULE_DEPENDENCIES in the test file.
+ modules = []
+ parsers = [
+ lit.TestRunner.IntegratedTestKeywordParser(
+ "MODULE_DEPENDENCIES:",
+ lit.TestRunner.ParserKind.SPACE_LIST,
+ initial_value=modules,
)
-
- def _executeShTest(self, test, litConfig, steps):
- if test.config.unsupported:
- return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported")
-
- script = parseScript(test, steps)
- if isinstance(script, lit.Test.Result):
- return script
-
- if litConfig.noExecute:
- return lit.Test.Result(
- lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS
- )
- else:
- _, tmpBase = _getTempPaths(test)
- useExternalSh = False
- return lit.TestRunner._runShTest(
- test, litConfig, useExternalSh, script, tmpBase
+ ]
+ lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, require_script=False)
+
+ # Build the modules if needed and tweak the compiler flags of the rest of the test so
+ # it knows about the just-built modules.
+ moduleCompileFlags = []
+ if modules:
+ _validateModuleDependencies(modules)
+
+ # Make sure the std module is built before std.compat. Libc++'s
+ # std.compat module depends on the std module. It is not
+ # known whether the compiler expects the modules in the order of
+ # their dependencies. However it's trivial to provide them in
+ # that order.
+ command = ("%dbg(MODULE std) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
+
+ if "std.compat" in modules:
+ command = ("%dbg(MODULE std.compat) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
+ "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"])
+
+ # Add compile flags required for the test to use the just-built modules
+ test.config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
+ test.config.substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
)
- def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
- generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
-
- # Make sure we have a directory to execute the generator test in
- generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
- os.makedirs(generatorExecDir, exist_ok=True)
-
- # Run the generator test
- steps = [] # Steps must already be in the script
- (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
- if exitCode != 0:
- raise RuntimeErr...
[truncated]
|
|
@llvm/pr-subscribers-libcxxabi Author: Louis Dionne (ldionne) ChangesThis allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++. With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient. Patch is 37.44 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/90803.diff 6 Files Affected:
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index 84b3270a8940ac..b220e5ebcafb17 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -18,7 +18,7 @@ import libcxx.test.format
# Basic configuration of the test suite
config.name = os.path.basename('@LIBCXX_TEST_CONFIG@')
config.test_source_root = os.path.join('@LIBCXX_SOURCE_DIR@', 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+config.test_format = libcxx.test.format.LibcxxTest()
config.recursiveExpansionLimit = 10
config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test')
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 387862ae6f496d..6177dc9dccd327 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -99,7 +99,7 @@ def _executeWithFakeConfig(test, commands):
order="smart",
params={},
)
- return libcxx.test.format._executeScriptInternal(test, litConfig, commands)
+ return lit.formats.standardlibrarytest._executeScriptInternal(test, litConfig, commands)
def _makeConfigTest(config):
@@ -121,12 +121,12 @@ def _makeConfigTest(config):
class TestWrapper(lit.Test.Test):
def __enter__(self):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
os.makedirs(testDir)
return self
def __exit__(self, *args):
- testDir, _ = libcxx.test.format._getTempPaths(self)
+ testDir, _ = lit.formats.standardlibrarytest._getTempPaths(self)
shutil.rmtree(testDir)
os.remove(tmp.name)
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index 7e5281c0b74064..e74e4838f7dfe6 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -7,50 +7,8 @@
# ===----------------------------------------------------------------------===##
import lit
-import libcxx.test.config as config
import lit.formats
import os
-import re
-
-
-def _getTempPaths(test):
- """
- Return the values to use for the %T and %t substitutions, respectively.
-
- The difference between this and Lit's default behavior is that we guarantee
- that %T is a path unique to the test being run.
- """
- tmpDir, _ = lit.TestRunner.getTempPaths(test)
- _, testName = os.path.split(test.getExecPath())
- tmpDir = os.path.join(tmpDir, testName + ".dir")
- tmpBase = os.path.join(tmpDir, "t")
- return tmpDir, tmpBase
-
-
-def _checkBaseSubstitutions(substitutions):
- substitutions = [s for (s, _) in substitutions]
- for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
- assert s in substitutions, "Required substitution {} was not provided".format(s)
-
-def _executeScriptInternal(test, litConfig, commands):
- """
- Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
-
- TODO: This really should be easier to access from Lit itself
- """
- parsedCommands = parseScript(test, preamble=commands)
-
- _, tmpBase = _getTempPaths(test)
- execDir = os.path.dirname(test.getExecPath())
- try:
- res = lit.TestRunner.executeScriptInternal(
- test, litConfig, tmpBase, parsedCommands, execDir, debug=False
- )
- except lit.TestRunner.ScriptFatal as e:
- res = ("", str(e), 127, None)
- (out, err, exitCode, timeoutInfo) = res
-
- return (out, err, exitCode, timeoutInfo, parsedCommands)
def _validateModuleDependencies(modules):
@@ -61,334 +19,66 @@ def _validateModuleDependencies(modules):
)
-def parseScript(test, preamble):
- """
- Extract the script from a test, with substitutions applied.
-
- Returns a list of commands ready to be executed.
-
- - test
- The lit.Test to parse.
-
- - preamble
- A list of commands to perform before any command in the test.
- These commands can contain unexpanded substitutions, but they
- must not be of the form 'RUN:' -- they must be proper commands
- once substituted.
- """
- # Get the default substitutions
- tmpDir, tmpBase = _getTempPaths(test)
+def _buildModule(test, litConfig, command):
+ tmpDir, tmpBase = lit.formats.standardlibrarytest._getTempPaths(test)
+ execDir = os.path.dirname(test.getExecPath())
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
- # Check base substitutions and add the %{build}, %{verify} and %{run} convenience substitutions
- #
- # Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as
- # errors, which doesn't make sense for clang-verify tests because we may want to check
- # for specific warning diagnostics.
- _checkBaseSubstitutions(substitutions)
- substitutions.append(
- ("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe")
+ substituted = lit.TestRunner.applySubstitutions(
+ [command], substitutions, recursion_limit=test.config.recursiveExpansionLimit
)
- substitutions.append(
- (
- "%{verify}",
- "%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0",
- )
- )
- substitutions.append(("%{run}", "%{exec} %t.exe"))
-
- # Parse the test file, including custom directives
- additionalCompileFlags = []
- fileDependencies = []
- modules = [] # The enabled modules
- moduleCompileFlags = [] # The compilation flags to use modules
- parsers = [
- lit.TestRunner.IntegratedTestKeywordParser(
- "FILE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.LIST,
- initial_value=fileDependencies,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- ),
- lit.TestRunner.IntegratedTestKeywordParser(
- "MODULE_DEPENDENCIES:",
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=modules,
- ),
- ]
-
- # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first
- # class support for conditional keywords in Lit, which would allow evaluating arbitrary
- # Lit boolean expressions instead.
- for feature in test.config.available_features:
- parser = lit.TestRunner.IntegratedTestKeywordParser(
- "ADDITIONAL_COMPILE_FLAGS({}):".format(feature),
- lit.TestRunner.ParserKind.SPACE_LIST,
- initial_value=additionalCompileFlags,
- )
- parsers.append(parser)
-
- scriptInTest = lit.TestRunner.parseIntegratedTestScript(
- test, additional_parsers=parsers, require_script=not preamble
- )
- if isinstance(scriptInTest, lit.Test.Result):
- return scriptInTest
-
- script = []
-
- # For each file dependency in FILE_DEPENDENCIES, inject a command to copy
- # that file to the execution directory. Execute the copy from %S to allow
- # relative paths from the test directory.
- for dep in fileDependencies:
- script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)]
- script += preamble
- script += scriptInTest
-
- # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
- # Modules need to be built with the same compilation flags as the
- # test. So add these flags before adding the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(additionalCompileFlags)
- )
-
- if modules:
- _validateModuleDependencies(modules)
-
- # The moduleCompileFlags are added to the %{compile_flags}, but
- # the modules need to be built without these flags. So expand the
- # %{compile_flags} eagerly and hardcode them in the build script.
- compileFlags = config._getSubstitution("%{compile_flags}", test.config)
-
- # Building the modules needs to happen before the other script
- # commands are executed. Therefore the commands are added to the
- # front of the list.
- if "std.compat" in modules:
- script.insert(
- 0,
- "%dbg(MODULE std.compat) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
- "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm",
- )
- moduleCompileFlags.extend(
- ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"]
- )
-
- # Make sure the std module is built before std.compat. Libc++'s
- # std.compat module depends on the std module. It is not
- # known whether the compiler expects the modules in the order of
- # their dependencies. However it's trivial to provide them in
- # that order.
- script.insert(
- 0,
- "%dbg(MODULE std) %{cxx} %{flags} "
- f"{compileFlags} "
- "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm",
- )
- moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
-
- # Add compile flags required for the modules.
- substitutions = config._appendToSubstitution(
- substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
+ (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(test, litConfig, tmpBase, substituted, execDir)
+ if exitCode != 0:
+ return lit.Test.Result(
+ lit.Test.UNRESOLVED, "Failed to build module std for '{}':\n{}\n{}".format(test.getFilePath(), out, err)
)
- # Perform substitutions in the script itself.
- script = lit.TestRunner.applySubstitutions(
- script, substitutions, recursion_limit=test.config.recursiveExpansionLimit
- )
-
- return script
-
-
-class CxxStandardLibraryTest(lit.formats.FileBasedTest):
- """
- Lit test format for the C++ Standard Library conformance test suite.
-
- Lit tests are contained in files that follow a certain pattern, which determines the semantics of the test.
- Under the hood, we basically generate a builtin Lit shell test that follows the ShTest format, and perform
- the appropriate operations (compile/link/run). See
- https://libcxx.llvm.org/TestingLibcxx.html#test-names
- for a complete description of those semantics.
-
- Substitution requirements
- ===============================
- The test format operates by assuming that each test's configuration provides
- the following substitutions, which it will reuse in the shell scripts it
- constructs:
- %{cxx} - A command that can be used to invoke the compiler
- %{compile_flags} - Flags to use when compiling a test case
- %{link_flags} - Flags to use when linking a test case
- %{flags} - Flags to use either when compiling or linking a test case
- %{exec} - A command to prefix the execution of executables
-
- Note that when building an executable (as opposed to only compiling a source
- file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
- in the same command line. In other words, the test format doesn't perform
- separate compilation and linking steps in this case.
-
- Additional provided substitutions and features
- ==============================================
- The test format will define the following substitutions for use inside tests:
-
- %{build}
- Expands to a command-line that builds the current source
- file with the %{flags}, %{compile_flags} and %{link_flags}
- substitutions, and that produces an executable named %t.exe.
-
- %{verify}
- Expands to a command-line that builds the current source
- file with the %{flags} and %{compile_flags} substitutions
- and enables clang-verify. This can be used to write .sh.cpp
- tests that use clang-verify. Note that this substitution can
- only be used when the 'verify-support' feature is available.
-
- %{run}
- Equivalent to `%{exec} %t.exe`. This is intended to be used
- in conjunction with the %{build} substitution.
- """
-
- def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
- SUPPORTED_SUFFIXES = [
- "[.]pass[.]cpp$",
- "[.]pass[.]mm$",
- "[.]compile[.]pass[.]cpp$",
- "[.]compile[.]pass[.]mm$",
- "[.]compile[.]fail[.]cpp$",
- "[.]link[.]pass[.]cpp$",
- "[.]link[.]pass[.]mm$",
- "[.]link[.]fail[.]cpp$",
- "[.]sh[.][^.]+$",
- "[.]gen[.][^.]+$",
- "[.]verify[.]cpp$",
- ]
-
- sourcePath = testSuite.getSourcePath(pathInSuite)
- filename = os.path.basename(sourcePath)
-
- # Ignore dot files, excluded tests and tests with an unsupported suffix
- hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
- if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
- return
-
- # If this is a generated test, run the generation step and add
- # as many Lit tests as necessary.
- if re.search('[.]gen[.][^.]+$', filename):
- for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
- yield test
- else:
- yield lit.Test.Test(testSuite, pathInSuite, localConfig)
+class LibcxxTest(lit.formats.StandardLibraryTest):
def execute(self, test, litConfig):
- supportsVerify = "verify-support" in test.config.available_features
- filename = test.path_in_suite[-1]
-
- if re.search("[.]sh[.][^.]+$", filename):
- steps = [] # The steps are already in the script
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.pass.cpp") or filename.endswith(
- ".compile.pass.mm"
- ):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".compile.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".link.fail.cpp"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
- "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- elif filename.endswith(".verify.cpp"):
- if not supportsVerify:
- return lit.Test.Result(
- lit.Test.UNSUPPORTED,
- "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(
- test.getFullName()
- ),
- )
- steps = ["%dbg(COMPILED WITH) %{verify}"]
- return self._executeShTest(test, litConfig, steps)
- # Make sure to check these ones last, since they will match other
- # suffixes above too.
- elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"):
- steps = [
- "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
- "%dbg(EXECUTED AS) %{exec} %t.exe",
- ]
- return self._executeShTest(test, litConfig, steps)
- else:
- return lit.Test.Result(
- lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
+ # Parse any MODULE_DEPENDENCIES in the test file.
+ modules = []
+ parsers = [
+ lit.TestRunner.IntegratedTestKeywordParser(
+ "MODULE_DEPENDENCIES:",
+ lit.TestRunner.ParserKind.SPACE_LIST,
+ initial_value=modules,
)
-
- def _executeShTest(self, test, litConfig, steps):
- if test.config.unsupported:
- return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported")
-
- script = parseScript(test, steps)
- if isinstance(script, lit.Test.Result):
- return script
-
- if litConfig.noExecute:
- return lit.Test.Result(
- lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS
- )
- else:
- _, tmpBase = _getTempPaths(test)
- useExternalSh = False
- return lit.TestRunner._runShTest(
- test, litConfig, useExternalSh, script, tmpBase
+ ]
+ lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, require_script=False)
+
+ # Build the modules if needed and tweak the compiler flags of the rest of the test so
+ # it knows about the just-built modules.
+ moduleCompileFlags = []
+ if modules:
+ _validateModuleDependencies(modules)
+
+ # Make sure the std module is built before std.compat. Libc++'s
+ # std.compat module depends on the std module. It is not
+ # known whether the compiler expects the modules in the order of
+ # their dependencies. However it's trivial to provide them in
+ # that order.
+ command = ("%dbg(MODULE std) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "--precompile -o %T/std.pcm -c %{module-dir}/std.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std=%T/std.pcm", "%T/std.pcm"])
+
+ if "std.compat" in modules:
+ command = ("%dbg(MODULE std.compat) %{cxx} %{flags} %{compile_flags} "
+ "-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
+ "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
+ "--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm")
+ res = _buildModule(test, litConfig, command)
+ if isinstance(res, lit.Test.Result):
+ return res
+ moduleCompileFlags.extend(["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"])
+
+ # Add compile flags required for the test to use the just-built modules
+ test.config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
+ test.config.substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
)
- def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
- generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
-
- # Make sure we have a directory to execute the generator test in
- generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
- os.makedirs(generatorExecDir, exist_ok=True)
-
- # Run the generator test
- steps = [] # Steps must already be in the script
- (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
- if exitCode != 0:
- raise RuntimeErr...
[truncated]
|
You can test this locally with the following command:darker --check --diff -r 0370beb230a35f00b7d07c50ab95f8777662a0c6...4871a82495e0056483759fa207ffbecd63b1e2f8 llvm/utils/lit/lit/formats/standardlibrarytest.py libcxx/utils/libcxx/test/config.py libcxx/utils/libcxx/test/dsl.py libcxx/utils/libcxx/test/format.py llvm/utils/lit/lit/formats/__init__.pyView the diff from darker here.--- libcxx/utils/libcxx/test/config.py 2024-05-22 19:18:56.000000 +0000
+++ libcxx/utils/libcxx/test/config.py 2024-05-22 19:21:52.557532 +0000
@@ -40,9 +40,13 @@
)
)
# Print the basic substitutions
for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{exec}"):
- note("Using {} substitution: '{}'".format(sub, lit.formats.standardlibrarytest._getSubstitution(sub, config)))
+ note(
+ "Using {} substitution: '{}'".format(
+ sub, lit.formats.standardlibrarytest._getSubstitution(sub, config)
+ )
+ )
# Print all available features
note("All available features: {}".format(", ".join(sorted(config.available_features))))
--- libcxx/utils/libcxx/test/dsl.py 2024-05-22 19:18:56.000000 +0000
+++ libcxx/utils/libcxx/test/dsl.py 2024-05-22 19:21:52.746771 +0000
@@ -97,11 +97,13 @@
debug=False,
isWindows=platform.system() == "Windows",
order="smart",
params={},
)
- return lit.formats.standardlibrarytest._executeScriptInternal(test, litConfig, commands)
+ return lit.formats.standardlibrarytest._executeScriptInternal(
+ test, litConfig, commands
+ )
def _makeConfigTest(config):
# Make sure the support directories exist, which is needed to create
# the temporary file %t below.
@@ -450,12 +452,14 @@
self._getFlag = lambda config: flag(config) if callable(flag) else flag
def applyTo(self, config):
flag = self._getFlag(config)
if hasCompileFlag(config, flag):
- config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
- config.substitutions, "%{flags}", flag
+ config.substitutions = (
+ lit.formats.standardlibrarytest._appendToSubstitution(
+ config.substitutions, "%{flags}", flag
+ )
)
def pretty(self, config, litParams):
return "add {} to %{{flags}}".format(self._getFlag(config))
@@ -540,12 +544,14 @@
def applyTo(self, config):
flag = self._getFlag(config)
# Use -Werror to make sure we see an error about the flag being unsupported.
if hasCompileFlag(config, "-Werror " + flag):
- config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
- config.substitutions, "%{compile_flags}", flag
+ config.substitutions = (
+ lit.formats.standardlibrarytest._appendToSubstitution(
+ config.substitutions, "%{compile_flags}", flag
+ )
)
def pretty(self, config, litParams):
return "add {} to %{{compile_flags}}".format(self._getFlag(config))
--- libcxx/utils/libcxx/test/format.py 2024-05-22 19:18:56.000000 +0000
+++ libcxx/utils/libcxx/test/format.py 2024-05-22 19:21:52.798207 +0000
@@ -25,14 +25,19 @@
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
substituted = lit.TestRunner.applySubstitutions(
commands, substitutions, recursion_limit=test.config.recursiveExpansionLimit
)
- (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(test, litConfig, tmpBase, substituted, execDir)
+ (out, err, exitCode, _) = lit.TestRunner.executeScriptInternal(
+ test, litConfig, tmpBase, substituted, execDir
+ )
if exitCode != 0:
return lit.Test.Result(
- lit.Test.UNRESOLVED, "Failed to build module std for '{}':\n{}\n{}".format(test.getFilePath(), out, err)
+ lit.Test.UNRESOLVED,
+ "Failed to build module std for '{}':\n{}\n{}".format(
+ test.getFilePath(), out, err
+ ),
)
class LibcxxTest(lit.formats.StandardLibraryTest):
def execute(self, test, litConfig):
@@ -47,11 +52,13 @@
"MODULE_DEPENDENCIES:",
lit.TestRunner.ParserKind.SPACE_LIST,
initial_value=modules,
)
]
- res = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, require_script=False)
+ res = lit.TestRunner.parseIntegratedTestScript(
+ test, additional_parsers=parsers, require_script=False
+ )
if isinstance(res, lit.Test.Result):
return res
# Build the modules if needed and tweak the compiler flags of the rest of the test so
# it knows about the just-built modules.
@@ -78,19 +85,25 @@
if "std.compat" in modules:
commands = [
"mkdir -p %T",
"%dbg(MODULE std.compat) %{cxx} %{flags} %{compile_flags} "
"-Wno-reserved-module-identifier -Wno-reserved-user-defined-literal "
- "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
+ "-fmodule-file=std=%T/std.pcm " # The std.compat module imports std.
"--precompile -o %T/std.compat.pcm -c %{module-dir}/std.compat.cppm",
]
res = _buildModule(test, litConfig, commands)
if isinstance(res, lit.Test.Result):
return res
- moduleCompileFlags.extend(["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"])
+ moduleCompileFlags.extend(
+ ["-fmodule-file=std.compat=%T/std.compat.pcm", "%T/std.compat.pcm"]
+ )
# Add compile flags required for the test to use the just-built modules
- test.config.substitutions = lit.formats.standardlibrarytest._appendToSubstitution(
- test.config.substitutions, "%{compile_flags}", " ".join(moduleCompileFlags)
+ test.config.substitutions = (
+ lit.formats.standardlibrarytest._appendToSubstitution(
+ test.config.substitutions,
+ "%{compile_flags}",
+ " ".join(moduleCompileFlags),
+ )
)
return super().execute(test, litConfig)
--- llvm/utils/lit/lit/formats/standardlibrarytest.py 2024-05-22 19:18:56.000000 +0000
+++ llvm/utils/lit/lit/formats/standardlibrarytest.py 2024-05-22 19:21:52.905759 +0000
@@ -14,26 +14,29 @@
def _getSubstitution(substitution, config):
"""
Helper function to get a specific substitution from a config object.
"""
- for (orig, replacement) in config.substitutions:
+ for orig, replacement in config.substitutions:
if orig == substitution:
return replacement
raise ValueError("Substitution {} is not in the config.".format(substitution))
+
def _appendToSubstitution(substitutions, key, value):
"""
Helper function to append a value to a specific substitution.
"""
return [(k, v + " " + value) if k == key else (k, v) for (k, v) in substitutions]
+
def _prependToSubstitution(substitutions, key, value):
"""
Helper function to prepend a value to a specific substitution.
"""
return [(k, value + " " + v) if k == key else (k, v) for (k, v) in substitutions]
+
def _getTempPaths(test):
"""
Return the values to use for the %T and %t substitutions, respectively.
@@ -68,11 +71,11 @@
# Note: We use -Wno-error with %{verify} to make sure that we don't treat all diagnostics as
# errors, which doesn't make sense for clang-verify tests because we may want to check
# for specific warning diagnostics.
"%{cxx} %s %{flags} %{compile_flags} -fsyntax-only -Wno-error -Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0",
),
- ("%{run}", "%{exec} %t.exe")
+ ("%{run}", "%{exec} %t.exe"),
]
def _executeScriptInternal(test, litConfig, commands):
"""
@@ -128,11 +131,11 @@
),
lit.TestRunner.IntegratedTestKeywordParser(
"ADDITIONAL_COMPILE_FLAGS:",
lit.TestRunner.ParserKind.SPACE_LIST,
initial_value=additionalCompileFlags,
- )
+ ),
]
# Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first
# class support for conditional keywords in Lit, which would allow evaluating arbitrary
# Lit boolean expressions instead.
@@ -234,18 +237,26 @@
sourcePath = testSuite.getSourcePath(pathInSuite)
filename = os.path.basename(sourcePath)
# Ignore dot files, excluded tests and tests with an unsupported suffix
- hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
- if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
+ hasSupportedSuffix = lambda f: any(
+ [re.search(ext, f) for ext in SUPPORTED_SUFFIXES]
+ )
+ if (
+ filename.startswith(".")
+ or filename in localConfig.excludes
+ or not hasSupportedSuffix(filename)
+ ):
return
# If this is a generated test, run the generation step and add
# as many Lit tests as necessary.
- if re.search('[.]gen[.][^.]+$', filename):
- for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
+ if re.search("[.]gen[.][^.]+$", filename):
+ for test in self._generateGenTest(
+ testSuite, pathInSuite, litConfig, localConfig
+ ):
yield test
else:
yield lit.Test.Test(testSuite, pathInSuite, localConfig)
def execute(self, test, litConfig):
@@ -326,34 +337,38 @@
# Make sure we have a directory to execute the generator test in
generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
os.makedirs(generatorExecDir, exist_ok=True)
# Run the generator test
- steps = [] # Steps must already be in the script
+ steps = [] # Steps must already be in the script
(out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
if exitCode != 0:
- raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
+ raise RuntimeError(
+ f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}"
+ )
# Split the generated output into multiple files and generate one test for each file
for subfile, content in self._splitFile(out):
generatedFile = testSuite.getExecPath(pathInSuite + (subfile,))
os.makedirs(os.path.dirname(generatedFile), exist_ok=True)
- with open(generatedFile, 'w') as f:
+ with open(generatedFile, "w") as f:
f.write(content)
yield lit.Test.Test(testSuite, (generatedFile,), localConfig)
def _splitFile(self, input):
- DELIM = r'^(//|#)---(.+)'
+ DELIM = r"^(//|#)---(.+)"
lines = input.splitlines()
currentFile = None
thisFileContent = []
for line in lines:
match = re.match(DELIM, line)
if match:
if currentFile is not None:
- yield (currentFile, '\n'.join(thisFileContent))
+ yield (currentFile, "\n".join(thisFileContent))
currentFile = match.group(2).strip()
thisFileContent = []
- assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}"
+ assert (
+ currentFile is not None
+ ), f"Some input to split-file doesn't belong to any file, input was:\n{input}"
thisFileContent.append(line)
if currentFile is not None:
- yield (currentFile, '\n'.join(thisFileContent))
+ yield (currentFile, "\n".join(thisFileContent))
|
ldionne
force-pushed
the
review/lit-format-move
branch
2 times, most recently
from
5d215af to
ecab1e5
Compare
mordante
reviewed
May 8, 2024
This allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++. With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient.
ldionne
force-pushed
the
review/lit-format-move
branch
from
ecab1e5 to
4871a82
Compare
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This allows the generally-useful parts of the test format to be shipped alongside Lit, which would make it usable for third-party developers as well. This came up at C++Now in the context of the Beman project where we'd like to set up a non-LLVM project that can use the same testing framework as libc++.
With the test format moved to Lit, the format would be available after installing Lit with pip, which is really convenient.