Closes #8520 - Fix up test execution for IntegrationTests (#8538)

SPM has functionality to skip a set of tests
using an undocumented environment variable,
'_SWIFTPM_SKIP_TESTS_LIST'. The presence of this variable causes test
output to change around starting/ending of test suites.

Test Suite 'All tests' started
vs.
Test Suite 'Selected tests' started

* Remove checking for the 'All tests' string.
* Check the return code of the test execution.
* Set up environment variables to match bootstrap in the
IntegrationTests execution.
* Change bootstrap to use a call() instead of call_output() as
call_output will block until the process has finished. A swift-test may
take a long time. Certain CI environments expects output to stream out,
and will cancel the build if none is detected for 30 mins.
* Update tests with using the 'checker', to output stdout/stderr for
easier debugging of failed expects.

Author: kcieplak

IntegrationTests/Sources/IntegrationTestSupport/Helpers.swift

@@ -14,6 +14,8 @@ import TSCBasic
 import TSCTestSupport
 import enum TSCUtility.Git
 
+public typealias ShReturnType = (stdout: String, stderr: String, returnCode: ProcessResult.ExitStatus)
+
 public let sdkRoot: AbsolutePath? = {
     if let environmentPath = ProcessInfo.processInfo.environment["SDK_ROOT"] {
         return try! AbsolutePath(validating: environmentPath)
@@ -131,7 +133,7 @@ public func sh(
     env: [String: String] = [:],
     file: StaticString = #file,
     line: UInt = #line
-) throws -> (stdout: String, stderr: String) {
+) throws -> ShReturnType {
     let result = try _sh(arguments, env: env, file: file, line: line)
     let stdout = try result.utf8Output()
     let stderr = try result.utf8stderrOutput()
@@ -145,7 +147,7 @@ public func sh(
             )
     }
 
-    return (stdout, stderr)
+    return (stdout, stderr, result.exitStatus)
 }
 
 @discardableResult
@@ -154,7 +156,7 @@ public func shFails(
     env: [String: String] = [:],
     file: StaticString = #file,
     line: UInt = #line
-) throws -> (stdout: String, stderr: String) {
+) throws -> ShReturnType {
     let result = try _sh(arguments, env: env, file: file, line: line)
     let stdout = try result.utf8Output()
     let stderr = try result.utf8stderrOutput()
@@ -168,7 +170,7 @@ public func shFails(
             )
     }
 
-    return (stdout, stderr)
+    return (stdout, stderr, result.exitStatus)
 }
 
 @discardableResult
@@ -385,4 +387,4 @@ extension ProcessResult {
         \((try? utf8stderrOutput()) ?? "")
         """
     }
-}
+}

IntegrationTests/Tests/IntegrationTests/BasicTests.swift

@@ -124,22 +124,22 @@ private struct BasicTests {
             let packagePath = tempDir.appending(component: "Project")
             try localFileSystem.createDirectory(packagePath)
             try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
-            let buildOutput = try sh(swiftBuild, "--package-path", packagePath).stdout
+            let packageOutput = try sh(swiftBuild, "--package-path", packagePath)
 
             // Check the build log.
-            let checker = StringChecker(string: buildOutput)
-            #expect(checker.check(.regex("Compiling .*Project.*")))
-            #expect(checker.check(.regex("Linking .*Project")))
-            #expect(checker.check(.contains("Build complete")))
+            let checker = StringChecker(string: packageOutput.stdout)
+            #expect(checker.check(.regex("Compiling .*Project.*")), "stdout: '\(packageOutput.stdout)'\n stderr:'\(packageOutput.stderr)'")
+            #expect(checker.check(.regex("Linking .*Project")), "stdout: '\(packageOutput.stdout)'\n stderr:'\(packageOutput.stderr)'")
+            #expect(checker.check(.contains("Build complete")), "stdout: '\(packageOutput.stdout)'\n stderr:'\(packageOutput.stderr)'")
 
             // Verify that the tool was built and works.
             let toolOutput = try sh(packagePath.appending(components: ".build", "debug", "Project"))
                 .stdout
             #expect(toolOutput.lowercased().contains("hello, world!"))
 
             // Check there were no compile errors or warnings.
-            #expect(buildOutput.contains("error") == false)
-            #expect(buildOutput.contains("warning") == false)
+            #expect(packageOutput.stdout.contains("error") == false)
+            #expect(packageOutput.stdout.contains("warning") == false)
         }
     }
 
@@ -151,17 +151,19 @@ private struct BasicTests {
             try localFileSystem.createDirectory(packagePath)
             withKnownIssue("error: no tests found; create a target in the 'Tests' directory") {
                 try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
-                let testOutput = try sh(swiftTest, "--package-path", packagePath).stdout
+                let packageOutput = try sh(swiftTest, "--package-path", packagePath, "--vv")
 
                 // Check the test log.
-                let checker = StringChecker(string: testOutput)
-                #expect(checker.check(.regex("Compiling .*ProjectTests.*")))
-                #expect(checker.check("Test Suite 'All tests' passed"))
-                #expect(checker.checkNext("Executed 1 test"))
+                let checker = StringChecker(string: packageOutput.stdout)
+                #expect(checker.check(.regex("Compiling .*ProjectTests.*")), "stdout: '\(packageOutput.stdout)'\n stderr:'\(packageOutput.stderr)'")
+                #expect(checker.checkNext("Executed 1 test"), "stdout: '\(packageOutput.stdout)'\n stderr:'\(packageOutput.stderr)'")
+
+                // Check the return code
+                #expect(packageOutput.returnCode == .terminated(code: 0))
 
                 // Check there were no compile errors or warnings.
-                #expect(testOutput.contains("error") == false)
-                #expect(testOutput.contains("warning") == false)
+                #expect(packageOutput.stdout.contains("error") == false)
+                #expect(packageOutput.stdout.contains("warning") == false)
             }
         }
     }
@@ -192,17 +194,14 @@ private struct BasicTests {
             let packagePath = tempDir.appending(component: "Project")
             try localFileSystem.createDirectory(packagePath)
             try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "library")
-            let testOutput = try sh(swiftTest, "--package-path", packagePath).stdout
+            let shOutput = try sh(swiftTest, "--package-path", packagePath)
 
-            // Check the test log.
-            let checker = StringChecker(string: testOutput)
-            #expect(checker.check(.contains("Test Suite 'All tests' started")))
-            #expect(checker.check(.contains("Test example() passed after")))
-            #expect(checker.checkNext(.contains("Test run with 1 test passed after")))
+            // Check the return code
+            #expect(shOutput.returnCode == .terminated(code: 0))
 
             // Check there were no compile errors or warnings.
-            #expect(testOutput.contains("error") == false)
-            #expect(testOutput.contains("warning") == false)
+            #expect(shOutput.stdout.contains("error") == false)
+            #expect(shOutput.stdout.contains("warning") == false)
         }
     }
 
@@ -248,11 +247,11 @@ private struct BasicTests {
             #expect(buildOutput.contains("Build complete"))
 
             // Verify that the tool exists and works.
-            let toolOutput = try sh(
+            let shOutput = try sh(
                 packagePath.appending(components: ".build", "debug", "special tool")
             ).stdout
 
-            #expect(toolOutput == "HI\(ProcessInfo.EOL)")
+            #expect(shOutput == "HI\(ProcessInfo.EOL)")
         }
     }
 
@@ -281,17 +280,20 @@ private struct BasicTests {
                     """
                 )
             )
-            let (runOutput, runError) = try sh(
+            let shOutput = try sh(
                 swiftRun, "--package-path", packagePath, "secho", "1", #""two""#
             )
 
             // Check the run log.
-            let checker = StringChecker(string: runError)
-            #expect(checker.check(.regex("Compiling .*secho.*")))
-            #expect(checker.check(.regex("Linking .*secho")))
-            #expect(checker.check(.contains("Build of product 'secho' complete")))
+            let checker = StringChecker(string: shOutput.stderr)
+            #expect(checker.check(.regex("Compiling .*secho.*")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.regex("Linking .*secho")),  "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.contains("Build of product 'secho' complete")),  "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+
+            #expect(shOutput.stdout == "1 \"two\"\(ProcessInfo.EOL)")
 
-            #expect(runOutput == "1 \"two\"\(ProcessInfo.EOL)")
+            // Check the return code
+            #expect(shOutput.returnCode == .terminated(code: 0))
         }
     }
 
@@ -318,16 +320,16 @@ private struct BasicTests {
                     """
                 )
             )
-            let testOutput = try sh(
+            let shOutput = try sh(
                 swiftTest, "--package-path", packagePath, "--filter", "MyTests.*", "--skip",
-                "testBaz"
-            ).stderr
+                "testBaz", "--vv"
+            )
 
             // Check the test log.
-            let checker = StringChecker(string: testOutput)
-            #expect(checker.check(.contains("Test Suite 'MyTests' started")))
-            #expect(checker.check(.contains("Test Suite 'MyTests' passed")))
-            #expect(checker.check(.contains("Executed 2 tests, with 0 failures")))
+            let checker = StringChecker(string: shOutput.stderr)
+            #expect(checker.check(.contains("Test Suite 'MyTests' started")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.contains("Test Suite 'MyTests' passed")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.contains("Executed 2 tests, with 0 failures")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
         }
     }
 
@@ -410,15 +412,15 @@ private struct BasicTests {
                 )
             )
 
-            let testOutput = try sh(
-                swiftTest, "--package-path", packagePath, "--filter", "MyTests.*"
-            ).stdout
+            let shOutput = try sh(
+                swiftTest, "--package-path", packagePath, "--filter", "MyTests.*", "--vv"
+            )
 
             // Check the test log.
-            let checker = StringChecker(string: testOutput)
-            #expect(checker.check(.contains("Test Suite 'MyTests' started")))
-            #expect(checker.check(.contains("Test Suite 'MyTests' passed")))
-            #expect(checker.check(.contains("Executed 2 tests, with 0 failures")))
+            let checker = StringChecker(string: shOutput.stdout)
+            #expect(checker.check(.contains("Test Suite 'MyTests' started")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.contains("Test Suite 'MyTests' passed")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
+            #expect(checker.check(.contains("Executed 2 tests, with 0 failures")), "stdout: '\(shOutput.stdout)'\n stderr:'\(shOutput.stderr)'")
         }
     }
 }

IntegrationTests/Tests/IntegrationTests/SwiftPMTests.swift

@@ -28,10 +28,10 @@ private struct SwiftPMTests {
             try binaryTargetsFixture { fixturePath in
                 do {
                     withKnownIssue("error: local binary target ... does not contain a binary artifact") {
-                        let (stdout, stderr) = try sh(swiftRun, "--package-path", fixturePath, "exe")
-                        #expect(!stderr.contains("error:"))
+                        let runOutput = try sh(swiftRun, "--package-path", fixturePath, "exe")
+                        #expect(!runOutput.stderr.contains("error:"))
                         #expect(
-                            stdout == """
+                            runOutput.stdout == """
                             SwiftFramework()
                             Library(framework: SwiftFramework.SwiftFramework())
 
@@ -42,31 +42,31 @@ private struct SwiftPMTests {
 
                 do {
                     withKnownIssue("error: local binary target ... does not contain a binary artifact") {
-                        let (stdout, stderr) = try sh(swiftRun, "--package-path", fixturePath, "cexe")
-                        #expect(!stderr.contains("error:"))
-                        #expect(stdout.contains("<CLibrary: "))
+                        let runOutput = try sh(swiftRun, "--package-path", fixturePath, "cexe")
+                        #expect(!runOutput.stderr.contains("error:"))
+                        #expect(runOutput.stdout.contains("<CLibrary: "))
                     }
                 }
 
                 do {
                     let invalidPath = fixturePath.appending(component: "SwiftFramework.xcframework")
-                    let (_, stderr) = try shFails(
+                    var packageOutput = try shFails(
                         swiftPackage, "--package-path", fixturePath, "compute-checksum", invalidPath
                     )
                     #expect(
                         // The order of supported extensions is not ordered, and changes.
                         //   '...supported extensions are: zip, tar.gz, tar'
                         //   '...supported extensions are: tar.gz, zip, tar'
                         // Only check for the start of that string.
-                        stderr.contains("error: unexpected file type; supported extensions are:")
+                        packageOutput.stderr.contains("error: unexpected file type; supported extensions are:")
                     )
 
                     let validPath = fixturePath.appending(component: "SwiftFramework.zip")
-                    let (stdout, _) = try sh(
+                    packageOutput = try sh(
                         swiftPackage, "--package-path", fixturePath, "compute-checksum", validPath
                     )
                     #expect(
-                        stdout.spm_chomp()
+                        packageOutput.stdout.spm_chomp()
                             == "d1f202b1bfe04dea30b2bc4038f8059dcd75a5a176f1d81fcaedb6d3597d1158"
                     )
                 }
@@ -106,18 +106,22 @@ private struct SwiftPMTests {
             try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
             try sh(swiftBuild, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue)
 
-            try withKnownIssue("Error while loading shared libraries: libswiftCore.so: cannot open shared object file: No such file or directory") {
-                // The 'native' build system uses 'swiftc' as the linker driver, which adds an RUNPATH to the swift runtime libraries in the SDK.
-                // 'swiftbuild' directly calls clang, which does not add the extra RUNPATH, so runtime libraries cannot be found.
-                let (stdout, stderr) = try sh(
+            try withKnownIssue(
+                "Error while loading shared libraries: libswiftCore.so: cannot open shared object file: No such file or directory"
+            ) {
+                // The 'native' build system uses 'swiftc' as the linker driver, which adds an RUNPATH to the swift
+                // runtime libraries in the SDK.
+                // 'swiftbuild' directly calls clang, which does not add the extra RUNPATH, so runtime libraries cannot
+                // be found.
+                let runOutput = try sh(
                     swiftRun, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue
                 )
-                #expect(!stderr.contains("error:"))
-                #expect(stdout.contains("Hello, world!"))
+                #expect(!runOutput.stderr.contains("error:"))
+                #expect(runOutput.stdout.contains("Hello, world!"))
             } when: {
                 buildSystemProvider == .swiftbuild && ProcessInfo.hostOperatingSystem == .linux
             }
-            }
+        }
     }
 
     @Test(
@@ -140,12 +144,13 @@ private struct SwiftPMTests {
                 """,
                 isIntermittent: true
             ) {
-                try sh(swiftBuild, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue, "--vv")
-                let (stdout, stderr) = try sh(
-                    swiftTest, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue, "--vv"
+                try sh(swiftBuild, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue)
+                let testOutput = try sh(
+                    swiftTest, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue
                 )
-                #expect(!stderr.contains("error:"))
-                #expect(stdout.contains("Test Suite 'All tests' passed"))
+                #expect(testOutput.returnCode == .terminated(code: 0))
+                #expect(!testOutput.stderr.contains("error:"))
+
             } when: {
                 buildSystemProvider == .swiftbuild
             }

IntegrationTests/Tests/IntegrationTests/XCBuildTests.swift

@@ -414,29 +414,29 @@ private struct XCBuildTests {
             let fooPath = path.appending(component: "Foo")
 
             do {
-                let (_, stderr) = try sh(swiftTest, "--package-path", fooPath, "--build-system", "xcode")
-                #expect(stderr.contains("Test Suite 'FooTests.xctest'"))
-                #expect(stderr.contains("Test Suite 'CFooTests.xctest'"))
+                let output = try sh(swiftTest, "--package-path", fooPath, "--build-system", "xcode")
+                #expect(output.stderr.contains("Test Suite 'FooTests.xctest'"))
+                #expect(output.stderr.contains("Test Suite 'CFooTests.xctest'"))
             }
 
             do {
-                let (_, stderr) = try sh(
+                let output = try sh(
                     swiftTest,
                     "--package-path",
                     fooPath,
                     "--build-system",
                     "xcode",
                     "--filter",
-                    "CFooTests"
+                    "CFooTests",
                 )
-                #expect(stderr.contains("Test Suite 'Selected tests' started"))
-                #expect(stderr.contains("Test Suite 'CFooTests.xctest'"))
+                #expect(output.stderr.contains("Test Suite 'Selected tests' started"))
+                #expect(output.stderr.contains("Test Suite 'CFooTests.xctest'"))
             }
 
             do {
-                let (stdout, _) = try sh(swiftTest, "--package-path", fooPath, "--build-system", "xcode", "--parallel")
-                #expect(stdout.contains("Testing FooTests"))
-                #expect(stdout.contains("Testing CFooTests"))
+                let output = try sh(swiftTest, "--package-path", fooPath, "--build-system", "xcode", "--parallel")
+                #expect(output.stdout.contains("Testing FooTests"))
+                #expect(output.stdout.contains("Testing CFooTests"))
             }
         }
     }

Utilities/bootstrap

@@ -810,7 +810,7 @@ def call_swiftpm(args, cmd, cwd=None):
     full_cmd = get_swiftpm_env_cmd(args) + cmd + get_swiftpm_flags(args)
     if cwd is None:
         cwd = args.project_root
-    call_output(full_cmd, cwd=cwd, stderr=True, verbose=True)
+    call(full_cmd, cwd=cwd)
 
 # -----------------------------------------------------------
 # Build-related helper functions