Handle generated resources from plugins

Up until now, we were treating all generated files as Swift source files. With this, we are involving `TargetSourcesBuilder` in the decision of how to treat a certain file, provided the tools version is 5.7 or newer.

Remaining limitations:
- generated files aren't treated the same way as non-generated ones, e.g. we will treat anything unknown as a resource to be processed and we won't respect declarations from the manifest
- files in Clang targets aren't treated at all at the moment

rdar://89693335

Author: neonichu

Fixtures/Miscellaneous/PluginGeneratedResources/Package.swift

@@ -0,0 +1,11 @@
+// swift-tools-version: 5.7
+
+import PackageDescription
+
+let package = Package(
+    name: "PluginGeneratedResources",
+    targets: [
+        .executableTarget(name: "PluginGeneratedResources", plugins: ["Generator"]),
+        .plugin(name: "Generator", capability: .buildTool()),
+    ]
+)

Fixtures/Miscellaneous/PluginGeneratedResources/Plugins/Generator/plugin.swift

@@ -0,0 +1,15 @@
+import PackagePlugin
+
+@main
+struct GeneratorPlugin: BuildToolPlugin {
+    func createBuildCommands(context: PluginContext, target: Target) async throws -> [Command] {
+        return [
+            .prebuildCommand(
+                displayName: "Generating empty file",
+                executable: .init("/usr/bin/touch"),
+                arguments: [context.pluginWorkDirectory.appending("best.txt")],
+                outputFilesDirectory: context.pluginWorkDirectory
+            )
+        ]
+    }
+}

Fixtures/Miscellaneous/PluginGeneratedResources/Sources/PluginGeneratedResources/PluginGeneratedResources.swift

@@ -0,0 +1,13 @@
+import Foundation
+
+@main
+public struct PluginGeneratedResources {
+    public private(set) var text = "Hello, World!"
+
+    public static func main() {
+        let path = Bundle.module.path(forResource: "best", ofType: "txt")
+        let exists = FileManager.default.fileExists(atPath: path!)
+        assert(exists, "generated file is missing")
+        print(PluginGeneratedResources().text)
+    }
+}

Sources/Build/BuildOperation.swift

@@ -13,6 +13,7 @@
 import Basics
 import LLBuildManifest
 import PackageGraph
+import PackageLoading
 import PackageModel
 import SPMBuildCore
 import SPMLLBuild
@@ -78,10 +79,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         (try? getBuildDescription())?.builtTestProducts ?? []
     }
 
+    /// File rules to determine resource handling behavior.
+    private let additionalFileRules: [FileRuleDescription]
+
     public init(
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
+        additionalFileRules: [FileRuleDescription],
         pluginScriptRunner: PluginScriptRunner,
         pluginWorkDirectory: AbsolutePath,
         disableSandboxForPluginCommands: Bool = false,
@@ -97,6 +102,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.buildParameters = buildParameters
         self.cacheBuildManifest = cacheBuildManifest
         self.packageGraphLoader = packageGraphLoader
+        self.additionalFileRules = additionalFileRules
         self.pluginScriptRunner = pluginScriptRunner
         self.pluginWorkDirectory = pluginWorkDirectory
         self.disableSandboxForPluginCommands = disableSandboxForPluginCommands
@@ -351,6 +357,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
+            additionalFileRules: additionalFileRules,
             buildToolPluginInvocationResults: buildToolPluginInvocationResults,
             prebuildCommandResults: prebuildCommandResults,
             fileSystem: self.fileSystem,
@@ -443,7 +450,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         return try pluginResults.map { pluginResult in
             // As we go we will collect a list of prebuild output directories whose contents should be input to the build,
             // and a list of the files in those directories after running the commands.
-            var derivedSourceFiles: [AbsolutePath] = []
+            var derivedFiles: [AbsolutePath] = []
             var prebuildOutputDirs: [AbsolutePath] = []
             for command in pluginResult.prebuildCommands {
                 self.observabilityScope.emit(info: "Running" + (command.configuration.displayName ?? command.configuration.executable.basename))
@@ -463,15 +470,15 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 // Add any files found in the output directory declared for the prebuild command after the command ends.
                 let outputFilesDir = command.outputFilesDirectory
                 if let swiftFiles = try? self.fileSystem.getDirectoryContents(outputFilesDir).sorted() {
-                    derivedSourceFiles.append(contentsOf: swiftFiles.map{ outputFilesDir.appending(component: $0) })
+                    derivedFiles.append(contentsOf: swiftFiles.map{ outputFilesDir.appending(component: $0) })
                 }
 
                 // Add the output directory to the list of directories whose structure should affect the build plan.
                 prebuildOutputDirs.append(outputFilesDir)
             }
 
             // Add the results of running any prebuild commands for this invocation.
-            return PrebuildCommandResult(derivedSourceFiles: derivedSourceFiles, outputDirectories: prebuildOutputDirs)
+            return PrebuildCommandResult(derivedFiles: derivedFiles, outputDirectories: prebuildOutputDirs)
         }
     }
 

Sources/Build/BuildPlan.swift

@@ -174,6 +174,17 @@ public enum TargetBuildDescription {
         }
     }
 
+    /// The resources in this target.
+    var resources: [Resource] {
+        switch self {
+        case .swift(let target):
+            return target.resources
+        case .clang(let target):
+            // TODO: Clang targets should support generated resources in the future.
+            return target.target.underlyingTarget.resources
+        }
+    }
+
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
         switch self {
@@ -239,7 +250,7 @@ public final class ClangTargetBuildDescription {
 
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        buildParameters.bundlePath(for: target)
+        target.underlyingTarget.bundleName.map(buildParameters.bundlePath(named:))
     }
 
     /// The modulemap file for this target, if any.
@@ -542,16 +553,28 @@ public final class SwiftTargetBuildDescription {
     /// These are the source files derived from plugins.
     private var pluginDerivedSources: Sources
 
+    /// These are the resource files derived from plugins.
+    private var pluginDerivedResources: [Resource]
+
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        buildParameters.bundlePath(for: target)
+        if let bundleName = target.underlyingTarget.potentialBundleName, !resources.isEmpty {
+            return buildParameters.bundlePath(named: bundleName)
+        } else {
+            return .none
+        }
     }
 
     /// The list of all source files in the target, including the derived ones.
     public var sources: [AbsolutePath] {
         target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
     }
 
+    /// The list of all resource files in the target, including the derived ones.
+    public var resources: [Resource] {
+        target.underlyingTarget.resources + pluginDerivedResources
+    }
+
     /// The objects in this target.
     public var objects: [AbsolutePath] {
         let relativePaths = target.sources.relativePaths + derivedSources.relativePaths + pluginDerivedSources.relativePaths
@@ -633,16 +656,21 @@ public final class SwiftTargetBuildDescription {
     /// The results of running any prebuild commands for this target.
     public let prebuildCommandResults: [PrebuildCommandResult]
 
+    /// ObservabilityScope with which to emit diagnostics
+    private let observabilityScope: ObservabilityScope
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         toolsVersion: ToolsVersion,
+        additionalFileRules: [FileRuleDescription] = [],
         buildParameters: BuildParameters,
         buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] = [],
         prebuildCommandResults: [PrebuildCommandResult] = [],
         isTestTarget: Bool? = nil,
         isTestDiscoveryTarget: Bool = false,
-        fileSystem: FileSystem
+        fileSystem: FileSystem,
+        observabilityScope: ObservabilityScope
     ) throws {
         guard target.underlyingTarget is SwiftTarget else {
             throw InternalError("underlying target type mismatch \(target)")
@@ -659,25 +687,31 @@ public final class SwiftTargetBuildDescription {
         self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
         self.buildToolPluginInvocationResults = buildToolPluginInvocationResults
         self.prebuildCommandResults = prebuildCommandResults
+        self.observabilityScope = observabilityScope
 
-        // Add any derived source files that were declared for any commands from plugin invocations.
+        // Add any derived files that were declared for any commands from plugin invocations.
+        var pluginDerivedFiles = [AbsolutePath]()
         for command in buildToolPluginInvocationResults.reduce([], { $0 + $1.buildCommands }) {
-            // TODO: What should we do if we find non-Swift sources here?
             for absPath in command.outputFiles {
-                let relPath = absPath.relative(to: self.pluginDerivedSources.root)
-                self.pluginDerivedSources.relativePaths.append(relPath)
+                pluginDerivedFiles.append(absPath)
             }
         }
 
-        // Add any derived source files that were discovered from output directories of prebuild commands.
+        // Add any derived files that were discovered from output directories of prebuild commands.
         for result in self.prebuildCommandResults {
-            // TODO: What should we do if we find non-Swift sources here?
-            for path in result.derivedSourceFiles {
-                let relPath = path.relative(to: self.pluginDerivedSources.root)
-                self.pluginDerivedSources.relativePaths.append(relPath)
+            for path in result.derivedFiles {
+                pluginDerivedFiles.append(path)
             }
         }
 
+        // Let `TargetSourcesBuilder` compute the treatment of plugin generated files.
+        let (derivedSources, derivedResources) = TargetSourcesBuilder.computeContents(for: pluginDerivedFiles, toolsVersion: toolsVersion, additionalFileRules: additionalFileRules, defaultLocalization: target.defaultLocalization, targetName: target.name, targetPath: target.underlyingTarget.path, observabilityScope: observabilityScope)
+        self.pluginDerivedResources = derivedResources
+        derivedSources.forEach { absPath in
+            let relPath = absPath.relative(to: self.pluginDerivedSources.root)
+            self.pluginDerivedSources.relativePaths.append(relPath)
+        }
+
         if shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
         }
@@ -1585,7 +1619,8 @@ public class BuildPlan {
                     toolsVersion: toolsVersion,
                     buildParameters: buildParameters,
                     isTestTarget: true,
-                    fileSystem: fileSystem
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope
                 )
 
                 result.append((testProduct, desc))
@@ -1621,7 +1656,8 @@ public class BuildPlan {
                     buildParameters: buildParameters,
                     isTestTarget: true,
                     isTestDiscoveryTarget: true,
-                    fileSystem: fileSystem
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope
                 )
 
                 result.append((testProduct, target))
@@ -1635,28 +1671,11 @@ public class BuildPlan {
         return result
     }
 
-    @available(*, deprecated, message: "use observability system instead")
-    public convenience init(
-        buildParameters: BuildParameters,
-        graph: PackageGraph,
-        buildToolPluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
-        prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
-        diagnostics: DiagnosticsEngine,
-        fileSystem: FileSystem
-    ) throws {
-        let observabilitySystem = ObservabilitySystem(diagnosticEngine: diagnostics)
-        try self.init(
-            buildParameters: buildParameters,
-            graph: graph,
-            fileSystem: fileSystem,
-            observabilityScope: observabilitySystem.topScope
-        )
-    }
-
     /// Create a build plan with build parameters and a package graph.
     public init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
+        additionalFileRules: [FileRuleDescription] = [],
         buildToolPluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
         prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         fileSystem: FileSystem,
@@ -1700,10 +1719,12 @@ public class BuildPlan {
                 targetMap[target] = try .swift(SwiftTargetBuildDescription(
                     target: target,
                     toolsVersion: toolsVersion,
+                    additionalFileRules: additionalFileRules,
                     buildParameters: buildParameters,
                     buildToolPluginInvocationResults: buildToolPluginInvocationResults[target] ?? [],
                     prebuildCommandResults: prebuildCommandResults[target] ?? [],
-                    fileSystem: fileSystem)
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope)
                 )
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(
@@ -2236,11 +2257,9 @@ private extension Basics.Diagnostic {
 }
 
 extension BuildParameters {
-    /// Returns a target's bundle path inside the build directory.
-    fileprivate func bundlePath(for target: ResolvedTarget) -> AbsolutePath? {
-        target.underlyingTarget.bundleName
-        .map{ $0 + triple.nsbundleExtension }
-        .map(buildPath.appending(component:))
+    /// Returns a named bundle's path inside the build directory.
+    fileprivate func bundlePath(named name: String) -> AbsolutePath {
+        return buildPath.appending(component: name + triple.nsbundleExtension)
     }
 }
 

Sources/Build/LLBuildManifestBuilder.swift

@@ -171,7 +171,7 @@ extension LLBuildManifestBuilder {
         let infoPlistDestination = RelativePath("Info.plist")
 
         // Create a copy command for each resource file.
-        for resource in target.target.underlyingTarget.resources {
+        for resource in target.resources {
             let destination = bundlePath.appending(resource.destination)
             let (_, output) = addCopyCommand(from: resource.path, to: destination)
             outputs.append(output)

Sources/Commands/SwiftTool.swift

@@ -752,6 +752,7 @@ public class SwiftTool {
             buildParameters: customBuildParameters ?? self.buildParameters(),
             cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
             packageGraphLoader: customPackageGraphLoader ?? graphLoader,
+            additionalFileRules: FileRuleDescription.swiftpmFileTypes,
             pluginScriptRunner: self.getPluginScriptRunner(),
             pluginWorkDirectory: try self.getActiveWorkspace().location.pluginWorkingDirectory,
             disableSandboxForPluginCommands: self.options.security.shouldDisableSandbox,

Sources/PackageLoading/PackageBuilder.swift

@@ -767,7 +767,7 @@ public final class PackageBuilder {
 
         // FIXME: use identity instead?
         // The name of the bundle, if one is being generated.
-        let bundleName = resources.isEmpty ? nil : self.manifest.displayName + "_" + potentialModule.name
+        let potentialBundleName = self.manifest.displayName + "_" + potentialModule.name
 
         if sources.relativePaths.isEmpty && resources.isEmpty {
             return nil
@@ -808,8 +808,9 @@ public final class PackageBuilder {
         if sources.hasSwiftSources {
             return SwiftTarget(
                 name: potentialModule.name,
-                bundleName: bundleName,
+                potentialBundleName: potentialBundleName,
                 type: targetType,
+                path: potentialModule.path,
                 sources: sources,
                 resources: resources,
                 ignored: ignored,
@@ -836,13 +837,14 @@ public final class PackageBuilder {
 
             return try ClangTarget(
                 name: potentialModule.name,
-                bundleName: bundleName,
+                potentialBundleName: potentialBundleName,
                 cLanguageStandard: manifest.cLanguageStandard,
                 cxxLanguageStandard: manifest.cxxLanguageStandard,
                 includeDir: publicHeadersPath,
                 moduleMapType: moduleMapType,
                 headers: headers,
                 type: targetType,
+                path: potentialModule.path,
                 sources: sources,
                 resources: resources,
                 ignored: ignored,
@@ -1362,6 +1364,7 @@ extension PackageBuilder {
                 return SwiftTarget(
                     name: name,
                     type: .snippet,
+                    path: .root,
                     sources: sources,
                     dependencies: dependencies,
                     swiftVersion: try swiftVersion(),