Handle generated resources from plugins

rdar://89693335

Author: neonichu

Sources/Build/BuildOperation.swift

@@ -13,6 +13,7 @@
 import Basics
 import LLBuildManifest
 import PackageGraph
+import PackageLoading
 import PackageModel
 import SPMBuildCore
 import SPMLLBuild
@@ -78,10 +79,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         (try? getBuildDescription())?.builtTestProducts ?? []
     }
 
+    /// File rules to determine resource handling behavior.
+    private let additionalFileRules: [FileRuleDescription]
+
     public init(
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
+        additionalFileRules: [FileRuleDescription],
         pluginScriptRunner: PluginScriptRunner,
         pluginWorkDirectory: AbsolutePath,
         disableSandboxForPluginCommands: Bool = false,
@@ -97,6 +102,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.buildParameters = buildParameters
         self.cacheBuildManifest = cacheBuildManifest
         self.packageGraphLoader = packageGraphLoader
+        self.additionalFileRules = additionalFileRules
         self.pluginScriptRunner = pluginScriptRunner
         self.pluginWorkDirectory = pluginWorkDirectory
         self.disableSandboxForPluginCommands = disableSandboxForPluginCommands
@@ -351,6 +357,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
+            additionalFileRules: additionalFileRules,
             buildToolPluginInvocationResults: buildToolPluginInvocationResults,
             prebuildCommandResults: prebuildCommandResults,
             fileSystem: self.fileSystem,
@@ -443,7 +450,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         return try pluginResults.map { pluginResult in
             // As we go we will collect a list of prebuild output directories whose contents should be input to the build,
             // and a list of the files in those directories after running the commands.
-            var derivedSourceFiles: [AbsolutePath] = []
+            var derivedFiles: [AbsolutePath] = []
             var prebuildOutputDirs: [AbsolutePath] = []
             for command in pluginResult.prebuildCommands {
                 self.observabilityScope.emit(info: "Running" + (command.configuration.displayName ?? command.configuration.executable.basename))
@@ -463,15 +470,15 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 // Add any files found in the output directory declared for the prebuild command after the command ends.
                 let outputFilesDir = command.outputFilesDirectory
                 if let swiftFiles = try? self.fileSystem.getDirectoryContents(outputFilesDir).sorted() {
-                    derivedSourceFiles.append(contentsOf: swiftFiles.map{ outputFilesDir.appending(component: $0) })
+                    derivedFiles.append(contentsOf: swiftFiles.map{ outputFilesDir.appending(component: $0) })
                 }
 
                 // Add the output directory to the list of directories whose structure should affect the build plan.
                 prebuildOutputDirs.append(outputFilesDir)
             }
 
             // Add the results of running any prebuild commands for this invocation.
-            return PrebuildCommandResult(derivedSourceFiles: derivedSourceFiles, outputDirectories: prebuildOutputDirs)
+            return PrebuildCommandResult(derivedFiles: derivedFiles, outputDirectories: prebuildOutputDirs)
         }
     }
 

Sources/Build/BuildPlan.swift

@@ -174,6 +174,17 @@ public enum TargetBuildDescription {
         }
     }
 
+    /// The resources in this target.
+    var resources: [Resource] {
+        switch self {
+        case .swift(let target):
+            return target.resources
+        case .clang(let target):
+            // TODO: Clang targets should support generated resources in the future.
+            return target.target.underlyingTarget.resources
+        }
+    }
+
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
         switch self {
@@ -239,7 +250,7 @@ public final class ClangTargetBuildDescription {
 
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        buildParameters.bundlePath(for: target)
+        target.underlyingTarget.bundleName.map(buildParameters.bundlePath(named:))
     }
 
     /// The modulemap file for this target, if any.
@@ -542,16 +553,28 @@ public final class SwiftTargetBuildDescription {
     /// These are the source files derived from plugins.
     private var pluginDerivedSources: Sources
 
+    /// These are the resource files derived from plugins.
+    private var pluginDerivedResources: [Resource]
+
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
-        buildParameters.bundlePath(for: target)
+        if let bundleName = target.underlyingTarget.potentialBundleName, !resources.isEmpty {
+            return buildParameters.bundlePath(named: bundleName)
+        } else {
+            return nil
+        }
     }
 
     /// The list of all source files in the target, including the derived ones.
     public var sources: [AbsolutePath] {
         target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
     }
 
+    /// The list of all resource files in the target, including the derived ones.
+    public var resources: [Resource] {
+        target.underlyingTarget.resources + pluginDerivedResources
+    }
+
     /// The objects in this target.
     public var objects: [AbsolutePath] {
         let relativePaths = target.sources.relativePaths + derivedSources.relativePaths + pluginDerivedSources.relativePaths
@@ -633,16 +656,21 @@ public final class SwiftTargetBuildDescription {
     /// The results of running any prebuild commands for this target.
     public let prebuildCommandResults: [PrebuildCommandResult]
 
+    /// ObservabilityScope with which to emit diagnostics
+    private let observabilityScope: ObservabilityScope
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         toolsVersion: ToolsVersion,
+        additionalFileRules: [FileRuleDescription] = [],
         buildParameters: BuildParameters,
         buildToolPluginInvocationResults: [BuildToolPluginInvocationResult] = [],
         prebuildCommandResults: [PrebuildCommandResult] = [],
         isTestTarget: Bool? = nil,
         isTestDiscoveryTarget: Bool = false,
-        fileSystem: FileSystem
+        fileSystem: FileSystem,
+        observabilityScope: ObservabilityScope
     ) throws {
         guard target.underlyingTarget is SwiftTarget else {
             throw InternalError("underlying target type mismatch \(target)")
@@ -657,24 +685,60 @@ public final class SwiftTargetBuildDescription {
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
         self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
+        self.pluginDerivedResources = []
         self.buildToolPluginInvocationResults = buildToolPluginInvocationResults
         self.prebuildCommandResults = prebuildCommandResults
+        self.observabilityScope = observabilityScope
+
+        func handlePluginDerivedFile(path absPath: AbsolutePath) {
+            // 5.6 handled treated all generated files as sources.
+            if toolsVersion <= .v5_6 {
+                let relPath = absPath.relative(to: self.pluginDerivedSources.root)
+                self.pluginDerivedSources.relativePaths.append(relPath)
+                return
+            }
+
+            var rule = TargetSourcesBuilder.computeRule(for: absPath, toolsVersion: toolsVersion, additionalFileRules: additionalFileRules, observabilityScope: observabilityScope)
+
+            // If we did not find a rule for a generated file, we treat it as to be processed for now. Eventually, we should handle generated files the same as other files and require explicit handling in the manifest for unknown types.
+            if rule == .none {
+                rule = .processResource(localization: .none)
+            }
+
+            switch rule {
+            case .compile:
+                if absPath.extension == "swift" {
+                    let relPath = absPath.relative(to: self.pluginDerivedSources.root)
+                    self.pluginDerivedSources.relativePaths.append(relPath)
+                } else {
+                    observabilityScope.emit(error: "Generated plugin source files can only use Swift: \(absPath)")
+                }
+            case .copy, .processResource:
+                if let resource = TargetSourcesBuilder.resource(for: absPath, with: rule, defaultLocalization: target.defaultLocalization, targetName: target.name, targetPath: target.underlyingTarget.path, observabilityScope: observabilityScope) {
+                    self.pluginDerivedResources.append(resource)
+                } else {
+                    // If this is reached, `TargetSourcesBuilder` already emitted a diagnostic, so we can ignore this case here.
+                }
+            case .header:
+                observabilityScope.emit(error: "Headers generated by plugins are not supported at this time: \(absPath)")
+            case .modulemap:
+                observabilityScope.emit(error: "Module maps generated by plugins are not supported at this time: \(absPath)")
+            case .ignored, .none:
+                break
+            }
+        }
 
         // Add any derived source files that were declared for any commands from plugin invocations.
         for command in buildToolPluginInvocationResults.reduce([], { $0 + $1.buildCommands }) {
-            // TODO: What should we do if we find non-Swift sources here?
             for absPath in command.outputFiles {
-                let relPath = absPath.relative(to: self.pluginDerivedSources.root)
-                self.pluginDerivedSources.relativePaths.append(relPath)
+                handlePluginDerivedFile(path: absPath)
             }
         }
 
         // Add any derived source files that were discovered from output directories of prebuild commands.
         for result in self.prebuildCommandResults {
-            // TODO: What should we do if we find non-Swift sources here?
-            for path in result.derivedSourceFiles {
-                let relPath = path.relative(to: self.pluginDerivedSources.root)
-                self.pluginDerivedSources.relativePaths.append(relPath)
+            for path in result.derivedFiles {
+                handlePluginDerivedFile(path: path)
             }
         }
 
@@ -1585,7 +1649,8 @@ public class BuildPlan {
                     toolsVersion: toolsVersion,
                     buildParameters: buildParameters,
                     isTestTarget: true,
-                    fileSystem: fileSystem
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope
                 )
 
                 result.append((testProduct, desc))
@@ -1621,7 +1686,8 @@ public class BuildPlan {
                     buildParameters: buildParameters,
                     isTestTarget: true,
                     isTestDiscoveryTarget: true,
-                    fileSystem: fileSystem
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope
                 )
 
                 result.append((testProduct, target))
@@ -1635,28 +1701,11 @@ public class BuildPlan {
         return result
     }
 
-    @available(*, deprecated, message: "use observability system instead")
-    public convenience init(
-        buildParameters: BuildParameters,
-        graph: PackageGraph,
-        buildToolPluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
-        prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
-        diagnostics: DiagnosticsEngine,
-        fileSystem: FileSystem
-    ) throws {
-        let observabilitySystem = ObservabilitySystem(diagnosticEngine: diagnostics)
-        try self.init(
-            buildParameters: buildParameters,
-            graph: graph,
-            fileSystem: fileSystem,
-            observabilityScope: observabilitySystem.topScope
-        )
-    }
-
     /// Create a build plan with build parameters and a package graph.
     public init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
+        additionalFileRules: [FileRuleDescription] = [],
         buildToolPluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]] = [:],
         prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         fileSystem: FileSystem,
@@ -1700,10 +1749,12 @@ public class BuildPlan {
                 targetMap[target] = try .swift(SwiftTargetBuildDescription(
                     target: target,
                     toolsVersion: toolsVersion,
+                    additionalFileRules: additionalFileRules,
                     buildParameters: buildParameters,
                     buildToolPluginInvocationResults: buildToolPluginInvocationResults[target] ?? [],
                     prebuildCommandResults: prebuildCommandResults[target] ?? [],
-                    fileSystem: fileSystem)
+                    fileSystem: fileSystem,
+                    observabilityScope: observabilityScope)
                 )
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(
@@ -2236,11 +2287,9 @@ private extension Basics.Diagnostic {
 }
 
 extension BuildParameters {
-    /// Returns a target's bundle path inside the build directory.
-    fileprivate func bundlePath(for target: ResolvedTarget) -> AbsolutePath? {
-        target.underlyingTarget.bundleName
-        .map{ $0 + triple.nsbundleExtension }
-        .map(buildPath.appending(component:))
+    /// Returns a named bundle's path inside the build directory.
+    fileprivate func bundlePath(named name: String) -> AbsolutePath {
+        return buildPath.appending(component: name + triple.nsbundleExtension)
     }
 }
 

Sources/Build/LLBuildManifestBuilder.swift

@@ -171,7 +171,7 @@ extension LLBuildManifestBuilder {
         let infoPlistDestination = RelativePath("Info.plist")
 
         // Create a copy command for each resource file.
-        for resource in target.target.underlyingTarget.resources {
+        for resource in target.resources {
             let destination = bundlePath.appending(resource.destination)
             let (_, output) = addCopyCommand(from: resource.path, to: destination)
             outputs.append(output)

Sources/Commands/SwiftTool.swift

@@ -752,6 +752,7 @@ public class SwiftTool {
             buildParameters: customBuildParameters ?? self.buildParameters(),
             cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
             packageGraphLoader: customPackageGraphLoader ?? graphLoader,
+            additionalFileRules: FileRuleDescription.swiftpmFileTypes,
             pluginScriptRunner: self.getPluginScriptRunner(),
             pluginWorkDirectory: try self.getActiveWorkspace().location.pluginWorkingDirectory,
             disableSandboxForPluginCommands: self.options.security.shouldDisableSandbox,

Sources/PackageLoading/PackageBuilder.swift

@@ -767,7 +767,7 @@ public final class PackageBuilder {
 
         // FIXME: use identity instead?
         // The name of the bundle, if one is being generated.
-        let bundleName = resources.isEmpty ? nil : self.manifest.displayName + "_" + potentialModule.name
+        let potentialBundleName = self.manifest.displayName + "_" + potentialModule.name
 
         if sources.relativePaths.isEmpty && resources.isEmpty {
             return nil
@@ -808,8 +808,9 @@ public final class PackageBuilder {
         if sources.hasSwiftSources {
             return SwiftTarget(
                 name: potentialModule.name,
-                bundleName: bundleName,
+                potentialBundleName: potentialBundleName,
                 type: targetType,
+                path: potentialModule.path,
                 sources: sources,
                 resources: resources,
                 ignored: ignored,
@@ -836,13 +837,14 @@ public final class PackageBuilder {
 
             return try ClangTarget(
                 name: potentialModule.name,
-                bundleName: bundleName,
+                potentialBundleName: potentialBundleName,
                 cLanguageStandard: manifest.cLanguageStandard,
                 cxxLanguageStandard: manifest.cxxLanguageStandard,
                 includeDir: publicHeadersPath,
                 moduleMapType: moduleMapType,
                 headers: headers,
                 type: targetType,
+                path: potentialModule.path,
                 sources: sources,
                 resources: resources,
                 ignored: ignored,
@@ -1362,6 +1364,7 @@ extension PackageBuilder {
                 return SwiftTarget(
                     name: name,
                     type: .snippet,
+                    path: .root,
                     sources: sources,
                     dependencies: dependencies,
                     swiftVersion: try swiftVersion(),