Support opening documents within the same workspace with sourcekitd/clangd from different toolchains

Sources/BuildSystemIntegration/BuildSystemManager.swift

@@ -202,6 +202,7 @@ private extension BuildSystemSpec {
       ) { connectionToSourceKitLSP in
         try JSONCompilationDatabaseBuildSystem(
           configPath: configPath,
+          toolchainRegistry: toolchainRegistry,
           connectionToSourceKitLSP: connectionToSourceKitLSP
         )
       }

Sources/BuildSystemIntegration/BuildTargetIdentifierExtensions.swift

@@ -43,9 +43,14 @@ package enum BuildDestinationIdentifier {
   }
 }
 
+// MARK: BuildTargetIdentifier for SwiftPM
+
 extension BuildTargetIdentifier {
   /// - Important: *For testing only*
-  package init(target: String, destination: BuildDestinationIdentifier) throws {
+  package static func createSwiftPM(
+    target: String,
+    destination: BuildDestinationIdentifier
+  ) throws -> BuildTargetIdentifier {
     var components = URLComponents()
     components.scheme = "swiftpm"
     components.host = "target"
@@ -67,12 +72,12 @@ extension BuildTargetIdentifier {
       throw FailedToConvertSwiftBuildTargetToUrlError(target: target, destination: destination.id)
     }
 
-    self.init(uri: URI(url))
+    return BuildTargetIdentifier(uri: URI(url))
   }
 
-  fileprivate static let forPackageManifest = BuildTargetIdentifier(uri: try! URI(string: "swiftpm://package-manifest"))
+  static let forPackageManifest = BuildTargetIdentifier(uri: try! URI(string: "swiftpm://package-manifest"))
 
-  fileprivate var targetProperties: (target: String, runDestination: String) {
+  var swiftpmTargetProperties: (target: String, runDestination: String) {
     get throws {
       struct InvalidTargetIdentifierError: Swift.Error, CustomStringConvertible {
         var target: BuildTargetIdentifier
@@ -96,6 +101,57 @@ extension BuildTargetIdentifier {
   }
 }
 
+// MARK: BuildTargetIdentifier for CompileCommands
+
+extension BuildTargetIdentifier {
+  /// - Important: *For testing only*
+  package static func createCompileCommands(compiler: String) throws -> BuildTargetIdentifier {
+    var components = URLComponents()
+    components.scheme = "compilecommands"
+    components.host = "target"
+    components.queryItems = [URLQueryItem(name: "compiler", value: compiler)]
+
+    struct FailedToConvertSwiftBuildTargetToUrlError: Swift.Error, CustomStringConvertible {
+      var compiler: String
+
+      var description: String {
+        return "Failed to generate URL for compiler: \(compiler)"
+      }
+    }
+
+    guard let url = components.url else {
+      throw FailedToConvertSwiftBuildTargetToUrlError(compiler: compiler)
+    }
+
+    return BuildTargetIdentifier(uri: URI(url))
+  }
+
+  var compileCommandsCompiler: String {
+    get throws {
+      struct InvalidTargetIdentifierError: Swift.Error, CustomStringConvertible {
+        var target: BuildTargetIdentifier
+
+        var description: String {
+          return "Invalid target identifier \(target)"
+        }
+      }
+      guard let components = URLComponents(url: self.uri.arbitrarySchemeURL, resolvingAgainstBaseURL: false) else {
+        throw InvalidTargetIdentifierError(target: self)
+      }
+      guard components.scheme == "compilecommands", components.host == "target" else {
+        throw InvalidTargetIdentifierError(target: self)
+      }
+      let compiler = components.queryItems?.last(where: { $0.name == "compiler" })?.value
+
+      guard let compiler else {
+        throw InvalidTargetIdentifierError(target: self)
+      }
+
+      return compiler
+    }
+  }
+}
+
 #if compiler(>=6)
 extension BuildTargetIdentifier: CustomLogStringConvertible {
   package var description: String {

Sources/BuildSystemIntegration/CompilationDatabase.swift

@@ -11,15 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #if compiler(>=6)
-package import BuildServerProtocol
 package import Foundation
 package import LanguageServerProtocol
 import LanguageServerProtocolExtensions
 import SKLogging
 import SwiftExtensions
 import TSCExtensions
 #else
-import BuildServerProtocol
 import Foundation
 import LanguageServerProtocol
 import LanguageServerProtocolExtensions
@@ -164,12 +162,6 @@ package struct JSONCompilationDatabase: Equatable, Codable {
     return []
   }
 
-  package var sourceItems: [SourceItem] {
-    return commands.map {
-      SourceItem(uri: $0.uri, kind: .file, generated: false)
-    }
-  }
-
   private mutating func add(_ command: CompilationDatabaseCompileCommand) {
     let uri = command.uri
     pathToCommands[uri, default: []].append(commands.count)

Sources/BuildSystemIntegration/JSONCompilationDatabaseBuildSystem.swift

@@ -17,12 +17,27 @@ import SwiftExtensions
 package import BuildServerProtocol
 package import Foundation
 package import LanguageServerProtocol
+package import ToolchainRegistry
 #else
 import BuildServerProtocol
 import Foundation
 import LanguageServerProtocol
+import ToolchainRegistry
 #endif
 
+fileprivate extension CompilationDatabaseCompileCommand {
+  /// The first entry in the command line identifies the compiler that should be used to compile the file and can thus
+  /// be used to infer the toolchain.
+  ///
+  /// Note that this compiler does not necessarily need to exist on disk. Eg. tools may just use `clang` as the compiler
+  /// without specifying a path.
+  ///
+  /// The absence of a compiler means we have an empty command line, which should never happen.
+  var compiler: String? {
+    return commandLine.first
+  }
+}
+
 /// A `BuildSystem` that provides compiler arguments from a `compile_commands.json` file.
 package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   package static let dbName: String = "compile_commands.json"
@@ -36,6 +51,8 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     }
   }
 
+  private let toolchainRegistry: ToolchainRegistry
+
   private let connectionToSourceKitLSP: any Connection
 
   package let configPath: URL
@@ -73,34 +90,55 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
 
   package init(
     configPath: URL,
+    toolchainRegistry: ToolchainRegistry,
     connectionToSourceKitLSP: any Connection
   ) throws {
     self.compdb = try JSONCompilationDatabase(file: configPath)
+    self.toolchainRegistry = toolchainRegistry
     self.connectionToSourceKitLSP = connectionToSourceKitLSP
-
     self.configPath = configPath
   }
 
   package func buildTargets(request: WorkspaceBuildTargetsRequest) async throws -> WorkspaceBuildTargetsResponse {
-    return WorkspaceBuildTargetsResponse(targets: [
-      BuildTarget(
-        id: .dummy,
+    let compilers = Set(compdb.commands.compactMap(\.compiler)).sorted { $0 < $1 }
+    let targets = try await compilers.asyncMap { compiler in
+      let toolchainUri: URI? =
+        if let toolchainPath = await toolchainRegistry.toolchain(withCompiler: URL(fileURLWithPath: compiler))?.path {
+          URI(toolchainPath)
+        } else {
+          nil
+        }
+      return BuildTarget(
+        id: try BuildTargetIdentifier.createCompileCommands(compiler: compiler),
         displayName: nil,
         baseDirectory: nil,
         tags: [.test],
         capabilities: BuildTargetCapabilities(),
         // Be conservative with the languages that might be used in the target. SourceKit-LSP doesn't use this property.
         languageIds: [.c, .cpp, .objective_c, .objective_cpp, .swift],
-        dependencies: []
+        dependencies: [],
+        dataKind: .sourceKit,
+        data: SourceKitBuildTarget(toolchain: toolchainUri).encodeToLSPAny()
       )
-    ])
+    }
+    return WorkspaceBuildTargetsResponse(targets: targets)
   }
 
   package func buildTargetSources(request: BuildTargetSourcesRequest) async throws -> BuildTargetSourcesResponse {
-    guard request.targets.contains(.dummy) else {
-      return BuildTargetSourcesResponse(items: [])
+    let items = request.targets.compactMap { (target) -> SourcesItem? in
+      guard let targetCompiler = orLog("Compiler for target", { try target.compileCommandsCompiler }) else {
+        return nil
+      }
+      let commandsWithRequestedCompilers = compdb.commands.lazy.filter { command in
+        return targetCompiler == command.compiler
+      }
+      let sources = commandsWithRequestedCompilers.map {
+        SourceItem(uri: $0.uri, kind: .file, generated: false)
+      }
+      return SourcesItem(target: target, sources: Array(sources))
     }
-    return BuildTargetSourcesResponse(items: [SourcesItem(target: .dummy, sources: compdb.sourceItems)])
+
+    return BuildTargetSourcesResponse(items: items)
   }
 
   package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) {
@@ -116,12 +154,16 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   package func sourceKitOptions(
     request: TextDocumentSourceKitOptionsRequest
   ) async throws -> TextDocumentSourceKitOptionsResponse? {
-    guard let cmd = compdb[request.textDocument.uri].first else {
+    let targetCompiler = try request.target.compileCommandsCompiler
+    let command = compdb[request.textDocument.uri].filter {
+      $0.compiler == targetCompiler
+    }.first
+    guard let command else {
       return nil
     }
     return TextDocumentSourceKitOptionsResponse(
-      compilerArguments: Array(cmd.commandLine.dropFirst()),
-      workingDirectory: cmd.directory
+      compilerArguments: Array(command.commandLine.dropFirst()),
+      workingDirectory: command.directory
     )
   }
 

Sources/BuildSystemIntegration/SwiftPMBuildSystem.swift

@@ -75,32 +75,10 @@ extension BuildDestinationIdentifier {
 
 extension BuildTargetIdentifier {
   fileprivate init(_ buildTarget: any SwiftBuildTarget) throws {
-    try self.init(target: buildTarget.name, destination: BuildDestinationIdentifier(buildTarget.destination))
-  }
-
-  fileprivate static let forPackageManifest = BuildTargetIdentifier(uri: try! URI(string: "swiftpm://package-manifest"))
-
-  fileprivate var targetProperties: (target: String, runDestination: String) {
-    get throws {
-      struct InvalidTargetIdentifierError: Swift.Error, CustomStringConvertible {
-        var target: BuildTargetIdentifier
-
-        var description: String {
-          return "Invalid target identifier \(target)"
-        }
-      }
-      guard let components = URLComponents(url: self.uri.arbitrarySchemeURL, resolvingAgainstBaseURL: false) else {
-        throw InvalidTargetIdentifierError(target: self)
-      }
-      let target = components.queryItems?.last(where: { $0.name == "target" })?.value
-      let runDestination = components.queryItems?.last(where: { $0.name == "destination" })?.value
-
-      guard let target, let runDestination else {
-        throw InvalidTargetIdentifierError(target: self)
-      }
-
-      return (target, runDestination)
-    }
+    self = try Self.createSwiftPM(
+      target: buildTarget.name,
+      destination: BuildDestinationIdentifier(buildTarget.destination)
+    )
   }
 }
 
@@ -608,7 +586,7 @@ package actor SwiftPMBuildSystem: BuiltInBuildSystem {
       "--package-path", try projectRoot.filePath,
       "--scratch-path", self.swiftPMWorkspace.location.scratchDirectory.pathString,
       "--disable-index-store",
-      "--target", try target.targetProperties.target,
+      "--target", try target.swiftpmTargetProperties.target,
     ]
     if options.swiftPMOrDefault.disableSandbox ?? false {
       arguments += ["--disable-sandbox"]

Sources/SKTestSupport/MultiFileTestProject.swift

@@ -10,18 +10,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+import SwiftExtensions
+
 #if compiler(>=6)
 package import Foundation
 package import LanguageServerProtocol
 package import SKOptions
 package import SourceKitLSP
-import SwiftExtensions
+package import ToolchainRegistry
 #else
 import Foundation
 import LanguageServerProtocol
 import SKOptions
 import SourceKitLSP
-import SwiftExtensions
+import ToolchainRegistry
 #endif
 
 /// The location of a test file within test workspace.
@@ -137,6 +139,7 @@ package class MultiFileTestProject {
     initializationOptions: LSPAny? = nil,
     capabilities: ClientCapabilities = ClientCapabilities(),
     options: SourceKitLSPOptions = .testDefault(),
+    toolchainRegistry: ToolchainRegistry = .forTesting,
     hooks: Hooks = Hooks(),
     enableBackgroundIndexing: Bool = false,
     usePullDiagnostics: Bool = true,
@@ -152,6 +155,7 @@ package class MultiFileTestProject {
       hooks: hooks,
       initializationOptions: initializationOptions,
       capabilities: capabilities,
+      toolchainRegistry: toolchainRegistry,
       usePullDiagnostics: usePullDiagnostics,
       enableBackgroundIndexing: enableBackgroundIndexing,
       workspaceFolders: workspaces(scratchDirectory),

Sources/SKTestSupport/TestSourceKitLSPClient.swift

@@ -10,34 +10,26 @@
 //
 //===----------------------------------------------------------------------===//
 
-#if compiler(>=6)
 import Foundation
 import InProcessClient
-package import LanguageServerProtocol
-import LanguageServerProtocolJSONRPC
 import LanguageServerProtocolExtensions
-package import SKOptions
+import LanguageServerProtocolJSONRPC
 import SKUtilities
 import SourceKitD
-package import SourceKitLSP
 import SwiftExtensions
 import SwiftSyntax
-import ToolchainRegistry
 import XCTest
+
+#if compiler(>=6)
+package import LanguageServerProtocol
+package import SKOptions
+package import SourceKitLSP
+package import ToolchainRegistry
 #else
-import Foundation
-import InProcessClient
 import LanguageServerProtocol
-import LanguageServerProtocolJSONRPC
-import LanguageServerProtocolExtensions
 import SKOptions
-import SKUtilities
-import SourceKitD
 import SourceKitLSP
-import SwiftExtensions
-import SwiftSyntax
 import ToolchainRegistry
-import XCTest
 #endif
 
 extension SourceKitLSPOptions {
@@ -150,6 +142,7 @@ package final class TestSourceKitLSPClient: MessageHandler, Sendable {
     initialize: Bool = true,
     initializationOptions: LSPAny? = nil,
     capabilities: ClientCapabilities = ClientCapabilities(),
+    toolchainRegistry: ToolchainRegistry = .forTesting,
     usePullDiagnostics: Bool = true,
     enableBackgroundIndexing: Bool = false,
     workspaceFolders: [WorkspaceFolder]? = nil,
@@ -175,7 +168,7 @@ package final class TestSourceKitLSPClient: MessageHandler, Sendable {
     self.serverToClientConnection = serverToClientConnection
     server = SourceKitLSPServer(
       client: serverToClientConnection,
-      toolchainRegistry: ToolchainRegistry.forTesting,
+      toolchainRegistry: toolchainRegistry,
       options: options,
       hooks: hooks,
       onExit: {