Merge main into release/6.1

.gitignore

@@ -1,7 +1,7 @@
 .DS_Store
 default.profraw
 Package.resolved
-/.build
+.build
 /.*-build
 /Packages
 /*.xcodeproj

CMakeLists.txt

@@ -17,6 +17,7 @@ add_compile_options("$<$<COMPILE_LANGUAGE:Swift>:SHELL:-enable-upcoming-feature
 find_package(dispatch QUIET)
 find_package(Foundation QUIET)
 find_package(TSC QUIET)
+find_package(LMDB QUIET)
 find_package(IndexStoreDB QUIET)
 find_package(SwiftPM QUIET)
 find_package(LLBuild QUIET)

CONTRIBUTING.md

@@ -115,6 +115,14 @@ swift package format-source-code
 
 If you are developing SourceKit-LSP in VS Code, you can also run the *Run swift-format* task from *Tasks: Run tasks* in the command palette.
 
+## Generate configuration schema
+
+If you modify the configuration options in [`SKOptions`](./Sources/SKOptions), you need to regenerate the configuration the JSON schema and the documentation by running the following command:
+
+```bash
+./sourcekit-lsp-dev-utils generate-config-schema
+```
+
 ## Authoring commits
 
 Prefer to squash the commits of your PR (*pull request*) and avoid adding commits like “Address review comments”. This creates a clearer git history, which doesn’t need to record the history of how the PR evolved.

Documentation/Configuration File.md

@@ -1,3 +1,5 @@
+<!-- DO NOT EDIT THIS FILE. This file is generated by ConfigSchemaGen/OptionDocument.swift. -->
+
 # Configuration File
 
 `.sourcekit-lsp/config.json` configuration files can be used to modify the behavior of SourceKit-LSP in various ways. The following locations are checked. Settings in later configuration files override settings in earlier configuration files
@@ -12,44 +14,46 @@ The structure of the file is currently not guaranteed to be stable. Options may
 ## Structure
 
 `config.json` is a JSON file with the following structure. All keys are optional and unknown keys are ignored.
-
-- `swiftPM`: Dictionary with the following keys, defining options for SwiftPM workspaces
+- `swiftPM`: Options for SwiftPM workspaces.
   - `configuration: "debug"|"release"`: The configuration to build the project for during background indexing and the configuration whose build folder should be used for Swift modules if background indexing is disabled. Equivalent to SwiftPM's `--configuration` option.
   - `scratchPath: string`: Build artifacts directory path. If nil, the build system may choose a default value. This path can be specified as a relative path, which will be interpreted relative to the project root. Equivalent to SwiftPM's `--scratch-path` option.
-  - `swiftSDKsDirectory: string`: Equivalent to SwiftPM's `--swift-sdks-path` option
-  - `swiftSDK: string`: Equivalent to SwiftPM's `--swift-sdk` option
-  - `triple: string`: Equivalent to SwiftPM's `--triple` option
+  - `swiftSDKsDirectory: string`: Equivalent to SwiftPM's `--swift-sdks-path` option.
+  - `swiftSDK: string`: Equivalent to SwiftPM's `--swift-sdk` option.
+  - `triple: string`: Equivalent to SwiftPM's `--triple` option.
   - `cCompilerFlags: string[]`: Extra arguments passed to the compiler for C files. Equivalent to SwiftPM's `-Xcc` option.
   - `cxxCompilerFlags: string[]`: Extra arguments passed to the compiler for C++ files. Equivalent to SwiftPM's `-Xcxx` option.
   - `swiftCompilerFlags: string[]`: Extra arguments passed to the compiler for Swift files. Equivalent to SwiftPM's `-Xswiftc` option.
   - `linkerFlags: string[]`: Extra arguments passed to the linker. Equivalent to SwiftPM's `-Xlinker` option.
-  - `disableSandbox: bool`: Disables running subprocesses from SwiftPM in a sandbox. Useful when running `sourcekit-lsp` in a sandbox because nested sandboxes are not supported.
-- `compilationDatabase`: Dictionary with the following keys, defining options for workspaces with a compilation database
+  - `disableSandbox: boolean`: Disables running subprocesses from SwiftPM in a sandbox. Equivalent to SwiftPM's `--disable-sandbox` option. Useful when running `sourcekit-lsp` in a sandbox because nested sandboxes are not supported.
+- `compilationDatabase`: Dictionary with the following keys, defining options for workspaces with a compilation database.
   - `searchPaths: string[]`: Additional paths to search for a compilation database, relative to a workspace root.
-- `fallbackBuildSystem`: Dictionary with the following keys, defining options for files that aren't managed by any build system
-  - `cCompilerFlags: string[]`: Extra arguments passed to the compiler for C files
-  - `cxxCompilerFlags: string[]`: Extra arguments passed to the compiler for C++ files
-  - `swiftCompilerFlags: string[]`: Extra arguments passed to the compiler for Swift files
+- `fallbackBuildSystem`: Dictionary with the following keys, defining options for files that aren't managed by any build system.
+  - `cCompilerFlags: string[]`: Extra arguments passed to the compiler for C files.
+  - `cxxCompilerFlags: string[]`: Extra arguments passed to the compiler for C++ files.
+  - `swiftCompilerFlags: string[]`: Extra arguments passed to the compiler for Swift files.
   - `sdk: string`: The SDK to use for fallback arguments. Default is to infer the SDK using `xcrun`.
-- `buildSettingsTimeout: int`: Number of milliseconds to wait for build settings from the build system before using fallback build settings.
-- `clangdOptions: string[]`: Extra command line arguments passed to `clangd` when launching it
-- `index`: Dictionary with the following keys, defining options related to indexing
+- `buildSettingsTimeout: integer`: Number of milliseconds to wait for build settings from the build system before using fallback build settings.
+- `clangdOptions: string[]`: Extra command line arguments passed to `clangd` when launching it.
+- `index`: Options related to indexing.
   - `indexStorePath: string`: Directory in which a separate compilation stores the index store. By default, inferred from the build system.
   - `indexDatabasePath: string`: Directory in which the indexstore-db should be stored. By default, inferred from the build system.
   - `indexPrefixMap: [string: string]`: Path remappings for remapping index data for local use.
-  - `maxCoresPercentageToUseForBackgroundIndexing: double`: A hint indicating how many cores background indexing should use at most (value between 0 and 1). Background indexing is not required to honor this setting
-  - `updateIndexStoreTimeout: int`: Number of seconds to wait for an update index store task to finish before killing it.
-- `logging`: Dictionary with the following keys, changing SourceKit-LSP’s logging behavior on non-Apple platforms. On Apple platforms, logging is done through the [system log](Diagnose%20Bundle.md#Enable%20Extended%20Logging). These options can only be set globally and not per workspace.
-  - `logLevel: "debug"|"info"|"default"|"error"|"fault"`: The level from which one onwards log messages should be written.
+  - `maxCoresPercentageToUseForBackgroundIndexing: number`: A hint indicating how many cores background indexing should use at most (value between 0 and 1). Background indexing is not required to honor this setting.
+  - `updateIndexStoreTimeout: integer`: Number of seconds to wait for an update index store task to finish before killing it.
+- `logging`: Options related to logging, changing SourceKit-LSP’s logging behavior on non-Apple platforms. On Apple platforms, logging is done through the [system log](Diagnose%20Bundle.md#Enable%20Extended%20Logging). These options can only be set globally and not per workspace.
+  - `level: "debug"|"info"|"default"|"error"|"fault"`: The level from which one onwards log messages should be written.
   - `privacyLevel: "public"|"private"|"sensitive"`: Whether potentially sensitive information should be redacted. Default is `public`, which redacts potentially sensitive information.
   - `inputMirrorDirectory: string`: Write all input received by SourceKit-LSP on stdin to a file in this directory. Useful to record and replay an entire SourceKit-LSP session.
-- `defaultWorkspaceType: "buildserver"|"compdb"|"swiftpm"`: Overrides workspace type selection logic.
+  - `outputMirrorDirectory: string`: Write all data sent from SourceKit-LSP to the client to a file in this directory. Useful to record the raw communication between SourceKit-LSP and the client on a low level.
+- `defaultWorkspaceType: "buildServer"|"compilationDatabase"|"swiftPM"`: Default workspace type. Overrides workspace type selection logic.
 - `generatedFilesPath: string`: Directory in which generated interfaces and macro expansions should be stored.
-- `backgroundIndexing: bool`: Explicitly enable or disable background indexing.
-- `backgroundPreparationMode: "build"|"noLazy"|"enabled"`: Determines how background indexing should prepare a target. Possible values are:
-  - `build`: Build a target to prepare it
-  - `noLazy`: Prepare a target without generating object files but do not do lazy type checking and function body skipping
-  - `enabled`: Prepare a target without generating object files and the like
-- `cancelTextDocumentRequestsOnEditAndClose: bool`: Whether sending a `textDocument/didChange` or `textDocument/didClose` notification for a document should cancel all pending requests for that document.
-- `experimentalFeatures: string[]`: Experimental features to enable. Available features: on-type-formatting
-- `swiftPublishDiagnosticsDebounceDuration: double`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.
+- `backgroundIndexing: boolean`: Whether background indexing is enabled.
+- `backgroundPreparationMode: "build"|"noLazy"|"enabled"`: Determines how background indexing should prepare a target.
+  - `build`: Build a target to prepare it.
+  - `noLazy`: Prepare a target without generating object files but do not do lazy type checking and function body skipping. This uses SwiftPM's `--experimental-prepare-for-indexing-no-lazy` flag.
+  - `enabled`: Prepare a target without generating object files.
+- `cancelTextDocumentRequestsOnEditAndClose: boolean`: Whether sending a `textDocument/didChange` or `textDocument/didClose` notification for a document should cancel all pending requests for that document.
+- `experimentalFeatures: ("on-type-formatting")[]`: Experimental features that are enabled.
+- `swiftPublishDiagnosticsDebounceDuration: number`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.
+- `workDoneProgressDebounceDuration: number`: When a task is started that should be displayed to the client as a work done progress, how many milliseconds to wait before actually starting the work done progress. This prevents flickering of the work done progress in the client for short-lived index tasks which end within this duration.
+- `sourcekitdRequestTimeout: number`: The maximum duration that a sourcekitd request should be allowed to execute before being declared as timed out. In general, editors should cancel requests that they are no longer interested in, but in case editors don't cancel requests, this ensures that a long-running non-cancelled request is not blocking sourcekitd and thus most semantic functionality. In particular, VS Code does not cancel the semantic tokens request, which can cause a long-running AST build that blocks sourcekitd.

SourceKitLSPDevUtils/Package.swift

@@ -0,0 +1,31 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+let package = Package(
+  name: "SourceKitLSPDevUtils",
+  platforms: [.macOS(.v10_15)],
+  products: [
+    .executable(name: "sourcekit-lsp-dev-utils", targets: ["SourceKitLSPDevUtils"])
+  ],
+  dependencies: [
+    .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "600.0.1"),
+    .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.5.0"),
+  ],
+  targets: [
+    .executableTarget(
+      name: "SourceKitLSPDevUtils",
+      dependencies: [
+        "ConfigSchemaGen",
+        .product(name: "ArgumentParser", package: "swift-argument-parser"),
+      ]
+    ),
+    .target(
+      name: "ConfigSchemaGen",
+      dependencies: [
+        .product(name: "SwiftSyntax", package: "swift-syntax"),
+        .product(name: "SwiftParser", package: "swift-syntax"),
+      ]
+    ),
+  ]
+)

SourceKitLSPDevUtils/README.md

@@ -0,0 +1,3 @@
+# sourcekit-lsp-dev-utils
+
+This directory contains utilities for developing SourceKit-LSP. [CONTRIBUTING.md](../CONTRIBUTING.md) covers how to use these utilities.

SourceKitLSPDevUtils/Sources/ConfigSchemaGen/ConfigSchemaGen.swift

@@ -0,0 +1,151 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import SwiftParser
+import SwiftSyntax
+
+/// The main entry point for generating a JSON schema and Markdown documentation
+/// for the SourceKit-LSP configuration file format
+/// (`.sourcekit-lsp/config.json`) from the Swift type definitions in
+/// `SKOptions` Swift module.
+package struct ConfigSchemaGen {
+  private struct WritePlan {
+    fileprivate let category: String
+    fileprivate let path: URL
+    fileprivate let contents: () throws -> Data
+
+    fileprivate func write() throws {
+      try contents().write(to: path)
+    }
+  }
+
+  private static let projectRoot = URL(fileURLWithPath: #filePath)
+    .deletingLastPathComponent()
+    .deletingLastPathComponent()
+    .deletingLastPathComponent()
+    .deletingLastPathComponent()
+  private static let sourceDir =
+    projectRoot
+    .appendingPathComponent("Sources")
+    .appendingPathComponent("SKOptions")
+  private static let configSchemaJSONPath =
+    projectRoot
+    .appendingPathComponent("config.schema.json")
+  private static let configSchemaDocPath =
+    projectRoot
+    .appendingPathComponent("Documentation")
+    .appendingPathComponent("Configuration File.md")
+
+  /// Generates and writes the JSON schema and documentation for the SourceKit-LSP configuration file format.
+  package static func generate() throws {
+    let plans = try plan()
+    for plan in plans {
+      print("Writing \(plan.category) to \"\(plan.path.path)\"")
+      try plan.write()
+    }
+  }
+
+  /// Verifies that the generated JSON schema and documentation in the current source tree
+  /// are up-to-date with the Swift type definitions in `SKOptions`.
+  /// - Returns: `true` if the generated files are up-to-date, `false` otherwise.
+  package static func verify() throws -> Bool {
+    let plans = try plan()
+    for plan in plans {
+      print("Verifying \(plan.category) at \"\(plan.path.path)\"")
+      let expectedContents = try plan.contents()
+      let actualContents = try Data(contentsOf: plan.path)
+      guard expectedContents == actualContents else {
+        print("error: \(plan.category) is out-of-date!")
+        print("Please run `./sourcekit-lsp-dev-utils generate-config-schema` to update it.")
+        return false
+      }
+    }
+    return true
+  }
+
+  private static func plan() throws -> [WritePlan] {
+    let sourceFiles = FileManager.default.enumerator(at: sourceDir, includingPropertiesForKeys: nil)!
+    let typeNameResolver = TypeDeclResolver()
+
+    for case let fileURL as URL in sourceFiles {
+      guard fileURL.pathExtension == "swift" else {
+        continue
+      }
+      let sourceText = try String(contentsOf: fileURL)
+      let sourceFile = Parser.parse(source: sourceText)
+      typeNameResolver.collect(from: sourceFile)
+    }
+    let rootTypeDecl = try typeNameResolver.lookupType(fullyQualified: ["SourceKitLSPOptions"])
+    let context = OptionSchemaContext(typeNameResolver: typeNameResolver)
+    var schema = try context.buildSchema(from: rootTypeDecl)
+
+    // Manually annotate the logging level enum since LogLevel type exists
+    // outside of the SKOptions module
+    schema["logging"]?["level"]?.kind = .enum(
+      OptionTypeSchama.Enum(
+        name: "LogLevel",
+        cases: ["debug", "info", "default", "error", "fault"].map {
+          OptionTypeSchama.Case(name: $0)
+        }
+      )
+    )
+    schema["logging"]?["privacyLevel"]?.kind = .enum(
+      OptionTypeSchama.Enum(
+        name: "PrivacyLevel",
+        cases: ["public", "private", "sensitive"].map {
+          OptionTypeSchama.Case(name: $0)
+        }
+      )
+    )
+
+    return [
+      WritePlan(
+        category: "JSON Schema",
+        path: configSchemaJSONPath,
+        contents: { try generateJSONSchema(from: schema, context: context) }
+      ),
+      WritePlan(
+        category: "Schema Documentation",
+        path: configSchemaDocPath,
+        contents: { try generateDocumentation(from: schema, context: context) }
+      ),
+    ]
+  }
+
+  private static func generateJSONSchema(from schema: OptionTypeSchama, context: OptionSchemaContext) throws -> Data {
+    let schemaBuilder = JSONSchemaBuilder(context: context)
+    var jsonSchema = try schemaBuilder.build(from: schema)
+    jsonSchema.title = "SourceKit-LSP Configuration"
+    jsonSchema.comment = "DO NOT EDIT THIS FILE. This file is generated by \(#fileID)."
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+    return try encoder.encode(jsonSchema)
+  }
+
+  private static func generateDocumentation(from schema: OptionTypeSchama, context: OptionSchemaContext) throws -> Data
+  {
+    let docBuilder = OptionDocumentBuilder(context: context)
+    guard let data = try docBuilder.build(from: schema).data(using: .utf8) else {
+      throw ConfigSchemaGenError("Failed to encode documentation as UTF-8")
+    }
+    return data
+  }
+}
+
+struct ConfigSchemaGenError: Error, CustomStringConvertible {
+  let description: String
+
+  init(_ description: String) {
+    self.description = description
+  }
+}