Merge pull request swiftlang#409 from natecook1000/main_as_of_12_may

Sync 5.7 branch with main

Author: natecook1000

Package.swift

@@ -7,10 +7,18 @@ let availabilityDefinition = PackageDescription.SwiftSetting.unsafeFlags([
     "-Xfrontend",
     "-define-availability",
     "-Xfrontend",
-    #"SwiftStdlib 5.7:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999"#,
+    "SwiftStdlib 5.7:macOS 9999, iOS 9999, watchOS 9999, tvOS 9999",
 ])
 
-let stdlibSettings: [PackageDescription.SwiftSetting] = [
+/// Swift settings for building a private stdlib-like module that is to be used
+/// by other stdlib-like modules only.
+let privateStdlibSettings: [PackageDescription.SwiftSetting] = [
+    .unsafeFlags(["-Xfrontend", "-disable-implicit-concurrency-module-import"]),
+    .unsafeFlags(["-Xfrontend", "-disable-implicit-string-processing-module-import"]),
+]
+
+/// Swift settings for building a user-facing stdlib-like module.
+let publicStdlibSettings: [PackageDescription.SwiftSetting] = [
     .unsafeFlags(["-enable-library-evolution"]),
     .unsafeFlags(["-Xfrontend", "-disable-implicit-concurrency-module-import"]),
     .unsafeFlags(["-Xfrontend", "-disable-implicit-string-processing-module-import"]),
@@ -43,7 +51,7 @@ let package = Package(
         .target(
             name: "_RegexParser",
             dependencies: [],
-            swiftSettings: stdlibSettings),
+            swiftSettings: privateStdlibSettings),
         .testTarget(
             name: "MatchingEngineTests",
             dependencies: [
@@ -55,16 +63,16 @@ let package = Package(
         .target(
             name: "_StringProcessing",
             dependencies: ["_RegexParser", "_CUnicode"],
-            swiftSettings: stdlibSettings),
+            swiftSettings: publicStdlibSettings),
         .target(
             name: "RegexBuilder",
             dependencies: ["_StringProcessing", "_RegexParser"],
-            swiftSettings: stdlibSettings),
+            swiftSettings: publicStdlibSettings),
         .testTarget(
             name: "RegexTests",
             dependencies: ["_StringProcessing"],
             swiftSettings: [
-                .unsafeFlags(["-Xfrontend", "-disable-availability-checking"])
+                .unsafeFlags(["-Xfrontend", "-disable-availability-checking"]),
             ]),
         .testTarget(
             name: "RegexBuilderTests",

Sources/PatternConverter/PatternConverter.swift

@@ -50,7 +50,7 @@ struct PatternConverter: ParsableCommand {
     print("Converting '\(delim)\(regex)\(delim)'")
 
     let ast = try _RegexParser.parse(
-      regex,
+      regex, .semantic,
       experimentalSyntax ? .experimental : .traditional)
 
     // Show rendered source ranges

Sources/_RegexParser/Regex/AST/AST.swift

@@ -29,7 +29,6 @@ extension AST {
 
 extension AST {
   /// A node in the regex AST.
-  @frozen
   public indirect enum Node:
     Hashable, _TreeNode //, _ASTPrintable ASTValue, ASTAction
   {
@@ -125,7 +124,9 @@ extension AST.Node {
     switch self {
     case .atom(let a):
       return a.isQuantifiable
-    case .group, .conditional, .customCharacterClass, .absentFunction:
+    case .group(let g):
+      return g.isQuantifiable
+    case .conditional, .customCharacterClass, .absentFunction:
       return true
     case .alternation, .concatenation, .quantification, .quote, .trivia,
         .empty:
@@ -247,7 +248,6 @@ extension AST {
   }
 
   public struct Reference: Hashable {
-    @frozen
     public enum Kind: Hashable {
       // \n \gn \g{n} \g<n> \g'n' (?n) (?(n)...
       // Oniguruma: \k<n>, \k'n'

Sources/_RegexParser/Regex/AST/Atom.swift

@@ -19,7 +19,6 @@ extension AST {
       self.location = loc
     }
 
-    @frozen
     public enum Kind: Hashable {
       /// Just a character
       ///
@@ -29,7 +28,13 @@ extension AST {
       /// A Unicode scalar value written as a literal
       ///
       /// \u{...}, \0dd, \x{...}, ...
-      case scalar(Unicode.Scalar)
+      case scalar(Scalar)
+
+      /// A whitespace-separated sequence of Unicode scalar values which are
+      /// implicitly splatted out.
+      ///
+      /// `\u{A B C}` -> `\u{A}\u{B}\u{C}`
+      case scalarSequence(ScalarSequence)
 
       /// A Unicode property, category, or script, including those written using
       /// POSIX syntax.
@@ -84,6 +89,7 @@ extension AST.Atom {
     switch kind {
     case .char(let v):                  return v
     case .scalar(let v):                return v
+    case .scalarSequence(let v):        return v
     case .property(let v):              return v
     case .escaped(let v):               return v
     case .keyboardControl(let v):       return v
@@ -106,6 +112,30 @@ extension AST.Atom {
   }
 }
 
+extension AST.Atom {
+  public struct Scalar: Hashable {
+    public var value: UnicodeScalar
+    public var location: SourceLocation
+
+    public init(_ value: UnicodeScalar, _ location: SourceLocation) {
+      self.value = value
+      self.location = location
+    }
+  }
+
+  public struct ScalarSequence: Hashable {
+    public var scalars: [Scalar]
+    public var trivia: [AST.Trivia]
+
+    public init(_ scalars: [Scalar], trivia: [AST.Trivia]) {
+      precondition(scalars.count > 1, "Expected multiple scalars")
+      self.scalars = scalars
+      self.trivia = trivia
+    }
+    public var scalarValues: [Unicode.Scalar] { scalars.map(\.value) }
+  }
+}
+
 extension AST.Atom {
 
   // TODO: We might scrap this and break out a few categories so
@@ -115,7 +145,6 @@ extension AST.Atom {
 
   // Characters, character types, literals, etc., derived from
   // an escape sequence.
-  @frozen
   public enum EscapedBuiltin: Hashable {
     // TODO: better doc comments
 
@@ -368,7 +397,6 @@ extension AST.Atom {
 }
 
 extension AST.Atom.CharacterProperty {
-  @frozen
   public enum Kind: Hashable {
     /// Matches any character, equivalent to Oniguruma's '\O'.
     case any
@@ -396,6 +424,9 @@ extension AST.Atom.CharacterProperty {
     case script(Unicode.Script)
     case scriptExtension(Unicode.Script)
 
+    /// Character name in the form `\p{name=...}`
+    case named(String)
+    
     case posix(Unicode.POSIXProperty)
 
     /// Some special properties implemented by PCRE and Oniguruma.
@@ -404,7 +435,6 @@ extension AST.Atom.CharacterProperty {
   }
 
   // TODO: erm, separate out or fold into something? splat it in?
-  @frozen
   public enum PCRESpecialCategory: String, Hashable {
     case alphanumeric     = "Xan"
     case posixSpace       = "Xps"
@@ -416,7 +446,6 @@ extension AST.Atom.CharacterProperty {
 
 extension AST.Atom {
   /// Anchors and other built-in zero-width assertions.
-  @frozen
   public enum AssertionKind: String {
     /// \A
     case startOfSubject = #"\A"#
@@ -665,6 +694,23 @@ extension AST.Atom.EscapedBuiltin {
       return nil
     }
   }
+
+  public var isQuantifiable: Bool {
+    switch self {
+    case .alarm, .escape, .formfeed, .newline, .carriageReturn, .tab,
+        .singleDataUnit, .decimalDigit, .notDecimalDigit, .horizontalWhitespace,
+        .notHorizontalWhitespace, .notNewline, .newlineSequence, .whitespace,
+        .notWhitespace, .verticalTab, .notVerticalTab, .wordCharacter,
+        .notWordCharacter, .backspace, .graphemeCluster, .trueAnychar:
+      return true
+
+    case .wordBoundary, .notWordBoundary, .startOfSubject,
+        .endOfSubjectBeforeNewline, .endOfSubject,
+        .firstMatchingPositionInSubject, .resetStartOfMatch, .textSegment,
+        .notTextSegment:
+      return false
+    }
+  }
 }
 
 extension AST.Atom {
@@ -677,7 +723,7 @@ extension AST.Atom {
     case .char(let c):
       return c
     case .scalar(let s):
-      return Character(s)
+      return Character(s.value)
 
     case .escaped(let c):
       return c.scalarValue.map(Character.init)
@@ -693,8 +739,9 @@ extension AST.Atom {
       // the AST? Or defer for the matching engine?
       return nil
 
-    case .property, .any, .startOfLine, .endOfLine, .backreference, .subpattern,
-        .callout, .backtrackingDirective, .changeMatchingOptions:
+    case .scalarSequence, .property, .any, .startOfLine, .endOfLine,
+        .backreference, .subpattern, .callout, .backtrackingDirective,
+        .changeMatchingOptions:
       return nil
     }
   }
@@ -716,13 +763,21 @@ extension AST.Atom {
   /// A string literal representation of the atom, if possible.
   ///
   /// Individual characters are returned as-is, and Unicode scalars are
-  /// presented using "\u{nnnn}" syntax.
+  /// presented using "\u{nn nn ...}" syntax.
   public var literalStringValue: String? {
+    func scalarLiteral(_ u: [UnicodeScalar]) -> String {
+      let digits = u.map { String($0.value, radix: 16, uppercase: true) }
+        .joined(separator: " ")
+      return "\\u{\(digits)}"
+    }
     switch kind {
     case .char(let c):
       return String(c)
     case .scalar(let s):
-      return "\\u{\(String(s.value, radix: 16, uppercase: true))}"
+      return scalarLiteral([s.value])
+
+    case .scalarSequence(let s):
+      return scalarLiteral(s.scalarValues)
 
     case .keyboardControl(let x):
       return "\\C-\(x)"
@@ -746,6 +801,10 @@ extension AST.Atom {
     case .changeMatchingOptions:
       return false
     // TODO: Are callouts quantifiable?
+    case .escaped(let esc):
+      return esc.isQuantifiable
+    case .startOfLine, .endOfLine:
+      return false
     default:
       return true
     }

Sources/_RegexParser/Regex/AST/CustomCharClass.swift

@@ -27,7 +27,6 @@ extension AST {
       self.location = sr
     }
 
-    @frozen
     public enum Member: Hashable {
       /// A nested custom character class `[[ab][cd]]`
       case custom(CustomCharacterClass)
@@ -59,13 +58,11 @@ extension AST {
         self.rhs = rhs
       }
     }
-    @frozen
     public enum SetOp: String, Hashable {
       case subtraction = "--"
       case intersection = "&&"
       case symmetricDifference = "~~"
     }
-    @frozen
     public enum Start: String {
       case normal = "["
       case inverted = "[^"

Sources/_RegexParser/Regex/AST/Group.swift

@@ -136,3 +136,18 @@ extension AST.Group {
     }
   }
 }
+
+extension AST.Group {
+  var isQuantifiable: Bool {
+    switch kind.value {
+    case .capture, .namedCapture, .balancedCapture, .nonCapture,
+        .nonCaptureReset, .atomicNonCapturing, .scriptRun, .atomicScriptRun,
+        .changeMatchingOptions:
+      return true
+      
+    case .lookahead, .negativeLookahead, .nonAtomicLookahead,
+        .lookbehind, .negativeLookbehind, .nonAtomicLookbehind:
+      return false
+    }
+  }
+}

Sources/_RegexParser/Regex/AST/Quantification.swift

@@ -36,7 +36,6 @@ extension AST {
       self.trivia = trivia
     }
 
-    @frozen
     public enum Amount: Hashable {
       case zeroOrMore              // *
       case oneOrMore               // +
@@ -47,7 +46,6 @@ extension AST {
       case range(Located<Int>, Located<Int>) // {n,m}
     }
 
-    @frozen
     public enum Kind: String, Hashable {
       case eager      = ""
       case reluctant  = "?"

Sources/_RegexParser/Regex/Parse/CaptureList.swift

@@ -26,15 +26,18 @@ extension CaptureList {
     public var name: String?
     public var type: Any.Type?
     public var optionalDepth: Int
+    public var location: SourceLocation
 
     public init(
       name: String? = nil,
       type: Any.Type? = nil,
-      optionalDepth: Int
+      optionalDepth: Int,
+      _ location: SourceLocation
     ) {
       self.name = name
       self.type = type
       self.optionalDepth = optionalDepth
+      self.location = location
     }
   }
 }
@@ -61,13 +64,14 @@ extension AST.Node {
     case let .group(g):
       switch g.kind.value {
       case .capture:
-        list.append(.init(optionalDepth: nesting))
+        list.append(.init(optionalDepth: nesting, g.location))
 
       case .namedCapture(let name):
-        list.append(.init(name: name.value, optionalDepth: nesting))
+        list.append(.init(name: name.value, optionalDepth: nesting, g.location))
 
       case .balancedCapture(let b):
-        list.append(.init(name: b.name?.value, optionalDepth: nesting))
+        list.append(.init(name: b.name?.value, optionalDepth: nesting,
+                          g.location))
 
       default: break
       }
@@ -124,7 +128,8 @@ extension CaptureList.Capture: Equatable {
   public static func == (lhs: Self, rhs: Self) -> Bool {
     lhs.name == rhs.name &&
     lhs.optionalDepth == rhs.optionalDepth &&
-    lhs.type == rhs.type
+    lhs.type == rhs.type &&
+    lhs.location == rhs.location
   }
 }
 extension CaptureList: Equatable {}

Sources/_RegexParser/Regex/Parse/CharacterPropertyClassification.swift

@@ -18,7 +18,7 @@ extension Source {
     // This follows the rules provided by UAX44-LM3, including trying to drop an
     // "is" prefix, which isn't required by UTS#18 RL1.2, but is nice for
     // consistency with other engines and the Unicode.Scalar.Properties names.
-    let str = str.filter { !$0.isWhitespace && $0 != "_" && $0 != "-" }
+    let str = str.filter { !$0.isPatternWhitespace && $0 != "_" && $0 != "-" }
                  .lowercased()
     if let m = match(str) {
       return m
@@ -428,6 +428,8 @@ extension Source {
         if let cat = classifyGeneralCategory(value) {
           return .generalCategory(cat)
         }
+      case "name", "na":
+        return .named(value)
       default:
         break
       }