Merge branch 'main' into match-scalar

Author: rctcwyvrn

Sources/_RegexParser/Regex/AST/CustomCharClass.swift

@@ -62,6 +62,10 @@ extension AST {
         self.rhs = rhs
         self.trivia = trivia
       }
+
+      public var location: SourceLocation {
+        lhs.location.union(with: rhs.location)
+      }
     }
     public enum SetOp: String, Hashable {
       case subtraction = "--"
@@ -108,6 +112,25 @@ extension CustomCC.Member {
   public var isSemantic: Bool {
     !isTrivia
   }
+
+  public var location: SourceLocation {
+    switch self {
+    case let .custom(c): return c.location
+    case let .range(r):  return r.location
+    case let .atom(a):   return a.location
+    case let .quote(q):  return q.location
+    case let .trivia(t): return t.location
+    case let .setOperation(lhs, dash, rhs):
+      var loc = dash.location
+      if let lhs = lhs.first {
+        loc = loc.union(with: lhs.location)
+      }
+      if let rhs = rhs.last {
+        loc = loc.union(with: rhs.location)
+      }
+      return loc
+    }
+  }
 }
 
 extension AST.CustomCharacterClass {

Sources/_RegexParser/Regex/Parse/Diagnostics.swift

@@ -94,6 +94,7 @@ enum ParseError: Error, Hashable {
   case invalidNamedReference(String)
   case duplicateNamedCapture(String)
   case invalidCharacterClassRangeOperand
+  case unsupportedDotNetSubtraction
   case invalidQuantifierRange(Int, Int)
   case invalidCharacterRange(from: Character, to: Character)
   case notQuantifiable
@@ -174,7 +175,9 @@ extension ParseError: CustomStringConvertible {
     case .expectedCustomCharacterClassMembers:
       return "expected custom character class members"
     case .invalidCharacterClassRangeOperand:
-      return "invalid character class range"
+      return "invalid bound for character class range"
+    case .unsupportedDotNetSubtraction:
+      return "subtraction with '-' is unsupported; use '--' instead"
     case .emptyProperty:
       return "empty property"
     case .unknownProperty(let key, let value):

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -1245,6 +1245,25 @@ extension Source {
     return nil
   }
 
+  /// Check to see if we can lex a .NET subtraction. Returns the
+  /// location of the `-`.
+  ///
+  ///     DotNetSubtraction -> Trivia* '-' Trivia* CustomCharClass
+  ///
+  func canLexDotNetCharClassSubtraction(
+    context: ParsingContext
+  ) -> SourceLocation? {
+    lookahead { src in
+      // We can lex '-' as a .NET subtraction if it precedes a custom character
+      // class.
+      while (try? src.lexTrivia(context: context)) != nil {}
+      guard let dashLoc = src.tryEatWithLoc("-") else { return nil }
+      while (try? src.lexTrivia(context: context)) != nil {}
+      guard src.lexCustomCCStart() != nil else { return nil }
+      return dashLoc
+    }
+  }
+
   private mutating func lexPOSIXCharacterProperty(
   ) throws -> Located<AST.Atom.CharacterProperty>? {
     try recordLoc { src in

Sources/_RegexParser/Regex/Parse/Parse.swift

@@ -502,6 +502,12 @@ extension Parser {
     var members: Array<Member> = []
     try parseCCCMembers(into: &members)
 
+    // Make sure we have at least one semantic member.
+    if members.none(\.isSemantic) {
+      throw Source.LocatedError(
+        ParseError.expectedCustomCharacterClassMembers, start.location)
+    }
+
     // If we have a binary set operator, parse it and the next members. Note
     // that this means we left associate for a chain of operators.
     // TODO: We may want to diagnose and require users to disambiguate, at least
@@ -511,16 +517,12 @@ extension Parser {
       var rhs: Array<Member> = []
       try parseCCCMembers(into: &rhs)
 
-      if members.none(\.isSemantic) || rhs.none(\.isSemantic) {
+      if rhs.none(\.isSemantic) {
         throw Source.LocatedError(
           ParseError.expectedCustomCharacterClassMembers, start.location)
       }
       members = [.setOperation(members, binOp, rhs)]
     }
-    if members.none(\.isSemantic) {
-      throw Source.LocatedError(
-        ParseError.expectedCustomCharacterClassMembers, start.location)
-    }
     try source.expect("]")
     return CustomCC(start, members, loc(start.location.start))
   }
@@ -550,48 +552,88 @@ extension Parser {
     return nil
   }
 
-  mutating func parseCCCMembers(
-    into members: inout Array<CustomCC.Member>
+  /// Attempt to parse a custom character class range into `members`, or regular
+  /// members if a range cannot be formed.
+  mutating func parsePotentialCCRange(
+    into members: inout [CustomCC.Member]
   ) throws {
-    // Parse members until we see the end of the custom char class or an
-    // operator.
-    while let member = try parseCCCMember() {
-      members.append(member)
-
-      // If we have an atom, we can try to parse a character class range. Each
-      // time we parse a component of the range, we append to `members` in case
-      // it ends up not being a range, and we bail. If we succeed in parsing, we
-      // remove the intermediate members.
-      if case .atom(let lhs) = member {
-        let membersBeforeRange = members.count - 1
-
-        while let t = try source.lexTrivia(context: context) {
-          members.append(.trivia(t))
-        }
+    guard let lhs = members.last, lhs.isSemantic else { return }
+
+    // Try and see if we can parse a character class range. Each time we parse
+    // a component of the range, we append to `members` in case it ends up not
+    // being a range, and we bail. If we succeed in parsing, we remove the
+    // intermediate members.
+    let membersBeforeRange = members.count - 1
+    while let t = try source.lexTrivia(context: context) {
+      members.append(.trivia(t))
+    }
+    guard let dash = source.lexCustomCharacterClassRangeOperator() else {
+      return
+    }
 
-        guard let dash = source.lexCustomCharacterClassRangeOperator() else {
-          continue
-        }
-        // If we can't parse a range, '-' becomes literal, e.g `[6-]`.
-        members.append(.atom(.init(.char("-"), dash)))
+    // If we can't parse a range, '-' becomes literal, e.g `[6-]`.
+    members.append(.atom(.init(.char("-"), dash)))
 
-        while let t = try source.lexTrivia(context: context) {
-          members.append(.trivia(t))
+    while let t = try source.lexTrivia(context: context) {
+      members.append(.trivia(t))
+    }
+    guard let rhs = try parseCCCMember() else { return }
+    members.append(rhs)
+
+    func makeOperand(_ m: CustomCC.Member, isLHS: Bool) throws -> AST.Atom {
+      switch m {
+      case .atom(let a):
+        return a
+      case .custom:
+        // Not supported. While .NET allows `x-[...]` to spell subtraction, we
+        // require `x--[...]`. We also ban `[...]-x` for consistency.
+        if isLHS {
+          throw Source.LocatedError(
+            ParseError.invalidCharacterClassRangeOperand, m.location)
+        } else {
+          throw Source.LocatedError(
+            ParseError.unsupportedDotNetSubtraction, m.location)
         }
-        guard let rhs = try parseCCCMember() else { continue }
-        members.append(rhs)
-
-        guard case let .atom(rhs) = rhs else { continue }
-
-        // We've successfully parsed an atom LHS and RHS, so form a range,
-        // collecting the trivia we've parsed, and replacing the members that
-        // would have otherwise been added to the custom character class.
-        let rangeMemberCount = members.count - membersBeforeRange
-        let trivia = members.suffix(rangeMemberCount).compactMap(\.asTrivia)
-        members.removeLast(rangeMemberCount)
-        members.append(.range(.init(lhs, dash, rhs, trivia: trivia)))
+      case .quote:
+        // Currently unsupported, we need to figure out what the semantics
+        // would be for grapheme/scalar modes.
+        throw Source.LocatedError(
+          ParseError.unsupported("range with quoted sequence"), m.location)
+      case .trivia:
+        throw Unreachable("Should have been lexed separately")
+      case .range, .setOperation:
+        throw Unreachable("Parsed later")
       }
     }
+    let lhsOp = try makeOperand(lhs, isLHS: true)
+    let rhsOp = try makeOperand(rhs, isLHS: false)
+
+    // We've successfully parsed an atom LHS and RHS, so form a range,
+    // collecting the trivia we've parsed, and replacing the members that
+    // would have otherwise been added to the custom character class.
+    let rangeMemberCount = members.count - membersBeforeRange
+    let trivia = members.suffix(rangeMemberCount).compactMap(\.asTrivia)
+    members.removeLast(rangeMemberCount)
+    members.append(.range(.init(lhsOp, dash, rhsOp, trivia: trivia)))
+
+    // We need to specially check if we can lex a .NET character class
+    // subtraction here as e.g `[a-c-[...]]` is allowed in .NET. Otherwise we'd
+    // treat the second `-` as literal.
+    if let dashLoc = source.canLexDotNetCharClassSubtraction(context: context) {
+      throw Source.LocatedError(
+        ParseError.unsupportedDotNetSubtraction, dashLoc)
+    }
+  }
+
+  mutating func parseCCCMembers(
+    into members: inout Array<CustomCC.Member>
+  ) throws {
+    // Parse members and ranges until we see the end of the custom char class
+    // or an operator.
+    while let member = try parseCCCMember() {
+      members.append(member)
+      try parsePotentialCCRange(into: &members)
+    }
   }
 }
 

Sources/_StringProcessing/ByteCodeGen.swift

@@ -8,8 +8,16 @@ extension Compiler {
     /// This is used to determine whether to apply initial options.
     var hasEmittedFirstMatchableAtom = false
 
-    init(options: MatchingOptions, captureList: CaptureList) {
+    private let compileOptions: CompileOptions
+    fileprivate var optimizationsEnabled: Bool { !compileOptions.contains(.disableOptimizations) }
+
+    init(
+      options: MatchingOptions,
+      compileOptions: CompileOptions,
+      captureList: CaptureList
+    ) {
       self.options = options
+      self.compileOptions = compileOptions
       self.builder.captureList = captureList
     }
   }
@@ -670,8 +678,16 @@ fileprivate extension Compiler.ByteCodeGen {
   mutating func emitCustomCharacterClass(
     _ ccc: DSLTree.CustomCharacterClass
   ) throws {
-    let consumer = try ccc.generateConsumer(options)
-    builder.buildConsume(by: consumer)
+    if let asciiBitset = ccc.asAsciiBitset(options),
+        options.semanticLevel == .graphemeCluster,
+        optimizationsEnabled {
+      // future work: add a bit to .matchBitset to consume either a character
+      // or a scalar so we can have this optimization in scalar mode
+      builder.buildMatchAsciiBitset(asciiBitset)
+    } else {
+      let consumer = try ccc.generateConsumer(options)
+      builder.buildConsume(by: consumer)
+    }
   }
 
   @discardableResult

Sources/_StringProcessing/Compiler.swift

@@ -16,6 +16,7 @@ class Compiler {
 
   // TODO: Or are these stored on the tree?
   var options = MatchingOptions()
+  private var compileOptions: CompileOptions = .default
 
   init(ast: AST) {
     self.tree = ast.dslTree
@@ -25,23 +26,22 @@ class Compiler {
     self.tree = tree
   }
 
+  init(tree: DSLTree, compileOptions: CompileOptions) {
+    self.tree = tree
+    self.compileOptions = compileOptions
+  }
+
   __consuming func emit() throws -> MEProgram {
     // TODO: Handle global options
     var codegen = ByteCodeGen(
-      options: options, captureList: tree.captureList
-    )
+      options: options,
+      compileOptions:
+        compileOptions,
+      captureList: tree.captureList)
     return try codegen.emitRoot(tree.root)
   }
 }
 
-func _compileRegex(
-  _ regex: String, _ syntax: SyntaxOptions = .traditional
-) throws -> Executor {
-  let ast = try parse(regex, .semantic, syntax)
-  let program = try Compiler(ast: ast).emit()
-  return Executor(program: program)
-}
-
 // An error produced when compiling a regular expression.
 enum RegexCompilationError: Error, CustomStringConvertible {
   // TODO: Source location?
@@ -54,3 +54,35 @@ enum RegexCompilationError: Error, CustomStringConvertible {
     }
   }
 }
+
+// Testing support
+@available(SwiftStdlib 5.7, *)
+func _compileRegex(
+  _ regex: String,
+  _ syntax: SyntaxOptions = .traditional,
+  _ semanticLevel: RegexSemanticLevel? = nil
+) throws -> Executor {
+  let ast = try parse(regex, .semantic, syntax)
+  let dsl: DSLTree
+
+  switch semanticLevel?.base {
+  case .graphemeCluster:
+    let sequence = AST.MatchingOptionSequence(adding: [.init(.graphemeClusterSemantics, location: .fake)])
+    dsl = DSLTree(.nonCapturingGroup(.init(ast: .changeMatchingOptions(sequence)), ast.dslTree.root))
+  case .unicodeScalar:
+    let sequence = AST.MatchingOptionSequence(adding: [.init(.unicodeScalarSemantics, location: .fake)])
+    dsl = DSLTree(.nonCapturingGroup(.init(ast: .changeMatchingOptions(sequence)), ast.dslTree.root))
+  case .none:
+    dsl = ast.dslTree
+  }
+  let program = try Compiler(tree: dsl).emit()
+  return Executor(program: program)
+}
+
+extension Compiler {
+  struct CompileOptions: OptionSet {
+    let rawValue: Int
+    static let disableOptimizations = CompileOptions(rawValue: 1)
+    static let `default`: CompileOptions = []
+  }
+}