Use a bitset for ascii-only character classes

Sources/_StringProcessing/ByteCodeGen.swift

@@ -3,13 +3,19 @@
 extension Compiler {
   struct ByteCodeGen {
     var options: MatchingOptions
+    private let compileOptions: CompileOptions
     var builder = MEProgram.Builder()
     /// A Boolean indicating whether the first matchable atom has been emitted.
     /// This is used to determine whether to apply initial options.
     var hasEmittedFirstMatchableAtom = false
 
-    init(options: MatchingOptions, captureList: CaptureList) {
+    init(
+      options: MatchingOptions,
+      compileOptions: CompileOptions,
+      captureList: CaptureList
+    ) {
       self.options = options
+      self.compileOptions = compileOptions
       self.builder.captureList = captureList
     }
   }
@@ -644,7 +650,8 @@ fileprivate extension Compiler.ByteCodeGen {
     _ ccc: DSLTree.CustomCharacterClass
   ) throws {
     if let asciiBitset = ccc.asAsciiBitset(options),
-        options.semanticLevel == .graphemeCluster {
+        options.semanticLevel == .graphemeCluster,
+        !compileOptions.contains(.unoptimized) {
       // future work: add a bit to .matchBitset to consume either a character
       // or a scalar so we can have this optimization in scalar mode
       builder.buildMatchAsciiBitset(asciiBitset)

Sources/_StringProcessing/Compiler.swift

@@ -16,6 +16,7 @@ class Compiler {
 
   // TODO: Or are these stored on the tree?
   var options = MatchingOptions()
+  private var compileOptions: CompileOptions = .default
 
   init(ast: AST) {
     self.tree = ast.dslTree
@@ -25,17 +26,36 @@ class Compiler {
     self.tree = tree
   }
 
+  init(tree: DSLTree, compileOptions: CompileOptions) {
+    self.tree = tree
+    self.compileOptions = compileOptions
+  }
+
   __consuming func emit() throws -> MEProgram {
     // TODO: Handle global options
     var codegen = ByteCodeGen(
-      options: options, captureList: tree.captureList
-    )
+      options: options,
+      compileOptions:
+        compileOptions,
+      captureList: tree.captureList)
     return try codegen.emitRoot(tree.root)
   }
 }
 
-/// Regex.Program and CompilerInterface.swift call these parse/compilation methods directly, this method is
-/// only for testing purposes (see CompileTest.swift)
+// An error produced when compiling a regular expression.
+enum RegexCompilationError: Error, CustomStringConvertible {
+  // TODO: Source location?
+  case uncapturedReference
+
+  var description: String {
+    switch self {
+    case .uncapturedReference:
+      return "Found a reference used before it captured any match."
+    }
+  }
+}
+
+// Testing support
 @available(SwiftStdlib 5.7, *)
 func _compileRegex(
   _ regex: String,
@@ -59,15 +79,10 @@ func _compileRegex(
   return Executor(program: program)
 }
 
-// An error produced when compiling a regular expression.
-enum RegexCompilationError: Error, CustomStringConvertible {
-  // TODO: Source location?
-  case uncapturedReference
-
-  var description: String {
-    switch self {
-    case .uncapturedReference:
-      return "Found a reference used before it captured any match."
-    }
+extension Compiler {
+  struct CompileOptions: OptionSet {
+    let rawValue: Int
+    static let unoptimized = CompileOptions(rawValue: 1)
+    static let `default`: CompileOptions = []
   }
 }

Sources/_StringProcessing/Regex/Core.swift

@@ -80,6 +80,9 @@ extension Regex {
     /// likely, compilation/caching.
     let tree: DSLTree
 
+    /// OptionSet of compiler options for testing purposes
+    fileprivate var compileOptions: Compiler.CompileOptions = .default
+
     private final class ProgramBox {
       let value: MEProgram
       init(_ value: MEProgram) { self.value = value }
@@ -93,7 +96,7 @@ extension Regex {
       if let loweredObject = _loweredProgramStorage as? ProgramBox {
         return loweredObject.value
       }
-      let lowered = try! Compiler(tree: tree).emit()
+      let lowered = try! Compiler(tree: tree, compileOptions: compileOptions).emit()
       _stdlib_atomicInitializeARCRef(object: &_loweredProgramStorage, desired: ProgramBox(lowered))
       return lowered
     }
@@ -132,3 +135,10 @@ extension Regex {
     self.program = Program(tree: .init(node))
   }
 }
+
+@available(SwiftStdlib 5.7, *)
+extension Regex {
+  internal mutating func _setCompilerOptionsForTesting(_ opts: Compiler.CompileOptions) {
+    program.compileOptions = opts
+  }
+}

Tests/RegexTests/MatchTests.swift

@@ -14,22 +14,32 @@ import XCTest
 @testable import _StringProcessing
 
 struct MatchError: Error {
-    var message: String
-    init(_ message: String) {
-        self.message = message
-    }
+  var message: String
+  init(_ message: String) {
+    self.message = message
+  }
 }
 
 func _firstMatch(
   _ regexStr: String,
   input: String,
+  validate: Bool,
   syntax: SyntaxOptions = .traditional
 ) throws -> (String, [String?]) {
   let regex = try Regex(regexStr, syntax: syntax)
   guard let result = try regex.firstMatch(in: input) else {
     throw MatchError("match not found for \(regexStr) in \(input)")
   }
   let caps = result.output.slices(from: input)
+
+  if validate {
+    var unoptRegex = try Regex(regexStr, syntax: syntax)
+    unoptRegex._setCompilerOptionsForTesting(.unoptimized)
+    guard let unoptResult = try unoptRegex.firstMatch(in: input) else {
+      throw MatchError("match not found for unoptimized \(regexStr) in \(input)")
+    }
+    XCTAssertEqual(String(input[result.range]), String(input[unoptResult.range]))
+  }
   return (String(input[result.range]), caps.map { $0.map(String.init) })
 }
 
@@ -41,6 +51,7 @@ func flatCaptureTest(
   syntax: SyntaxOptions = .traditional,
   dumpAST: Bool = false,
   xfail: Bool = false,
+  validate: Bool = true,
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -49,6 +60,7 @@ func flatCaptureTest(
       guard var (_, caps) = try? _firstMatch(
         regex,
         input: test,
+        validate: validate,
         syntax: syntax
       ) else {
         if expect == nil {
@@ -98,6 +110,7 @@ func matchTest(
   enableTracing: Bool = false,
   dumpAST: Bool = false,
   xfail: Bool = false,
+  validate: Bool = true,
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -110,6 +123,7 @@ func matchTest(
       enableTracing: enableTracing,
       dumpAST: dumpAST,
       xfail: xfail,
+      validate: validate,
       file: file,
       line: line)
   }
@@ -126,13 +140,15 @@ func firstMatchTest(
   enableTracing: Bool = false,
   dumpAST: Bool = false,
   xfail: Bool = false,
+  validate: Bool = true,
   file: StaticString = #filePath,
   line: UInt = #line
 ) {
   do {
     let (found, _) = try _firstMatch(
       regex,
       input: input,
+      validate: validate,
       syntax: syntax)
 
     if xfail {