Fully generalize "whole match" in the engine and enable transforming custom types

* Track the whole match as an element of the "capture list" in the matching engine. Do so by emitting code as an implicit `capture` around the root node.
* No longer handle `matcher` as a special case within `capture` lowering, because the matcher can be arbitrarily nested within "output-forwarding" nodes, such as a `changeMatchingOptions` non-capturing group. Instead, make the bytecode emitter carry a result value so that a custom output can be propagated through any forwarding nodes.
  ```swift
  Regex {
    Capture(
      SemanticVersionParser()
        .ignoringCase()
        .matchingSemantics(.unicodeScalar)
    ) // This would not work previously.
  }
  ```
* Collapse DSLTree node `transform` into `capture`, because a transform can never be standalone (without a `capture` parent). This greatly simplifies `capture` lowering.
* Make the bytecode's capture transform use type `(Input, _StoredCapture) -> Any` so that it can transform any whole match, not just `Substring`. This means you can now transform any captured value, including a custom-consuming regex component's result!
  ```swift
   Regex {
    "version:"
    OneOrMore(.whitespace)
    Capture {
      SemanticVersionParser() // Regex<SemanticVersion>
    } transform: {
      // (SemanticVersion) -> SomethingElse
    }
  }
  ```
  The transforms of `Capture` and `TryCapture` are now generalized from taking `Substring` to taking generic parameter `W` (the whole match).

Author: rxwei

Sources/RegexBuilder/Variadics.swift

@@ -646,28 +646,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping @Sendable (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any
-            },
-            component.regex.root)))
+          self.init(node: .capture(
+            component.regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
           as reference: Reference<NewCapture>,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping @Sendable (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any
-              },
-              component.regex.root)))
+            component.regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -676,28 +671,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping @Sendable (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any?
-            },
-            component.regex.root)))
+          self.init(node: .capture(
+          component.regex.root,
+          CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
           as reference: Reference<NewCapture>,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping @Sendable (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any?
-              },
-              component.regex.root)))
+            component.regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -725,28 +715,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping @Sendable (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any
-            },
-            component().regex.root)))
+          self.init(node: .capture(
+            component().regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           as reference: Reference<NewCapture>,
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping @Sendable (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any
-              },
-              component().regex.root)))
+            component().regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -755,28 +740,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping @Sendable (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any?
-            },
-            component().regex.root)))
+          self.init(node: .capture(
+            component().regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           as reference: Reference<NewCapture>,
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping @Sendable (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any?
-              },
-              component().regex.root)))
+            component().regex.root,
+            CaptureTransform(transform)))
         }
       }
 

Sources/VariadicsGenerator/VariadicsGenerator.swift

@@ -24,13 +24,13 @@ public struct CaptureList {
 extension CaptureList {
   public struct Capture {
     public var name: String?
-    public var type: Any.Type?
+    public var type: Any.Type
     public var optionalDepth: Int
     public var location: SourceLocation
 
     public init(
       name: String? = nil,
-      type: Any.Type? = nil,
+      type: Any.Type = Substring.self,
       optionalDepth: Int,
       _ location: SourceLocation
     ) {
@@ -125,6 +125,7 @@ extension AST.Node {
 
   public var _captureList: CaptureList {
     var caps = CaptureList()
+    caps.append(.init(optionalDepth: 0, .fake))
     self._addCaptures(to: &caps, optionalNesting: 0)
     return caps
   }
@@ -151,12 +152,7 @@ extension CaptureList: Equatable {}
 
 extension CaptureList.Capture: CustomStringConvertible {
   public var description: String {
-    let typeStr: String
-    if let ty = type {
-      typeStr = "\(ty)"
-    } else {
-      typeStr = "Substring"
-    }
+    let typeStr = String(describing: type)
     let suffix = String(repeating: "?", count: optionalDepth)
     return typeStr + suffix
   }

Sources/_RegexParser/Regex/Parse/CaptureList.swift

@@ -246,10 +246,7 @@ extension CaptureList {
 extension CaptureList.Capture {
   func _captureStructure(nestOptionals: Bool) -> CaptureStructure {
     if optionalDepth == 0 {
-      if let ty = type {
-        return .atom(name: name, type: .init(ty))
-      }
-      return .atom(name: name)
+      return .atom(name: name, type: type == Substring.self ? nil : .init(type))
     }
     var copy = self
     copy.optionalDepth = 0

Sources/_RegexParser/Regex/Parse/CaptureStructure.swift

@@ -77,7 +77,7 @@ extension RegexValidator {
     }
     switch ref.kind {
     case .absolute(let i):
-      guard i <= captures.captures.count else {
+      guard i < captures.captures.count else {
         throw error(.invalidReference(i), at: ref.innerLoc)
       }
     case .named(let name):

Sources/_RegexParser/Regex/Parse/Sema.swift

@@ -9,16 +9,20 @@ extension Compiler {
       self.options = options
       self.builder.captureList = captureList
     }
-
-    mutating func finish(
-    ) throws -> Program {
-      builder.buildAccept()
-      return try builder.assemble()
-    }
   }
 }
 
 extension Compiler.ByteCodeGen {
+  mutating func emitRoot(_ root: DSLTree.Node) throws -> Program {
+    // The whole match (`.0` element of output) is equivalent to an implicit
+    // capture over the entire regex.
+    try emitNode(.capture(name: nil, reference: nil, root))
+    builder.buildAccept()
+    return try builder.assemble()
+  }
+}
+
+fileprivate extension Compiler.ByteCodeGen {
   mutating func emitAtom(_ a: DSLTree.Atom) throws {
     switch a {
     case .any:
@@ -65,8 +69,7 @@ extension Compiler.ByteCodeGen {
 
     switch ref.kind {
     case .absolute(let i):
-      // Backreferences number starting at 1
-      builder.buildBackreference(.init(i-1))
+      builder.buildBackreference(.init(i))
     case .named(let name):
       try builder.buildNamedReference(name)
     case .relative:
@@ -329,9 +332,8 @@ extension Compiler.ByteCodeGen {
   }
 
   mutating func emitMatcher(
-    _ matcher: @escaping _MatcherInterface,
-    into capture: CaptureRegister? = nil
-  ) {
+    _ matcher: @escaping _MatcherInterface
+  ) -> ValueRegister {
 
     // TODO: Consider emitting consumer interface if
     // not captured. This may mean we should store
@@ -343,26 +345,7 @@ extension Compiler.ByteCodeGen {
 
     let valReg = builder.makeValueRegister()
     builder.buildMatcher(matcher, into: valReg)
-
-    // TODO: Instruction to store directly
-    if let cap = capture {
-      builder.buildMove(valReg, into: cap)
-    }
-  }
-
-  mutating func emitTransform(
-    _ t: CaptureTransform,
-    _ child: DSLTree.Node,
-    into cap: CaptureRegister
-  ) throws {
-    let transform = builder.makeTransformFunction {
-      input, range in
-      try t(input[range])
-    }
-    builder.buildBeginCapture(cap)
-    try emitNode(child)
-    builder.buildEndCapture(cap)
-    builder.buildTransformCapture(cap, transform)
+    return valReg
   }
 
   mutating func emitNoncapturingGroup(
@@ -612,7 +595,8 @@ extension Compiler.ByteCodeGen {
     builder.buildConsume(by: consumer)
   }
 
-  mutating func emitNode(_ node: DSLTree.Node) throws {
+  @discardableResult
+  mutating func emitNode(_ node: DSLTree.Node) throws -> ValueRegister? {
     switch node {
 
     case let .orderedChoice(children):
@@ -623,20 +607,34 @@ extension Compiler.ByteCodeGen {
         try emitConcatenationComponent(child)
       }
 
-    case let .capture(name, refId, child):
+    case let .capture(name, refId, child, transform):
       options.beginScope()
       defer { options.endScope() }
 
       let cap = builder.makeCapture(id: refId, name: name)
-      switch child {
-      case let .matcher(_, m):
-        emitMatcher(m, into: cap)
-      case let .transform(t, child):
-        try emitTransform(t, child, into: cap)
-      default:
-        builder.buildBeginCapture(cap)
-        try emitNode(child)
-        builder.buildEndCapture(cap)
+      builder.buildBeginCapture(cap)
+      let value = try emitNode(child)
+      builder.buildEndCapture(cap)
+      // If the child node produced a custom capture value, e.g. the result of
+      // a matcher, this should override the captured substring.
+      if let value {
+        builder.buildMove(value, into: cap)
+      }
+      // If there's a capture transform, apply it now.
+      if let transform = transform {
+        let fn = builder.makeTransformFunction { input, storedCapture in
+          // If it's a substring capture with no custom value, apply the
+          // transform directly to the substring to avoid existential traffic.
+          if let cap = storedCapture.latest, cap.value == nil {
+            return try transform(input[cap.range])
+          }
+          let value = constructExistentialOutputComponent(
+             from: input,
+             component: storedCapture.latest,
+             optionalCount: 0)
+          return try transform(value)
+        }
+        builder.buildTransformCapture(cap, fn)
       }
 
     case let .nonCapturingGroup(kind, child):
@@ -704,29 +702,25 @@ extension Compiler.ByteCodeGen {
       }
 
     case let .regexLiteral(l):
-      try emitNode(l.ast.dslTreeNode)
+      return try emitNode(l.ast.dslTreeNode)
 
     case let .convertedRegexLiteral(n, _):
-      try emitNode(n)
+      return try emitNode(n)
 
     case .absentFunction:
       throw Unsupported("absent function")
     case .consumer:
       throw Unsupported("consumer")
 
     case let .matcher(_, f):
-      emitMatcher(f)
-
-    case .transform:
-      throw Unreachable(
-        "Transforms only directly inside captures")
+      return emitMatcher(f)
 
     case .characterPredicate:
       throw Unsupported("character predicates")
 
     case .trivia, .empty:
-      return
+      break
     }
+    return nil
   }
 }
-