Introduce ASTStage parameter to parse

This allows specifying whether or not to perform
semantic checks on the AST. Some clients, e.g
syntax coloring, only care about the syntactic
structure. But other clients want errors to be
emitted for e.g unsupported constructs.

Author: hamishknight

Sources/PatternConverter/PatternConverter.swift

@@ -50,7 +50,7 @@ struct PatternConverter: ParsableCommand {
     print("Converting '\(delim)\(regex)\(delim)'")
 
     let ast = try _RegexParser.parse(
-      regex,
+      regex, .semantic,
       experimentalSyntax ? .experimental : .traditional)
 
     // Show rendered source ranges

Sources/_RegexParser/Regex/Parse/CompilerInterface.swift

@@ -96,7 +96,7 @@ public func swiftCompilerParseRegexLiteral(
   _ input: String, captureBufferOut: UnsafeMutableRawBufferPointer
 ) throws -> (regexToEmit: String, version: Int) {
   do {
-    let ast = try parseWithDelimiters(input)
+    let ast = try parseWithDelimiters(input, .semantic)
     // Serialize the capture structure for later type inference.
     assert(captureBufferOut.count >= input.utf8.count)
     ast.captureStructure.encode(to: captureBufferOut)

Sources/_RegexParser/Regex/Parse/Parse.swift

@@ -558,15 +558,34 @@ extension Parser {
   }
 }
 
+public enum ASTStage {
+  /// The regex is parsed, and a syntactically valid AST is returned. Otherwise
+  /// an error is thrown. This is useful for e.g syntax coloring.
+  case syntactic
+
+  /// The regex is parsed, and a syntactically and semantically valid AST is
+  /// returned. Otherwise an error is thrown. A semantically valid AST has been
+  /// checked for e.g unsupported constructs and invalid backreferences.
+  case semantic
+}
+
 public func parse<S: StringProtocol>(
-  _ regex: S, _ syntax: SyntaxOptions
+  _ regex: S, _ stage: ASTStage, _ syntax: SyntaxOptions
 ) throws -> AST where S.SubSequence == Substring
 {
   let source = Source(String(regex))
   var parser = Parser(source, syntax: syntax)
   return try parser.parse()
 }
 
+@available(*, deprecated, renamed: "parse(_:_:_:)")
+public func parse<S: StringProtocol>(
+  _ regex: S, _ syntax: SyntaxOptions
+) throws -> AST where S.SubSequence == Substring
+{
+  try parse(regex, .syntactic, syntax)
+}
+
 /// Retrieve the default set of syntax options that a delimiter and literal
 /// contents indicates.
 fileprivate func defaultSyntaxOptions(
@@ -588,14 +607,23 @@ fileprivate func defaultSyntaxOptions(
   }
 }
 
+@available(*, deprecated, renamed: "parseWithDelimiters(_:_:)")
+public func parseWithDelimiters<S: StringProtocol>(
+  _ regex: S
+) throws -> AST where S.SubSequence == Substring
+{
+  try parseWithDelimiters(regex, .syntactic)
+}
+
 /// Parses a given regex string with delimiters, inferring the syntax options
 /// from the delimiters used.
 public func parseWithDelimiters<S: StringProtocol>(
-  _ regex: S
+  _ regex: S, _ stage: ASTStage
 ) throws -> AST where S.SubSequence == Substring {
   let (contents, delim) = droppingRegexDelimiters(String(regex))
   do {
-    return try parse(contents, defaultSyntaxOptions(delim, contents: contents))
+    let syntax = defaultSyntaxOptions(delim, contents: contents)
+    return try parse(contents, stage, syntax)
   } catch let error as LocatedErrorProtocol {
     // Convert the range in 'contents' to the range in 'regex'.
     let delimCount = delim.opening.count

Sources/_StringProcessing/Compiler.swift

@@ -38,7 +38,7 @@ class Compiler {
 func _compileRegex(
   _ regex: String, _ syntax: SyntaxOptions = .traditional
 ) throws -> Executor {
-  let ast = try parse(regex, syntax)
+  let ast = try parse(regex, .semantic, syntax)
   let program = try Compiler(ast: ast).emit()
   return Executor(program: program)
 }

Sources/_StringProcessing/Regex/AnyRegexOutput.swift

@@ -17,7 +17,7 @@ extension Regex where Output == AnyRegexOutput {
   ///
   /// - Parameter pattern: The regular expression.
   public init(_ pattern: String) throws {
-    self.init(ast: try parse(pattern, .traditional))
+    self.init(ast: try parse(pattern, .semantic, .traditional))
   }
 }
 
@@ -31,7 +31,7 @@ extension Regex {
     _ pattern: String,
     as: Output.Type = Output.self
   ) throws {
-    self.init(ast: try parse(pattern, .traditional))
+    self.init(ast: try parse(pattern, .semantic, .traditional))
   }
 }
 

Sources/_StringProcessing/Regex/Core.swift

@@ -44,7 +44,7 @@ public struct Regex<Output>: RegexComponent {
   // Compiler interface. Do not change independently.
   @usableFromInline
   init(_regexString pattern: String) {
-    self.init(ast: try! parse(pattern, .traditional))
+    self.init(ast: try! parse(pattern, .semantic, .traditional))
   }
 
   // Compiler interface. Do not change independently.
@@ -53,7 +53,7 @@ public struct Regex<Output>: RegexComponent {
     assert(version == currentRegexLiteralFormatVersion)
     // The version argument is passed by the compiler using the value defined
     // in libswiftParseRegexLiteral.
-    self.init(ast: try! parseWithDelimiters(pattern))
+    self.init(ast: try! parseWithDelimiters(pattern, .semantic))
   }
 
   public var regex: Regex<Output> {

Tests/RegexTests/CaptureTests.swift

@@ -150,7 +150,7 @@ func captureTest(
   file: StaticString = #file,
   line: UInt = #line
 ) {
-  let ast = try! parse(regex, .traditional)
+  let ast = try! parse(regex, .semantic, .traditional)
   let capList = ast.root._captureList
   guard capList == expected else {
     XCTFail("""

Tests/RegexTests/DiagnosticTests.swift

@@ -20,7 +20,7 @@ extension RegexTests {
     XCTAssert(SourceLocation.fake.isFake)
     XCTAssert(group(.capture, "a").location.isFake)
 
-    let ast = try! parse("(a)", .traditional).root
+    let ast = try! parse("(a)", .semantic, .traditional).root
     XCTAssert(ast.location.isReal)
   }
 
@@ -31,7 +31,7 @@ extension RegexTests {
     //
     // Input should be a concatenation or alternation
     func flatTest(_ str: String, _ expected: [String]) {
-      guard let ast = try? parse(str, .traditional).root else {
+      guard let ast = try? parse(str, .semantic, .traditional).root else {
         XCTFail("Fail to parse: \(str)")
         return
       }
@@ -54,7 +54,7 @@ extension RegexTests {
 
     func renderTest(_ str: String, _ expected: [String]) {
       let lines = try! parse(
-        str, .traditional
+        str, .semantic, .traditional
       )._render(in: str)
       func fail() {
         XCTFail("""

Tests/RegexTests/ParseTests.swift

@@ -170,8 +170,8 @@ func parseNotEqualTest(
   syntax: SyntaxOptions = .traditional,
   file: StaticString = #file, line: UInt = #line
 ) {
-  let lhsAST = try! parse(lhs, syntax)
-  let rhsAST = try! parse(rhs, syntax)
+  let lhsAST = try! parse(lhs, .syntactic, syntax)
+  let rhsAST = try! parse(rhs, .syntactic, syntax)
   if lhsAST == rhsAST || lhsAST._dump() == rhsAST._dump() {
     XCTFail("""
               AST: \(lhsAST._dump())
@@ -187,7 +187,7 @@ func rangeTest(
   at locFn: (AST.Node) -> SourceLocation = \.location,
   file: StaticString = #file, line: UInt = #line
 ) {
-  let ast = try! parse(input, syntax).root
+  let ast = try! parse(input, .syntactic, syntax).root
   let range = input.offsets(of: locFn(ast).range)
   let expected = expectedRange(input)
 
@@ -207,7 +207,7 @@ func diagnosticTest(
   file: StaticString = #file, line: UInt = #line
 ) {
   do {
-    let ast = try parse(input, syntax)
+    let ast = try parse(input, .semantic, syntax)
     XCTFail("""
 
       Passed \(ast)
@@ -236,7 +236,7 @@ func diagnosticWithDelimitersTest(
     input, ignoreTrailing: ignoreTrailing, file: file, line: line)
 
   do {
-    let orig = try parseWithDelimiters(literal)
+    let orig = try parseWithDelimiters(literal, .semantic)
     let ast = orig.root
     XCTFail("""