First ver

Author: rctcwyvrn

Sources/_StringProcessing/ByteCodeGen.swift

@@ -208,11 +208,20 @@ fileprivate extension Compiler.ByteCodeGen {
       builder.buildConsume(by: consumeScalar {
         $0.properties.lowercaseMapping == s.properties.lowercaseMapping
       })
-    } else {
-      builder.buildConsume(by: consumeScalar {
-        $0 == s
-      })
+      return
     }
+    
+    if s.value < 0x300 {
+      // lily todo: make sure this is correct + add compiler option check after it's merged in
+      // we unconditionally match against the scalar using consumeScalar in the else case
+      // so maybe this check is uneccessary??
+      builder.buildMatchScalar(s, boundaryCheck: false)
+      return
+    }
+    
+    builder.buildConsume(by: consumeScalar {
+      $0 == s
+    })
   }
 
   mutating func emitCharacter(_ c: Character) throws {
@@ -233,9 +242,27 @@ fileprivate extension Compiler.ByteCodeGen {
           ? input.index(after: bounds.lowerBound)
           : nil
       }
-    } else {
-      builder.buildMatch(c)
     }
+    
+//    if c.unicodeScalars.count == 1,
+//        let first = c.unicodeScalars.first,
+//        first.value < 0x300 { // lily todo: check this more carefully
+      // if we have a single scalar then this must not be an extended grapheme cluster
+      // so it must be a character that can be exactly matched by its first scalar
+      // cr-lf has two scalars right? yes it has two
+      
+      // i think one these two checks are redundant, I think we only need the second?
+      // ask alex?
+    
+    // we can only match against characters that have a single cannonical equivalence
+    // so I think that rules out any latin in here, so just use ascii for now
+    // we also need to exclude our good non-single-scalar-ascii friend cr-lf
+    if c.isASCII && c != "\r\n" {
+      builder.buildMatchScalar(c.unicodeScalars.first!, boundaryCheck: true)
+      return
+    }
+      
+    builder.buildMatch(c)
   }
 
   mutating func emitAny() {
@@ -732,7 +759,22 @@ fileprivate extension Compiler.ByteCodeGen {
             return currentIndex
           }
         } else {
-          builder.buildMatchSequence(s)
+          // if we have any extended latin in our characters then we have to
+          // respect cannoical equivalence, so we cannot match against scalars exactly
+          // so match against all single scalar ascii
+          
+          // lily todo: which strings are nfc invariant and matchable by direct scalar comparison?
+          // alternatively: loop over characters in s and emit either matchScalar or matchCharacter depending on if it is NFC invariant
+          // getting rid of matchSeq entirely does also get rid of the weird ARC
+          if s.allSatisfy({c in c.isASCII && c != "\r\n"}) {
+            for scalar in s.unicodeScalars.dropLast(1) {
+              builder.buildMatchScalar(scalar, boundaryCheck: false)
+            }
+            // check that we are on a boundary at the end and there isn't a combining character after this scalar
+            builder.buildMatchScalar(s.unicodeScalars.last!, boundaryCheck: true)
+          } else {
+            builder.buildMatchSequence(s)
+          }
         }
       } else {
         builder.buildConsume {

Sources/_StringProcessing/Engine/InstPayload.swift

@@ -146,6 +146,13 @@ extension Instruction.Payload {
   var string: StringRegister {
     interpret()
   }
+  
+  init(scalar: Unicode.Scalar) {
+    self.init(UInt64(scalar.value))
+  }
+  var scalar: Unicode.Scalar {
+    return Unicode.Scalar(_value: UInt32(self.rawValue))
+  }
 
   init(sequence: SequenceRegister) {
     self.init(sequence)

Sources/_StringProcessing/Engine/Instruction.swift

@@ -83,6 +83,9 @@ extension Instruction {
     ///
     /// Operand: Sequence register to compare against.
     case matchSequence
+    
+    case matchScalar
+    case matchScalarUnchecked
 
     /// TODO: builtin assertions and anchors
     case builtinAssertion

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -146,6 +146,14 @@ extension MEProgram.Builder {
       .matchSequence,
       .init(sequence: sequences.store(.init(s)))))
   }
+  
+  mutating func buildMatchScalar(_ s: Unicode.Scalar, boundaryCheck: Bool) {
+    if boundaryCheck {
+      instructions.append(.init(.matchScalar, .init(scalar: s)))
+    } else {
+      instructions.append(.init(.matchScalarUnchecked, .init(scalar: s)))
+    }
+  }
 
   mutating func buildConsume(
     by p: @escaping MEProgram.ConsumeFunction

Sources/_StringProcessing/Engine/Processor.swift

@@ -226,6 +226,23 @@ extension Processor {
     }
     return true
   }
+  
+  func loadScalar() -> Unicode.Scalar? {
+    currentPosition < end ? input.unicodeScalars[currentPosition] : nil
+  }
+  
+  mutating func matchScalar(_ s: Unicode.Scalar, boundaryCheck: Bool) -> Bool {
+    guard let curScalar = loadScalar(),
+      curScalar == s,
+      let idx = input.unicodeScalars.index(currentPosition, offsetBy: 1, limitedBy: end),
+      (!boundaryCheck || input.isOnGraphemeClusterBoundary(idx))
+    else {
+      signalFailure()
+      return false
+    }
+    currentPosition = idx
+    return true
+  }
 
   mutating func signalFailure() {
     guard let (pc, pos, stackEnd, capEnds, intRegisters) =
@@ -363,7 +380,17 @@ extension Processor {
       if matchSeq(seq) {
         controller.step()
       }
-
+    
+    case .matchScalar:
+      let scalar = payload.scalar
+      if matchScalar(scalar, boundaryCheck: true) {
+        controller.step()
+      }
+    case .matchScalarUnchecked:
+      let scalar = payload.scalar
+      if matchScalar(scalar, boundaryCheck: false) {
+        controller.step()
+      }
     case .consumeBy:
       let reg = payload.consumer
       guard currentPosition < searchBounds.upperBound,