[String] Refactor validation and grapheme breaking

Internal refactoring. Validation now produces a (discardable) reason validation failed (only checked on slow-path). _hasGraphemeBreakBetween has been refactored to represent its quick-check nature.

Author: milseman

stdlib/public/core/StringCreate.swift

@@ -117,7 +117,7 @@ extension String {
       return unsafe (String._uncheckedFromUTF8(
         input, asciiPreScanResult: extraInfo.isASCII
       ), false)
-    case .error(let initialRange):
+    case .error(_, let initialRange):
         return unsafe (repairUTF8(input, firstKnownBrokenRange: initialRange), true)
     }
   }
@@ -139,7 +139,7 @@ extension String {
         newIsASCII: info.isASCII
       )
       return result.asString
-    case .error(let initialRange):
+    case .error(_, let initialRange):
       defer { _fixLifetime(result) }
       //This could be optimized to use excess tail capacity
       return unsafe repairUTF8(result.codeUnits, firstKnownBrokenRange: initialRange)

stdlib/public/core/StringGraphemeBreaking.swift

@@ -13,14 +13,18 @@
 import SwiftShims
 
 /// CR and LF are common special cases in grapheme breaking logic
-private var _CR: UInt8 { return 0x0d }
-private var _LF: UInt8 { return 0x0a }
-
-internal func _hasGraphemeBreakBetween(
+private var _CR: UInt8 { return 0x0D }
+private var _LF: UInt8 { return 0x0A }
+
+/// Perform a quick-check to determine if there's a grapheme-break between two
+/// scalars, without consulting the data tables. Returns true if there
+/// definitely is a break, false if there definitely is none, and nil if a
+/// break couldn't be determined
+internal func _quickHasGraphemeBreakBetween(
   _ lhs: Unicode.Scalar, _ rhs: Unicode.Scalar
-) -> Bool {
-
-  // CR-LF is a special case: no break between these
+) -> Bool? {
+  // GB3:
+  //   CR-LF is a special case: no break between these
   if lhs == Unicode.Scalar(_CR) && rhs == Unicode.Scalar(_LF) {
     return false
   }
@@ -80,7 +84,10 @@ internal func _hasGraphemeBreakBetween(
     default: return false
     }
   }
-  return hasBreakWhenPaired(lhs) && hasBreakWhenPaired(rhs)
+  if hasBreakWhenPaired(lhs) && hasBreakWhenPaired(rhs) {
+    return true
+  }
+  return nil
 }
 
 extension _StringGuts {
@@ -523,13 +530,7 @@ extension Unicode {
       between scalar1: Unicode.Scalar,
       and scalar2: Unicode.Scalar
     ) -> Bool? {
-      if scalar1.value == 0xD, scalar2.value == 0xA {
-        return false
-      }
-      if _hasGraphemeBreakBetween(scalar1, scalar2) {
-        return true
-      }
-      return nil
+      _quickHasGraphemeBreakBetween(scalar1, scalar2)
     }
 
     /// Initialize a new character recognizer at the _start of text_ (sot)
@@ -708,13 +709,8 @@ extension _GraphemeBreakingState {
     between scalar1: Unicode.Scalar,
     and scalar2: Unicode.Scalar
   ) -> Bool {
-    // GB3
-    if scalar1.value == 0xD, scalar2.value == 0xA {
-      return false
-    }
-
-    if _hasGraphemeBreakBetween(scalar1, scalar2) {
-      return true
+    if let result = _quickHasGraphemeBreakBetween(scalar1, scalar2) {
+      return result
     }
 
     let x = Unicode._GraphemeBreakProperty(from: scalar1)
@@ -883,13 +879,8 @@ extension _StringGuts {
     at index: Int,
     with previousScalar: (Int) -> (scalar: Unicode.Scalar, start: Int)?
   ) -> Bool {
-    // GB3
-    if scalar1.value == 0xD, scalar2.value == 0xA {
-      return false
-    }
-
-    if _hasGraphemeBreakBetween(scalar1, scalar2) {
-      return true
+    if let result = _quickHasGraphemeBreakBetween(scalar1, scalar2) {
+      return result
     }
 
     let x = Unicode._GraphemeBreakProperty(from: scalar1)

stdlib/public/core/StringUTF8Validation.swift

@@ -18,30 +18,97 @@ private func _isNotOverlong_F0(_ x: UInt8) -> Bool {
   return (0x90...0xBF).contains(x)
 }
 
-private func _isNotOverlong_F4(_ x: UInt8) -> Bool {
+private func _isNotInvalid_F4(_ x: UInt8) -> Bool {
   return UTF8.isContinuation(x) && x <= 0x8F
 }
 
 private func _isNotOverlong_E0(_ x: UInt8) -> Bool {
   return (0xA0...0xBF).contains(x)
 }
 
-private func _isNotOverlong_ED(_ x: UInt8) -> Bool {
+private func _isNotInvalid_ED(_ x: UInt8) -> Bool {
   return UTF8.isContinuation(x) && x <= 0x9F
 }
 
 internal struct UTF8ExtraInfo: Equatable {
   public var isASCII: Bool
 }
 
+@inline(never) // slow-path
+private func _diagnoseInvalidUTF8MultiByteLeading(
+  _ x: UInt8
+) -> _UTF8EncodingErrorKind {
+  _internalInvariant(x >= 0x80)
+  _internalInvariant(!_isUTF8MultiByteLeading(x))
+  switch x {
+  case 0x80...0xBF:
+    return .unexpectedContinuationByte
+  case 0xC0..<0xC2:
+    return .overlongEncodingByte
+  default:
+    _internalInvariant(x > 0xF4)
+    return .invalidNonSurrogateCodePointByte
+  }
+}
+
 internal enum UTF8ValidationResult {
   case success(UTF8ExtraInfo)
-  case error(toBeReplaced: Range<Int>)
+  case error(
+    kind: _UTF8EncodingErrorKind, toBeReplaced: Range<Int>
+  )
 }
 
-extension UTF8ValidationResult: Equatable {}
+// FIXME: refactor other parts of stdlib to avoid this dumb mirror enum
+//
+// Mirror of UTF8.EncodingError.Kind, available on 6.1
+internal struct _UTF8EncodingErrorKind: Error, Sendable, Hashable
+// TODO: embedded?, Codable
+  , RawRepresentable {
+  internal var rawValue: UInt8
+
+  @available(SwiftStdlib 6.1, *)
+  internal var _publicKind: UTF8.EncodingError.Kind {
+    .init(rawValue: self.rawValue)
+  }
+
+  @inlinable
+  internal init(rawValue: UInt8) {
+    self.rawValue = rawValue
+  }
+
+  /// A continuation byte (`10xxxxxx`) outside of a multi-byte sequence
+  @_alwaysEmitIntoClient
+  internal static var unexpectedContinuationByte: Self {
+    .init(rawValue: 0)
+  }
+
+  /// A byte in a surrogate code point (`U+D800..U+DFFF`) sequence
+  @_alwaysEmitIntoClient
+  internal static var surrogateCodePointByte: Self {
+    .init(rawValue: 1)
+  }
 
-private struct UTF8ValidationError: Error {}
+  /// A byte in an invalid, non-surrogate code point (`>U+10FFFF`) sequence
+  @_alwaysEmitIntoClient
+  internal static var invalidNonSurrogateCodePointByte: Self {
+    .init(rawValue: 2)
+  }
+
+  /// A byte in an overlong encoding sequence
+  @_alwaysEmitIntoClient
+  internal static var overlongEncodingByte: Self {
+    .init(rawValue: 3)
+  }
+
+  /// A multi-byte sequence that is the start of a valid multi-byte scalar
+  /// but is cut off before ending correctly
+  @_alwaysEmitIntoClient
+  internal static var truncatedScalar: Self {
+    .init(rawValue: 4)
+  }
+}
+
+extension UTF8ValidationResult: Equatable {}
 
 internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationResult {
   if unsafe _allASCII(buf) {
@@ -51,12 +118,20 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
   var iter = unsafe buf.makeIterator()
   var lastValidIndex = buf.startIndex
 
-  @inline(__always) func guaranteeIn(_ f: (UInt8) -> Bool) throws(UTF8ValidationError) {
-    guard let cu = unsafe iter.next() else { throw UTF8ValidationError() }
-    guard f(cu) else { throw UTF8ValidationError() }
+  @inline(__always) func guarantee(
+    _ f: (UInt8) -> Bool,
+    _ err: _UTF8EncodingErrorKind
+  ) throws(_UTF8EncodingErrorKind) {
+    guard let cu = unsafe iter.next() else {
+      throw .truncatedScalar
+    }
+    guard f(cu) else {
+      throw err
+    }
   }
-  @inline(__always) func guaranteeContinuation() throws(UTF8ValidationError) {
-    try guaranteeIn(UTF8.isContinuation)
+  @inline(__always) func guaranteeContinuation(
+  ) throws(_UTF8EncodingErrorKind) {
+    try guarantee(UTF8.isContinuation, .truncatedScalar)
   }
 
   func _legacyInvalidLengthCalculation(_ _buffer: (_storage: UInt32, ())) -> Int {
@@ -117,37 +192,56 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
     return unsafe _legacyNarrowIllegalRange(buf: buf[illegalRange])
   }
 
-  do {
+  do throws(_UTF8EncodingErrorKind) {
+
+    /*
+    The table of valid UTF-8 is:
+
+     ╔════════════════════╦════════╦════════╦════════╦════════╗
+     ║    Scalar value    ║ Byte 0 ║ Byte 1 ║ Byte 2 ║ Byte 3 ║
+     ╠════════════════════╬════════╬════════╬════════╬════════╣
+     ║ U+0000..U+007F     ║ 00..7F ║        ║        ║        ║
+     ║ U+0080..U+07FF     ║ C2..DF ║ Contin ║        ║        ║
+     ║ U+0800..U+0FFF     ║ E0     ║ A0..BF ║ Contin ║        ║
+     ║ U+1000..U+CFFF     ║ E1..EC ║ Contin ║ Contin ║        ║
+     ║ U+D000..U+D7FF     ║ ED     ║ 80..9F ║ Contin ║        ║
+     ║ U+E000..U+FFFF     ║ EE..EF ║ Contin ║ Contin ║        ║
+     ║ U+10000..U+3FFFF   ║ F0     ║ 90..BF ║ Contin ║ Contin ║
+     ║ U+40000..U+FFFFF   ║ F1..F3 ║ Contin ║ Contin ║ Contin ║
+     ║ U+100000..U+10FFFF ║ F4     ║ 80..8F ║ Contin ║ Contin ║
+     ╚════════════════════╩════════╩════════╩════════╩════════╝
+
+     "Contin" is any continuation byte, i.e. 80..BF or 10xxxxxx
+     */
     var isASCII = true
     while let cu = unsafe iter.next() {
       if UTF8.isASCII(cu) { lastValidIndex &+= 1; continue }
       isASCII = false
       if _slowPath(!_isUTF8MultiByteLeading(cu)) {
-        func fail() throws(UTF8ValidationError) { throw UTF8ValidationError() }
-        try fail()
+        throw _diagnoseInvalidUTF8MultiByteLeading(cu)
       }
       switch cu {
       case 0xC2...0xDF:
         try guaranteeContinuation()
         lastValidIndex &+= 2
       case 0xE0:
-        try guaranteeIn(_isNotOverlong_E0)
+        try guarantee(_isNotOverlong_E0, .overlongEncodingByte)
         try guaranteeContinuation()
         lastValidIndex &+= 3
       case 0xE1...0xEC:
         try guaranteeContinuation()
         try guaranteeContinuation()
         lastValidIndex &+= 3
       case 0xED:
-        try guaranteeIn(_isNotOverlong_ED)
+        try guarantee(_isNotInvalid_ED, .surrogateCodePointByte)
         try guaranteeContinuation()
         lastValidIndex &+= 3
       case 0xEE...0xEF:
         try guaranteeContinuation()
         try guaranteeContinuation()
         lastValidIndex &+= 3
       case 0xF0:
-        try guaranteeIn(_isNotOverlong_F0)
+        try guarantee(_isNotOverlong_F0, .overlongEncodingByte)
         try guaranteeContinuation()
         try guaranteeContinuation()
         lastValidIndex &+= 4
@@ -157,7 +251,8 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
         try guaranteeContinuation()
         lastValidIndex &+= 4
       case 0xF4:
-        try guaranteeIn(_isNotOverlong_F4)
+        try guarantee(
+          _isNotInvalid_F4, .invalidNonSurrogateCodePointByte)
         try guaranteeContinuation()
         try guaranteeContinuation()
         lastValidIndex &+= 4
@@ -167,7 +262,9 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
     }
     return .success(UTF8ExtraInfo(isASCII: isASCII))
   } catch {
-    return unsafe .error(toBeReplaced: findInvalidRange(buf[lastValidIndex...]))
+    return unsafe .error(
+      kind: error,
+      toBeReplaced: findInvalidRange(buf[lastValidIndex...]))
   }
 }
 
@@ -214,7 +311,7 @@ internal func repairUTF8(_ input: UnsafeBufferPointer<UInt8>, firstKnownBrokenRa
     case .success:
       unsafe result.appendInPlace(remainingInput, isASCII: false)
       return String(result)
-    case .error(let newBrokenRange):
+    case .error(_, let newBrokenRange):
       brokenRange = newBrokenRange
     }
   } while !remainingInput.isEmpty