SQUASH ME: unsafe annotations, final API

Author: milseman

stdlib/public/core/StringNormalization.swift

@@ -126,7 +126,7 @@ extension _StringGutsSlice {
 
   internal func _fastNFCCheck(_ isNFCQC: inout Bool, _ prevCCC: inout UInt8) {
     unsafe withFastUTF8 { utf8 in
-      isNFCQC = _nfcQuickCheck(utf8, prevCCC: &prevCCC)
+      isNFCQC = unsafe _nfcQuickCheck(utf8, prevCCC: &prevCCC)
     }
   }
 }

stdlib/public/core/UTF8EncodingError.swift

@@ -69,7 +69,7 @@ extension Unicode.UTF8 {
    multi-byte scalar but is cut off before ending correctly). For all other
    errors (including overlong encodings, surrogates, and invalid code
    points), it will produce an error per byte.
-  
+
    // FIXME: without a checkAllErrors, we don't have these classification distinctions, should we drop it, ensure we will do it, or what?
 
    Since overlong encodings, surrogates, and invalid code points are erroneous
@@ -210,3 +210,52 @@ extension UTF8.ValidationError: CustomStringConvertible {
     "UTF8.ValidationError(\(kind), \(byteOffsets))"
   }
 }
+
+@available(SwiftStdlib 6.1, *)
+ extension UTF8 {
+    @usableFromInline // for testing purposes
+    internal static func _checkAllErrors(
+      _ s: some Sequence<UInt8>
+    ) -> Array<UTF8.ValidationError> {
+      // TODO: Span fast path
+      // TODO: Fixed size buffer for non-contig inputs
+      // TODO: Lifetime-dependent result variant
+      let cus = Array(s)
+      return unsafe cus.withUnsafeBytes {
+        var bufPtr = unsafe $0
+        var start = 0
+        var errors: Array<UTF8.ValidationError> = []
+
+        // Remember the previous error, so that we can
+        // apply it to subsequent bytes instead of reporting
+        // just `.unexpectedContinuation`.
+        var priorError: UTF8.ValidationError? = nil
+        while true {
+          do throws(UTF8.ValidationError) {
+            _ = unsafe try bufPtr.baseAddress!._validateUTF8(limitedBy: bufPtr.count)
+            return errors
+          } catch {
+            let adjustedRange =
+              error.byteOffsets.lowerBound + start ..< error.byteOffsets.upperBound + start
+
+            let kind: UTF8.ValidationError.Kind
+            if let prior = priorError,
+               prior.byteOffsets.upperBound == adjustedRange.lowerBound,
+               error.kind == .unexpectedContinuationByte
+            {
+              kind = prior.kind
+            } else {
+              kind = error.kind
+            }
+            let adjustedErr = UTF8.ValidationError(kind, adjustedRange)
+            priorError = adjustedErr
+
+            let errEnd = error.byteOffsets.upperBound
+            start += errEnd
+            unsafe bufPtr = .init(rebasing: bufPtr[errEnd...])
+            errors.append(adjustedErr)
+          }
+        }
+      }
+    }
+ }

stdlib/public/core/UTF8Span.swift

@@ -3,6 +3,7 @@
 
 /// TODO: docs
 @frozen
+@safe
 @available(SwiftStdlib 6.1, *)
 public struct UTF8Span: Copyable, ~Escapable, BitwiseCopyable {
   @usableFromInline
@@ -25,26 +26,36 @@ public struct UTF8Span: Copyable, ~Escapable, BitwiseCopyable {
 
   // @_alwaysEmitIntoClient
   @inline(__always)
-  @lifetime(borrow start)
+  @lifetime(borrow start) // TODO: borrow or copy?
   internal init(
     _unsafeAssumingValidUTF8 start: borrowing UnsafeRawPointer,
     _countAndFlags: UInt64
   ) {
-    self._unsafeBaseAddress = copy start
+    unsafe self._unsafeBaseAddress = copy start
     self._countAndFlags = _countAndFlags
 
     _invariantCheck()
   }
 
+  @unsafe
+  @lifetime(copy uncheckedCodeUnits) // TODO: borrow or copy?
+  public init(
+    unsafeAssumingValidUTF8 uncheckedCodeUnits: Span<UInt8>,
+    isKnownASCII: Bool = false
+  ) {
+    self.init(
+      _uncheckedAssumingValidUTF8: uncheckedCodeUnits,
+      isKnownASCII: isKnownASCII,
+      isKnownNFC: false
+    )
+  }
+
   // FIXME: we need to make sure ALL API are nil safe, that is they
   // at least check the count first
   @_alwaysEmitIntoClient
   internal func _start() -> UnsafeRawPointer {
-    _unsafeBaseAddress._unsafelyUnwrappedUnchecked
+    unsafe _unsafeBaseAddress._unsafelyUnwrappedUnchecked
   }
-
-  // HACK: working around lack of internal plumbing work
-  internal var _str: String { _start()._str(0..<count) }
 }
 
 // TODO: try to convert code to be ran on Span instead of URP
@@ -67,18 +78,16 @@ extension UTF8Span {
   internal init(
     _validating codeUnits: consuming Span<UInt8>
   ) throws(UTF8.ValidationError) {
-    guard let ptr = codeUnits._pointer else {
-      self._unsafeBaseAddress = nil
+    guard let basePtr = unsafe codeUnits._pointer else {
+      unsafe self._unsafeBaseAddress = nil
       self._countAndFlags = 0
       return
     }
 
-    // FIXME: handle empty/null span
-    let basePtr = codeUnits._start()
     let count = codeUnits._count
-    let isASCII = try basePtr._validateUTF8(limitedBy: count)
+    let isASCII = unsafe try basePtr._validateUTF8(limitedBy: count)
 
-    self._unsafeBaseAddress = .init(basePtr)
+    unsafe self._unsafeBaseAddress = .init(basePtr)
     self._countAndFlags = UInt64(truncatingIfNeeded: count)
     if isASCII {
       _setIsASCII()
@@ -93,13 +102,13 @@ extension UTF8Span {
     isKnownASCII: Bool,
     isKnownNFC: Bool
   ) {
-    guard let ptr = codeUnits._pointer else {
-      self._unsafeBaseAddress = nil
+    guard let ptr = unsafe codeUnits._pointer else {
+      unsafe self._unsafeBaseAddress = nil
       self._countAndFlags = 0
       return
     }
 
-    self._unsafeBaseAddress = codeUnits._start()
+    unsafe self._unsafeBaseAddress = ptr
     self._countAndFlags = UInt64(truncatingIfNeeded: codeUnits.count)
     if isKnownASCII {
       _setIsASCII()
@@ -109,6 +118,9 @@ extension UTF8Span {
     }
     _internalInvariant(self.count == codeUnits.count)
   }
+
+  // HACK: working around lack of internal plumbing work
+  internal var _str: String { unsafe _start()._str(0..<count) }
 }
 
 
@@ -135,7 +147,7 @@ extension UTF8Span {
   >(
     _ body: (_ buffer: /*borrowing*/ UnsafeBufferPointer<UInt8>) throws(E) -> Result
   ) throws(E) -> Result {
-    try body(_start()._ubp(0..<count))
+    try unsafe body(_start()._ubp(0..<count))
   }
 
   // TODO: withSpan or similar?
@@ -164,29 +176,43 @@ extension UTF8Span {
   public var span: Span<UInt8> {
     @lifetime(copy self)
     get {
-      Span(_unchecked: _unsafeBaseAddress, count: self.count)
+      unsafe Span(_unchecked: _unsafeBaseAddress, count: self.count)
     }
   }
 
 
 }
 
-func UNSUPPORTED(_ message: String) -> Never {
-  fatalError("UNSUPPORTED: \(message)")
-}
-
 // TODO(toolchain): decide if we rebase on top of Guillaume's work
 @available(SwiftStdlib 6.1, *)
 extension String {
   public var utf8Span: UTF8Span {
-    UNSUPPORTED("utf8Span property pending compiler fixes")
+    @lifetime(borrow self)
+    borrowing get {
+      let isKnownASCII = _guts.isASCII
+      let utf8 = self.utf8
+      let span = utf8.span
+      let result = unsafe UTF8Span(
+        unsafeAssumingValidUTF8: span,
+        isKnownASCII: isKnownASCII)
+      return unsafe _overrideLifetime(result, borrowing: self)
+    }
   }
 }
 
 @available(SwiftStdlib 6.1, *)
 extension Substring {
   public var utf8Span: UTF8Span {
-    UNSUPPORTED("utf8Span property pending compiler fixes")
+    @lifetime(borrow self)
+    borrowing get {
+      let isKnownASCII = base._guts.isASCII
+      let utf8 = self.utf8
+      let span = utf8.span
+      let result = unsafe UTF8Span(
+        unsafeAssumingValidUTF8: span,
+        isKnownASCII: isKnownASCII)
+      return unsafe _overrideLifetime(result, borrowing: self)
+    }
   }
 }
 

stdlib/public/core/UTF8SpanBits.swift

@@ -24,8 +24,8 @@ extension UTF8Span {
   public mutating func checkForASCII() -> Bool {
     if isKnownASCII { return true }
 
-    let result = _withUnsafeBufferPointer {
-      _allASCII($0)
+    let result = unsafe _withUnsafeBufferPointer {
+      unsafe _allASCII($0)
     }
     if result {
       _setIsASCII()
@@ -72,9 +72,9 @@ extension UTF8Span {
     if isKnownNFC { return true }
 
     if quickCheck {
-      let result = _withUnsafeBufferPointer { utf8 in
+      let result = unsafe _withUnsafeBufferPointer { utf8 in
         var prevCCC: UInt8 = 0
-        return _nfcQuickCheck(utf8, prevCCC: &prevCCC)
+        return unsafe _nfcQuickCheck(utf8, prevCCC: &prevCCC)
       }
       if result {
         self._countAndFlags |= Self._nfcBit
@@ -84,7 +84,7 @@ extension UTF8Span {
 
     // TODO: use faster internal algorithm
     let normalized = _str._nfcCodeUnits
-    guard _start()._urbp(
+    guard unsafe _start()._urbp(
       0..<count
     ).elementsEqual(normalized) else {
       return false

stdlib/public/core/UTF8SpanComparisons.swift

@@ -6,7 +6,7 @@ extension UTF8Span {
   /// Whether this span has the same bytes as `other`.
   @_alwaysEmitIntoClient
   public func bytesEqual(to other: some Sequence<UInt8>) -> Bool {
-    _withUnsafeBufferPointer { $0.elementsEqual(other) }
+    unsafe _withUnsafeBufferPointer { unsafe $0.elementsEqual(other) }
   }
 
   /// Whether this span has the same `Unicode.Scalar`s as `other`.
@@ -57,9 +57,9 @@ extension UTF8Span {
   public func isCanonicallyEquivalent(
     to other: UTF8Span
   ) -> Bool {
-    self._withUnsafeBufferPointer { selfBufPtr in
-      other._withUnsafeBufferPointer { otherBufPtr in
-        _stringCompareFastUTF8(
+    unsafe self._withUnsafeBufferPointer { selfBufPtr in
+      unsafe other._withUnsafeBufferPointer { otherBufPtr in
+        unsafe _stringCompareFastUTF8(
           selfBufPtr,
           otherBufPtr,
           expecting: .equal,
@@ -73,9 +73,9 @@ extension UTF8Span {
   public func isCanonicallyLessThan(
     _ other: UTF8Span
   ) -> Bool {
-    self._withUnsafeBufferPointer { selfBufPtr in
-      other._withUnsafeBufferPointer { otherBufPtr in
-        _stringCompareFastUTF8(
+    unsafe self._withUnsafeBufferPointer { selfBufPtr in
+      unsafe other._withUnsafeBufferPointer { otherBufPtr in
+        unsafe _stringCompareFastUTF8(
           selfBufPtr,
           otherBufPtr,
           expecting: .less,
@@ -85,15 +85,16 @@ extension UTF8Span {
   }
 }
 
-@available(SwiftStdlib 6.1, *)
-extension UTF8Span {
-  public static func ~=(_ lhs: StaticString, _ rhs: UTF8Span) -> Bool {
-    return lhs.withUTF8Buffer { str in
-      rhs._withUnsafeBufferPointer { span in
-        str.elementsEqual(span)
-      }
-    }
-  }
-}
+// // FIXME: remove
+// @available(SwiftStdlib 6.1, *)
+// extension UTF8Span {
+//   public static func ~=(_ lhs: StaticString, _ rhs: UTF8Span) -> Bool {
+//     return lhs.withUTF8Buffer { str in
+//       rhs._withUnsafeBufferPointer { span in
+//         str.elementsEqual(span)
+//       }
+//     }
+//   }
+// }