Skip to content

Add regex-specific Matches and Ranges collections #460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jun 17, 2022
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 39 additions & 3 deletions Sources/_StringProcessing/Algorithms/Algorithms/Ranges.swift
Original file line number Diff line number Diff line change
Expand Up @@ -226,17 +226,53 @@ extension BidirectionalCollection where Element: Comparable {
// }
}

@available(SwiftStdlib 5.7, *)
struct RegexRangesCollection<Output> {
let base: RegexMatchesCollection<Output>

init(string: Substring, regex: Regex<Output>) {
self.base = RegexMatchesCollection(base: string, regex: regex)
}
}

@available(SwiftStdlib 5.7, *)
extension RegexRangesCollection: Sequence {
struct Iterator: IteratorProtocol {
var matchesBase: RegexMatchesCollection<Output>.Iterator

mutating func next() -> Range<String.Index>? {
matchesBase.next().map(\.range)
}
}

func makeIterator() -> Iterator {
Iterator(matchesBase: base.makeIterator())
}
}

@available(SwiftStdlib 5.7, *)
extension RegexRangesCollection: Collection {
typealias Index = RegexMatchesCollection<Output>.Index

var startIndex: Index { base.startIndex }
var endIndex: Index { base.endIndex }
func index(after i: Index) -> Index { base.index(after: i) }
subscript(position: Index) -> Range<String.Index> { base[position].range }
}

// MARK: Regex algorithms

extension BidirectionalCollection where SubSequence == Substring {
extension Collection where SubSequence == Substring {
@available(SwiftStdlib 5.7, *)
@_disfavoredOverload
func _ranges<R: RegexComponent>(
of regex: R
) -> RangesCollection<RegexConsumer<R, Self>> {
_ranges(of: RegexConsumer(regex))
) -> RegexRangesCollection<R.RegexOutput> {
RegexRangesCollection(string: self[...], regex: regex.regex)
}
}

extension BidirectionalCollection where SubSequence == Substring {
@available(SwiftStdlib 5.7, *)
func _rangesFromBack<R: RegexComponent>(
of regex: R
Expand Down
43 changes: 12 additions & 31 deletions Sources/_StringProcessing/Algorithms/Algorithms/Replace.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,19 @@
// MARK: `CollectionSearcher` algorithms

extension RangeReplaceableCollection {
func _replacing<Searcher: CollectionSearcher, Replacement: Collection>(
_ searcher: Searcher,
func _replacing<Ranges: Collection, Replacement: Collection>(
_ ranges: Ranges,
with replacement: Replacement,
subrange: Range<Index>,
maxReplacements: Int = .max
) -> Self where Searcher.Searched == SubSequence,
) -> Self where Ranges.Element == Range<Index>,
Replacement.Element == Element
{
precondition(maxReplacements >= 0)

var index = subrange.lowerBound
var result = Self()
result.append(contentsOf: self[..<index])
var index = startIndex

for range in self[subrange]._ranges(of: searcher).prefix(maxReplacements) {
for range in ranges.prefix(maxReplacements) {
result.append(contentsOf: self[index..<range.lowerBound])
result.append(contentsOf: replacement)
index = range.upperBound
Expand All @@ -36,29 +34,15 @@ extension RangeReplaceableCollection {
return result
}

func _replacing<Searcher: CollectionSearcher, Replacement: Collection>(
_ searcher: Searcher,
with replacement: Replacement,
maxReplacements: Int = .max
) -> Self where Searcher.Searched == SubSequence,
Replacement.Element == Element
{
_replacing(
searcher,
with: replacement,
subrange: startIndex..<endIndex,
maxReplacements: maxReplacements)
}

mutating func _replace<
Searcher: CollectionSearcher, Replacement: Collection
Ranges: Collection, Replacement: Collection
>(
_ searcher: Searcher,
_ ranges: Ranges,
with replacement: Replacement,
maxReplacements: Int = .max
) where Searcher.Searched == SubSequence, Replacement.Element == Element {
) where Ranges.Element == Range<Index>, Replacement.Element == Element {
self = _replacing(
searcher,
ranges,
with: replacement,
maxReplacements: maxReplacements)
}
Expand All @@ -85,9 +69,8 @@ extension RangeReplaceableCollection where Element: Equatable {
maxReplacements: Int = .max
) -> Self where C.Element == Element, Replacement.Element == Element {
_replacing(
ZSearcher(pattern: Array(other), by: ==),
self[subrange]._ranges(of: other),
with: replacement,
subrange: subrange,
maxReplacements: maxReplacements)
}

Expand Down Expand Up @@ -143,9 +126,8 @@ extension RangeReplaceableCollection
maxReplacements: Int = .max
) -> Self where C.Element == Element, Replacement.Element == Element {
_replacing(
PatternOrEmpty(searcher: TwoWaySearcher(pattern: Array(other))),
self[subrange]._ranges(of: other),
with: replacement,
subrange: subrange,
maxReplacements: maxReplacements)
}

Expand Down Expand Up @@ -195,9 +177,8 @@ extension RangeReplaceableCollection where SubSequence == Substring {
maxReplacements: Int = .max
) -> Self where Replacement.Element == Element {
_replacing(
RegexConsumer(regex),
self[subrange]._ranges(of: regex),
with: replacement,
subrange: subrange,
maxReplacements: maxReplacements)
}

Expand Down
173 changes: 146 additions & 27 deletions Sources/_StringProcessing/Algorithms/Matching/Matches.swift
Original file line number Diff line number Diff line change
Expand Up @@ -183,13 +183,156 @@ extension BidirectionalCollection {

// MARK: Regex algorithms

@available(SwiftStdlib 5.7, *)
struct RegexMatchesCollection<Output> {
let input: Substring
let regex: Regex<Output>
let startIndex: Index

init(base: Substring, regex: Regex<Output>) {
self.input = base
self.regex = regex
self.startIndex = base.firstMatch(of: regex).map(Index.match) ?? .end
}
}

@available(SwiftStdlib 5.7, *)
extension RegexMatchesCollection: Sequence {
/// Returns the index to start searching for the next match after `match`.
fileprivate func searchIndex(after match: Regex<Output>.Match) -> String.Index? {
if !match.range.isEmpty {
return match.range.upperBound
}

// If the last match was an empty match, advance by one position and
// run again, unless at the end of `input`.
if match.range.lowerBound == input.endIndex {
return nil
}

switch regex.initialOptions.semanticLevel {
case .graphemeCluster:
return input.index(after: match.range.upperBound)
case .unicodeScalar:
return input.unicodeScalars.index(after: match.range.upperBound)
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've seen this code in a few places. I wonder if it should be an underscored helper method somewhere that takes the semantic level as a parameter?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I'll add it as part of #435.

}

struct Iterator: IteratorProtocol {
let base: RegexMatchesCollection

// Because `RegexMatchesCollection` eagerly computes the first match for
// its `startIndex`, the iterator begins with this current match populated.
// For subsequent calls to `next()`, this value is `nil`, and `nextStart`
// is used to search for the next match.
var currentMatch: Regex<Output>.Match?
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we double store it? Is this just a Bool?

var nextStart: String.Index?
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the difference between initialIterator == true and nextStart == nil? Do we have 4 cases, 3 cases, or 2? What's the difference between nextStart is nil vs endIndex?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are three cases:

  1. initialIteration == true: We use the match found (or not) that's stored in base.startIndex
  2. nextStart != nil: We search for the next match starting at nextStart
  3. nextStart == nil: End of iteration, no need to search

nextStart == endIndex is a valid search-starting index, just like any other, so it can't represent the "finished iterating" state.


init(_ matches: RegexMatchesCollection) {
self.base = matches
self.currentMatch = matches.startIndex.match
self.nextStart = currentMatch.flatMap(base.searchIndex(after:))
}

mutating func next() -> Regex<Output>.Match? {
// Initial case with pre-computed first match
if let match = currentMatch {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we never set currentMatch, then it's just a bool telling you whether to grab the one from the underlying collection vs computing one yourself.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, nice 👍🏻

currentMatch = nil
return match
}

// `nextStart` is `nil` when iteration has completed
guard let start = nextStart else {
return nil
}

// Otherwise, find the next match (if any) and compute `nextStart`
let match = try! base.regex.firstMatch(in: base.input[start...])
nextStart = match.flatMap(base.searchIndex(after:))
return match
}
}

func makeIterator() -> Iterator {
Iterator(self)
}
}

@available(SwiftStdlib 5.7, *)
extension RegexMatchesCollection: Collection {
enum Index: Comparable {
case match(Regex<Output>.Match)
case end

var match: Regex<Output>.Match? {
switch self {
case .match(let match): return match
case .end: return nil
}
}

static func == (lhs: Self, rhs: Self) -> Bool {
switch (lhs, rhs) {
case (.match(let lhs), .match(let rhs)):
return lhs.range == rhs.range
case (.end, .end):
return true
case (.end, .match), (.match, .end):
return false
}
}

static func < (lhs: Self, rhs: Self) -> Bool {
switch (lhs, rhs) {
case (.match(let lhs), .match(let rhs)):
// This implementation uses a tuple comparison so that an empty
// range `i..<i` will be ordered before a non-empty range at that
// same starting point `i..<j`. As of 2022-05-30, `Regex` does not
// return matches of this kind, but that is one behavior under
// discussion for regexes like /a*|b/ when matched against "b".
return (lhs.range.lowerBound, lhs.range.upperBound)
< (rhs.range.lowerBound, rhs.range.upperBound)
case (.match, .end):
return true
case (.end, .match), (.end, .end):
return false
}
}
}

var endIndex: Index {
Index.end
}

func index(after i: Index) -> Index {
guard let currentMatch = i.match else {
fatalError("Can't advance past the 'endIndex' of a match collection.")
}

guard
let start = searchIndex(after: currentMatch),
let nextMatch = try! regex.firstMatch(in: input[start...])
else {
return .end
}
return Index.match(nextMatch)
}

subscript(position: Index) -> Regex<Output>.Match {
guard let match = position.match else {
fatalError("Can't subscript the 'endIndex' of a match collection.")
}
return match
}
}

extension BidirectionalCollection where SubSequence == Substring {
@available(SwiftStdlib 5.7, *)
@_disfavoredOverload
func _matches<R: RegexComponent>(
of regex: R
) -> MatchesCollection<RegexConsumer<R, Self>> {
_matches(of: RegexConsumer(regex))
) -> RegexMatchesCollection<R.RegexOutput> {
RegexMatchesCollection(base: self[...], regex: regex.regex)
}

@available(SwiftStdlib 5.7, *)
Expand All @@ -207,30 +350,6 @@ extension BidirectionalCollection where SubSequence == Substring {
public func matches<Output>(
of r: some RegexComponent<Output>
) -> [Regex<Output>.Match] {
let slice = self[...]
var start = self.startIndex
let end = self.endIndex
let regex = r.regex

var result = [Regex<Output>.Match]()
while start <= end {
guard let match = try? regex._firstMatch(
slice.base, in: start..<end
) else {
break
}
result.append(match)
if match.range.isEmpty {
if match.range.upperBound == end {
break
}
// FIXME: semantic level
start = slice.index(after: match.range.upperBound)
} else {
start = match.range.upperBound
}
}
return result
Array(_matches(of: r))
}

}
35 changes: 35 additions & 0 deletions Tests/RegexTests/AlgorithmsInternalsTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,39 @@ extension AlgorithmTests {
XCTAssertEqual("x", "axb"._trimming(r))
XCTAssertEqual("x", "axbb"._trimming(r))
}

func testMatchesCollection() {
let r = try! Regex("a|b+|c*", as: Substring.self)

let str = "zaabbbbbbcde"
let matches = str._matches(of: r)
let expected: [Substring] = [
"", // before 'z'
"a",
"a",
"bbbbbb",
"c",
"", // after 'c'
"", // after 'd'
"", // after 'e'
]

// Make sure we're getting the right collection type
let _: RegexMatchesCollection<Substring> = matches

XCTAssertEqual(matches.map(\.output), expected)

let i = matches.index(matches.startIndex, offsetBy: 3)
XCTAssertEqual(matches[i].output, expected[3])
let j = matches.index(i, offsetBy: 5)
XCTAssertEqual(j, matches.endIndex)

var index = matches.startIndex
while index < matches.endIndex {
XCTAssertEqual(
matches[index].output,
expected[matches.distance(from: matches.startIndex, to: index)])
matches.formIndex(after: &index)
}
}
}
Loading