Skip to content

Commit 005e0fb

Browse files
authored
Merge pull request #491 from rctcwyvrn/benchmarker
Add regex benchmarker
2 parents 0878029 + 73b0482 commit 005e0fb

11 files changed

+2397
-1
lines changed

Package.swift

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ let package = Package(
4141
targets: ["_RegexParser"]),
4242
.executable(
4343
name: "VariadicsGenerator",
44-
targets: ["VariadicsGenerator"])
44+
targets: ["VariadicsGenerator"]),
45+
.executable(
46+
name: "RegexBenchmark",
47+
targets: ["RegexBenchmark"])
4548
],
4649
dependencies: [
4750
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.0.0"),
@@ -112,6 +115,17 @@ let package = Package(
112115
"_RegexParser",
113116
"_StringProcessing"
114117
]),
118+
.executableTarget(
119+
name: "RegexBenchmark",
120+
dependencies: [
121+
.product(name: "ArgumentParser", package: "swift-argument-parser"),
122+
"_RegexParser",
123+
"_StringProcessing",
124+
"RegexBuilder"
125+
],
126+
swiftSettings: [
127+
.unsafeFlags(["-Xfrontend", "-disable-availability-checking"]),
128+
]),
115129

116130
// MARK: Exercises
117131
.target(
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import _StringProcessing
2+
import Foundation
3+
4+
public protocol RegexBenchmark {
5+
var name: String { get }
6+
func run()
7+
}
8+
9+
public struct Benchmark: RegexBenchmark {
10+
public let name: String
11+
let regex: Regex<Substring>
12+
let ty: MatchType
13+
let target: String
14+
15+
public enum MatchType {
16+
case whole
17+
case first
18+
case allMatches
19+
}
20+
21+
public func run() {
22+
switch ty {
23+
case .whole: blackHole(target.wholeMatch(of: regex))
24+
case .allMatches: blackHole(target.matches(of: regex))
25+
case .first: blackHole(target.firstMatch(of: regex))
26+
}
27+
}
28+
}
29+
30+
public struct NSBenchmark: RegexBenchmark {
31+
public let name: String
32+
let regex: NSRegularExpression
33+
let ty: NSMatchType
34+
let target: String
35+
36+
var range: NSRange {
37+
NSRange(target.startIndex..<target.endIndex, in: target)
38+
}
39+
40+
public enum NSMatchType {
41+
case all
42+
case first
43+
}
44+
45+
public func run() {
46+
switch ty {
47+
case .all: blackHole(regex.matches(in: target, range: range))
48+
case .first: blackHole(regex.firstMatch(in: target, range: range))
49+
}
50+
}
51+
}
52+
53+
public struct BenchmarkRunner {
54+
// Register instances of Benchmark and run them
55+
let suiteName: String
56+
var suite: [any RegexBenchmark]
57+
let samples: Int
58+
59+
public init(_ suiteName: String) {
60+
self.suiteName = suiteName
61+
self.suite = []
62+
self.samples = 20
63+
}
64+
65+
public init(_ suiteName: String, _ n: Int) {
66+
self.suiteName = suiteName
67+
self.suite = []
68+
self.samples = n
69+
}
70+
71+
public mutating func register(_ new: some RegexBenchmark) {
72+
suite.append(new)
73+
}
74+
75+
func measure(benchmark: some RegexBenchmark) -> Time {
76+
var times: [Time] = []
77+
78+
// initial run to make sure the regex has been compiled
79+
benchmark.run()
80+
81+
// fixme: use suspendingclock?
82+
for _ in 0..<samples {
83+
let start = Tick.now
84+
benchmark.run()
85+
let end = Tick.now
86+
let time = end.elapsedTime(since: start)
87+
times.append(time)
88+
}
89+
// todo: compute stdev and warn if it's too large
90+
91+
// return median time
92+
times.sort()
93+
return times[samples/2]
94+
}
95+
96+
public func run() {
97+
print("Running")
98+
for b in suite {
99+
print("- \(b.name) \(measure(benchmark: b))")
100+
}
101+
}
102+
103+
public func profile() {
104+
print("Starting")
105+
for b in suite {
106+
print("- \(b.name)")
107+
b.run()
108+
print("- done")
109+
}
110+
}
111+
}
112+
113+
// nom nom nom, consume the argument
114+
@inline(never)
115+
public func blackHole<T>(_ x: T) {
116+
}

Sources/RegexBenchmark/CLI.swift

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import ArgumentParser
2+
3+
@main
4+
struct Runner: ParsableCommand {
5+
@Argument(help: "Names of benchmarks to run")
6+
var specificBenchmarks: [String] = []
7+
8+
@Option(help: "Run only once for profiling purposes")
9+
var profile = false
10+
11+
@Option(help: "How many samples to collect for each benchmark")
12+
var samples = 20
13+
14+
func makeRunner() -> BenchmarkRunner {
15+
var benchmark = BenchmarkRunner("RegexBench", samples)
16+
benchmark.addReluctantQuant()
17+
benchmark.addBacktracking()
18+
benchmark.addCSS()
19+
benchmark.addFirstMatch()
20+
return benchmark
21+
}
22+
mutating func run() throws {
23+
var runner = makeRunner()
24+
if !self.specificBenchmarks.isEmpty {
25+
runner.suite = runner.suite.filter { b in specificBenchmarks.contains(b.name) }
26+
}
27+
if profile {
28+
runner.profile()
29+
} else {
30+
runner.run()
31+
}
32+
}
33+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import _StringProcessing
2+
import RegexBuilder
3+
import Foundation
4+
5+
// Tests that involve heavy backtracking
6+
7+
extension BenchmarkRunner {
8+
mutating func addBacktracking() {
9+
let r = "^ +A"
10+
let s = String(repeating: " ", count: 10000)
11+
12+
let basicBacktrack = Benchmark(
13+
name: "BasicBacktrack",
14+
regex: try! Regex(r),
15+
ty: .allMatches,
16+
target: s
17+
)
18+
19+
let basicBacktrackNS = NSBenchmark(
20+
name: "BasicBacktrackNS",
21+
regex: try! NSRegularExpression(pattern: r),
22+
ty: .all,
23+
target: s
24+
)
25+
26+
let basicBacktrackFirstMatch = Benchmark(
27+
name: "BasicBacktrackFirstMatch",
28+
regex: try! Regex(r),
29+
ty: .first,
30+
target: s
31+
)
32+
33+
let basicBacktrackNSFirstMatch = NSBenchmark(
34+
name: "BasicBacktrackNSFirstMatch",
35+
regex: try! NSRegularExpression(pattern: r),
36+
ty: .first,
37+
target: s
38+
)
39+
40+
register(basicBacktrack)
41+
register(basicBacktrackNS)
42+
register(basicBacktrackFirstMatch)
43+
register(basicBacktrackNSFirstMatch)
44+
}
45+
}

0 commit comments

Comments
 (0)