Skip to content

Commit e0af639

Browse files
rctcwyvrnrctcwyvrn
authored
Add more benchmarks and benchmarker functionality (swiftlang#505)
* Add debug mode * Fix typo in css regex * Add HTML benchmark * Add email regex benchmarks * Add save/compare functionality to the benchmarker * Clean up compare and add cli flags
1 parent 94f5d33 commit e0af639

File tree

11 files changed

+2931
-82
lines changed

11 files changed

+2931
-82
lines changed

‎Sources/RegexBenchmark/Benchmark.swift

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import Foundation
44
public protocol RegexBenchmark {
55
var name: String { get }
66
func run()
7+
func debug()
78
}
89

910
public struct Benchmark: RegexBenchmark {
1011
public let name: String
11-
let regex: Regex<Substring>
12+
let regex: Regex<AnyRegexOutput>
1213
let type: MatchType
1314
let target: String
1415

@@ -50,66 +51,6 @@ public struct NSBenchmark: RegexBenchmark {
5051
}
5152
}
5253

53-
public struct BenchmarkRunner {
54-
// Register instances of Benchmark and run them
55-
let suiteName: String
56-
var suite: [any RegexBenchmark]
57-
let samples: Int
58-
59-
public init(_ suiteName: String) {
60-
self.suiteName = suiteName
61-
self.suite = []
62-
self.samples = 20
63-
}
64-
65-
public init(_ suiteName: String, _ n: Int) {
66-
self.suiteName = suiteName
67-
self.suite = []
68-
self.samples = n
69-
}
70-
71-
public mutating func register(_ new: some RegexBenchmark) {
72-
suite.append(new)
73-
}
74-
75-
func measure(benchmark: some RegexBenchmark) -> Time {
76-
var times: [Time] = []
77-
78-
// initial run to make sure the regex has been compiled
79-
benchmark.run()
80-
81-
// fixme: use suspendingclock?
82-
for _ in 0..<samples {
83-
let start = Tick.now
84-
benchmark.run()
85-
let end = Tick.now
86-
let time = end.elapsedTime(since: start)
87-
times.append(time)
88-
}
89-
// todo: compute stdev and warn if it's too large
90-
91-
// return median time
92-
times.sort()
93-
return times[samples/2]
94-
}
95-
96-
public func run() {
97-
print("Running")
98-
for b in suite {
99-
print("- \(b.name) \(measure(benchmark: b))")
100-
}
101-
}
102-
103-
public func profile() {
104-
print("Starting")
105-
for b in suite {
106-
print("- \(b.name)")
107-
b.run()
108-
print("- done")
109-
}
110-
}
111-
}
112-
11354
/// A benchmark meant to be ran across multiple engines
11455
struct CrossBenchmark {
11556
/// The base name of the benchmark
@@ -130,7 +71,7 @@ struct CrossBenchmark {
13071
var isWhole: Bool = false
13172

13273
func register(_ runner: inout BenchmarkRunner) {
133-
let swiftRegex = try! Regex(regex, as: Substring.self)
74+
let swiftRegex = try! Regex(regex)
13475

13576
let nsPattern = isWhole ? "^" + regex + "$" : regex
13677
let nsRegex: NSRegularExpression

‎Sources/RegexBenchmark/BenchmarkRunner.swift

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import Foundation
2+
3+
public struct BenchmarkRunner {
4+
let suiteName: String
5+
var suite: [any RegexBenchmark] = []
6+
let samples: Int
7+
var results: SuiteResult = SuiteResult()
8+
9+
// Outputting
10+
let startTime = Date()
11+
let outputPath: String
12+
13+
public init(_ suiteName: String, _ n: Int, _ outputPath: String) {
14+
self.suiteName = suiteName
15+
self.samples = n
16+
self.outputPath = outputPath
17+
}
18+
19+
public mutating func register(_ new: some RegexBenchmark) {
20+
suite.append(new)
21+
}
22+
23+
mutating func measure(benchmark: some RegexBenchmark) -> Time {
24+
var times: [Time] = []
25+
26+
// initial run to make sure the regex has been compiled
27+
// todo: measure compile times, or at least how much this first run
28+
// differs from the later ones
29+
benchmark.run()
30+
31+
// fixme: use suspendingclock?
32+
for _ in 0..<samples {
33+
let start = Tick.now
34+
benchmark.run()
35+
let end = Tick.now
36+
let time = end.elapsedTime(since: start)
37+
times.append(time)
38+
}
39+
// todo: compute stdev and warn if it's too large
40+
41+
// return median time
42+
times.sort()
43+
let median = times[samples/2]
44+
self.results.add(name: benchmark.name, time: median)
45+
return median
46+
}
47+
48+
public mutating func run() {
49+
print("Running")
50+
for b in suite {
51+
print("- \(b.name) \(measure(benchmark: b))")
52+
}
53+
}
54+
55+
public func profile() {
56+
print("Starting")
57+
for b in suite {
58+
print("- \(b.name)")
59+
b.run()
60+
print("- done")
61+
}
62+
}
63+
64+
public mutating func debug() {
65+
print("Debugging")
66+
print("========================")
67+
for b in suite {
68+
print("- \(b.name) \(measure(benchmark: b))")
69+
b.debug()
70+
print("========================")
71+
}
72+
}
73+
}
74+
75+
extension BenchmarkRunner {
76+
var dateStyle: Date.FormatStyle {
77+
Date.FormatStyle()
78+
.year(.twoDigits)
79+
.month(.twoDigits)
80+
.day(.twoDigits)
81+
.hour(.twoDigits(amPM: .omitted))
82+
.minute(.twoDigits)
83+
}
84+
85+
var outputFolderUrl: URL {
86+
let url = URL(fileURLWithPath: outputPath, isDirectory: true)
87+
if !FileManager.default.fileExists(atPath: url.path) {
88+
try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true)
89+
}
90+
return url
91+
}
92+
93+
public func save() throws {
94+
let now = startTime.formatted(dateStyle)
95+
let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json")
96+
print("Saving result to \(resultJsonUrl.path)")
97+
try results.save(to: resultJsonUrl)
98+
}
99+
100+
func fetchLatestResult() throws -> (Date, SuiteResult) {
101+
var pastResults: [Date: SuiteResult] = [:]
102+
for resultFile in try FileManager.default.contentsOfDirectory(
103+
at: outputFolderUrl,
104+
includingPropertiesForKeys: nil
105+
) {
106+
let dateString = resultFile.lastPathComponent.replacingOccurrences(
107+
of: "-result.json",
108+
with: "")
109+
let date = try dateStyle.parse(dateString)
110+
pastResults.updateValue(try SuiteResult.load(from: resultFile), forKey: date)
111+
}
112+
113+
let sorted = pastResults
114+
.sorted(by: {(kv1,kv2) in kv1.0 > kv2.0})
115+
return sorted[0]
116+
}
117+
118+
public func compare() throws {
119+
// It just compares by the latest result for now, we probably want a CLI
120+
// flag to set which result we want to compare against
121+
let (compareDate, compareResult) = try fetchLatestResult()
122+
let diff = results.compare(with: compareResult)
123+
let regressions = diff.filter({(_, change) in change.seconds > 0})
124+
let improvements = diff.filter({(_, change) in change.seconds < 0})
125+
126+
print("Comparing against benchmark done on \(compareDate.formatted(dateStyle))")
127+
print("=== Regressions ====================================================")
128+
for item in regressions {
129+
let oldVal = compareResult.results[item.key]!
130+
let newVal = results.results[item.key]!
131+
let percentage = item.value.seconds / oldVal.seconds
132+
print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%")
133+
}
134+
print("=== Improvements ====================================================")
135+
for item in improvements {
136+
let oldVal = compareResult.results[item.key]!
137+
let newVal = results.results[item.key]!
138+
let percentage = item.value.seconds / oldVal.seconds
139+
print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%")
140+
}
141+
}
142+
}
143+
144+
struct SuiteResult {
145+
var results: [String: Time] = [:]
146+
147+
public mutating func add(name: String, time: Time) {
148+
results.updateValue(time, forKey: name)
149+
}
150+
151+
public func compare(with other: SuiteResult) -> [String: Time] {
152+
var output: [String: Time] = [:]
153+
for item in results {
154+
if let otherVal = other.results[item.key] {
155+
let diff = item.value - otherVal
156+
// note: is this enough time difference?
157+
if diff.abs() > Time.millisecond {
158+
output.updateValue(diff, forKey: item.key)
159+
}
160+
}
161+
}
162+
return output
163+
}
164+
}
165+
166+
extension SuiteResult: Codable {
167+
public func save(to url: URL) throws {
168+
let encoder = JSONEncoder()
169+
let data = try encoder.encode(self)
170+
try data.write(to: url, options: .atomic)
171+
}
172+
173+
public static func load(from url: URL) throws -> SuiteResult {
174+
let decoder = JSONDecoder()
175+
let data = try Data(contentsOf: url)
176+
return try decoder.decode(SuiteResult.self, from: data)
177+
}
178+
}

‎Sources/RegexBenchmark/CLI.swift

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,53 @@ struct Runner: ParsableCommand {
55
@Argument(help: "Names of benchmarks to run")
66
var specificBenchmarks: [String] = []
77

8-
@Option(help: "Run only once for profiling purposes")
8+
@Flag(help: "Run only once for profiling purposes")
99
var profile = false
1010

1111
@Option(help: "How many samples to collect for each benchmark")
1212
var samples = 20
13+
14+
@Flag(help: "Debug benchmark regexes")
15+
var debug = false
16+
17+
@Option(help: "Output folder")
18+
var outputPath = "./results/"
19+
20+
@Flag(help: "Should the results be saved")
21+
var save = false
22+
23+
@Flag(help: "Compare this result with the latest saved result")
24+
var compare = false
1325

1426
func makeRunner() -> BenchmarkRunner {
15-
var benchmark = BenchmarkRunner("RegexBench", samples)
27+
var benchmark = BenchmarkRunner("RegexBench", samples, outputPath)
1628
benchmark.addReluctantQuant()
1729
benchmark.addCSS()
1830
benchmark.addNotFound()
1931
benchmark.addGraphemeBreak()
2032
benchmark.addHangulSyllable()
33+
benchmark.addHTML()
34+
benchmark.addEmail()
2135
return benchmark
2236
}
37+
2338
mutating func run() throws {
2439
var runner = makeRunner()
2540
if !self.specificBenchmarks.isEmpty {
2641
runner.suite = runner.suite.filter { b in specificBenchmarks.contains(b.name) }
2742
}
28-
if profile {
29-
runner.profile()
30-
} else {
43+
switch (profile, debug) {
44+
case (true, true): print("Cannot run both profile and debug")
45+
case (true, false): runner.profile()
46+
case (false, true): runner.debug()
47+
case (false, false):
3148
runner.run()
49+
if compare {
50+
try runner.compare()
51+
}
52+
if save {
53+
try runner.save()
54+
}
3255
}
3356
}
3457
}

0 commit comments

Comments
 (0)