-
Notifications
You must be signed in to change notification settings - Fork 49
Add more benchmarks and benchmarker functionality #505
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
5fd8840
[benchmark] Add no-capture version of grapheme breaking exercise
milseman 03fe8d6
[benchmark] Add cross-engine benchmark helpers
milseman 5667705
[benchmark] Hangul Syllable finding benchmark
milseman bde259b
Add debug mode
rctcwyvrn bf95e81
Fix typo in css regex
rctcwyvrn 243ec7b
Add HTML benchmark
rctcwyvrn eeb0852
Add email regex benchmarks
rctcwyvrn 49efd67
Add save/compare functionality to the benchmarker
rctcwyvrn b3a61a7
Clean up compare and add cli flags
rctcwyvrn 926d208
Merge branch 'main' into more_more_benchmarks
milseman File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,178 @@ | ||
| import Foundation | ||
|
|
||
| public struct BenchmarkRunner { | ||
| let suiteName: String | ||
| var suite: [any RegexBenchmark] = [] | ||
| let samples: Int | ||
| var results: SuiteResult = SuiteResult() | ||
|
|
||
| // Outputting | ||
| let startTime = Date() | ||
| let outputPath: String | ||
|
|
||
| public init(_ suiteName: String, _ n: Int, _ outputPath: String) { | ||
| self.suiteName = suiteName | ||
| self.samples = n | ||
| self.outputPath = outputPath | ||
| } | ||
|
|
||
| public mutating func register(_ new: some RegexBenchmark) { | ||
| suite.append(new) | ||
| } | ||
|
|
||
| mutating func measure(benchmark: some RegexBenchmark) -> Time { | ||
| var times: [Time] = [] | ||
|
|
||
| // initial run to make sure the regex has been compiled | ||
| // todo: measure compile times, or at least how much this first run | ||
| // differs from the later ones | ||
| benchmark.run() | ||
|
|
||
| // fixme: use suspendingclock? | ||
| for _ in 0..<samples { | ||
| let start = Tick.now | ||
| benchmark.run() | ||
| let end = Tick.now | ||
| let time = end.elapsedTime(since: start) | ||
| times.append(time) | ||
| } | ||
| // todo: compute stdev and warn if it's too large | ||
|
|
||
| // return median time | ||
| times.sort() | ||
| let median = times[samples/2] | ||
| self.results.add(name: benchmark.name, time: median) | ||
| return median | ||
| } | ||
|
|
||
| public mutating func run() { | ||
| print("Running") | ||
| for b in suite { | ||
| print("- \(b.name) \(measure(benchmark: b))") | ||
| } | ||
| } | ||
|
|
||
| public func profile() { | ||
| print("Starting") | ||
| for b in suite { | ||
| print("- \(b.name)") | ||
| b.run() | ||
| print("- done") | ||
| } | ||
| } | ||
|
|
||
| public mutating func debug() { | ||
| print("Debugging") | ||
| print("========================") | ||
| for b in suite { | ||
| print("- \(b.name) \(measure(benchmark: b))") | ||
| b.debug() | ||
| print("========================") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| extension BenchmarkRunner { | ||
| var dateStyle: Date.FormatStyle { | ||
| Date.FormatStyle() | ||
| .year(.twoDigits) | ||
| .month(.twoDigits) | ||
| .day(.twoDigits) | ||
| .hour(.twoDigits(amPM: .omitted)) | ||
| .minute(.twoDigits) | ||
| } | ||
|
|
||
| var outputFolderUrl: URL { | ||
| let url = URL(fileURLWithPath: outputPath, isDirectory: true) | ||
| if !FileManager.default.fileExists(atPath: url.path) { | ||
| try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true) | ||
| } | ||
| return url | ||
| } | ||
|
|
||
| public func save() throws { | ||
| let now = startTime.formatted(dateStyle) | ||
| let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json") | ||
| print("Saving result to \(resultJsonUrl.path)") | ||
| try results.save(to: resultJsonUrl) | ||
| } | ||
|
|
||
| func fetchLatestResult() throws -> (Date, SuiteResult) { | ||
| var pastResults: [Date: SuiteResult] = [:] | ||
| for resultFile in try FileManager.default.contentsOfDirectory( | ||
| at: outputFolderUrl, | ||
| includingPropertiesForKeys: nil | ||
| ) { | ||
| let dateString = resultFile.lastPathComponent.replacingOccurrences( | ||
| of: "-result.json", | ||
| with: "") | ||
| let date = try dateStyle.parse(dateString) | ||
| pastResults.updateValue(try SuiteResult.load(from: resultFile), forKey: date) | ||
| } | ||
|
|
||
| let sorted = pastResults | ||
| .sorted(by: {(kv1,kv2) in kv1.0 > kv2.0}) | ||
| return sorted[0] | ||
| } | ||
|
|
||
| public func compare() throws { | ||
| // It just compares by the latest result for now, we probably want a CLI | ||
| // flag to set which result we want to compare against | ||
| let (compareDate, compareResult) = try fetchLatestResult() | ||
| let diff = results.compare(with: compareResult) | ||
| let regressions = diff.filter({(_, change) in change.seconds > 0}) | ||
| let improvements = diff.filter({(_, change) in change.seconds < 0}) | ||
|
|
||
| print("Comparing against benchmark done on \(compareDate.formatted(dateStyle))") | ||
| print("=== Regressions ====================================================") | ||
| for item in regressions { | ||
| let oldVal = compareResult.results[item.key]! | ||
| let newVal = results.results[item.key]! | ||
| let percentage = item.value.seconds / oldVal.seconds | ||
| print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") | ||
| } | ||
| print("=== Improvements ====================================================") | ||
| for item in improvements { | ||
| let oldVal = compareResult.results[item.key]! | ||
| let newVal = results.results[item.key]! | ||
| let percentage = item.value.seconds / oldVal.seconds | ||
| print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| struct SuiteResult { | ||
| var results: [String: Time] = [:] | ||
|
|
||
| public mutating func add(name: String, time: Time) { | ||
| results.updateValue(time, forKey: name) | ||
| } | ||
|
|
||
| public func compare(with other: SuiteResult) -> [String: Time] { | ||
| var output: [String: Time] = [:] | ||
| for item in results { | ||
| if let otherVal = other.results[item.key] { | ||
| let diff = item.value - otherVal | ||
| // note: is this enough time difference? | ||
| if diff.abs() > Time.millisecond { | ||
| output.updateValue(diff, forKey: item.key) | ||
| } | ||
| } | ||
| } | ||
| return output | ||
| } | ||
| } | ||
|
|
||
| extension SuiteResult: Codable { | ||
| public func save(to url: URL) throws { | ||
| let encoder = JSONEncoder() | ||
| let data = try encoder.encode(self) | ||
| try data.write(to: url, options: .atomic) | ||
| } | ||
|
|
||
| public static func load(from url: URL) throws -> SuiteResult { | ||
| let decoder = JSONDecoder() | ||
| let data = try Data(contentsOf: url) | ||
| return try decoder.decode(SuiteResult.self, from: data) | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We don't normalize our execution times, and some of them run in under a millisecond total. Should we be doing something relative?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Or we may want to remove this just for now and let the user decide. We're not doing long-term perf tracking with this (at least not yet), but it's very useful for development-cycle perf testing.