|
| 1 | +import Foundation |
| 2 | + |
| 3 | +public struct BenchmarkRunner { |
| 4 | + let suiteName: String |
| 5 | + var suite: [any RegexBenchmark] = [] |
| 6 | + |
| 7 | + let samples: Int |
| 8 | + var results: SuiteResult = SuiteResult() |
| 9 | + |
| 10 | + // Outputting |
| 11 | + let startTime = Date() |
| 12 | + let outputPath: String |
| 13 | + |
| 14 | + public init(_ suiteName: String, _ n: Int, _ outputPath: String) { |
| 15 | + self.suiteName = suiteName |
| 16 | + self.samples = n |
| 17 | + self.outputPath = outputPath |
| 18 | + } |
| 19 | + |
| 20 | + public mutating func register(_ new: some RegexBenchmark) { |
| 21 | + suite.append(new) |
| 22 | + } |
| 23 | + |
| 24 | + mutating func measure(benchmark: some RegexBenchmark) -> Time { |
| 25 | + var times: [Time] = [] |
| 26 | + |
| 27 | + // initial run to make sure the regex has been compiled |
| 28 | + // todo: measure compile times, or at least how much this first run |
| 29 | + // differs from the later ones |
| 30 | + benchmark.run() |
| 31 | + |
| 32 | + // fixme: use suspendingclock? |
| 33 | + for _ in 0..<samples { |
| 34 | + let start = Tick.now |
| 35 | + benchmark.run() |
| 36 | + let end = Tick.now |
| 37 | + let time = end.elapsedTime(since: start) |
| 38 | + times.append(time) |
| 39 | + } |
| 40 | + // todo: compute stdev and warn if it's too large |
| 41 | + |
| 42 | + // return median time |
| 43 | + times.sort() |
| 44 | + let median = times[samples/2] |
| 45 | + self.results.add(name: benchmark.name, time: median) |
| 46 | + return median |
| 47 | + } |
| 48 | + |
| 49 | + public mutating func run() { |
| 50 | + print("Running") |
| 51 | + for b in suite { |
| 52 | + print("- \(b.name) \(measure(benchmark: b))") |
| 53 | + } |
| 54 | + } |
| 55 | + |
| 56 | + public func profile() { |
| 57 | + print("Starting") |
| 58 | + for b in suite { |
| 59 | + print("- \(b.name)") |
| 60 | + b.run() |
| 61 | + print("- done") |
| 62 | + } |
| 63 | + } |
| 64 | + |
| 65 | + public mutating func debug() { |
| 66 | + print("Debugging") |
| 67 | + print("========================") |
| 68 | + for b in suite { |
| 69 | + print("- \(b.name) \(measure(benchmark: b))") |
| 70 | + b.debug() |
| 71 | + print("========================") |
| 72 | + } |
| 73 | + } |
| 74 | +} |
| 75 | + |
| 76 | +extension BenchmarkRunner { |
| 77 | + |
| 78 | +#if _runtime(_ObjC) |
| 79 | + var dateStyle: Date.ISO8601FormatStyle { Date.ISO8601FormatStyle() } |
| 80 | + |
| 81 | + func format(_ date: Date) -> String { |
| 82 | + return dateStyle.format(date) |
| 83 | + } |
| 84 | +#else |
| 85 | + func format(_ date: Date) -> String { |
| 86 | + return date.description |
| 87 | + } |
| 88 | +#endif |
| 89 | + |
| 90 | + var outputFolderUrl: URL { |
| 91 | + let url = URL(fileURLWithPath: outputPath, isDirectory: true) |
| 92 | + if !FileManager.default.fileExists(atPath: url.path) { |
| 93 | + try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true) |
| 94 | + } |
| 95 | + return url |
| 96 | + } |
| 97 | + |
| 98 | + public func save() throws { |
| 99 | + let now = format(startTime) |
| 100 | + let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json") |
| 101 | + print("Saving result to \(resultJsonUrl.path)") |
| 102 | + try results.save(to: resultJsonUrl) |
| 103 | + } |
| 104 | + |
| 105 | + func fetchLatestResult() throws -> (String, SuiteResult) { |
| 106 | +#if _runtime(_ObjC) |
| 107 | + var pastResults: [Date: (String, SuiteResult)] = [:] |
| 108 | + for resultFile in try FileManager.default.contentsOfDirectory( |
| 109 | + at: outputFolderUrl, |
| 110 | + includingPropertiesForKeys: nil |
| 111 | + ) { |
| 112 | + do { |
| 113 | + let dateString = resultFile.lastPathComponent.replacingOccurrences( |
| 114 | + of: "-result.json", |
| 115 | + with: "") |
| 116 | + let date = try dateStyle.parse(dateString) |
| 117 | + let result = try SuiteResult.load(from: resultFile) |
| 118 | + pastResults.updateValue((resultFile.lastPathComponent, result), forKey: date) |
| 119 | + } catch { |
| 120 | + print("Warning: Found invalid result file \(resultFile.lastPathComponent) in results directory, skipping") |
| 121 | + } |
| 122 | + } |
| 123 | + |
| 124 | + let sorted = pastResults |
| 125 | + .sorted(by: {(kv1,kv2) in kv1.0 > kv2.0}) |
| 126 | + return sorted[0].1 |
| 127 | +#else |
| 128 | + // corelibs-foundation lacks Date.FormatStyle entirely, so we don't have |
| 129 | + // any way of parsing the dates. So use the filename sorting to pick out the |
| 130 | + // latest one... this sucks |
| 131 | + let items = try FileManager.default.contentsOfDirectory( |
| 132 | + at: outputFolderUrl, |
| 133 | + includingPropertiesForKeys: nil |
| 134 | + ) |
| 135 | + let resultFile = items[items.count - 1] |
| 136 | + let pastResult = try SuiteResult.load(from: resultFile) |
| 137 | + return (resultFile.lastPathComponent, pastResult) |
| 138 | +#endif |
| 139 | + } |
| 140 | + |
| 141 | + public func compare(against: String?) throws { |
| 142 | + let compareFile: String |
| 143 | + let compareResult: SuiteResult |
| 144 | + |
| 145 | + if let compareFilePath = against { |
| 146 | + let compareFileURL = URL(fileURLWithPath: compareFilePath) |
| 147 | + compareResult = try SuiteResult.load(from: compareFileURL) |
| 148 | + compareFile = compareFileURL.lastPathComponent |
| 149 | + } else { |
| 150 | + (compareFile, compareResult) = try fetchLatestResult() |
| 151 | + } |
| 152 | + |
| 153 | + let diff = results.compare(with: compareResult) |
| 154 | + let regressions = diff.filter({(_, change) in change.seconds > 0}) |
| 155 | + let improvements = diff.filter({(_, change) in change.seconds < 0}) |
| 156 | + |
| 157 | + print("Comparing against benchmark result file \(compareFile)") |
| 158 | + print("=== Regressions ====================================================") |
| 159 | + for item in regressions { |
| 160 | + let oldVal = compareResult.results[item.key]! |
| 161 | + let newVal = results.results[item.key]! |
| 162 | + let percentage = item.value.seconds / oldVal.seconds |
| 163 | + print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") |
| 164 | + } |
| 165 | + print("=== Improvements ====================================================") |
| 166 | + for item in improvements { |
| 167 | + let oldVal = compareResult.results[item.key]! |
| 168 | + let newVal = results.results[item.key]! |
| 169 | + let percentage = item.value.seconds / oldVal.seconds |
| 170 | + print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%") |
| 171 | + } |
| 172 | + } |
| 173 | +} |
| 174 | + |
| 175 | +struct SuiteResult { |
| 176 | + var results: [String: Time] = [:] |
| 177 | + |
| 178 | + public mutating func add(name: String, time: Time) { |
| 179 | + results.updateValue(time, forKey: name) |
| 180 | + } |
| 181 | + |
| 182 | + public func compare(with other: SuiteResult) -> [String: Time] { |
| 183 | + var output: [String: Time] = [:] |
| 184 | + for item in results { |
| 185 | + if let otherVal = other.results[item.key] { |
| 186 | + let diff = item.value - otherVal |
| 187 | + if abs(100 * diff.seconds / otherVal.seconds) > 0.5 { |
| 188 | + output.updateValue(diff, forKey: item.key) |
| 189 | + } |
| 190 | + } |
| 191 | + } |
| 192 | + return output |
| 193 | + } |
| 194 | +} |
| 195 | + |
| 196 | +extension SuiteResult: Codable { |
| 197 | + public func save(to url: URL) throws { |
| 198 | + let encoder = JSONEncoder() |
| 199 | + let data = try encoder.encode(self) |
| 200 | + try data.write(to: url, options: .atomic) |
| 201 | + } |
| 202 | + |
| 203 | + public static func load(from url: URL) throws -> SuiteResult { |
| 204 | + let decoder = JSONDecoder() |
| 205 | + let data = try Data(contentsOf: url) |
| 206 | + return try decoder.decode(SuiteResult.self, from: data) |
| 207 | + } |
| 208 | +} |
0 commit comments