Skip to content

Add more benchmarks and benchmarker functionality #505

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jun 21, 2022
Merged
65 changes: 3 additions & 62 deletions Sources/RegexBenchmark/Benchmark.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import Foundation
public protocol RegexBenchmark {
var name: String { get }
func run()
func debug()
}

public struct Benchmark: RegexBenchmark {
public let name: String
let regex: Regex<Substring>
let regex: Regex<AnyRegexOutput>
let type: MatchType
let target: String

Expand Down Expand Up @@ -50,66 +51,6 @@ public struct NSBenchmark: RegexBenchmark {
}
}

public struct BenchmarkRunner {
// Register instances of Benchmark and run them
let suiteName: String
var suite: [any RegexBenchmark]
let samples: Int

public init(_ suiteName: String) {
self.suiteName = suiteName
self.suite = []
self.samples = 20
}

public init(_ suiteName: String, _ n: Int) {
self.suiteName = suiteName
self.suite = []
self.samples = n
}

public mutating func register(_ new: some RegexBenchmark) {
suite.append(new)
}

func measure(benchmark: some RegexBenchmark) -> Time {
var times: [Time] = []

// initial run to make sure the regex has been compiled
benchmark.run()

// fixme: use suspendingclock?
for _ in 0..<samples {
let start = Tick.now
benchmark.run()
let end = Tick.now
let time = end.elapsedTime(since: start)
times.append(time)
}
// todo: compute stdev and warn if it's too large

// return median time
times.sort()
return times[samples/2]
}

public func run() {
print("Running")
for b in suite {
print("- \(b.name) \(measure(benchmark: b))")
}
}

public func profile() {
print("Starting")
for b in suite {
print("- \(b.name)")
b.run()
print("- done")
}
}
}

/// A benchmark meant to be ran across multiple engines
struct CrossBenchmark {
/// The base name of the benchmark
Expand All @@ -130,7 +71,7 @@ struct CrossBenchmark {
var isWhole: Bool = false

func register(_ runner: inout BenchmarkRunner) {
let swiftRegex = try! Regex(regex, as: Substring.self)
let swiftRegex = try! Regex(regex)

let nsPattern = isWhole ? "^" + regex + "$" : regex
let nsRegex: NSRegularExpression
Expand Down
178 changes: 178 additions & 0 deletions Sources/RegexBenchmark/BenchmarkRunner.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
import Foundation

public struct BenchmarkRunner {
let suiteName: String
var suite: [any RegexBenchmark] = []
let samples: Int
var results: SuiteResult = SuiteResult()

// Outputting
let startTime = Date()
let outputPath: String

public init(_ suiteName: String, _ n: Int, _ outputPath: String) {
self.suiteName = suiteName
self.samples = n
self.outputPath = outputPath
}

public mutating func register(_ new: some RegexBenchmark) {
suite.append(new)
}

mutating func measure(benchmark: some RegexBenchmark) -> Time {
var times: [Time] = []

// initial run to make sure the regex has been compiled
// todo: measure compile times, or at least how much this first run
// differs from the later ones
benchmark.run()

// fixme: use suspendingclock?
for _ in 0..<samples {
let start = Tick.now
benchmark.run()
let end = Tick.now
let time = end.elapsedTime(since: start)
times.append(time)
}
// todo: compute stdev and warn if it's too large

// return median time
times.sort()
let median = times[samples/2]
self.results.add(name: benchmark.name, time: median)
return median
}

public mutating func run() {
print("Running")
for b in suite {
print("- \(b.name) \(measure(benchmark: b))")
}
}

public func profile() {
print("Starting")
for b in suite {
print("- \(b.name)")
b.run()
print("- done")
}
}

public mutating func debug() {
print("Debugging")
print("========================")
for b in suite {
print("- \(b.name) \(measure(benchmark: b))")
b.debug()
print("========================")
}
}
}

extension BenchmarkRunner {
var dateStyle: Date.FormatStyle {
Date.FormatStyle()
.year(.twoDigits)
.month(.twoDigits)
.day(.twoDigits)
.hour(.twoDigits(amPM: .omitted))
.minute(.twoDigits)
}

var outputFolderUrl: URL {
let url = URL(fileURLWithPath: outputPath, isDirectory: true)
if !FileManager.default.fileExists(atPath: url.path) {
try! FileManager.default.createDirectory(atPath: url.path, withIntermediateDirectories: true)
}
return url
}

public func save() throws {
let now = startTime.formatted(dateStyle)
let resultJsonUrl = outputFolderUrl.appendingPathComponent(now + "-result.json")
print("Saving result to \(resultJsonUrl.path)")
try results.save(to: resultJsonUrl)
}

func fetchLatestResult() throws -> (Date, SuiteResult) {
var pastResults: [Date: SuiteResult] = [:]
for resultFile in try FileManager.default.contentsOfDirectory(
at: outputFolderUrl,
includingPropertiesForKeys: nil
) {
let dateString = resultFile.lastPathComponent.replacingOccurrences(
of: "-result.json",
with: "")
let date = try dateStyle.parse(dateString)
pastResults.updateValue(try SuiteResult.load(from: resultFile), forKey: date)
}

let sorted = pastResults
.sorted(by: {(kv1,kv2) in kv1.0 > kv2.0})
return sorted[0]
}

public func compare() throws {
// It just compares by the latest result for now, we probably want a CLI
// flag to set which result we want to compare against
let (compareDate, compareResult) = try fetchLatestResult()
let diff = results.compare(with: compareResult)
let regressions = diff.filter({(_, change) in change.seconds > 0})
let improvements = diff.filter({(_, change) in change.seconds < 0})

print("Comparing against benchmark done on \(compareDate.formatted(dateStyle))")
print("=== Regressions ====================================================")
for item in regressions {
let oldVal = compareResult.results[item.key]!
let newVal = results.results[item.key]!
let percentage = item.value.seconds / oldVal.seconds
print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%")
}
print("=== Improvements ====================================================")
for item in improvements {
let oldVal = compareResult.results[item.key]!
let newVal = results.results[item.key]!
let percentage = item.value.seconds / oldVal.seconds
print("- \(item.key)\t\t\(newVal)\t\(oldVal)\t\(item.value)\t\((percentage * 100).rounded())%")
}
}
}

struct SuiteResult {
var results: [String: Time] = [:]

public mutating func add(name: String, time: Time) {
results.updateValue(time, forKey: name)
}

public func compare(with other: SuiteResult) -> [String: Time] {
var output: [String: Time] = [:]
for item in results {
if let otherVal = other.results[item.key] {
let diff = item.value - otherVal
// note: is this enough time difference?
if diff.abs() > Time.millisecond {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't normalize our execution times, and some of them run in under a millisecond total. Should we be doing something relative?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or we may want to remove this just for now and let the user decide. We're not doing long-term perf tracking with this (at least not yet), but it's very useful for development-cycle perf testing.

output.updateValue(diff, forKey: item.key)
}
}
}
return output
}
}

extension SuiteResult: Codable {
public func save(to url: URL) throws {
let encoder = JSONEncoder()
let data = try encoder.encode(self)
try data.write(to: url, options: .atomic)
}

public static func load(from url: URL) throws -> SuiteResult {
let decoder = JSONDecoder()
let data = try Data(contentsOf: url)
return try decoder.decode(SuiteResult.self, from: data)
}
}
33 changes: 28 additions & 5 deletions Sources/RegexBenchmark/CLI.swift
Original file line number Diff line number Diff line change
Expand Up @@ -5,30 +5,53 @@ struct Runner: ParsableCommand {
@Argument(help: "Names of benchmarks to run")
var specificBenchmarks: [String] = []

@Option(help: "Run only once for profiling purposes")
@Flag(help: "Run only once for profiling purposes")
var profile = false

@Option(help: "How many samples to collect for each benchmark")
var samples = 20

@Flag(help: "Debug benchmark regexes")
var debug = false

@Option(help: "Output folder")
var outputPath = "./results/"

@Flag(help: "Should the results be saved")
var save = false

@Flag(help: "Compare this result with the latest saved result")
var compare = false

func makeRunner() -> BenchmarkRunner {
var benchmark = BenchmarkRunner("RegexBench", samples)
var benchmark = BenchmarkRunner("RegexBench", samples, outputPath)
benchmark.addReluctantQuant()
benchmark.addCSS()
benchmark.addNotFound()
benchmark.addGraphemeBreak()
benchmark.addHangulSyllable()
benchmark.addHTML()
benchmark.addEmail()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any way that the benchmarks can add themselves instead of having a second listing?

return benchmark
}

mutating func run() throws {
var runner = makeRunner()
if !self.specificBenchmarks.isEmpty {
runner.suite = runner.suite.filter { b in specificBenchmarks.contains(b.name) }
}
if profile {
runner.profile()
} else {
switch (profile, debug) {
case (true, true): print("Cannot run both profile and debug")
case (true, false): runner.profile()
case (false, true): runner.debug()
case (false, false):
runner.run()
if compare {
try runner.compare()
}
if save {
try runner.save()
}
}
}
}
Loading