Skip to content

[5.7] Merge benchmarker improvements and character class bitset optimization #532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 76 additions & 61 deletions Sources/RegexBenchmark/Benchmark.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ import Foundation
public protocol RegexBenchmark {
var name: String { get }
func run()
func debug()
}

public struct Benchmark: RegexBenchmark {
public let name: String
let regex: Regex<Substring>
let ty: MatchType
let regex: Regex<AnyRegexOutput>
let type: MatchType
let target: String

public enum MatchType {
Expand All @@ -19,7 +20,7 @@ public struct Benchmark: RegexBenchmark {
}

public func run() {
switch ty {
switch type {
case .whole: blackHole(target.wholeMatch(of: regex))
case .allMatches: blackHole(target.matches(of: regex))
case .first: blackHole(target.firstMatch(of: regex))
Expand All @@ -30,86 +31,100 @@ public struct Benchmark: RegexBenchmark {
public struct NSBenchmark: RegexBenchmark {
public let name: String
let regex: NSRegularExpression
let ty: NSMatchType
let type: NSMatchType
let target: String

var range: NSRange {
NSRange(target.startIndex..<target.endIndex, in: target)
}

public enum NSMatchType {
case all
case allMatches
case first
}

public func run() {
switch ty {
case .all: blackHole(regex.matches(in: target, range: range))
switch type {
case .allMatches: blackHole(regex.matches(in: target, range: range))
case .first: blackHole(regex.firstMatch(in: target, range: range))
}
}
}

public struct BenchmarkRunner {
// Register instances of Benchmark and run them
let suiteName: String
var suite: [any RegexBenchmark]
let samples: Int

public init(_ suiteName: String) {
self.suiteName = suiteName
self.suite = []
self.samples = 20
}

public init(_ suiteName: String, _ n: Int) {
self.suiteName = suiteName
self.suite = []
self.samples = n
}
/// A benchmark meant to be ran across multiple engines
struct CrossBenchmark {
/// The base name of the benchmark
var baseName: String

public mutating func register(_ new: some RegexBenchmark) {
suite.append(new)
}

func measure(benchmark: some RegexBenchmark) -> Time {
var times: [Time] = []

// initial run to make sure the regex has been compiled
benchmark.run()

// fixme: use suspendingclock?
for _ in 0..<samples {
let start = Tick.now
benchmark.run()
let end = Tick.now
let time = end.elapsedTime(since: start)
times.append(time)
}
// todo: compute stdev and warn if it's too large

// return median time
times.sort()
return times[samples/2]
}

public func run() {
print("Running")
for b in suite {
print("- \(b.name) \(measure(benchmark: b))")
/// The string to compile in differnet engines
var regex: String

/// The text to search
var input: String

// TODO: var output, for validation

/// Whether this is whole string matching or a searching benchmark
///
/// TODO: Probably better ot have a whole-line vs search anywhere, maybe
/// accomodate multi-line matching, etc.
var isWhole: Bool = false

func register(_ runner: inout BenchmarkRunner) {
let swiftRegex = try! Regex(regex)

let nsPattern = isWhole ? "^" + regex + "$" : regex
let nsRegex: NSRegularExpression
if isWhole {
nsRegex = try! NSRegularExpression(pattern: "^" + regex + "$")
} else {
nsRegex = try! NSRegularExpression(pattern: regex)
}
}

public func profile() {
print("Starting")
for b in suite {
print("- \(b.name)")
b.run()
print("- done")

if isWhole {
runner.register(
Benchmark(
name: baseName + "Whole",
regex: swiftRegex,
type: .whole,
target: input))
runner.register(
NSBenchmark(
name: baseName + "Whole_NS",
regex: nsRegex,
type: .first,
target: input))
} else {
runner.register(
Benchmark(
name: baseName + "First",
regex: swiftRegex,
type: .first,
target: input))
runner.register(
Benchmark(
name: baseName + "All",
regex: swiftRegex,
type: .allMatches,
target: input))
runner.register(
NSBenchmark(
name: baseName + "First_NS",
regex: nsRegex,
type: .first,
target: input))
runner.register(
NSBenchmark(
name: baseName + "All_NS",
regex: nsRegex,
type: .allMatches,
target: input))
}
}
}

// TODO: Capture-containing benchmarks

// nom nom nom, consume the argument
@inline(never)
public func blackHole<T>(_ x: T) {
Expand Down
22 changes: 22 additions & 0 deletions Sources/RegexBenchmark/BenchmarkRegistration.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// This file has lines generated by createBenchmark.py
// Do not remove the start of registration or end of registration markers

extension BenchmarkRunner {
public static func makeRunner(
_ samples: Int,
_ outputPath: String
) -> BenchmarkRunner {
var benchmark = BenchmarkRunner("RegexBench", samples, outputPath)
// -- start of registrations --
benchmark.addReluctantQuant()
benchmark.addCSS()
benchmark.addNotFound()
benchmark.addGraphemeBreak()
benchmark.addHangulSyllable()
benchmark.addHTML()
benchmark.addEmail()
benchmark.addCustomCharacterClasses()
// -- end of registrations --
return benchmark
}
}
Loading