Skip to content

Commit f670112

Browse files
[llvm-exegesis] Add support for validation counters (#76653)
This patch adds support for validation counters. Validation counters can be used to measure events that occur during snippet execution like cache misses to ensure that certain assumed invariants about the benchmark actually hold. Validation counters are setup within a perf event group, so are turned on and off at exactly the same time as the "group leader" counter that measures the desired value.
1 parent 4619147 commit f670112

19 files changed

+333
-102
lines changed
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# REQUIRES: exegesis-can-measure-latency, exegesis-can-measure-uops, x86_64-linux
2+
3+
# Check that when specifying validation counters, the validation counter is
4+
# collected and the information is displayed in the output. Test across
5+
# multiple configurations that need to be wired up separately for validation
6+
# counter support.
7+
8+
# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -opcode-name=ADD64rr --validation-counter=instructions-retired | FileCheck %s
9+
# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -opcode-name=ADD64rr --validation-counter=instructions-retired -execution-mode=subprocess | FileCheck %s
10+
# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=uops -opcode-name=ADD64rr --validation-counter=instructions-retired -execution-mode=subprocess | FileCheck %s
11+
12+
# CHECK: instructions-retired: {{[0-9]+}}

llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "llvm/ADT/StringRef.h"
1515
#include "llvm/ADT/bit.h"
1616
#include "llvm/ObjectYAML/YAML.h"
17+
#include "llvm/Support/Errc.h"
1718
#include "llvm/Support/FileOutputBuffer.h"
1819
#include "llvm/Support/FileSystem.h"
1920
#include "llvm/Support/Format.h"
@@ -192,6 +193,41 @@ template <> struct SequenceElementTraits<exegesis::BenchmarkMeasure> {
192193
static const bool flow = false;
193194
};
194195

196+
const char *validationEventToString(exegesis::ValidationEvent VE) {
197+
switch (VE) {
198+
case exegesis::ValidationEvent::InstructionRetired:
199+
return "instructions-retired";
200+
}
201+
}
202+
203+
Expected<exegesis::ValidationEvent> stringToValidationEvent(StringRef Input) {
204+
if (Input == "instructions-retired")
205+
return exegesis::ValidationEvent::InstructionRetired;
206+
else
207+
return make_error<StringError>("Invalid validation event string",
208+
errc::invalid_argument);
209+
}
210+
211+
template <>
212+
struct CustomMappingTraits<std::map<exegesis::ValidationEvent, int64_t>> {
213+
static void inputOne(IO &Io, StringRef KeyStr,
214+
std::map<exegesis::ValidationEvent, int64_t> &VI) {
215+
Expected<exegesis::ValidationEvent> Key = stringToValidationEvent(KeyStr);
216+
if (!Key) {
217+
Io.setError("Key is not a valid validation event");
218+
return;
219+
}
220+
Io.mapRequired(KeyStr.str().c_str(), VI[*Key]);
221+
}
222+
223+
static void output(IO &Io, std::map<exegesis::ValidationEvent, int64_t> &VI) {
224+
for (auto &IndividualVI : VI) {
225+
Io.mapRequired(validationEventToString(IndividualVI.first),
226+
IndividualVI.second);
227+
}
228+
}
229+
};
230+
195231
// exegesis::Measure is rendererd as a flow instead of a list.
196232
// e.g. { "key": "the key", "value": 0123 }
197233
template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
@@ -203,6 +239,7 @@ template <> struct MappingTraits<exegesis::BenchmarkMeasure> {
203239
}
204240
Io.mapRequired("value", Obj.PerInstructionValue);
205241
Io.mapOptional("per_snippet_value", Obj.PerSnippetValue);
242+
Io.mapOptional("validation_counters", Obj.ValidationCounters);
206243
}
207244
static const bool flow = true;
208245
};

llvm/tools/llvm-exegesis/lib/BenchmarkResult.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ class Error;
3232

3333
namespace exegesis {
3434

35+
enum ValidationEvent { InstructionRetired };
36+
3537
enum class BenchmarkPhaseSelectorE {
3638
PrepareSnippet,
3739
PrepareAndAssembleSnippet,
@@ -77,8 +79,10 @@ struct BenchmarkKey {
7779

7880
struct BenchmarkMeasure {
7981
// A helper to create an unscaled BenchmarkMeasure.
80-
static BenchmarkMeasure Create(std::string Key, double Value) {
81-
return {Key, Value, Value};
82+
static BenchmarkMeasure
83+
Create(std::string Key, double Value,
84+
std::map<ValidationEvent, int64_t> ValCounters) {
85+
return {Key, Value, Value, ValCounters};
8286
}
8387
std::string Key;
8488
// This is the per-instruction value, i.e. measured quantity scaled per
@@ -87,6 +91,8 @@ struct BenchmarkMeasure {
8791
// This is the per-snippet value, i.e. measured quantity for one repetition of
8892
// the whole snippet.
8993
double PerSnippetValue;
94+
// These are the validation counter values.
95+
std::map<ValidationEvent, int64_t> ValidationCounters;
9096
};
9197

9298
// The result of an instruction benchmark.

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp

Lines changed: 69 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,11 @@ namespace exegesis {
5454

5555
BenchmarkRunner::BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
5656
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
57-
ExecutionModeE ExecutionMode)
57+
ExecutionModeE ExecutionMode,
58+
ArrayRef<ValidationEvent> ValCounters)
5859
: State(State), Mode(Mode), BenchmarkPhaseSelector(BenchmarkPhaseSelector),
59-
ExecutionMode(ExecutionMode), Scratch(std::make_unique<ScratchSpace>()) {}
60+
ExecutionMode(ExecutionMode), ValidationCounters(ValCounters),
61+
Scratch(std::make_unique<ScratchSpace>()) {}
6062

6163
BenchmarkRunner::~BenchmarkRunner() = default;
6264

@@ -71,16 +73,18 @@ void BenchmarkRunner::FunctionExecutor::accumulateCounterValues(
7173
}
7274

7375
Expected<llvm::SmallVector<int64_t, 4>>
74-
BenchmarkRunner::FunctionExecutor::runAndSample(const char *Counters) const {
76+
BenchmarkRunner::FunctionExecutor::runAndSample(
77+
const char *Counters, ArrayRef<const char *> ValidationCounters,
78+
SmallVectorImpl<int64_t> &ValidationCounterValues) const {
7579
// We sum counts when there are several counters for a single ProcRes
7680
// (e.g. P23 on SandyBridge).
7781
llvm::SmallVector<int64_t, 4> CounterValues;
7882
SmallVector<StringRef, 2> CounterNames;
7983
StringRef(Counters).split(CounterNames, '+');
8084
for (auto &CounterName : CounterNames) {
8185
CounterName = CounterName.trim();
82-
Expected<SmallVector<int64_t, 4>> ValueOrError =
83-
runWithCounter(CounterName);
86+
Expected<SmallVector<int64_t, 4>> ValueOrError = runWithCounter(
87+
CounterName, ValidationCounters, ValidationCounterValues);
8488
if (!ValueOrError)
8589
return ValueOrError.takeError();
8690
accumulateCounterValues(ValueOrError.get(), &CounterValues);
@@ -120,11 +124,13 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
120124
(*Result)[I] += NewValues[I];
121125
}
122126

123-
Expected<llvm::SmallVector<int64_t, 4>>
124-
runWithCounter(StringRef CounterName) const override {
127+
Expected<llvm::SmallVector<int64_t, 4>> runWithCounter(
128+
StringRef CounterName, ArrayRef<const char *> ValidationCounters,
129+
SmallVectorImpl<int64_t> &ValidationCounterValues) const override {
125130
const ExegesisTarget &ET = State.getExegesisTarget();
126131
char *const ScratchPtr = Scratch->ptr();
127-
auto CounterOrError = ET.createCounter(CounterName, State);
132+
auto CounterOrError =
133+
ET.createCounter(CounterName, State, ValidationCounters);
128134

129135
if (!CounterOrError)
130136
return CounterOrError.takeError();
@@ -156,6 +162,14 @@ class InProcessFunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
156162
}
157163
}
158164

165+
auto ValidationValuesOrErr = Counter->readValidationCountersOrError();
166+
if (!ValidationValuesOrErr)
167+
return ValidationValuesOrErr.takeError();
168+
169+
ArrayRef RealValidationValues = *ValidationValuesOrErr;
170+
for (size_t I = 0; I < RealValidationValues.size(); ++I)
171+
ValidationCounterValues[I] = RealValidationValues[I];
172+
159173
return Counter->readOrError(Function.getFunctionBytes());
160174
}
161175

@@ -266,7 +280,9 @@ class SubProcessFunctionExecutorImpl
266280
}
267281

268282
Error createSubProcessAndRunBenchmark(
269-
StringRef CounterName, SmallVectorImpl<int64_t> &CounterValues) const {
283+
StringRef CounterName, SmallVectorImpl<int64_t> &CounterValues,
284+
ArrayRef<const char *> ValidationCounters,
285+
SmallVectorImpl<int64_t> &ValidationCounterValues) const {
270286
int PipeFiles[2];
271287
int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles);
272288
if (PipeSuccessOrErr != 0) {
@@ -306,8 +322,8 @@ class SubProcessFunctionExecutorImpl
306322
}
307323

308324
const ExegesisTarget &ET = State.getExegesisTarget();
309-
auto CounterOrError =
310-
ET.createCounter(CounterName, State, ParentOrChildPID);
325+
auto CounterOrError = ET.createCounter(
326+
CounterName, State, ValidationCounters, ParentOrChildPID);
311327

312328
if (!CounterOrError)
313329
return CounterOrError.takeError();
@@ -362,6 +378,14 @@ class SubProcessFunctionExecutorImpl
362378
return CounterValueOrErr.takeError();
363379
CounterValues = std::move(*CounterValueOrErr);
364380

381+
auto ValidationValuesOrErr = Counter->readValidationCountersOrError();
382+
if (!ValidationValuesOrErr)
383+
return ValidationValuesOrErr.takeError();
384+
385+
ArrayRef RealValidationValues = *ValidationValuesOrErr;
386+
for (size_t I = 0; I < RealValidationValues.size(); ++I)
387+
ValidationCounterValues[I] = RealValidationValues[I];
388+
365389
return Error::success();
366390
}
367391
// The child exited, but not successfully
@@ -460,15 +484,15 @@ class SubProcessFunctionExecutorImpl
460484
exit(0);
461485
}
462486

463-
Expected<llvm::SmallVector<int64_t, 4>>
464-
runWithCounter(StringRef CounterName) const override {
487+
Expected<llvm::SmallVector<int64_t, 4>> runWithCounter(
488+
StringRef CounterName, ArrayRef<const char *> ValidationCounters,
489+
SmallVectorImpl<int64_t> &ValidationCounterValues) const override {
465490
SmallVector<int64_t, 4> Value(1, 0);
466-
Error PossibleBenchmarkError =
467-
createSubProcessAndRunBenchmark(CounterName, Value);
491+
Error PossibleBenchmarkError = createSubProcessAndRunBenchmark(
492+
CounterName, Value, ValidationCounters, ValidationCounterValues);
468493

469-
if (PossibleBenchmarkError) {
494+
if (PossibleBenchmarkError)
470495
return std::move(PossibleBenchmarkError);
471-
}
472496

473497
return Value;
474498
}
@@ -642,6 +666,34 @@ BenchmarkRunner::writeObjectFile(StringRef Buffer, StringRef FileName) const {
642666
return std::string(ResultPath);
643667
}
644668

669+
static bool EventLessThan(const std::pair<ValidationEvent, const char *> LHS,
670+
const ValidationEvent RHS) {
671+
return static_cast<int>(LHS.first) < static_cast<int>(RHS);
672+
}
673+
674+
Error BenchmarkRunner::getValidationCountersToRun(
675+
SmallVector<const char *> &ValCountersToRun) const {
676+
const PfmCountersInfo &PCI = State.getPfmCounters();
677+
ValCountersToRun.reserve(ValidationCounters.size());
678+
679+
ValCountersToRun.reserve(ValidationCounters.size());
680+
ArrayRef TargetValidationEvents(PCI.ValidationEvents,
681+
PCI.NumValidationEvents);
682+
for (const ValidationEvent RequestedValEvent : ValidationCounters) {
683+
auto ValCounterIt =
684+
lower_bound(TargetValidationEvents, RequestedValEvent, EventLessThan);
685+
if (ValCounterIt == TargetValidationEvents.end() ||
686+
ValCounterIt->first != RequestedValEvent)
687+
return make_error<Failure>("Cannot create validation counter");
688+
689+
assert(ValCounterIt->first == RequestedValEvent &&
690+
"The array of validation events from the target should be sorted");
691+
ValCountersToRun.push_back(ValCounterIt->second);
692+
}
693+
694+
return Error::success();
695+
}
696+
645697
BenchmarkRunner::FunctionExecutor::~FunctionExecutor() {}
646698

647699
} // namespace exegesis

llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ class BenchmarkRunner {
3838

3939
explicit BenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
4040
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
41-
ExecutionModeE ExecutionMode);
41+
ExecutionModeE ExecutionMode,
42+
ArrayRef<ValidationEvent> ValCounters);
4243

4344
virtual ~BenchmarkRunner();
4445

@@ -93,14 +94,18 @@ class BenchmarkRunner {
9394
virtual ~FunctionExecutor();
9495

9596
Expected<llvm::SmallVector<int64_t, 4>>
96-
runAndSample(const char *Counters) const;
97+
runAndSample(const char *Counters,
98+
ArrayRef<const char *> ValidationCounters,
99+
SmallVectorImpl<int64_t> &ValidationCounterValues) const;
97100

98101
protected:
99102
static void
100103
accumulateCounterValues(const llvm::SmallVectorImpl<int64_t> &NewValues,
101104
llvm::SmallVectorImpl<int64_t> *Result);
102105
virtual Expected<llvm::SmallVector<int64_t, 4>>
103-
runWithCounter(StringRef CounterName) const = 0;
106+
runWithCounter(StringRef CounterName,
107+
ArrayRef<const char *> ValidationCounters,
108+
SmallVectorImpl<int64_t> &ValidationCounterValues) const = 0;
104109
};
105110

106111
protected:
@@ -109,6 +114,11 @@ class BenchmarkRunner {
109114
const BenchmarkPhaseSelectorE BenchmarkPhaseSelector;
110115
const ExecutionModeE ExecutionMode;
111116

117+
SmallVector<ValidationEvent> ValidationCounters;
118+
119+
Error
120+
getValidationCountersToRun(SmallVector<const char *> &ValCountersToRun) const;
121+
112122
private:
113123
virtual Expected<std::vector<BenchmarkMeasure>>
114124
runMeasurements(const FunctionExecutor &Executor) const = 0;

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ LatencyBenchmarkRunner::LatencyBenchmarkRunner(
2222
const LLVMState &State, Benchmark::ModeE Mode,
2323
BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
2424
Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode,
25-
unsigned BenchmarkRepeatCount)
26-
: BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
25+
ArrayRef<ValidationEvent> ValCounters, unsigned BenchmarkRepeatCount)
26+
: BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode,
27+
ValCounters) {
2728
assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
2829
"invalid mode");
2930
ResultAggMode = ResultAgg;
@@ -72,11 +73,21 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
7273
// ResultAggMode.
7374
llvm::SmallVector<int64_t, 4> AccumulatedValues;
7475
double MinVariance = std::numeric_limits<double>::infinity();
75-
const char *CounterName = State.getPfmCounters().CycleCounter;
76+
const PfmCountersInfo &PCI = State.getPfmCounters();
77+
const char *CounterName = PCI.CycleCounter;
78+
79+
SmallVector<const char *> ValCountersToRun;
80+
Error ValCounterErr = getValidationCountersToRun(ValCountersToRun);
81+
if (ValCounterErr)
82+
return std::move(ValCounterErr);
83+
84+
SmallVector<int64_t> ValCounterValues(ValCountersToRun.size(), 0);
7685
// Values count for each run.
7786
int ValuesCount = 0;
7887
for (size_t I = 0; I < NumMeasurements; ++I) {
79-
auto ExpectedCounterValues = Executor.runAndSample(CounterName);
88+
SmallVector<int64_t> IterationValCounterValues(ValCountersToRun.size(), -1);
89+
auto ExpectedCounterValues = Executor.runAndSample(
90+
CounterName, ValCountersToRun, IterationValCounterValues);
8091
if (!ExpectedCounterValues)
8192
return ExpectedCounterValues.takeError();
8293
ValuesCount = ExpectedCounterValues.get().size();
@@ -90,8 +101,15 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
90101
MinVariance = Variance;
91102
}
92103
}
104+
105+
for (size_t I = 0; I < ValCounterValues.size(); ++I)
106+
ValCounterValues[I] += IterationValCounterValues[I];
93107
}
94108

109+
std::map<ValidationEvent, int64_t> ValidationInfo;
110+
for (size_t I = 0; I < ValidationCounters.size(); ++I)
111+
ValidationInfo[ValidationCounters[I]] = ValCounterValues[I];
112+
95113
std::string ModeName;
96114
switch (Mode) {
97115
case Benchmark::Latency:
@@ -112,25 +130,26 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
112130
std::vector<BenchmarkMeasure> Result;
113131
Result.reserve(AccumulatedValues.size());
114132
for (const int64_t Value : AccumulatedValues)
115-
Result.push_back(BenchmarkMeasure::Create(ModeName, Value));
133+
Result.push_back(
134+
BenchmarkMeasure::Create(ModeName, Value, ValidationInfo));
116135
return std::move(Result);
117136
}
118137
case Benchmark::Min: {
119138
std::vector<BenchmarkMeasure> Result;
120-
Result.push_back(
121-
BenchmarkMeasure::Create(ModeName, findMin(AccumulatedValues)));
139+
Result.push_back(BenchmarkMeasure::Create(
140+
ModeName, findMin(AccumulatedValues), ValidationInfo));
122141
return std::move(Result);
123142
}
124143
case Benchmark::Max: {
125144
std::vector<BenchmarkMeasure> Result;
126-
Result.push_back(
127-
BenchmarkMeasure::Create(ModeName, findMax(AccumulatedValues)));
145+
Result.push_back(BenchmarkMeasure::Create(
146+
ModeName, findMax(AccumulatedValues), ValidationInfo));
128147
return std::move(Result);
129148
}
130149
case Benchmark::Mean: {
131150
std::vector<BenchmarkMeasure> Result;
132-
Result.push_back(
133-
BenchmarkMeasure::Create(ModeName, findMean(AccumulatedValues)));
151+
Result.push_back(BenchmarkMeasure::Create(
152+
ModeName, findMean(AccumulatedValues), ValidationInfo));
134153
return std::move(Result);
135154
}
136155
}

0 commit comments

Comments
 (0)