@@ -38,76 +38,127 @@ static const AnalyzerOptions DefaultOpts = [] {
38
38
39
39
// Remember to update the tests in this file when these values change.
40
40
// Also update the doc comment of `interpretQueryResult`.
41
- assert (Config.Z3CrosscheckRLimitThreshold == 400'000 );
42
- assert (Config.Z3CrosscheckTimeoutThreshold == 300_ms );
41
+ assert (Config.Z3CrosscheckRLimitThreshold == 0 );
42
+ assert (Config.Z3CrosscheckTimeoutThreshold == 15 ' 000_ms );
43
43
// Usually, when the timeout/rlimit threshold is reached, Z3 only slightly
44
44
// overshoots until it realizes that it overshoot and needs to back off.
45
45
// Consequently, the measured timeout should be fairly close to the threshold.
46
46
// Same reasoning applies to the rlimit too.
47
47
return Config;
48
48
}();
49
49
50
+ static const AnalyzerOptions LimitedOpts = [] {
51
+ AnalyzerOptions Config = DefaultOpts;
52
+ Config.Z3CrosscheckEQClassTimeoutThreshold = 700_ms;
53
+ Config.Z3CrosscheckTimeoutThreshold = 300_step;
54
+ Config.Z3CrosscheckRLimitThreshold = 400' 000_step;
55
+ return Config;
56
+ }();
57
+
50
58
namespace {
51
59
60
+ template <const AnalyzerOptions &Opts>
52
61
class Z3CrosscheckOracleTest : public testing ::Test {
53
62
public:
54
63
Z3Decision interpretQueryResult (const Z3Result &Result) {
55
64
return Oracle.interpretQueryResult (Result);
56
65
}
57
66
58
67
private:
59
- Z3CrosscheckOracle Oracle = Z3CrosscheckOracle(DefaultOpts );
68
+ Z3CrosscheckOracle Oracle = Z3CrosscheckOracle(Opts );
60
69
};
61
70
62
- TEST_F (Z3CrosscheckOracleTest, AcceptsFirstSAT) {
71
+ using DefaultZ3CrosscheckOracleTest = Z3CrosscheckOracleTest<DefaultOpts>;
72
+ using LimitedZ3CrosscheckOracleTest = Z3CrosscheckOracleTest<LimitedOpts>;
73
+
74
+ TEST_F (DefaultZ3CrosscheckOracleTest, AcceptsFirstSAT) {
75
+ ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 25_ms, 1000_step}));
76
+ }
77
+ TEST_F (LimitedZ3CrosscheckOracleTest, AcceptsFirstSAT) {
63
78
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 25_ms, 1000_step}));
64
79
}
65
80
66
- TEST_F (Z3CrosscheckOracleTest, AcceptsSAT) {
81
+ TEST_F (DefaultZ3CrosscheckOracleTest, AcceptsSAT) {
82
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
83
+ ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 25_ms, 1000_step}));
84
+ }
85
+ TEST_F (LimitedZ3CrosscheckOracleTest, AcceptsSAT) {
67
86
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
68
87
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 25_ms, 1000_step}));
69
88
}
70
89
71
- TEST_F (Z3CrosscheckOracleTest, SATWhenItGoesOverTime) {
90
+ TEST_F (DefaultZ3CrosscheckOracleTest, SATWhenItGoesOverTime) {
91
+ // Even if it times out, if it is SAT, we should accept it.
92
+ ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 15' 010_ms, 1000_step}));
93
+ }
94
+ TEST_F (LimitedZ3CrosscheckOracleTest, SATWhenItGoesOverTime) {
72
95
// Even if it times out, if it is SAT, we should accept it.
73
96
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 310_ms, 1000_step}));
74
97
}
75
98
76
- TEST_F (Z3CrosscheckOracleTest, UNSATWhenItGoesOverTime) {
99
+ TEST_F (DefaultZ3CrosscheckOracleTest, UNSATWhenItGoesOverTime) {
100
+ ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNSAT, 15' 010_ms, 1000_step}));
101
+ }
102
+ TEST_F (LimitedZ3CrosscheckOracleTest, UNSATWhenItGoesOverTime) {
77
103
ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNSAT, 310_ms, 1000_step}));
78
104
}
79
105
80
- TEST_F (Z3CrosscheckOracleTest, RejectsTimeout) {
106
+ TEST_F (DefaultZ3CrosscheckOracleTest, RejectsTimeout) {
107
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
108
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
109
+ ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNDEF, 15' 010_ms, 1000_step}));
110
+ }
111
+ TEST_F (LimitedZ3CrosscheckOracleTest, RejectsTimeout) {
81
112
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
82
113
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
83
114
ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNDEF, 310_ms, 1000_step}));
84
115
}
85
116
86
- TEST_F (Z3CrosscheckOracleTest, RejectsUNSATs) {
117
+ TEST_F (DefaultZ3CrosscheckOracleTest, RejectsUNSATs) {
118
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
119
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
120
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
121
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
122
+ }
123
+ TEST_F (LimitedZ3CrosscheckOracleTest, RejectsUNSATs) {
87
124
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
88
125
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
89
126
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
90
127
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
91
128
}
92
129
93
- // Testing cut heuristics:
94
- // =======================
130
+ // Testing cut heuristics of the two configurations :
131
+ // =================================================
95
132
96
- TEST_F (Z3CrosscheckOracleTest, RejectEQClassIfSpendsTooMuchTotalTime) {
133
+ TEST_F (DefaultZ3CrosscheckOracleTest, RejectEQClassIfSpendsTooMuchTotalTime) {
134
+ // Simulate long queries, that barely doesn't trigger the timeout.
135
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 14' 990_ms, 1000_step}));
136
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 14' 990_ms, 1000_step}));
137
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 14' 990_ms, 1000_step}));
138
+ }
139
+ TEST_F (LimitedZ3CrosscheckOracleTest, RejectEQClassIfSpendsTooMuchTotalTime) {
97
140
// Simulate long queries, that barely doesn't trigger the timeout.
98
141
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 290_ms, 1000_step}));
99
142
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 290_ms, 1000_step}));
100
143
ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNSAT, 290_ms, 1000_step}));
101
144
}
102
145
103
- TEST_F (Z3CrosscheckOracleTest, SATWhenItSpendsTooMuchTotalTime) {
146
+ TEST_F (DefaultZ3CrosscheckOracleTest, SATWhenItSpendsTooMuchTotalTime) {
147
+ // Simulate long queries, that barely doesn't trigger the timeout.
148
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 14' 990_ms, 1000_step}));
149
+ ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 14' 990_ms, 1000_step}));
150
+ ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 14' 990_ms, 1000_step}));
151
+ }
152
+ TEST_F (LimitedZ3CrosscheckOracleTest, SATWhenItSpendsTooMuchTotalTime) {
104
153
// Simulate long queries, that barely doesn't trigger the timeout.
105
154
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 290_ms, 1000_step}));
106
155
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 290_ms, 1000_step}));
107
156
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 290_ms, 1000_step}));
108
157
}
109
158
110
- TEST_F (Z3CrosscheckOracleTest, RejectEQClassIfAttemptsManySmallQueries) {
159
+ // Z3CrosscheckEQClassTimeoutThreshold is disabled in default configuration, so
160
+ // it doesn't make sense to test that.
161
+ TEST_F (LimitedZ3CrosscheckOracleTest, RejectEQClassIfAttemptsManySmallQueries) {
111
162
// Simulate quick, but many queries: 35 quick UNSAT queries.
112
163
// 35*20ms = 700ms, which is equal to the 700ms threshold.
113
164
for (int i = 0 ; i < 35 ; ++i) {
@@ -117,7 +168,9 @@ TEST_F(Z3CrosscheckOracleTest, RejectEQClassIfAttemptsManySmallQueries) {
117
168
ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNSAT, 1_ms, 1000_step}));
118
169
}
119
170
120
- TEST_F (Z3CrosscheckOracleTest, SATWhenIfAttemptsManySmallQueries) {
171
+ // Z3CrosscheckEQClassTimeoutThreshold is disabled in default configuration, so
172
+ // it doesn't make sense to test that.
173
+ TEST_F (LimitedZ3CrosscheckOracleTest, SATWhenItAttemptsManySmallQueries) {
121
174
// Simulate quick, but many queries: 35 quick UNSAT queries.
122
175
// 35*20ms = 700ms, which is equal to the 700ms threshold.
123
176
for (int i = 0 ; i < 35 ; ++i) {
@@ -128,16 +181,34 @@ TEST_F(Z3CrosscheckOracleTest, SATWhenIfAttemptsManySmallQueries) {
128
181
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 200_ms, 1000_step}));
129
182
}
130
183
131
- TEST_F (Z3CrosscheckOracleTest, RejectEQClassIfExhaustsRLimit) {
184
+ // Z3CrosscheckRLimitThreshold is disabled in default configuration, so it
185
+ // doesn't make sense to test that.
186
+ TEST_F (LimitedZ3CrosscheckOracleTest, RejectEQClassIfExhaustsRLimit) {
132
187
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
133
188
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
134
189
ASSERT_EQ (RejectEQClass, interpretQueryResult ({UNDEF, 25_ms, 405' 000_step}));
135
190
}
136
191
137
- TEST_F (Z3CrosscheckOracleTest, SATWhenItExhaustsRLimit) {
192
+ // Z3CrosscheckRLimitThreshold is disabled in default configuration, so it
193
+ // doesn't make sense to test that.
194
+ TEST_F (LimitedZ3CrosscheckOracleTest, SATWhenItExhaustsRLimit) {
138
195
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
139
196
ASSERT_EQ (RejectReport, interpretQueryResult ({UNSAT, 25_ms, 1000_step}));
140
197
ASSERT_EQ (AcceptReport, interpretQueryResult ({SAT, 25_ms, 405' 000_step}));
141
198
}
142
199
200
+ // Demonstrate the weaknesses of the default configuration:
201
+ // ========================================================
202
+
203
+ TEST_F (DefaultZ3CrosscheckOracleTest, ManySlowQueriesHangTheAnalyzer) {
204
+ // Simulate many slow queries: 250 slow UNSAT queries.
205
+ // 250*14000ms = 3500s, ~1 hour. Since we disabled the total time limitation,
206
+ // this eqclass would take roughly 1 hour to process.
207
+ // It doesn't matter what rlimit the queries consume.
208
+ for (int i = 0 ; i < 250 ; ++i) {
209
+ ASSERT_EQ (RejectReport,
210
+ interpretQueryResult ({UNSAT, 14' 000_ms, 1'000' 000_step}));
211
+ }
212
+ }
213
+
143
214
} // namespace
0 commit comments