reporter_output_test.cc (30456B)
1 2 #undef NDEBUG 3 #include <utility> 4 5 #include "benchmark/benchmark.h" 6 #include "output_test.h" 7 8 // ========================================================================= // 9 // ---------------------- Testing Prologue Output -------------------------- // 10 // ========================================================================= // 11 12 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next}, 13 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, 14 {"^[-]+$", MR_Next}}); 15 static int AddContextCases() { 16 AddCases(TC_ConsoleErr, 17 { 18 {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, 19 {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, 20 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, 21 }); 22 AddCases(TC_JSONOut, 23 {{"^\\{", MR_Default}, 24 {"\"context\":", MR_Next}, 25 {"\"date\": \"", MR_Next}, 26 {"\"host_name\":", MR_Next}, 27 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",", 28 MR_Next}, 29 {"\"num_cpus\": %int,$", MR_Next}, 30 {"\"mhz_per_cpu\": %float,$", MR_Next}, 31 {"\"cpu_scaling_enabled\": ", MR_Next}, 32 {"\"caches\": \\[$", MR_Next}}); 33 auto const& Info = benchmark::CPUInfo::Get(); 34 auto const& Caches = Info.caches; 35 if (!Caches.empty()) { 36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}}); 37 } 38 for (size_t I = 0; I < Caches.size(); ++I) { 39 std::string num_caches_str = 40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$"; 41 AddCases( 42 TC_ConsoleErr, 43 {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}}); 44 AddCases(TC_JSONOut, {{"\\{$", MR_Next}, 45 {"\"type\": \"", MR_Next}, 46 {"\"level\": %int,$", MR_Next}, 47 {"\"size\": %int,$", MR_Next}, 48 {"\"num_sharing\": %int$", MR_Next}, 49 {"}[,]{0,1}$", MR_Next}}); 50 } 51 AddCases(TC_JSONOut, {{"],$"}}); 52 auto const& LoadAvg = Info.load_avg; 53 if (!LoadAvg.empty()) { 54 AddCases(TC_ConsoleErr, 55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); 56 } 57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); 58 return 0; 59 } 60 int dummy_register = AddContextCases(); 61 ADD_CASES(TC_CSVOut, {{"%csv_header"}}); 62 63 // ========================================================================= // 64 // ------------------------ Testing Basic Output --------------------------- // 65 // ========================================================================= // 66 67 void BM_basic(benchmark::State& state) { 68 for (auto _ : state) { 69 } 70 } 71 BENCHMARK(BM_basic); 72 73 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); 74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, 75 {"\"run_name\": \"BM_basic\",$", MR_Next}, 76 {"\"run_type\": \"iteration\",$", MR_Next}, 77 {"\"iterations\": %int,$", MR_Next}, 78 {"\"real_time\": %float,$", MR_Next}, 79 {"\"cpu_time\": %float,$", MR_Next}, 80 {"\"time_unit\": \"ns\"$", MR_Next}, 81 {"}", MR_Next}}); 82 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); 83 84 // ========================================================================= // 85 // ------------------------ Testing Bytes per Second Output ---------------- // 86 // ========================================================================= // 87 88 void BM_bytes_per_second(benchmark::State& state) { 89 for (auto _ : state) { 90 } 91 state.SetBytesProcessed(1); 92 } 93 BENCHMARK(BM_bytes_per_second); 94 95 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " 96 "bytes_per_second=%float[kM]{0,1}/s$"}}); 97 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, 98 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, 99 {"\"run_type\": \"iteration\",$", MR_Next}, 100 {"\"iterations\": %int,$", MR_Next}, 101 {"\"real_time\": %float,$", MR_Next}, 102 {"\"cpu_time\": %float,$", MR_Next}, 103 {"\"time_unit\": \"ns\",$", MR_Next}, 104 {"\"bytes_per_second\": %float$", MR_Next}, 105 {"}", MR_Next}}); 106 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); 107 108 // ========================================================================= // 109 // ------------------------ Testing Items per Second Output ---------------- // 110 // ========================================================================= // 111 112 void BM_items_per_second(benchmark::State& state) { 113 for (auto _ : state) { 114 } 115 state.SetItemsProcessed(1); 116 } 117 BENCHMARK(BM_items_per_second); 118 119 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " 120 "items_per_second=%float[kM]{0,1}/s$"}}); 121 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, 122 {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, 123 {"\"run_type\": \"iteration\",$", MR_Next}, 124 {"\"iterations\": %int,$", MR_Next}, 125 {"\"real_time\": %float,$", MR_Next}, 126 {"\"cpu_time\": %float,$", MR_Next}, 127 {"\"time_unit\": \"ns\",$", MR_Next}, 128 {"\"items_per_second\": %float$", MR_Next}, 129 {"}", MR_Next}}); 130 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); 131 132 // ========================================================================= // 133 // ------------------------ Testing Label Output --------------------------- // 134 // ========================================================================= // 135 136 void BM_label(benchmark::State& state) { 137 for (auto _ : state) { 138 } 139 state.SetLabel("some label"); 140 } 141 BENCHMARK(BM_label); 142 143 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); 144 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, 145 {"\"run_name\": \"BM_label\",$", MR_Next}, 146 {"\"run_type\": \"iteration\",$", MR_Next}, 147 {"\"iterations\": %int,$", MR_Next}, 148 {"\"real_time\": %float,$", MR_Next}, 149 {"\"cpu_time\": %float,$", MR_Next}, 150 {"\"time_unit\": \"ns\",$", MR_Next}, 151 {"\"label\": \"some label\"$", MR_Next}, 152 {"}", MR_Next}}); 153 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " 154 "label\"%csv_label_report_end$"}}); 155 156 // ========================================================================= // 157 // ------------------------ Testing Error Output --------------------------- // 158 // ========================================================================= // 159 160 void BM_error(benchmark::State& state) { 161 state.SkipWithError("message"); 162 for (auto _ : state) { 163 } 164 } 165 BENCHMARK(BM_error); 166 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); 167 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, 168 {"\"run_name\": \"BM_error\",$", MR_Next}, 169 {"\"run_type\": \"iteration\",$", MR_Next}, 170 {"\"error_occurred\": true,$", MR_Next}, 171 {"\"error_message\": \"message\",$", MR_Next}}); 172 173 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); 174 175 // ========================================================================= // 176 // ------------------------ Testing No Arg Name Output ----------------------- 177 // // 178 // ========================================================================= // 179 180 void BM_no_arg_name(benchmark::State& state) { 181 for (auto _ : state) { 182 } 183 } 184 BENCHMARK(BM_no_arg_name)->Arg(3); 185 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); 186 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, 187 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, 188 {"\"run_type\": \"iteration\",$", MR_Next}}); 189 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); 190 191 // ========================================================================= // 192 // ------------------------ Testing Arg Name Output ----------------------- // 193 // ========================================================================= // 194 195 void BM_arg_name(benchmark::State& state) { 196 for (auto _ : state) { 197 } 198 } 199 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); 200 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); 201 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, 202 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, 203 {"\"run_type\": \"iteration\",$", MR_Next}}); 204 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); 205 206 // ========================================================================= // 207 // ------------------------ Testing Arg Names Output ----------------------- // 208 // ========================================================================= // 209 210 void BM_arg_names(benchmark::State& state) { 211 for (auto _ : state) { 212 } 213 } 214 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); 215 ADD_CASES(TC_ConsoleOut, 216 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); 217 ADD_CASES(TC_JSONOut, 218 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, 219 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, 220 {"\"run_type\": \"iteration\",$", MR_Next}}); 221 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); 222 223 // ========================================================================= // 224 // ------------------------ Testing Big Args Output ------------------------ // 225 // ========================================================================= // 226 227 void BM_BigArgs(benchmark::State& state) { 228 for (auto _ : state) { 229 } 230 } 231 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U); 232 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, 233 {"^BM_BigArgs/2147483648 %console_report$"}}); 234 235 // ========================================================================= // 236 // ----------------------- Testing Complexity Output ----------------------- // 237 // ========================================================================= // 238 239 void BM_Complexity_O1(benchmark::State& state) { 240 for (auto _ : state) { 241 } 242 state.SetComplexityN(state.range(0)); 243 } 244 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 245 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, 246 {"%RMS", "[ ]*[0-9]+ %"}}); 247 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, 248 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); 249 250 // ========================================================================= // 251 // ----------------------- Testing Aggregate Output ------------------------ // 252 // ========================================================================= // 253 254 // Test that non-aggregate data is printed by default 255 void BM_Repeat(benchmark::State& state) { 256 for (auto _ : state) { 257 } 258 } 259 // need two repetitions min to be able to output any aggregate output 260 BENCHMARK(BM_Repeat)->Repetitions(2); 261 ADD_CASES(TC_ConsoleOut, 262 {{"^BM_Repeat/repeats:2 %console_report$"}, 263 {"^BM_Repeat/repeats:2 %console_report$"}, 264 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"}, 265 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, 266 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); 267 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, 268 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, 269 {"\"run_type\": \"iteration\",$", MR_Next}, 270 {"\"name\": \"BM_Repeat/repeats:2\",$"}, 271 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 272 {"\"run_type\": \"iteration\",$", MR_Next}, 273 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, 274 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 275 {"\"run_type\": \"aggregate\",$", MR_Next}, 276 {"\"aggregate_name\": \"mean\",$", MR_Next}, 277 {"\"iterations\": 2,$", MR_Next}, 278 {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, 279 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 280 {"\"run_type\": \"aggregate\",$", MR_Next}, 281 {"\"aggregate_name\": \"median\",$", MR_Next}, 282 {"\"iterations\": 2,$", MR_Next}, 283 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, 284 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, 285 {"\"run_type\": \"aggregate\",$", MR_Next}, 286 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 287 {"\"iterations\": 2,$", MR_Next}}); 288 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, 289 {"^\"BM_Repeat/repeats:2\",%csv_report$"}, 290 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, 291 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"}, 292 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); 293 // but for two repetitions, mean and median is the same, so let's repeat.. 294 BENCHMARK(BM_Repeat)->Repetitions(3); 295 ADD_CASES(TC_ConsoleOut, 296 {{"^BM_Repeat/repeats:3 %console_report$"}, 297 {"^BM_Repeat/repeats:3 %console_report$"}, 298 {"^BM_Repeat/repeats:3 %console_report$"}, 299 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"}, 300 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, 301 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); 302 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, 303 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 304 {"\"run_type\": \"iteration\",$", MR_Next}, 305 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 306 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 307 {"\"run_type\": \"iteration\",$", MR_Next}, 308 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 309 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 310 {"\"run_type\": \"iteration\",$", MR_Next}, 311 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, 312 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 313 {"\"run_type\": \"aggregate\",$", MR_Next}, 314 {"\"aggregate_name\": \"mean\",$", MR_Next}, 315 {"\"iterations\": 3,$", MR_Next}, 316 {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, 317 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 318 {"\"run_type\": \"aggregate\",$", MR_Next}, 319 {"\"aggregate_name\": \"median\",$", MR_Next}, 320 {"\"iterations\": 3,$", MR_Next}, 321 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, 322 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, 323 {"\"run_type\": \"aggregate\",$", MR_Next}, 324 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 325 {"\"iterations\": 3,$", MR_Next}}); 326 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, 327 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 328 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 329 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, 330 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"}, 331 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); 332 // median differs between even/odd number of repetitions, so just to be sure 333 BENCHMARK(BM_Repeat)->Repetitions(4); 334 ADD_CASES(TC_ConsoleOut, 335 {{"^BM_Repeat/repeats:4 %console_report$"}, 336 {"^BM_Repeat/repeats:4 %console_report$"}, 337 {"^BM_Repeat/repeats:4 %console_report$"}, 338 {"^BM_Repeat/repeats:4 %console_report$"}, 339 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"}, 340 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, 341 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); 342 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, 343 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 344 {"\"run_type\": \"iteration\",$", MR_Next}, 345 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 346 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 347 {"\"run_type\": \"iteration\",$", MR_Next}, 348 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 349 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 350 {"\"run_type\": \"iteration\",$", MR_Next}, 351 {"\"name\": \"BM_Repeat/repeats:4\",$"}, 352 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 353 {"\"run_type\": \"iteration\",$", MR_Next}, 354 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, 355 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 356 {"\"run_type\": \"aggregate\",$", MR_Next}, 357 {"\"aggregate_name\": \"mean\",$", MR_Next}, 358 {"\"iterations\": 4,$", MR_Next}, 359 {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, 360 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 361 {"\"run_type\": \"aggregate\",$", MR_Next}, 362 {"\"aggregate_name\": \"median\",$", MR_Next}, 363 {"\"iterations\": 4,$", MR_Next}, 364 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, 365 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, 366 {"\"run_type\": \"aggregate\",$", MR_Next}, 367 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 368 {"\"iterations\": 4,$", MR_Next}}); 369 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, 370 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 371 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 372 {"^\"BM_Repeat/repeats:4\",%csv_report$"}, 373 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"}, 374 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"}, 375 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}}); 376 377 // Test that a non-repeated test still prints non-aggregate results even when 378 // only-aggregate reports have been requested 379 void BM_RepeatOnce(benchmark::State& state) { 380 for (auto _ : state) { 381 } 382 } 383 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); 384 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); 385 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, 386 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, 387 {"\"run_type\": \"iteration\",$", MR_Next}}); 388 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); 389 390 // Test that non-aggregate data is not reported 391 void BM_SummaryRepeat(benchmark::State& state) { 392 for (auto _ : state) { 393 } 394 } 395 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); 396 ADD_CASES( 397 TC_ConsoleOut, 398 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 399 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"}, 400 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"}, 401 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); 402 ADD_CASES(TC_JSONOut, 403 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 404 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, 405 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 406 {"\"run_type\": \"aggregate\",$", MR_Next}, 407 {"\"aggregate_name\": \"mean\",$", MR_Next}, 408 {"\"iterations\": 3,$", MR_Next}, 409 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, 410 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 411 {"\"run_type\": \"aggregate\",$", MR_Next}, 412 {"\"aggregate_name\": \"median\",$", MR_Next}, 413 {"\"iterations\": 3,$", MR_Next}, 414 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, 415 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, 416 {"\"run_type\": \"aggregate\",$", MR_Next}, 417 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 418 {"\"iterations\": 3,$", MR_Next}}); 419 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 420 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, 421 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, 422 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); 423 424 // Test that non-aggregate data is not displayed. 425 // NOTE: this test is kinda bad. we are only testing the display output. 426 // But we don't check that the file output still contains everything... 427 void BM_SummaryDisplay(benchmark::State& state) { 428 for (auto _ : state) { 429 } 430 } 431 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); 432 ADD_CASES( 433 TC_ConsoleOut, 434 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 435 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"}, 436 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"}, 437 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}}); 438 ADD_CASES(TC_JSONOut, 439 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 440 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, 441 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 442 {"\"run_type\": \"aggregate\",$", MR_Next}, 443 {"\"aggregate_name\": \"mean\",$", MR_Next}, 444 {"\"iterations\": 2,$", MR_Next}, 445 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, 446 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 447 {"\"run_type\": \"aggregate\",$", MR_Next}, 448 {"\"aggregate_name\": \"median\",$", MR_Next}, 449 {"\"iterations\": 2,$", MR_Next}, 450 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, 451 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, 452 {"\"run_type\": \"aggregate\",$", MR_Next}, 453 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 454 {"\"iterations\": 2,$", MR_Next}}); 455 ADD_CASES(TC_CSVOut, 456 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, 457 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, 458 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"}, 459 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}}); 460 461 // Test repeats with custom time unit. 462 void BM_RepeatTimeUnit(benchmark::State& state) { 463 for (auto _ : state) { 464 } 465 } 466 BENCHMARK(BM_RepeatTimeUnit) 467 ->Repetitions(3) 468 ->ReportAggregatesOnly() 469 ->Unit(benchmark::kMicrosecond); 470 ADD_CASES( 471 TC_ConsoleOut, 472 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 473 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"}, 474 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ " 475 "]*3$"}, 476 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ " 477 "]*3$"}}); 478 ADD_CASES(TC_JSONOut, 479 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 480 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, 481 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 482 {"\"run_type\": \"aggregate\",$", MR_Next}, 483 {"\"aggregate_name\": \"mean\",$", MR_Next}, 484 {"\"iterations\": 3,$", MR_Next}, 485 {"\"time_unit\": \"us\",?$"}, 486 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, 487 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 488 {"\"run_type\": \"aggregate\",$", MR_Next}, 489 {"\"aggregate_name\": \"median\",$", MR_Next}, 490 {"\"iterations\": 3,$", MR_Next}, 491 {"\"time_unit\": \"us\",?$"}, 492 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, 493 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, 494 {"\"run_type\": \"aggregate\",$", MR_Next}, 495 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 496 {"\"iterations\": 3,$", MR_Next}, 497 {"\"time_unit\": \"us\",?$"}}); 498 ADD_CASES(TC_CSVOut, 499 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 500 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, 501 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"}, 502 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); 503 504 // ========================================================================= // 505 // -------------------- Testing user-provided statistics ------------------- // 506 // ========================================================================= // 507 508 const auto UserStatistics = [](const std::vector<double>& v) { 509 return v.back(); 510 }; 511 void BM_UserStats(benchmark::State& state) { 512 for (auto _ : state) { 513 state.SetIterationTime(150 / 10e8); 514 } 515 } 516 // clang-format off 517 BENCHMARK(BM_UserStats) 518 ->Repetitions(3) 519 ->Iterations(5) 520 ->UseManualTime() 521 ->ComputeStatistics("", UserStatistics); 522 // clang-format on 523 524 // check that user-provided stats is calculated, and is after the default-ones 525 // empty string as name is intentional, it would sort before anything else 526 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 527 "]* 150 ns %time [ ]*5$"}, 528 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 529 "]* 150 ns %time [ ]*5$"}, 530 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " 531 "]* 150 ns %time [ ]*5$"}, 532 {"^BM_UserStats/iterations:5/repeats:3/" 533 "manual_time_mean [ ]* 150 ns %time [ ]*3$"}, 534 {"^BM_UserStats/iterations:5/repeats:3/" 535 "manual_time_median [ ]* 150 ns %time [ ]*3$"}, 536 {"^BM_UserStats/iterations:5/repeats:3/" 537 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"}, 538 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ " 539 "[ ]* 150 ns %time [ ]*3$"}}); 540 ADD_CASES( 541 TC_JSONOut, 542 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 543 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 544 MR_Next}, 545 {"\"run_type\": \"iteration\",$", MR_Next}, 546 {"\"iterations\": 5,$", MR_Next}, 547 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 548 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 549 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 550 MR_Next}, 551 {"\"run_type\": \"iteration\",$", MR_Next}, 552 {"\"iterations\": 5,$", MR_Next}, 553 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 554 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, 555 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 556 MR_Next}, 557 {"\"run_type\": \"iteration\",$", MR_Next}, 558 {"\"iterations\": 5,$", MR_Next}, 559 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 560 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, 561 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 562 MR_Next}, 563 {"\"run_type\": \"aggregate\",$", MR_Next}, 564 {"\"aggregate_name\": \"mean\",$", MR_Next}, 565 {"\"iterations\": 3,$", MR_Next}, 566 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 567 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, 568 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 569 MR_Next}, 570 {"\"run_type\": \"aggregate\",$", MR_Next}, 571 {"\"aggregate_name\": \"median\",$", MR_Next}, 572 {"\"iterations\": 3,$", MR_Next}, 573 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, 574 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, 575 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 576 MR_Next}, 577 {"\"run_type\": \"aggregate\",$", MR_Next}, 578 {"\"aggregate_name\": \"stddev\",$", MR_Next}, 579 {"\"iterations\": 3,$", MR_Next}, 580 {"\"real_time\": %float,$", MR_Next}, 581 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, 582 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", 583 MR_Next}, 584 {"\"run_type\": \"aggregate\",$", MR_Next}, 585 {"\"aggregate_name\": \"\",$", MR_Next}, 586 {"\"iterations\": 3,$", MR_Next}, 587 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); 588 ADD_CASES( 589 TC_CSVOut, 590 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 591 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 592 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, 593 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"}, 594 {"^\"BM_UserStats/iterations:5/repeats:3/" 595 "manual_time_median\",%csv_report$"}, 596 {"^\"BM_UserStats/iterations:5/repeats:3/" 597 "manual_time_stddev\",%csv_report$"}, 598 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); 599 600 // ========================================================================= // 601 // --------------------------- TEST CASES END ------------------------------ // 602 // ========================================================================= // 603 604 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }