skip_with_error_test.cc (6295B)
1 2 #undef NDEBUG 3 #include <cassert> 4 #include <vector> 5 6 #include "../src/check.h" // NOTE: check.h is for internal use only! 7 #include "benchmark/benchmark.h" 8 9 namespace { 10 11 class TestReporter : public benchmark::ConsoleReporter { 12 public: 13 virtual bool ReportContext(const Context& context) { 14 return ConsoleReporter::ReportContext(context); 15 }; 16 17 virtual void ReportRuns(const std::vector<Run>& report) { 18 all_runs_.insert(all_runs_.end(), begin(report), end(report)); 19 ConsoleReporter::ReportRuns(report); 20 } 21 22 TestReporter() {} 23 virtual ~TestReporter() {} 24 25 mutable std::vector<Run> all_runs_; 26 }; 27 28 struct TestCase { 29 std::string name; 30 bool error_occurred; 31 std::string error_message; 32 33 typedef benchmark::BenchmarkReporter::Run Run; 34 35 void CheckRun(Run const& run) const { 36 CHECK(name == run.benchmark_name()) 37 << "expected " << name << " got " << run.benchmark_name(); 38 CHECK(error_occurred == run.error_occurred); 39 CHECK(error_message == run.error_message); 40 if (error_occurred) { 41 // CHECK(run.iterations == 0); 42 } else { 43 CHECK(run.iterations != 0); 44 } 45 } 46 }; 47 48 std::vector<TestCase> ExpectedResults; 49 50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) { 51 for (auto TC : v) { 52 TC.name = base_name + TC.name; 53 ExpectedResults.push_back(std::move(TC)); 54 } 55 return 0; 56 } 57 58 #define CONCAT(x, y) CONCAT2(x, y) 59 #define CONCAT2(x, y) x##y 60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) 61 62 } // end namespace 63 64 void BM_error_before_running(benchmark::State& state) { 65 state.SkipWithError("error message"); 66 while (state.KeepRunning()) { 67 assert(false); 68 } 69 } 70 BENCHMARK(BM_error_before_running); 71 ADD_CASES("BM_error_before_running", {{"", true, "error message"}}); 72 73 void BM_error_before_running_batch(benchmark::State& state) { 74 state.SkipWithError("error message"); 75 while (state.KeepRunningBatch(17)) { 76 assert(false); 77 } 78 } 79 BENCHMARK(BM_error_before_running_batch); 80 ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}}); 81 82 void BM_error_before_running_range_for(benchmark::State& state) { 83 state.SkipWithError("error message"); 84 for (auto _ : state) { 85 assert(false); 86 } 87 } 88 BENCHMARK(BM_error_before_running_range_for); 89 ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}}); 90 91 void BM_error_during_running(benchmark::State& state) { 92 int first_iter = true; 93 while (state.KeepRunning()) { 94 if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { 95 assert(first_iter); 96 first_iter = false; 97 state.SkipWithError("error message"); 98 } else { 99 state.PauseTiming(); 100 state.ResumeTiming(); 101 } 102 } 103 } 104 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); 105 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"}, 106 {"/1/threads:2", true, "error message"}, 107 {"/1/threads:4", true, "error message"}, 108 {"/1/threads:8", true, "error message"}, 109 {"/2/threads:1", false, ""}, 110 {"/2/threads:2", false, ""}, 111 {"/2/threads:4", false, ""}, 112 {"/2/threads:8", false, ""}}); 113 114 void BM_error_during_running_ranged_for(benchmark::State& state) { 115 assert(state.max_iterations > 3 && "test requires at least a few iterations"); 116 int first_iter = true; 117 // NOTE: Users should not write the for loop explicitly. 118 for (auto It = state.begin(), End = state.end(); It != End; ++It) { 119 if (state.range(0) == 1) { 120 assert(first_iter); 121 first_iter = false; 122 state.SkipWithError("error message"); 123 // Test the unfortunate but documented behavior that the ranged-for loop 124 // doesn't automatically terminate when SkipWithError is set. 125 assert(++It != End); 126 break; // Required behavior 127 } 128 } 129 } 130 BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5); 131 ADD_CASES("BM_error_during_running_ranged_for", 132 {{"/1/iterations:5", true, "error message"}, 133 {"/2/iterations:5", false, ""}}); 134 135 void BM_error_after_running(benchmark::State& state) { 136 for (auto _ : state) { 137 benchmark::DoNotOptimize(state.iterations()); 138 } 139 if (state.thread_index <= (state.threads / 2)) 140 state.SkipWithError("error message"); 141 } 142 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); 143 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"}, 144 {"/threads:2", true, "error message"}, 145 {"/threads:4", true, "error message"}, 146 {"/threads:8", true, "error message"}}); 147 148 void BM_error_while_paused(benchmark::State& state) { 149 bool first_iter = true; 150 while (state.KeepRunning()) { 151 if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { 152 assert(first_iter); 153 first_iter = false; 154 state.PauseTiming(); 155 state.SkipWithError("error message"); 156 } else { 157 state.PauseTiming(); 158 state.ResumeTiming(); 159 } 160 } 161 } 162 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8); 163 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"}, 164 {"/1/threads:2", true, "error message"}, 165 {"/1/threads:4", true, "error message"}, 166 {"/1/threads:8", true, "error message"}, 167 {"/2/threads:1", false, ""}, 168 {"/2/threads:2", false, ""}, 169 {"/2/threads:4", false, ""}, 170 {"/2/threads:8", false, ""}}); 171 172 int main(int argc, char* argv[]) { 173 benchmark::Initialize(&argc, argv); 174 175 TestReporter test_reporter; 176 benchmark::RunSpecifiedBenchmarks(&test_reporter); 177 178 typedef benchmark::BenchmarkReporter::Run Run; 179 auto EB = ExpectedResults.begin(); 180 181 for (Run const& run : test_reporter.all_runs_) { 182 assert(EB != ExpectedResults.end()); 183 EB->CheckRun(run); 184 ++EB; 185 } 186 assert(EB == ExpectedResults.end()); 187 188 return 0; 189 }