output_test.h (7863B)
1 #ifndef TEST_OUTPUT_TEST_H 2 #define TEST_OUTPUT_TEST_H 3 4 #undef NDEBUG 5 #include <functional> 6 #include <initializer_list> 7 #include <memory> 8 #include <sstream> 9 #include <string> 10 #include <utility> 11 #include <vector> 12 13 #include "../src/re.h" 14 #include "benchmark/benchmark.h" 15 16 #define CONCAT2(x, y) x##y 17 #define CONCAT(x, y) CONCAT2(x, y) 18 19 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) 20 21 #define SET_SUBSTITUTIONS(...) \ 22 int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) 23 24 enum MatchRules { 25 MR_Default, // Skip non-matching lines until a match is found. 26 MR_Next, // Match must occur on the next line. 27 MR_Not // No line between the current position and the next match matches 28 // the regex 29 }; 30 31 struct TestCase { 32 TestCase(std::string re, int rule = MR_Default); 33 34 std::string regex_str; 35 int match_rule; 36 std::string substituted_regex; 37 std::shared_ptr<benchmark::Regex> regex; 38 }; 39 40 enum TestCaseID { 41 TC_ConsoleOut, 42 TC_ConsoleErr, 43 TC_JSONOut, 44 TC_JSONErr, 45 TC_CSVOut, 46 TC_CSVErr, 47 48 TC_NumID // PRIVATE 49 }; 50 51 // Add a list of test cases to be run against the output specified by 52 // 'ID' 53 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il); 54 55 // Add or set a list of substitutions to be performed on constructed regex's 56 // See 'output_test_helper.cc' for a list of default substitutions. 57 int SetSubstitutions( 58 std::initializer_list<std::pair<std::string, std::string>> il); 59 60 // Run all output tests. 61 void RunOutputTests(int argc, char* argv[]); 62 63 // Count the number of 'pat' substrings in the 'haystack' string. 64 int SubstrCnt(const std::string& haystack, const std::string& pat); 65 66 // Run registered benchmarks with file reporter enabled, and return the content 67 // outputted by the file reporter. 68 std::string GetFileReporterOutput(int argc, char* argv[]); 69 70 // ========================================================================= // 71 // ------------------------- Results checking ------------------------------ // 72 // ========================================================================= // 73 74 // Call this macro to register a benchmark for checking its results. This 75 // should be all that's needed. It subscribes a function to check the (CSV) 76 // results of a benchmark. This is done only after verifying that the output 77 // strings are really as expected. 78 // bm_name_pattern: a name or a regex pattern which will be matched against 79 // all the benchmark names. Matching benchmarks 80 // will be the subject of a call to checker_function 81 // checker_function: should be of type ResultsCheckFn (see below) 82 #define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ 83 size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) 84 85 struct Results; 86 typedef std::function<void(Results const&)> ResultsCheckFn; 87 88 size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn); 89 90 // Class holding the results of a benchmark. 91 // It is passed in calls to checker functions. 92 struct Results { 93 // the benchmark name 94 std::string name; 95 // the benchmark fields 96 std::map<std::string, std::string> values; 97 98 Results(const std::string& n) : name(n) {} 99 100 int NumThreads() const; 101 102 double NumIterations() const; 103 104 typedef enum { kCpuTime, kRealTime } BenchmarkTime; 105 106 // get cpu_time or real_time in seconds 107 double GetTime(BenchmarkTime which) const; 108 109 // get the real_time duration of the benchmark in seconds. 110 // it is better to use fuzzy float checks for this, as the float 111 // ASCII formatting is lossy. 112 double DurationRealTime() const { 113 return NumIterations() * GetTime(kRealTime); 114 } 115 // get the cpu_time duration of the benchmark in seconds 116 double DurationCPUTime() const { 117 return NumIterations() * GetTime(kCpuTime); 118 } 119 120 // get the string for a result by name, or nullptr if the name 121 // is not found 122 const std::string* Get(const char* entry_name) const { 123 auto it = values.find(entry_name); 124 if (it == values.end()) return nullptr; 125 return &it->second; 126 } 127 128 // get a result by name, parsed as a specific type. 129 // NOTE: for counters, use GetCounterAs instead. 130 template <class T> 131 T GetAs(const char* entry_name) const; 132 133 // counters are written as doubles, so they have to be read first 134 // as a double, and only then converted to the asked type. 135 template <class T> 136 T GetCounterAs(const char* entry_name) const { 137 double dval = GetAs<double>(entry_name); 138 T tval = static_cast<T>(dval); 139 return tval; 140 } 141 }; 142 143 template <class T> 144 T Results::GetAs(const char* entry_name) const { 145 auto* sv = Get(entry_name); 146 CHECK(sv != nullptr && !sv->empty()); 147 std::stringstream ss; 148 ss << *sv; 149 T out; 150 ss >> out; 151 CHECK(!ss.fail()); 152 return out; 153 } 154 155 //---------------------------------- 156 // Macros to help in result checking. Do not use them with arguments causing 157 // side-effects. 158 159 // clang-format off 160 161 #define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ 162 CONCAT(CHECK_, relationship) \ 163 (entry.getfn< var_type >(var_name), (value)) << "\n" \ 164 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ 165 << __FILE__ << ":" << __LINE__ << ": " \ 166 << "expected (" << #var_type << ")" << (var_name) \ 167 << "=" << (entry).getfn< var_type >(var_name) \ 168 << " to be " #relationship " to " << (value) << "\n" 169 170 // check with tolerance. eps_factor is the tolerance window, which is 171 // interpreted relative to value (eg, 0.1 means 10% of value). 172 #define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ 173 CONCAT(CHECK_FLOAT_, relationship) \ 174 (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ 175 << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ 176 << __FILE__ << ":" << __LINE__ << ": " \ 177 << "expected (" << #var_type << ")" << (var_name) \ 178 << "=" << (entry).getfn< var_type >(var_name) \ 179 << " to be " #relationship " to " << (value) << "\n" \ 180 << __FILE__ << ":" << __LINE__ << ": " \ 181 << "with tolerance of " << (eps_factor) * (value) \ 182 << " (" << (eps_factor)*100. << "%), " \ 183 << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \ 184 << " (" << (((entry).getfn< var_type >(var_name) - (value)) \ 185 / \ 186 ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \ 187 << "%)" 188 189 #define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ 190 _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) 191 192 #define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ 193 _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) 194 195 #define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \ 196 _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor) 197 198 #define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ 199 _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) 200 201 // clang-format on 202 203 // ========================================================================= // 204 // --------------------------- Misc Utilities ------------------------------ // 205 // ========================================================================= // 206 207 namespace { 208 209 const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; 210 211 } // end namespace 212 213 #endif // TEST_OUTPUT_TEST_H