libcxx

output_test_helper.cc (16870B)

---

 #include <cstdio>
 #include <cstring>
 #include <fstream>
 #include <iostream>
 #include <map>
 #include <memory>
 #include <random>
 #include <sstream>
 #include <streambuf>
 
 #include "../src/benchmark_api_internal.h"
 #include "../src/check.h"  // NOTE: check.h is for internal use only!
 #include "../src/re.h"     // NOTE: re.h is for internal use only
 #include "output_test.h"
 
 // ========================================================================= //
 // ------------------------------ Internals -------------------------------- //
 // ========================================================================= //
 namespace internal {
 namespace {
 
 using TestCaseList = std::vector<TestCase>;
 
 // Use a vector because the order elements are added matters during iteration.
 // std::map/unordered_map don't guarantee that.
 // For example:
 //  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
 //     Substitute("%HelloWorld") // Always expands to Hello.
 using SubMap = std::vector<std::pair<std::string, std::string>>;
 
 TestCaseList& GetTestCaseList(TestCaseID ID) {
   // Uses function-local statics to ensure initialization occurs
   // before first use.
   static TestCaseList lists[TC_NumID];
   return lists[ID];
 }
 
 SubMap& GetSubstitutions() {
   // Don't use 'dec_re' from header because it may not yet be initialized.
   // clang-format off
   static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
   static std::string time_re = "([0-9]+[.])?[0-9]+";
   static SubMap map = {
       {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
       // human-readable float
       {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
       {"%int", "[ ]*[0-9]+"},
       {" %s ", "[ ]+"},
       {"%time", "[ ]*" + time_re + "[ ]+ns"},
       {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
       {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
       {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
       {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
       {"%csv_header",
        "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
        "items_per_second,label,error_occurred,error_message"},
       {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
       {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
       {"%csv_bytes_report",
        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
       {"%csv_items_report",
        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
       {"%csv_bytes_items_report",
        "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
        "," + safe_dec_re + ",,,"},
       {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
       {"%csv_label_report_end", ",,"}};
   // clang-format on
   return map;
 }
 
 std::string PerformSubstitutions(std::string source) {
   SubMap const& subs = GetSubstitutions();
   using SizeT = std::string::size_type;
   for (auto const& KV : subs) {
     SizeT pos;
     SizeT next_start = 0;
     while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
       next_start = pos + KV.second.size();
       source.replace(pos, KV.first.size(), KV.second);
     }
   }
   return source;
 }
 
 void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
                TestCaseList const& not_checks) {
   std::string first_line;
   bool on_first = true;
   std::string line;
   while (remaining_output.eof() == false) {
     CHECK(remaining_output.good());
     std::getline(remaining_output, line);
     if (on_first) {
       first_line = line;
       on_first = false;
     }
     for (const auto& NC : not_checks) {
       CHECK(!NC.regex->Match(line))
           << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
           << NC.regex_str << "\""
           << "\n    actual regex string \"" << TC.substituted_regex << "\""
           << "\n    started matching near: " << first_line;
     }
     if (TC.regex->Match(line)) return;
     CHECK(TC.match_rule != MR_Next)
         << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
         << "\""
         << "\n    actual regex string \"" << TC.substituted_regex << "\""
         << "\n    started matching near: " << first_line;
   }
   CHECK(remaining_output.eof() == false)
       << "End of output reached before match for regex \"" << TC.regex_str
       << "\" was found"
       << "\n    actual regex string \"" << TC.substituted_regex << "\""
       << "\n    started matching near: " << first_line;
 }
 
 void CheckCases(TestCaseList const& checks, std::stringstream& output) {
   std::vector<TestCase> not_checks;
   for (size_t i = 0; i < checks.size(); ++i) {
     const auto& TC = checks[i];
     if (TC.match_rule == MR_Not) {
       not_checks.push_back(TC);
       continue;
     }
     CheckCase(output, TC, not_checks);
     not_checks.clear();
   }
 }
 
 class TestReporter : public benchmark::BenchmarkReporter {
  public:
   TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
       : reporters_(reps) {}
 
   virtual bool ReportContext(const Context& context) {
     bool last_ret = false;
     bool first = true;
     for (auto rep : reporters_) {
       bool new_ret = rep->ReportContext(context);
       CHECK(first || new_ret == last_ret)
           << "Reports return different values for ReportContext";
       first = false;
       last_ret = new_ret;
     }
     (void)first;
     return last_ret;
   }
 
   void ReportRuns(const std::vector<Run>& report) {
     for (auto rep : reporters_) rep->ReportRuns(report);
   }
   void Finalize() {
     for (auto rep : reporters_) rep->Finalize();
   }
 
  private:
   std::vector<benchmark::BenchmarkReporter*> reporters_;
 };
 }  // namespace
 
 }  // end namespace internal
 
 // ========================================================================= //
 // -------------------------- Results checking ----------------------------- //
 // ========================================================================= //
 
 namespace internal {
 
 // Utility class to manage subscribers for checking benchmark results.
 // It works by parsing the CSV output to read the results.
 class ResultsChecker {
  public:
   struct PatternAndFn : public TestCase {  // reusing TestCase for its regexes
     PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
         : TestCase(rx), fn(fn_) {}
     ResultsCheckFn fn;
   };
 
   std::vector<PatternAndFn> check_patterns;
   std::vector<Results> results;
   std::vector<std::string> field_names;
 
   void Add(const std::string& entry_pattern, ResultsCheckFn fn);
 
   void CheckResults(std::stringstream& output);
 
  private:
   void SetHeader_(const std::string& csv_header);
   void SetValues_(const std::string& entry_csv_line);
 
   std::vector<std::string> SplitCsv_(const std::string& line);
 };
 
 // store the static ResultsChecker in a function to prevent initialization
 // order problems
 ResultsChecker& GetResultsChecker() {
   static ResultsChecker rc;
   return rc;
 }
 
 // add a results checker for a benchmark
 void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
   check_patterns.emplace_back(entry_pattern, fn);
 }
 
 // check the results of all subscribed benchmarks
 void ResultsChecker::CheckResults(std::stringstream& output) {
   // first reset the stream to the start
   {
     auto start = std::stringstream::pos_type(0);
     // clear before calling tellg()
     output.clear();
     // seek to zero only when needed
     if (output.tellg() > start) output.seekg(start);
     // and just in case
     output.clear();
   }
   // now go over every line and publish it to the ResultsChecker
   std::string line;
   bool on_first = true;
   while (output.eof() == false) {
     CHECK(output.good());
     std::getline(output, line);
     if (on_first) {
       SetHeader_(line);  // this is important
       on_first = false;
       continue;
     }
     SetValues_(line);
   }
   // finally we can call the subscribed check functions
   for (const auto& p : check_patterns) {
     VLOG(2) << "--------------------------------\n";
     VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
     for (const auto& r : results) {
       if (!p.regex->Match(r.name)) {
         VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
         continue;
       } else {
         VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
       }
       VLOG(1) << "Checking results of " << r.name << ": ... \n";
       p.fn(r);
       VLOG(1) << "Checking results of " << r.name << ": OK.\n";
     }
   }
 }
 
 // prepare for the names in this header
 void ResultsChecker::SetHeader_(const std::string& csv_header) {
   field_names = SplitCsv_(csv_header);
 }
 
 // set the values for a benchmark
 void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
   if (entry_csv_line.empty()) return;  // some lines are empty
   CHECK(!field_names.empty());
   auto vals = SplitCsv_(entry_csv_line);
   CHECK_EQ(vals.size(), field_names.size());
   results.emplace_back(vals[0]);  // vals[0] is the benchmark name
   auto& entry = results.back();
   for (size_t i = 1, e = vals.size(); i < e; ++i) {
     entry.values[field_names[i]] = vals[i];
   }
 }
 
 // a quick'n'dirty csv splitter (eliminating quotes)
 std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) {
   std::vector<std::string> out;
   if (line.empty()) return out;
   if (!field_names.empty()) out.reserve(field_names.size());
   size_t prev = 0, pos = line.find_first_of(','), curr = pos;
   while (pos != line.npos) {
     CHECK(curr > 0);
     if (line[prev] == '"') ++prev;
     if (line[curr - 1] == '"') --curr;
     out.push_back(line.substr(prev, curr - prev));
     prev = pos + 1;
     pos = line.find_first_of(',', pos + 1);
     curr = pos;
   }
   curr = line.size();
   if (line[prev] == '"') ++prev;
   if (line[curr - 1] == '"') --curr;
   out.push_back(line.substr(prev, curr - prev));
   return out;
 }
 
 }  // end namespace internal
 
 size_t AddChecker(const char* bm_name, ResultsCheckFn fn) {
   auto& rc = internal::GetResultsChecker();
   rc.Add(bm_name, fn);
   return rc.results.size();
 }
 
 int Results::NumThreads() const {
   auto pos = name.find("/threads:");
   if (pos == name.npos) return 1;
   auto end = name.find('/', pos + 9);
   std::stringstream ss;
   ss << name.substr(pos + 9, end);
   int num = 1;
   ss >> num;
   CHECK(!ss.fail());
   return num;
 }
 
 double Results::NumIterations() const {
   return GetAs<double>("iterations");
 }
 
 double Results::GetTime(BenchmarkTime which) const {
   CHECK(which == kCpuTime || which == kRealTime);
   const char* which_str = which == kCpuTime ? "cpu_time" : "real_time";
   double val = GetAs<double>(which_str);
   auto unit = Get("time_unit");
   CHECK(unit);
   if (*unit == "ns") {
     return val * 1.e-9;
   } else if (*unit == "us") {
     return val * 1.e-6;
   } else if (*unit == "ms") {
     return val * 1.e-3;
   } else if (*unit == "s") {
     return val;
   } else {
     CHECK(1 == 0) << "unknown time unit: " << *unit;
     return 0;
   }
 }
 
 // ========================================================================= //
 // -------------------------- Public API Definitions------------------------ //
 // ========================================================================= //
 
 TestCase::TestCase(std::string re, int rule)
     : regex_str(std::move(re)),
       match_rule(rule),
       substituted_regex(internal::PerformSubstitutions(regex_str)),
       regex(std::make_shared<benchmark::Regex>()) {
   std::string err_str;
   regex->Init(substituted_regex, &err_str);
   CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
                          << "\""
                          << "\n    originally \"" << regex_str << "\""
                          << "\n    got error: " << err_str;
 }
 
 int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
   auto& L = internal::GetTestCaseList(ID);
   L.insert(L.end(), il);
   return 0;
 }
 
 int SetSubstitutions(
     std::initializer_list<std::pair<std::string, std::string>> il) {
   auto& subs = internal::GetSubstitutions();
   for (auto KV : il) {
     bool exists = false;
     KV.second = internal::PerformSubstitutions(KV.second);
     for (auto& EKV : subs) {
       if (EKV.first == KV.first) {
         EKV.second = std::move(KV.second);
         exists = true;
         break;
       }
     }
     if (!exists) subs.push_back(std::move(KV));
   }
   return 0;
 }
 
 void RunOutputTests(int argc, char* argv[]) {
   using internal::GetTestCaseList;
   benchmark::Initialize(&argc, argv);
   auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true);
   benchmark::ConsoleReporter CR(options);
   benchmark::JSONReporter JR;
   benchmark::CSVReporter CSVR;
   struct ReporterTest {
     const char* name;
     std::vector<TestCase>& output_cases;
     std::vector<TestCase>& error_cases;
     benchmark::BenchmarkReporter& reporter;
     std::stringstream out_stream;
     std::stringstream err_stream;
 
     ReporterTest(const char* n, std::vector<TestCase>& out_tc,
                  std::vector<TestCase>& err_tc,
                  benchmark::BenchmarkReporter& br)
         : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
       reporter.SetOutputStream(&out_stream);
       reporter.SetErrorStream(&err_stream);
     }
   } TestCases[] = {
       {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
        GetTestCaseList(TC_ConsoleErr), CR},
       {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
        JR},
       {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
        CSVR},
   };
 
   // Create the test reporter and run the benchmarks.
   std::cout << "Running benchmarks...\n";
   internal::TestReporter test_rep({&CR, &JR, &CSVR});
   benchmark::RunSpecifiedBenchmarks(&test_rep);
 
   for (auto& rep_test : TestCases) {
     std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
     std::string banner(msg.size() - 1, '-');
     std::cout << banner << msg << banner << "\n";
 
     std::cerr << rep_test.err_stream.str();
     std::cout << rep_test.out_stream.str();
 
     internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
     internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
 
     std::cout << "\n";
   }
 
   // now that we know the output is as expected, we can dispatch
   // the checks to subscribees.
   auto& csv = TestCases[2];
   // would use == but gcc spits a warning
   CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
   internal::GetResultsChecker().CheckResults(csv.out_stream);
 }
 
 int SubstrCnt(const std::string& haystack, const std::string& pat) {
   if (pat.length() == 0) return 0;
   int count = 0;
   for (size_t offset = haystack.find(pat); offset != std::string::npos;
        offset = haystack.find(pat, offset + pat.length()))
     ++count;
   return count;
 }
 
 static char ToHex(int ch) {
   return ch < 10 ? static_cast<char>('0' + ch)
                  : static_cast<char>('a' + (ch - 10));
 }
 
 static char RandomHexChar() {
   static std::mt19937 rd{std::random_device{}()};
   static std::uniform_int_distribution<int> mrand{0, 15};
   return ToHex(mrand(rd));
 }
 
 static std::string GetRandomFileName() {
   std::string model = "test.%%%%%%";
   for (auto & ch :  model) {
     if (ch == '%')
       ch = RandomHexChar();
   }
   return model;
 }
 
 static bool FileExists(std::string const& name) {
   std::ifstream in(name.c_str());
   return in.good();
 }
 
 static std::string GetTempFileName() {
   // This function attempts to avoid race conditions where two tests
   // create the same file at the same time. However, it still introduces races
   // similar to tmpnam.
   int retries = 3;
   while (--retries) {
     std::string name = GetRandomFileName();
     if (!FileExists(name))
       return name;
   }
   std::cerr << "Failed to create unique temporary file name" << std::endl;
   std::abort();
 }
 
 std::string GetFileReporterOutput(int argc, char* argv[]) {
   std::vector<char*> new_argv(argv, argv + argc);
   assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
 
   std::string tmp_file_name = GetTempFileName();
   std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
 
   std::string tmp = "--benchmark_out=";
   tmp += tmp_file_name;
   new_argv.emplace_back(const_cast<char*>(tmp.c_str()));
 
   argc = int(new_argv.size());
 
   benchmark::Initialize(&argc, new_argv.data());
   benchmark::RunSpecifiedBenchmarks();
 
   // Read the output back from the file, and delete the file.
   std::ifstream tmp_stream(tmp_file_name);
   std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
                                    std::istreambuf_iterator<char>());
   std::remove(tmp_file_name.c_str());
 
   return output;
 }