You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
211 lines
7.7 KiB
211 lines
7.7 KiB
#ifndef TEST_OUTPUT_TEST_H
|
|
#define TEST_OUTPUT_TEST_H
|
|
|
|
#undef NDEBUG
|
|
#include <functional>
|
|
#include <initializer_list>
|
|
#include <memory>
|
|
#include <sstream>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "../src/re.h"
|
|
#include "benchmark/benchmark.h"
|
|
|
|
#define CONCAT2(x, y) x##y
|
|
#define CONCAT(x, y) CONCAT2(x, y)
|
|
|
|
#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__)
|
|
|
|
#define SET_SUBSTITUTIONS(...) \
|
|
int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__)
|
|
|
|
enum MatchRules {
|
|
MR_Default, // Skip non-matching lines until a match is found.
|
|
MR_Next, // Match must occur on the next line.
|
|
MR_Not // No line between the current position and the next match matches
|
|
// the regex
|
|
};
|
|
|
|
struct TestCase {
|
|
TestCase(std::string re, int rule = MR_Default);
|
|
|
|
std::string regex_str;
|
|
int match_rule;
|
|
std::string substituted_regex;
|
|
std::shared_ptr<benchmark::Regex> regex;
|
|
};
|
|
|
|
enum TestCaseID {
|
|
TC_ConsoleOut,
|
|
TC_ConsoleErr,
|
|
TC_JSONOut,
|
|
TC_JSONErr,
|
|
TC_CSVOut,
|
|
TC_CSVErr,
|
|
|
|
TC_NumID // PRIVATE
|
|
};
|
|
|
|
// Add a list of test cases to be run against the output specified by
|
|
// 'ID'
|
|
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il);
|
|
|
|
// Add or set a list of substitutions to be performed on constructed regex's
|
|
// See 'output_test_helper.cc' for a list of default substitutions.
|
|
int SetSubstitutions(
|
|
std::initializer_list<std::pair<std::string, std::string>> il);
|
|
|
|
// Run all output tests.
|
|
void RunOutputTests(int argc, char* argv[]);
|
|
|
|
// Count the number of 'pat' substrings in the 'haystack' string.
|
|
int SubstrCnt(const std::string& haystack, const std::string& pat);
|
|
|
|
// Run registered benchmarks with file reporter enabled, and return the content
|
|
// outputted by the file reporter.
|
|
std::string GetFileReporterOutput(int argc, char* argv[]);
|
|
|
|
// ========================================================================= //
|
|
// ------------------------- Results checking ------------------------------ //
|
|
// ========================================================================= //
|
|
|
|
// Call this macro to register a benchmark for checking its results. This
|
|
// should be all that's needed. It subscribes a function to check the (CSV)
|
|
// results of a benchmark. This is done only after verifying that the output
|
|
// strings are really as expected.
|
|
// bm_name_pattern: a name or a regex pattern which will be matched against
|
|
// all the benchmark names. Matching benchmarks
|
|
// will be the subject of a call to checker_function
|
|
// checker_function: should be of type ResultsCheckFn (see below)
|
|
#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \
|
|
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function)
|
|
|
|
struct Results;
|
|
typedef std::function<void(Results const&)> ResultsCheckFn;
|
|
|
|
size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn);
|
|
|
|
// Class holding the results of a benchmark.
|
|
// It is passed in calls to checker functions.
|
|
struct Results {
|
|
// the benchmark name
|
|
std::string name;
|
|
// the benchmark fields
|
|
std::map<std::string, std::string> values;
|
|
|
|
Results(const std::string& n) : name(n) {}
|
|
|
|
int NumThreads() const;
|
|
|
|
double NumIterations() const;
|
|
|
|
typedef enum { kCpuTime, kRealTime } BenchmarkTime;
|
|
|
|
// get cpu_time or real_time in seconds
|
|
double GetTime(BenchmarkTime which) const;
|
|
|
|
// get the real_time duration of the benchmark in seconds.
|
|
// it is better to use fuzzy float checks for this, as the float
|
|
// ASCII formatting is lossy.
|
|
double DurationRealTime() const {
|
|
return NumIterations() * GetTime(kRealTime);
|
|
}
|
|
// get the cpu_time duration of the benchmark in seconds
|
|
double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); }
|
|
|
|
// get the string for a result by name, or nullptr if the name
|
|
// is not found
|
|
const std::string* Get(const std::string& entry_name) const {
|
|
auto it = values.find(entry_name);
|
|
if (it == values.end()) return nullptr;
|
|
return &it->second;
|
|
}
|
|
|
|
// get a result by name, parsed as a specific type.
|
|
// NOTE: for counters, use GetCounterAs instead.
|
|
template <class T>
|
|
T GetAs(const std::string& entry_name) const;
|
|
|
|
// counters are written as doubles, so they have to be read first
|
|
// as a double, and only then converted to the asked type.
|
|
template <class T>
|
|
T GetCounterAs(const std::string& entry_name) const {
|
|
double dval = GetAs<double>(entry_name);
|
|
T tval = static_cast<T>(dval);
|
|
return tval;
|
|
}
|
|
};
|
|
|
|
template <class T>
|
|
T Results::GetAs(const std::string& entry_name) const {
|
|
auto* sv = Get(entry_name);
|
|
BM_CHECK(sv != nullptr && !sv->empty());
|
|
std::stringstream ss;
|
|
ss << *sv;
|
|
T out;
|
|
ss >> out;
|
|
BM_CHECK(!ss.fail());
|
|
return out;
|
|
}
|
|
|
|
//----------------------------------
|
|
// Macros to help in result checking. Do not use them with arguments causing
|
|
// side-effects.
|
|
|
|
// clang-format off
|
|
|
|
#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \
|
|
CONCAT(BM_CHECK_, relationship) \
|
|
(entry.getfn< var_type >(var_name), (value)) << "\n" \
|
|
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
|
|
<< __FILE__ << ":" << __LINE__ << ": " \
|
|
<< "expected (" << #var_type << ")" << (var_name) \
|
|
<< "=" << (entry).getfn< var_type >(var_name) \
|
|
<< " to be " #relationship " to " << (value) << "\n"
|
|
|
|
// check with tolerance. eps_factor is the tolerance window, which is
|
|
// interpreted relative to value (eg, 0.1 means 10% of value).
|
|
#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \
|
|
CONCAT(BM_CHECK_FLOAT_, relationship) \
|
|
(entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \
|
|
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \
|
|
<< __FILE__ << ":" << __LINE__ << ": " \
|
|
<< "expected (" << #var_type << ")" << (var_name) \
|
|
<< "=" << (entry).getfn< var_type >(var_name) \
|
|
<< " to be " #relationship " to " << (value) << "\n" \
|
|
<< __FILE__ << ":" << __LINE__ << ": " \
|
|
<< "with tolerance of " << (eps_factor) * (value) \
|
|
<< " (" << (eps_factor)*100. << "%), " \
|
|
<< "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \
|
|
<< " (" << (((entry).getfn< var_type >(var_name) - (value)) \
|
|
/ \
|
|
((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \
|
|
<< "%)"
|
|
|
|
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \
|
|
CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value)
|
|
|
|
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \
|
|
CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value)
|
|
|
|
#define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \
|
|
CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor)
|
|
|
|
#define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \
|
|
CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor)
|
|
|
|
// clang-format on
|
|
|
|
// ========================================================================= //
|
|
// --------------------------- Misc Utilities ------------------------------ //
|
|
// ========================================================================= //
|
|
|
|
namespace {
|
|
|
|
const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
|
|
|
|
} // end namespace
|
|
|
|
#endif // TEST_OUTPUT_TEST_H
|