You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
432 lines
18 KiB
432 lines
18 KiB
#!/usr/bin/env python3
|
|
|
|
import unittest
|
|
"""
|
|
compare.py - versatile benchmark output compare tool
|
|
"""
|
|
|
|
import argparse
|
|
from argparse import ArgumentParser
|
|
import json
|
|
import sys
|
|
import os
|
|
import gbench
|
|
from gbench import util, report
|
|
|
|
|
|
def check_inputs(in1, in2, flags):
|
|
"""
|
|
Perform checking on the user provided inputs and diagnose any abnormalities
|
|
"""
|
|
in1_kind, in1_err = util.classify_input_file(in1)
|
|
in2_kind, in2_err = util.classify_input_file(in2)
|
|
output_file = util.find_benchmark_flag('--benchmark_out=', flags)
|
|
output_type = util.find_benchmark_flag('--benchmark_out_format=', flags)
|
|
if in1_kind == util.IT_Executable and in2_kind == util.IT_Executable and output_file:
|
|
print(("WARNING: '--benchmark_out=%s' will be passed to both "
|
|
"benchmarks causing it to be overwritten") % output_file)
|
|
if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
|
|
# When both sides are JSON the only supported flag is
|
|
# --benchmark_filter=
|
|
for flag in util.remove_benchmark_flags('--benchmark_filter=', flags):
|
|
print("WARNING: passing %s has no effect since both "
|
|
"inputs are JSON" % flag)
|
|
if output_type is not None and output_type != 'json':
|
|
print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
|
|
" is not supported.") % output_type)
|
|
sys.exit(1)
|
|
|
|
|
|
def create_parser():
|
|
parser = ArgumentParser(
|
|
description='versatile benchmark output compare tool')
|
|
|
|
parser.add_argument(
|
|
'-a',
|
|
'--display_aggregates_only',
|
|
dest='display_aggregates_only',
|
|
action="store_true",
|
|
help="If there are repetitions, by default, we display everything - the"
|
|
" actual runs, and the aggregates computed. Sometimes, it is "
|
|
"desirable to only view the aggregates. E.g. when there are a lot "
|
|
"of repetitions. Do note that only the display is affected. "
|
|
"Internally, all the actual runs are still used, e.g. for U test.")
|
|
|
|
parser.add_argument(
|
|
'--no-color',
|
|
dest='color',
|
|
default=True,
|
|
action="store_false",
|
|
help="Do not use colors in the terminal output"
|
|
)
|
|
|
|
parser.add_argument(
|
|
'-d',
|
|
'--dump_to_json',
|
|
dest='dump_to_json',
|
|
help="Additionally, dump benchmark comparison output to this file in JSON format.")
|
|
|
|
utest = parser.add_argument_group()
|
|
utest.add_argument(
|
|
'--no-utest',
|
|
dest='utest',
|
|
default=True,
|
|
action="store_false",
|
|
help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS))
|
|
alpha_default = 0.05
|
|
utest.add_argument(
|
|
"--alpha",
|
|
dest='utest_alpha',
|
|
default=alpha_default,
|
|
type=float,
|
|
help=("significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)") %
|
|
alpha_default)
|
|
|
|
subparsers = parser.add_subparsers(
|
|
help='This tool has multiple modes of operation:',
|
|
dest='mode')
|
|
|
|
parser_a = subparsers.add_parser(
|
|
'benchmarks',
|
|
help='The most simple use-case, compare all the output of these two benchmarks')
|
|
baseline = parser_a.add_argument_group(
|
|
'baseline', 'The benchmark baseline')
|
|
baseline.add_argument(
|
|
'test_baseline',
|
|
metavar='test_baseline',
|
|
type=argparse.FileType('r'),
|
|
nargs=1,
|
|
help='A benchmark executable or JSON output file')
|
|
contender = parser_a.add_argument_group(
|
|
'contender', 'The benchmark that will be compared against the baseline')
|
|
contender.add_argument(
|
|
'test_contender',
|
|
metavar='test_contender',
|
|
type=argparse.FileType('r'),
|
|
nargs=1,
|
|
help='A benchmark executable or JSON output file')
|
|
parser_a.add_argument(
|
|
'benchmark_options',
|
|
metavar='benchmark_options',
|
|
nargs=argparse.REMAINDER,
|
|
help='Arguments to pass when running benchmark executables')
|
|
|
|
parser_b = subparsers.add_parser(
|
|
'filters', help='Compare filter one with the filter two of benchmark')
|
|
baseline = parser_b.add_argument_group(
|
|
'baseline', 'The benchmark baseline')
|
|
baseline.add_argument(
|
|
'test',
|
|
metavar='test',
|
|
type=argparse.FileType('r'),
|
|
nargs=1,
|
|
help='A benchmark executable or JSON output file')
|
|
baseline.add_argument(
|
|
'filter_baseline',
|
|
metavar='filter_baseline',
|
|
type=str,
|
|
nargs=1,
|
|
help='The first filter, that will be used as baseline')
|
|
contender = parser_b.add_argument_group(
|
|
'contender', 'The benchmark that will be compared against the baseline')
|
|
contender.add_argument(
|
|
'filter_contender',
|
|
metavar='filter_contender',
|
|
type=str,
|
|
nargs=1,
|
|
help='The second filter, that will be compared against the baseline')
|
|
parser_b.add_argument(
|
|
'benchmark_options',
|
|
metavar='benchmark_options',
|
|
nargs=argparse.REMAINDER,
|
|
help='Arguments to pass when running benchmark executables')
|
|
|
|
parser_c = subparsers.add_parser(
|
|
'benchmarksfiltered',
|
|
help='Compare filter one of first benchmark with filter two of the second benchmark')
|
|
baseline = parser_c.add_argument_group(
|
|
'baseline', 'The benchmark baseline')
|
|
baseline.add_argument(
|
|
'test_baseline',
|
|
metavar='test_baseline',
|
|
type=argparse.FileType('r'),
|
|
nargs=1,
|
|
help='A benchmark executable or JSON output file')
|
|
baseline.add_argument(
|
|
'filter_baseline',
|
|
metavar='filter_baseline',
|
|
type=str,
|
|
nargs=1,
|
|
help='The first filter, that will be used as baseline')
|
|
contender = parser_c.add_argument_group(
|
|
'contender', 'The benchmark that will be compared against the baseline')
|
|
contender.add_argument(
|
|
'test_contender',
|
|
metavar='test_contender',
|
|
type=argparse.FileType('r'),
|
|
nargs=1,
|
|
help='The second benchmark executable or JSON output file, that will be compared against the baseline')
|
|
contender.add_argument(
|
|
'filter_contender',
|
|
metavar='filter_contender',
|
|
type=str,
|
|
nargs=1,
|
|
help='The second filter, that will be compared against the baseline')
|
|
parser_c.add_argument(
|
|
'benchmark_options',
|
|
metavar='benchmark_options',
|
|
nargs=argparse.REMAINDER,
|
|
help='Arguments to pass when running benchmark executables')
|
|
|
|
return parser
|
|
|
|
|
|
def main():
|
|
# Parse the command line flags
|
|
parser = create_parser()
|
|
args, unknown_args = parser.parse_known_args()
|
|
if args.mode is None:
|
|
parser.print_help()
|
|
exit(1)
|
|
assert not unknown_args
|
|
benchmark_options = args.benchmark_options
|
|
|
|
if args.mode == 'benchmarks':
|
|
test_baseline = args.test_baseline[0].name
|
|
test_contender = args.test_contender[0].name
|
|
filter_baseline = ''
|
|
filter_contender = ''
|
|
|
|
# NOTE: if test_baseline == test_contender, you are analyzing the stdev
|
|
|
|
description = 'Comparing %s to %s' % (test_baseline, test_contender)
|
|
elif args.mode == 'filters':
|
|
test_baseline = args.test[0].name
|
|
test_contender = args.test[0].name
|
|
filter_baseline = args.filter_baseline[0]
|
|
filter_contender = args.filter_contender[0]
|
|
|
|
# NOTE: if filter_baseline == filter_contender, you are analyzing the
|
|
# stdev
|
|
|
|
description = 'Comparing %s to %s (from %s)' % (
|
|
filter_baseline, filter_contender, args.test[0].name)
|
|
elif args.mode == 'benchmarksfiltered':
|
|
test_baseline = args.test_baseline[0].name
|
|
test_contender = args.test_contender[0].name
|
|
filter_baseline = args.filter_baseline[0]
|
|
filter_contender = args.filter_contender[0]
|
|
|
|
# NOTE: if test_baseline == test_contender and
|
|
# filter_baseline == filter_contender, you are analyzing the stdev
|
|
|
|
description = 'Comparing %s (from %s) to %s (from %s)' % (
|
|
filter_baseline, test_baseline, filter_contender, test_contender)
|
|
else:
|
|
# should never happen
|
|
print("Unrecognized mode of operation: '%s'" % args.mode)
|
|
parser.print_help()
|
|
exit(1)
|
|
|
|
check_inputs(test_baseline, test_contender, benchmark_options)
|
|
|
|
if args.display_aggregates_only:
|
|
benchmark_options += ['--benchmark_display_aggregates_only=true']
|
|
|
|
options_baseline = []
|
|
options_contender = []
|
|
|
|
if filter_baseline and filter_contender:
|
|
options_baseline = ['--benchmark_filter=%s' % filter_baseline]
|
|
options_contender = ['--benchmark_filter=%s' % filter_contender]
|
|
|
|
# Run the benchmarks and report the results
|
|
json1 = json1_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark(
|
|
test_baseline, benchmark_options + options_baseline))
|
|
json2 = json2_orig = gbench.util.sort_benchmark_results(gbench.util.run_or_load_benchmark(
|
|
test_contender, benchmark_options + options_contender))
|
|
|
|
# Now, filter the benchmarks so that the difference report can work
|
|
if filter_baseline and filter_contender:
|
|
replacement = '[%s vs. %s]' % (filter_baseline, filter_contender)
|
|
json1 = gbench.report.filter_benchmark(
|
|
json1_orig, filter_baseline, replacement)
|
|
json2 = gbench.report.filter_benchmark(
|
|
json2_orig, filter_contender, replacement)
|
|
|
|
diff_report = gbench.report.get_difference_report(
|
|
json1, json2, args.utest)
|
|
output_lines = gbench.report.print_difference_report(
|
|
diff_report,
|
|
args.display_aggregates_only,
|
|
args.utest, args.utest_alpha, args.color)
|
|
print(description)
|
|
for ln in output_lines:
|
|
print(ln)
|
|
|
|
# Optionally, diff and output to JSON
|
|
if args.dump_to_json is not None:
|
|
with open(args.dump_to_json, 'w') as f_json:
|
|
json.dump(diff_report, f_json)
|
|
|
|
class TestParser(unittest.TestCase):
|
|
def setUp(self):
|
|
self.parser = create_parser()
|
|
testInputs = os.path.join(
|
|
os.path.dirname(
|
|
os.path.realpath(__file__)),
|
|
'gbench',
|
|
'Inputs')
|
|
self.testInput0 = os.path.join(testInputs, 'test1_run1.json')
|
|
self.testInput1 = os.path.join(testInputs, 'test1_run2.json')
|
|
|
|
def test_benchmarks_basic(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarks', self.testInput0, self.testInput1])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_without_utest(self):
|
|
parsed = self.parser.parse_args(
|
|
['--no-utest', 'benchmarks', self.testInput0, self.testInput1])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertFalse(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.05)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_display_aggregates_only(self):
|
|
parsed = self.parser.parse_args(
|
|
['-a', 'benchmarks', self.testInput0, self.testInput1])
|
|
self.assertTrue(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_with_utest_alpha(self):
|
|
parsed = self.parser.parse_args(
|
|
['--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.314)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_basic_without_utest_with_utest_alpha(self):
|
|
parsed = self.parser.parse_args(
|
|
['--no-utest', '--alpha=0.314', 'benchmarks', self.testInput0, self.testInput1])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertFalse(parsed.utest)
|
|
self.assertEqual(parsed.utest_alpha, 0.314)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarks_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarks', self.testInput0, self.testInput1, 'd'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.benchmark_options, ['d'])
|
|
|
|
def test_benchmarks_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarks')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.benchmark_options, ['e'])
|
|
|
|
def test_filters_basic(self):
|
|
parsed = self.parser.parse_args(
|
|
['filters', self.testInput0, 'c', 'd'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'filters')
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.filter_contender[0], 'd')
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_filters_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
['filters', self.testInput0, 'c', 'd', 'e'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'filters')
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.filter_contender[0], 'd')
|
|
self.assertEqual(parsed.benchmark_options, ['e'])
|
|
|
|
def test_filters_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
['filters', self.testInput0, 'c', 'd', '--', 'f'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'filters')
|
|
self.assertEqual(parsed.test[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.filter_contender[0], 'd')
|
|
self.assertEqual(parsed.benchmark_options, ['f'])
|
|
|
|
def test_benchmarksfiltered_basic(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], 'e')
|
|
self.assertFalse(parsed.benchmark_options)
|
|
|
|
def test_benchmarksfiltered_with_remainder(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], 'e')
|
|
self.assertEqual(parsed.benchmark_options[0], 'f')
|
|
|
|
def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
|
|
parsed = self.parser.parse_args(
|
|
['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
|
|
self.assertFalse(parsed.display_aggregates_only)
|
|
self.assertTrue(parsed.utest)
|
|
self.assertEqual(parsed.mode, 'benchmarksfiltered')
|
|
self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
|
|
self.assertEqual(parsed.filter_baseline[0], 'c')
|
|
self.assertEqual(parsed.test_contender[0].name, self.testInput1)
|
|
self.assertEqual(parsed.filter_contender[0], 'e')
|
|
self.assertEqual(parsed.benchmark_options[0], 'g')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# unittest.main()
|
|
main()
|
|
|
|
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
|
# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
|
|
# kate: indent-mode python; remove-trailing-spaces modified;
|