Added script to run benchmarks.

1 files changed, 94 insertions, 0 deletions

diff --git a/meson_benchmark.py b/meson_benchmark.py
new file mode 100755
index 0000000..3cde98b
--- /dev/null
+++ b/meson_benchmark.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+# Copyright 2015 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess, sys, os, argparse
+import pickle, statistics, json
+import meson_test
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--wd', default=None, dest='wd',
+                    help='directory to cd into before running')
+parser.add_argument('args', nargs='+')
+
+def print_stats(numlen, num_tests, name, res, i, duration, stdev):
+    startpad = ' '*(numlen - len('%d' % (i+1)))
+    num = '%s%d/%d' % (startpad, i+1, num_tests)
+    padding1 = ' '*(38-len(name))
+    padding2 = ' '*(8-len(res))
+    result_str = '%s %s  %s%s%s%5.5f s +- %5.5f s' % \
+        (num, name, padding1, res, padding2, duration, stdev)
+    print(result_str)
+#    write_json_log(jsonlogfile, name, result)
+
+def print_json_log(jsonlogfile, rawruns, test_name, i):
+    jsonobj = {'name' : test_name}
+    runs = []
+    for r in rawruns:
+        runobj = {'duration': r.duration,
+                  'stdout': r.stdo,
+                  'stderr': r.stde,
+                  'returncode' : r.returncode,
+                  'duration' : r.duration}
+        runs.append(runobj)
+    jsonobj['runs'] = runs
+    jsonlogfile.write(json.dumps(jsonobj) + '\n')
+    jsonlogfile.flush()
+
+def run_benchmarks(options, datafile):
+    failed_tests = 0
+    logfile_base = 'meson-logs/benchmarklog'
+    jsonlogfilename = logfile_base+ '.json'
+    jsonlogfile = open(jsonlogfilename, 'w')
+    tests = pickle.load(open(datafile, 'rb'))
+    num_tests = len(tests)
+    iteration_count = 5
+    wrap = [] # Benchmarks on cross builds are pointless so don't support them.
+    for i, test in enumerate(tests):
+        runs = []
+        durations = []
+        failed = False
+        for _ in range(iteration_count):
+            res = meson_test.run_single_test(wrap, test)
+            runs.append(res)
+            durations.append(res.duration)
+            if res.returncode != 0:
+                failed = True
+        mean = statistics.mean(durations)
+        stddev = statistics.stdev(durations)
+        if failed:
+            resultstr = 'FAIL'
+            failed_tests += 1
+        else:
+            resultstr = 'OK'
+        print_stats(3, num_tests, test.name, resultstr, i, mean, stddev)
+        print_json_log(jsonlogfile, runs, test.name, i)
+    return failed_tests
+
+def run(args):
+    global failed_tests
+    options = parser.parse_args(args)
+    if len(options.args) != 1:
+        print('Benchmark runner for Meson. Do not run on your own, mmm\'kay?')
+        print('%s [data file]' % sys.argv[0])
+    if options.wd is not None:
+        os.chdir(options.wd)
+    datafile = options.args[0]
+    returncode = run_benchmarks(options, datafile)
+    print('\nFull log written to meson-logs/benchmarklog.json.')
+    return returncode
+
+if __name__ == '__main__':
+    sys.exit(run(sys.argv[1:]))