diff options
| -rw-r--r-- | tests/perf_bench/benchrun.py | 26 | ||||
| -rwxr-xr-x | tests/run-perfbench.py | 241 |
2 files changed, 267 insertions, 0 deletions
diff --git a/tests/perf_bench/benchrun.py b/tests/perf_bench/benchrun.py new file mode 100644 index 000000000..9cbc9695a --- /dev/null +++ b/tests/perf_bench/benchrun.py @@ -0,0 +1,26 @@ +def bm_run(N, M): + try: + from utime import ticks_us, ticks_diff + except ImportError: + import time + ticks_us = lambda: int(time.perf_counter() * 1000000) + ticks_diff = lambda a, b: a - b + + # Pick sensible parameters given N, M + cur_nm = (0, 0) + param = None + for nm, p in bm_params.items(): + if 10 * nm[0] <= 12 * N and nm[1] <= M and nm > cur_nm: + cur_nm = nm + param = p + if param is None: + print(-1, -1, 'no matching params') + return + + # Run and time benchmark + run, result = bm_setup(param) + t0 = ticks_us() + run() + t1 = ticks_us() + norm, out = result() + print(ticks_diff(t1, t0), norm, out) diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py new file mode 100755 index 000000000..d8100ef81 --- /dev/null +++ b/tests/run-perfbench.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +# This file is part of the MicroPython project, http://micropython.org/ +# The MIT License (MIT) +# Copyright (c) 2019 Damien P. George + +import os +import subprocess +import sys +import argparse +from glob import glob + +sys.path.append('../tools') +import pyboard + +# Paths for host executables +if os.name == 'nt': + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe') +else: + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython') + +PYTHON_TRUTH = CPYTHON3 + +BENCH_SCRIPT_DIR = 'perf_bench/' + +def compute_stats(lst): + avg = 0 + var = 0 + for x in lst: + avg += x + var += x * x + avg /= len(lst) + var = max(0, var / len(lst) - avg ** 2) + return avg, var ** 0.5 + +def run_script_on_target(target, script): + output = b'' + err = None + + if isinstance(target, pyboard.Pyboard): + # Run via pyboard interface + try: + target.enter_raw_repl() + output = target.exec_(script) + except pyboard.PyboardError as er: + err = er + else: + # Run local executable + try: + p = subprocess.run([target], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script) + output = p.stdout + except subprocess.CalledProcessError as er: + err = er + + return str(output.strip(), 'ascii'), err + +def run_feature_test(target, test): + with open('feature_check/' + test + '.py', 'rb') as f: + script = f.read() + output, err = run_script_on_target(target, script) + if err is None: + return output + else: + return 'CRASH: %r' % err + +def run_benchmark_on_target(target, script): + output, err = run_script_on_target(target, script) + if err is None: + time, norm, result = output.split(None, 2) + try: + return int(time), int(norm), result + except ValueError: + return -1, -1, 'CRASH: %r' % output + else: + return -1, -1, 'CRASH: %r' % err + +def run_benchmarks(target, param_n, param_m, n_average, test_list): + skip_native = run_feature_test(target, 'native_check') != '' + + for test_file in sorted(test_list): + print(test_file + ': ', end='') + + # Check if test should be skipped + skip = skip_native and test_file.find('viper_') != -1 + if skip: + print('skip') + continue + + # Create test script + with open(test_file, 'rb') as f: + test_script = f.read() + with open(BENCH_SCRIPT_DIR + 'benchrun.py', 'rb') as f: + test_script += f.read() + test_script += b'bm_run(%u, %u)\n' % (param_n, param_m) + + # Write full test script if needed + if 0: + with open('%s.full' % test_file, 'wb') as f: + f.write(test_script) + + # Run MicroPython a given number of times + times = [] + scores = [] + error = None + result_out = None + for _ in range(n_average): + time, norm, result = run_benchmark_on_target(target, test_script) + if time < 0 or norm < 0: + error = result + break + if result_out is None: + result_out = result + elif result != result_out: + error = 'FAIL self' + break + times.append(time) + scores.append(1e6 * norm / time) + + # Check result against truth if needed + if error is None and result_out != 'None': + _, _, result_exp = run_benchmark_on_target(PYTHON_TRUTH, test_script) + if result_out != result_exp: + error = 'FAIL truth' + break + + if error is not None: + print(error) + else: + t_avg, t_sd = compute_stats(times) + s_avg, s_sd = compute_stats(scores) + print('{:.2f} {:.4f} {:.2f} {:.4f}'.format(t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg)) + if 0: + print(' times: ', times) + print(' scores:', scores) + + sys.stdout.flush() + +def parse_output(filename): + with open(filename) as f: + params = f.readline() + n, m, _ = params.strip().split() + n = int(n.split('=')[1]) + m = int(m.split('=')[1]) + data = [] + for l in f: + if l.find(': ') != -1 and l.find(': skip') == -1 and l.find('CRASH: ') == -1: + name, values = l.strip().split(': ') + values = tuple(float(v) for v in values.split()) + data.append((name,) + values) + return n, m, data + +def compute_diff(file1, file2, diff_score): + # Parse output data from previous runs + n1, m1, d1 = parse_output(file1) + n2, m2, d2 = parse_output(file2) + + # Print header + if diff_score: + print('diff of scores (higher is better)') + else: + print('diff of microsecond times (lower is better)') + if n1 == n2 and m1 == m2: + hdr = 'N={} M={}'.format(n1, m1) + else: + hdr = 'N={} M={} vs N={} M={}'.format(n1, m1, n2, m2) + print('{:24} {:>10} -> {:>10} {:>10} {:>7}% (error%)'.format(hdr, file1, file2, 'diff', 'diff')) + + # Print entries + while d1 and d2: + if d1[0][0] == d2[0][0]: + # Found entries with matching names + entry1 = d1.pop(0) + entry2 = d2.pop(0) + name = entry1[0].rsplit('/')[-1] + av1, sd1 = entry1[1 + 2 * diff_score], entry1[2 + 2 * diff_score] + av2, sd2 = entry2[1 + 2 * diff_score], entry2[2 + 2 * diff_score] + sd1 *= av1 / 100 # convert from percent sd to absolute sd + sd2 *= av2 / 100 # convert from percent sd to absolute sd + av_diff = av2 - av1 + sd_diff = (sd1 ** 2 + sd2 ** 2) ** 0.5 + percent = 100 * av_diff / av1 + percent_sd = 100 * sd_diff / av1 + print('{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)'.format(name, av1, av2, av_diff, percent, percent_sd)) + elif d1[0][0] < d2[0][0]: + d1.pop(0) + else: + d2.pop(0) + +def main(): + cmd_parser = argparse.ArgumentParser(description='Run benchmarks for MicroPython') + cmd_parser.add_argument('-t', '--diff-time', action='store_true', help='diff time outputs from a previous run') + cmd_parser.add_argument('-s', '--diff-score', action='store_true', help='diff score outputs from a previous run') + cmd_parser.add_argument('-p', '--pyboard', action='store_true', help='run tests via pyboard.py') + cmd_parser.add_argument('-d', '--device', default='/dev/ttyACM0', help='the device for pyboard.py') + cmd_parser.add_argument('-a', '--average', default='8', help='averaging number') + cmd_parser.add_argument('N', nargs=1, help='N parameter (approximate target CPU frequency)') + cmd_parser.add_argument('M', nargs=1, help='M parameter (approximate target heap in kbytes)') + cmd_parser.add_argument('files', nargs='*', help='input test files') + args = cmd_parser.parse_args() + + if args.diff_time or args.diff_score: + compute_diff(args.N[0], args.M[0], args.diff_score) + sys.exit(0) + + # N, M = 50, 25 # esp8266 + # N, M = 100, 100 # pyboard, esp32 + # N, M = 1000, 1000 # PC + N = int(args.N[0]) + M = int(args.M[0]) + n_average = int(args.average) + + if args.pyboard: + target = pyboard.Pyboard(args.device) + target.enter_raw_repl() + else: + target = MICROPYTHON + + if len(args.files) == 0: + tests_skip = ('benchrun.py',) + if M <= 25: + # These scripts are too big to be compiled by the target + tests_skip += ('bm_chaos.py', 'bm_hexiom.py', 'misc_raytrace.py') + tests = sorted( + BENCH_SCRIPT_DIR + test_file for test_file in os.listdir(BENCH_SCRIPT_DIR) + if test_file.endswith('.py') and test_file not in tests_skip + ) + else: + tests = sorted(args.files) + + print('N={} M={} n_average={}'.format(N, M, n_average)) + + run_benchmarks(target, N, M, n_average, tests) + + if isinstance(target, pyboard.Pyboard): + target.exit_raw_repl() + target.close() + +if __name__ == "__main__": + main() |