tests/run-internalbench.py: Allow running internalbench on hardware.

Signed-off-by: Anson Mansfield <amansfield@mantaro.com>

2 files changed, 109 insertions, 44 deletions

diff --git a/tests/run-internalbench.py b/tests/run-internalbench.py
index c9f783e47..99c6304af 100755
--- a/tests/run-internalbench.py
+++ b/tests/run-internalbench.py
@@ -8,6 +8,10 @@ import re
 from glob import glob
 from collections import defaultdict
 
+run_tests_module = __import__("run-tests")
+sys.path.append(run_tests_module.base_path("../tools"))
+import pyboard
+
 if os.name == "nt":
     MICROPYTHON = os.getenv(
         "MICROPY_MICROPYTHON", "../ports/windows/build-standard/micropython.exe"
@@ -15,13 +19,39 @@ if os.name == "nt":
 else:
     MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython")
 
+injected_bench_code = b"""
+import time
+
+class bench_class:
+    ITERS = 20000000
+
+    @staticmethod
+    def run(test):
+        t = time.ticks_us()
+        test(bench_class.ITERS)
+        t = time.ticks_diff(time.ticks_us(), t)
+        s, us = divmod(t, 1_000_000)
+        print("{}.{:06}".format(s, us))
+
+import sys
+sys.modules['bench'] = bench_class
+"""
+
 
-def run_tests(pyb, test_dict):
+def execbench(pyb, filename, iters):
+    with open(filename, "rb") as f:
+        pyfile = f.read()
+    code = (injected_bench_code + pyfile).replace(b"20000000", str(iters).encode("utf-8"))
+    return pyb.exec(code).replace(b"\r\n", b"\n")
+
+
+def run_tests(pyb, test_dict, iters):
     test_count = 0
     testcase_count = 0
 
     for base_test, tests in sorted(test_dict.items()):
         print(base_test + ":")
+        baseline = None
         for test_file in tests:
             # run MicroPython
             if pyb is None:
@@ -36,20 +66,25 @@ def run_tests(pyb, test_dict):
                 # run on pyboard
                 pyb.enter_raw_repl()
                 try:
-                    output_mupy = pyb.execfile(test_file).replace(b"\r\n", b"\n")
+                    output_mupy = execbench(pyb, test_file[0], iters)
                 except pyboard.PyboardError:
                     output_mupy = b"CRASH"
 
-            output_mupy = float(output_mupy.strip())
+            try:
+                output_mupy = float(output_mupy.strip())
+            except ValueError:
+                output_mupy = -1
             test_file[1] = output_mupy
             testcase_count += 1
 
-        test_count += 1
-        baseline = None
-        for t in tests:
             if baseline is None:
-                baseline = t[1]
-            print("    %.3fs (%+06.2f%%) %s" % (t[1], (t[1] * 100 / baseline) - 100, t[0]))
+                baseline = test_file[1]
+            print(
+                "    %.3fs (%+06.2f%%) %s"
+                % (test_file[1], (test_file[1] * 100 / baseline) - 100, test_file[0])
+            )
+
+        test_count += 1
 
     print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
 
@@ -58,27 +93,47 @@ def run_tests(pyb, test_dict):
 
 
 def main():
-    cmd_parser = argparse.ArgumentParser(description="Run tests for MicroPython.")
-    cmd_parser.add_argument("--pyboard", action="store_true", help="run the tests on the pyboard")
+    cmd_parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=f"""Run and manage tests for MicroPython.
+
+{run_tests_module.test_instance_description}
+{run_tests_module.test_directory_description}
+""",
+        epilog=run_tests_module.test_instance_epilog,
+    )
+    cmd_parser.add_argument(
+        "-t", "--test-instance", default="unix", help="the MicroPython instance to test"
+    )
+    cmd_parser.add_argument(
+        "-b", "--baudrate", default=115200, help="the baud rate of the serial device"
+    )
+    cmd_parser.add_argument("-u", "--user", default="micro", help="the telnet login username")
+    cmd_parser.add_argument("-p", "--password", default="python", help="the telnet login password")
+    cmd_parser.add_argument(
+        "-d", "--test-dirs", nargs="*", help="input test directories (if no files given)"
+    )
+    cmd_parser.add_argument(
+        "-I",
+        "--iters",
+        type=int,
+        default=200_000,
+        help="number of test iterations, only for remote instances (default 200,000)",
+    )
     cmd_parser.add_argument("files", nargs="*", help="input test files")
     args = cmd_parser.parse_args()
 
     # Note pyboard support is copied over from run-tests.py, not tests, and likely needs revamping
-    if args.pyboard:
-        import pyboard
-
-        pyb = pyboard.Pyboard("/dev/ttyACM0")
-        pyb.enter_raw_repl()
-    else:
-        pyb = None
+    pyb = run_tests_module.get_test_instance(
+        args.test_instance, args.baudrate, args.user, args.password
+    )
 
     if len(args.files) == 0:
-        if pyb is None:
-            # run PC tests
-            test_dirs = ("internal_bench",)
+        if args.test_dirs:
+            test_dirs = tuple(args.test_dirs)
         else:
-            # run pyboard tests
-            test_dirs = ("basics", "float", "pyb")
+            test_dirs = ("internal_bench",)
+
         tests = sorted(
             test_file
             for test_files in (glob("{}/*.py".format(dir)) for dir in test_dirs)
@@ -95,7 +150,7 @@ def main():
             continue
         test_dict[m.group(1)].append([t, None])
 
-    if not run_tests(pyb, test_dict):
+    if not run_tests(pyb, test_dict, args.iters):
         sys.exit(1)
 
 
diff --git a/tests/run-tests.py b/tests/run-tests.py
index a428665db..59aec327f 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -1145,29 +1145,11 @@ class append_filter(argparse.Action):
         args.filters.append((option, re.compile(value)))
 
 
-def main():
-    global injected_import_hook_code
-
-    cmd_parser = argparse.ArgumentParser(
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        description="""Run and manage tests for MicroPython.
-
+test_instance_description = """\
 By default the tests are run against the unix port of MicroPython. To run it
 against something else, use the -t option.  See below for details.
-
-Tests are discovered by scanning test directories for .py files or using the
-specified test files. If test files nor directories are specified, the script
-expects to be ran in the tests directory (where this file is located) and the
-builtin tests suitable for the target platform are ran.
-
-When running tests, run-tests.py compares the MicroPython output of the test with the output
-produced by running the test through CPython unless a <test>.exp file is found, in which
-case it is used as comparison.
-
-If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
-directory with the MicroPython output and the expectations, respectively.
-""",
-        epilog="""\
+"""
+test_instance_epilog = """\
 The -t option accepts the following for the test instance:
 - unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON
   environment variable (which defaults to the standard variant of either the unix
@@ -1183,7 +1165,35 @@ The -t option accepts the following for the test instance:
 - execpty:<command> - execute a command and attach to the printed /dev/pts/<n> device
 - <a>.<b>.<c>.<d> - connect to the given IPv4 address
 - anything else specifies a serial port
+"""
 
+test_directory_description = """\
+Tests are discovered by scanning test directories for .py files or using the
+specified test files. If test files nor directories are specified, the script
+expects to be ran in the tests directory (where this file is located) and the
+builtin tests suitable for the target platform are ran.
+"""
+
+
+def main():
+    global injected_import_hook_code
+
+    cmd_parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=f"""Run and manage tests for MicroPython.
+
+{test_instance_description}
+{test_directory_description}
+
+When running tests, run-tests.py compares the MicroPython output of the test with the output
+produced by running the test through CPython unless a <test>.exp file is found, in which
+case it is used as comparison.
+
+If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
+directory with the MicroPython output and the expectations, respectively.
+""",
+        epilog=f"""\
+{test_instance_epilog}
 Options -i and -e can be multiple and processed in the order given. Regex
 "search" (vs "match") operation is used. An action (include/exclude) of
 the last matching regex is used: