tests: Add script to run dynamic-native-module tests.

1 files changed, 187 insertions, 0 deletions

diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py
new file mode 100755
index 000000000..186c30b83
--- /dev/null
+++ b/tests/run-natmodtests.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+
+# This file is part of the MicroPython project, http://micropython.org/
+# The MIT License (MIT)
+# Copyright (c) 2019 Damien P. George
+
+import os
+import subprocess
+import sys
+import argparse
+
+sys.path.append('../tools')
+import pyboard
+
+# Paths for host executables
+CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
+MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython_coverage')
+
+NATMOD_EXAMPLE_DIR = '../examples/natmod/'
+
+# Supported tests and their corresponding mpy module
+TEST_MAPPINGS = {
+    'btree': 'btree/btree_$(ARCH).mpy',
+    'framebuf': 'framebuf/framebuf_$(ARCH).mpy',
+    'uheapq': 'uheapq/uheapq_$(ARCH).mpy',
+    'ure': 'ure/ure_$(ARCH).mpy',
+    'uzlib': 'uzlib/uzlib_$(ARCH).mpy',
+}
+
+# Code to allow a target MicroPython to import an .mpy from RAM
+injected_import_hook_code = """\
+import sys, uos, uio
+class __File(uio.IOBase):
+  def __init__(self):
+    self.off = 0
+  def ioctl(self, request, arg):
+    return 0
+  def readinto(self, buf):
+    buf[:] = memoryview(__buf)[self.off:self.off + len(buf)]
+    self.off += len(buf)
+    return len(buf)
+class __FS:
+  def mount(self, readonly, mkfs):
+    pass
+  def chdir(self, path):
+    pass
+  def stat(self, path):
+    if path == '__injected.mpy':
+      return tuple(0 for _ in range(10))
+    else:
+      raise OSError(-2) # ENOENT
+  def open(self, path, mode):
+    return __File()
+uos.mount(__FS(), '/__remote')
+uos.chdir('/__remote')
+sys.modules['{}'] = __import__('__injected')
+"""
+
+class TargetSubprocess:
+    def __init__(self, cmd):
+        self.cmd = cmd
+
+    def close(self):
+        pass
+
+    def run_script(self, script):
+        try:
+            p = subprocess.run(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script)
+            return p.stdout, None
+        except subprocess.CalledProcessError as er:
+            return b'', er
+
+class TargetPyboard:
+    def __init__(self, pyb):
+        self.pyb = pyb
+        self.pyb.enter_raw_repl()
+
+    def close(self):
+        self.pyb.exit_raw_repl()
+        self.pyb.close()
+
+    def run_script(self, script):
+        try:
+            self.pyb.enter_raw_repl()
+            output = self.pyb.exec_(script)
+            output = output.replace(b'\r\n', b'\n')
+            return output, None
+        except pyboard.PyboardError as er:
+            return b'', er
+
+def run_tests(target_truth, target, args, stats):
+    for test_file in args.files:
+        # Find supported test
+        for k, v in TEST_MAPPINGS.items():
+            if test_file.find(k) != -1:
+                test_module = k
+                test_mpy = v.replace('$(ARCH)', args.arch)
+                break
+        else:
+            print('----  {} - no matching mpy'.format(test_file))
+            continue
+
+        # Read test script
+        with open(test_file, 'rb') as f:
+            test_file_data = f.read()
+
+        # Create full test with embedded .mpy
+        try:
+            with open(NATMOD_EXAMPLE_DIR + test_mpy, 'rb') as f:
+                test_script = b'__buf=' + bytes(repr(f.read()), 'ascii') + b'\n'
+        except OSError:
+            print('----  {} - mpy file not compiled'.format(test_file))
+            continue
+        test_script += bytes(injected_import_hook_code.format(test_module), 'ascii')
+        test_script += test_file_data
+
+        # Run test under MicroPython
+        result_out, error = target.run_script(test_script)
+
+        # Work out result of test
+        extra = ''
+        if error is None and result_out == b'SKIP\n':
+            result = 'SKIP'
+        elif error is not None:
+            result = 'FAIL'
+            extra = ' - ' + str(error)
+        else:
+            # Check result against truth
+            try:
+                with open(test_file + '.exp', 'rb') as f:
+                    result_exp = f.read()
+                error = None
+            except OSError:
+                result_exp, error = target_truth.run_script(test_file_data)
+            if error is not None:
+                result = 'TRUTH FAIL'
+            elif result_out != result_exp:
+                result = 'FAIL'
+                print(result_out)
+            else:
+                result = 'pass'
+
+        # Accumulate statistics
+        stats['total'] += 1
+        if result == 'pass':
+            stats['pass'] += 1
+        elif result == 'SKIP':
+            stats['skip'] += 1
+        else:
+            stats['fail'] += 1
+
+        # Print result
+        print('{:4}  {}{}'.format(result, test_file, extra))
+
+def main():
+    cmd_parser = argparse.ArgumentParser(description='Run dynamic-native-module tests under MicroPython')
+    cmd_parser.add_argument('-p', '--pyboard', action='store_true', help='run tests via pyboard.py')
+    cmd_parser.add_argument('-d', '--device', default='/dev/ttyACM0', help='the device for pyboard.py')
+    cmd_parser.add_argument('-a', '--arch', default='x64', help='native architecture of the target')
+    cmd_parser.add_argument('files', nargs='*', help='input test files')
+    args = cmd_parser.parse_args()
+
+    target_truth = TargetSubprocess([CPYTHON3])
+
+    if args.pyboard:
+        target = TargetPyboard(pyboard.Pyboard(args.device))
+    else:
+        target = TargetSubprocess([MICROPYTHON])
+
+    stats = {'total': 0, 'pass': 0, 'fail':0, 'skip': 0}
+    run_tests(target_truth, target, args, stats)
+
+    target.close()
+    target_truth.close()
+
+    print('{} tests performed'.format(stats['total']))
+    print('{} tests passed'.format(stats['pass']))
+    if stats['fail']:
+        print('{} tests failed'.format(stats['fail']))
+    if stats['skip']:
+        print('{} tests skipped'.format(stats['skip']))
+
+    if stats['fail']:
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()