tests: Rename run-tests to run-tests.py for consistency.

Signed-off-by: Damien George <damien@micropython.org>

1 files changed, 691 insertions, 0 deletions

diff --git a/tests/run-tests.py b/tests/run-tests.py
new file mode 100755
index 000000000..5863b23b4
--- /dev/null
+++ b/tests/run-tests.py
@@ -0,0 +1,691 @@
+#! /usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import platform
+import argparse
+import inspect
+import re
+from glob import glob
+
+# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0]
+# are guaranteed to always work, this one should though.
+BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None)))
+
+def base_path(*p):
+    return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/')
+
+# Tests require at least CPython 3.3. If your default python3 executable
+# is of lower version, you can point MICROPY_CPYTHON3 environment var
+# to the correct executable.
+if os.name == 'nt':
+    CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python')
+    MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe'))
+else:
+    CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
+    MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython'))
+
+# Use CPython options to not save .pyc files, to only access the core standard library
+# (not site packages which may clash with u-module names), and improve start up time.
+CPYTHON3_CMD = [CPYTHON3, "-BS"]
+
+# mpy-cross is only needed if --via-mpy command-line arg is passed
+MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross'))
+
+# For diff'ing test output
+DIFF = os.getenv('MICROPY_DIFF', 'diff -u')
+
+# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
+os.environ['PYTHONIOENCODING'] = 'utf-8'
+
+def rm_f(fname):
+    if os.path.exists(fname):
+        os.remove(fname)
+
+
+# unescape wanted regex chars and escape unwanted ones
+def convert_regex_escapes(line):
+    cs = []
+    escape = False
+    for c in str(line, 'utf8'):
+        if escape:
+            escape = False
+            cs.append(c)
+        elif c == '\\':
+            escape = True
+        elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):
+            cs.append('\\' + c)
+        else:
+            cs.append(c)
+    # accept carriage-return(s) before final newline
+    if cs[-1] == '\n':
+        cs[-1] = '\r*\n'
+    return bytes(''.join(cs), 'utf8')
+
+
+def run_micropython(pyb, args, test_file, is_special=False):
+    special_tests = (
+        'micropython/meminfo.py', 'basics/bytes_compare3.py',
+        'basics/builtin_help.py', 'thread/thread_exc2.py',
+        'esp32/partition_ota.py',
+    )
+    had_crash = False
+    if pyb is None:
+        # run on PC
+        if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests:
+            # special handling for tests of the unix cmdline program
+            is_special = True
+
+        if is_special:
+            # check for any cmdline options needed for this test
+            args = [MICROPYTHON]
+            with open(test_file, 'rb') as f:
+                line = f.readline()
+                if line.startswith(b'# cmdline:'):
+                    # subprocess.check_output on Windows only accepts strings, not bytes
+                    args += [str(c, 'utf-8') for c in line[10:].strip().split()]
+
+            # run the test, possibly with redirected input
+            try:
+                if 'repl_' in test_file:
+                    # Need to use a PTY to test command line editing
+                    try:
+                        import pty
+                    except ImportError:
+                        # in case pty module is not available, like on Windows
+                        return b'SKIP\n'
+                    import select
+
+                    def get(required=False):
+                        rv = b''
+                        while True:
+                            ready = select.select([master], [], [], 0.02)
+                            if ready[0] == [master]:
+                                rv += os.read(master, 1024)
+                            else:
+                                if not required or rv:
+                                    return rv
+
+                    def send_get(what):
+                        os.write(master, what)
+                        return get()
+
+                    with open(test_file, 'rb') as f:
+                        # instead of: output_mupy = subprocess.check_output(args, stdin=f)
+                        master, slave = pty.openpty()
+                        p = subprocess.Popen(args, stdin=slave, stdout=slave,
+                                             stderr=subprocess.STDOUT, bufsize=0)
+                        banner = get(True)
+                        output_mupy = banner + b''.join(send_get(line) for line in f)
+                        send_get(b'\x04') # exit the REPL, so coverage info is saved
+                        # At this point the process might have exited already, but trying to
+                        # kill it 'again' normally doesn't result in exceptions as Python and/or
+                        # the OS seem to try to handle this nicely. When running Linux on WSL
+                        # though, the situation differs and calling Popen.kill after the process
+                        # terminated results in a ProcessLookupError. Just catch that one here
+                        # since we just want the process to be gone and that's the case.
+                        try:
+                            p.kill()
+                        except ProcessLookupError:
+                            pass
+                        os.close(master)
+                        os.close(slave)
+                else:
+                    output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError:
+                return b'CRASH'
+
+        else:
+            # a standard test run on PC
+
+            # create system command
+            cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]
+            if args.heapsize is not None:
+                cmdlist.extend(['-X', 'heapsize=' + args.heapsize])
+
+            # if running via .mpy, first compile the .py file
+            if args.via_mpy:
+                subprocess.check_output([MPYCROSS] + args.mpy_cross_flags.split() + ['-o', 'mpytest.mpy', '-X', 'emit=' + args.emit, test_file])
+                cmdlist.extend(['-m', 'mpytest'])
+            else:
+                cmdlist.append(test_file)
+
+            # run the actual test
+            try:
+                output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as er:
+                had_crash = True
+                output_mupy = er.output + b'CRASH'
+
+            # clean up if we had an intermediate .mpy file
+            if args.via_mpy:
+                rm_f('mpytest.mpy')
+
+    else:
+        # run on pyboard
+        pyb.enter_raw_repl()
+        try:
+            output_mupy = pyb.execfile(test_file)
+        except pyboard.PyboardError as e:
+            had_crash = True
+            if not is_special and e.args[0] == 'exception':
+                output_mupy = e.args[1] + e.args[2] + b'CRASH'
+            else:
+                output_mupy = b'CRASH'
+
+    # canonical form for all ports/platforms is to use \n for end-of-line
+    output_mupy = output_mupy.replace(b'\r\n', b'\n')
+
+    # don't try to convert the output if we should skip this test
+    if had_crash or output_mupy in (b'SKIP\n', b'CRASH'):
+        return output_mupy
+
+    if is_special or test_file in special_tests:
+        # convert parts of the output that are not stable across runs
+        with open(test_file + '.exp', 'rb') as f:
+            lines_exp = []
+            for line in f.readlines():
+                if line == b'########\n':
+                    line = (line,)
+                else:
+                    line = (line, re.compile(convert_regex_escapes(line)))
+                lines_exp.append(line)
+        lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
+        if output_mupy.endswith(b'\n'):
+            lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
+        i_mupy = 0
+        for i in range(len(lines_exp)):
+            if lines_exp[i][0] == b'########\n':
+                # 8x #'s means match 0 or more whole lines
+                line_exp = lines_exp[i + 1]
+                skip = 0
+                while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):
+                    skip += 1
+                if i_mupy + skip >= len(lines_mupy):
+                    lines_mupy[i_mupy] = b'######## FAIL\n'
+                    break
+                del lines_mupy[i_mupy:i_mupy + skip]
+                lines_mupy.insert(i_mupy, b'########\n')
+                i_mupy += 1
+            else:
+                # a regex
+                if lines_exp[i][1].match(lines_mupy[i_mupy]):
+                    lines_mupy[i_mupy] = lines_exp[i][0]
+                else:
+                    #print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
+                    pass
+                i_mupy += 1
+            if i_mupy >= len(lines_mupy):
+                break
+        output_mupy = b''.join(lines_mupy)
+
+    return output_mupy
+
+
+def run_feature_check(pyb, args, base_path, test_file):
+    if pyb is not None and test_file.startswith("repl_"):
+        # REPL feature tests will not run via pyboard because they require prompt interactivity
+        return b""
+    return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True)
+
+
+def run_tests(pyb, tests, args, result_dir):
+    test_count = 0
+    testcase_count = 0
+    passed_count = 0
+    failed_tests = []
+    skipped_tests = []
+
+    skip_tests = set()
+    skip_native = False
+    skip_int_big = False
+    skip_bytearray = False
+    skip_set_type = False
+    skip_slice = False
+    skip_async = False
+    skip_const = False
+    skip_revops = False
+    skip_io_module = False
+    skip_endian = False
+    has_complex = True
+    has_coverage = False
+
+    upy_float_precision = 32
+
+    # If we're asked to --list-tests, we can't assume that there's a
+    # connection to target, so we can't run feature checks usefully.
+    if not (args.list_tests or args.write_exp):
+        # Even if we run completely different tests in a different directory,
+        # we need to access feature_checks from the same directory as the
+        # run-tests.py script itself so use base_path.
+
+        # Check if micropython.native is supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'native_check.py')
+        if output == b'CRASH':
+            skip_native = True
+
+        # Check if arbitrary-precision integers are supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'int_big.py')
+        if output != b'1000000000000000000000000000000000000000000000\n':
+            skip_int_big = True
+
+        # Check if bytearray is supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'bytearray.py')
+        if output != b'bytearray\n':
+            skip_bytearray = True
+
+        # Check if set type (and set literals) is supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'set_check.py')
+        if output == b'CRASH':
+            skip_set_type = True
+
+        # Check if slice is supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'slice.py')
+        if output != b'slice\n':
+            skip_slice = True
+
+        # Check if async/await keywords are supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'async_check.py')
+        if output == b'CRASH':
+            skip_async = True
+
+        # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'const.py')
+        if output == b'CRASH':
+            skip_const = True
+
+        # Check if __rOP__ special methods are supported, and skip such tests if it's not
+        output = run_feature_check(pyb, args, base_path, 'reverse_ops.py')
+        if output == b'TypeError\n':
+            skip_revops = True
+
+        # Check if uio module exists, and skip such tests if it doesn't
+        output = run_feature_check(pyb, args, base_path, 'uio_module.py')
+        if output != b'uio\n':
+            skip_io_module = True
+
+        # Check if emacs repl is supported, and skip such tests if it's not
+        t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py')
+        if 'True' not in str(t, 'ascii'):
+            skip_tests.add('cmdline/repl_emacs_keys.py')
+
+        # Check if words movement in repl is supported, and skip such tests if it's not
+        t = run_feature_check(pyb, args, base_path, 'repl_words_move_check.py')
+        if 'True' not in str(t, 'ascii'):
+            skip_tests.add('cmdline/repl_words_move.py')
+
+        upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py')
+        upy_float_precision = run_feature_check(pyb, args, base_path, 'float.py')
+        if upy_float_precision == b'CRASH':
+            upy_float_precision = 0
+        else:
+            upy_float_precision = int(upy_float_precision)
+        has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n'
+        has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n'
+        cpy_byteorder = subprocess.check_output(CPYTHON3_CMD + [base_path('feature_check/byteorder.py')])
+        skip_endian = (upy_byteorder != cpy_byteorder)
+
+    # These tests don't test slice explicitly but rather use it to perform the test
+    misc_slice_tests = (
+        'builtin_range',
+        'class_super',
+        'containment',
+        'errno1',
+        'fun_str',
+        'generator1',
+        'globals_del',
+        'memoryview1',
+        'memoryview_gc',
+        'object1',
+        'python34',
+        'struct_endian',
+    )
+
+    # Some tests shouldn't be run on GitHub Actions
+    if os.getenv('GITHUB_ACTIONS') == 'true':
+        skip_tests.add('thread/stress_schedule.py') # has reliability issues
+
+    if upy_float_precision == 0:
+        skip_tests.add('extmod/uctypes_le_float.py')
+        skip_tests.add('extmod/uctypes_native_float.py')
+        skip_tests.add('extmod/uctypes_sizeof_float.py')
+        skip_tests.add('extmod/ujson_dumps_float.py')
+        skip_tests.add('extmod/ujson_loads_float.py')
+        skip_tests.add('extmod/urandom_extra_float.py')
+        skip_tests.add('misc/rge_sm.py')
+    if upy_float_precision < 32:
+        skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead
+        skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead
+        skip_tests.add('float/bytes_construct.py') # requires fp32
+        skip_tests.add('float/bytearray_construct.py') # requires fp32
+    if upy_float_precision < 64:
+        skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead
+        skip_tests.add('float/float2int_doubleprec_intbig.py')
+        skip_tests.add('float/float_parse_doubleprec.py')
+
+    if not has_complex:
+        skip_tests.add('float/complex1.py')
+        skip_tests.add('float/complex1_intbig.py')
+        skip_tests.add('float/complex_special_methods.py')
+        skip_tests.add('float/int_big_float.py')
+        skip_tests.add('float/true_value.py')
+        skip_tests.add('float/types.py')
+
+    if not has_coverage:
+        skip_tests.add('cmdline/cmd_parsetree.py')
+
+    # Some tests shouldn't be run on a PC
+    if args.target == 'unix':
+        # unix build does not have the GIL so can't run thread mutation tests
+        for t in tests:
+            if t.startswith('thread/mutate_'):
+                skip_tests.add(t)
+
+    # Some tests shouldn't be run on pyboard
+    if args.target != 'unix':
+        skip_tests.add('basics/exception_chain.py') # warning is not printed
+        skip_tests.add('micropython/meminfo.py') # output is very different to PC output
+        skip_tests.add('extmod/machine_mem.py') # raw memory access not supported
+
+        if args.target == 'wipy':
+            skip_tests.add('misc/print_exception.py')       # requires error reporting full
+            skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes
+            skip_tests.add('extmod/zlibd_decompress.py')    # requires zlib
+            skip_tests.add('extmod/uheapq1.py')             # uheapq not supported by WiPy
+            skip_tests.add('extmod/urandom_basic.py')       # requires urandom
+            skip_tests.add('extmod/urandom_extra.py')       # requires urandom
+        elif args.target == 'esp8266':
+            skip_tests.add('misc/rge_sm.py')                # too large
+        elif args.target == 'minimal':
+            skip_tests.add('basics/class_inplace_op.py')    # all special methods not supported
+            skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support
+            skip_tests.add('misc/rge_sm.py')                # too large
+            skip_tests.add('micropython/opt_level.py')      # don't assume line numbers are stored
+        elif args.target == 'nrf':
+            skip_tests.add('basics/memoryview1.py')         # no item assignment for memoryview
+            skip_tests.add('extmod/urandom_basic.py')       # unimplemented: urandom.seed
+            skip_tests.add('micropython/opt_level.py')      # no support for line numbers
+            skip_tests.add('misc/non_compliant.py')         # no item assignment for bytearray
+            for t in tests:
+                if t.startswith('basics/io_'):
+                    skip_tests.add(t)
+        elif args.target == 'qemu-arm':
+            skip_tests.add('misc/print_exception.py')       # requires sys stdfiles
+
+    # Some tests are known to fail on 64-bit machines
+    if pyb is None and platform.architecture()[0] == '64bit':
+        pass
+
+    # Some tests use unsupported features on Windows
+    if os.name == 'nt':
+        skip_tests.add('import/import_file.py') # works but CPython prints forward slashes
+
+    # Some tests are known to fail with native emitter
+    # Remove them from the below when they work
+    if args.emit == 'native':
+        skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from_close generator_name'.split()}) # require raise_varargs, generator name
+        skip_tests.update({'basics/async_%s.py' % t for t in 'with with2 with_break with_return'.split()}) # require async_with
+        skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs
+        skip_tests.add('basics/annotate_var.py') # requires checking for unbound local
+        skip_tests.add('basics/del_deref.py') # requires checking for unbound local
+        skip_tests.add('basics/del_local.py') # requires checking for unbound local
+        skip_tests.add('basics/exception_chain.py') # raise from is not supported
+        skip_tests.add('basics/scope_implicit.py') # requires checking for unbound local
+        skip_tests.add('basics/try_finally_return2.py') # requires raise_varargs
+        skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local
+        skip_tests.add('extmod/uasyncio_event.py') # unknown issue
+        skip_tests.add('extmod/uasyncio_lock.py') # requires async with
+        skip_tests.add('extmod/uasyncio_micropython.py') # unknown issue
+        skip_tests.add('extmod/uasyncio_wait_for.py') # unknown issue
+        skip_tests.add('misc/features.py') # requires raise_varargs
+        skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info
+        skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native
+        skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info
+        skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info
+        skip_tests.add('micropython/opt_level_lineno.py') # native doesn't have proper traceback info
+        skip_tests.add('micropython/schedule.py') # native code doesn't check pending events
+
+    for test_file in tests:
+        test_file = test_file.replace('\\', '/')
+
+        if args.filters:
+            # Default verdict is the opposit of the first action
+            verdict = "include" if args.filters[0][0] == "exclude" else "exclude"
+            for action, pat in args.filters:
+                if pat.search(test_file):
+                    verdict = action
+            if verdict == "exclude":
+                continue
+
+        test_basename = test_file.replace('..', '_').replace('./', '').replace('/', '_')
+        test_name = os.path.splitext(os.path.basename(test_file))[0]
+        is_native = test_name.startswith("native_") or test_name.startswith("viper_") or args.emit == "native"
+        is_endian = test_name.endswith("_endian")
+        is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
+        is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray")
+        is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
+        is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests
+        is_async = test_name.startswith(("async_", "uasyncio_"))
+        is_const = test_name.startswith("const")
+        is_io_module = test_name.startswith("io_")
+
+        skip_it = test_file in skip_tests
+        skip_it |= skip_native and is_native
+        skip_it |= skip_endian and is_endian
+        skip_it |= skip_int_big and is_int_big
+        skip_it |= skip_bytearray and is_bytearray
+        skip_it |= skip_set_type and is_set_type
+        skip_it |= skip_slice and is_slice
+        skip_it |= skip_async and is_async
+        skip_it |= skip_const and is_const
+        skip_it |= skip_revops and "reverse_op" in test_name
+        skip_it |= skip_io_module and is_io_module
+
+        if args.list_tests:
+            if not skip_it:
+                print(test_file)
+            continue
+
+        if skip_it:
+            print("skip ", test_file)
+            skipped_tests.append(test_name)
+            continue
+
+        # get expected output
+        test_file_expected = test_file + '.exp'
+        if os.path.isfile(test_file_expected):
+            # expected output given by a file, so read that in
+            with open(test_file_expected, 'rb') as f:
+                output_expected = f.read()
+        else:
+            # run CPython to work out expected output
+            try:
+                output_expected = subprocess.check_output(CPYTHON3_CMD + [test_file])
+                if args.write_exp:
+                    with open(test_file_expected, 'wb') as f:
+                        f.write(output_expected)
+            except subprocess.CalledProcessError:
+                output_expected = b'CPYTHON3 CRASH'
+
+        # canonical form for all host platforms is to use \n for end-of-line
+        output_expected = output_expected.replace(b'\r\n', b'\n')
+
+        if args.write_exp:
+            continue
+
+        # run MicroPython
+        output_mupy = run_micropython(pyb, args, test_file)
+
+        if output_mupy == b'SKIP\n':
+            print("skip ", test_file)
+            skipped_tests.append(test_name)
+            continue
+
+        testcase_count += len(output_expected.splitlines())
+
+        filename_expected = os.path.join(result_dir, test_basename + ".exp")
+        filename_mupy = os.path.join(result_dir, test_basename + ".out")
+
+        if output_expected == output_mupy:
+            print("pass ", test_file)
+            passed_count += 1
+            rm_f(filename_expected)
+            rm_f(filename_mupy)
+        else:
+            with open(filename_expected, "wb") as f:
+                f.write(output_expected)
+            with open(filename_mupy, "wb") as f:
+                f.write(output_mupy)
+            print("FAIL ", test_file)
+            failed_tests.append(test_name)
+
+        test_count += 1
+
+    if args.list_tests:
+        return True
+
+    print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
+    print("{} tests passed".format(passed_count))
+
+    if len(skipped_tests) > 0:
+        print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))
+    if len(failed_tests) > 0:
+        print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))
+        return False
+
+    # all tests succeeded
+    return True
+
+
+class append_filter(argparse.Action):
+
+    def __init__(self, option_strings, dest, **kwargs):
+        super().__init__(option_strings, dest, default=[], **kwargs)
+
+    def __call__(self, parser, args, value, option):
+        if not hasattr(args, self.dest):
+            args.filters = []
+        if option.startswith(("-e", "--e")):
+            option = "exclude"
+        else:
+            option = "include"
+        args.filters.append((option, re.compile(value)))
+
+
+def main():
+    cmd_parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='''Run and manage tests for MicroPython.
+
+Tests are discovered by scanning test directories for .py files or using the
+specified test files. If test files nor directories are specified, the script
+expects to be ran in the tests directory (where this file is located) and the
+builtin tests suitable for the target platform are ran.
+When running tests, run-tests.py compares the MicroPython output of the test with the output
+produced by running the test through CPython unless a <test>.exp file is found, in which
+case it is used as comparison.
+If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
+directory with the MicroPython output and the expectations, respectively.
+''',
+        epilog='''\
+Options -i and -e can be multiple and processed in the order given. Regex
+"search" (vs "match") operation is used. An action (include/exclude) of
+the last matching regex is used:
+  run-tests.py -i async - exclude all, then include tests containing "async" anywhere
+  run-tests.py -e '/big.+int' - include all, then exclude by regex
+  run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo
+''')
+    cmd_parser.add_argument('--target', default='unix', help='the target platform')
+    cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
+    cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
+    cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
+    cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
+    cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
+    cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results')
+    cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py')
+    cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py')
+    cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython')
+    cmd_parser.add_argument('--list-tests', action='store_true', help='list tests instead of running them')
+    cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')
+    cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
+    cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
+    cmd_parser.add_argument('--mpy-cross-flags', default='-mcache-lookup-bc', help='flags to pass to mpy-cross')
+    cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
+    cmd_parser.add_argument('files', nargs='*', help='input test files')
+    cmd_parser.add_argument('--print-failures', action='store_true', help='print the diff of expected vs. actual output for failed tests and exit')
+    cmd_parser.add_argument('--clean-failures', action='store_true', help='delete the .exp and .out files from failed tests and exit')
+    args = cmd_parser.parse_args()
+
+    if args.print_failures:
+        for exp in glob(os.path.join(args.result_dir, "*.exp")):
+            testbase = exp[:-4]
+            print()
+            print("FAILURE {0}".format(testbase))
+            os.system("{0} {1}.exp {1}.out".format(DIFF, testbase))
+
+        sys.exit(0)
+
+    if args.clean_failures:
+        for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")):
+            os.remove(f)
+
+        sys.exit(0)
+
+    LOCAL_TARGETS = ('unix', 'qemu-arm',)
+    EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'esp32', 'minimal', 'nrf')
+    if args.target in LOCAL_TARGETS or args.list_tests:
+        pyb = None
+    elif args.target in EXTERNAL_TARGETS:
+        global pyboard
+        sys.path.append(base_path('../tools'))
+        import pyboard
+        pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
+        pyb.enter_raw_repl()
+    else:
+        raise ValueError('target must be one of %s' % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
+
+    if len(args.files) == 0:
+        if args.test_dirs is None:
+            test_dirs = ('basics', 'micropython', 'misc', 'extmod',)
+            if args.target == 'pyboard':
+                # run pyboard tests
+                test_dirs += ('float', 'stress', 'pyb', 'pybnative', 'inlineasm')
+            elif args.target in ('esp8266', 'esp32', 'minimal', 'nrf'):
+                test_dirs += ('float',)
+            elif args.target == 'wipy':
+                # run WiPy tests
+                test_dirs += ('wipy',)
+            elif args.target == 'unix':
+                # run PC tests
+                test_dirs += ('float', 'import', 'io', 'stress', 'unicode', 'unix', 'cmdline',)
+            elif args.target == 'qemu-arm':
+                if not args.write_exp:
+                    raise ValueError('--target=qemu-arm must be used with --write-exp')
+                # Generate expected output files for qemu run.
+                # This list should match the test_dirs tuple in tinytest-codegen.py.
+                test_dirs += ('float', 'inlineasm', 'qemu-arm',)
+        else:
+            # run tests from these directories
+            test_dirs = args.test_dirs
+        tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
+    else:
+        # tests explicitly given
+        tests = args.files
+
+    if not args.keep_path:
+        # clear search path to make sure tests use only builtin modules and those in extmod
+        os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod')
+
+    try:
+        os.makedirs(args.result_dir, exist_ok=True)
+        res = run_tests(pyb, tests, args, args.result_dir)
+    finally:
+        if pyb:
+            pyb.close()
+
+    if not res:
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()