diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/gen-cpydiff.py | 224 | ||||
| -rwxr-xr-x | tools/mpy-tool.py | 23 | ||||
| -rwxr-xr-x | tools/mpy_cross_all.py | 38 | ||||
| -rwxr-xr-x | tools/pyboard.py | 91 | ||||
| -rwxr-xr-x | tools/tinytest-codegen.py | 3 | ||||
| -rw-r--r-- | tools/upip.py | 89 |
6 files changed, 418 insertions, 50 deletions
diff --git a/tools/gen-cpydiff.py b/tools/gen-cpydiff.py new file mode 100644 index 000000000..4b273d97f --- /dev/null +++ b/tools/gen-cpydiff.py @@ -0,0 +1,224 @@ +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2016 Rami Ali +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" gen-cpydiff generates documentation which outlines operations that differ between MicroPython + and CPython. This script is called by the docs Makefile for html and Latex and may be run + manually using the command make gen-cpydiff. """ + +import os +import errno +import subprocess +import time +import re +from collections import namedtuple + +# Micropython supports syntax of CPython 3.4 with some features from 3.5, and +# such version should be used to test for differences. If your default python3 +# executable is of lower version, you can point MICROPY_CPYTHON3 environment var +# to the correct executable. +if os.name == 'nt': + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../windows/micropython.exe') +else: + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../unix/micropython') + +TESTPATH = '../tests/cpydiff/' +DOCPATH = '../docs/genrst/' +INDEXTEMPLATE = '../docs/differences/index_template.txt' +INDEX = 'index.rst' + +HEADER = '.. This document was generated by tools/gen-cpydiff.py\n\n' +UIMPORTLIST = {'struct', 'collections', 'json'} +CLASSMAP = {'Core': 'Core Language', 'Types': 'Builtin Types'} +INDEXPRIORITY = ['syntax', 'core_language', 'builtin_types', 'modules'] +RSTCHARS = ['=', '-', '~', '`', ':'] +SPLIT = '"""\n|categories: |description: |cause: |workaround: ' +TAB = ' ' + +Output = namedtuple('output', ['name', 'class_', 'desc', 'cause', 'workaround', 'code', + 'output_cpy', 'output_upy', 'status']) + +def readfiles(): + """ Reads test files """ + tests = list(filter(lambda x: x.endswith('.py'), os.listdir(TESTPATH))) + tests.sort() + files = [] + + for test in tests: + text = open(TESTPATH + test, 'r').read() + + try: + class_, desc, cause, workaround, code = [x.rstrip() for x in \ + list(filter(None, re.split(SPLIT, text)))] + output = Output(test, class_, desc, cause, workaround, code, '', '', '') + files.append(output) + except IndexError: + print('Incorrect format in file ' + TESTPATH + test) + + return files + +def uimports(code): + """ converts CPython module names into MicroPython equivalents """ + for uimport in UIMPORTLIST: + uimport = bytes(uimport, 'utf8') + code = code.replace(uimport, b'u' + uimport) + return code + +def run_tests(tests): + """ executes all tests """ + results = [] + for test in tests: + with open(TESTPATH + test.name, 'rb') as f: + input_cpy = f.read() + input_upy = uimports(input_cpy) + + process = subprocess.Popen(CPYTHON3, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + output_cpy = [com.decode('utf8') for com in process.communicate(input_cpy)] + + process = subprocess.Popen(MICROPYTHON, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + output_upy = [com.decode('utf8') for com in process.communicate(input_upy)] + + if output_cpy[0] == output_upy[0] and output_cpy[1] == output_upy[1]: + status = 'Supported' + print('Supported operation!\nFile: ' + TESTPATH + test.name) + else: + status = 'Unsupported' + + output = Output(test.name, test.class_, test.desc, test.cause, + test.workaround, test.code, output_cpy, output_upy, status) + results.append(output) + + results.sort(key=lambda x: x.class_) + return results + +def indent(block, spaces): + """ indents paragraphs of text for rst formatting """ + new_block = '' + for line in block.split('\n'): + new_block += spaces + line + '\n' + return new_block + +def gen_table(contents): + """ creates a table given any set of columns """ + xlengths = [] + ylengths = [] + for column in contents: + col_len = 0 + for entry in column: + lines = entry.split('\n') + for line in lines: + col_len = max(len(line) + 2, col_len) + xlengths.append(col_len) + for i in range(len(contents[0])): + ymax = 0 + for j in range(len(contents)): + ymax = max(ymax, len(contents[j][i].split('\n'))) + ylengths.append(ymax) + + table_divider = '+' + ''.join(['-' * i + '+' for i in xlengths]) + '\n' + table = table_divider + for i in range(len(ylengths)): + row = [column[i] for column in contents] + row = [entry + '\n' * (ylengths[i]-len(entry.split('\n'))) for entry in row] + row = [entry.split('\n') for entry in row] + for j in range(ylengths[i]): + k = 0 + for entry in row: + width = xlengths[k] + table += ''.join(['| {:{}}'.format(entry[j], width - 1)]) + k += 1 + table += '|\n' + table += table_divider + return table + '\n' + +def gen_rst(results): + """ creates restructured text documents to display tests """ + + # make sure the destination directory exists + try: + os.mkdir(DOCPATH) + except OSError as e: + if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR: + raise + + toctree = [] + class_ = [] + for output in results: + section = output.class_.split(',') + for i in range(len(section)): + section[i] = section[i].rstrip() + if section[i] in CLASSMAP: + section[i] = CLASSMAP[section[i]] + if i >= len(class_) or section[i] != class_[i]: + if i == 0: + filename = section[i].replace(' ', '_').lower() + rst = open(DOCPATH + filename + '.rst', 'w') + rst.write(HEADER) + rst.write(section[i] + '\n') + rst.write(RSTCHARS[0] * len(section[i])) + rst.write(time.strftime("\nGenerated %a %d %b %Y %X UTC\n\n", time.gmtime())) + toctree.append(filename) + else: + rst.write(section[i] + '\n') + rst.write(RSTCHARS[min(i, len(RSTCHARS)-1)] * len(section[i])) + rst.write('\n\n') + class_ = section + rst.write('**' + output.desc + '**\n\n') + if output.cause != 'Unknown': + rst.write('**Cause:** ' + output.cause + '\n\n') + if output.workaround != 'Unknown': + rst.write('**Workaround:** ' + output.workaround + '\n\n') + + rst.write('Sample code::\n\n' + indent(output.code, TAB) + '\n') + output_cpy = indent(''.join(output.output_cpy[0:2]), TAB).rstrip() + output_cpy = ('::\n\n' if output_cpy != '' else '') + output_cpy + output_upy = indent(''.join(output.output_upy[0:2]), TAB).rstrip() + output_upy = ('::\n\n' if output_upy != '' else '') + output_upy + table = gen_table([['CPy output:', output_cpy], ['uPy output:', output_upy]]) + rst.write(table) + + template = open(INDEXTEMPLATE, 'r') + index = open(DOCPATH + INDEX, 'w') + index.write(HEADER) + index.write(template.read()) + for section in INDEXPRIORITY: + if section in toctree: + index.write(indent(section + '.rst', TAB)) + toctree.remove(section) + for section in toctree: + index.write(indent(section + '.rst', TAB)) + +def main(): + """ Main function """ + + # set search path so that test scripts find the test modules (and no other ones) + os.environ['PYTHONPATH'] = TESTPATH + os.environ['MICROPYPATH'] = TESTPATH + + files = readfiles() + results = run_tests(files) + gen_rst(results) + +main() diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index ce373a4f5..544f90cc8 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -45,7 +45,7 @@ import sys import struct from collections import namedtuple -sys.path.append('../py') +sys.path.append(sys.path[0] + '/../py') import makeqstrdata as qstrutil class FreezeError(Exception): @@ -57,6 +57,7 @@ class FreezeError(Exception): return 'error while freezing %s: %s' % (self.rawcode.source_file, self.msg) class Config: + MPY_VERSION = 2 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -93,7 +94,7 @@ def make_opcode_format(): OC4(U, U, U, U), # 0x0c-0x0f OC4(B, B, B, U), # 0x10-0x13 OC4(V, U, Q, V), # 0x14-0x17 - OC4(B, U, V, V), # 0x18-0x1b + OC4(B, V, V, Q), # 0x18-0x1b OC4(Q, Q, Q, Q), # 0x1c-0x1f OC4(B, B, V, V), # 0x20-0x23 OC4(Q, Q, Q, B), # 0x24-0x27 @@ -331,25 +332,25 @@ class RawCode: raise FreezeError(self, 'freezing of object %r is not implemented' % (obj,)) # generate constant table - print('STATIC const mp_uint_t const_table_data_%s[%u] = {' + print('STATIC const mp_rom_obj_t const_table_data_%s[%u] = {' % (self.escaped_name, len(self.qstrs) + len(self.objs) + len(self.raw_codes))) for qst in self.qstrs: - print(' (mp_uint_t)MP_OBJ_NEW_QSTR(%s),' % global_qstrs[qst].qstr_id) + print(' MP_ROM_QSTR(%s),' % global_qstrs[qst].qstr_id) for i in range(len(self.objs)): if type(self.objs[i]) is float: print('#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B') - print(' (mp_uint_t)&const_obj_%s_%u,' % (self.escaped_name, i)) + print(' MP_ROM_PTR(&const_obj_%s_%u),' % (self.escaped_name, i)) print('#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C') n = struct.unpack('<I', struct.pack('<f', self.objs[i]))[0] n = ((n & ~0x3) | 2) + 0x80800000 - print(' (mp_uint_t)0x%08x,' % (n,)) + print(' (mp_rom_obj_t)(0x%08x),' % (n,)) print('#else') print('#error "MICROPY_OBJ_REPR_D not supported with floats in frozen mpy files"') print('#endif') else: - print(' (mp_uint_t)&const_obj_%s_%u,' % (self.escaped_name, i)) + print(' MP_ROM_PTR(&const_obj_%s_%u),' % (self.escaped_name, i)) for rc in self.raw_codes: - print(' (mp_uint_t)&raw_code_%s,' % rc.escaped_name) + print(' MP_ROM_PTR(&raw_code_%s),' % rc.escaped_name) print('};') # generate module @@ -361,7 +362,7 @@ class RawCode: print(' .n_pos_args = %u,' % self.prelude[3]) print(' .data.u_byte = {') print(' .bytecode = bytecode_data_%s,' % self.escaped_name) - print(' .const_table = const_table_data_%s,' % self.escaped_name) + print(' .const_table = (mp_uint_t*)const_table_data_%s,' % self.escaped_name) print(' #if MICROPY_PERSISTENT_CODE_SAVE') print(' .bc_len = %u,' % len(self.bytecode)) print(' .n_obj = %u,' % len(self.objs)) @@ -438,8 +439,8 @@ def read_mpy(filename): header = bytes_cons(f.read(4)) if header[0] != ord('M'): raise Exception('not a valid .mpy file') - if header[1] != 0: - raise Exception('incompatible version') + if header[1] != config.MPY_VERSION: + raise Exception('incompatible .mpy version') feature_flags = header[2] config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = (feature_flags & 1) != 0 config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_flags & 2) != 0 diff --git a/tools/mpy_cross_all.py b/tools/mpy_cross_all.py new file mode 100755 index 000000000..2bda71e9b --- /dev/null +++ b/tools/mpy_cross_all.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import argparse +import os +import os.path + +argparser = argparse.ArgumentParser(description="Compile all .py files to .mpy recursively") +argparser.add_argument("-o", "--out", help="output directory (default: input dir)") +argparser.add_argument("--target", help="select MicroPython target config") +argparser.add_argument("-mcache-lookup-bc", action="store_true", help="cache map lookups in the bytecode") +argparser.add_argument("dir", help="input directory") +args = argparser.parse_args() + +TARGET_OPTS = { + "unix": "-mcache-lookup-bc", + "baremetal": "", +} + +args.dir = args.dir.rstrip("/") + +if not args.out: + args.out = args.dir + +path_prefix_len = len(args.dir) + 1 + +for path, subdirs, files in os.walk(args.dir): + for f in files: + if f.endswith(".py"): + fpath = path + "/" + f + #print(fpath) + out_fpath = args.out + "/" + fpath[path_prefix_len:-3] + ".mpy" + out_dir = os.path.dirname(out_fpath) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + cmd = "mpy-cross -v -v %s -s %s %s -o %s" % (TARGET_OPTS.get(args.target, ""), + fpath[path_prefix_len:], fpath, out_fpath) + #print(cmd) + res = os.system(cmd) + assert res == 0 diff --git a/tools/pyboard.py b/tools/pyboard.py index d4ce8b788..5eac030bd 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -39,6 +39,7 @@ Or: import sys import time +import os try: stdout = sys.stdout.buffer @@ -116,9 +117,93 @@ class TelnetToSerial: else: return n_waiting + +class ProcessToSerial: + "Execute a process and emulate serial connection using its stdin/stdout." + + def __init__(self, cmd): + import subprocess + self.subp = subprocess.Popen(cmd.split(), bufsize=0, shell=True, preexec_fn=os.setsid, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + # Initially was implemented with selectors, but that adds Python3 + # dependency. However, there can be race conditions communicating + # with a particular child process (like QEMU), and selectors may + # still work better in that case, so left inplace for now. + # + #import selectors + #self.sel = selectors.DefaultSelector() + #self.sel.register(self.subp.stdout, selectors.EVENT_READ) + + import select + self.poll = select.poll() + self.poll.register(self.subp.stdout.fileno()) + + def close(self): + import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) + + def read(self, size=1): + data = b"" + while len(data) < size: + data += self.subp.stdout.read(size - len(data)) + return data + + def write(self, data): + self.subp.stdin.write(data) + return len(data) + + def inWaiting(self): + #res = self.sel.select(0) + res = self.poll.poll(0) + if res: + return 1 + return 0 + + +class ProcessPtyToTerminal: + """Execute a process which creates a PTY and prints slave PTY as + first line of its output, and emulate serial connection using + this PTY.""" + + def __init__(self, cmd): + import subprocess + import re + import serial + self.subp = subprocess.Popen(cmd.split(), bufsize=0, shell=False, preexec_fn=os.setsid, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + pty_line = self.subp.stderr.readline().decode("utf-8") + m = re.search(r"/dev/pts/[0-9]+", pty_line) + if not m: + print("Error: unable to find PTY device in startup line:", pty_line) + self.close() + sys.exit(1) + pty = m.group() + # rtscts, dsrdtr params are to workaround pyserial bug: + # http://stackoverflow.com/questions/34831131/pyserial-does-not-play-well-with-virtual-port + self.ser = serial.Serial(pty, interCharTimeout=1, rtscts=True, dsrdtr=True) + + def close(self): + import signal + os.killpg(os.getpgid(self.subp.pid), signal.SIGTERM) + + def read(self, size=1): + return self.ser.read(size) + + def write(self, data): + return self.ser.write(data) + + def inWaiting(self): + return self.ser.inWaiting() + + class Pyboard: def __init__(self, device, baudrate=115200, user='micro', password='python', wait=0): - if device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3: + if device.startswith("exec:"): + self.serial = ProcessToSerial(device[len("exec:"):]) + elif device.startswith("execpty:"): + self.serial = ProcessPtyToTerminal(device[len("qemupty:"):]) + elif device and device[0].isdigit() and device[-1].isdigit() and device.count('.') == 3: # device looks like an IP address self.serial = TelnetToSerial(device, user, password, read_timeout=10) else: @@ -234,7 +319,7 @@ class Pyboard: # check if we could exec command data = self.serial.read(2) if data != b'OK': - raise PyboardError('could not exec command') + raise PyboardError('could not exec command (response: %s)' % data) def exec_raw(self, command, timeout=10, data_consumer=None): self.exec_raw_no_follow(command); @@ -300,6 +385,7 @@ def main(): pyb.enter_raw_repl() except PyboardError as er: print(er) + pyb.close() sys.exit(1) def execbuffer(buf): @@ -307,6 +393,7 @@ def main(): ret, ret_err = pyb.exec_raw(buf, timeout=None, data_consumer=stdout_write_bytes) except PyboardError as er: print(er) + pyb.close() sys.exit(1) except KeyboardInterrupt: sys.exit(1) diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py index 3436d0f45..dadfea1cc 100755 --- a/tools/tinytest-codegen.py +++ b/tools/tinytest-codegen.py @@ -48,10 +48,9 @@ testgroup_member = ( # currently these tests are selected because they pass on qemu-arm test_dirs = ('basics', 'micropython', 'float', 'extmod', 'inlineasm') # 'import', 'io', 'misc') exclude_tests = ( - 'float/float2int_doubleprec.py', # requires double precision floating point to work + 'float/float2int_doubleprec_intbig.py', # requires double precision floating point to work 'inlineasm/asmfpaddsub.py', 'inlineasm/asmfpcmp.py', 'inlineasm/asmfpldrstr.py', 'inlineasm/asmfpmuldiv.py', 'inlineasm/asmfpsqrt.py', 'extmod/ticks_diff.py', 'extmod/time_ms_us.py', 'extmod/uheapq_timeq.py', - 'extmod/machine_pinbase.py', 'extmod/machine_pulse.py', 'extmod/vfs_fat_ramdisk.py', 'extmod/vfs_fat_fileio.py', 'extmod/vfs_fat_fsusermount.py', 'extmod/vfs_fat_oldproto.py', ) diff --git a/tools/upip.py b/tools/upip.py index db18a7427..7b85c718f 100644 --- a/tools/upip.py +++ b/tools/upip.py @@ -104,48 +104,62 @@ import usocket warn_ussl = True def url_open(url): global warn_ussl + + if debug: + print(url) + proto, _, host, urlpath = url.split('/', 3) - ai = usocket.getaddrinfo(host, 443) + try: + ai = usocket.getaddrinfo(host, 443) + except OSError as e: + fatal("Unable to resolve %s (no Internet?)" % host, e) #print("Address infos:", ai) addr = ai[0][4] s = usocket.socket(ai[0][0]) - #print("Connect address:", addr) - s.connect(addr) - - if proto == "https:": - s = ussl.wrap_socket(s) - if warn_ussl: - print("Warning: %s SSL certificate is not validated" % host) - warn_ussl = False - - # MicroPython rawsocket module supports file interface directly - s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host)) - l = s.readline() - protover, status, msg = l.split(None, 2) - if status != b"200": - if status == b"404": - print("Package not found") - raise ValueError(status) - while 1: + try: + #print("Connect address:", addr) + s.connect(addr) + + if proto == "https:": + s = ussl.wrap_socket(s) + if warn_ussl: + print("Warning: %s SSL certificate is not validated" % host) + warn_ussl = False + + # MicroPython rawsocket module supports file interface directly + s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host)) l = s.readline() - if not l: - raise ValueError("Unexpected EOF") - if l == b'\r\n': - break + protover, status, msg = l.split(None, 2) + if status != b"200": + if status == b"404" or status == b"301": + raise NotFoundError("Package not found") + raise ValueError(status) + while 1: + l = s.readline() + if not l: + raise ValueError("Unexpected EOF in HTTP headers") + if l == b'\r\n': + break + except Exception as e: + s.close() + raise e return s def get_pkg_metadata(name): f = url_open("https://pypi.python.org/pypi/%s/json" % name) - s = f.read() - f.close() - return json.loads(s) + try: + return json.load(f) + finally: + f.close() -def fatal(msg): - print(msg) +def fatal(msg, exc=None): + print("Error:", msg) + if exc and debug: + raise exc sys.exit(1) def install_pkg(pkg_spec, install_path): @@ -160,10 +174,12 @@ def install_pkg(pkg_spec, install_path): print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) package_fname = op_basename(package_url) f1 = url_open(package_url) - f2 = uzlib.DecompIO(f1, gzdict_sz) - f3 = tarfile.TarFile(fileobj=f2) - meta = install_tar(f3, install_path) - f1.close() + try: + f2 = uzlib.DecompIO(f1, gzdict_sz) + f3 = tarfile.TarFile(fileobj=f2) + meta = install_tar(f3, install_path) + finally: + f1.close() del f3 del f2 gc.collect() @@ -200,9 +216,10 @@ def install(to_install, install_path=None): if deps: deps = deps.decode("utf-8").split("\n") to_install.extend(deps) - except NotFoundError: - print("Error: cannot find '%s' package (or server error), packages may be partially installed" \ - % pkg_spec, file=sys.stderr) + except Exception as e: + print("Error installing '{}': {}, packages may be partially installed".format( + pkg_spec, e), + file=sys.stderr) def get_install_path(): global install_path @@ -267,6 +284,8 @@ def main(): l = f.readline() if not l: break + if l[0] == "#": + continue to_install.append(l.rstrip()) elif opt == "--debug": debug = True |