diff options
Diffstat (limited to 'tools/makemanifest.py')
| -rw-r--r-- | tools/makemanifest.py | 302 |
1 files changed, 61 insertions, 241 deletions
diff --git a/tools/makemanifest.py b/tools/makemanifest.py index e69698d3f..800a25435 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -29,127 +29,10 @@ import sys import os import subprocess - -########################################################################### -# Public functions to be used in the manifest - - -def include(manifest, **kwargs): - """Include another manifest. - - The manifest argument can be a string (filename) or an iterable of - strings. - - Relative paths are resolved with respect to the current manifest file. - - Optional kwargs can be provided which will be available to the - included script via the `options` variable. - - e.g. include("path.py", extra_features=True) - - in path.py: - options.defaults(standard_features=True) - - # freeze minimal modules. - if options.standard_features: - # freeze standard modules. - if options.extra_features: - # freeze extra modules. - """ - - if not isinstance(manifest, str): - for m in manifest: - include(m) - else: - manifest = convert_path(manifest) - with open(manifest) as f: - # Make paths relative to this manifest file while processing it. - # Applies to includes and input files. - prev_cwd = os.getcwd() - os.chdir(os.path.dirname(manifest)) - exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)}) - os.chdir(prev_cwd) - - -def freeze(path, script=None, opt=0): - """Freeze the input, automatically determining its type. A .py script - will be compiled to a .mpy first then frozen, and a .mpy file will be - frozen directly. - - `path` must be a directory, which is the base directory to search for - files from. When importing the resulting frozen modules, the name of - the module will start after `path`, ie `path` is excluded from the - module name. - - If `path` is relative, it is resolved to the current manifest.py. - Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need - to access specific paths. - - If `script` is None all files in `path` will be frozen. - - If `script` is an iterable then freeze() is called on all items of the - iterable (with the same `path` and `opt` passed through). - - If `script` is a string then it specifies the file or directory to - freeze, and can include extra directories before the file or last - directory. The file or directory will be searched for in `path`. If - `script` is a directory then all files in that directory will be frozen. - - `opt` is the optimisation level to pass to mpy-cross when compiling .py - to .mpy. - """ - - freeze_internal(KIND_AUTO, path, script, opt) - - -def freeze_as_str(path): - """Freeze the given `path` and all .py scripts within it as a string, - which will be compiled upon import. - """ - - freeze_internal(KIND_AS_STR, path, None, 0) - - -def freeze_as_mpy(path, script=None, opt=0): - """Freeze the input (see above) by first compiling the .py scripts to - .mpy files, then freezing the resulting .mpy files. - """ - - freeze_internal(KIND_AS_MPY, path, script, opt) - - -def freeze_mpy(path, script=None, opt=0): - """Freeze the input (see above), which must be .mpy files that are - frozen directly. - """ - - freeze_internal(KIND_MPY, path, script, opt) - - -########################################################################### -# Internal implementation - -KIND_AUTO = 0 -KIND_AS_STR = 1 -KIND_AS_MPY = 2 -KIND_MPY = 3 +import manifestfile VARS = {} -manifest_list = [] - - -class IncludeOptions: - def __init__(self, **kwargs): - self._kwargs = kwargs - self._defaults = {} - - def defaults(self, **kwargs): - self._defaults = kwargs - - def __getattr__(self, name): - return self._kwargs.get(name, self._defaults.get(name, None)) - class FreezeError(Exception): pass @@ -163,15 +46,6 @@ def system(cmd): return -1, er.output -def convert_path(path): - # Perform variable substituion. - for name, value in VARS.items(): - path = path.replace("$({})".format(name), value) - # Convert to absolute path (so that future operations don't rely on - # still being chdir'ed). - return os.path.abspath(path) - - def get_timestamp(path, default=None): try: stat = os.stat(path) @@ -182,119 +56,64 @@ def get_timestamp(path, default=None): return default -def get_timestamp_newest(path): - ts_newest = 0 - for dirpath, dirnames, filenames in os.walk(path, followlinks=True): - for f in filenames: - ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f))) - return ts_newest - - def mkdir(filename): path = os.path.dirname(filename) if not os.path.isdir(path): os.makedirs(path) -def freeze_internal(kind, path, script, opt): - path = convert_path(path) - if not os.path.isdir(path): - raise FreezeError("freeze path must be a directory: {}".format(path)) - if script is None and kind == KIND_AS_STR: - manifest_list.append((KIND_AS_STR, path, script, opt)) - elif script is None or isinstance(script, str) and script.find(".") == -1: - # Recursively search `path` for files to freeze, optionally restricted - # to a subdirectory specified by `script` - if script is None: - subdir = "" - else: - subdir = "/" + script - for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True): - for f in filenames: - freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt) - elif not isinstance(script, str): - # `script` is an iterable of items to freeze - for s in script: - freeze_internal(kind, path, s, opt) - else: - # `script` should specify an individual file to be frozen - extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"} - if kind == KIND_AUTO: - for k, ext in extension_kind.items(): - if script.endswith(ext): - kind = k - break - else: - print("warn: unsupported file type, skipped freeze: {}".format(script)) - return - wanted_extension = extension_kind[kind] - if not script.endswith(wanted_extension): - raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script)) - manifest_list.append((kind, path, script, opt)) - - # Formerly make-frozen.py. # This generates: # - MP_FROZEN_STR_NAMES macro # - mp_frozen_str_sizes # - mp_frozen_str_content -def generate_frozen_str_content(paths): - def module_name(f): - return f - - modules = [] - output = [b"#include <stdint.h>\n"] - - for path in paths: - root = path.rstrip("/") - root_len = len(root) - - for dirpath, dirnames, filenames in os.walk(root): - for f in filenames: - fullpath = dirpath + "/" + f - st = os.stat(fullpath) - modules.append((path, fullpath[root_len + 1 :], st)) - - output.append(b"#define MP_FROZEN_STR_NAMES \\\n") - for _path, f, st in modules: - m = module_name(f) - output.append(b'"%s\\0" \\\n' % m.encode()) +def generate_frozen_str_content(modules): + output = [ + b"#include <stdint.h>\n", + b"#define MP_FROZEN_STR_NAMES \\\n", + ] + + for _, target_path in modules: + print("STR", target_path) + output.append(b'"%s\\0" \\\n' % target_path.encode()) output.append(b"\n") output.append(b"const uint32_t mp_frozen_str_sizes[] = { ") - for _path, f, st in modules: + for full_path, _ in modules: + st = os.stat(full_path) output.append(b"%d, " % st.st_size) output.append(b"0 };\n") output.append(b"const char mp_frozen_str_content[] = {\n") - for path, f, st in modules: - data = open(path + "/" + f, "rb").read() - - # We need to properly escape the script data to create a C string. - # When C parses hex characters of the form \x00 it keeps parsing the hex - # data until it encounters a non-hex character. Thus one must create - # strings of the form "data\x01" "abc" to properly encode this kind of - # data. We could just encode all characters as hex digits but it's nice - # to be able to read the resulting C code as ASCII when possible. - - data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} - output.append(b'"') - break_str = False - for c in data: - try: - output.append(esc_dict[c]) - except KeyError: - if 32 <= c <= 126: - if break_str: - output.append(b'" "') - break_str = False - output.append(chr(c).encode()) - else: - output.append(b"\\x%02x" % c) - break_str = True - output.append(b'\\0"\n') + for full_path, _ in modules: + with open(full_path, "rb") as f: + data = f.read() + + # We need to properly escape the script data to create a C string. + # When C parses hex characters of the form \x00 it keeps parsing the hex + # data until it encounters a non-hex character. Thus one must create + # strings of the form "data\x01" "abc" to properly encode this kind of + # data. We could just encode all characters as hex digits but it's nice + # to be able to read the resulting C code as ASCII when possible. + + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} + output.append(b'"') + break_str = False + for c in data: + try: + output.append(esc_dict[c]) + except KeyError: + if 32 <= c <= 126: + if break_str: + output.append(b'" "') + break_str = False + output.append(chr(c).encode()) + else: + output.append(b"\\x%02x" % c) + break_str = True + output.append(b'\\0"\n') output.append(b'"\\0"\n};\n\n') return b"".join(output) @@ -340,14 +159,13 @@ def main(): print("mpy-cross not found at {}, please build it first".format(MPY_CROSS)) sys.exit(1) + manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS) + # Include top-level inputs, to generate the manifest for input_manifest in args.files: try: - if input_manifest.endswith(".py"): - include(input_manifest) - else: - exec(input_manifest) - except FreezeError as er: + manifest.execute(input_manifest) + except manifestfile.ManifestFileError as er: print('freeze error executing "{}": {}'.format(input_manifest, er.args[0])) sys.exit(1) @@ -355,22 +173,25 @@ def main(): str_paths = [] mpy_files = [] ts_newest = 0 - for kind, path, script, opt in manifest_list: - if kind == KIND_AS_STR: - str_paths.append(path) - ts_outfile = get_timestamp_newest(path) - elif kind == KIND_AS_MPY: - infile = "{}/{}".format(path, script) - outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3]) - ts_infile = get_timestamp(infile) + for full_path, target_path, timestamp, kind, version, opt in manifest.files(): + if kind == manifestfile.KIND_FREEZE_AS_STR: + str_paths.append( + ( + full_path, + target_path, + ) + ) + ts_outfile = timestamp + elif kind == manifestfile.KIND_FREEZE_AS_MPY: + outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, target_path[:-3]) ts_outfile = get_timestamp(outfile, 0) - if ts_infile >= ts_outfile: - print("MPY", script) + if timestamp >= ts_outfile: + print("MPY", target_path) mkdir(outfile) res, out = system( [MPY_CROSS] + args.mpy_cross_flags.split() - + ["-o", outfile, "-s", script, "-O{}".format(opt), infile] + + ["-o", outfile, "-s", target_path, "-O{}".format(opt), full_path] ) if res != 0: print("error compiling {}:".format(infile)) @@ -379,10 +200,9 @@ def main(): ts_outfile = get_timestamp(outfile) mpy_files.append(outfile) else: - assert kind == KIND_MPY - infile = "{}/{}".format(path, script) - mpy_files.append(infile) - ts_outfile = get_timestamp(infile) + assert kind == manifestfile.KIND_FREEZE_MPY + mpy_files.append(full_path) + ts_outfile = timestamp ts_newest = max(ts_newest, ts_outfile) # Check if output file needs generating |