tools/manifestfile.py: Add library for working with manifests.

This splits the manifest file loading logic from makemanifest.py and
updates makemanifest.py to use it.

This will allow non-freezing uses of manifests, such as defining packages
and dependencies in micropython-lib.

Also adds additional methods to the manifest "API":
 - require() - to get a package from micropython-lib.
 - module() - to define a single-file module
 - package() - to define a multi-file package

module() and package() should replace most uses of freeze() and can also
be also used in non-freezing scenarios.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

1 files changed, 61 insertions, 241 deletions

diff --git a/tools/makemanifest.py b/tools/makemanifest.py
index e69698d3f..800a25435 100644
--- a/tools/makemanifest.py
+++ b/tools/makemanifest.py
@@ -29,127 +29,10 @@ import sys
 import os
 import subprocess
 
-
-###########################################################################
-# Public functions to be used in the manifest
-
-
-def include(manifest, **kwargs):
-    """Include another manifest.
-
-    The manifest argument can be a string (filename) or an iterable of
-    strings.
-
-    Relative paths are resolved with respect to the current manifest file.
-
-    Optional kwargs can be provided which will be available to the
-    included script via the `options` variable.
-
-    e.g. include("path.py", extra_features=True)
-
-    in path.py:
-        options.defaults(standard_features=True)
-
-        # freeze minimal modules.
-        if options.standard_features:
-            # freeze standard modules.
-        if options.extra_features:
-            # freeze extra modules.
-    """
-
-    if not isinstance(manifest, str):
-        for m in manifest:
-            include(m)
-    else:
-        manifest = convert_path(manifest)
-        with open(manifest) as f:
-            # Make paths relative to this manifest file while processing it.
-            # Applies to includes and input files.
-            prev_cwd = os.getcwd()
-            os.chdir(os.path.dirname(manifest))
-            exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)})
-            os.chdir(prev_cwd)
-
-
-def freeze(path, script=None, opt=0):
-    """Freeze the input, automatically determining its type.  A .py script
-    will be compiled to a .mpy first then frozen, and a .mpy file will be
-    frozen directly.
-
-    `path` must be a directory, which is the base directory to search for
-    files from.  When importing the resulting frozen modules, the name of
-    the module will start after `path`, ie `path` is excluded from the
-    module name.
-
-    If `path` is relative, it is resolved to the current manifest.py.
-    Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need
-    to access specific paths.
-
-    If `script` is None all files in `path` will be frozen.
-
-    If `script` is an iterable then freeze() is called on all items of the
-    iterable (with the same `path` and `opt` passed through).
-
-    If `script` is a string then it specifies the file or directory to
-    freeze, and can include extra directories before the file or last
-    directory.  The file or directory will be searched for in `path`.  If
-    `script` is a directory then all files in that directory will be frozen.
-
-    `opt` is the optimisation level to pass to mpy-cross when compiling .py
-    to .mpy.
-    """
-
-    freeze_internal(KIND_AUTO, path, script, opt)
-
-
-def freeze_as_str(path):
-    """Freeze the given `path` and all .py scripts within it as a string,
-    which will be compiled upon import.
-    """
-
-    freeze_internal(KIND_AS_STR, path, None, 0)
-
-
-def freeze_as_mpy(path, script=None, opt=0):
-    """Freeze the input (see above) by first compiling the .py scripts to
-    .mpy files, then freezing the resulting .mpy files.
-    """
-
-    freeze_internal(KIND_AS_MPY, path, script, opt)
-
-
-def freeze_mpy(path, script=None, opt=0):
-    """Freeze the input (see above), which must be .mpy files that are
-    frozen directly.
-    """
-
-    freeze_internal(KIND_MPY, path, script, opt)
-
-
-###########################################################################
-# Internal implementation
-
-KIND_AUTO = 0
-KIND_AS_STR = 1
-KIND_AS_MPY = 2
-KIND_MPY = 3
+import manifestfile
 
 VARS = {}
 
-manifest_list = []
-
-
-class IncludeOptions:
-    def __init__(self, **kwargs):
-        self._kwargs = kwargs
-        self._defaults = {}
-
-    def defaults(self, **kwargs):
-        self._defaults = kwargs
-
-    def __getattr__(self, name):
-        return self._kwargs.get(name, self._defaults.get(name, None))
-
 
 class FreezeError(Exception):
     pass
@@ -163,15 +46,6 @@ def system(cmd):
         return -1, er.output
 
 
-def convert_path(path):
-    # Perform variable substituion.
-    for name, value in VARS.items():
-        path = path.replace("$({})".format(name), value)
-    # Convert to absolute path (so that future operations don't rely on
-    # still being chdir'ed).
-    return os.path.abspath(path)
-
-
 def get_timestamp(path, default=None):
     try:
         stat = os.stat(path)
@@ -182,119 +56,64 @@ def get_timestamp(path, default=None):
         return default
 
 
-def get_timestamp_newest(path):
-    ts_newest = 0
-    for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
-        for f in filenames:
-            ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f)))
-    return ts_newest
-
-
 def mkdir(filename):
     path = os.path.dirname(filename)
     if not os.path.isdir(path):
         os.makedirs(path)
 
 
-def freeze_internal(kind, path, script, opt):
-    path = convert_path(path)
-    if not os.path.isdir(path):
-        raise FreezeError("freeze path must be a directory: {}".format(path))
-    if script is None and kind == KIND_AS_STR:
-        manifest_list.append((KIND_AS_STR, path, script, opt))
-    elif script is None or isinstance(script, str) and script.find(".") == -1:
-        # Recursively search `path` for files to freeze, optionally restricted
-        # to a subdirectory specified by `script`
-        if script is None:
-            subdir = ""
-        else:
-            subdir = "/" + script
-        for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True):
-            for f in filenames:
-                freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt)
-    elif not isinstance(script, str):
-        # `script` is an iterable of items to freeze
-        for s in script:
-            freeze_internal(kind, path, s, opt)
-    else:
-        # `script` should specify an individual file to be frozen
-        extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"}
-        if kind == KIND_AUTO:
-            for k, ext in extension_kind.items():
-                if script.endswith(ext):
-                    kind = k
-                    break
-            else:
-                print("warn: unsupported file type, skipped freeze: {}".format(script))
-                return
-        wanted_extension = extension_kind[kind]
-        if not script.endswith(wanted_extension):
-            raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script))
-        manifest_list.append((kind, path, script, opt))
-
-
 # Formerly make-frozen.py.
 # This generates:
 # - MP_FROZEN_STR_NAMES macro
 # - mp_frozen_str_sizes
 # - mp_frozen_str_content
-def generate_frozen_str_content(paths):
-    def module_name(f):
-        return f
-
-    modules = []
-    output = [b"#include <stdint.h>\n"]
-
-    for path in paths:
-        root = path.rstrip("/")
-        root_len = len(root)
-
-        for dirpath, dirnames, filenames in os.walk(root):
-            for f in filenames:
-                fullpath = dirpath + "/" + f
-                st = os.stat(fullpath)
-                modules.append((path, fullpath[root_len + 1 :], st))
-
-    output.append(b"#define MP_FROZEN_STR_NAMES \\\n")
-    for _path, f, st in modules:
-        m = module_name(f)
-        output.append(b'"%s\\0" \\\n' % m.encode())
+def generate_frozen_str_content(modules):
+    output = [
+        b"#include <stdint.h>\n",
+        b"#define MP_FROZEN_STR_NAMES \\\n",
+    ]
+
+    for _, target_path in modules:
+        print("STR", target_path)
+        output.append(b'"%s\\0" \\\n' % target_path.encode())
     output.append(b"\n")
 
     output.append(b"const uint32_t mp_frozen_str_sizes[] = { ")
 
-    for _path, f, st in modules:
+    for full_path, _ in modules:
+        st = os.stat(full_path)
         output.append(b"%d, " % st.st_size)
     output.append(b"0 };\n")
 
     output.append(b"const char mp_frozen_str_content[] = {\n")
-    for path, f, st in modules:
-        data = open(path + "/" + f, "rb").read()
-
-        # We need to properly escape the script data to create a C string.
-        # When C parses hex characters of the form \x00 it keeps parsing the hex
-        # data until it encounters a non-hex character.  Thus one must create
-        # strings of the form "data\x01" "abc" to properly encode this kind of
-        # data.  We could just encode all characters as hex digits but it's nice
-        # to be able to read the resulting C code as ASCII when possible.
-
-        data = bytearray(data)  # so Python2 extracts each byte as an integer
-        esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"}
-        output.append(b'"')
-        break_str = False
-        for c in data:
-            try:
-                output.append(esc_dict[c])
-            except KeyError:
-                if 32 <= c <= 126:
-                    if break_str:
-                        output.append(b'" "')
-                        break_str = False
-                    output.append(chr(c).encode())
-                else:
-                    output.append(b"\\x%02x" % c)
-                    break_str = True
-        output.append(b'\\0"\n')
+    for full_path, _ in modules:
+        with open(full_path, "rb") as f:
+            data = f.read()
+
+            # We need to properly escape the script data to create a C string.
+            # When C parses hex characters of the form \x00 it keeps parsing the hex
+            # data until it encounters a non-hex character.  Thus one must create
+            # strings of the form "data\x01" "abc" to properly encode this kind of
+            # data.  We could just encode all characters as hex digits but it's nice
+            # to be able to read the resulting C code as ASCII when possible.
+
+            data = bytearray(data)  # so Python2 extracts each byte as an integer
+            esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"}
+            output.append(b'"')
+            break_str = False
+            for c in data:
+                try:
+                    output.append(esc_dict[c])
+                except KeyError:
+                    if 32 <= c <= 126:
+                        if break_str:
+                            output.append(b'" "')
+                            break_str = False
+                        output.append(chr(c).encode())
+                    else:
+                        output.append(b"\\x%02x" % c)
+                        break_str = True
+            output.append(b'\\0"\n')
 
     output.append(b'"\\0"\n};\n\n')
     return b"".join(output)
@@ -340,14 +159,13 @@ def main():
         print("mpy-cross not found at {}, please build it first".format(MPY_CROSS))
         sys.exit(1)
 
+    manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS)
+
     # Include top-level inputs, to generate the manifest
     for input_manifest in args.files:
         try:
-            if input_manifest.endswith(".py"):
-                include(input_manifest)
-            else:
-                exec(input_manifest)
-        except FreezeError as er:
+            manifest.execute(input_manifest)
+        except manifestfile.ManifestFileError as er:
             print('freeze error executing "{}": {}'.format(input_manifest, er.args[0]))
             sys.exit(1)
 
@@ -355,22 +173,25 @@ def main():
     str_paths = []
     mpy_files = []
     ts_newest = 0
-    for kind, path, script, opt in manifest_list:
-        if kind == KIND_AS_STR:
-            str_paths.append(path)
-            ts_outfile = get_timestamp_newest(path)
-        elif kind == KIND_AS_MPY:
-            infile = "{}/{}".format(path, script)
-            outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3])
-            ts_infile = get_timestamp(infile)
+    for full_path, target_path, timestamp, kind, version, opt in manifest.files():
+        if kind == manifestfile.KIND_FREEZE_AS_STR:
+            str_paths.append(
+                (
+                    full_path,
+                    target_path,
+                )
+            )
+            ts_outfile = timestamp
+        elif kind == manifestfile.KIND_FREEZE_AS_MPY:
+            outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, target_path[:-3])
             ts_outfile = get_timestamp(outfile, 0)
-            if ts_infile >= ts_outfile:
-                print("MPY", script)
+            if timestamp >= ts_outfile:
+                print("MPY", target_path)
                 mkdir(outfile)
                 res, out = system(
                     [MPY_CROSS]
                     + args.mpy_cross_flags.split()
-                    + ["-o", outfile, "-s", script, "-O{}".format(opt), infile]
+                    + ["-o", outfile, "-s", target_path, "-O{}".format(opt), full_path]
                 )
                 if res != 0:
                     print("error compiling {}:".format(infile))
@@ -379,10 +200,9 @@ def main():
                 ts_outfile = get_timestamp(outfile)
             mpy_files.append(outfile)
         else:
-            assert kind == KIND_MPY
-            infile = "{}/{}".format(path, script)
-            mpy_files.append(infile)
-            ts_outfile = get_timestamp(infile)
+            assert kind == manifestfile.KIND_FREEZE_MPY
+            mpy_files.append(full_path)
+            ts_outfile = timestamp
         ts_newest = max(ts_newest, ts_outfile)
 
     # Check if output file needs generating