tools/manifestfile.py: Add library for working with manifests.

This splits the manifest file loading logic from makemanifest.py and
updates makemanifest.py to use it.

This will allow non-freezing uses of manifests, such as defining packages
and dependencies in micropython-lib.

Also adds additional methods to the manifest "API":
 - require() - to get a package from micropython-lib.
 - module() - to define a single-file module
 - package() - to define a multi-file package

module() and package() should replace most uses of freeze() and can also
be also used in non-freezing scenarios.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

2 files changed, 477 insertions, 241 deletions

diff --git a/tools/makemanifest.py b/tools/makemanifest.py
index e69698d3f..800a25435 100644
--- a/tools/makemanifest.py
+++ b/tools/makemanifest.py
@@ -29,127 +29,10 @@ import sys
 import os
 import subprocess
 
-
-###########################################################################
-# Public functions to be used in the manifest
-
-
-def include(manifest, **kwargs):
-    """Include another manifest.
-
-    The manifest argument can be a string (filename) or an iterable of
-    strings.
-
-    Relative paths are resolved with respect to the current manifest file.
-
-    Optional kwargs can be provided which will be available to the
-    included script via the `options` variable.
-
-    e.g. include("path.py", extra_features=True)
-
-    in path.py:
-        options.defaults(standard_features=True)
-
-        # freeze minimal modules.
-        if options.standard_features:
-            # freeze standard modules.
-        if options.extra_features:
-            # freeze extra modules.
-    """
-
-    if not isinstance(manifest, str):
-        for m in manifest:
-            include(m)
-    else:
-        manifest = convert_path(manifest)
-        with open(manifest) as f:
-            # Make paths relative to this manifest file while processing it.
-            # Applies to includes and input files.
-            prev_cwd = os.getcwd()
-            os.chdir(os.path.dirname(manifest))
-            exec(f.read(), globals(), {"options": IncludeOptions(**kwargs)})
-            os.chdir(prev_cwd)
-
-
-def freeze(path, script=None, opt=0):
-    """Freeze the input, automatically determining its type.  A .py script
-    will be compiled to a .mpy first then frozen, and a .mpy file will be
-    frozen directly.
-
-    `path` must be a directory, which is the base directory to search for
-    files from.  When importing the resulting frozen modules, the name of
-    the module will start after `path`, ie `path` is excluded from the
-    module name.
-
-    If `path` is relative, it is resolved to the current manifest.py.
-    Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need
-    to access specific paths.
-
-    If `script` is None all files in `path` will be frozen.
-
-    If `script` is an iterable then freeze() is called on all items of the
-    iterable (with the same `path` and `opt` passed through).
-
-    If `script` is a string then it specifies the file or directory to
-    freeze, and can include extra directories before the file or last
-    directory.  The file or directory will be searched for in `path`.  If
-    `script` is a directory then all files in that directory will be frozen.
-
-    `opt` is the optimisation level to pass to mpy-cross when compiling .py
-    to .mpy.
-    """
-
-    freeze_internal(KIND_AUTO, path, script, opt)
-
-
-def freeze_as_str(path):
-    """Freeze the given `path` and all .py scripts within it as a string,
-    which will be compiled upon import.
-    """
-
-    freeze_internal(KIND_AS_STR, path, None, 0)
-
-
-def freeze_as_mpy(path, script=None, opt=0):
-    """Freeze the input (see above) by first compiling the .py scripts to
-    .mpy files, then freezing the resulting .mpy files.
-    """
-
-    freeze_internal(KIND_AS_MPY, path, script, opt)
-
-
-def freeze_mpy(path, script=None, opt=0):
-    """Freeze the input (see above), which must be .mpy files that are
-    frozen directly.
-    """
-
-    freeze_internal(KIND_MPY, path, script, opt)
-
-
-###########################################################################
-# Internal implementation
-
-KIND_AUTO = 0
-KIND_AS_STR = 1
-KIND_AS_MPY = 2
-KIND_MPY = 3
+import manifestfile
 
 VARS = {}
 
-manifest_list = []
-
-
-class IncludeOptions:
-    def __init__(self, **kwargs):
-        self._kwargs = kwargs
-        self._defaults = {}
-
-    def defaults(self, **kwargs):
-        self._defaults = kwargs
-
-    def __getattr__(self, name):
-        return self._kwargs.get(name, self._defaults.get(name, None))
-
 
 class FreezeError(Exception):
     pass
@@ -163,15 +46,6 @@ def system(cmd):
         return -1, er.output
 
 
-def convert_path(path):
-    # Perform variable substituion.
-    for name, value in VARS.items():
-        path = path.replace("$({})".format(name), value)
-    # Convert to absolute path (so that future operations don't rely on
-    # still being chdir'ed).
-    return os.path.abspath(path)
-
-
 def get_timestamp(path, default=None):
     try:
         stat = os.stat(path)
@@ -182,119 +56,64 @@ def get_timestamp(path, default=None):
         return default
 
 
-def get_timestamp_newest(path):
-    ts_newest = 0
-    for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
-        for f in filenames:
-            ts_newest = max(ts_newest, get_timestamp(os.path.join(dirpath, f)))
-    return ts_newest
-
-
 def mkdir(filename):
     path = os.path.dirname(filename)
     if not os.path.isdir(path):
         os.makedirs(path)
 
 
-def freeze_internal(kind, path, script, opt):
-    path = convert_path(path)
-    if not os.path.isdir(path):
-        raise FreezeError("freeze path must be a directory: {}".format(path))
-    if script is None and kind == KIND_AS_STR:
-        manifest_list.append((KIND_AS_STR, path, script, opt))
-    elif script is None or isinstance(script, str) and script.find(".") == -1:
-        # Recursively search `path` for files to freeze, optionally restricted
-        # to a subdirectory specified by `script`
-        if script is None:
-            subdir = ""
-        else:
-            subdir = "/" + script
-        for dirpath, dirnames, filenames in os.walk(path + subdir, followlinks=True):
-            for f in filenames:
-                freeze_internal(kind, path, (dirpath + "/" + f)[len(path) + 1 :], opt)
-    elif not isinstance(script, str):
-        # `script` is an iterable of items to freeze
-        for s in script:
-            freeze_internal(kind, path, s, opt)
-    else:
-        # `script` should specify an individual file to be frozen
-        extension_kind = {KIND_AS_MPY: ".py", KIND_MPY: ".mpy"}
-        if kind == KIND_AUTO:
-            for k, ext in extension_kind.items():
-                if script.endswith(ext):
-                    kind = k
-                    break
-            else:
-                print("warn: unsupported file type, skipped freeze: {}".format(script))
-                return
-        wanted_extension = extension_kind[kind]
-        if not script.endswith(wanted_extension):
-            raise FreezeError("expecting a {} file, got {}".format(wanted_extension, script))
-        manifest_list.append((kind, path, script, opt))
-
-
 # Formerly make-frozen.py.
 # This generates:
 # - MP_FROZEN_STR_NAMES macro
 # - mp_frozen_str_sizes
 # - mp_frozen_str_content
-def generate_frozen_str_content(paths):
-    def module_name(f):
-        return f
-
-    modules = []
-    output = [b"#include <stdint.h>\n"]
-
-    for path in paths:
-        root = path.rstrip("/")
-        root_len = len(root)
-
-        for dirpath, dirnames, filenames in os.walk(root):
-            for f in filenames:
-                fullpath = dirpath + "/" + f
-                st = os.stat(fullpath)
-                modules.append((path, fullpath[root_len + 1 :], st))
-
-    output.append(b"#define MP_FROZEN_STR_NAMES \\\n")
-    for _path, f, st in modules:
-        m = module_name(f)
-        output.append(b'"%s\\0" \\\n' % m.encode())
+def generate_frozen_str_content(modules):
+    output = [
+        b"#include <stdint.h>\n",
+        b"#define MP_FROZEN_STR_NAMES \\\n",
+    ]
+
+    for _, target_path in modules:
+        print("STR", target_path)
+        output.append(b'"%s\\0" \\\n' % target_path.encode())
     output.append(b"\n")
 
     output.append(b"const uint32_t mp_frozen_str_sizes[] = { ")
 
-    for _path, f, st in modules:
+    for full_path, _ in modules:
+        st = os.stat(full_path)
         output.append(b"%d, " % st.st_size)
     output.append(b"0 };\n")
 
     output.append(b"const char mp_frozen_str_content[] = {\n")
-    for path, f, st in modules:
-        data = open(path + "/" + f, "rb").read()
-
-        # We need to properly escape the script data to create a C string.
-        # When C parses hex characters of the form \x00 it keeps parsing the hex
-        # data until it encounters a non-hex character.  Thus one must create
-        # strings of the form "data\x01" "abc" to properly encode this kind of
-        # data.  We could just encode all characters as hex digits but it's nice
-        # to be able to read the resulting C code as ASCII when possible.
-
-        data = bytearray(data)  # so Python2 extracts each byte as an integer
-        esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"}
-        output.append(b'"')
-        break_str = False
-        for c in data:
-            try:
-                output.append(esc_dict[c])
-            except KeyError:
-                if 32 <= c <= 126:
-                    if break_str:
-                        output.append(b'" "')
-                        break_str = False
-                    output.append(chr(c).encode())
-                else:
-                    output.append(b"\\x%02x" % c)
-                    break_str = True
-        output.append(b'\\0"\n')
+    for full_path, _ in modules:
+        with open(full_path, "rb") as f:
+            data = f.read()
+
+            # We need to properly escape the script data to create a C string.
+            # When C parses hex characters of the form \x00 it keeps parsing the hex
+            # data until it encounters a non-hex character.  Thus one must create
+            # strings of the form "data\x01" "abc" to properly encode this kind of
+            # data.  We could just encode all characters as hex digits but it's nice
+            # to be able to read the resulting C code as ASCII when possible.
+
+            data = bytearray(data)  # so Python2 extracts each byte as an integer
+            esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"}
+            output.append(b'"')
+            break_str = False
+            for c in data:
+                try:
+                    output.append(esc_dict[c])
+                except KeyError:
+                    if 32 <= c <= 126:
+                        if break_str:
+                            output.append(b'" "')
+                            break_str = False
+                        output.append(chr(c).encode())
+                    else:
+                        output.append(b"\\x%02x" % c)
+                        break_str = True
+            output.append(b'\\0"\n')
 
     output.append(b'"\\0"\n};\n\n')
     return b"".join(output)
@@ -340,14 +159,13 @@ def main():
         print("mpy-cross not found at {}, please build it first".format(MPY_CROSS))
         sys.exit(1)
 
+    manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS)
+
     # Include top-level inputs, to generate the manifest
     for input_manifest in args.files:
         try:
-            if input_manifest.endswith(".py"):
-                include(input_manifest)
-            else:
-                exec(input_manifest)
-        except FreezeError as er:
+            manifest.execute(input_manifest)
+        except manifestfile.ManifestFileError as er:
             print('freeze error executing "{}": {}'.format(input_manifest, er.args[0]))
             sys.exit(1)
 
@@ -355,22 +173,25 @@ def main():
     str_paths = []
     mpy_files = []
     ts_newest = 0
-    for kind, path, script, opt in manifest_list:
-        if kind == KIND_AS_STR:
-            str_paths.append(path)
-            ts_outfile = get_timestamp_newest(path)
-        elif kind == KIND_AS_MPY:
-            infile = "{}/{}".format(path, script)
-            outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, script[:-3])
-            ts_infile = get_timestamp(infile)
+    for full_path, target_path, timestamp, kind, version, opt in manifest.files():
+        if kind == manifestfile.KIND_FREEZE_AS_STR:
+            str_paths.append(
+                (
+                    full_path,
+                    target_path,
+                )
+            )
+            ts_outfile = timestamp
+        elif kind == manifestfile.KIND_FREEZE_AS_MPY:
+            outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, target_path[:-3])
             ts_outfile = get_timestamp(outfile, 0)
-            if ts_infile >= ts_outfile:
-                print("MPY", script)
+            if timestamp >= ts_outfile:
+                print("MPY", target_path)
                 mkdir(outfile)
                 res, out = system(
                     [MPY_CROSS]
                     + args.mpy_cross_flags.split()
-                    + ["-o", outfile, "-s", script, "-O{}".format(opt), infile]
+                    + ["-o", outfile, "-s", target_path, "-O{}".format(opt), full_path]
                 )
                 if res != 0:
                     print("error compiling {}:".format(infile))
@@ -379,10 +200,9 @@ def main():
                 ts_outfile = get_timestamp(outfile)
             mpy_files.append(outfile)
         else:
-            assert kind == KIND_MPY
-            infile = "{}/{}".format(path, script)
-            mpy_files.append(infile)
-            ts_outfile = get_timestamp(infile)
+            assert kind == manifestfile.KIND_FREEZE_MPY
+            mpy_files.append(full_path)
+            ts_outfile = timestamp
         ts_newest = max(ts_newest, ts_outfile)
 
     # Check if output file needs generating
diff --git a/tools/manifestfile.py b/tools/manifestfile.py
new file mode 100644
index 000000000..cb155da21
--- /dev/null
+++ b/tools/manifestfile.py
@@ -0,0 +1,416 @@
+#!/usr/bin/env python3
+#
+# This file is part of the MicroPython project, http://micropython.org/
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2022 Jim Mussared
+# Copyright (c) 2019 Damien P. George
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import os
+import sys
+import glob
+
+__all__ = ["ManifestFileError", "ManifestFile"]
+
+# Allow freeze*() etc.
+MODE_FREEZE = 1
+# Only allow include/require/module/package.
+MODE_COMPILE = 2
+
+
+# In compile mode, .py -> KIND_COMPILE_AS_MPY
+# In freeze mode, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY
+KIND_AUTO = 1
+# Freeze-mode only, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY
+KIND_FREEZE_AUTO = 2
+
+# Freeze-mode only, The .py file will be frozen as text.
+KIND_FREEZE_AS_STR = 3
+# Freeze-mode only, The .py file will be compiled and frozen as bytecode.
+KIND_FREEZE_AS_MPY = 4
+# Freeze-mode only, The .mpy file will be frozen directly.
+KIND_FREEZE_MPY = 5
+# Compile mode only, the .py file should be compiled to .mpy.
+KIND_COMPILE_AS_MPY = 6
+
+# File on the local filesystem.
+FILE_TYPE_LOCAL = 1
+# URL to file. (TODO)
+FILE_TYPE_HTTP = 2
+
+
+class ManifestFileError(Exception):
+    pass
+
+
+# Turns a dict of options into a object with attributes used to turn the
+# kwargs passed to include() and require into the "options" global in the
+# included manifest.
+#   options = IncludeOptions(foo="bar", blah="stuff")
+#   options.foo # "bar"
+#   options.blah # "stuff"
+class IncludeOptions:
+    def __init__(self, **kwargs):
+        self._kwargs = kwargs
+        self._defaults = {}
+
+    def defaults(self, **kwargs):
+        self._defaults = kwargs
+
+    def __getattr__(self, name):
+        return self._kwargs.get(name, self._defaults.get(name, None))
+
+
+class ManifestFile:
+    def __init__(self, mode, path_vars=None):
+        # Either MODE_FREEZE or MODE_COMPILE.
+        self._mode = mode
+        # Path substition variables.
+        self._path_vars = path_vars or {}
+        # List of files references by this manifest.
+        # Tuple of (file_type, full_path, target_path, timestamp, kind, version, opt)
+        self._manifest_files = []
+        # Don't allow including the same file twice.
+        self._visited = set()
+
+    def _resolve_path(self, path):
+        # Convert path to an absolute path, applying variable substitutions.
+        for name, value in self._path_vars.items():
+            if value is not None:
+                path = path.replace("$({})".format(name), value)
+        return os.path.abspath(path)
+
+    def _manifest_globals(self, kwargs):
+        # This is the "API" available to a manifest file.
+        return {
+            "metadata": self.metadata,
+            "include": self.include,
+            "require": self.require,
+            "package": self.package,
+            "module": self.module,
+            "freeze": self.freeze,
+            "freeze_as_str": self.freeze_as_str,
+            "freeze_as_mpy": self.freeze_as_mpy,
+            "freeze_mpy": self.freeze_mpy,
+            "options": IncludeOptions(**kwargs),
+        }
+
+    def files(self):
+        return self._manifest_files
+
+    def execute(self, manifest_file):
+        if manifest_file.endswith(".py"):
+            # Execute file from filesystem.
+            self.include(manifest_file)
+        else:
+            # Execute manifest code snippet.
+            try:
+                exec(manifest_file, self._manifest_globals({}))
+            except Exception as er:
+                raise ManifestFileError("Error in manifest: {}".format(er))
+
+    def _add_file(self, full_path, target_path, kind=KIND_AUTO, version=None, opt=None):
+        # Check file exists and get timestamp.
+        try:
+            stat = os.stat(full_path)
+            timestamp = stat.st_mtime
+        except OSError:
+            raise ManifestFileError("cannot stat {}".format(full_path))
+
+        # Map the AUTO kinds to their actual kind based on mode and extension.
+        _, ext = os.path.splitext(full_path)
+        if self._mode == MODE_FREEZE:
+            if kind in (
+                KIND_AUTO,
+                KIND_FREEZE_AUTO,
+            ):
+                if ext.lower() == ".py":
+                    kind = KIND_FREEZE_AS_MPY
+                elif ext.lower() == ".mpy":
+                    kind = KIND_FREEZE_MPY
+        else:
+            if kind != KIND_AUTO:
+                raise ManifestFileError("Not in freeze mode")
+            if ext.lower() != ".py":
+                raise ManifestFileError("Expected .py file")
+            kind = KIND_COMPILE_AS_MPY
+
+        self._manifest_files.append(
+            (FILE_TYPE_LOCAL, full_path, target_path, timestamp, kind, version, opt)
+        )
+
+    def _search(self, base_path, package_path, files, exts, kind, opt=None, strict=False):
+        base_path = self._resolve_path(base_path)
+
+        if files:
+            # Use explicit list of files (relative to package_path).
+            for file in files:
+                if package_path:
+                    file = os.path.join(package_path, file)
+                self._add_file(
+                    os.path.join(base_path, file), file, kind=kind, version=None, opt=opt
+                )
+        else:
+            if base_path:
+                prev_cwd = os.getcwd()
+                os.chdir(self._resolve_path(base_path))
+
+            # Find all candidate files.
+            for dirpath, _, filenames in os.walk(package_path or ".", followlinks=True):
+                for file in filenames:
+                    file = os.path.relpath(os.path.join(dirpath, file), ".")
+                    _, ext = os.path.splitext(file)
+                    if ext.lower() in exts:
+                        self._add_file(
+                            os.path.join(base_path, file),
+                            file,
+                            kind=kind,
+                            version=None,
+                            opt=opt,
+                        )
+                    elif strict:
+                        raise ManifestFileError("Unexpected file type")
+
+            if base_path:
+                os.chdir(prev_cwd)
+
+    def metadata(self, description=None, version=None):
+        # TODO
+        pass
+
+    def include_maybe(self, manifest_path, **kwargs):
+        """
+        Include the manifest file if it exists. See docs for include().
+        """
+        if os.path.exists(manifest_path):
+            self.include(manifest_path, **kwargs)
+
+    def include(self, manifest_path, **kwargs):
+        """
+        Include another manifest.
+
+        The manifest argument can be a string (filename) or an iterable of
+        strings.
+
+        Relative paths are resolved with respect to the current manifest file.
+
+        Optional kwargs can be provided which will be available to the
+        included script via the `options` variable.
+
+        e.g. include("path.py", extra_features=True)
+
+        in path.py:
+            options.defaults(standard_features=True)
+
+            # freeze minimal modules.
+            if options.standard_features:
+                # freeze standard modules.
+            if options.extra_features:
+                # freeze extra modules.
+        """
+        if not isinstance(manifest_path, str):
+            for m in manifest_path:
+                self.include(m)
+        else:
+            manifest_path = self._resolve_path(manifest_path)
+            if manifest_path in self._visited:
+                return
+            self._visited.add(manifest_path)
+            with open(manifest_path) as f:
+                # Make paths relative to this manifest file while processing it.
+                # Applies to includes and input files.
+                prev_cwd = os.getcwd()
+                os.chdir(os.path.dirname(manifest_path))
+                try:
+                    exec(f.read(), self._manifest_globals(kwargs))
+                except Exception as er:
+                    raise ManifestFileError(
+                        "Error in manifest file: {}: {}".format(manifest_path, er)
+                    )
+                os.chdir(prev_cwd)
+
+    def require(self, name, version=None, **kwargs):
+        """
+        Require a module by name from micropython-lib.
+
+        This is a shortcut for
+        """
+        if self._path_vars["MPY_LIB_DIR"]:
+            for manifest_path in glob.glob(
+                os.path.join(self._path_vars["MPY_LIB_DIR"], "**", name, "manifest.py"),
+                recursive=True,
+            ):
+                self.include(manifest_path, **kwargs)
+                return
+            raise ValueError("Library not found in local micropython-lib: {}".format(name))
+        else:
+            # TODO: HTTP request to obtain URLs from manifest.json.
+            raise ValueError("micropython-lib not available for require('{}').", name)
+
+    def package(self, package_path, files=None, base_path=".", opt=None):
+        """
+        Define a package, optionally restricting to a set of files.
+
+        Simple case, a package in the current directory:
+            package("foo")
+        will include all .py files in foo, and will be stored as foo/bar/baz.py.
+
+        If the package isn't in the current directory, use base_path:
+            package("foo", base_path="src")
+
+        To restrict to certain files in the package use files (note: paths should be relative to the package):
+            package("foo", files=["bar/baz.py"])
+        """
+        # Include "base_path/package_path/**/*.py" --> "package_path/**/*.py"
+        self._search(base_path, package_path, files, exts=(".py",), kind=KIND_AUTO, opt=opt)
+
+    def module(self, module_path, base_path=".", opt=None):
+        """
+        Include a single Python file as a module.
+
+        If the file is in the current directory:
+            module("foo.py")
+
+        Otherwise use base_path to locate the file:
+            module("foo.py", "src/drivers")
+        """
+        # Include "base_path/module_path" --> "module_path"
+        base_path = self._resolve_path(base_path)
+        _, ext = os.path.splitext(module_path)
+        if ext.lower() != ".py":
+            raise ManifestFileError("module must be .py file")
+        # TODO: version None
+        self._add_file(os.path.join(base_path, module_path), module_path, version=None, opt=opt)
+
+    def _freeze_internal(self, path, script, exts, kind, opt):
+        if script is None:
+            self._search(path, None, None, exts=exts, kind=kind, opt=opt)
+        elif isinstance(script, str) and os.path.isdir(os.path.join(path, script)):
+            self._search(path, script, None, exts=exts, kind=kind, opt=opt)
+        elif not isinstance(script, str):
+            self._search(path, None, script, exts=exts, kind=kind, opt=opt)
+        else:
+            self._search(path, None, (script,), exts=exts, kind=kind, opt=opt)
+
+    def freeze(self, path, script=None, opt=None):
+        """
+        Freeze the input, automatically determining its type.  A .py script
+        will be compiled to a .mpy first then frozen, and a .mpy file will be
+        frozen directly.
+
+        `path` must be a directory, which is the base directory to _search for
+        files from.  When importing the resulting frozen modules, the name of
+        the module will start after `path`, ie `path` is excluded from the
+        module name.
+
+        If `path` is relative, it is resolved to the current manifest.py.
+        Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need
+        to access specific paths.
+
+        If `script` is None all files in `path` will be frozen.
+
+        If `script` is an iterable then freeze() is called on all items of the
+        iterable (with the same `path` and `opt` passed through).
+
+        If `script` is a string then it specifies the file or directory to
+        freeze, and can include extra directories before the file or last
+        directory.  The file or directory will be _searched for in `path`.  If
+        `script` is a directory then all files in that directory will be frozen.
+
+        `opt` is the optimisation level to pass to mpy-cross when compiling .py
+        to .mpy.
+        """
+        self._freeze_internal(path, script, exts=(".py", ".mpy"), kind=KIND_FREEZE_AUTO, opt=opt)
+
+    def freeze_as_str(self, path):
+        """
+        Freeze the given `path` and all .py scripts within it as a string,
+        which will be compiled upon import.
+        """
+        self._search(path, None, None, exts=(".py"), kind=KIND_FREEZE_AS_STR)
+
+    def freeze_as_mpy(self, path, script=None, opt=None):
+        """
+        Freeze the input (see above) by first compiling the .py scripts to
+        .mpy files, then freezing the resulting .mpy files.
+        """
+        self._freeze_internal(path, script, exts=(".py"), kind=KIND_FREEZE_AS_MPY, opt=opt)
+
+    def freeze_mpy(self, path, script=None, opt=None):
+        """
+        Freeze the input (see above), which must be .mpy files that are
+        frozen directly.
+        """
+        self._freeze_internal(path, script, exts=(".mpy"), kind=KIND_FREEZE_MPY, opt=opt)
+
+
+def main():
+    import argparse
+
+    cmd_parser = argparse.ArgumentParser(description="List the files referenced by a manifest.")
+    cmd_parser.add_argument("--freeze", action="store_true", help="freeze mode")
+    cmd_parser.add_argument("--compile", action="store_true", help="compile mode")
+    cmd_parser.add_argument(
+        "--lib",
+        default=os.path.join(os.path.dirname(__file__), "../lib/micropython-lib"),
+        help="path to micropython-lib repo",
+    )
+    cmd_parser.add_argument("--port", default=None, help="path to port dir")
+    cmd_parser.add_argument("--board", default=None, help="path to board dir")
+    cmd_parser.add_argument(
+        "--top",
+        default=os.path.join(os.path.dirname(__file__), ".."),
+        help="path to micropython repo",
+    )
+    cmd_parser.add_argument("files", nargs="+", help="input manifest.py")
+    args = cmd_parser.parse_args()
+
+    path_vars = {
+        "MPY_DIR": os.path.abspath(args.top) if args.top else None,
+        "BOARD_DIR": os.path.abspath(args.board) if args.board else None,
+        "PORT_DIR": os.path.abspath(args.port) if args.port else None,
+        "MPY_LIB_DIR": os.path.abspath(args.lib) if args.lib else None,
+    }
+
+    mode = None
+    if args.freeze:
+        mode = MODE_FREEZE
+    elif args.compile:
+        mode = MODE_COMPILE
+    else:
+        print("Error: No mode specified.", file=sys.stderr)
+        exit(1)
+
+    m = ManifestFile(mode, path_vars)
+    for manifest_file in args.files:
+        try:
+            m.execute(manifest_file)
+        except ManifestFileError as er:
+            print(er, file=sys.stderr)
+            exit(1)
+    for f in m.files():
+        print(f)
+
+
+if __name__ == "__main__":
+    main()