py/persistentcode: Add architecture flags compatibility checks.

This commit extends the MPY file format in a backwards-compatible way to
store an encoded form of architecture-specific flags that have been
specified in the "mpy-cross" command line, or that have been explicitly
set as part of a native emitter configuration.

The file format changes are as follows:

* The features byte, previously containing the target native
  architecture and the minor file format version, now claims bit 6 as a
  flag indicating the presence of an encoded architecture flags integer
* If architecture flags need to be stored, they are placed right after
  the MPY file header.

This means that properly-written MPY parsers, if encountering a MPY file
containing encoded architecture flags, should raise an error since no
architecture identifiers have been defined that make use of bits 6 and
7 in the referenced header byte.  This should give enough guarantees of
backwards compatibility when this feature is used (improper parsers were
subjected to breakage anyway).

The encoded architecture flags could have been placed at the end, but:

* Having them right after the header makes the architecture
  compatibility checks occur before having read the whole file in memory
  (which still happens on certain platforms as the reader may be backed
  by a memory buffer), and prevents eventual memory allocations that do
  not take place if the module is rejected early
* Properly-written MPY file parsers should have checked the upper two
  bits of the flags byte to be actually zero according to the format
  specification available right before this change, so no assumptions
  should have been made on the exact order of the chunks for an
  unexpected format.

The meaning of the architecture flags value is backend-specific, with
the only common characteristic of being a variable-encoded unsigned
integer for the time being.

The changes made to the file format effectively limit the number of
possible target architectures to 16, of which 13 are already claimed.
There aren't that many new architectures planned to be supported for the
lifetime of the current MPY file format, so this change still leaves
space for architecture updates if needed.

Signed-off-by: Alessandro Gatti <a.gatti@frob.it>

3 files changed, 19 insertions, 3 deletions

diff --git a/py/bc.h b/py/bc.h
index f24510ea7..8caac6d4e 100644
--- a/py/bc.h
+++ b/py/bc.h
@@ -220,6 +220,7 @@ typedef struct _mp_compiled_module_t {
     bool has_native;
     size_t n_qstr;
     size_t n_obj;
+    size_t arch_flags;
     #endif
 } mp_compiled_module_t;
 
diff --git a/py/persistentcode.c b/py/persistentcode.c
index 6ec0717f9..7d71cfd98 100644
--- a/py/persistentcode.c
+++ b/py/persistentcode.c
@@ -471,7 +471,7 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
         || header[3] > MP_SMALL_INT_BITS) {
         mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
     }
-    if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) {
+    if (arch != MP_NATIVE_ARCH_NONE) {
         if (!MPY_FEATURE_ARCH_TEST(arch)) {
             if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) {
                 // On supported ports this can be resolved by enabling feature, eg
@@ -483,6 +483,12 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
         }
     }
 
+    size_t arch_flags = 0;
+    if (MPY_FEATURE_ARCH_FLAGS_TEST(header[2])) {
+        (void)arch_flags;
+        mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
+    }
+
     size_t n_qstr = read_uint(reader);
     size_t n_obj = read_uint(reader);
     mp_module_context_alloc_tables(cm->context, n_qstr, n_obj);
@@ -504,6 +510,7 @@ void mp_raw_code_load(mp_reader_t *reader, mp_compiled_module_t *cm) {
     cm->has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE;
     cm->n_qstr = n_qstr;
     cm->n_obj = n_obj;
+    cm->arch_flags = arch_flags;
     #endif
 
     // Deregister exception handler and close the reader.
@@ -672,7 +679,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
     byte header[4] = {
         'M',
         MPY_VERSION,
-        cm->has_native ? MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC) : 0,
+        (cm->arch_flags != 0 ? MPY_FEATURE_ARCH_FLAGS : 0) | (cm->has_native ? MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC) : 0),
         #if MICROPY_DYNAMIC_COMPILER
         mp_dynamic_compiler.small_int_bits,
         #else
@@ -681,6 +688,10 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
     };
     mp_print_bytes(print, header, sizeof(header));
 
+    if (cm->arch_flags) {
+        mp_print_uint(print, cm->arch_flags);
+    }
+
     // Number of entries in constant table.
     mp_print_uint(print, cm->n_qstr);
     mp_print_uint(print, cm->n_obj);
diff --git a/py/persistentcode.h b/py/persistentcode.h
index a45d5fd18..8fd7068e3 100644
--- a/py/persistentcode.h
+++ b/py/persistentcode.h
@@ -45,7 +45,7 @@
 
 // Macros to encode/decode native architecture to/from the feature byte
 #define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2)
-#define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2)
+#define MPY_FEATURE_DECODE_ARCH(feat) (((feat) >> 2) & 0x2F)
 
 // Define the host architecture
 #if MICROPY_EMIT_X86
@@ -85,6 +85,10 @@
 #define MPY_FILE_HEADER_INT (MPY_VERSION \
     | (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
 
+// Architecture-specific flags are present in the .mpy file
+#define MPY_FEATURE_ARCH_FLAGS (0x40)
+#define MPY_FEATURE_ARCH_FLAGS_TEST(x) (((x) & MPY_FEATURE_ARCH_FLAGS) == MPY_FEATURE_ARCH_FLAGS)
+
 enum {
     MP_NATIVE_ARCH_NONE = 0,
     MP_NATIVE_ARCH_X86,