py/qstr: Separate hash and len from string data.

This allows the compiler to merge strings: e.g. "update",
"difference_update" and "symmetric_difference_update" will all point to the
same memory.

No functional change.

The size reduction depends on the number of qstrs in the build.  The change
this commit brings is:

   bare-arm:    -4 -0.007%
minimal x86:  +150 +0.092% [incl +48(data)]
   unix x64:  -608 -0.118%
unix nanbox:  -572 -0.126% [incl +32(data)]
      stm32: -1392 -0.352% PYBV10
     cc3200:  -448 -0.244%
    esp8266: -1208 -0.173% GENERIC
      esp32: -1028 -0.068% GENERIC[incl -1020(data)]
        nrf:  -440 -0.252% pca10040
        rp2: -1072 -0.217% PICO
       samd:  -368 -0.264% ADAFRUIT_ITSYBITSY_M4_EXPRESS

Performance is also improved (on bare metal at least) for the
core_import_mpy_multi.py, core_import_mpy_single.py and core_qstr.py
performance benchmarks.

Originally at adafruit#4583

Signed-off-by: Artyom Skrobov <tyomitch@gmail.com>

1 files changed, 21 insertions, 8 deletions

diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py
index aa0272111..337f580b6 100755
--- a/tools/mpy-tool.py
+++ b/tools/mpy-tool.py
@@ -814,7 +814,7 @@ def freeze_mpy(base_qstrs, raw_codes):
         # don't add duplicates
         if q is None or q.qstr_esc in base_qstrs or q.qstr_esc in new:
             continue
-        new[q.qstr_esc] = (len(new), q.qstr_esc, q.str)
+        new[q.qstr_esc] = (len(new), q.qstr_esc, q.str, bytes_cons(q.str, "utf8"))
     new = sorted(new.values(), key=lambda x: x[0])
 
     print('#include "py/mpconfig.h"')
@@ -865,20 +865,33 @@ def freeze_mpy(base_qstrs, raw_codes):
     qstr_pool_alloc = min(len(new), 10)
 
     print()
+    print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {")
+    qstr_size = {"metadata": 0, "data": 0}
+    for _, _, _, qbytes in new:
+        qhash = qstrutil.compute_hash(qbytes, config.MICROPY_QSTR_BYTES_IN_HASH)
+        print("    %d," % qhash)
+    print("};")
+    print()
+    print("const qstr_len_t mp_qstr_frozen_const_lengths[] = {")
+    for _, _, _, qbytes in new:
+        print("    %d," % len(qbytes))
+        qstr_size["metadata"] += (
+            config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH
+        )
+        qstr_size["data"] += len(qbytes)
+    print("};")
+    print()
     print("extern const qstr_pool_t mp_qstr_const_pool;")
     print("const qstr_pool_t mp_qstr_frozen_const_pool = {")
     print("    (qstr_pool_t*)&mp_qstr_const_pool, // previous pool")
     print("    MP_QSTRnumber_of, // previous pool size")
     print("    %u, // allocated entries" % qstr_pool_alloc)
     print("    %u, // used entries" % len(new))
+    print("    (qstr_hash_t *)mp_qstr_frozen_const_hashes,")
+    print("    (qstr_len_t *)mp_qstr_frozen_const_lengths,")
     print("    {")
-    for _, _, qstr in new:
-        print(
-            "        %s,"
-            % qstrutil.make_bytes(
-                config.MICROPY_QSTR_BYTES_IN_LEN, config.MICROPY_QSTR_BYTES_IN_HASH, qstr
-            )
-        )
+    for _, _, qstr, qbytes in new:
+        print('        "%s",' % qstrutil.escape_bytes(qstr, qbytes))
     print("    },")
     print("};")