diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/ar_util.py | 236 | ||||
| -rwxr-xr-x | tools/ci.sh | 44 | ||||
| -rwxr-xr-x | tools/mpy_ld.py | 140 |
3 files changed, 357 insertions, 63 deletions
diff --git a/tools/ar_util.py b/tools/ar_util.py new file mode 100644 index 000000000..b90d37903 --- /dev/null +++ b/tools/ar_util.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2024 Volodymyr Shymanskyy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import re +import hashlib +import functools +import pickle + +from elftools.elf import elffile +from collections import defaultdict + +try: + from ar import Archive +except: + Archive = None + + +class PickleCache: + def __init__(self, path, prefix=""): + self.path = path + self._get_fn = lambda key: os.path.join(path, prefix + key[:24]) + + def store(self, key, data): + os.makedirs(self.path, exist_ok=True) + # See also https://bford.info/cachedir/ + cachedir_tag_path = os.path.join(self.path, "CACHEDIR.TAG") + if not os.path.exists(cachedir_tag_path): + with open(cachedir_tag_path, "w") as f: + f.write( + "Signature: 8a477f597d28d172789f06886806bc55\n" + "# This file is a cache directory tag created by MicroPython.\n" + "# For information about cache directory tags see https://bford.info/cachedir/\n" + ) + with open(self._get_fn(key), "wb") as f: + pickle.dump(data, f) + + def load(self, key): + with open(self._get_fn(key), "rb") as f: + return pickle.load(f) + + +def cached(key, cache): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + cache_key = key(*args, **kwargs) + try: + d = cache.load(cache_key) + if d["key"] != cache_key: + raise Exception("Cache key mismatch") + return d["data"] + except Exception: + res = func(*args, **kwargs) + try: + cache.store( + cache_key, + { + "key": cache_key, + "data": res, + }, + ) + except Exception: + pass + return res + + return wrapper + + return decorator + + +class CachedArFile: + def __init__(self, fn): + if not Archive: + raise RuntimeError("Please run 'pip install ar' to link .a files") + self.fn = fn + self._archive = Archive(open(fn, "rb")) + info = self.load_symbols() + self.objs = info["objs"] + self.symbols = info["symbols"] + + def open(self, obj): + return self._archive.open(obj, "rb") + + def _cache_key(self): + sha = hashlib.sha256() + with open(self.fn, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + sha.update(chunk) + # Change this salt if the cache data format changes + sha.update(bytes.fromhex("00000000000000000000000000000001")) + return sha.hexdigest() + + @cached(key=_cache_key, cache=PickleCache(path=".mpy_ld_cache", prefix="ar_")) + def load_symbols(self): + print("Loading", self.fn) + objs = defaultdict(lambda: {"def": set(), "undef": set(), "weak": set()}) + symbols = {} + for entry in self._archive: + obj_name = entry.name + elf = elffile.ELFFile(self.open(obj_name)) + symtab = elf.get_section_by_name(".symtab") + if not symtab: + continue + + obj = objs[obj_name] + + for symbol in symtab.iter_symbols(): + sym_name = symbol.name + sym_bind = symbol["st_info"]["bind"] + + if sym_bind in ("STB_GLOBAL", "STB_WEAK"): + if symbol.entry["st_shndx"] != "SHN_UNDEF": + obj["def"].add(sym_name) + symbols[sym_name] = obj_name + else: + obj["undef"].add(sym_name) + + if sym_bind == "STB_WEAK": + obj["weak"].add(sym_name) + + return {"objs": dict(objs), "symbols": symbols} + + +def resolve(archives, symbols): + resolved_objs = [] # Object files needed to resolve symbols + unresolved_symbols = set() + provided_symbols = {} # Which symbol is provided by which object + symbol_stack = list(symbols) + + # A helper function to handle symbol resolution from a particular object + def add_obj(archive, symbol): + obj_name = archive.symbols[symbol] + obj_info = archive.objs[obj_name] + + obj_tuple = (archive, obj_name) + if obj_tuple in resolved_objs: + return # Already processed this object + + resolved_objs.append(obj_tuple) + + # Add the symbols this object defines + for defined_symbol in obj_info["def"]: + if defined_symbol in provided_symbols and not defined_symbol.startswith( + "__x86.get_pc_thunk." + ): + if defined_symbol in obj_info["weak"]: + continue + else: + raise RuntimeError(f"Multiple definitions for {defined_symbol}") + provided_symbols[defined_symbol] = obj_name # TODO: mark weak if needed + + # Recursively add undefined symbols from this object + for undef_symbol in obj_info["undef"]: + if undef_symbol in obj_info["weak"]: + print(f"Skippping weak dependency: {undef_symbol}") + continue + if undef_symbol not in provided_symbols: + symbol_stack.append(undef_symbol) # Add undefined symbol to resolve + + while symbol_stack: + symbol = symbol_stack.pop(0) + + if symbol in provided_symbols: + continue # Symbol is already resolved + + found = False + for archive in archives: + if symbol in archive.symbols: + add_obj(archive, symbol) + found = True + break + + if not found: + unresolved_symbols.add(symbol) + + return resolved_objs, list(unresolved_symbols) + + +def expand_ld_script(fn): + # This function parses a subset of ld scripts + # Typically these are just groups of static lib references + group_pattern = re.compile(r"GROUP\s*\(\s*([^\)]+)\s*\)", re.MULTILINE) + output_format_pattern = re.compile(r"OUTPUT_FORMAT\s*\(\s*([^\)]+)\s*\)", re.MULTILINE) + comment_pattern = re.compile(r"/\*.*?\*/", re.MULTILINE | re.DOTALL) + + with open(fn, "r") as f: + content = f.read() + content = comment_pattern.sub("", content).strip() + + # Ensure no unrecognized instructions + leftovers = content + for pattern in (group_pattern, output_format_pattern): + leftovers = pattern.sub("", leftovers) + if leftovers.strip(): + raise ValueError("Invalid instruction found in the ld script:" + leftovers) + + # Extract files from GROUP instructions + files = [] + for match in group_pattern.findall(content): + files.extend([file.strip() for file in re.split(r"[,\s]+", match) if file.strip()]) + + return files + + +def load_archive(fn): + ar_header = b"!<arch>\012" + with open(fn, "rb") as f: + is_ar_file = f.read(len(ar_header)) == ar_header + if is_ar_file: + return [CachedArFile(fn)] + else: + return [CachedArFile(item) for item in expand_ld_script(fn)] diff --git a/tools/ci.sh b/tools/ci.sh index 682c3ae50..5d57175c8 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -155,12 +155,15 @@ PYTHON_VER=$(python --version | cut -d' ' -f2) export IDF_CCACHE_ENABLE=1 function ci_esp32_idf_setup { - pip3 install pyelftools git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git # doing a treeless clone isn't quite as good as --shallow-submodules, but it # is smaller than full clones and works when the submodule commit isn't a head. git -C esp-idf submodule update --init --recursive --filter=tree:0 ./esp-idf/install.sh + # Install additional packages for mpy_ld into the IDF env + source esp-idf/export.sh + pip3 install pyelftools + pip3 install ar } function ci_esp32_build_common { @@ -287,6 +290,7 @@ function ci_qemu_setup_arm { sudo apt-get update sudo apt-get install qemu-system sudo pip3 install pyelftools + sudo pip3 install ar qemu-system-arm --version } @@ -295,6 +299,7 @@ function ci_qemu_setup_rv32 { sudo apt-get update sudo apt-get install qemu-system sudo pip3 install pyelftools + sudo pip3 install ar qemu-system-riscv32 --version } @@ -385,6 +390,7 @@ function ci_samd_build { function ci_stm32_setup { ci_gcc_arm_setup pip3 install pyelftools + pip3 install ar pip3 install pyhy } @@ -503,18 +509,40 @@ function ci_native_mpy_modules_build { else arch=$1 fi - for natmod in features1 features3 features4 deflate framebuf heapq random re + for natmod in features1 features3 features4 heapq re do + make -C examples/natmod/$natmod clean make -C examples/natmod/$natmod ARCH=$arch done - # btree requires thread local storage support on rv32imc. - if [ $arch != rv32imc ]; then - make -C examples/natmod/btree ARCH=$arch + + # deflate, framebuf, and random currently cannot build on xtensa due to + # some symbols that have been removed from the compiler's runtime, in + # favour of being provided from ROM. + if [ $arch != "xtensa" ]; then + for natmod in deflate framebuf random + do + make -C examples/natmod/$natmod clean + make -C examples/natmod/$natmod ARCH=$arch + done fi - # features2 requires soft-float on armv7m and rv32imc. - if [ $arch != rv32imc ] && [ $arch != armv7m ]; then + + # features2 requires soft-float on armv7m, rv32imc, and xtensa. On armv6m + # the compiler generates absolute relocations in the object file + # referencing soft-float functions, which is not supported at the moment. + make -C examples/natmod/features2 clean + if [ $arch = "rv32imc" ] || [ $arch = "armv7m" ] || [ $arch = "xtensa" ]; then + make -C examples/natmod/features2 ARCH=$arch MICROPY_FLOAT_IMPL=float + elif [ $arch != "armv6m" ]; then make -C examples/natmod/features2 ARCH=$arch fi + + # btree requires thread local storage support on rv32imc, whilst on xtensa + # it relies on symbols that are provided from ROM but not exposed to + # natmods at the moment. + if [ $arch != "rv32imc" ] && [ $arch != "xtensa" ]; then + make -C examples/natmod/btree clean + make -C examples/natmod/btree ARCH=$arch + fi } function ci_native_mpy_modules_32bit_build { @@ -550,6 +578,7 @@ function ci_unix_standard_v2_run_tests { function ci_unix_coverage_setup { sudo pip3 install setuptools sudo pip3 install pyelftools + sudo pip3 install ar gcc --version python3 --version } @@ -598,6 +627,7 @@ function ci_unix_32bit_setup { sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 python2.7 sudo pip3 install setuptools sudo pip3 install pyelftools + sudo pip3 install ar gcc --version python2.7 --version python3 --version diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 54295208f..44a76bdee 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -30,6 +30,7 @@ Link .o files to .mpy import sys, os, struct, re from elftools.elf import elffile +import ar_util sys.path.append(os.path.dirname(__file__) + "/../py") import makeqstrdata as qstrutil @@ -664,7 +665,7 @@ def do_relocation_text(env, text_addr, r): R_XTENSA_PDIFF32, R_XTENSA_ASM_EXPAND, ): - if s.section.name.startswith(".text"): + if not hasattr(s, "section") or s.section.name.startswith(".text"): # it looks like R_XTENSA_[P]DIFF32 into .text is already correctly relocated, # and expand relaxations cannot occur in non-executable sections. return @@ -1075,59 +1076,59 @@ def process_riscv32_relocation(env, text_addr, r): return addr, value -def load_object_file(env, felf): - with open(felf, "rb") as f: - elf = elffile.ELFFile(f) - env.check_arch(elf["e_machine"]) - - # Get symbol table - symtab = list(elf.get_section_by_name(".symtab").iter_symbols()) - - # Load needed sections from ELF file - sections_shndx = {} # maps elf shndx to Section object - for idx, s in enumerate(elf.iter_sections()): - if s.header.sh_type in ("SHT_PROGBITS", "SHT_NOBITS"): - if s.data_size == 0: - # Ignore empty sections - pass - elif s.name.startswith((".literal", ".text", ".rodata", ".data.rel.ro", ".bss")): - sec = Section.from_elfsec(s, felf) - sections_shndx[idx] = sec - if s.name.startswith(".literal"): - env.literal_sections.append(sec) - else: - env.sections.append(sec) - elif s.name.startswith(".data"): - raise LinkError("{}: {} non-empty".format(felf, s.name)) +def load_object_file(env, f, felf): + elf = elffile.ELFFile(f) + env.check_arch(elf["e_machine"]) + + # Get symbol table + symtab = list(elf.get_section_by_name(".symtab").iter_symbols()) + + # Load needed sections from ELF file + sections_shndx = {} # maps elf shndx to Section object + for idx, s in enumerate(elf.iter_sections()): + if s.header.sh_type in ("SHT_PROGBITS", "SHT_NOBITS"): + if s.data_size == 0: + # Ignore empty sections + pass + elif s.name.startswith((".literal", ".text", ".rodata", ".data.rel.ro", ".bss")): + sec = Section.from_elfsec(s, felf) + sections_shndx[idx] = sec + if s.name.startswith(".literal"): + env.literal_sections.append(sec) else: - # Ignore section - pass - elif s.header.sh_type in ("SHT_REL", "SHT_RELA"): - shndx = s.header.sh_info - if shndx in sections_shndx: - sec = sections_shndx[shndx] - sec.reloc_name = s.name - sec.reloc = list(s.iter_relocations()) - for r in sec.reloc: - r.sym = symtab[r["r_info_sym"]] - - # Link symbols to their sections, and update known and unresolved symbols - for sym in symtab: - sym.filename = felf - shndx = sym.entry["st_shndx"] + env.sections.append(sec) + elif s.name.startswith(".data"): + raise LinkError("{}: {} non-empty".format(felf, s.name)) + else: + # Ignore section + pass + elif s.header.sh_type in ("SHT_REL", "SHT_RELA"): + shndx = s.header.sh_info if shndx in sections_shndx: - # Symbol with associated section - sym.section = sections_shndx[shndx] - if sym["st_info"]["bind"] in ("STB_GLOBAL", "STB_WEAK"): - # Defined global symbol - if sym.name in env.known_syms and not sym.name.startswith( - "__x86.get_pc_thunk." - ): - raise LinkError("duplicate symbol: {}".format(sym.name)) - env.known_syms[sym.name] = sym - elif sym.entry["st_shndx"] == "SHN_UNDEF" and sym["st_info"]["bind"] == "STB_GLOBAL": - # Undefined global symbol, needs resolving - env.unresolved_syms.append(sym) + sec = sections_shndx[shndx] + sec.reloc_name = s.name + sec.reloc = list(s.iter_relocations()) + for r in sec.reloc: + r.sym = symtab[r["r_info_sym"]] + + # Link symbols to their sections, and update known and unresolved symbols + dup_errors = [] + for sym in symtab: + sym.filename = felf + shndx = sym.entry["st_shndx"] + if shndx in sections_shndx: + # Symbol with associated section + sym.section = sections_shndx[shndx] + if sym["st_info"]["bind"] in ("STB_GLOBAL", "STB_WEAK"): + # Defined global symbol + if sym.name in env.known_syms and not sym.name.startswith("__x86.get_pc_thunk."): + dup_errors.append("duplicate symbol: {}".format(sym.name)) + env.known_syms[sym.name] = sym + elif sym.entry["st_shndx"] == "SHN_UNDEF" and sym["st_info"]["bind"] == "STB_GLOBAL": + # Undefined global symbol, needs resolving + env.unresolved_syms.append(sym) + if len(dup_errors): + raise LinkError("\n".join(dup_errors)) def link_objects(env, native_qstr_vals_len): @@ -1188,6 +1189,8 @@ def link_objects(env, native_qstr_vals_len): ] ) } + + undef_errors = [] for sym in env.unresolved_syms: assert sym["st_value"] == 0 if sym.name == "_GLOBAL_OFFSET_TABLE_": @@ -1205,7 +1208,10 @@ def link_objects(env, native_qstr_vals_len): sym.section = mp_fun_table_sec sym.mp_fun_table_offset = fun_table[sym.name] else: - raise LinkError("{}: undefined symbol: {}".format(sym.filename, sym.name)) + undef_errors.append("{}: undefined symbol: {}".format(sym.filename, sym.name)) + + if len(undef_errors): + raise LinkError("\n".join(undef_errors)) # Align sections, assign their addresses, and create full_text env.full_text = bytearray(env.arch.asm_jump(8)) # dummy, to be filled in later @@ -1446,8 +1452,27 @@ def do_link(args): log(LOG_LEVEL_2, "qstr vals: " + ", ".join(native_qstr_vals)) env = LinkEnv(args.arch) try: - for file in args.files: - load_object_file(env, file) + # Load object files + for fn in args.files: + with open(fn, "rb") as f: + load_object_file(env, f, fn) + + if args.libs: + # Load archive info + archives = [] + for item in args.libs: + archives.extend(ar_util.load_archive(item)) + # List symbols to look for + syms = set(sym.name for sym in env.unresolved_syms) + # Resolve symbols from libs + lib_objs, _ = ar_util.resolve(archives, syms) + # Load extra object files from libs + for ar, obj in lib_objs: + obj_name = ar.fn + ":" + obj + log(LOG_LEVEL_2, "using " + obj_name) + with ar.open(obj) as f: + load_object_file(env, f, obj_name) + link_objects(env, len(native_qstr_vals)) build_mpy(env, env.find_addr("mpy_init"), args.output, native_qstr_vals) except LinkError as er: @@ -1458,7 +1483,7 @@ def do_link(args): def main(): import argparse - cmd_parser = argparse.ArgumentParser(description="Run scripts on the pyboard.") + cmd_parser = argparse.ArgumentParser(description="Link native object files into a MPY bundle.") cmd_parser.add_argument( "--verbose", "-v", action="count", default=1, help="increase verbosity" ) @@ -1466,6 +1491,9 @@ def main(): cmd_parser.add_argument("--preprocess", action="store_true", help="preprocess source files") cmd_parser.add_argument("--qstrs", default=None, help="file defining additional qstrs") cmd_parser.add_argument( + "--libs", "-l", dest="libs", action="append", help="static .a libraries to link" + ) + cmd_parser.add_argument( "--output", "-o", default=None, help="output .mpy file (default to input with .o->.mpy)" ) cmd_parser.add_argument("files", nargs="+", help="input files") |