Skip to content

Commit ceabe58

Browse files
vshymanskyyvshymanskyy
committed
natmod: Allow linking static libraries.
Signed-off-by: Volodymyr Shymanskyy <vshymanskyi@gmail.com>
1 parent f1bdac3 commit ceabe58

File tree

3 files changed

+256
-7
lines changed

3 files changed

+256
-7
lines changed

‎py/dynruntime.mk

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@ endif
106106
MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]')
107107
CFLAGS += -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER)
108108

109+
ifeq ($(MPY_LINK_RUNTIME),1)
110+
MPY_LD_FLAGS += -l$(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-libgcc-file-name))
111+
MPY_LD_FLAGS += -l$(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-file-name=libm.a))
112+
endif
113+
109114
CFLAGS += $(CFLAGS_EXTRA)
110115

111116
################################################################################
@@ -147,7 +152,7 @@ $(BUILD)/%.mpy: %.py
147152
# Build native .mpy from object files
148153
$(BUILD)/$(MOD).native.mpy: $(SRC_O)
149154
$(ECHO) "LINK $<"
150-
$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^
155+
$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) $(MPY_LD_FLAGS) -o $@ $^
151156

152157
# Build final .mpy from all intermediate .mpy files
153158
$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)

‎tools/ar_util.py

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
#!/usr/bin/env python3
2+
#
3+
# This file is part of the MicroPython project, http://micropython.org/
4+
#
5+
# The MIT License (MIT)
6+
#
7+
# Copyright (c) 2024 Volodymyr Shymanskyy
8+
#
9+
# Permission is hereby granted, free of charge, to any person obtaining a copy
10+
# of this software and associated documentation files (the "Software"), to deal
11+
# in the Software without restriction, including without limitation the rights
12+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13+
# copies of the Software, and to permit persons to whom the Software is
14+
# furnished to do so, subject to the following conditions:
15+
#
16+
# The above copyright notice and this permission notice shall be included in
17+
# all copies or substantial portions of the Software.
18+
#
19+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25+
# THE SOFTWARE.
26+
27+
import os
28+
import hashlib
29+
import functools
30+
import pickle
31+
32+
from elftools.elf import elffile
33+
from collections import defaultdict
34+
35+
try:
36+
from ar import Archive
37+
except:
38+
Archive = None
39+
40+
41+
class PickleCache:
42+
def __init__(self, path=".cache", prefix=""):
43+
self.path = path
44+
self._get_fn = lambda key: os.path.join(path, prefix + key[:24])
45+
46+
def store(self, key, data):
47+
os.makedirs(self.path, exist_ok=True)
48+
with open(self._get_fn(key), "wb") as f:
49+
pickle.dump(data, f)
50+
51+
def load(self, key):
52+
with open(self._get_fn(key), "rb") as f:
53+
return pickle.load(f)
54+
55+
56+
def cached(key, cache):
57+
def decorator(func):
58+
@functools.wraps(func)
59+
def wrapper(*args, **kwargs):
60+
cache_key = key(*args, **kwargs)
61+
try:
62+
d = cache.load(cache_key)
63+
if d["key"] != cache_key:
64+
raise Exception("Cache key mismatch")
65+
return d["data"]
66+
except Exception:
67+
res = func(*args, **kwargs)
68+
try:
69+
cache.store(
70+
cache_key,
71+
{
72+
"key": cache_key,
73+
"data": res,
74+
},
75+
)
76+
except Exception:
77+
pass
78+
return res
79+
80+
return wrapper
81+
82+
return decorator
83+
84+
85+
class CachedArFile:
86+
def __init__(self, fn):
87+
if not Archive:
88+
raise RuntimeError("Please run 'pip install ar' to link .a files")
89+
self.fn = fn
90+
self._archive = Archive(open(fn, "rb"))
91+
info = self.load_symbols()
92+
self.objs = info["objs"]
93+
self.symbols = info["symbols"]
94+
95+
def open(self, obj):
96+
return self._archive.open(obj, "rb")
97+
98+
def _cache_key(self):
99+
with open(self.fn, "rb") as f:
100+
digest = hashlib.file_digest(f, "sha3_256")
101+
# Change this salt if the cache data format changes
102+
digest.update(bytes.fromhex("45155db4bc868fa78cb99c3448b2bf2b"))
103+
return digest.hexdigest()
104+
105+
@cached(key=_cache_key, cache=PickleCache(prefix="ar_"))
106+
def load_symbols(self):
107+
print("Loading", self.fn)
108+
objs = defaultdict(lambda: {"def": set(), "undef": set(), "weak": set()})
109+
symbols = {}
110+
for entry in self._archive:
111+
obj_name = entry.name
112+
elf = elffile.ELFFile(self.open(obj_name))
113+
symtab = elf.get_section_by_name(".symtab")
114+
if not symtab:
115+
continue
116+
117+
obj = objs[obj_name]
118+
119+
for symbol in symtab.iter_symbols():
120+
sym_name = symbol.name
121+
sym_bind = symbol["st_info"]["bind"]
122+
123+
if sym_bind in ("STB_GLOBAL", "STB_WEAK"):
124+
if symbol.entry["st_shndx"] != "SHN_UNDEF":
125+
obj["def"].add(sym_name)
126+
symbols[sym_name] = obj_name
127+
else:
128+
obj["undef"].add(sym_name)
129+
130+
if sym_bind == "STB_WEAK":
131+
obj["weak"].add(sym_name)
132+
133+
return {"objs": dict(objs), "symbols": symbols}
134+
135+
136+
def resolve(archives, symbols):
137+
resolved_objs = [] # Object files needed to resolve symbols
138+
unresolved_symbols = set()
139+
provided_symbols = {} # Which symbol is provided by which object
140+
symbol_stack = list(symbols)
141+
142+
# A helper function to handle symbol resolution from a particular object
143+
def add_obj(archive, symbol):
144+
obj_name = archive.symbols[symbol]
145+
obj_info = archive.objs[obj_name]
146+
147+
obj_tuple = (archive, obj_name)
148+
if obj_tuple in resolved_objs:
149+
return # Already processed this object
150+
151+
resolved_objs.append(obj_tuple)
152+
153+
# Add the symbols this object defines
154+
for defined_symbol in obj_info["def"]:
155+
if defined_symbol in provided_symbols:
156+
raise RuntimeError(f"Multiple definitions for {defined_symbol}")
157+
provided_symbols[defined_symbol] = obj_name # TODO: save if week
158+
159+
# Recursively add undefined symbols from this object
160+
for undef_symbol in obj_info["undef"]:
161+
if undef_symbol in obj_info["weak"]:
162+
print(f"Skippping weak dependency: {undef_symbol}")
163+
continue
164+
if undef_symbol not in provided_symbols:
165+
symbol_stack.append(undef_symbol) # Add undefined symbol to resolve
166+
167+
while symbol_stack:
168+
symbol = symbol_stack.pop(0)
169+
170+
if symbol in provided_symbols:
171+
continue # Symbol is already resolved
172+
173+
found = False
174+
for archive in archives:
175+
if symbol in archive.symbols:
176+
add_obj(archive, symbol)
177+
found = True
178+
break
179+
180+
if not found:
181+
unresolved_symbols.add(symbol)
182+
183+
return resolved_objs, list(unresolved_symbols)
184+
185+
186+
if __name__ == "__main__":
187+
import argparse
188+
from pathlib import Path
189+
190+
parser = argparse.ArgumentParser(description="Resolve symbols from AR files.")
191+
parser.add_argument("--arch", help="Target architecture to extract objects to")
192+
parser.add_argument("-v", "--verbose", help="Verbose logging", action="store_true")
193+
parser.add_argument("inputs", nargs="+", help="AR files and symbols to resolve")
194+
args = parser.parse_args()
195+
196+
# Separate files and symbols
197+
archives = [CachedArFile(item) for item in args.inputs if item.endswith(".a")]
198+
symbols = [item for item in args.inputs if not item.endswith(".a")]
199+
200+
result, unresolved = resolve(archives, symbols)
201+
202+
if unresolved:
203+
raise RuntimeError("Unresolved symbols: " + ", ".join(unresolved))
204+
205+
# Extract files
206+
for ar, obj in result:
207+
print(Path(ar.fn).stem + "/" + obj)
208+
if args.verbose:
209+
print(" def:", ", ".join(ar.objs[obj]["def"]))
210+
print(" req:", ", ".join(ar.objs[obj]["undef"]))
211+
weak = ar.objs[obj]["weak"]
212+
if weak:
213+
print(" weak:", ", ".join(weak))
214+
if args.arch:
215+
content = ar.open(obj).read()
216+
with open(f"runtime/libgcc-{args.arch}/{obj}", "wb") as output:
217+
output.write(content)

‎tools/mpy_ld.py

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
import sys, os, struct, re
3232
from elftools.elf import elffile
33+
import ar_util
3334

3435
sys.path.append(os.path.dirname(__file__) + "/../py")
3536
import makeqstrdata as qstrutil
@@ -668,8 +669,8 @@ def do_relocation_data(env, text_addr, r):
668669
assert 0, r_info_type
669670

670671

671-
def load_object_file(env, felf):
672-
with open(felf, "rb") as f:
672+
def load_object_file(env, f, felf):
673+
if 1: # Temporary, to preserve indent
673674
elf = elffile.ELFFile(f)
674675
env.check_arch(elf["e_machine"])
675676

@@ -705,6 +706,7 @@ def load_object_file(env, felf):
705706
r.sym = symtab[r["r_info_sym"]]
706707

707708
# Link symbols to their sections, and update known and unresolved symbols
709+
dup_errors = []
708710
for sym in symtab:
709711
sym.filename = felf
710712
shndx = sym.entry["st_shndx"]
@@ -716,11 +718,13 @@ def load_object_file(env, felf):
716718
if sym.name in env.known_syms and not sym.name.startswith(
717719
"__x86.get_pc_thunk."
718720
):
719-
raise LinkError("duplicate symbol: {}".format(sym.name))
721+
dup_errors.append("duplicate symbol: {}".format(sym.name))
720722
env.known_syms[sym.name] = sym
721723
elif sym.entry["st_shndx"] == "SHN_UNDEF" and sym["st_info"]["bind"] == "STB_GLOBAL":
722724
# Undefined global symbol, needs resolving
723725
env.unresolved_syms.append(sym)
726+
if len(dup_errors):
727+
raise LinkError("\n".join(dup_errors))
724728

725729

726730
def link_objects(env, native_qstr_vals_len):
@@ -781,6 +785,8 @@ def link_objects(env, native_qstr_vals_len):
781785
]
782786
)
783787
}
788+
789+
undef_errors = []
784790
for sym in env.unresolved_syms:
785791
assert sym["st_value"] == 0
786792
if sym.name == "_GLOBAL_OFFSET_TABLE_":
@@ -798,7 +804,10 @@ def link_objects(env, native_qstr_vals_len):
798804
sym.section = mp_fun_table_sec
799805
sym.mp_fun_table_offset = fun_table[sym.name]
800806
else:
801-
raise LinkError("{}: undefined symbol: {}".format(sym.filename, sym.name))
807+
undef_errors.append("{}: undefined symbol: {}".format(sym.filename, sym.name))
808+
809+
if len(undef_errors):
810+
raise LinkError("\n".join(undef_errors))
802811

803812
# Align sections, assign their addresses, and create full_text
804813
env.full_text = bytearray(env.arch.asm_jump(8)) # dummy, to be filled in later
@@ -1039,8 +1048,25 @@ def do_link(args):
10391048
log(LOG_LEVEL_2, "qstr vals: " + ", ".join(native_qstr_vals))
10401049
env = LinkEnv(args.arch)
10411050
try:
1042-
for file in args.files:
1043-
load_object_file(env, file)
1051+
# Load object files
1052+
for fn in args.files:
1053+
with open(fn, "rb") as f:
1054+
load_object_file(env, f, fn)
1055+
1056+
if args.libs:
1057+
# Load archive info
1058+
archives = [ar_util.CachedArFile(item) for item in args.libs]
1059+
# List symbols to look for
1060+
syms = set(sym.name for sym in env.unresolved_syms)
1061+
# Resolve symbols from libs
1062+
lib_objs, _ = ar_util.resolve(archives, syms)
1063+
# Load extra object files from libs
1064+
for ar, obj in lib_objs:
1065+
obj_name = ar.fn + ":" + obj
1066+
log(LOG_LEVEL_2, "using " + obj_name)
1067+
with ar.open(obj) as f:
1068+
load_object_file(env, f, obj_name)
1069+
10441070
link_objects(env, len(native_qstr_vals))
10451071
build_mpy(env, env.find_addr("mpy_init"), args.output, native_qstr_vals)
10461072
except LinkError as er:
@@ -1058,6 +1084,7 @@ def main():
10581084
cmd_parser.add_argument("--arch", default="x64", help="architecture")
10591085
cmd_parser.add_argument("--preprocess", action="store_true", help="preprocess source files")
10601086
cmd_parser.add_argument("--qstrs", default=None, help="file defining additional qstrs")
1087+
cmd_parser.add_argument("-l", dest="libs", action="append", help="Static .a libraries to link")
10611088
cmd_parser.add_argument(
10621089
"--output", "-o", default=None, help="output .mpy file (default to input with .o->.mpy)"
10631090
)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy