Skip to content

Commit 61d74fd

Browse files
pfalconpfalcon
committed
tools, unix: Replace upip tarball with just source files.
To make its inclusion as frozen modules in multiple ports less magic. Ports are just expected to symlink 2 files into their scripts/modules subdirs. Unix port updated to use this and in general follow frozen modules setup tested and tried on baremetal ports, where there's "scripts" predefined dir (overridable with FROZEN_DIR make var), and a user just drops Python files there.
1 parent bc4441a commit 61d74fd

File tree

4 files changed

+385
-28
lines changed

4 files changed

+385
-28
lines changed

‎tools/micropython-upip-1.1.4.tar.gz

-3.99 KB
Binary file not shown.

‎tools/upip.py

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
import sys
2+
import gc
3+
import uos as os
4+
import uerrno as errno
5+
import ujson as json
6+
import uzlib
7+
import upip_utarfile as tarfile
8+
gc.collect()
9+
10+
11+
debug = False
12+
install_path = None
13+
cleanup_files = []
14+
gzdict_sz = 16 + 15
15+
16+
file_buf = bytearray(512)
17+
18+
class NotFoundError(Exception):
19+
pass
20+
21+
def op_split(path):
22+
if path == "":
23+
return ("", "")
24+
r = path.rsplit("/", 1)
25+
if len(r) == 1:
26+
return ("", path)
27+
head = r[0]
28+
if not head:
29+
head = "/"
30+
return (head, r[1])
31+
32+
def op_basename(path):
33+
return op_split(path)[1]
34+
35+
# Expects *file* name
36+
def _makedirs(name, mode=0o777):
37+
ret = False
38+
s = ""
39+
comps = name.rstrip("/").split("/")[:-1]
40+
if comps[0] == "":
41+
s = "/"
42+
for c in comps:
43+
if s and s[-1] != "/":
44+
s += "/"
45+
s += c
46+
try:
47+
os.mkdir(s)
48+
ret = True
49+
except OSError as e:
50+
if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
51+
raise
52+
ret = False
53+
return ret
54+
55+
56+
def save_file(fname, subf):
57+
global file_buf
58+
with open(fname, "wb") as outf:
59+
while True:
60+
sz = subf.readinto(file_buf)
61+
if not sz:
62+
break
63+
outf.write(file_buf, sz)
64+
65+
def install_tar(f, prefix):
66+
meta = {}
67+
for info in f:
68+
#print(info)
69+
fname = info.name
70+
try:
71+
fname = fname[fname.index("/") + 1:]
72+
except ValueError:
73+
fname = ""
74+
75+
save = True
76+
for p in ("setup.", "PKG-INFO", "README"):
77+
#print(fname, p)
78+
if fname.startswith(p) or ".egg-info" in fname:
79+
if fname.endswith("/requires.txt"):
80+
meta["deps"] = f.extractfile(info).read()
81+
save = False
82+
if debug:
83+
print("Skipping", fname)
84+
break
85+
86+
if save:
87+
outfname = prefix + fname
88+
if info.type != tarfile.DIRTYPE:
89+
if debug:
90+
print("Extracting " + outfname)
91+
_makedirs(outfname)
92+
subf = f.extractfile(info)
93+
save_file(outfname, subf)
94+
return meta
95+
96+
def expandhome(s):
97+
if "~/" in s:
98+
h = os.getenv("HOME")
99+
s = s.replace("~/", h + "/")
100+
return s
101+
102+
import ussl
103+
import usocket
104+
warn_ussl = True
105+
def url_open(url):
106+
global warn_ussl
107+
proto, _, host, urlpath = url.split('/', 3)
108+
ai = usocket.getaddrinfo(host, 443)
109+
#print("Address infos:", ai)
110+
addr = ai[0][4]
111+
112+
s = usocket.socket(ai[0][0])
113+
#print("Connect address:", addr)
114+
s.connect(addr)
115+
116+
if proto == "https:":
117+
s = ussl.wrap_socket(s)
118+
if warn_ussl:
119+
print("Warning: %s SSL certificate is not validated" % host)
120+
warn_ussl = False
121+
122+
# MicroPython rawsocket module supports file interface directly
123+
s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
124+
l = s.readline()
125+
protover, status, msg = l.split(None, 2)
126+
if status != b"200":
127+
if status == b"404":
128+
print("Package not found")
129+
raise ValueError(status)
130+
while 1:
131+
l = s.readline()
132+
if not l:
133+
raise ValueError("Unexpected EOF")
134+
if l == b'\r\n':
135+
break
136+
137+
return s
138+
139+
140+
def get_pkg_metadata(name):
141+
f = url_open("https://pypi.python.org/pypi/%s/json" % name)
142+
s = f.read()
143+
f.close()
144+
return json.loads(s)
145+
146+
147+
def fatal(msg):
148+
print(msg)
149+
sys.exit(1)
150+
151+
def install_pkg(pkg_spec, install_path):
152+
data = get_pkg_metadata(pkg_spec)
153+
154+
latest_ver = data["info"]["version"]
155+
packages = data["releases"][latest_ver]
156+
del data
157+
gc.collect()
158+
assert len(packages) == 1
159+
package_url = packages[0]["url"]
160+
print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
161+
package_fname = op_basename(package_url)
162+
f1 = url_open(package_url)
163+
f2 = uzlib.DecompIO(f1, gzdict_sz)
164+
f3 = tarfile.TarFile(fileobj=f2)
165+
meta = install_tar(f3, install_path)
166+
f1.close()
167+
del f3
168+
del f2
169+
gc.collect()
170+
return meta
171+
172+
def install(to_install, install_path=None):
173+
# Calculate gzip dictionary size to use
174+
global gzdict_sz
175+
sz = gc.mem_free() + gc.mem_alloc()
176+
if sz <= 65536:
177+
gzdict_sz = 16 + 12
178+
179+
if install_path is None:
180+
install_path = get_install_path()
181+
if install_path[-1] != "/":
182+
install_path += "/"
183+
if not isinstance(to_install, list):
184+
to_install = [to_install]
185+
print("Installing to: " + install_path)
186+
# sets would be perfect here, but don't depend on them
187+
installed = []
188+
try:
189+
while to_install:
190+
if debug:
191+
print("Queue:", to_install)
192+
pkg_spec = to_install.pop(0)
193+
if pkg_spec in installed:
194+
continue
195+
meta = install_pkg(pkg_spec, install_path)
196+
installed.append(pkg_spec)
197+
if debug:
198+
print(meta)
199+
deps = meta.get("deps", "").rstrip()
200+
if deps:
201+
deps = deps.decode("utf-8").split("\n")
202+
to_install.extend(deps)
203+
except NotFoundError:
204+
print("Error: cannot find '%s' package (or server error), packages may be partially installed" \
205+
% pkg_spec, file=sys.stderr)
206+
207+
def get_install_path():
208+
global install_path
209+
if install_path is None:
210+
# sys.path[0] is current module's path
211+
install_path = sys.path[1]
212+
install_path = expandhome(install_path)
213+
return install_path
214+
215+
def cleanup():
216+
for fname in cleanup_files:
217+
try:
218+
os.unlink(fname)
219+
except OSError:
220+
print("Warning: Cannot delete " + fname)
221+
222+
def help():
223+
print("""\
224+
upip - Simple PyPI package manager for MicroPython
225+
Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt>
226+
import upip; upip.install(package_or_list, [<path>])
227+
228+
If <path> is not given, packages will be installed into sys.path[1]
229+
(can be set from MICROPYPATH environment variable, if current system
230+
supports that).""")
231+
print("Current value of sys.path[1]:", sys.path[1])
232+
print("""\
233+
234+
Note: only MicroPython packages (usually, named micropython-*) are supported
235+
for installation, upip does not support arbitrary code in setup.py.
236+
""")
237+
238+
def main():
239+
global debug
240+
global install_path
241+
install_path = None
242+
243+
if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
244+
help()
245+
return
246+
247+
if sys.argv[1] != "install":
248+
fatal("Only 'install' command supported")
249+
250+
to_install = []
251+
252+
i = 2
253+
while i < len(sys.argv) and sys.argv[i][0] == "-":
254+
opt = sys.argv[i]
255+
i += 1
256+
if opt == "-h" or opt == "--help":
257+
help()
258+
return
259+
elif opt == "-p":
260+
install_path = sys.argv[i]
261+
i += 1
262+
elif opt == "-r":
263+
list_file = sys.argv[i]
264+
i += 1
265+
with open(list_file) as f:
266+
while True:
267+
l = f.readline()
268+
if not l:
269+
break
270+
to_install.append(l.rstrip())
271+
elif opt == "--debug":
272+
debug = True
273+
else:
274+
fatal("Unknown/unsupported option: " + opt)
275+
276+
to_install.extend(sys.argv[i:])
277+
if not to_install:
278+
help()
279+
return
280+
281+
install(to_install)
282+
283+
if not debug:
284+
cleanup()
285+
286+
287+
if __name__ == "__main__":
288+
main()

‎tools/upip_utarfile.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import uctypes
2+
3+
# http://www.gnu.org/software/tar/manual/html_node/Standard.html
4+
TAR_HEADER = {
5+
"name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100),
6+
"size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12),
7+
}
8+
9+
DIRTYPE = "dir"
10+
REGTYPE = "file"
11+
12+
def roundup(val, align):
13+
return (val + align - 1) & ~(align - 1)
14+
15+
class FileSection:
16+
17+
def __init__(self, f, content_len, aligned_len):
18+
self.f = f
19+
self.content_len = content_len
20+
self.align = aligned_len - content_len
21+
22+
def read(self, sz=65536):
23+
if self.content_len == 0:
24+
return b""
25+
if sz > self.content_len:
26+
sz = self.content_len
27+
data = self.f.read(sz)
28+
sz = len(data)
29+
self.content_len -= sz
30+
return data
31+
32+
def readinto(self, buf):
33+
if self.content_len == 0:
34+
return 0
35+
if len(buf) > self.content_len:
36+
buf = memoryview(buf)[:self.content_len]
37+
sz = self.f.readinto(buf)
38+
self.content_len -= sz
39+
return sz
40+
41+
def skip(self):
42+
sz = self.content_len + self.align
43+
if sz:
44+
buf = bytearray(16)
45+
while sz:
46+
s = min(sz, 16)
47+
self.f.readinto(buf, s)
48+
sz -= s
49+
50+
class TarInfo:
51+
52+
def __str__(self):
53+
return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size)
54+
55+
class TarFile:
56+
57+
def __init__(self, name=None, fileobj=None):
58+
if fileobj:
59+
self.f = fileobj
60+
else:
61+
self.f = open(name, "rb")
62+
self.subf = None
63+
64+
def next(self):
65+
if self.subf:
66+
self.subf.skip()
67+
buf = self.f.read(512)
68+
if not buf:
69+
return None
70+
71+
h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN)
72+
73+
# Empty block means end of archive
74+
if h.name[0] == 0:
75+
return None
76+
77+
d = TarInfo()
78+
d.name = str(h.name, "utf-8").rstrip()
79+
d.size = int(bytes(h.size).rstrip(), 8)
80+
d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"]
81+
self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512))
82+
return d
83+
84+
def __iter__(self):
85+
return self
86+
87+
def __next__(self):
88+
v = self.next()
89+
if v is None:
90+
raise StopIteration
91+
return v
92+
93+
def extractfile(self, tarinfo):
94+
return tarinfo.subf

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy