diff --git a/wasm/lib/Lib/_microdistlib.py b/wasm/lib/Lib/_microdistlib.py new file mode 100644 index 000000000..77e3afad6 --- /dev/null +++ b/wasm/lib/Lib/_microdistlib.py @@ -0,0 +1,129 @@ +# taken from https://bitbucket.org/pypa/distlib/src/master/distlib/util.py + +from types import SimpleNamespace as Container +import re + +IDENTIFIER = re.compile(r'^([\w\.-]+)\s*') +VERSION_IDENTIFIER = re.compile(r'^([\w\.*+-]+)\s*') +COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*') +NON_SPACE = re.compile(r'(\S+)\s*') + +def parse_requirement(req): + """ + Parse a requirement passed in as a string. Return a Container + whose attributes contain the various parts of the requirement. + """ + remaining = req.strip() + if not remaining or remaining.startswith('#'): + return None + m = IDENTIFIER.match(remaining) + if not m: + raise SyntaxError('name expected: %s' % remaining) + distname = m.groups()[0] + remaining = remaining[m.end():] + extras = mark_expr = versions = uri = None + if remaining and remaining[0] == '[': + i = remaining.find(']', 1) + if i < 0: + raise SyntaxError('unterminated extra: %s' % remaining) + s = remaining[1:i] + remaining = remaining[i + 1:].lstrip() + extras = [] + while s: + m = IDENTIFIER.match(s) + if not m: + raise SyntaxError('malformed extra: %s' % s) + extras.append(m.groups()[0]) + s = s[m.end():] + if not s: + break + if s[0] != ',': + raise SyntaxError('comma expected in extras: %s' % s) + s = s[1:].lstrip() + if not extras: + extras = None + if remaining: + if remaining[0] == '@': + # it's a URI + remaining = remaining[1:].lstrip() + m = NON_SPACE.match(remaining) + if not m: + raise SyntaxError('invalid URI: %s' % remaining) + uri = m.groups()[0] + t = urlparse(uri) + # there are issues with Python and URL parsing, so this test + # is a bit crude. See bpo-20271, bpo-23505. Python doesn't + # always parse invalid URLs correctly - it should raise + # exceptions for malformed URLs + if not (t.scheme and t.netloc): + raise SyntaxError('Invalid URL: %s' % uri) + remaining = remaining[m.end():].lstrip() + else: + + def get_versions(ver_remaining): + """ + Return a list of operator, version tuples if any are + specified, else None. + """ + m = COMPARE_OP.match(ver_remaining) + versions = None + if m: + versions = [] + while True: + op = m.groups()[0] + ver_remaining = ver_remaining[m.end():] + m = VERSION_IDENTIFIER.match(ver_remaining) + if not m: + raise SyntaxError('invalid version: %s' % ver_remaining) + v = m.groups()[0] + versions.append((op, v)) + ver_remaining = ver_remaining[m.end():] + if not ver_remaining or ver_remaining[0] != ',': + break + ver_remaining = ver_remaining[1:].lstrip() + m = COMPARE_OP.match(ver_remaining) + if not m: + raise SyntaxError('invalid constraint: %s' % ver_remaining) + if not versions: + versions = None + return versions, ver_remaining + + if remaining[0] != '(': + versions, remaining = get_versions(remaining) + else: + i = remaining.find(')', 1) + if i < 0: + raise SyntaxError('unterminated parenthesis: %s' % remaining) + s = remaining[1:i] + remaining = remaining[i + 1:].lstrip() + # As a special diversion from PEP 508, allow a version number + # a.b.c in parentheses as a synonym for ~= a.b.c (because this + # is allowed in earlier PEPs) + if COMPARE_OP.match(s): + versions, _ = get_versions(s) + else: + m = VERSION_IDENTIFIER.match(s) + if not m: + raise SyntaxError('invalid constraint: %s' % s) + v = m.groups()[0] + s = s[m.end():].lstrip() + if s: + raise SyntaxError('invalid constraint: %s' % s) + versions = [('~=', v)] + + if remaining: + if remaining[0] != ';': + raise SyntaxError('invalid requirement: %s' % remaining) + remaining = remaining[1:].lstrip() + + mark_expr, remaining = parse_marker(remaining) + + if remaining and remaining[0] != '#': + raise SyntaxError('unexpected trailing data: %s' % remaining) + + if not versions: + rs = distname + else: + rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions])) + return Container(name=distname, extras=extras, constraints=versions, + marker=mark_expr, url=uri, requirement=rs) diff --git a/wasm/lib/Lib/asyncweb.py b/wasm/lib/Lib/asyncweb.py index 2e45d4a5e..7110846bf 100644 --- a/wasm/lib/Lib/asyncweb.py +++ b/wasm/lib/Lib/asyncweb.py @@ -14,6 +14,12 @@ def run(coro): """ _Runner(coro) +def spawn(coro): + """ + Run a coroutine. Like run(), but returns a promise that resolves with + the result of the coroutine. + """ + return Promise.resolve(_CoroPromise(coro)) class _Runner: def __init__(self, coro): @@ -77,7 +83,7 @@ class _CallbackMap: if self.done == -1: if error: - return _call_resolve(error, self._error) + return _call_resolve(error, self._result) else: return self elif self.done == 1: @@ -87,7 +93,6 @@ class _CallbackMap: return self if success: - # def onsuccess(then= self._successes.append(success) if error: self._errors.append(error) diff --git a/wasm/lib/Lib/whlimport.py b/wasm/lib/Lib/whlimport.py index 106196813..075ccc24d 100644 --- a/wasm/lib/Lib/whlimport.py +++ b/wasm/lib/Lib/whlimport.py @@ -2,7 +2,10 @@ import browser import zipfile import asyncweb import io +import re +from urllib.parse import urlparse, unquote import _frozen_importlib as _bootstrap +import _microdistlib _IS_SETUP = False @@ -13,17 +16,17 @@ def setup(*, log=print): if not _IS_SETUP: import sys - sys.meta_path.insert(0, WheelFinder) + sys.meta_path.insert(0, ZipFinder) _IS_SETUP = True - if not log: + if log: + + LOG_FUNC = log + else: def LOG_FUNC(log): pass - else: - LOG_FUNC = log - async def load_package(*args): await asyncweb.wait_all(_load_package(pkg) for pkg in args) @@ -33,12 +36,38 @@ _loaded_packages = {} LOG_FUNC = print +_http_url = re.compile("^http[s]?://") + async def _load_package(pkg): - # TODO: support pkg==X.Y semver specifiers as well as arbitrary URLs - info = await browser.fetch( - f"https://pypi.org/pypi/{pkg}/json", response_format="json" + if isinstance(pkg, str) and _http_url.match(pkg): + urlobj = urlparse(pkg) + fname = posixpath.basename(urlobj.path) + name, url, size, deps = fname, pkg, None, [] + else: + # TODO: load dependencies as well + name, fname, url, size, deps = await _load_info_pypi(pkg) + if name in _loaded_packages: + return + deps = asyncweb.spawn(asyncweb.wait_all(_load_package for dep in deps)) + size_str = format_size(size) if size is not None else "unknown size" + LOG_FUNC(f"Downloading {fname} ({size_str})...") + zip_data = io.BytesIO(await browser.fetch(url, response_format="array_buffer")) + size = len(zip_data.getbuffer()) + LOG_FUNC(f"{fname} done!") + _loaded_packages[name] = zipfile.ZipFile(zip_data) + await deps + + +async def _load_info_pypi(pkg): + pkg = _microdistlib.parse_requirement(pkg) + # TODO: use VersionMatcher from distlib + api_url = ( + f"https://pypi.org/pypi/{pkg.name}/json" + if not pkg.constraints + else f"https://pypi.org/pypi/{pkg}/{pkg.constraints[0][1]}/json" ) + info = await browser.fetch(api_url, response_format="json") name = info["info"]["name"] ver = info["info"]["version"] ver_downloads = info["releases"][ver] @@ -46,16 +75,7 @@ async def _load_package(pkg): dl = next(dl for dl in ver_downloads if dl["packagetype"] == "bdist_wheel") except StopIteration: raise ValueError(f"no wheel available for package {Name!r} {ver}") - if name in _loaded_packages: - return - fname = dl["filename"] - LOG_FUNC(f"Downloading {fname} ({format_size(dl['size'])})...") - zip_data = io.BytesIO( - await browser.fetch(dl["url"], response_format="array_buffer") - ) - size = len(zip_data.getbuffer()) - LOG_FUNC(f"{fname} done!") - _loaded_packages[name] = zipfile.ZipFile(zip_data) + return name, dl["filename"], dl["url"], dl["size"], info["info"]["requires_dist"] or [] def format_size(bytes): @@ -70,7 +90,7 @@ def format_size(bytes): return "{} bytes".format(int(bytes)) -class WheelFinder: +class ZipFinder: _packages = _loaded_packages @classmethod @@ -80,7 +100,7 @@ class WheelFinder: mi, fullpath = _get_module_info(z, path) if mi is not None: return _bootstrap.spec_from_loader( - fullname, cls, origin=f"wheel:{zname}/{fullpath}", is_package=mi + fullname, cls, origin=f"zip:{zname}/{fullpath}", is_package=mi ) return None @@ -99,10 +119,10 @@ class WheelFinder: @classmethod def _get_source(cls, spec): origin = spec.origin - if not origin or not origin.startswith("wheel:"): + if not origin or not origin.startswith("zip:"): raise ImportError(f"{module.__spec__.name!r} is not a zip module") - zipname, slash, path = origin[len("wheel:") :].partition("/") + zipname, slash, path = origin[len("zip:") :].partition("/") return cls._packages[zipname].read(path).decode() @classmethod @@ -116,9 +136,9 @@ class WheelFinder: _zip_searchorder = ( - # (path_sep + '__init__.pyc', True, True), + ("/__init__.pyc", True, True), ("/__init__.py", False, True), - # ('.pyc', True, False), + (".pyc", True, False), (".py", False, False), )