Version specification for whlimport

wasm/lib/Lib/_microdistlib.py

@@ -0,0 +1,129 @@
+# taken from https://bitbucket.org/pypa/distlib/src/master/distlib/util.py
+
+from types import SimpleNamespace as Container
+import re
+
+IDENTIFIER = re.compile(r'^([\w\.-]+)\s*')
+VERSION_IDENTIFIER = re.compile(r'^([\w\.*+-]+)\s*')
+COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*')
+NON_SPACE = re.compile(r'(\S+)\s*')
+
+def parse_requirement(req):
+    """
+    Parse a requirement passed in as a string. Return a Container
+    whose attributes contain the various parts of the requirement.
+    """
+    remaining = req.strip()
+    if not remaining or remaining.startswith('#'):
+        return None
+    m = IDENTIFIER.match(remaining)
+    if not m:
+        raise SyntaxError('name expected: %s' % remaining)
+    distname = m.groups()[0]
+    remaining = remaining[m.end():]
+    extras = mark_expr = versions = uri = None
+    if remaining and remaining[0] == '[':
+        i = remaining.find(']', 1)
+        if i < 0:
+            raise SyntaxError('unterminated extra: %s' % remaining)
+        s = remaining[1:i]
+        remaining = remaining[i + 1:].lstrip()
+        extras = []
+        while s:
+            m = IDENTIFIER.match(s)
+            if not m:
+                raise SyntaxError('malformed extra: %s' % s)
+            extras.append(m.groups()[0])
+            s = s[m.end():]
+            if not s:
+                break
+            if s[0] != ',':
+                raise SyntaxError('comma expected in extras: %s' % s)
+            s = s[1:].lstrip()
+        if not extras:
+            extras = None
+    if remaining:
+        if remaining[0] == '@':
+            # it's a URI
+            remaining = remaining[1:].lstrip()
+            m = NON_SPACE.match(remaining)
+            if not m:
+                raise SyntaxError('invalid URI: %s' % remaining)
+            uri = m.groups()[0]
+            t = urlparse(uri)
+            # there are issues with Python and URL parsing, so this test
+            # is a bit crude. See bpo-20271, bpo-23505. Python doesn't
+            # always parse invalid URLs correctly - it should raise
+            # exceptions for malformed URLs
+            if not (t.scheme and t.netloc):
+                raise SyntaxError('Invalid URL: %s' % uri)
+            remaining = remaining[m.end():].lstrip()
+        else:
+
+            def get_versions(ver_remaining):
+                """
+                Return a list of operator, version tuples if any are
+                specified, else None.
+                """
+                m = COMPARE_OP.match(ver_remaining)
+                versions = None
+                if m:
+                    versions = []
+                    while True:
+                        op = m.groups()[0]
+                        ver_remaining = ver_remaining[m.end():]
+                        m = VERSION_IDENTIFIER.match(ver_remaining)
+                        if not m:
+                            raise SyntaxError('invalid version: %s' % ver_remaining)
+                        v = m.groups()[0]
+                        versions.append((op, v))
+                        ver_remaining = ver_remaining[m.end():]
+                        if not ver_remaining or ver_remaining[0] != ',':
+                            break
+                        ver_remaining = ver_remaining[1:].lstrip()
+                        m = COMPARE_OP.match(ver_remaining)
+                        if not m:
+                            raise SyntaxError('invalid constraint: %s' % ver_remaining)
+                    if not versions:
+                        versions = None
+                return versions, ver_remaining
+
+            if remaining[0] != '(':
+                versions, remaining = get_versions(remaining)
+            else:
+                i = remaining.find(')', 1)
+                if i < 0:
+                    raise SyntaxError('unterminated parenthesis: %s' % remaining)
+                s = remaining[1:i]
+                remaining = remaining[i + 1:].lstrip()
+                # As a special diversion from PEP 508, allow a version number
+                # a.b.c in parentheses as a synonym for ~= a.b.c (because this
+                # is allowed in earlier PEPs)
+                if COMPARE_OP.match(s):
+                    versions, _ = get_versions(s)
+                else:
+                    m = VERSION_IDENTIFIER.match(s)
+                    if not m:
+                        raise SyntaxError('invalid constraint: %s' % s)
+                    v = m.groups()[0]
+                    s = s[m.end():].lstrip()
+                    if s:
+                        raise SyntaxError('invalid constraint: %s' % s)
+                    versions = [('~=', v)]
+
+    if remaining:
+        if remaining[0] != ';':
+            raise SyntaxError('invalid requirement: %s' % remaining)
+        remaining = remaining[1:].lstrip()
+
+        mark_expr, remaining = parse_marker(remaining)
+
+    if remaining and remaining[0] != '#':
+        raise SyntaxError('unexpected trailing data: %s' % remaining)
+
+    if not versions:
+        rs = distname
+    else:
+        rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions]))
+    return Container(name=distname, extras=extras, constraints=versions,
+                     marker=mark_expr, url=uri, requirement=rs)

wasm/lib/Lib/asyncweb.py

@@ -14,6 +14,12 @@ def run(coro):
     """
     _Runner(coro)
 
+def spawn(coro):
+    """
+    Run a coroutine. Like run(), but returns a promise that resolves with
+    the result of the coroutine.
+    """
+    return Promise.resolve(_CoroPromise(coro))
 
 class _Runner:
     def __init__(self, coro):
@@ -77,7 +83,7 @@ class _CallbackMap:
 
         if self.done == -1:
             if error:
-                return _call_resolve(error, self._error)
+                return _call_resolve(error, self._result)
             else:
                 return self
         elif self.done == 1:
@@ -87,7 +93,6 @@ class _CallbackMap:
                 return self
 
         if success:
-            # def onsuccess(then=
             self._successes.append(success)
         if error:
             self._errors.append(error)

wasm/lib/Lib/whlimport.py

@@ -2,7 +2,10 @@ import browser
 import zipfile
 import asyncweb
 import io
+import re
+from urllib.parse import urlparse, unquote
 import _frozen_importlib as _bootstrap
+import _microdistlib
 
 _IS_SETUP = False
 
@@ -13,17 +16,17 @@ def setup(*, log=print):
     if not _IS_SETUP:
         import sys
 
-        sys.meta_path.insert(0, WheelFinder)
+        sys.meta_path.insert(0, ZipFinder)
         _IS_SETUP = True
 
-    if not log:
+    if log:
+
+        LOG_FUNC = log
+    else:
 
         def LOG_FUNC(log):
             pass
 
-    else:
-        LOG_FUNC = log
-
 
 async def load_package(*args):
     await asyncweb.wait_all(_load_package(pkg) for pkg in args)
@@ -33,12 +36,38 @@ _loaded_packages = {}
 
 LOG_FUNC = print
 
+_http_url = re.compile("^http[s]?://")
+
 
 async def _load_package(pkg):
-    # TODO: support pkg==X.Y semver specifiers as well as arbitrary URLs
-    info = await browser.fetch(
-        f"https://pypi.org/pypi/{pkg}/json", response_format="json"
+    if isinstance(pkg, str) and _http_url.match(pkg):
+        urlobj = urlparse(pkg)
+        fname = posixpath.basename(urlobj.path)
+        name, url, size, deps = fname, pkg, None, []
+    else:
+        # TODO: load dependencies as well
+        name, fname, url, size, deps = await _load_info_pypi(pkg)
+    if name in _loaded_packages:
+        return
+    deps = asyncweb.spawn(asyncweb.wait_all(_load_package for dep in deps))
+    size_str = format_size(size) if size is not None else "unknown size"
+    LOG_FUNC(f"Downloading {fname} ({size_str})...")
+    zip_data = io.BytesIO(await browser.fetch(url, response_format="array_buffer"))
+    size = len(zip_data.getbuffer())
+    LOG_FUNC(f"{fname} done!")
+    _loaded_packages[name] = zipfile.ZipFile(zip_data)
+    await deps
+
+
+async def _load_info_pypi(pkg):
+    pkg = _microdistlib.parse_requirement(pkg)
+    # TODO: use VersionMatcher from distlib
+    api_url = (
+        f"https://pypi.org/pypi/{pkg.name}/json"
+        if not pkg.constraints
+        else f"https://pypi.org/pypi/{pkg}/{pkg.constraints[0][1]}/json"
     )
+    info = await browser.fetch(api_url, response_format="json")
     name = info["info"]["name"]
     ver = info["info"]["version"]
     ver_downloads = info["releases"][ver]
@@ -46,16 +75,7 @@ async def _load_package(pkg):
         dl = next(dl for dl in ver_downloads if dl["packagetype"] == "bdist_wheel")
     except StopIteration:
         raise ValueError(f"no wheel available for package {Name!r} {ver}")
-    if name in _loaded_packages:
-        return
-    fname = dl["filename"]
-    LOG_FUNC(f"Downloading {fname} ({format_size(dl['size'])})...")
-    zip_data = io.BytesIO(
-        await browser.fetch(dl["url"], response_format="array_buffer")
-    )
-    size = len(zip_data.getbuffer())
-    LOG_FUNC(f"{fname} done!")
-    _loaded_packages[name] = zipfile.ZipFile(zip_data)
+    return name, dl["filename"], dl["url"], dl["size"], info["info"]["requires_dist"] or []
 
 
 def format_size(bytes):
@@ -70,7 +90,7 @@ def format_size(bytes):
         return "{} bytes".format(int(bytes))
 
 
-class WheelFinder:
+class ZipFinder:
     _packages = _loaded_packages
 
     @classmethod
@@ -80,7 +100,7 @@ class WheelFinder:
             mi, fullpath = _get_module_info(z, path)
             if mi is not None:
                 return _bootstrap.spec_from_loader(
-                    fullname, cls, origin=f"wheel:{zname}/{fullpath}", is_package=mi
+                    fullname, cls, origin=f"zip:{zname}/{fullpath}", is_package=mi
                 )
         return None
 
@@ -99,10 +119,10 @@ class WheelFinder:
     @classmethod
     def _get_source(cls, spec):
         origin = spec.origin
-        if not origin or not origin.startswith("wheel:"):
+        if not origin or not origin.startswith("zip:"):
             raise ImportError(f"{module.__spec__.name!r} is not a zip module")
 
-        zipname, slash, path = origin[len("wheel:") :].partition("/")
+        zipname, slash, path = origin[len("zip:") :].partition("/")
         return cls._packages[zipname].read(path).decode()
 
     @classmethod
@@ -116,9 +136,9 @@ class WheelFinder:
 
 
 _zip_searchorder = (
-    # (path_sep + '__init__.pyc', True, True),
+    ("/__init__.pyc", True, True),
     ("/__init__.py", False, True),
-    # ('.pyc', True, False),
+    (".pyc", True, False),
     (".py", False, False),
 )