Detect incompatible function signatures in whats_left

.gitignore

@@ -9,7 +9,6 @@ __pycache__
 .vscode
 wasm-pack.log
 .idea/
-extra_tests/snippets/resources
 
 flame-graph.html
 flame.txt
@@ -17,3 +16,6 @@ flamescope.json
 /wapm.lock
 /wapm_packages
 /.cargo/config
+
+extra_tests/snippets/resources
+extra_tests/snippets/not_impl.py

extra_tests/generator/not_impl_methods_footer.txt

@@ -1,26 +0,0 @@
-
-def attr_is_not_inherited(type_, attr):
-    """
-    returns True if type_'s attr is not inherited from any of its base classes
-    """
-    bases = type_.__mro__[1:]
-
-    return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases)
-
-
-not_implementeds = []
-for name, (typ, methods) in expected_methods.items():
-    for method in methods:
-        has_method = hasattr(typ, method)
-        is_inherited = has_method and not attr_is_not_inherited(typ, method)
-        if has_method and not is_inherited:
-            continue
-        not_implementeds.append((name, method, is_inherited))
-
-for r in not_implementeds:
-    print(r[0], ".", r[1], " (inherited)" if r[2] else "", sep="")
-if not not_implementeds:
-    print("Not much \\o/")
-
-if platform.python_implementation() == "CPython":
-    assert len(not_implementeds) == 0, "CPython should have all the methods"

extra_tests/generator/not_impl_methods_header.txt

@@ -1,7 +0,0 @@
-# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
-# EDIT tests/not_impl_gen.py, NOT THIS FILE.
-# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
-# VERSION USED TO RUN not_impl_gen.py
-
-import platform
-

extra_tests/generator/not_impl_modules_footer.txt

@@ -1,33 +0,0 @@
-
-
-def dir_of_mod_or_default(module, default=None):
-    try:
-        mod = set(dir(__import__(module)))
-        return mod
-    except (ImportError, AttributeError):
-        return default
-
-mod_names = [
-    name.decode() for name, ext in map(os.path.splitext, os.listdir(libdir))
-    if ext == b'.py' or os.path.isdir(os.path.join(libdir, name))]
-mod_names += list(sys.builtin_module_names)
-
-not_imported = {'not imported'}
-rustpymods = {
-    mod: (dir_of_mod_or_default(mod, not_imported) if mod not in ("this", "antigravity") else None)
-    for mod in mod_names
-}
-
-for modname, cpymod in cpymods.items():
-    if modname in rustpymods:
-        rustpymod = rustpymods[modname]
-        if rustpymod:
-            if rustpymod == not_imported:
-                print(f"{modname} (existing but not importable)")
-                continue
-            for item in cpymod - rustpymod:
-                print(f"{modname}.{item}")
-    else:
-        print(f"{modname} (entire module)")
-
-

extra_tests/generator/not_impl_modules_header.txt

@@ -1,8 +0,0 @@
-# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
-# EDIT tests/not_impl_mods_gen.sh, NOT THIS FILE.
-# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
-# VERSION AND PYTHON ENVIRONMENT USED
-# TO RUN not_impl_mods_gen.py
-
-import sys
-import os

extra_tests/not_impl_gen.py

@@ -1,13 +1,20 @@
 # It's recommended to run this with `python3 -I not_impl_gen.py`, to make sure
 # that nothing in your global Python environment interferes with what's being
 # extracted here.
+#
+# This script generates Lib/snippets/whats_left_data.py with these variables defined:
+# expected_methods - a dictionary mapping builtin objects to their methods
+# cpymods - a dictionary mapping module names to their contents
+# libdir - the location of RustPython's Lib/ directory.
 
-import pkgutil
+import re
 import os
 import sys
 import warnings
+import inspect
 from pydoc import ModuleScanner
 
+
 sys.path = list(
     filter(
         lambda path: "site-packages" not in path and "dist-packages" not in path,
@@ -20,13 +27,24 @@ def attr_is_not_inherited(type_, attr):
     """
     returns True if type_'s attr is not inherited from any of its base classes
     """
-
     bases = type_.__mro__[1:]
-
     return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases)
 
 
-def gen_methods(header, footer, output):
+# TODO: move this function to a shared library both CPython and RustPython import
+def extra_info(obj):
+    if callable(obj):
+        # TODO: check for the correct thing above and remove try
+        try:
+            sig = str(inspect.signature(obj))
+            # remove function memory addresses
+            return re.sub(" at 0x[0-9A-Fa-f]+", " at 0xdeadbeef", sig)
+        except Exception:
+            return None
+    return None
+
+
+def gen_methods():
     types = [
         bool,
         bytearray,
@@ -49,51 +67,50 @@ def gen_methods(header, footer, output):
         tuple,
         object,
         zip,
-
         classmethod,
         staticmethod,
         property,
-
         Exception,
         BaseException,
     ]
-    objects = [(t.__name__, t.__name__) for t in types]
-    objects.extend([
-        ('NoneType', 'type(None)'),
-    ])
+    objects = [t.__name__ for t in types]
+    objects.append("type(None)")
 
     iters = [
-        ('bytearray_iterator', 'type(bytearray().__iter__())'),
-        ('bytes_iterator', 'type(bytes().__iter__())'),
-        ('dict_keyiterator', 'type(dict().__iter__())'),
-        ('dict_valueiterator', 'type(dict().values().__iter__())'),
-        ('dict_itemiterator', 'type(dict().items().__iter__())'),
-        ('dict_values', 'type(dict().values())'),
-        ('dict_items', 'type(dict().items())'),
-        ('set_iterator', 'type(set().__iter__())'),
-        ('list_iterator', 'type(list().__iter__())'),
-        ('range_iterator', 'type(range(0).__iter__())'),
-        ('str_iterator', 'type(str().__iter__())'),
-        ('tuple_iterator', 'type(tuple().__iter__())'),
+        "type(bytearray().__iter__())",
+        "type(bytes().__iter__())",
+        "type(dict().__iter__())",
+        "type(dict().values().__iter__())",
+        "type(dict().items().__iter__())",
+        "type(dict().values())",
+        "type(dict().items())",
+        "type(set().__iter__())",
+        "type(list().__iter__())",
+        "type(range(0).__iter__())",
+        "type(str().__iter__())",
+        "type(tuple().__iter__())",
     ]
 
-    output.write(header.read())
-    output.write("expected_methods = {\n")
-
-    for name, typ_code in objects + iters:
+    methods = {}
+    for typ_code in objects + iters:
         typ = eval(typ_code)
-        output.write(f" '{name}': ({typ_code}, [\n")
-        output.write(
-            "\n".join(
-                f"    {attr!r},"
-                for attr in dir(typ)
-                if attr_is_not_inherited(typ, attr)
-            )
+        methods[typ.__name__] = (
+            typ_code,
+            extra_info(typ),
+            [(attr, extra_info(attr)) for attr in dir(typ) if attr_is_not_inherited(typ, attr)],
         )
-        output.write("\n ])," + ("\n" if objects[-1] == typ else "\n\n"))
 
-    output.write("}\n\n")
-    output.write(footer.read())
+    output = "expected_methods = {\n"
+    for name, (typ_code, extra, attrs) in methods.items():
+        output += f" '{name}': ({typ_code}, {extra!r}, [\n"
+        for attr, attr_extra in attrs:
+            output += f"    ({attr!r}, {attr_extra!r}),\n"
+        output += " ]),\n"
+        # TODO: why?
+        if typ_code != objects[-1]:
+            output += "\n"
+    output += "}\n\n"
+    return output
 
 
 def scan_modules():
@@ -107,6 +124,7 @@ def scan_modules():
             modname = modname[:-9] + " (package)"
         if modname.find(".") < 0:
             modules[modname] = 1
+
     def onerror(modname):
         callback(None, modname, None)
 
@@ -116,42 +134,168 @@ def scan_modules():
     return list(modules.keys())
 
 
-def get_module_methods(name):
+# TODO: move this function to a shared library both CPython and RustPython import
+def dir_of_mod_or_error(module_name):
     with warnings.catch_warnings():
         # ignore warnings caused by importing deprecated modules
         warnings.filterwarnings("ignore", category=DeprecationWarning)
         try:
-            return set(dir(__import__(name))) if name not in ("this", "antigravity") else None
-        except ModuleNotFoundError:
-            return None
+            module = __import__(module_name)
         except Exception as e:
-            print("!!! {} skipped because {}: {}".format(name, type(e).__name__, str(e)))
+            return e
+    item_names = sorted(set(dir(module)))
+    result = {}
+    for item_name in item_names:
+        item = getattr(module, item_name)
+        result[item_name] = extra_info(item)
+    return result
 
 
-def gen_modules(header, footer, output):
-    output.write(header.read())
-
+def gen_modules():
     # check name because modules listed have side effects on import,
     # e.g. printing something or opening a webpage
-    modules = {mod_name: get_module_methods(mod_name) for mod_name in scan_modules()}
-
-    print(
-        f"""
+    modules = {}
+    for mod_name in scan_modules():
+        if mod_name in ("this", "antigravity"):
+            continue
+        dir_result = dir_of_mod_or_error(mod_name)
+        if isinstance(dir_result, Exception):
+            print(
+                f"!!! {mod_name} skipped because {type(dir_result).__name__}: {str(dir_result)}",
+                file=sys.stderr,
+            )
+            continue
+        modules[mod_name] = dir_result
+    return f"""
 cpymods = {modules!r}
 libdir = {os.path.abspath("../Lib/").encode('utf8')!r}
-""",
-        file=output,
-    )
 
-    output.write(footer.read())
+"""
 
 
-gen_funcs = {"methods": gen_methods, "modules": gen_modules}
+output = """\
+# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
+# EDIT extra_tests/not_impl_gen.sh, NOT THIS FILE.
+# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
+# VERSION AND PYTHON ENVIRONMENT USED
+# TO RUN not_impl_mods_gen.py
+
+"""
+
+output += gen_methods()
+output += gen_modules()
+
+# Prevent missing variable linter errors from compare()
+expected_methods = {}
+cpymods = {}
+libdir = ""
+# This function holds the source code that will be run under RustPython
+def compare():
+    import re
+    import os
+    import sys
+    import warnings
+    import inspect
+    import platform
+
+    def attr_is_not_inherited(type_, attr):
+        """
+        returns True if type_'s attr is not inherited from any of its base classes
+        """
+        bases = type_.__mro__[1:]
+        return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases)
+
+    # TODO: move this function to a shared library both CPython and RustPython import
+    def extra_info(obj):
+        if callable(obj):
+            # TODO: check for the correct thing above and remove try
+            try:
+                sig = str(inspect.signature(obj))
+                # remove function memory addresses
+                return re.sub(" at 0x[0-9A-Fa-f]+", " at 0xdeadbeef", sig)
+            except Exception:
+                return None
+        return None
+
+    not_implementeds = []
+    for name, (typ, real_value, methods) in expected_methods.items():
+        for method, method_extra in methods:
+            has_method = hasattr(typ, method)
+            is_inherited = has_method and not attr_is_not_inherited(typ, method)
+            value = extra_info(method)
+            if has_method and not is_inherited and value == real_value:
+                continue
+
+            if not has_method:
+                reason = ""
+            elif is_inherited:
+                reason = "inherited"
+            else:
+                reason = f"{value} != {real_value}"
+            not_implementeds.append((name, method, reason))
+
+    for module, method, reason in not_implementeds:
+        print(f"{module}.{method}" + (f" {reason}" if reason else ""))
+    if not not_implementeds:
+        print("Not much \\o/")
+
+    if platform.python_implementation() == "CPython":
+        if not_implementeds:
+            sys.exit("ERROR: CPython should have all the methods")
+
+    # TODO: move this function to a shared library both CPython and RustPython import
+    def dir_of_mod_or_error(module_name):
+        with warnings.catch_warnings():
+            # ignore warnings caused by importing deprecated modules
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            try:
+                module = __import__(module_name)
+            except Exception as e:
+                return e
+        item_names = sorted(set(dir(module)))
+        result = {}
+        for item_name in item_names:
+            item = getattr(module, item_name)
+            result[item_name] = extra_info(item)
+        return result
+
+    mod_names = [
+        name.decode()
+        for name, ext in map(os.path.splitext, os.listdir(libdir))
+        if ext == b".py" or os.path.isdir(os.path.join(libdir, name))
+    ]
+    mod_names += list(sys.builtin_module_names)
+    # Remove easter egg modules
+    mod_names = sorted(set(mod_names) - {"this", "antigravity"})
+
+    rustpymods = {mod: dir_of_mod_or_error(mod) for mod in mod_names}
+
+    for modname, cpymod in cpymods.items():
+        rustpymod = rustpymods.get(modname, {})
+        if isinstance(rustpymod, ImportError):
+            print(modname, "(entire module)")
+        elif isinstance(rustpymod, Exception):
+            print(f"{modname} (exists but not importable: {rustpymod})")
+        else:
+            implemented_items = sorted(set(cpymod) & set(rustpymod))
+            missing_items = set(cpymod) - set(rustpymod)
+            if not rustpymod and cpymod:
+                print(modname, "(entire module)")  # TODO: why do I have this twice
+            else:
+                for item in missing_items:
+                    print(f"{modname}.{item}")
+                for item in implemented_items:
+                    if rustpymod[item] != cpymod[item] and cpymod[item] != "None":
+                        print(f"{modname}.{item}: {rustpymod[item]} != {cpymod[item]}")
 
 
-for name, gen_func in gen_funcs.items():
-    gen_func(
-        header=open(f"generator/not_impl_{name}_header.txt"),
-        footer=open(f"generator/not_impl_{name}_footer.txt"),
-        output=open(f"snippets/whats_left_{name}.py", "w"),
-    )
+def remove_one_indent(s):
+    indent = "    "
+    return s[len(indent) :] if s.startswith(indent) else s
+
+
+compare_src = inspect.getsourcelines(compare)[0][1:]
+output += "".join(remove_one_indent(line) for line in compare_src)
+
+with open("snippets/not_impl.py", "w") as f:
+    f.write(output + "\n")

whats_left.sh

@@ -1,16 +1,22 @@
 #!/bin/bash
 set -e
 
+# This script runs a Python script which finds all modules it has available and
+# creates a Python dictionary mapping module names to their contents, which is
+# in turn used to generate a second Python script that also finds which modules
+# it has available and compares that against the first dictionary we generated.
+# We then run this second generated script with RustPython.
+
 ALL_SECTIONS=(methods modules)
 
 GREEN=''
 BOLD=''
-NC='(B'
+NC=''
 
-h() {
+print_header() {
   # uppercase input
   header_name=$(echo "$@" | tr "[:lower:]" "[:upper:]")
-  echo "$GREEN$BOLD===== $header_name =====$NC"
+  echo "$GREEN""$BOLD"===== "$header_name" ====="$NC"
 }
 
 cd "$(dirname "$0")"
@@ -23,23 +29,13 @@ export RUSTPYTHONPATH=Lib
   python3 -I not_impl_gen.py
 )
 
-# show the building first, so people aren't confused why it's taking so long to
-# run whats_left_to_implement
-cargo build --release
-
-if [ $# -eq 0 ]; then
-  sections=(${ALL_SECTIONS[@]})
-else
-  sections=($@)
+# This takes a while
+if command -v black &> /dev/null; then
+    black -q extra_tests/snippets/not_impl.py
 fi
 
-for section in "${sections[@]}"; do
-  section=$(echo "$section" | tr "[:upper:]" "[:lower:]")
-  snippet=extra_tests/snippets/whats_left_$section.py
-  if ! [[ -f $snippet ]]; then
-    echo "Invalid section $section" >&2
-    continue
-  fi
-  h "$section" >&2
-  cargo run --release -q -- "$snippet"
-done
+# show the building first, so people aren't confused why it's taking so long to
+# run whats_left
+cargo build --release
+
+cargo run --release -q -- extra_tests/snippets/not_impl.py