diff --git a/tests/.gitignore b/tests/.gitignore
index 05ba24c43..8829b7581 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1 +1 @@
-snippets/whats_left_to_implement.py
+snippets/whats_left_*.py
diff --git a/tests/generator/not_impl_footer.txt b/tests/generator/not_impl_methods_footer.txt
similarity index 100%
rename from tests/generator/not_impl_footer.txt
rename to tests/generator/not_impl_methods_footer.txt
diff --git a/tests/generator/not_impl_header.txt b/tests/generator/not_impl_methods_header.txt
similarity index 100%
rename from tests/generator/not_impl_header.txt
rename to tests/generator/not_impl_methods_header.txt
diff --git a/tests/generator/not_impl_modules_footer.txt b/tests/generator/not_impl_modules_footer.txt
new file mode 100644
index 000000000..1faffd0e5
--- /dev/null
+++ b/tests/generator/not_impl_modules_footer.txt
@@ -0,0 +1,15 @@
+
+rustpymods = set(
+    map(
+        lambda mod: mod[0],
+        filter(
+            lambda mod: mod[1] == "" or mod[1] == ".py",
+            map(os.path.splitext, os.listdir(libdir)),
+        ),
+    )
+)
+rustpymods |= set(sys.builtin_module_names)
+
+for mod in cpymods - rustpymods:
+    print(mod)
+
diff --git a/tests/generator/not_impl_modules_header.txt b/tests/generator/not_impl_modules_header.txt
new file mode 100644
index 000000000..8879230d4
--- /dev/null
+++ b/tests/generator/not_impl_modules_header.txt
@@ -0,0 +1,8 @@
+# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE
+# EDIT tests/not_impl_mods_gen.sh, NOT THIS FILE.
+# RESULTS OF THIS TEST DEPEND ON THE CPYTHON
+# VERSION AND PYTHON ENVIRONMENT USED
+# TO RUN not_impl_mods_gen.py
+
+import sys
+import os
diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py
index 2f1df44a2..dba3df827 100644
--- a/tests/not_impl_gen.py
+++ b/tests/not_impl_gen.py
@@ -1,20 +1,6 @@
-objects = [
-    bool,
-    bytearray,
-    bytes,
-    complex,
-    dict,
-    float,
-    frozenset,
-    int,
-    list,
-    memoryview,
-    range,
-    set,
-    str,
-    tuple,
-    object,
-]
+import pkgutil
+import os
+import sys
 
 
 def attr_is_not_inherited(type_, attr):
@@ -22,27 +8,71 @@ def attr_is_not_inherited(type_, attr):
     returns True if type_'s attr is not inherited from any of its base classes
     """
 
-    bases = obj.__mro__[1:]
+    bases = type_.__mro__[1:]
 
-    return getattr(obj, attr) not in (
-        getattr(base, attr, None) for base in bases)
+    return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases)
 
 
-header = open("generator/not_impl_header.txt")
-footer = open("generator/not_impl_footer.txt")
-output = open("snippets/whats_left_to_implement.py", "w")
+def gen_methods(header, footer, output):
+    objects = [
+        bool,
+        bytearray,
+        bytes,
+        complex,
+        dict,
+        float,
+        frozenset,
+        int,
+        list,
+        memoryview,
+        range,
+        set,
+        str,
+        tuple,
+        object,
+    ]
 
-output.write(header.read())
-output.write("expected_methods = {\n")
+    output.write(header.read())
+    output.write("expected_methods = {\n")
 
-for obj in objects:
-    output.write(f" '{obj.__name__}': ({obj.__name__}, [\n")
-    output.write("\n".join(
-        f"  {attr!r},"
-        for attr in dir(obj)
-        if attr_is_not_inherited(obj, attr)
-    ))
-    output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n"))
+    for obj in objects:
+        output.write(f" '{obj.__name__}': ({obj.__name__}, [\n")
+        output.write(
+            "\n".join(
+                f"    {attr!r},"
+                for attr in dir(obj)
+                if attr_is_not_inherited(obj, attr)
+            )
+        )
+        output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n"))
+
+    output.write("}\n\n")
+    output.write(footer.read())
+
+
+def gen_modules(header, footer, output):
+    output.write(header.read())
+
+    modules = set(map(lambda mod: mod.name, pkgutil.iter_modules()))
+
+    print(
+        f"""
+cpymods = {modules!r}
+libdir = {os.path.abspath("../Lib/")!r}
+""",
+        file=output,
+    )
+
+    output.write(footer.read())
+
+
+gen_funcs = {"methods": gen_methods, "modules": gen_modules}
+
+
+for name, gen_func in gen_funcs.items():
+    gen_func(
+        header=open(f"generator/not_impl_{name}_header.txt"),
+        footer=open(f"generator/not_impl_{name}_footer.txt"),
+        output=open(f"snippets/whats_left_{name}.py", "w"),
+    )
 
-output.write("}\n\n")
-output.write(footer.read())
diff --git a/whats_left.sh b/whats_left.sh
index 66e64f1a3..72601c4e2 100755
--- a/whats_left.sh
+++ b/whats_left.sh
@@ -1,11 +1,32 @@
 #!/bin/sh
 set -e
 
+GREEN=''
+BOLD=''
+NC='(B'
+
+h() {
+  # uppercase input
+  header_name=$(echo "$@" | tr "[:lower:]" "[:upper:]")
+  echo "$GREEN$BOLD===== $header_name =====$NC"
+}
+
 cd "$(dirname "$0")"
-cd tests
 
-python3 not_impl_gen.py
+(
+  cd tests
+  # -I means isolate from environment; we don't want any pip packages to be listed
+  python3 -I not_impl_gen.py
+)
 
-cd ..
+# show the building first, so people aren't confused why it's taking so long to
+# run whats_left_to_implement
+cargo build
 
-cargo run -- tests/snippets/whats_left_to_implement.py
\ No newline at end of file
+whats_left_section() {
+  h "$1"
+  cargo run -q -- tests/snippets/whats_left_"$1".py
+}
+
+whats_left_section methods
+whats_left_section modules