Merge branch 'RustPython:main' into 4845

.github/workflows/ci.yaml

@@ -124,8 +124,8 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
-          choco install llvm openssl
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          choco install llvm openssl --no-progress
+          echo "OPENSSL_DIR=C:\Program Files\OpenSSL" >>$GITHUB_ENV
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
         run: brew install autoconf automake libtool
@@ -263,8 +263,8 @@ jobs:
       - name: Set up the Windows environment
         shell: bash
         run: |
-          choco install llvm openssl
-          echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV
+          choco install llvm openssl --no-progress
+          echo "OPENSSL_DIR=C:\Program Files\OpenSSL" >>$GITHUB_ENV
         if: runner.os == 'Windows'
       - name: Set up the Mac environment
         run: brew install autoconf automake libtool openssl@3
@@ -308,6 +308,11 @@ jobs:
         run: |
           target/release/rustpython -m ensurepip
           target/release/rustpython -c "import pip"
+      - if: runner.os != 'Windows'
+        name: Check if pip inside venv is functional
+        run: |
+          target/release/rustpython -m venv testvenv
+          testvenv/bin/rustpython -m pip install wheel
       - name: Check whats_left is not broken
         run: python -I whats_left.py
 

.vscode/launch.json

@@ -17,6 +17,9 @@
             "preLaunchTask": "Build RustPython Debug",
             "program": "target/debug/rustpython",
             "args": [],
+            "env": {
+                "RUST_BACKTRACE": "1"
+            },
             "cwd": "${workspaceFolder}"
         },
         {

Cargo.lock

@@ -950,6 +950,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df"
 
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.53"
@@ -2137,6 +2146,7 @@ dependencies = [
  "foreign-types-shared",
  "gethostname",
  "hex",
+ "hmac",
  "itertools",
  "libc",
  "libsqlite3-sys",

Lib/concurrent/futures/thread.py

@@ -37,7 +37,8 @@ def _python_exit():
 threading._register_atexit(_python_exit)
 
 # At fork, reinitialize the `_global_shutdown_lock` lock in the child process
-if hasattr(os, 'register_at_fork'):
+# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet
+if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'):
     os.register_at_fork(before=_global_shutdown_lock.acquire,
                         after_in_child=_global_shutdown_lock._at_fork_reinit,
                         after_in_parent=_global_shutdown_lock.release)

Lib/hashlib.py

@@ -0,0 +1,316 @@
+#.  Copyright (C) 2005-2010   Gregory P. Smith (greg@krypto.org)
+#  Licensed to PSF under a Contributor Agreement.
+#
+
+__doc__ = """hashlib module - A common interface to many hash functions.
+
+new(name, data=b'', **kwargs) - returns a new hash object implementing the
+                                given hash function; initializing the hash
+                                using the given binary data.
+
+Named constructor functions are also available, these are faster
+than using new(name):
+
+md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(),
+sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256.
+
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
+
+NOTE: If you want the adler32 or crc32 hash functions they are available in
+the zlib module.
+
+Choose your hash function wisely.  Some have known collision weaknesses.
+sha384 and sha512 will be slow on 32 bit platforms.
+
+Hash objects have these methods:
+ - update(data): Update the hash object with the bytes in data. Repeated calls
+                 are equivalent to a single call with the concatenation of all
+                 the arguments.
+ - digest():     Return the digest of the bytes passed to the update() method
+                 so far as a bytes object.
+ - hexdigest():  Like digest() except the digest is returned as a string
+                 of double length, containing only hexadecimal digits.
+ - copy():       Return a copy (clone) of the hash object. This can be used to
+                 efficiently compute the digests of datas that share a common
+                 initial substring.
+
+For example, to obtain the digest of the byte string 'Nobody inspects the
+spammish repetition':
+
+    >>> import hashlib
+    >>> m = hashlib.md5()
+    >>> m.update(b"Nobody inspects")
+    >>> m.update(b" the spammish repetition")
+    >>> m.digest()
+    b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9'
+
+More condensed:
+
+    >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest()
+    'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2'
+
+"""
+
+# This tuple and __get_builtin_constructor() must be modified if a new
+# always available algorithm is added.
+__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
+                      'blake2b', 'blake2s',
+                      'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+                      'shake_128', 'shake_256')
+
+
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'pbkdf2_hmac', 'file_digest')
+
+
+__builtin_constructor_cache = {}
+
+# Prefer our blake2 implementation
+# OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. The OpenSSL
+# implementations neither support keyed blake2 (blake2 MAC) nor advanced
+# features like salt, personalization, or tree hashing. OpenSSL hash-only
+# variants are available as 'blake2b512' and 'blake2s256', though.
+__block_openssl_constructor = {
+    'blake2b', 'blake2s',
+}
+
+def __get_builtin_constructor(name):
+    cache = __builtin_constructor_cache
+    constructor = cache.get(name)
+    if constructor is not None:
+        return constructor
+    try:
+        if name in {'SHA1', 'sha1'}:
+            import _sha1
+            cache['SHA1'] = cache['sha1'] = _sha1.sha1
+        elif name in {'MD5', 'md5'}:
+            import _md5
+            cache['MD5'] = cache['md5'] = _md5.md5
+        elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
+            import _sha256
+            cache['SHA224'] = cache['sha224'] = _sha256.sha224
+            cache['SHA256'] = cache['sha256'] = _sha256.sha256
+        elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
+            import _sha512
+            cache['SHA384'] = cache['sha384'] = _sha512.sha384
+            cache['SHA512'] = cache['sha512'] = _sha512.sha512
+        elif name in {'blake2b', 'blake2s'}:
+            import _blake2
+            cache['blake2b'] = _blake2.blake2b
+            cache['blake2s'] = _blake2.blake2s
+        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512'}:
+            import _sha3
+            cache['sha3_224'] = _sha3.sha3_224
+            cache['sha3_256'] = _sha3.sha3_256
+            cache['sha3_384'] = _sha3.sha3_384
+            cache['sha3_512'] = _sha3.sha3_512
+        elif name in {'shake_128', 'shake_256'}:
+            import _sha3
+            cache['shake_128'] = _sha3.shake_128
+            cache['shake_256'] = _sha3.shake_256
+    except ImportError:
+        pass  # no extension module, this hash is unsupported.'''
+    
+    constructor = cache.get(name)
+    if constructor is not None:
+        return constructor
+
+    raise ValueError('unsupported hash type ' + name)
+
+
+def __get_openssl_constructor(name):
+    if name in __block_openssl_constructor:
+        # Prefer our builtin blake2 implementation.
+        return __get_builtin_constructor(name)
+    try:
+        # MD5, SHA1, and SHA2 are in all supported OpenSSL versions
+        # SHA3/shake are available in OpenSSL 1.1.1+
+        f = getattr(_hashlib, 'openssl_' + name)
+        # Allow the C module to raise ValueError.  The function will be
+        # defined but the hash not actually available.  Don't fall back to
+        # builtin if the current security policy blocks a digest, bpo#40695.
+        f(usedforsecurity=False)
+        # Use the C function directly (very fast)
+        return f
+    except (AttributeError, ValueError):
+        return __get_builtin_constructor(name)
+
+
+def __py_new(name, data=b'', **kwargs):
+    """new(name, data=b'', **kwargs) - Return a new hashing object using the
+    named algorithm; optionally initialized with data (which must be
+    a bytes-like object).
+    """
+    return __get_builtin_constructor(name)(data, **kwargs)
+
+
+def __hash_new(name, data=b'', **kwargs):
+    """new(name, data=b'') - Return a new hashing object using the named algorithm;
+    optionally initialized with data (which must be a bytes-like object).
+    """
+    if name in __block_openssl_constructor:
+        # Prefer our builtin blake2 implementation.
+        return __get_builtin_constructor(name)(data, **kwargs)
+    try:
+        return _hashlib.new(name, data, **kwargs)
+    except ValueError:
+        # If the _hashlib module (OpenSSL) doesn't support the named
+        # hash, try using our builtin implementations.
+        # This allows for SHA224/256 and SHA384/512 support even though
+        # the OpenSSL library prior to 0.9.8 doesn't provide them.
+        return __get_builtin_constructor(name)(data)
+
+
+try:
+    import _hashlib
+    new = __hash_new
+    __get_hash = __get_openssl_constructor
+    # TODO: RUSTPYTHON set in _hashlib instance PyFrozenSet algorithms_available
+    '''algorithms_available = algorithms_available.union(
+            _hashlib.openssl_md_meth_names)'''
+except ImportError:
+    _hashlib = None
+    new = __py_new
+    __get_hash = __get_builtin_constructor
+
+try:
+    # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
+    from _hashlib import pbkdf2_hmac
+except ImportError:
+    from warnings import warn as _warn
+    _trans_5C = bytes((x ^ 0x5C) for x in range(256))
+    _trans_36 = bytes((x ^ 0x36) for x in range(256))
+
+    def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None):
+        """Password based key derivation function 2 (PKCS #5 v2.0)
+
+        This Python implementations based on the hmac module about as fast
+        as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster
+        for long passwords.
+        """
+        _warn(
+            "Python implementation of pbkdf2_hmac() is deprecated.",
+            category=DeprecationWarning,
+            stacklevel=2
+        )
+        if not isinstance(hash_name, str):
+            raise TypeError(hash_name)
+
+        if not isinstance(password, (bytes, bytearray)):
+            password = bytes(memoryview(password))
+        if not isinstance(salt, (bytes, bytearray)):
+            salt = bytes(memoryview(salt))
+
+        # Fast inline HMAC implementation
+        inner = new(hash_name)
+        outer = new(hash_name)
+        blocksize = getattr(inner, 'block_size', 64)
+        if len(password) > blocksize:
+            password = new(hash_name, password).digest()
+        password = password + b'\x00' * (blocksize - len(password))
+        inner.update(password.translate(_trans_36))
+        outer.update(password.translate(_trans_5C))
+
+        def prf(msg, inner=inner, outer=outer):
+            # PBKDF2_HMAC uses the password as key. We can re-use the same
+            # digest objects and just update copies to skip initialization.
+            icpy = inner.copy()
+            ocpy = outer.copy()
+            icpy.update(msg)
+            ocpy.update(icpy.digest())
+            return ocpy.digest()
+
+        if iterations < 1:
+            raise ValueError(iterations)
+        if dklen is None:
+            dklen = outer.digest_size
+        if dklen < 1:
+            raise ValueError(dklen)
+
+        dkey = b''
+        loop = 1
+        from_bytes = int.from_bytes
+        while len(dkey) < dklen:
+            prev = prf(salt + loop.to_bytes(4))
+            # endianness doesn't matter here as long to / from use the same
+            rkey = from_bytes(prev)
+            for i in range(iterations - 1):
+                prev = prf(prev)
+                # rkey = rkey ^ prev
+                rkey ^= from_bytes(prev)
+            loop += 1
+            dkey += rkey.to_bytes(inner.digest_size)
+
+        return dkey[:dklen]
+
+try:
+    # OpenSSL's scrypt requires OpenSSL 1.1+
+    from _hashlib import scrypt
+except ImportError:
+    pass
+
+
+def file_digest(fileobj, digest, /, *, _bufsize=2**18):
+    """Hash the contents of a file-like object. Returns a digest object.
+
+    *fileobj* must be a file-like object opened for reading in binary mode.
+    It accepts file objects from open(), io.BytesIO(), and SocketIO objects.
+    The function may bypass Python's I/O and use the file descriptor *fileno*
+    directly.
+
+    *digest* must either be a hash algorithm name as a *str*, a hash
+    constructor, or a callable that returns a hash object.
+    """
+    # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy
+    # hashing with hardware acceleration.
+    if isinstance(digest, str):
+        digestobj = new(digest)
+    else:
+        digestobj = digest()
+
+    if hasattr(fileobj, "getbuffer"):
+        # io.BytesIO object, use zero-copy buffer
+        digestobj.update(fileobj.getbuffer())
+        return digestobj
+
+    # Only binary files implement readinto().
+    if not (
+        hasattr(fileobj, "readinto")
+        and hasattr(fileobj, "readable")
+        and fileobj.readable()
+    ):
+        raise ValueError(
+            f"'{fileobj!r}' is not a file-like object in binary reading mode."
+        )
+
+    # binary file, socket.SocketIO object
+    # Note: socket I/O uses different syscalls than file I/O.
+    buf = bytearray(_bufsize)  # Reusable buffer to reduce allocations.
+    view = memoryview(buf)
+    while True:
+        size = fileobj.readinto(buf)
+        if size == 0:
+            break  # EOF
+        digestobj.update(view[:size])
+
+    return digestobj
+
+
+for __func_name in __always_supported:
+    # try them all, some may not work due to the OpenSSL
+    # version not supporting that algorithm.
+    try:
+        globals()[__func_name] = __get_hash(__func_name)
+    except ValueError:
+        import logging
+        logging.exception('code for hash %s was not found.', __func_name)
+
+
+# Cleanup locals()
+del __always_supported, __func_name, __get_hash
+del __py_new, __hash_new, __get_openssl_constructor

Lib/site.py

@@ -275,7 +275,8 @@ def _getuserbase():
 
     if os.name == "nt":
         base = os.environ.get("APPDATA") or "~"
-        return joinuser(base, "Python")
+        # XXX: RUSTPYTHON; please keep this change for site-packages
+        return joinuser(base, "RustPython")
 
     if sys.platform == "darwin" and sys._framework:
         return joinuser("~", "Library", sys._framework,
@@ -368,7 +369,8 @@ def getsitepackages(prefixes=None):
 
             for libdir in libdirs:
                 path = os.path.join(prefix, libdir,
-                                    "python%d.%d" % sys.version_info[:2],
+                                    # XXX: RUSTPYTHON; please keep this change for site-packages
+                                    "rustpython%d.%d" % sys.version_info[:2],
                                     "site-packages")
                 sitepackages.append(path)
         else:

Lib/sysconfig.py

@@ -1,3 +1,5 @@
+# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython
+exec(r'''
 """Access to Python's configuration information."""
 
 import os
@@ -851,10 +853,6 @@ def _main():
     print()
     _print_dict('Variables', get_config_vars())
 
-# XXX RUSTPYTHON: replace python with rustpython in all these paths
-for group in _INSTALL_SCHEMES.values():
-    for key in group.keys():
-        group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython")
-
 if __name__ == '__main__':
     _main()
+'''.replace("Python", "RustPython").replace("/python", "/rustpython"))

Lib/test/test_builtin.py

@@ -37,6 +37,8 @@ try:
 except ImportError:
     pty = signal = None
 
+import threading  # XXX: RUSTPYTHON; to skip _at_fork_reinit
+
 
 class Squares:
 
@@ -2269,6 +2271,7 @@ class PtyTests(unittest.TestCase):
         # Check stdin/stdout error handler is used when invoking PyOS_Readline()
         self.check_input_tty("prompté", b"quux\xe9", "ascii")
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_input_no_stdout_fileno(self):
         # Issue #24402: If stdin is the original terminal but stdout.fileno()
         # fails, do not use the original stdout file descriptor

Lib/test/test_enum.py

@@ -2889,9 +2889,7 @@ class TestIntFlag(unittest.TestCase):
         self.assertEqual(Color.ALL.value, 7)
         self.assertEqual(str(Color.BLUE), 'blue')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading")
+    @unittest.skip("TODO: RUSTPYTHON, inconsistent test result due to threading")
     @threading_helper.reap_threads
     def test_unique_composite(self):
         # override __eq__ to be identity only

Lib/test/test_fcntl.py

@@ -4,6 +4,7 @@ import platform
 import os
 import struct
 import sys
+import threading  # XXX: RUSTPYTHON
 import unittest
 from multiprocessing import Process
 from test.support import verbose, cpython_only
@@ -155,8 +156,9 @@ class TestFcntl(unittest.TestCase):
         self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH)
         self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH)
 
-    # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork'
+    # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_exclusive(self):
         self.f = open(TESTFN, 'wb+')
@@ -167,9 +169,9 @@ class TestFcntl(unittest.TestCase):
         p.join()
         fcntl.lockf(self.f, fcntl.LOCK_UN)
         self.assertEqual(p.exitcode, 0)
-
-    # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork'
+    # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_share(self):
         self.f = open(TESTFN, 'wb+')

Lib/test/test_hashlib.py

@@ -10,6 +10,7 @@ import array
 from binascii import unhexlify
 import hashlib
 import importlib
+import io
 import itertools
 import os
 import sys
@@ -20,6 +21,7 @@ import warnings
 from test import support
 from test.support import _4G, bigmemtest
 from test.support.import_helper import import_fresh_module
+from test.support import os_helper
 from test.support import threading_helper
 from test.support import warnings_helper
 from http.client import HTTPException
@@ -102,8 +104,7 @@ class HashLibTestCase(unittest.TestCase):
                              'sha384', 'SHA384', 'sha512', 'SHA512',
                              'blake2b', 'blake2s',
                              'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-                             'shake_128', 'shake_256'
-                            )
+                             'shake_128', 'shake_256')
 
     shakes = {'shake_128', 'shake_256'}
 
@@ -214,15 +215,11 @@ class HashLibTestCase(unittest.TestCase):
             else:
                 c.hexdigest()
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_algorithms_guaranteed(self):
         self.assertEqual(hashlib.algorithms_guaranteed,
             set(_algo for _algo in self.supported_hash_names
                   if _algo.islower()))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_algorithms_available(self):
         self.assertTrue(set(hashlib.algorithms_guaranteed).
                             issubset(hashlib.algorithms_available))
@@ -262,8 +259,6 @@ class HashLibTestCase(unittest.TestCase):
     def test_new_upper_to_lower(self):
         self.assertEqual(hashlib.new("SHA256").name, "sha256")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_get_builtin_constructor(self):
         get_builtin_constructor = getattr(hashlib,
                                           '__get_builtin_constructor')
@@ -384,6 +379,39 @@ class HashLibTestCase(unittest.TestCase):
             if not shake:
                 self.assertEqual(len(digest), m.digest_size)
 
+        if not shake and kwargs.get("key") is None:
+            # skip shake and blake2 extended parameter tests
+            self.check_file_digest(name, data, hexdigest)
+
+    def check_file_digest(self, name, data, hexdigest):
+        hexdigest = hexdigest.lower()
+        try:
+            hashlib.new(name)
+        except ValueError:
+            # skip, algorithm is blocked by security policy.
+            return
+        digests = [name]
+        digests.extend(self.constructors_to_test[name])
+
+        with open(os_helper.TESTFN, "wb") as f:
+            f.write(data)
+
+        try:
+            for digest in digests:
+                buf = io.BytesIO(data)
+                buf.seek(0)
+                '''self.assertEqual(
+                    dir(hashlib), None
+                )'''
+                self.assertEqual(
+                    hashlib.file_digest(buf, digest).hexdigest(), hexdigest
+                )
+                with open(os_helper.TESTFN, "rb") as f:
+                    digestobj = hashlib.file_digest(f, digest)
+                self.assertEqual(digestobj.hexdigest(), hexdigest)
+        finally:
+            os.unlink(os_helper.TESTFN)
+
     def check_no_unicode(self, algorithm_name):
         # Unicode objects are not allowed as input.
         constructors = self.constructors_to_test[algorithm_name]
@@ -475,6 +503,8 @@ class HashLibTestCase(unittest.TestCase):
         self.check_sha3('shake_128', 256, 1344, b'\x1f')
         self.check_sha3('shake_256', 512, 1088, b'\x1f')
 
+    # TODO: RUSTPYTHON implement all blake2 params
+    @unittest.expectedFailure
     @requires_blake2
     def test_blocksize_name_blake2(self):
         self.check_blocksize_name('blake2b', 128, 64)
@@ -718,6 +748,8 @@ class HashLibTestCase(unittest.TestCase):
                 outer.update(keyed.digest())
         return outer.hexdigest()
 
+    # TODO: RUSTPYTHON add to constructor const value
+    @unittest.expectedFailure
     @requires_blake2
     def test_blake2b(self):
         self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1)
@@ -739,6 +771,8 @@ class HashLibTestCase(unittest.TestCase):
           "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+
           "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923")
 
+    # TODO: RUSTPYTHON implement all blake2 fields
+    @unittest.expectedFailure
     @requires_blake2
     def test_case_blake2b_all_parameters(self):
         # This checks that all the parameters work in general, and also that
@@ -763,6 +797,8 @@ class HashLibTestCase(unittest.TestCase):
             key = bytes.fromhex(key)
             self.check('blake2b', msg, md, key=key)
 
+    # TODO: RUSTPYTHON add to constructor const value
+    @unittest.expectedFailure
     @requires_blake2
     def test_blake2s(self):
         self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1)
@@ -782,6 +818,8 @@ class HashLibTestCase(unittest.TestCase):
         self.check('blake2s', b"abc",
           "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982")
 
+    # TODO: RUSTPYTHON implement all blake2 fields
+    @unittest.expectedFailure
     @requires_blake2
     def test_case_blake2s_all_parameters(self):
         # This checks that all the parameters work in general, and also that
@@ -898,6 +936,7 @@ class HashLibTestCase(unittest.TestCase):
         )
 
     @threading_helper.reap_threads
+    @threading_helper.requires_working_threading()
     def test_threaded_hashing(self):
         # Updating the same hash object from several threads at once
         # using data chunk sizes containing the same byte sequences.
@@ -1136,12 +1175,37 @@ class KDFTests(unittest.TestCase):
                 hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1,
                                dklen=dklen)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_normalized_name(self):
         self.assertNotIn("blake2b512", hashlib.algorithms_available)
         self.assertNotIn("sha3-512", hashlib.algorithms_available)
 
+    def test_file_digest(self):
+        data = b'a' * 65536
+        d1 = hashlib.sha256()
+        self.addCleanup(os.unlink, os_helper.TESTFN)
+        with open(os_helper.TESTFN, "wb") as f:
+            for _ in range(10):
+                d1.update(data)
+                f.write(data)
+
+        with open(os_helper.TESTFN, "rb") as f:
+            d2 = hashlib.file_digest(f, hashlib.sha256)
+
+        self.assertEqual(d1.hexdigest(), d2.hexdigest())
+        self.assertEqual(d1.name, d2.name)
+        self.assertIs(type(d1), type(d2))
+
+        with self.assertRaises(ValueError):
+            hashlib.file_digest(None, "sha256")
+
+        with self.assertRaises(ValueError):
+            with open(os_helper.TESTFN, "r") as f:
+                hashlib.file_digest(f, "sha256")
+
+        with self.assertRaises(ValueError):
+            with open(os_helper.TESTFN, "wb") as f:
+                hashlib.file_digest(f, "sha256")
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_hmac.py

@@ -39,6 +39,8 @@ def ignore_warning(func):
 
 class TestVectorsTestCase(unittest.TestCase):
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def assert_hmac_internals(
             self, h, digest, hashname, digest_size, block_size
     ):
@@ -48,6 +50,8 @@ class TestVectorsTestCase(unittest.TestCase):
         self.assertEqual(h.digest_size, digest_size)
         self.assertEqual(h.block_size, block_size)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def assert_hmac(
         self, key, data, digest, hashfunc, hashname, digest_size, block_size
     ):
@@ -122,6 +126,8 @@ class TestVectorsTestCase(unittest.TestCase):
                 h, digest, hashname, digest_size, block_size
             )
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('md5', openssl=True)
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
@@ -164,6 +170,8 @@ class TestVectorsTestCase(unittest.TestCase):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha1', openssl=True)
     def test_sha_vectors(self):
         def shatest(key, data, digest):
@@ -323,18 +331,26 @@ class TestVectorsTestCase(unittest.TestCase):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha224', openssl=True)
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256', openssl=True)
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha384', openssl=True)
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha512', openssl=True)
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
@@ -380,6 +396,8 @@ class ConstructorTestCase(unittest.TestCase):
         "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb"
     )
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_normal(self):
         # Standard constructor call.
@@ -402,6 +420,8 @@ class ConstructorTestCase(unittest.TestCase):
         with self.assertRaises(TypeError):
             h = hmac.new("key", digestmod='sha256')
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_withtext(self):
         # Constructor call with text.
@@ -411,6 +431,8 @@ class ConstructorTestCase(unittest.TestCase):
             self.fail("Constructor call with text argument raised exception.")
         self.assertEqual(h.hexdigest(), self.expected)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_with_bytearray(self):
         try:
@@ -420,6 +442,8 @@ class ConstructorTestCase(unittest.TestCase):
             self.fail("Constructor call with bytearray arguments raised exception.")
         self.assertEqual(h.hexdigest(), self.expected)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_with_memoryview_msg(self):
         try:
@@ -428,6 +452,8 @@ class ConstructorTestCase(unittest.TestCase):
             self.fail("Constructor call with memoryview msg raised exception.")
         self.assertEqual(h.hexdigest(), self.expected)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_withmodule(self):
         # Constructor call with text and digest module.
@@ -436,6 +462,8 @@ class ConstructorTestCase(unittest.TestCase):
         except Exception:
             self.fail("Constructor call with hashlib.sha256 raised exception.")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @unittest.skipUnless(C_HMAC is not None, 'need _hashlib')
     def test_internal_types(self):
         # internal types like _hashlib.C_HMAC are not constructable
@@ -443,6 +471,8 @@ class ConstructorTestCase(unittest.TestCase):
         with self.assertRaisesRegex(TypeError, "immutable type"):
             C_HMAC.value = None
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @unittest.skipUnless(sha256_module is not None, 'need _sha256')
     def test_with_sha256_module(self):
         h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256)
@@ -455,6 +485,8 @@ class ConstructorTestCase(unittest.TestCase):
 
 class SanityTestCase(unittest.TestCase):
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_exercise_all_methods(self):
         # Exercising all methods once.
@@ -496,6 +528,8 @@ class CopyTestCase(unittest.TestCase):
             "No real copy of the attribute 'outer'.")
         self.assertIs(h1._hmac, None)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl")
     @hashlib_helper.requires_hashdigest('sha256')
     def test_realcopy_hmac(self):
@@ -504,6 +538,8 @@ class CopyTestCase(unittest.TestCase):
         h2 = h1.copy()
         self.assertTrue(id(h1._hmac) != id(h2._hmac))
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_equality(self):
         # Testing if the copy has the same digests.
@@ -515,6 +551,8 @@ class CopyTestCase(unittest.TestCase):
         self.assertEqual(h1.hexdigest(), h2.hexdigest(),
             "Hexdigest of copy doesn't match original hexdigest.")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @hashlib_helper.requires_hashdigest('sha256')
     def test_equality_new(self):
         # Testing if the copy has the same digests with hmac.new().
@@ -532,6 +570,8 @@ class CopyTestCase(unittest.TestCase):
 
 class CompareDigestTestCase(unittest.TestCase):
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_hmac_compare_digest(self):
         self._test_compare_digest(hmac.compare_digest)
         if openssl_compare_digest is not None:
@@ -542,6 +582,8 @@ class CompareDigestTestCase(unittest.TestCase):
     def test_operator_compare_digest(self):
         self._test_compare_digest(operator_compare_digest)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     @unittest.skipIf(openssl_compare_digest is None, "test requires _hashlib")
     def test_openssl_compare_digest(self):
         self._test_compare_digest(openssl_compare_digest)

Lib/test/test_httpservers.py

@@ -607,7 +607,8 @@ for k, v in os.environ.items():
 print("</pre>")
 """
 
-
+@unittest.skipIf(not hasattr(os, '_exit'),
+        "TODO: RUSTPYTHON, run_cgi in http/server.py gets stuck as os._exit(127) doesn't currently kill forked processes")
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
 class CGIHTTPServerTestCase(BaseTestCase):
@@ -765,6 +766,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
                                  msg='path = %r\nGot:    %r\nWanted: %r' %
                                  (path, actual, expected))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_headers_and_content(self):
         res = self.request('/cgi-bin/file1.py')
         self.assertEqual(
@@ -775,6 +777,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
         res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh')
         self.assertEqual(res.status, HTTPStatus.NOT_FOUND)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_post(self):
         params = urllib.parse.urlencode(
             {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456})
@@ -788,6 +791,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
         res.read()
         self.assertEqual(res.status, HTTPStatus.NOT_FOUND)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_authorization(self):
         headers = {b'Authorization' : b'Basic ' +
                    base64.b64encode(b'username:pass')}
@@ -796,6 +800,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_no_leading_slash(self):
         # http://bugs.python.org/issue2254
         res = self.request('cgi-bin/file1.py')
@@ -803,6 +808,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_os_environ_is_not_altered(self):
         signature = "Test CGI Server"
         os.environ['SERVER_SOFTWARE'] = signature
@@ -812,24 +818,28 @@ class CGIHTTPServerTestCase(BaseTestCase):
             (res.read(), res.getheader('Content-type'), res.status))
         self.assertEqual(os.environ['SERVER_SOFTWARE'], signature)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_urlquote_decoding_in_cgi_check(self):
         res = self.request('/cgi-bin%2ffile1.py')
         self.assertEqual(
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_nested_cgi_path_issue21323(self):
         res = self.request('/cgi-bin/child-dir/file3.py')
         self.assertEqual(
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_query_with_multiple_question_mark(self):
         res = self.request('/cgi-bin/file4.py?a=b?c=d')
         self.assertEqual(
             (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_query_with_continuous_slashes(self):
         res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
         self.assertEqual(
@@ -837,6 +847,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
              'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_cgi_path_in_sub_directories(self):
         try:
             CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin')
@@ -847,6 +858,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
         finally:
             CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin')
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_accept(self):
         browser_accept = \
                     'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

Lib/test/test_importlib/test_threaded_import.py

@@ -6,6 +6,7 @@
 # randrange, and then Python hangs.
 
 import _imp as imp
+import _multiprocessing  # TODO: RUSTPYTHON
 import os
 import importlib
 import sys
@@ -260,6 +261,7 @@ class ThreadedImportTests(unittest.TestCase):
                           'partial', 'cfimport.py')
         script_helper.assert_python_ok(fn)
 
+    @unittest.skipUnless(hasattr(_multiprocessing, "SemLock"), "TODO: RUSTPYTHON, pool_in_threads.py needs _multiprocessing.SemLock")                                                                                      
     @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
     def test_multiprocessing_pool_circular_import(self):
         # Regression test for bpo-41567

Lib/test/test_os.py

@@ -69,6 +69,10 @@ try:
 except ImportError:
     INT_MAX = PY_SSIZE_T_MAX = sys.maxsize
 
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 from test.support.script_helper import assert_python_ok
 from test.support import unix_shell
@@ -3067,11 +3071,13 @@ class PidTests(unittest.TestCase):
         self.assertEqual(pid2, pid)
 
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_waitpid(self):
         self.check_waitpid(code='pass', exitcode=0)
 
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_waitstatus_to_exitcode(self):
         exitcode = 23
@@ -3103,7 +3109,8 @@ class PidTests(unittest.TestCase):
             os.waitstatus_to_exitcode((max_exitcode + 1) << 8)
         with self.assertRaises(OverflowError):
             os.waitstatus_to_exitcode(-1)
-
+            
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
     @unittest.expectedFailure
     # Skip the test on Windows
@@ -3146,31 +3153,36 @@ class SpawnTests(unittest.TestCase):
                         for k, v in self.env.items()}
 
         return args
-
+    
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnl')
     def test_spawnl(self):
         args = self.create_args()
         exitcode = os.spawnl(os.P_WAIT, args[0], *args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnle')
     def test_spawnle(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnlp')
     def test_spawnlp(self):
         args = self.create_args()
         exitcode = os.spawnlp(os.P_WAIT, args[0], *args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnlpe')
     def test_spawnlpe(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnv')
     def test_spawnv(self):
         args = self.create_args()
@@ -3181,30 +3193,35 @@ class SpawnTests(unittest.TestCase):
         exitcode = os.spawnv(os.P_WAIT, FakePath(args[0]), args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvp')
     def test_spawnvp(self):
         args = self.create_args()
         exitcode = os.spawnvp(os.P_WAIT, args[0], args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvpe')
     def test_spawnvpe(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnv')
     def test_nowait(self):
         args = self.create_args()
         pid = os.spawnv(os.P_NOWAIT, args[0], args)
         support.wait_process(pid, exitcode=self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve_bytes(self):
         # Test bytes handling in parse_arglist and parse_envlist (#28114)
@@ -3286,10 +3303,12 @@ class SpawnTests(unittest.TestCase):
         exitcode = spawn(os.P_WAIT, args[0], args, newenv)
         self.assertEqual(exitcode, 0)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve_invalid_env(self):
         self._test_invalid_env(os.spawnve)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvpe')
     def test_spawnvpe_invalid_env(self):
         self._test_invalid_env(os.spawnvpe)
@@ -4660,6 +4679,35 @@ class ForkTests(unittest.TestCase):
         assert_python_ok("-c", code)
         assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
 
+    @unittest.skipIf(_testcapi is None, 'TODO: RUSTPYTHON; needs _testcapi')
+    @unittest.skipUnless(sys.platform in ("linux", "darwin"),
+                         "Only Linux and macOS detect this today.")
+    def test_fork_warns_when_non_python_thread_exists(self):
+        code = """if 1:
+            import os, threading, warnings
+            from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread
+            _spawn_pthread_waiter()
+            try:
+                with warnings.catch_warnings(record=True) as ws:
+                    warnings.filterwarnings(
+                            "always", category=DeprecationWarning)
+                    if os.fork() == 0:
+                        assert not ws, f"unexpected warnings in child: {ws}"
+                        os._exit(0)  # child
+                    else:
+                        assert ws[0].category == DeprecationWarning, ws[0]
+                        assert 'fork' in str(ws[0].message), ws[0]
+                        # Waiting allows an error in the child to hit stderr.
+                        exitcode = os.wait()[1]
+                        assert exitcode == 0, f"child exited {exitcode}"
+                assert threading.active_count() == 1, threading.enumerate()
+            finally:
+                _end_spawned_pthread()
+        """
+        _, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0')
+        self.assertEqual(err.decode("utf-8"), "")
+        self.assertEqual(out.decode("utf-8"), "")
+
 
 # Only test if the C version is provided, otherwise TestPEP519 already tested
 # the pure Python implementation.

Lib/test/test_platform.py

@@ -346,8 +346,6 @@ class PlatformTest(unittest.TestCase):
             else:
                 self.assertEqual(res[2], 'PowerPC')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @unittest.skipUnless(sys.platform == 'darwin', "OSX only test")
     def test_mac_ver_with_fork(self):
         # Issue7895: platform.mac_ver() crashes when using fork without exec

Lib/test/test_poll.py

@@ -0,0 +1,240 @@
+# Test case for the os.poll() function
+
+import os
+import subprocess
+import random
+import select
+import threading
+import time
+import unittest
+from test.support import (
+    cpython_only, requires_subprocess, requires_working_socket
+)
+from test.support import threading_helper
+from test.support.os_helper import TESTFN
+
+
+try:
+    select.poll
+except AttributeError:
+    raise unittest.SkipTest("select.poll not defined")
+
+requires_working_socket(module=True)
+
+def find_ready_matching(ready, flag):
+    match = []
+    for fd, mode in ready:
+        if mode & flag:
+            match.append(fd)
+    return match
+
+class PollTests(unittest.TestCase):
+
+    def test_poll1(self):
+        # Basic functional test of poll object
+        # Create a bunch of pipe and test that poll works with them.
+
+        p = select.poll()
+
+        NUM_PIPES = 12
+        MSG = b" This is a test."
+        MSG_LEN = len(MSG)
+        readers = []
+        writers = []
+        r2w = {}
+        w2r = {}
+
+        for i in range(NUM_PIPES):
+            rd, wr = os.pipe()
+            p.register(rd)
+            p.modify(rd, select.POLLIN)
+            p.register(wr, select.POLLOUT)
+            readers.append(rd)
+            writers.append(wr)
+            r2w[rd] = wr
+            w2r[wr] = rd
+
+        bufs = []
+
+        while writers:
+            ready = p.poll()
+            ready_writers = find_ready_matching(ready, select.POLLOUT)
+            if not ready_writers:
+                raise RuntimeError("no pipes ready for writing")
+            wr = random.choice(ready_writers)
+            os.write(wr, MSG)
+
+            ready = p.poll()
+            ready_readers = find_ready_matching(ready, select.POLLIN)
+            if not ready_readers:
+                raise RuntimeError("no pipes ready for reading")
+            rd = random.choice(ready_readers)
+            buf = os.read(rd, MSG_LEN)
+            self.assertEqual(len(buf), MSG_LEN)
+            bufs.append(buf)
+            os.close(r2w[rd]) ; os.close( rd )
+            p.unregister( r2w[rd] )
+            p.unregister( rd )
+            writers.remove(r2w[rd])
+
+        self.assertEqual(bufs, [MSG] * NUM_PIPES)
+
+    def test_poll_unit_tests(self):
+        # returns NVAL for invalid file descriptor
+        FD, w = os.pipe()
+        os.close(FD)
+        os.close(w)
+        p = select.poll()
+        p.register(FD)
+        r = p.poll()
+        self.assertEqual(r[0], (FD, select.POLLNVAL))
+
+        with open(TESTFN, 'w') as f:
+            fd = f.fileno()
+            p = select.poll()
+            p.register(f)
+            r = p.poll()
+            self.assertEqual(r[0][0], fd)
+        r = p.poll()
+        self.assertEqual(r[0], (fd, select.POLLNVAL))
+        os.unlink(TESTFN)
+
+        # type error for invalid arguments
+        p = select.poll()
+        self.assertRaises(TypeError, p.register, p)
+        self.assertRaises(TypeError, p.unregister, p)
+
+        # can't unregister non-existent object
+        p = select.poll()
+        self.assertRaises(KeyError, p.unregister, 3)
+
+        # Test error cases
+        pollster = select.poll()
+        class Nope:
+            pass
+
+        class Almost:
+            def fileno(self):
+                return 'fileno'
+
+        self.assertRaises(TypeError, pollster.register, Nope(), 0)
+        self.assertRaises(TypeError, pollster.register, Almost(), 0)
+
+    # Another test case for poll().  This is copied from the test case for
+    # select(), modified to use poll() instead.
+
+    @requires_subprocess()
+    def test_poll2(self):
+        cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done'
+        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                                bufsize=0)
+        self.enterContext(proc)
+        p = proc.stdout
+        pollster = select.poll()
+        pollster.register( p, select.POLLIN )
+        for tout in (0, 1000, 2000, 4000, 8000, 16000) + (-1,)*10:
+            fdlist = pollster.poll(tout)
+            if (fdlist == []):
+                continue
+            fd, flags = fdlist[0]
+            if flags & select.POLLHUP:
+                line = p.readline()
+                if line != b"":
+                    self.fail('error: pipe seems to be closed, but still returns data')
+                continue
+
+            elif flags & select.POLLIN:
+                line = p.readline()
+                if not line:
+                    break
+                self.assertEqual(line, b'testing...\n')
+                continue
+            else:
+                self.fail('Unexpected return value from select.poll: %s' % fdlist)
+
+    # TODO: RUSTPYTHON int overflow
+    @unittest.expectedFailure
+    def test_poll3(self):
+        # test int overflow
+        pollster = select.poll()
+        pollster.register(1)
+
+        self.assertRaises(OverflowError, pollster.poll, 1 << 64)
+
+        x = 2 + 3
+        if x != 5:
+            self.fail('Overflow must have occurred')
+
+        # Issues #15989, #17919
+        self.assertRaises(ValueError, pollster.register, 0, -1)
+        self.assertRaises(OverflowError, pollster.register, 0, 1 << 64)
+        self.assertRaises(ValueError, pollster.modify, 1, -1)
+        self.assertRaises(OverflowError, pollster.modify, 1, 1 << 64)
+
+    @cpython_only
+    def test_poll_c_limits(self):
+        from _testcapi import USHRT_MAX, INT_MAX, UINT_MAX
+        pollster = select.poll()
+        pollster.register(1)
+
+        # Issues #15989, #17919
+        self.assertRaises(OverflowError, pollster.register, 0, USHRT_MAX + 1)
+        self.assertRaises(OverflowError, pollster.modify, 1, USHRT_MAX + 1)
+        self.assertRaises(OverflowError, pollster.poll, INT_MAX + 1)
+        self.assertRaises(OverflowError, pollster.poll, UINT_MAX + 1)
+
+    @unittest.skip("TODO: RUSTPYTHON fd reallocation")
+    @threading_helper.reap_threads
+    def test_threaded_poll(self):
+        r, w = os.pipe()
+        self.addCleanup(os.close, r)
+        self.addCleanup(os.close, w)
+        rfds = []
+        for i in range(10):
+            fd = os.dup(r)
+            self.addCleanup(os.close, fd)
+            rfds.append(fd)
+        pollster = select.poll()
+        for fd in rfds:
+            pollster.register(fd, select.POLLIN)
+        t = threading.Thread(target=pollster.poll)
+        t.start()
+        try:
+            time.sleep(0.5)
+            # trigger ufds array reallocation
+            for fd in rfds:
+                pollster.unregister(fd)
+            pollster.register(w, select.POLLOUT)
+            self.assertRaises(RuntimeError, pollster.poll)
+        finally:
+            # and make the call to poll() from the thread return
+            os.write(w, b'spam')
+            t.join()
+
+    # TODO: RUSTPYTHON add support for negative timeout
+    @unittest.expectedFailure    
+    @unittest.skipUnless(threading, 'Threading required for this test.')
+    @threading_helper.reap_threads
+    def test_poll_blocks_with_negative_ms(self):
+        for timeout_ms in [None, -1000, -1, -1.0, -0.1, -1e-100]:
+            # Create two file descriptors. This will be used to unlock
+            # the blocking call to poll.poll inside the thread
+            r, w = os.pipe()
+            pollster = select.poll()
+            pollster.register(r, select.POLLIN)
+
+            poll_thread = threading.Thread(target=pollster.poll, args=(timeout_ms,))
+            poll_thread.start()
+            poll_thread.join(timeout=0.1)
+            self.assertTrue(poll_thread.is_alive())
+
+            # Write to the pipe so pollster.poll unblocks and the thread ends.
+            os.write(w, b'spam')
+            poll_thread.join()
+            self.assertFalse(poll_thread.is_alive())
+            os.close(r)
+            os.close(w)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_property.py

@@ -226,6 +226,23 @@ class PropertyTests(unittest.TestCase):
             ):
                 p.__set_name__(*([0] * i))
 
+    def test_property_setname_on_property_subclass(self):
+        # https://github.com/python/cpython/issues/100942
+        # Copy was setting the name field without first
+        # verifying that the copy was an actual property
+        # instance.  As a result, the code below was
+        # causing a segfault.
+
+        class pro(property):
+            def __new__(typ, *args, **kwargs):
+                return "abcdef"
+
+        class A:
+            pass
+
+        p = property.__new__(pro)
+        p.__set_name__(A, 1)
+        np = p.getter(lambda self: 1)
 
 # Issue 5890: subclasses of property do not preserve method __doc__ strings
 class PropertySub(property):
@@ -341,43 +358,35 @@ class _PropertyUnreachableAttribute:
     def setUpClass(cls):
         cls.obj = cls.cls()
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_get_property(self):
-        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("unreadable attribute")):
+        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no getter")):
             self.obj.foo
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_set_property(self):
-        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("can't set attribute")):
+        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no setter")):
             self.obj.foo = None
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_del_property(self):
-        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("can't delete attribute")):
+        with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no deleter")):
             del self.obj.foo
 
 
 class PropertyUnreachableAttributeWithName(_PropertyUnreachableAttribute, unittest.TestCase):
-    msg_format = "^{} 'foo'$"
+    msg_format = r"^property 'foo' of 'PropertyUnreachableAttributeWithName\.cls' object {}$"
 
     class cls:
         foo = property()
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    def test_get_property(self):  # TODO: RUSTPYTHON; remove this function when the test is fixed
-        super().test_get_property()
-
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    def test_set_property(self):  # TODO: RUSTPYTHON; remove this function when the test is fixed
-        super().test_get_property()
-
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    def test_del_property(self):  # TODO: RUSTPYTHON; remove this function when the test is fixed
-        super().test_get_property()
 
 
 class PropertyUnreachableAttributeNoName(_PropertyUnreachableAttribute, unittest.TestCase):
-    msg_format = "^{}$"
+    msg_format = r"^property of 'PropertyUnreachableAttributeNoName\.cls' object {}$"
 
     class cls:
         pass

Lib/test/test_pty.py

@@ -1,3 +1,4 @@
+import threading  # XXX: RUSTPYTHON
 from test.support import verbose, reap_children
 from test.support.import_helper import import_module
 
@@ -211,6 +212,7 @@ class PtyTest(unittest.TestCase):
         self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2))
 
     # TODO: RUSTPYTHON
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_fork(self):
         debug("calling pty.fork()")
@@ -314,6 +316,7 @@ class PtyTest(unittest.TestCase):
         self.assertEqual(data, b"")
 
     # TODO: RUSTPYTHON; no os.fork
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_spawn_doesnt_hang(self):
         pty.spawn([sys.executable, '-c', 'print("hi there")'])

Lib/test/test_pulldom.py

@@ -0,0 +1,356 @@
+import io
+import unittest
+import xml.sax
+
+from xml.sax.xmlreader import AttributesImpl
+from xml.sax.handler import feature_external_ges
+from xml.dom import pulldom
+
+from test.support import findfile
+
+
+tstfile = findfile("test.xml", subdir="xmltestdata")
+
+# A handy XML snippet, containing attributes, a namespace prefix, and a
+# self-closing tag:
+SMALL_SAMPLE = """<?xml version="1.0"?>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books">
+<!-- A comment -->
+<title>Introduction to XSL</title>
+<hr/>
+<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p>
+</html>"""
+
+
+class PullDOMTestCase(unittest.TestCase):
+    # TODO: RUSTPYTHON FileNotFoundError: [Errno 2] No such file or directory (os error 2): 'xmltestdata/test.xml' -> 'None'
+    @unittest.expectedFailure
+    def test_parse(self):
+        """Minimal test of DOMEventStream.parse()"""
+
+        # This just tests that parsing from a stream works. Actual parser
+        # semantics are tested using parseString with a more focused XML
+        # fragment.
+
+        # Test with a filename:
+        handler = pulldom.parse(tstfile)
+        self.addCleanup(handler.stream.close)
+        list(handler)
+
+        # Test with a file object:
+        with open(tstfile, "rb") as fin:
+            list(pulldom.parse(fin))
+
+    # TODO: RUSTPYTHON implement DOM semantic
+    @unittest.expectedFailure
+    def test_parse_semantics(self):
+        """Test DOMEventStream parsing semantics."""
+
+        items = pulldom.parseString(SMALL_SAMPLE)
+        evt, node = next(items)
+        # Just check the node is a Document:
+        self.assertTrue(hasattr(node, "createElement"))
+        self.assertEqual(pulldom.START_DOCUMENT, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+        self.assertEqual(2, len(node.attributes))
+        self.assertEqual(node.attributes.getNamedItem("xmlns:xdc").value,
+              "http://www.xml.com/books")
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt) # Line break
+        evt, node = next(items)
+        # XXX - A comment should be reported here!
+        # self.assertEqual(pulldom.COMMENT, evt)
+        # Line break after swallowed comment:
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual("title", node.tagName)
+        title_node = node
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        self.assertEqual("Introduction to XSL", node.data)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("title", node.tagName)
+        self.assertTrue(title_node is node)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("hr", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("hr", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("xdc:author", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("xdc:author", node.tagName)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        evt, node = next(items)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        # XXX No END_DOCUMENT item is ever obtained:
+        #evt, node = next(items)
+        #self.assertEqual(pulldom.END_DOCUMENT, evt)
+
+    # TODO: RUSTPYTHON pulldom.parseString(SMALL_SAMPLE) return iterator with tuple with 2 elements
+    @unittest.expectedFailure
+    def test_expandItem(self):
+        """Ensure expandItem works as expected."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        # Loop through the nodes until we get to a "title" start tag:
+        for evt, item in items:
+            if evt == pulldom.START_ELEMENT and item.tagName == "title":
+                items.expandNode(item)
+                self.assertEqual(1, len(item.childNodes))
+                break
+        else:
+            self.fail("No \"title\" element detected in SMALL_SAMPLE!")
+        # Loop until we get to the next start-element:
+        for evt, node in items:
+            if evt == pulldom.START_ELEMENT:
+                break
+        self.assertEqual("hr", node.tagName,
+            "expandNode did not leave DOMEventStream in the correct state.")
+        # Attempt to expand a standalone element:
+        items.expandNode(node)
+        self.assertEqual(next(items)[0], pulldom.CHARACTERS)
+        evt, node = next(items)
+        self.assertEqual(node.tagName, "p")
+        items.expandNode(node)
+        next(items) # Skip character data
+        evt, node = next(items)
+        self.assertEqual(node.tagName, "html")
+        with self.assertRaises(StopIteration):
+            next(items)
+        items.clear()
+        self.assertIsNone(items.parser)
+        self.assertIsNone(items.stream)
+
+    @unittest.expectedFailure
+    def test_comment(self):
+        """PullDOM does not receive "comment" events."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        for evt, _ in items:
+            if evt == pulldom.COMMENT:
+                break
+        else:
+            self.fail("No comment was encountered")
+
+    @unittest.expectedFailure
+    def test_end_document(self):
+        """PullDOM does not receive "end-document" events."""
+        items = pulldom.parseString(SMALL_SAMPLE)
+        # Read all of the nodes up to and including </html>:
+        for evt, node in items:
+            if evt == pulldom.END_ELEMENT and node.tagName == "html":
+                break
+        try:
+            # Assert that the next node is END_DOCUMENT:
+            evt, node = next(items)
+            self.assertEqual(pulldom.END_DOCUMENT, evt)
+        except StopIteration:
+            self.fail(
+                "Ran out of events, but should have received END_DOCUMENT")
+
+    def test_external_ges_default(self):
+        parser = pulldom.parseString(SMALL_SAMPLE)
+        saxparser = parser.parser
+        ges = saxparser.getFeature(feature_external_ges)
+        self.assertEqual(ges, False)
+
+
+class ThoroughTestCase(unittest.TestCase):
+    """Test the hard-to-reach parts of pulldom."""
+
+    def test_thorough_parse(self):
+        """Test some of the hard-to-reach parts of PullDOM."""
+        self._test_thorough(pulldom.parse(None, parser=SAXExerciser()))
+
+    @unittest.expectedFailure
+    def test_sax2dom_fail(self):
+        """SAX2DOM can"t handle a PI before the root element."""
+        pd = SAX2DOMTestHelper(None, SAXExerciser(), 12)
+        self._test_thorough(pd)
+
+    def test_thorough_sax2dom(self):
+        """Test some of the hard-to-reach parts of SAX2DOM."""
+        pd = SAX2DOMTestHelper(None, SAX2DOMExerciser(), 12)
+        self._test_thorough(pd, False)
+
+    def _test_thorough(self, pd, before_root=True):
+        """Test some of the hard-to-reach parts of the parser, using a mock
+        parser."""
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_DOCUMENT, evt)
+        # Just check the node is a Document:
+        self.assertTrue(hasattr(node, "createElement"))
+
+        if before_root:
+            evt, node = next(pd)
+            self.assertEqual(pulldom.COMMENT, evt)
+            self.assertEqual("a comment", node.data)
+            evt, node = next(pd)
+            self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt)
+            self.assertEqual("target", node.target)
+            self.assertEqual("data", node.data)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.COMMENT, evt)
+        self.assertEqual("a comment", node.data)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt)
+        self.assertEqual("target", node.target)
+        self.assertEqual("data", node.data)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.START_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+
+        evt, node = next(pd)
+        self.assertEqual(pulldom.CHARACTERS, evt)
+        self.assertEqual("text", node.data)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("p", node.tagName)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_ELEMENT, evt)
+        self.assertEqual("html", node.tagName)
+        evt, node = next(pd)
+        self.assertEqual(pulldom.END_DOCUMENT, evt)
+
+
+class SAXExerciser(object):
+    """A fake sax parser that calls some of the harder-to-reach sax methods to
+    ensure it emits the correct events"""
+
+    def setContentHandler(self, handler):
+        self._handler = handler
+
+    def parse(self, _):
+        h = self._handler
+        h.startDocument()
+
+        # The next two items ensure that items preceding the first
+        # start_element are properly stored and emitted:
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+
+        h.startElement("html", AttributesImpl({}))
+
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+
+        h.startElement("p", AttributesImpl({"class": "paraclass"}))
+        h.characters("text")
+        h.endElement("p")
+        h.endElement("html")
+        h.endDocument()
+
+    def stub(self, *args, **kwargs):
+        """Stub method. Does nothing."""
+        pass
+    setProperty = stub
+    setFeature = stub
+
+
+class SAX2DOMExerciser(SAXExerciser):
+    """The same as SAXExerciser, but without the processing instruction and
+    comment before the root element, because S2D can"t handle it"""
+
+    def parse(self, _):
+        h = self._handler
+        h.startDocument()
+        h.startElement("html", AttributesImpl({}))
+        h.comment("a comment")
+        h.processingInstruction("target", "data")
+        h.startElement("p", AttributesImpl({"class": "paraclass"}))
+        h.characters("text")
+        h.endElement("p")
+        h.endElement("html")
+        h.endDocument()
+
+
+class SAX2DOMTestHelper(pulldom.DOMEventStream):
+    """Allows us to drive SAX2DOM from a DOMEventStream."""
+
+    def reset(self):
+        self.pulldom = pulldom.SAX2DOM()
+        # This content handler relies on namespace support
+        self.parser.setFeature(xml.sax.handler.feature_namespaces, 1)
+        self.parser.setContentHandler(self.pulldom)
+
+
+class SAX2DOMTestCase(unittest.TestCase):
+
+    def confirm(self, test, testname="Test"):
+        self.assertTrue(test, testname)
+
+    # TODO: RUSTPYTHON read from stream io
+    @unittest.expectedFailure
+    def test_basic(self):
+        """Ensure SAX2DOM can parse from a stream."""
+        with io.StringIO(SMALL_SAMPLE) as fin:
+            sd = SAX2DOMTestHelper(fin, xml.sax.make_parser(),
+                                   len(SMALL_SAMPLE))
+            for evt, node in sd:
+                if evt == pulldom.START_ELEMENT and node.tagName == "html":
+                    break
+            # Because the buffer is the same length as the XML, all the
+            # nodes should have been parsed and added:
+            self.assertGreater(len(node.childNodes), 0)
+
+    def testSAX2DOM(self):
+        """Ensure SAX2DOM expands nodes as expected."""
+        sax2dom = pulldom.SAX2DOM()
+        sax2dom.startDocument()
+        sax2dom.startElement("doc", {})
+        sax2dom.characters("text")
+        sax2dom.startElement("subelm", {})
+        sax2dom.characters("text")
+        sax2dom.endElement("subelm")
+        sax2dom.characters("text")
+        sax2dom.endElement("doc")
+        sax2dom.endDocument()
+
+        doc = sax2dom.document
+        root = doc.documentElement
+        (text1, elm1, text2) = root.childNodes
+        text3 = elm1.childNodes[0]
+
+        self.assertIsNone(text1.previousSibling)
+        self.assertIs(text1.nextSibling, elm1)
+        self.assertIs(elm1.previousSibling, text1)
+        self.assertIs(elm1.nextSibling, text2)
+        self.assertIs(text2.previousSibling, elm1)
+        self.assertIsNone(text2.nextSibling)
+        self.assertIsNone(text3.previousSibling)
+        self.assertIsNone(text3.nextSibling)
+
+        self.assertIs(root.parentNode, doc)
+        self.assertIs(text1.parentNode, root)
+        self.assertIs(elm1.parentNode, root)
+        self.assertIs(text2.parentNode, root)
+        self.assertIs(text3.parentNode, elm1)
+        doc.unlink()
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_site.py

@@ -288,13 +288,15 @@ class HelperFunctionsTests(unittest.TestCase):
             if sys.platlibdir != "lib":
                 self.assertEqual(len(dirs), 2)
                 wanted = os.path.join('xoxo', sys.platlibdir,
-                                      'python%d.%d' % sys.version_info[:2],
+                                      # XXX: RUSTPYTHON
+                                      'rustpython%d.%d' % sys.version_info[:2],
                                       'site-packages')
                 self.assertEqual(dirs[0], wanted)
             else:
                 self.assertEqual(len(dirs), 1)
             wanted = os.path.join('xoxo', 'lib',
-                                  'python%d.%d' % sys.version_info[:2],
+                                  # XXX: RUSTPYTHON
+                                  'rustpython%d.%d' % sys.version_info[:2],
                                   'site-packages')
             self.assertEqual(dirs[-1], wanted)
         else:

Lib/test/test_socketserver.py

@@ -55,7 +55,7 @@ if HAVE_UNIX_SOCKETS and HAVE_FORKING:
                                     socketserver.UnixDatagramServer):
         pass
 
-
+@test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems
 @contextlib.contextmanager
 def simple_subprocess(testcase):
     """Tests that a custom child process is not waited on (Issue 1540386)"""

Lib/test/test_sqlite3/test_hooks.py

@@ -36,8 +36,6 @@ class CollationTests(unittest.TestCase):
         with self.assertRaises(TypeError):
             con.create_collation(None, lambda x, y: (x > y) - (x < y))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_create_collation_not_callable(self):
         con = sqlite.connect(":memory:")
         with self.assertRaises(TypeError) as cm:

Lib/test/test_support.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+import threading  # XXX: RUSTPYTHON
 import time
 import unittest
 import warnings
@@ -453,6 +454,7 @@ class TestSupport(unittest.TestCase):
 
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'),
                          'need os.waitpid() and os.WNOHANG')
     @support.requires_fork()

Lib/test/test_tempfile.py

@@ -6,6 +6,7 @@ import os
 import pathlib
 import sys
 import re
+import threading  # XXX: RUSTPYTHON; to check `_at_fork_reinit`
 import warnings
 import contextlib
 import stat
@@ -198,6 +199,7 @@ class TestRandomNameSequence(BaseTestCase):
             if i == 20:
                 break
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'),
         "os.fork is required for this test")
     def test_process_awareness(self):
@@ -465,6 +467,7 @@ class TestMkstempInner(TestBadTempdir, BaseTestCase):
             expected = user * (1 + 8 + 64)
         self.assertEqual(mode, expected)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(has_spawnl, 'os.spawnl not available')
     def test_noinherit(self):
         # _mkstemp_inner file handles are not inherited by child processes

Lib/test/test_threading.py

@@ -509,6 +509,7 @@ class ThreadTests(BaseTestCase):
         t = threading.Thread(daemon=True)
         self.assertTrue(t.daemon)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, exit_handler needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()')
     def test_fork_at_exit(self):
         # bpo-42350: Calling os.fork() after threading._shutdown() must
@@ -537,6 +538,7 @@ class ThreadTests(BaseTestCase):
         self.assertEqual(out, b'')
         self.assertEqual(err.rstrip(), b'child process ok')
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()')
     def test_dummy_thread_after_fork(self):
         # Issue #14308: a dummy thread in the active list doesn't mess up
@@ -564,6 +566,7 @@ class ThreadTests(BaseTestCase):
         self.assertEqual(out, b'')
         self.assertEqual(err, b'')
 
+    @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "TODO: RUSTPYTHON, needs sys.getswitchinterval()")
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
     def test_is_alive_after_fork(self):
         # Try hard to trigger #18418: is_alive() could sometimes be True on
@@ -598,6 +601,7 @@ class ThreadTests(BaseTestCase):
         th.start()
         th.join()
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()")
     @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()")
     def test_main_thread_after_fork(self):
@@ -1003,6 +1007,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
             """
         self._run_and_join(script)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
     @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
     def test_2_join_in_forked_process(self):
@@ -1024,6 +1029,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
             """
         self._run_and_join(script)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
     @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
     def test_3_join_in_forked_from_thread(self):
@@ -1094,6 +1100,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
         rc, out, err = assert_python_ok('-c', script)
         self.assertFalse(err)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
     @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
     def test_reinit_tls_after_fork(self):
@@ -1118,6 +1125,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
         for t in threads:
             t.join()
 
+    @unittest.skipUnless(hasattr(sys, '_current_frames'), "TODO: RUSTPYTHON, needs sys._current_frames()")
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
     def test_clear_threads_states_after_fork(self):
         # Issue #17094: check that threads states are cleared after fork()

Lib/test/test_uuid.py

@@ -1,3 +1,4 @@
+import threading  # XXX: RUSTPYTHON; to check `_at_fork_reinit`
 import unittest
 from test import support
 from test.support import import_helper
@@ -641,7 +642,8 @@ class BaseTestUUID:
 
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
+    @support.requires_fork()
     def testIssue8621(self):
         # On at least some versions of OSX self.uuid.uuid4 generates
         # the same sequence of UUIDs in the parent and any

Lib/test/xmltestdata/test.xml

@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<HTML xmlns:pp="http://www.isogen.com/paul/post-processor">
+<TITLE>Introduction to XSL</TITLE>
+<H1>Introduction to XSL</H1>
+	
+
+	
+		<HR/>
+		<H2>Overview
+</H2>
+		<UL>
+		
+	<LI>1.Intro</LI>
+
+	<LI>2.History</LI>
+
+	<LI>3.XSL Basics</LI>
+
+	<LI>Lunch</LI>
+
+	<LI>4.An XML Data Model</LI>
+
+	<LI>5.XSL Patterns</LI>
+
+	<LI>6.XSL Templates</LI>
+
+	<LI>7.XSL Formatting Model
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR/>
+		<H2>Intro</H2>
+		<UL>
+		
+	<LI>Who am I?</LI>
+
+	<LI>Who are you?</LI>
+
+	<LI>Why are we here?
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR/>
+		<H2>History: XML and SGML</H2>
+		<UL>
+		
+	<LI>XML is a subset of SGML.</LI>
+
+	<LI>SGML allows the separation of abstract content from formatting.</LI>
+
+	<LI>Also one of XML's primary virtues (in the doc publishing domain).
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR/>
+		<H2>History: What are stylesheets?</H2>
+		<UL>
+		
+	<LI>Stylesheets specify the formatting of SGML/XML documents.</LI>
+
+	<LI>Stylesheets put the &quot;style&quot; back into documents.</LI>
+
+	<LI>New York Times content+NYT Stylesheet = NYT paper
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR/>
+		<H2>History: FOSI</H2>
+		<UL>
+		
+	<LI>FOSI: &quot;Formatted Output Specification Instance&quot;
+<UL>
+	<LI>MIL-STD-28001
+	</LI>
+
+	<LI>FOSI's are SGML documents
+	</LI>
+
+	<LI>A stylesheet for another document
+	</LI>
+</UL></LI>
+
+	<LI>Obsolete but implemented...
+</LI>
+
+		</UL>
+	
+
+µ
+	
+</HTML>

Lib/test/xmltestdata/test.xml.out

@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<HTML xmlns:pp="http://www.isogen.com/paul/post-processor">
+<TITLE>Introduction to XSL</TITLE>
+<H1>Introduction to XSL</H1>
+	
+
+	
+		<HR></HR>
+		<H2>Overview
+</H2>
+		<UL>
+		
+	<LI>1.Intro</LI>
+
+	<LI>2.History</LI>
+
+	<LI>3.XSL Basics</LI>
+
+	<LI>Lunch</LI>
+
+	<LI>4.An XML Data Model</LI>
+
+	<LI>5.XSL Patterns</LI>
+
+	<LI>6.XSL Templates</LI>
+
+	<LI>7.XSL Formatting Model
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR></HR>
+		<H2>Intro</H2>
+		<UL>
+		
+	<LI>Who am I?</LI>
+
+	<LI>Who are you?</LI>
+
+	<LI>Why are we here?
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR></HR>
+		<H2>History: XML and SGML</H2>
+		<UL>
+		
+	<LI>XML is a subset of SGML.</LI>
+
+	<LI>SGML allows the separation of abstract content from formatting.</LI>
+
+	<LI>Also one of XML's primary virtues (in the doc publishing domain).
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR></HR>
+		<H2>History: What are stylesheets?</H2>
+		<UL>
+		
+	<LI>Stylesheets specify the formatting of SGML/XML documents.</LI>
+
+	<LI>Stylesheets put the "style" back into documents.</LI>
+
+	<LI>New York Times content+NYT Stylesheet = NYT paper
+</LI>
+
+		</UL>
+	
+
+
+	
+
+	
+		<HR></HR>
+		<H2>History: FOSI</H2>
+		<UL>
+		
+	<LI>FOSI: "Formatted Output Specification Instance"
+<UL>
+	<LI>MIL-STD-28001
+	</LI>
+
+	<LI>FOSI's are SGML documents
+	</LI>
+
+	<LI>A stylesheet for another document
+	</LI>
+</UL></LI>
+
+	<LI>Obsolete but implemented...
+</LI>
+
+		</UL>
+	
+
+µ
+	
+</HTML>

Lib/test/xmltests.py

@@ -0,0 +1,21 @@
+# Convenience test module to run all of the XML-related tests in the
+# standard library.
+
+import sys
+import test.support
+
+test.support.verbose = 0
+
+def runtest(name):
+    __import__(name)
+    module = sys.modules[name]
+    if hasattr(module, "test_main"):
+        module.test_main()
+
+runtest("test.test_minidom")
+runtest("test.test_pyexpat")
+runtest("test.test_sax")
+runtest("test.test_xml_dom_minicompat")
+runtest("test.test_xml_etree")
+runtest("test.test_xml_etree_c")
+runtest("test.test_xmlrpc")

Lib/xml/dom/pulldom.py

@@ -216,19 +216,6 @@ class DOMEventStream:
         self.parser.setFeature(xml.sax.handler.feature_namespaces, 1)
         self.parser.setContentHandler(self.pulldom)
 
-    def __getitem__(self, pos):
-        import warnings
-        warnings.warn(
-            "DOMEventStream's __getitem__ method ignores 'pos' parameter. "
-            "Use iterator protocol instead.",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        rc = self.getEvent()
-        if rc:
-            return rc
-        raise IndexError
-
     def __next__(self):
         rc = self.getEvent()
         if rc:

compiler/ast/asdl_rs.py

@@ -474,7 +474,7 @@ class ExtendModuleVisitor(EmitVisitor):
     def visitModule(self, mod):
         depth = 0
         self.emit(
-            "pub fn extend_module_nodes(vm: &VirtualMachine, module: &PyObject) {",
+            "pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) {",
             depth,
         )
         self.emit("extend_module!(vm, module, {", depth + 1)
@@ -671,6 +671,24 @@ def write_ast_def(mod, typeinfo, f):
             pub fn new(location: Location, end_location: Location, node: T) -> Self {
                 Self { location, end_location: Some(end_location), custom: (), node }
             }
+            
+            pub const fn start(&self) -> Location {
+                self.location
+            }
+            
+            /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if 
+            /// [`end_location`](Located::end_location) is `None`.
+            pub fn end(&self) -> Location {
+                self.end_location.unwrap_or(self.location)
+            }      
+        }
+        
+        impl<T, U> std::ops::Deref for Located<T, U> {
+            type Target = T;
+            
+            fn deref(&self) -> &Self::Target {
+                &self.node
+            }
         }
         \n
     """.lstrip()

compiler/ast/src/ast_gen.rs

@@ -24,6 +24,24 @@ impl<T> Located<T> {
             node,
         }
     }
+
+    pub const fn start(&self) -> Location {
+        self.location
+    }
+
+    /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if
+    /// [`end_location`](Located::end_location) is `None`.
+    pub fn end(&self) -> Location {
+        self.end_location.unwrap_or(self.location)
+    }
+}
+
+impl<T, U> std::ops::Deref for Located<T, U> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.node
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]

compiler/parser/src/function.rs

@@ -35,12 +35,12 @@ pub(crate) fn validate_arguments(
 
     let mut all_arg_names = FxHashSet::with_hasher(Default::default());
     for arg in all_args {
-        let arg_name = &arg.node.arg;
+        let arg_name = &arg.arg;
         // Check for duplicate arguments in the function definition.
         if !all_arg_names.insert(arg_name) {
             return Err(LexicalError {
                 error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
-                location: arg.location,
+                location: arg.start(),
             });
         }
     }
@@ -64,7 +64,7 @@ pub(crate) fn parse_params(
             // have defaults.
             return Err(LexicalError {
                 error: LexicalErrorType::DefaultArgumentError,
-                location: name.location,
+                location: name.start(),
             });
         }
         Ok(())
@@ -126,14 +126,14 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
                 if !keywords.is_empty() && !is_starred(&value) {
                     return Err(LexicalError {
                         error: LexicalErrorType::PositionalArgumentError,
-                        location: value.location,
+                        location: value.start(),
                     });
                 // Allow starred arguments after keyword arguments but
                 // not after double-starred arguments.
                 } else if double_starred {
                     return Err(LexicalError {
                         error: LexicalErrorType::UnpackedArgumentError,
-                        location: value.location,
+                        location: value.start(),
                     });
                 }
 

derive-impl/src/lib.rs

@@ -18,6 +18,7 @@ mod pyclass;
 mod pymodule;
 mod pypayload;
 mod pystructseq;
+mod pytraverse;
 
 use error::{extract_spans, Diagnostic};
 use proc_macro2::TokenStream;
@@ -39,19 +40,18 @@ pub fn derive_from_args(input: DeriveInput) -> TokenStream {
 
 pub fn pyclass(attr: AttributeArgs, item: Item) -> TokenStream {
     if matches!(item, syn::Item::Impl(_) | syn::Item::Trait(_)) {
-        result_to_tokens(pyclass::impl_pyimpl(attr, item))
+        result_to_tokens(pyclass::impl_pyclass_impl(attr, item))
     } else {
         result_to_tokens(pyclass::impl_pyclass(attr, item))
     }
 }
 
-pub use pyclass::PyExceptionDef;
-pub fn define_exception(exc_def: PyExceptionDef) -> TokenStream {
-    result_to_tokens(pyclass::impl_define_exception(exc_def))
-}
-
 pub fn pyexception(attr: AttributeArgs, item: Item) -> TokenStream {
-    result_to_tokens(pyclass::impl_pyexception(attr, item))
+    if matches!(item, syn::Item::Impl(_)) {
+        result_to_tokens(pyclass::impl_pyexception_impl(attr, item))
+    } else {
+        result_to_tokens(pyclass::impl_pyexception(attr, item))
+    }
 }
 
 pub fn pymodule(attr: AttributeArgs, item: Item) -> TokenStream {
@@ -77,3 +77,7 @@ pub fn py_freeze(input: TokenStream, compiler: &dyn Compiler) -> TokenStream {
 pub fn pypayload(input: DeriveInput) -> TokenStream {
     result_to_tokens(pypayload::impl_pypayload(input))
 }
+
+pub fn pytraverse(item: DeriveInput) -> TokenStream {
+    result_to_tokens(pytraverse::impl_pytraverse(item))
+}

derive-impl/src/pyclass.rs

@@ -1,18 +1,15 @@
 use super::Diagnostic;
 use crate::util::{
-    format_doc, pyclass_ident_and_attrs, text_signature, ClassItemMeta, ContentItem,
-    ContentItemInner, ErrorVec, ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta,
-    ALL_ALLOWED_NAMES,
+    format_doc, pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature,
+    ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, ItemMeta,
+    ItemMetaInner, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES,
 };
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, quote_spanned, ToTokens};
 use std::collections::HashMap;
 use std::str::FromStr;
 use syn::{
-    parse::{Parse, ParseStream, Result as ParsingResult},
-    parse_quote,
-    spanned::Spanned,
-    Attribute, AttributeArgs, Ident, Item, LitStr, Meta, NestedMeta, Result, Token,
+    parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Meta, NestedMeta, Result,
 };
 use syn_ext::ext::*;
 
@@ -99,7 +96,7 @@ fn extract_items_into_context<'a, Item>(
     context.errors.ok_or_push(context.member_items.validate());
 }
 
-pub(crate) fn impl_pyimpl(attr: AttributeArgs, item: Item) -> Result<TokenStream> {
+pub(crate) fn impl_pyclass_impl(attr: AttributeArgs, item: Item) -> Result<TokenStream> {
     let mut context = ImplContext::default();
     let mut tokens = match item {
         Item::Impl(mut imp) => {
@@ -340,8 +337,8 @@ fn generate_class_def(
     let base_class = if is_pystruct {
         Some(quote! { rustpython_vm::builtins::PyTuple })
     } else {
-        base.map(|typ| {
-            let typ = Ident::new(&typ, ident.span());
+        base.as_ref().map(|typ| {
+            let typ = Ident::new(typ, ident.span());
             quote_spanned! { ident.span() => #typ }
         })
     }
@@ -364,6 +361,13 @@ fn generate_class_def(
         }
     });
 
+    let base_or_object = if let Some(base) = base {
+        let base = Ident::new(&base, ident.span());
+        quote! { #base }
+    } else {
+        quote! { ::rustpython_vm::builtins::PyBaseObject }
+    };
+
     let tokens = quote! {
         impl ::rustpython_vm::class::PyClassDef for #ident {
             const NAME: &'static str = #name;
@@ -372,6 +376,8 @@ fn generate_class_def(
             const DOC: Option<&'static str> = #doc;
             const BASICSIZE: usize = #basicsize;
             const UNHASHABLE: bool = #unhashable;
+
+            type Base = #base_or_object;
         }
 
         impl ::rustpython_vm::class::StaticType for #ident {
@@ -413,9 +419,87 @@ pub(crate) fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result<TokenStrea
         attrs,
     )?;
 
+    const ALLOWED_TRAVERSE_OPTS: &[&str] = &["manual"];
+    // try to know if it have a `#[pyclass(trace)]` exist on this struct
+    // TODO(discord9): rethink on auto detect `#[Derive(PyTrace)]`
+
+    // 1. no `traverse` at all: generate a dummy try_traverse
+    // 2. `traverse = "manual"`: generate a try_traverse, but not #[derive(Traverse)]
+    // 3. `traverse`: generate a try_traverse, and #[derive(Traverse)]
+    let (maybe_trace_code, derive_trace) = {
+        if class_meta.inner()._has_key("traverse")? {
+            let maybe_trace_code = quote! {
+                impl ::rustpython_vm::object::MaybeTraverse for #ident {
+                    const IS_TRACE: bool = true;
+                    fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) {
+                        ::rustpython_vm::object::Traverse::traverse(self, tracer_fn);
+                    }
+                }
+            };
+            // if the key `traverse` exist but not as key-value, _optional_str return Err(...)
+            // so we need to check if it is Ok(Some(...))
+            let value = class_meta.inner()._optional_str("traverse");
+            let derive_trace = if let Ok(Some(s)) = value {
+                if !ALLOWED_TRAVERSE_OPTS.contains(&s.as_str()) {
+                    bail_span!(
+                        item,
+                        "traverse attribute only accept {ALLOWED_TRAVERSE_OPTS:?} as value or no value at all",
+                    );
+                }
+                assert_eq!(s, "manual");
+                quote! {}
+            } else {
+                quote! {#[derive(Traverse)]}
+            };
+            (maybe_trace_code, derive_trace)
+        } else {
+            (
+                // a dummy impl, which do nothing
+                // #attrs
+                quote! {
+                    impl ::rustpython_vm::object::MaybeTraverse for #ident {
+                        fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) {
+                            // do nothing
+                        }
+                    }
+                },
+                quote! {},
+            )
+        }
+    };
+
+    let impl_payload = if let Some(ctx_type_name) = class_meta.ctx_name()? {
+        let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); // FIXME span
+
+        // We need this to make extend mechanism work:
+        quote! {
+            impl ::rustpython_vm::PyPayload for #ident {
+                fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> {
+                    ctx.types.#ctx_type_ident
+                }
+            }
+        }
+    } else {
+        quote! {}
+    };
+
+    let empty_impl = if let Some(attrs) = class_meta.impl_attrs()? {
+        let attrs: Meta = parse_quote! (#attrs);
+        quote! {
+            #[pyclass(#attrs)]
+            impl #ident {}
+        }
+    } else {
+        quote! {}
+    };
+
     let ret = quote! {
+        #derive_trace
         #item
+        #maybe_trace_code
         #class_def
+        #impl_payload
+        #empty_impl
     };
     Ok(ret)
 }
@@ -429,87 +513,125 @@ pub(crate) fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result<TokenStrea
 /// to add non-literal attributes to `pyclass`.
 /// That's why we have to use this proxy.
 pub(crate) fn impl_pyexception(attr: AttributeArgs, item: Item) -> Result<TokenStream> {
-    let class_name = parse_vec_ident(&attr, &item, 0, "first 'class_name'")?;
-    let base_class_name = parse_vec_ident(&attr, &item, 1, "second 'base_class_name'")?;
+    let (ident, _attrs) = pyexception_ident_and_attrs(&item)?;
+    let fake_ident = Ident::new("pyclass", item.span());
+    let class_meta = ExceptionItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?;
+    let class_name = class_meta.class_name()?;
 
-    // We also need to strip `Py` prefix from `class_name`,
-    // due to implementation and Python naming conventions mismatch:
-    // `PyKeyboardInterrupt` -> `KeyboardInterrupt`
-    let class_name = class_name
-        .strip_prefix("Py")
-        .ok_or_else(|| err_span!(item, "We require 'class_name' to have 'Py' prefix"))?;
+    let base_class_name = class_meta.base()?;
+    let impl_payload = if let Some(ctx_type_name) = class_meta.ctx_name()? {
+        let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); // FIXME span
+
+        // We need this to make extend mechanism work:
+        quote! {
+            impl ::rustpython_vm::PyPayload for #ident {
+                fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> {
+                    ctx.exceptions.#ctx_type_ident
+                }
+            }
+        }
+    } else {
+        quote! {}
+    };
+    let impl_pyclass = if class_meta.has_impl()? {
+        quote! {
+            #[pyexception]
+            impl #ident {}
+        }
+    } else {
+        quote! {}
+    };
 
-    // We just "proxy" it into `pyclass` macro, because, exception is a class.
     let ret = quote! {
         #[pyclass(module = false, name = #class_name, base = #base_class_name)]
         #item
+        #impl_payload
+        #impl_pyclass
     };
     Ok(ret)
 }
 
-pub(crate) fn impl_define_exception(exc_def: PyExceptionDef) -> Result<TokenStream> {
-    let PyExceptionDef {
-        class_name,
-        base_class,
-        ctx_name,
-        docs,
-        slot_new,
-        init,
-    } = exc_def;
-
-    // We need this method, because of how `CPython` copies `__new__`
-    // from `BaseException` in `SimpleExtendsException` macro.
-    // See: `BaseException_new`
-    let slot_new_impl = match slot_new {
-        Some(slot_call) => quote! { #slot_call(cls, args, vm) },
-        None => quote! { #base_class::slot_new(cls, args, vm) },
+pub(crate) fn impl_pyexception_impl(attr: AttributeArgs, item: Item) -> Result<TokenStream> {
+    let Item::Impl(imp) = item else {
+        return Ok(item.into_token_stream());
     };
 
-    // We need this method, because of how `CPython` copies `__init__`
-    // from `BaseException` in `SimpleExtendsException` macro.
-    // See: `(initproc)BaseException_init`
-    // spell-checker:ignore initproc
-    let init_method = match init {
-        Some(init_def) => quote! { #init_def(zelf, args, vm) },
-        None => quote! { #base_class::slot_init(zelf, args, vm) },
-    };
+    if !attr.is_empty() {
+        return Err(syn::Error::new_spanned(
+            &attr[0],
+            "#[pyexception] impl doesn't allow attrs. Use #[pyclass] instead.",
+        ));
+    }
 
-    let ret = quote! {
-        #[pyexception(#class_name, #base_class)]
-        #[derive(Debug)]
-        #[doc = #docs]
-        pub struct #class_name {}
-
-        // We need this to make extend mechanism work:
-        impl ::rustpython_vm::PyPayload for #class_name {
-            fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> {
-                ctx.exceptions.#ctx_name
+    let mut has_slot_new = false;
+    let mut has_slot_init = false;
+    let syn::ItemImpl {
+        generics,
+        self_ty,
+        items,
+        ..
+    } = &imp;
+    for item in items {
+        // FIXME: better detection or correct wrapper implementation
+        let Some(ident) = item.get_ident() else {
+            continue;
+        };
+        let item_name = ident.to_string();
+        match item_name.as_str() {
+            "slot_new" => {
+                has_slot_new = true;
             }
+            "slot_init" => {
+                has_slot_init = true;
+            }
+            _ => continue,
         }
+    }
 
-        #[pyclass(flags(BASETYPE, HAS_DICT))]
-        impl #class_name {
+    let slot_new = if has_slot_new {
+        quote!()
+    } else {
+        quote! {
             #[pyslot]
             pub(crate) fn slot_new(
                 cls: ::rustpython_vm::builtins::PyTypeRef,
                 args: ::rustpython_vm::function::FuncArgs,
                 vm: &::rustpython_vm::VirtualMachine,
             ) -> ::rustpython_vm::PyResult {
-                #slot_new_impl
-            }
-
-            #[pyslot]
-            #[pymethod(name="__init__")]
-            pub(crate) fn slot_init(
-                zelf: PyObjectRef,
-                args: ::rustpython_vm::function::FuncArgs,
-                vm: &::rustpython_vm::VirtualMachine,
-            ) -> ::rustpython_vm::PyResult<()> {
-                #init_method
+                <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_new(cls, args, vm)
             }
         }
     };
-    Ok(ret)
+
+    // We need this method, because of how `CPython` copies `__init__`
+    // from `BaseException` in `SimpleExtendsException` macro.
+    // See: `(initproc)BaseException_init`
+    // spell-checker:ignore initproc
+    let slot_init = if has_slot_init {
+        quote!()
+    } else {
+        // FIXME: this is a generic logic for types not only for exceptions
+        quote! {
+            #[pyslot]
+            #[pymethod(name="__init__")]
+            pub(crate) fn slot_init(
+                zelf: ::rustpython_vm::PyObjectRef,
+                args: ::rustpython_vm::function::FuncArgs,
+                vm: &::rustpython_vm::VirtualMachine,
+            ) -> ::rustpython_vm::PyResult<()> {
+                <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_init(zelf, args, vm)
+            }
+        }
+    };
+    Ok(quote! {
+        #[pyclass(flags(BASETYPE, HAS_DICT))]
+        impl #generics #self_ty {
+            #(#items)*
+
+            #slot_new
+            #slot_init
+        }
+    })
 }
 
 /// #[pymethod] and #[pyclassmethod]
@@ -1425,50 +1547,7 @@ where
     Ok((result, cfgs))
 }
 
-#[derive(Debug)]
-pub struct PyExceptionDef {
-    pub class_name: Ident,
-    pub base_class: Ident,
-    pub ctx_name: Ident,
-    pub docs: LitStr,
-
-    /// Holds optional `slot_new` slot to be used instead of a default one:
-    pub slot_new: Option<Ident>,
-    /// We also store `__init__` magic method, that can
-    pub init: Option<Ident>,
-}
-
-impl Parse for PyExceptionDef {
-    fn parse(input: ParseStream) -> ParsingResult<Self> {
-        let class_name: Ident = input.parse()?;
-        input.parse::<Token![,]>()?;
-
-        let base_class: Ident = input.parse()?;
-        input.parse::<Token![,]>()?;
-
-        let ctx_name: Ident = input.parse()?;
-        input.parse::<Token![,]>()?;
-
-        let docs: LitStr = input.parse()?;
-        input.parse::<Option<Token![,]>>()?;
-
-        let slot_new: Option<Ident> = input.parse()?;
-        input.parse::<Option<Token![,]>>()?;
-
-        let init: Option<Ident> = input.parse()?;
-        input.parse::<Option<Token![,]>>()?; // leading `,`
-
-        Ok(PyExceptionDef {
-            class_name,
-            base_class,
-            ctx_name,
-            docs,
-            slot_new,
-            init,
-        })
-    }
-}
-
+#[allow(dead_code)]
 fn parse_vec_ident(
     attr: &[NestedMeta],
     item: &Item,

derive-impl/src/pymodule.rs

@@ -2,7 +2,7 @@ use crate::error::Diagnostic;
 use crate::util::{
     format_doc, iter_use_idents, pyclass_ident_and_attrs, text_signature, AttrItemMeta,
     AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery,
-    SimpleItemMeta, ALL_ALLOWED_NAMES,
+    ModuleItemMeta, SimpleItemMeta, ALL_ALLOWED_NAMES,
 };
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, quote_spanned, ToTokens};
@@ -45,7 +45,9 @@ impl FromStr for AttrName {
 #[derive(Default)]
 struct ModuleContext {
     name: String,
-    module_extend_items: ItemNursery,
+    function_items: FunctionNursery,
+    attribute_items: ItemNursery,
+    has_extend_module: bool, // TODO: check if `fn extend_module` exists
     errors: Vec<syn::Error>,
 }
 
@@ -56,7 +58,7 @@ pub fn impl_pymodule(attr: AttributeArgs, module_item: Item) -> Result<TokenStre
     };
     let fake_ident = Ident::new("pymodule", module_item.span());
     let module_meta =
-        SimpleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?;
+        ModuleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?;
 
     // generation resources
     let mut context = ModuleContext {
@@ -91,7 +93,8 @@ pub fn impl_pymodule(attr: AttributeArgs, module_item: Item) -> Result<TokenStre
 
     // append additional items
     let module_name = context.name.as_str();
-    let module_extend_items = context.module_extend_items.validate()?;
+    let function_items = context.function_items.validate()?;
+    let attribute_items = context.attribute_items.validate()?;
     let doc = doc.or_else(|| {
         crate::doc::Database::shared()
             .try_path(module_name)
@@ -104,29 +107,99 @@ pub fn impl_pymodule(attr: AttributeArgs, module_item: Item) -> Result<TokenStre
     } else {
         quote!(None)
     };
+    let is_submodule = module_meta.sub()?;
+    let withs = module_meta.with()?;
+    if !is_submodule {
+        items.extend(iter_chain![
+            parse_quote! {
+                pub(crate) const MODULE_NAME: &'static str = #module_name;
+            },
+            parse_quote! {
+                pub(crate) const DOC: Option<&'static str> = #doc;
+            },
+            parse_quote! {
+                pub(crate) fn __module_def(
+                    ctx: &::rustpython_vm::Context,
+                ) -> &'static ::rustpython_vm::builtins::PyModuleDef {
+                    DEF.get_or_init(|| {
+                        let mut def = ::rustpython_vm::builtins::PyModuleDef {
+                            name: ctx.intern_str(MODULE_NAME),
+                            doc: DOC.map(|doc| ctx.intern_str(doc)),
+                            slots: Default::default(),
+                        };
+                        def.slots.exec = Some(extend_module);
+                        def
+                    })
+                }
+            },
+            parse_quote! {
+                #[allow(dead_code)]
+                pub(crate) fn make_module(
+                    vm: &::rustpython_vm::VirtualMachine
+                ) -> ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> {
+                    use ::rustpython_vm::PyPayload;
+                    let module = ::rustpython_vm::builtins::PyModule::from_def(__module_def(&vm.ctx)).into_ref(&vm.ctx);
+                    __init_dict(vm, &module);
+                    extend_module(vm, &module).unwrap();
+                    module
+                }
+            },
+        ]);
+    }
+    if !is_submodule && !context.has_extend_module {
+        items.push(parse_quote! {
+            pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> {
+                __extend_module(vm, module);
+                Ok(())
+            }
+        });
+    }
     items.extend(iter_chain![
         parse_quote! {
-            pub(crate) const MODULE_NAME: &'static str = #module_name;
-        },
-        parse_quote! {
-            pub(crate) const DOC: Option<&'static str> = #doc;
-        },
-        parse_quote! {
-            pub(crate) fn extend_module(
-                vm: &::rustpython_vm::VirtualMachine,
-                module: &::rustpython_vm::PyObject,
-            ) {
-                #module_extend_items
+            ::rustpython_vm::common::static_cell! {
+                pub(crate) static DEF: ::rustpython_vm::builtins::PyModuleDef;
             }
         },
         parse_quote! {
-            #[allow(dead_code)]
-            pub(crate) fn make_module(
-                vm: &::rustpython_vm::VirtualMachine
-            ) -> ::rustpython_vm::PyObjectRef {
-                let module = vm.new_module(MODULE_NAME, vm.ctx.new_dict(), DOC);
-                extend_module(vm, &module);
-                module
+            pub(crate) fn __init_attributes(
+                vm: &::rustpython_vm::VirtualMachine,
+                module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
+            ) {
+                #(
+                    super::#withs::__init_attributes(vm, module);
+                )*
+                let ctx = &vm.ctx;
+                #attribute_items
+            }
+        },
+        parse_quote! {
+            pub(crate) fn __extend_module(
+                vm: &::rustpython_vm::VirtualMachine,
+                module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
+            ) {
+                __init_methods(vm, module);
+                __init_attributes(vm, module);
+            }
+        },
+        parse_quote! {
+            // TODO: remove once PyMethodDef done
+            pub(crate) fn __init_methods(
+                vm: &::rustpython_vm::VirtualMachine,
+                module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
+            ) {
+                #(
+                    super::#withs::__init_methods(vm, module);
+                )*
+                let ctx = &vm.ctx;
+                #function_items
+            }
+        },
+        parse_quote! {
+            pub(crate) fn __init_dict(
+                vm: &::rustpython_vm::VirtualMachine,
+                module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
+            ) {
+                ::rustpython_vm::builtins::PyModule::__init_dict_from_def(vm, module);
             }
         },
     ]);
@@ -248,6 +321,53 @@ where
     Ok((result, cfgs))
 }
 
+#[derive(Default)]
+struct FunctionNursery {
+    items: Vec<FunctionNurseryItem>,
+}
+
+struct FunctionNurseryItem {
+    py_names: Vec<String>,
+    cfgs: Vec<Attribute>,
+    ident: Ident,
+    #[allow(dead_code)]
+    doc: String,
+    tokens: TokenStream,
+}
+
+impl FunctionNursery {
+    fn add_item(&mut self, item: FunctionNurseryItem) {
+        self.items.push(item);
+    }
+
+    fn validate(self) -> Result<ValidatedFunctionNursery> {
+        let mut name_set = HashSet::new();
+        for item in &self.items {
+            for py_name in &item.py_names {
+                if !name_set.insert((py_name.to_owned(), &item.cfgs)) {
+                    bail_span!(item.ident, "duplicate method name `{}`", py_name);
+                }
+            }
+        }
+        Ok(ValidatedFunctionNursery(self))
+    }
+}
+
+struct ValidatedFunctionNursery(FunctionNursery);
+
+impl ToTokens for ValidatedFunctionNursery {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        for item in &self.0.items {
+            let cfgs = &item.cfgs;
+            let item_tokens = &item.tokens;
+            tokens.extend(quote! {
+                #(#cfgs)*
+                #item_tokens
+            });
+        }
+    }
+}
+
 /// #[pyfunction]
 struct FunctionItem {
     inner: ContentItemInner<AttrName>,
@@ -318,7 +438,7 @@ impl ModuleItem for FunctionItem {
         let py_name = item_meta.simple_name()?;
         let sig_doc = text_signature(func.sig(), &py_name);
 
-        let (tokens, py_names) = {
+        let (tokens, py_names, doc) = {
             let module = args.module_name();
             let doc = args.attrs.doc().or_else(|| {
                 crate::doc::Database::shared()
@@ -332,10 +452,10 @@ impl ModuleItem for FunctionItem {
             } else {
                 sig_doc
             };
-            let doc = quote!(.with_doc(#doc.to_owned(), &vm.ctx));
+            let with_doc = quote!(.with_doc(#doc.to_owned(), &vm.ctx));
             let new_func = quote_spanned!(ident.span()=>
                 vm.ctx.make_func_def(vm.ctx.intern_str(#py_name), #ident)
-                    #doc
+                    #with_doc
                     .into_function()
                     .with_module(vm.new_pyobj(#module.to_owned()))
                     .into_ref(&vm.ctx)
@@ -348,6 +468,7 @@ impl ModuleItem for FunctionItem {
                         vm.__module_set_attr(module, #py_name, func).unwrap();
                     }},
                     vec![py_name],
+                    doc,
                 )
             } else {
                 let mut py_names = HashSet::new();
@@ -381,17 +502,18 @@ impl ModuleItem for FunctionItem {
                         }
                     }},
                     py_names,
+                    doc,
                 )
             }
         };
 
-        args.context.module_extend_items.add_item(
-            ident.clone(),
+        args.context.function_items.add_item(FunctionNurseryItem {
+            ident: ident.to_owned(),
             py_names,
-            args.cfgs.to_vec(),
+            cfgs: args.cfgs.to_vec(),
+            doc,
             tokens,
-            10,
-        )?;
+        });
         Ok(())
     }
 }
@@ -432,8 +554,8 @@ impl ModuleItem for ClassItem {
                 class_meta.class_name()?
             };
             let class_new = quote_spanned!(ident.span() =>
-                let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_class(&vm.ctx);
-                new_class.set_attr(rustpython_vm::identifier!(vm, __module__), vm.new_pyobj(#module_name));
+                let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx);
+                new_class.set_attr(rustpython_vm::identifier!(ctx, __module__), vm.new_pyobj(#module_name));
             );
             (class_name, class_new)
         };
@@ -473,7 +595,7 @@ impl ModuleItem for ClassItem {
             },
         };
 
-        args.context.module_extend_items.add_item(
+        args.context.attribute_items.add_item(
             ident.clone(),
             py_names,
             args.cfgs.to_vec(),
@@ -561,7 +683,7 @@ impl ModuleItem for AttributeItem {
                     let tokens = quote_spanned! { ident.span() =>
                         vm.__module_set_attr(module, #py_name, vm.new_pyobj(#ident)).unwrap();
                     };
-                    args.context.module_extend_items.add_item(
+                    args.context.attribute_items.add_item(
                         ident.clone(),
                         vec![py_name],
                         cfgs.clone(),
@@ -624,7 +746,7 @@ impl ModuleItem for AttributeItem {
         };
 
         args.context
-            .module_extend_items
+            .attribute_items
             .add_item(ident, py_names, cfgs, tokens, 1)?;
 
         Ok(())

derive-impl/src/pytraverse.rs

@@ -0,0 +1,138 @@
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::{Attribute, DeriveInput, Field, Meta, MetaList, NestedMeta, Result};
+
+struct TraverseAttr {
+    /// set to `true` if the attribute is `#[pytraverse(skip)]`
+    skip: bool,
+}
+
+const ATTR_TRAVERSE: &str = "pytraverse";
+
+/// get the `#[pytraverse(..)]` attribute from the struct
+fn valid_get_traverse_attr_from_meta_list(list: &MetaList) -> Result<TraverseAttr> {
+    let find_skip_and_only_skip = || {
+        let len = list.nested.len();
+        if len != 1 {
+            return None;
+        }
+        let mut iter = list.nested.iter();
+        // we have checked the length, so unwrap is safe
+        let first_arg = iter.next().unwrap();
+        let skip = match first_arg {
+            NestedMeta::Meta(Meta::Path(path)) => match path.is_ident("skip") {
+                true => true,
+                false => return None,
+            },
+            _ => return None,
+        };
+        Some(skip)
+    };
+    let skip = find_skip_and_only_skip().ok_or_else(|| {
+        err_span!(
+            list,
+            "only support attr is #[pytraverse(skip)], got arguments: {:?}",
+            list.nested
+        )
+    })?;
+    Ok(TraverseAttr { skip })
+}
+
+/// only accept `#[pytraverse(skip)]` for now
+fn pytraverse_arg(attr: &Attribute) -> Option<Result<TraverseAttr>> {
+    if !attr.path.is_ident(ATTR_TRAVERSE) {
+        return None;
+    }
+    let ret = || {
+        let parsed = attr.parse_meta()?;
+        if let Meta::List(list) = parsed {
+            valid_get_traverse_attr_from_meta_list(&list)
+        } else {
+            bail_span!(attr, "pytraverse must be a list, like #[pytraverse(skip)]")
+        }
+    };
+    Some(ret())
+}
+
+fn field_to_traverse_code(field: &Field) -> Result<TokenStream> {
+    let pytraverse_attrs = field
+        .attrs
+        .iter()
+        .filter_map(pytraverse_arg)
+        .collect::<std::result::Result<Vec<_>, _>>()?;
+    let do_trace = if pytraverse_attrs.len() > 1 {
+        bail_span!(
+            field,
+            "found multiple #[pytraverse] attributes on the same field, expect at most one"
+        )
+    } else if pytraverse_attrs.is_empty() {
+        // default to always traverse every field
+        true
+    } else {
+        !pytraverse_attrs[0].skip
+    };
+    let name = field.ident.as_ref().ok_or_else(|| {
+        syn::Error::new_spanned(
+            field.clone(),
+            "Field should have a name in non-tuple struct",
+        )
+    })?;
+    if do_trace {
+        Ok(quote!(
+            ::rustpython_vm::object::Traverse::traverse(&self.#name, tracer_fn);
+        ))
+    } else {
+        Ok(quote!())
+    }
+}
+
+/// not trace corresponding field
+fn gen_trace_code(item: &mut DeriveInput) -> Result<TokenStream> {
+    match &mut item.data {
+        syn::Data::Struct(s) => {
+            let fields = &mut s.fields;
+            match fields {
+                syn::Fields::Named(ref mut fields) => {
+                    let res: Vec<TokenStream> = fields
+                        .named
+                        .iter_mut()
+                        .map(|f| -> Result<TokenStream> { field_to_traverse_code(f) })
+                        .collect::<Result<_>>()?;
+                    let res = res.into_iter().collect::<TokenStream>();
+                    Ok(res)
+                }
+                syn::Fields::Unnamed(fields) => {
+                    let res: TokenStream = (0..fields.unnamed.len())
+                        .map(|i| {
+                            let i = syn::Index::from(i);
+                            quote!(
+                                ::rustpython_vm::object::Traverse::traverse(&self.#i, tracer_fn);
+                            )
+                        })
+                        .collect();
+                    Ok(res)
+                }
+                _ => Err(syn::Error::new_spanned(
+                    fields,
+                    "Only named and unnamed fields are supported",
+                )),
+            }
+        }
+        _ => Err(syn::Error::new_spanned(item, "Only structs are supported")),
+    }
+}
+
+pub(crate) fn impl_pytraverse(mut item: DeriveInput) -> Result<TokenStream> {
+    let trace_code = gen_trace_code(&mut item)?;
+
+    let ty = &item.ident;
+
+    let ret = quote! {
+        unsafe impl ::rustpython_vm::object::Traverse for #ty {
+            fn traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) {
+                #trace_code
+            }
+        }
+    };
+    Ok(ret)
+}

derive-impl/src/util.rs

@@ -178,6 +178,10 @@ impl ItemMetaInner {
         Ok(value)
     }
 
+    pub fn _has_key(&self, key: &str) -> Result<bool> {
+        Ok(matches!(self.meta_map.get(key), Some((_, _))))
+    }
+
     pub fn _bool(&self, key: &str) -> Result<bool> {
         let value = if let Some((_, meta)) = self.meta_map.get(key) {
             match meta {
@@ -193,6 +197,21 @@ impl ItemMetaInner {
         };
         Ok(value)
     }
+
+    pub fn _optional_list(
+        &self,
+        key: &str,
+    ) -> Result<Option<impl std::iter::Iterator<Item = &'_ NestedMeta>>> {
+        let value = if let Some((_, meta)) = self.meta_map.get(key) {
+            let Meta::List(syn::MetaList { path: _, nested, .. }) = meta else {
+                bail_span!(meta, "#[{}({}(...))] must be a list", self.meta_name(), key)
+            };
+            Some(nested.into_iter())
+        } else {
+            None
+        };
+        Ok(value)
+    }
 }
 
 pub(crate) trait ItemMeta: Sized {
@@ -247,6 +266,38 @@ impl ItemMeta for SimpleItemMeta {
     }
 }
 
+pub(crate) struct ModuleItemMeta(pub ItemMetaInner);
+
+impl ItemMeta for ModuleItemMeta {
+    const ALLOWED_NAMES: &'static [&'static str] = &["name", "with", "sub"];
+
+    fn from_inner(inner: ItemMetaInner) -> Self {
+        Self(inner)
+    }
+    fn inner(&self) -> &ItemMetaInner {
+        &self.0
+    }
+}
+
+impl ModuleItemMeta {
+    pub fn sub(&self) -> Result<bool> {
+        self.inner()._bool("sub")
+    }
+    pub fn with(&self) -> Result<Vec<&syn::Path>> {
+        let mut withs = Vec::new();
+        let Some(nested) = self.inner()._optional_list("with")? else {
+            return Ok(withs);
+        };
+        for meta in nested {
+            let NestedMeta::Meta(Meta::Path(path)) = meta else {
+                bail_span!(meta, "#[pymodule(with(...))] arguments should be paths")
+            };
+            withs.push(path);
+        }
+        Ok(withs)
+    }
+}
+
 pub(crate) struct AttrItemMeta(pub ItemMetaInner);
 
 impl ItemMeta for AttrItemMeta {
@@ -263,8 +314,15 @@ impl ItemMeta for AttrItemMeta {
 pub(crate) struct ClassItemMeta(ItemMetaInner);
 
 impl ItemMeta for ClassItemMeta {
-    const ALLOWED_NAMES: &'static [&'static str] =
-        &["module", "name", "base", "metaclass", "unhashable"];
+    const ALLOWED_NAMES: &'static [&'static str] = &[
+        "module",
+        "name",
+        "base",
+        "metaclass",
+        "unhashable",
+        "impl",
+        "traverse",
+    ];
 
     fn from_inner(inner: ItemMetaInner) -> Self {
         Self(inner)
@@ -296,6 +354,10 @@ impl ClassItemMeta {
         )
     }
 
+    pub fn ctx_name(&self) -> Result<Option<String>> {
+        self.inner()._optional_str("ctx")
+    }
+
     pub fn base(&self) -> Result<Option<String>> {
         self.inner()._optional_str("base")
     }
@@ -339,6 +401,10 @@ impl ClassItemMeta {
         Ok(value)
     }
 
+    pub fn impl_attrs(&self) -> Result<Option<String>> {
+        self.inner()._optional_str("impl")
+    }
+
     // pub fn mandatory_module(&self) -> Result<String> {
     //     let inner = self.inner();
     //     let value = self.module().ok().flatten().
@@ -351,6 +417,64 @@ impl ClassItemMeta {
     // }
 }
 
+pub(crate) struct ExceptionItemMeta(ClassItemMeta);
+
+impl ItemMeta for ExceptionItemMeta {
+    const ALLOWED_NAMES: &'static [&'static str] = &["name", "base", "unhashable", "ctx", "impl"];
+
+    fn from_inner(inner: ItemMetaInner) -> Self {
+        Self(ClassItemMeta(inner))
+    }
+    fn inner(&self) -> &ItemMetaInner {
+        &self.0 .0
+    }
+}
+
+impl ExceptionItemMeta {
+    pub fn class_name(&self) -> Result<String> {
+        const KEY: &str = "name";
+        let inner = self.inner();
+        if let Some((_, meta)) = inner.meta_map.get(KEY) {
+            match meta {
+                Meta::NameValue(syn::MetaNameValue {
+                    lit: syn::Lit::Str(lit),
+                    ..
+                }) => return Ok(lit.value()),
+                Meta::Path(_) => {
+                    return Ok({
+                        let type_name = inner.item_name();
+                        let Some(py_name) = type_name.as_str().strip_prefix("Py") else {
+                        bail_span!(
+                            inner.item_ident,
+                            "#[pyexception] expects its underlying type to be named `Py` prefixed"
+                        )
+                    };
+                        py_name.to_string()
+                    })
+                }
+                _ => {}
+            }
+        }
+        bail_span!(
+            inner.meta_ident,
+            "#[{attr_name}(name = ...)] must exist as a string. Try \
+             #[{attr_name}(name)] to use rust type name.",
+            attr_name = inner.meta_name()
+        )
+    }
+
+    pub fn has_impl(&self) -> Result<bool> {
+        self.inner()._bool("impl")
+    }
+}
+
+impl std::ops::Deref for ExceptionItemMeta {
+    type Target = ClassItemMeta;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
 pub(crate) trait AttributeExt: SynAttributeExt {
     fn promoted_nested(&self) -> Result<PunctuatedNestedMeta>;
     fn ident_and_promoted_nested(&self) -> Result<(&Ident, PunctuatedNestedMeta)>;
@@ -458,6 +582,17 @@ pub(crate) fn pyclass_ident_and_attrs(item: &syn::Item) -> Result<(&Ident, &[Att
     })
 }
 
+pub(crate) fn pyexception_ident_and_attrs(item: &syn::Item) -> Result<(&Ident, &[Attribute])> {
+    use syn::Item::*;
+    Ok(match item {
+        Struct(syn::ItemStruct { ident, attrs, .. }) => (ident, attrs),
+        Enum(syn::ItemEnum { ident, attrs, .. }) => (ident, attrs),
+        other => {
+            bail_span!(other, "#[pyexception] can only be on a struct or enum",)
+        }
+    })
+}
+
 pub(crate) trait ErrorVec: Sized {
     fn into_error(self) -> Option<syn::Error>;
     fn into_result(self) -> Result<()> {

derive/src/lib.rs

@@ -19,24 +19,14 @@ pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream {
     derive_impl::pyclass(attr, item).into()
 }
 
+/// Helper macro to define `Exception` types.
+/// More-or-less is an alias to `pyclass` macro.
+///
 /// This macro serves a goal of generating multiple
 /// `BaseException` / `Exception`
 /// subtypes in a uniform and convenient manner.
 /// It looks like `SimpleExtendsException` in `CPython`.
 /// <https://github.com/python/cpython/blob/main/Objects/exceptions.c>
-///
-/// We need `ctx` to be ready to add
-/// `properties` / `custom` constructors / slots / methods etc.
-/// So, we use `extend_class!` macro as the second
-/// step in exception type definition.
-#[proc_macro]
-pub fn define_exception(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input);
-    derive_impl::define_exception(input).into()
-}
-
-/// Helper macro to define `Exception` types.
-/// More-or-less is an alias to `pyclass` macro.
 #[proc_macro_attribute]
 pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream {
     let attr = parse_macro_input!(attr);
@@ -91,3 +81,27 @@ pub fn pypayload(input: TokenStream) -> TokenStream {
     let input = parse_macro_input!(input);
     derive_impl::pypayload(input).into()
 }
+
+/// use on struct with named fields like `struct A{x:PyRef<B>, y:PyRef<C>}` to impl `Traverse` for datatype.
+///
+/// use `#[pytraverse(skip)]` on fields you wish not to trace
+///
+/// add `trace` attr to `#[pyclass]` to make it impl `MaybeTraverse` that will call `Traverse`'s `traverse` method so make it
+/// traceable(Even from type-erased PyObject)(i.e. write `#[pyclass(trace)]`).
+/// # Example
+/// ```rust, ignore
+/// #[pyclass(module = false, traverse)]
+/// #[derive(Default, Traverse)]
+/// pub struct PyList {
+///     elements: PyRwLock<Vec<PyObjectRef>>,
+///     #[pytraverse(skip)]
+///     len: AtomicCell<usize>,
+/// }
+/// ```
+/// This create both `MaybeTraverse` that call `Traverse`'s `traverse` method and `Traverse` that impl `Traverse`
+/// for `PyList` which call elements' `traverse` method and ignore `len` field.
+#[proc_macro_derive(Traverse, attributes(pytraverse))]
+pub fn pytraverse(item: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let item = parse_macro_input!(item);
+    derive_impl::pytraverse(item).into()
+}

extra_tests/snippets/builtin_exceptions.py

@@ -239,13 +239,13 @@ assert BaseException.__new__.__qualname__ == 'BaseException.__new__'
 assert BaseException.__init__.__qualname__ == 'BaseException.__init__'
 assert BaseException().__dict__ == {}
 
-assert Exception.__new__.__qualname__ == 'Exception.__new__'
-assert Exception.__init__.__qualname__ == 'Exception.__init__'
+assert Exception.__new__.__qualname__ == 'Exception.__new__', Exception.__new__.__qualname__
+assert Exception.__init__.__qualname__ == 'Exception.__init__', Exception.__init__.__qualname__
 assert Exception().__dict__ == {}
 
 
 # Extends `BaseException`, simple:
-assert KeyboardInterrupt.__new__.__qualname__ == 'KeyboardInterrupt.__new__'
+assert KeyboardInterrupt.__new__.__qualname__ == 'KeyboardInterrupt.__new__', KeyboardInterrupt.__new__.__qualname__
 assert KeyboardInterrupt.__init__.__qualname__ == 'KeyboardInterrupt.__init__'
 assert KeyboardInterrupt().__dict__ == {}
 

extra_tests/snippets/stdlib_collections_deque.py

@@ -17,20 +17,20 @@ def test_deque_iterator__new__positional_index():
     klass = type(iter(deque()))
 
     # index between 0 and len
-    for s in ('abcd', range(2000)):
+    for s in ('abcd', range(200)):
         for i in range(len(s)):
             d = klass(deque(s), i)
             assert (list(d) == list(s)[i:])
 
     # negative index
-    for s in ('abcd', range(2000)):
-        for i in range(-1000, 0):
+    for s in ('abcd', range(200)):
+        for i in range(-100, 0):
             d = klass(deque(s), i)
             assert (list(d) == list(s))
 
     # index ge len
-    for s in ('abcd', range(2000)):
-        for i in range(len(s), 4000):
+    for s in ('abcd', range(200)):
+        for i in range(len(s), 400):
             d = klass(deque(s), i)
             assert (list(d) == list())
 
@@ -41,8 +41,8 @@ test_deque_iterator__new__positional_index()
 def test_deque_iterator__new__not_using_keyword_index():
     klass = type(iter(deque()))
 
-    for s in ('abcd', range(2000)):
-        for i in range(-1000, 4000):
+    for s in ('abcd', range(200)):
+        for i in range(-100, 400):
             d = klass(deque(s), index=i)
             assert (list(d) == list(s))
 
@@ -54,20 +54,20 @@ def test_deque_reverse_iterator__new__positional_index():
     klass = type(reversed(deque()))
 
     # index between 0 and len
-    for s in ('abcd', range(2000)):
+    for s in ('abcd', range(200)):
         for i in range(len(s)):
             d = klass(deque(s), i)
             assert (list(d) == list(reversed(s))[i:])
 
     # negative index
-    for s in ('abcd', range(2000)):
-        for i in range(-1000, 0):
+    for s in ('abcd', range(200)):
+        for i in range(-100, 0):
             d = klass(deque(s), i)
             assert (list(d) == list(reversed(s)))
 
     # index ge len
-    for s in ('abcd', range(2000)):
-        for i in range(len(s), 4000):
+    for s in ('abcd', range(200)):
+        for i in range(len(s), 400):
             d = klass(deque(s), i)
             assert (list(d) == list())
 
@@ -78,8 +78,8 @@ test_deque_reverse_iterator__new__positional_index()
 def test_deque_reverse_iterator__new__not_using_keyword_index():
     klass = type(reversed(deque()))
 
-    for s in ('abcd', range(2000)):
-        for i in range(-1000, 4000):
+    for s in ('abcd', range(200)):
+        for i in range(-100, 400):
             d = klass(deque(s), index=i)
             assert (list(d) == list(reversed(s)))
 

src/lib.rs

@@ -99,15 +99,12 @@ fn setup_main_module(vm: &VirtualMachine) -> PyResult<Scope> {
     let main_module = vm.new_module("__main__", scope.globals.clone(), None);
     main_module
         .dict()
-        .and_then(|d| {
-            d.set_item("__annotations__", vm.ctx.new_dict().into(), vm)
-                .ok()
-        })
+        .set_item("__annotations__", vm.ctx.new_dict().into(), vm)
         .expect("Failed to initialize __main__.__annotations__");
 
     vm.sys_module
         .get_attr("modules", vm)?
-        .set_item("__main__", main_module, vm)?;
+        .set_item("__main__", main_module.into(), vm)?;
 
     Ok(scope)
 }

stdlib/Cargo.toml

@@ -55,6 +55,7 @@ sha-1 = "0.10.0"
 sha2 = "0.10.2"
 sha3 = "0.10.1"
 blake2 = "0.10.4"
+hmac = "0.12.1"
 
 ## unicode stuff
 unicode_names2 = { workspace = true }

stdlib/src/array.rs

@@ -1,8 +1,8 @@
 // spell-checker:ignore typecode tofile tolist fromfile
 
-use rustpython_vm::{PyObjectRef, VirtualMachine};
+use rustpython_vm::{builtins::PyModule, PyRef, VirtualMachine};
 
-pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     let module = array::make_module(vm);
 
     let array = module
@@ -62,7 +62,7 @@ mod array {
             },
             types::{
                 AsBuffer, AsMapping, AsSequence, Comparable, Constructor, IterNext,
-                IterNextIterable, Iterable, PyComparisonOp,
+                IterNextIterable, Iterable, PyComparisonOp, Representable,
             },
             AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
         },
@@ -721,7 +721,15 @@ mod array {
 
     #[pyclass(
         flags(BASETYPE),
-        with(Comparable, AsBuffer, AsMapping, AsSequence, Iterable, Constructor)
+        with(
+            Comparable,
+            AsBuffer,
+            AsMapping,
+            AsSequence,
+            Iterable,
+            Constructor,
+            Representable
+        )
     )]
     impl PyArray {
         fn read(&self) -> PyRwLockReadGuard<'_, ArrayContentType> {
@@ -1113,23 +1121,6 @@ mod array {
             Ok(zelf)
         }
 
-        #[pymethod(magic)]
-        fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
-            let class = zelf.class();
-            let class_name = class.name();
-            if zelf.read().typecode() == 'u' {
-                if zelf.len() == 0 {
-                    return Ok(format!("{class_name}('u')"));
-                }
-                return Ok(format!(
-                    "{}('u', {})",
-                    class_name,
-                    crate::common::str::repr(&zelf.tounicode(vm)?)
-                ));
-            }
-            zelf.read().repr(&class_name, vm)
-        }
-
         #[pymethod(magic)]
         pub(crate) fn len(&self) -> usize {
             self.read().len()
@@ -1292,6 +1283,25 @@ mod array {
         }
     }
 
+    impl Representable for PyArray {
+        #[inline]
+        fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
+            let class = zelf.class();
+            let class_name = class.name();
+            if zelf.read().typecode() == 'u' {
+                if zelf.len() == 0 {
+                    return Ok(format!("{class_name}('u')"));
+                }
+                return Ok(format!(
+                    "{}('u', {})",
+                    class_name,
+                    crate::common::str::repr(&zelf.tounicode(vm)?)
+                ));
+            }
+            zelf.read().repr(&class_name, vm)
+        }
+    }
+
     static BUFFER_METHODS: BufferMethods = BufferMethods {
         obj_bytes: |buffer| buffer.obj_as::<PyArray>().get_bytes().into(),
         obj_bytes_mut: |buffer| buffer.obj_as::<PyArray>().get_bytes_mut().into(),
@@ -1389,7 +1399,7 @@ mod array {
     }
 
     #[pyattr]
-    #[pyclass(name = "arrayiterator")]
+    #[pyclass(name = "arrayiterator", traverse)]
     #[derive(Debug, PyPayload)]
     pub struct PyArrayIter {
         internal: PyMutex<PositionIterInternal<PyArrayRef>>,
@@ -1424,12 +1434,13 @@ mod array {
         }
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct ReconstructorArgs {
         #[pyarg(positional)]
         arraytype: PyTypeRef,
         #[pyarg(positional)]
         typecode: PyStrRef,
+        #[pytraverse(skip)]
         #[pyarg(positional)]
         mformat_code: MachineFormatCode,
         #[pyarg(positional)]

stdlib/src/bisect.rs

@@ -8,7 +8,7 @@ mod _bisect {
         PyObjectRef, PyResult, VirtualMachine,
     };
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct BisectArgs {
         a: PyObjectRef,
         x: PyObjectRef,

stdlib/src/blake2.rs

@@ -0,0 +1,19 @@
+// spell-checker:ignore usedforsecurity HASHXOF
+
+pub(crate) use _blake2::make_module;
+
+#[pymodule]
+mod _blake2 {
+    use crate::hashlib::_hashlib::{local_blake2b, local_blake2s, BlakeHashArgs};
+    use crate::vm::{PyPayload, PyResult, VirtualMachine};
+
+    #[pyfunction]
+    fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_blake2b(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_blake2s(args).into_pyobject(vm))
+    }
+}

stdlib/src/contextvars.rs

@@ -5,8 +5,8 @@ mod _contextvars {
     use crate::vm::{
         builtins::{PyFunction, PyStrRef, PyTypeRef},
         function::{ArgCallable, FuncArgs, OptionalArg},
-        types::Initializer,
-        PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
+        types::{Initializer, Representable},
+        Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
     };
 
     #[pyattr]
@@ -80,9 +80,10 @@ mod _contextvars {
     }
 
     #[pyattr]
-    #[pyclass(name)]
+    #[pyclass(name, traverse)]
     #[derive(Debug, PyPayload)]
     struct ContextVar {
+        #[pytraverse(skip)]
         #[allow(dead_code)] // TODO: RUSTPYTHON
         name: String,
         #[allow(dead_code)] // TODO: RUSTPYTHON
@@ -99,7 +100,7 @@ mod _contextvars {
         default: OptionalArg<PyObjectRef>,
     }
 
-    #[pyclass(with(Initializer))]
+    #[pyclass(with(Initializer, Representable))]
     impl ContextVar {
         #[pygetset]
         fn name(&self) -> String {
@@ -133,17 +134,6 @@ mod _contextvars {
         fn class_getitem(_cls: PyTypeRef, _key: PyStrRef, _vm: &VirtualMachine) -> PyResult<()> {
             unimplemented!("ContextVar.__class_getitem__() is currently under construction")
         }
-
-        #[pymethod(magic)]
-        fn repr(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> String {
-            unimplemented!("<ContextVar name={{}} default={{}} at {{}}")
-            // format!(
-            //     "<ContextVar name={} default={:?} at {:#x}>",
-            //     zelf.name.as_str(),
-            //     zelf.default.map_or("", |x| PyStr::from(*x).as_str()),
-            //     zelf.get_id()
-            // )
-        }
     }
 
     impl Initializer for ContextVar {
@@ -154,12 +144,25 @@ mod _contextvars {
         }
     }
 
+    impl Representable for ContextVar {
+        #[inline]
+        fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
+            unimplemented!("<ContextVar name={{}} default={{}} at {{}}")
+            // format!(
+            //     "<ContextVar name={} default={:?} at {:#x}>",
+            //     zelf.name.as_str(),
+            //     zelf.default.map_or("", |x| PyStr::from(*x).as_str()),
+            //     zelf.get_id()
+            // )
+        }
+    }
+
     #[pyattr]
     #[pyclass(name = "Token")]
     #[derive(Debug, PyPayload)]
     struct ContextToken {}
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct ContextTokenOptions {
         #[pyarg(positional)]
         #[allow(dead_code)] // TODO: RUSTPYTHON
@@ -172,7 +175,7 @@ mod _contextvars {
         old_value: PyObjectRef,
     }
 
-    #[pyclass(with(Initializer))]
+    #[pyclass(with(Initializer, Representable))]
     impl ContextToken {
         #[pygetset]
         fn var(&self, _vm: &VirtualMachine) -> PyObjectRef {
@@ -183,11 +186,6 @@ mod _contextvars {
         fn old_value(&self, _vm: &VirtualMachine) -> PyObjectRef {
             unimplemented!("Token.old_value() is currently under construction")
         }
-
-        #[pymethod(magic)]
-        fn repr(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> String {
-            unimplemented!("<Token {{}}var={{}} at {{}}>")
-        }
     }
 
     impl Initializer for ContextToken {
@@ -198,6 +196,13 @@ mod _contextvars {
         }
     }
 
+    impl Representable for ContextToken {
+        #[inline]
+        fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
+            unimplemented!("<Token {{}}var={{}} at {{}}>")
+        }
+    }
+
     #[pyfunction]
     fn copy_context() {}
 }

stdlib/src/csv.rs

@@ -152,10 +152,11 @@ mod _csv {
         reader: csv_core::Reader,
     }
 
-    #[pyclass(no_attr, module = "_csv", name = "reader")]
+    #[pyclass(no_attr, module = "_csv", name = "reader", traverse)]
     #[derive(PyPayload)]
     pub(super) struct Reader {
         iter: PyIter,
+        #[pytraverse(skip)]
         state: PyMutex<ReadState>,
     }
 
@@ -242,10 +243,11 @@ mod _csv {
         writer: csv_core::Writer,
     }
 
-    #[pyclass(no_attr, module = "_csv", name = "writer")]
+    #[pyclass(no_attr, module = "_csv", name = "writer", traverse)]
     #[derive(PyPayload)]
     pub(super) struct Writer {
         write: PyObjectRef,
+        #[pytraverse(skip)]
         state: PyMutex<WriteState>,
     }
 

stdlib/src/grp.rs

@@ -13,11 +13,14 @@ mod grp {
     use std::ptr::NonNull;
 
     #[pyattr]
-    #[pyclass(module = "grp", name = "struct_group")]
+    #[pyclass(module = "grp", name = "struct_group", traverse)]
     #[derive(PyStructSequence)]
     struct Group {
+        #[pytraverse(skip)]
         gr_name: String,
+        #[pytraverse(skip)]
         gr_passwd: String,
+        #[pytraverse(skip)]
         gr_gid: u32,
         gr_mem: PyListRef,
     }

stdlib/src/hashlib.rs

@@ -1,13 +1,15 @@
 // spell-checker:ignore usedforsecurity HASHXOF
 
-pub(crate) use hashlib::make_module;
+pub(crate) use _hashlib::make_module;
 
 #[pymodule]
-mod hashlib {
+pub mod _hashlib {
     use crate::common::lock::PyRwLock;
     use crate::vm::{
         builtins::{PyBytes, PyStrRef, PyTypeRef},
-        function::{ArgBytesLike, FuncArgs, OptionalArg},
+        convert::ToPyObject,
+        function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg},
+        protocol::PyBuffer,
         PyObjectRef, PyPayload, PyResult, VirtualMachine,
     };
     use blake2::{Blake2b512, Blake2s256};
@@ -19,7 +21,7 @@ mod hashlib {
     use sha2::{Sha224, Sha256, Sha384, Sha512};
     use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256};
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Debug)]
     #[allow(unused)]
     struct NewHashArgs {
         #[pyarg(positional)]
@@ -32,9 +34,9 @@ mod hashlib {
 
     #[derive(FromArgs)]
     #[allow(unused)]
-    struct BlakeHashArgs {
+    pub struct BlakeHashArgs {
         #[pyarg(positional, optional)]
-        data: OptionalArg<ArgBytesLike>,
+        pub data: OptionalArg<ArgBytesLike>,
         #[pyarg(named, default = "true")]
         usedforsecurity: bool,
     }
@@ -48,11 +50,11 @@ mod hashlib {
         }
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Debug)]
     #[allow(unused)]
-    struct HashArgs {
+    pub struct HashArgs {
         #[pyarg(any, optional)]
-        string: OptionalArg<ArgBytesLike>,
+        pub string: OptionalArg<ArgBytesLike>,
         #[pyarg(named, default = "true")]
         usedforsecurity: bool,
     }
@@ -81,11 +83,11 @@ mod hashlib {
     }
 
     #[pyattr]
-    #[pyclass(module = "hashlib", name = "HASH")]
+    #[pyclass(module = "_hashlib", name = "HASH")]
     #[derive(PyPayload)]
-    struct PyHasher {
-        name: String,
-        ctx: PyRwLock<HashWrapper>,
+    pub struct PyHasher {
+        pub name: String,
+        pub ctx: PyRwLock<HashWrapper>,
     }
 
     impl std::fmt::Debug for PyHasher {
@@ -105,7 +107,7 @@ mod hashlib {
 
         #[pyslot]
         fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult {
-            Err(vm.new_type_error("cannot create 'hashlib.HASH' instances".into()))
+            Err(vm.new_type_error("cannot create '_hashlib.HASH' instances".into()))
         }
 
         #[pygetset]
@@ -145,9 +147,9 @@ mod hashlib {
     }
 
     #[pyattr]
-    #[pyclass(module = "hashlib", name = "HASHXOF")]
+    #[pyclass(module = "_hashlib", name = "HASHXOF")]
     #[derive(PyPayload)]
-    struct PyHasherXof {
+    pub struct PyHasherXof {
         name: String,
         ctx: PyRwLock<HashXofWrapper>,
     }
@@ -169,7 +171,7 @@ mod hashlib {
 
         #[pyslot]
         fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult {
-            Err(vm.new_type_error("cannot create 'hashlib.HASHXOF' instances".into()))
+            Err(vm.new_type_error("cannot create '_hashlib.HASHXOF' instances".into()))
         }
 
         #[pygetset]
@@ -211,108 +213,147 @@ mod hashlib {
     #[pyfunction(name = "new")]
     fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
         match args.name.as_str().to_lowercase().as_str() {
-            "md5" => Ok(md5(args.into()).into_pyobject(vm)),
-            "sha1" => Ok(sha1(args.into()).into_pyobject(vm)),
-            "sha224" => Ok(sha224(args.into()).into_pyobject(vm)),
-            "sha256" => Ok(sha256(args.into()).into_pyobject(vm)),
-            "sha384" => Ok(sha384(args.into()).into_pyobject(vm)),
-            "sha512" => Ok(sha512(args.into()).into_pyobject(vm)),
-            "sha3_224" => Ok(sha3_224(args.into()).into_pyobject(vm)),
-            "sha3_256" => Ok(sha3_256(args.into()).into_pyobject(vm)),
-            "sha3_384" => Ok(sha3_384(args.into()).into_pyobject(vm)),
-            "sha3_512" => Ok(sha3_512(args.into()).into_pyobject(vm)),
-            "shake_128" => Ok(shake_128(args.into()).into_pyobject(vm)),
-            "shake_256" => Ok(shake_256(args.into()).into_pyobject(vm)),
-            "blake2b" => Ok(blake2b(args.into()).into_pyobject(vm)),
-            "blake2s" => Ok(blake2s(args.into()).into_pyobject(vm)),
+            "md5" => Ok(local_md5(args.into()).into_pyobject(vm)),
+            "sha1" => Ok(local_sha1(args.into()).into_pyobject(vm)),
+            "sha224" => Ok(local_sha224(args.into()).into_pyobject(vm)),
+            "sha256" => Ok(local_sha256(args.into()).into_pyobject(vm)),
+            "sha384" => Ok(local_sha384(args.into()).into_pyobject(vm)),
+            "sha512" => Ok(local_sha512(args.into()).into_pyobject(vm)),
+            "sha3_224" => Ok(local_sha3_224(args.into()).into_pyobject(vm)),
+            "sha3_256" => Ok(local_sha3_256(args.into()).into_pyobject(vm)),
+            "sha3_384" => Ok(local_sha3_384(args.into()).into_pyobject(vm)),
+            "sha3_512" => Ok(local_sha3_512(args.into()).into_pyobject(vm)),
+            "shake_128" => Ok(local_shake_128(args.into()).into_pyobject(vm)),
+            "shake_256" => Ok(local_shake_256(args.into()).into_pyobject(vm)),
+            "blake2b" => Ok(local_blake2b(args.into()).into_pyobject(vm)),
+            "blake2s" => Ok(local_blake2s(args.into()).into_pyobject(vm)),
             other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {other}"))),
         }
     }
 
-    #[pyfunction]
-    fn md5(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_md5")]
+    pub fn local_md5(args: HashArgs) -> PyHasher {
         PyHasher::new("md5", HashWrapper::new::<Md5>(args.string))
     }
 
-    #[pyfunction]
-    fn sha1(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha1")]
+    pub fn local_sha1(args: HashArgs) -> PyHasher {
         PyHasher::new("sha1", HashWrapper::new::<Sha1>(args.string))
     }
 
-    #[pyfunction]
-    fn sha224(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha224")]
+    pub fn local_sha224(args: HashArgs) -> PyHasher {
         PyHasher::new("sha224", HashWrapper::new::<Sha224>(args.string))
     }
 
-    #[pyfunction]
-    fn sha256(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha256")]
+    pub fn local_sha256(args: HashArgs) -> PyHasher {
         PyHasher::new("sha256", HashWrapper::new::<Sha256>(args.string))
     }
 
-    #[pyfunction]
-    fn sha384(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha384")]
+    pub fn local_sha384(args: HashArgs) -> PyHasher {
         PyHasher::new("sha384", HashWrapper::new::<Sha384>(args.string))
     }
 
-    #[pyfunction]
-    fn sha512(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha512")]
+    pub fn local_sha512(args: HashArgs) -> PyHasher {
         PyHasher::new("sha512", HashWrapper::new::<Sha512>(args.string))
     }
 
-    #[pyfunction]
-    fn sha3_224(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha3_224")]
+    pub fn local_sha3_224(args: HashArgs) -> PyHasher {
         PyHasher::new("sha3_224", HashWrapper::new::<Sha3_224>(args.string))
     }
 
-    #[pyfunction]
-    fn sha3_256(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha3_256")]
+    pub fn local_sha3_256(args: HashArgs) -> PyHasher {
         PyHasher::new("sha3_256", HashWrapper::new::<Sha3_256>(args.string))
     }
 
-    #[pyfunction]
-    fn sha3_384(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha3_384")]
+    pub fn local_sha3_384(args: HashArgs) -> PyHasher {
         PyHasher::new("sha3_384", HashWrapper::new::<Sha3_384>(args.string))
     }
 
-    #[pyfunction]
-    fn sha3_512(args: HashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_sha3_512")]
+    pub fn local_sha3_512(args: HashArgs) -> PyHasher {
         PyHasher::new("sha3_512", HashWrapper::new::<Sha3_512>(args.string))
     }
 
-    #[pyfunction]
-    fn shake_128(args: HashArgs) -> PyHasherXof {
+    #[pyfunction(name = "openssl_shake_128")]
+    pub fn local_shake_128(args: HashArgs) -> PyHasherXof {
         PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string))
     }
 
-    #[pyfunction]
-    fn shake_256(args: HashArgs) -> PyHasherXof {
+    #[pyfunction(name = "openssl_shake_256")]
+    pub fn local_shake_256(args: HashArgs) -> PyHasherXof {
         PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string))
     }
 
-    #[pyfunction]
-    fn blake2b(args: BlakeHashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_blake2b")]
+    pub fn local_blake2b(args: BlakeHashArgs) -> PyHasher {
         PyHasher::new("blake2b", HashWrapper::new::<Blake2b512>(args.data))
     }
 
-    #[pyfunction]
-    fn blake2s(args: BlakeHashArgs) -> PyHasher {
+    #[pyfunction(name = "openssl_blake2s")]
+    pub fn local_blake2s(args: BlakeHashArgs) -> PyHasher {
         PyHasher::new("blake2s", HashWrapper::new::<Blake2s256>(args.data))
     }
 
-    trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {}
+    #[pyfunction]
+    fn compare_digest(
+        a: ArgStrOrBytesLike,
+        b: ArgStrOrBytesLike,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyObjectRef> {
+        fn is_str(arg: &ArgStrOrBytesLike) -> bool {
+            matches!(arg, ArgStrOrBytesLike::Str(_))
+        }
+
+        if is_str(&a) != is_str(&b) {
+            return Err(vm.new_type_error(format!(
+                "a bytes-like object is required, not '{}'",
+                b.as_object().class().name()
+            )));
+        }
+
+        let a_hash = a.borrow_bytes().to_vec();
+        let b_hash = b.borrow_bytes().to_vec();
+
+        Ok((a_hash == b_hash).to_pyobject(vm))
+    }
+
+    #[derive(FromArgs, Debug)]
+    #[allow(unused)]
+    pub struct NewHMACHashArgs {
+        #[pyarg(positional)]
+        name: PyBuffer,
+        #[pyarg(any, optional)]
+        data: OptionalArg<ArgBytesLike>,
+        #[pyarg(named, default = "true")]
+        digestmod: bool, // TODO: RUSTPYTHON support functions & name functions
+    }
+
+    #[pyfunction]
+    fn hmac_new(_args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
+        Err(vm.new_type_error("cannot create 'hmac' instances".into())) // TODO: RUSTPYTHON support hmac
+    }
+
+    pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {}
     impl<T> ThreadSafeDynDigest for T where T: DynClone + DynDigest + Sync + Send {}
 
     clone_trait_object!(ThreadSafeDynDigest);
 
     /// Generic wrapper patching around the hashing libraries.
     #[derive(Clone)]
-    struct HashWrapper {
+    pub struct HashWrapper {
         block_size: usize,
         inner: Box<dyn ThreadSafeDynDigest>,
     }
 
     impl HashWrapper {
-        fn new<D>(data: OptionalArg<ArgBytesLike>) -> Self
+        pub fn new<D>(data: OptionalArg<ArgBytesLike>) -> Self
         where
             D: ThreadSafeDynDigest + BlockSizeUser + Default + 'static,
         {
@@ -345,13 +386,13 @@ mod hashlib {
     }
 
     #[derive(Clone)]
-    enum HashXofWrapper {
+    pub enum HashXofWrapper {
         Shake128(Shake128),
         Shake256(Shake256),
     }
 
     impl HashXofWrapper {
-        fn new_shake_128(data: OptionalArg<ArgBytesLike>) -> Self {
+        pub fn new_shake_128(data: OptionalArg<ArgBytesLike>) -> Self {
             let mut h = HashXofWrapper::Shake128(Shake128::default());
             if let OptionalArg::Present(d) = data {
                 d.with_ref(|bytes| h.update(bytes));
@@ -359,7 +400,7 @@ mod hashlib {
             h
         }
 
-        fn new_shake_256(data: OptionalArg<ArgBytesLike>) -> Self {
+        pub fn new_shake_256(data: OptionalArg<ArgBytesLike>) -> Self {
             let mut h = HashXofWrapper::Shake256(Shake256::default());
             if let OptionalArg::Present(d) = data {
                 d.with_ref(|bytes| h.update(bytes));

stdlib/src/json.rs

@@ -16,9 +16,10 @@ mod _json {
     use std::str::FromStr;
 
     #[pyattr(name = "make_scanner")]
-    #[pyclass(name = "Scanner")]
+    #[pyclass(name = "Scanner", traverse)]
     #[derive(Debug, PyPayload)]
     struct JsonScanner {
+        #[pytraverse(skip)]
         strict: bool,
         object_hook: Option<PyObjectRef>,
         object_pairs_hook: Option<PyObjectRef>,

stdlib/src/json/machinery.rs

@@ -105,10 +105,8 @@ pub struct DecodeError {
 }
 impl DecodeError {
     fn new(msg: impl Into<String>, pos: usize) -> Self {
-        Self {
-            msg: msg.into(),
-            pos,
-        }
+        let msg = msg.into();
+        Self { msg, pos }
     }
 }
 

stdlib/src/lib.rs

@@ -13,7 +13,14 @@ mod contextvars;
 mod csv;
 mod dis;
 mod gc;
+
+mod blake2;
 mod hashlib;
+mod md5;
+mod sha1;
+mod sha256;
+mod sha3;
+
 mod json;
 #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))]
 mod locale;
@@ -100,7 +107,13 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_csv" => csv::make_module,
             "_dis" => dis::make_module,
             "gc" => gc::make_module,
-            "hashlib" => hashlib::make_module,
+            "_hashlib" => hashlib::make_module,
+            "_sha1" => sha1::make_module,
+            "_sha3" => sha3::make_module,
+            "_sha256" => sha256::make_module,
+            // "_sha512" => sha512::make_module, // TODO: RUSPYTHON fix strange fail on vm: 'static type has not been initialized'
+            "_md5" => md5::make_module,
+            "_blake2" => blake2::make_module,
             "_json" => json::make_module,
             "math" => math::make_module,
             "pyexpat" => pyexpat::make_module,

stdlib/src/math.rs

@@ -885,7 +885,7 @@ mod math {
         }
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct ProdArgs {
         #[pyarg(positional)]
         iterable: ArgIterable<PyObjectRef>,

stdlib/src/md5.rs

@@ -0,0 +1,12 @@
+pub(crate) use _md5::make_module;
+
+#[pymodule]
+mod _md5 {
+    use crate::hashlib::_hashlib::{local_md5, HashArgs};
+    use crate::vm::{PyPayload, PyResult, VirtualMachine};
+
+    #[pyfunction]
+    fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_md5(args).into_pyobject(vm))
+    }
+}

stdlib/src/mmap.rs

@@ -112,9 +112,28 @@ mod mmap {
     #[pyattr]
     use libc::{
         MADV_DODUMP, MADV_DOFORK, MADV_DONTDUMP, MADV_DONTFORK, MADV_HUGEPAGE, MADV_HWPOISON,
-        MADV_MERGEABLE, MADV_NOHUGEPAGE, MADV_REMOVE, MADV_SOFT_OFFLINE, MADV_UNMERGEABLE,
+        MADV_MERGEABLE, MADV_NOHUGEPAGE, MADV_REMOVE, MADV_UNMERGEABLE,
     };
 
+    #[cfg(any(
+        target_os = "android",
+        all(
+            target_os = "linux",
+            any(
+                target_arch = "aarch64",
+                target_arch = "arm",
+                target_arch = "powerpc",
+                target_arch = "powerpc64",
+                target_arch = "s390x",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "sparc64"
+            )
+        )
+    ))]
+    #[pyattr]
+    use libc::MADV_SOFT_OFFLINE;
+
     #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
     #[pyattr]
     use libc::{MAP_DENYWRITE, MAP_EXECUTABLE, MAP_POPULATE};
@@ -225,8 +244,9 @@ mod mmap {
     }
 
     #[cfg(not(target_os = "redox"))]
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     pub struct AdviseOptions {
+        #[pytraverse(skip)]
         #[pyarg(positional)]
         option: libc::c_int,
         #[pyarg(positional, default)]

stdlib/src/pyexpat.rs

@@ -3,12 +3,12 @@
 *
 */
 
-use crate::vm::{extend_module, PyObjectRef, VirtualMachine};
+use crate::vm::{builtins::PyModule, extend_module, PyRef, VirtualMachine};
 
-pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     let module = _pyexpat::make_module(vm);
 
-    extend_module!(vm, module, {
+    extend_module!(vm, &module, {
          "errors" => _errors::make_module(vm),
          "model" => _model::make_module(vm),
     });
@@ -43,7 +43,7 @@ mod _pyexpat {
     type MutableObject = PyRwLock<PyObjectRef>;
 
     #[pyattr]
-    #[pyclass(name = "xmlparser", module = false)]
+    #[pyclass(name = "xmlparser", module = false, traverse)]
     #[derive(Debug, PyPayload)]
     pub struct PyExpatLikeXmlParser {
         start_element: MutableObject,
@@ -156,7 +156,7 @@ mod _pyexpat {
         }
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     #[allow(dead_code)]
     struct ParserCreateArgs {
         #[pyarg(any, optional)]

stdlib/src/pystruct.rs

@@ -20,6 +20,7 @@ pub(crate) mod _struct {
     };
     use crossbeam_utils::atomic::AtomicCell;
 
+    #[derive(Traverse)]
     struct IntoStructFormatBytes(PyStrRef);
 
     impl TryFromObject for IntoStructFormatBytes {
@@ -133,9 +134,10 @@ pub(crate) mod _struct {
         buffer.with_ref(|buf| format_spec.unpack(buf, vm))
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct UpdateFromArgs {
         buffer: ArgBytesLike,
+        #[pytraverse(skip)]
         #[pyarg(any, default = "0")]
         offset: isize,
     }
@@ -154,11 +156,13 @@ pub(crate) mod _struct {
     }
 
     #[pyattr]
-    #[pyclass(name = "unpack_iterator")]
+    #[pyclass(name = "unpack_iterator", traverse)]
     #[derive(Debug, PyPayload)]
     struct UnpackIterator {
+        #[pytraverse(skip)]
         format_spec: FormatSpec,
         buffer: ArgBytesLike,
+        #[pytraverse(skip)]
         offset: AtomicCell<usize>,
     }
 
@@ -231,9 +235,10 @@ pub(crate) mod _struct {
     }
 
     #[pyattr]
-    #[pyclass(name = "Struct")]
+    #[pyclass(name = "Struct", traverse)]
     #[derive(Debug, PyPayload)]
     struct PyStruct {
+        #[pytraverse(skip)]
         spec: FormatSpec,
         format: PyStrRef,
     }

stdlib/src/re.rs

@@ -76,10 +76,11 @@ mod re {
 
     /// Inner data for a match object.
     #[pyattr]
-    #[pyclass(module = "re", name = "Match")]
-    #[derive(PyPayload)]
+    #[pyclass(module = "re", name = "Match", traverse)]
+    #[derive(PyPayload, Traverse)]
     struct PyMatch {
         haystack: PyStrRef,
+        #[pytraverse(skip)]
         captures: Vec<Option<Range<usize>>>,
     }
 

stdlib/src/select.rs

@@ -1,9 +1,10 @@
 use crate::vm::{
-    builtins::PyListRef, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine,
+    builtins::PyListRef, builtins::PyModule, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject,
+    VirtualMachine,
 };
 use std::{io, mem};
 
-pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     #[cfg(windows)]
     crate::vm::stdlib::nt::init_winsock();
 
@@ -67,8 +68,10 @@ mod platform {
 pub use platform::timeval;
 use platform::RawFd;
 
+#[derive(Traverse)]
 struct Selectable {
     obj: PyObjectRef,
+    #[pytraverse(skip)]
     fno: RawFd,
 }
 

stdlib/src/sha1.rs

@@ -0,0 +1,12 @@
+pub(crate) use _sha1::make_module;
+
+#[pymodule]
+mod _sha1 {
+    use crate::hashlib::_hashlib::{local_sha1, HashArgs};
+    use crate::vm::{PyPayload, PyResult, VirtualMachine};
+
+    #[pyfunction]
+    fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha1(args).into_pyobject(vm))
+    }
+}

stdlib/src/sha256.rs

@@ -0,0 +1,17 @@
+pub(crate) use _sha256::make_module;
+
+#[pymodule]
+mod _sha256 {
+    use crate::hashlib::_hashlib::{local_sha224, local_sha256, HashArgs};
+    use crate::vm::{PyPayload, PyResult, VirtualMachine};
+
+    #[pyfunction]
+    fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha224(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha256(args).into_pyobject(vm))
+    }
+}

stdlib/src/sha3.rs

@@ -0,0 +1,40 @@
+pub(crate) use _sha3::make_module;
+
+#[pymodule]
+mod _sha3 {
+    use crate::hashlib::_hashlib::{
+        local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128,
+        local_shake_256, HashArgs,
+    };
+    use crate::vm::{PyPayload, PyResult, VirtualMachine};
+
+    #[pyfunction]
+    fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha3_224(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha3_256(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha3_384(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_sha3_512(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_shake_128(args).into_pyobject(vm))
+    }
+
+    #[pyfunction]
+    fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult {
+        Ok(local_shake_256(args).into_pyobject(vm))
+    }
+}

stdlib/src/sha512.rs

@@ -0,0 +1,20 @@
+// spell-checker:ignore usedforsecurity HASHXOF
+
+pub(crate) use _sha512::make_module;
+
+#[pymodule]
+mod _sha512 {
+    use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher};
+    use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine};
+    use sha2::{Sha384, Sha512};
+
+    #[pyfunction(name = "sha384")]
+    fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
+        Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(args.string)).into_pyobject(vm))
+    }
+
+    #[pyfunction(name = "sha512")]
+    fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
+        Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(args.string)).into_pyobject(vm))
+    }
+}

stdlib/src/socket.rs

@@ -1,8 +1,8 @@
-use crate::vm::{PyObjectRef, VirtualMachine};
+use crate::vm::{builtins::PyModule, PyRef, VirtualMachine};
 #[cfg(feature = "ssl")]
 pub(super) use _socket::{sock_select, timeout_error_msg, PySocket, SelectKind};
 
-pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     #[cfg(windows)]
     crate::vm::stdlib::nt::init_winsock();
     _socket::make_module(vm)
@@ -15,9 +15,9 @@ mod _socket {
         builtins::{PyBaseExceptionRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef},
         convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject},
         function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption},
-        types::{DefaultConstructor, Initializer},
+        types::{DefaultConstructor, Initializer, Representable},
         utils::ToCString,
-        AsObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
+        AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
     };
     use crossbeam_utils::atomic::AtomicCell;
     use num_traits::ToPrimitive;
@@ -309,6 +309,7 @@ mod _socket {
             any(
                 target_arch = "aarch64",
                 target_arch = "i686",
+                target_arch = "loongarch64",
                 target_arch = "mips",
                 target_arch = "powerpc",
                 target_arch = "powerpc64",
@@ -359,6 +360,7 @@ mod _socket {
             any(
                 target_arch = "aarch64",
                 target_arch = "i686",
+                target_arch = "loongarch64",
                 target_arch = "mips",
                 target_arch = "powerpc",
                 target_arch = "powerpc64",
@@ -1015,7 +1017,21 @@ mod _socket {
         }
     }
 
-    #[pyclass(with(DefaultConstructor, Initializer), flags(BASETYPE))]
+    impl Representable for PySocket {
+        #[inline]
+        fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
+            Ok(format!(
+                "<socket object, fd={}, family={}, type={}, proto={}>",
+                // cast because INVALID_SOCKET is unsigned, so would show usize::MAX instead of -1
+                zelf.fileno() as i64,
+                zelf.family.load(),
+                zelf.kind.load(),
+                zelf.proto.load(),
+            ))
+        }
+    }
+
+    #[pyclass(with(DefaultConstructor, Initializer, Representable), flags(BASETYPE))]
     impl PySocket {
         fn _init(
             zelf: PyRef<Self>,
@@ -1463,18 +1479,6 @@ mod _socket {
         fn proto(&self) -> i32 {
             self.proto.load()
         }
-
-        #[pymethod(magic)]
-        fn repr(&self) -> String {
-            format!(
-                "<socket object, fd={}, family={}, type={}, proto={}>",
-                // cast because INVALID_SOCKET is unsigned, so would show usize::MAX instead of -1
-                self.fileno() as i64,
-                self.family.load(),
-                self.kind.load(),
-                self.proto.load(),
-            )
-        }
     }
 
     struct Address {

stdlib/src/sqlite.rs

@@ -8,13 +8,13 @@
 // spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath
 // spell-checker:ignore dbmoved vnode nbytes
 
-use rustpython_vm::{PyObjectRef, VirtualMachine};
+use rustpython_vm::{builtins::PyModule, AsObject, PyRef, VirtualMachine};
 
 // pub(crate) use _sqlite::make_module;
-pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     // TODO: sqlite version check
     let module = _sqlite::make_module(vm);
-    _sqlite::setup_module(&module, vm);
+    _sqlite::setup_module(module.as_object(), vm);
     module
 }
 
@@ -69,6 +69,7 @@ mod _sqlite {
         AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
         TryFromBorrowedObject, VirtualMachine,
         __exports::paste,
+        object::{Traverse, TraverseFn},
     };
     use std::{
         ffi::{c_int, c_longlong, c_uint, c_void, CStr},
@@ -311,6 +312,13 @@ mod _sqlite {
         uri: bool,
     }
 
+    unsafe impl Traverse for ConnectArgs {
+        fn traverse(&self, tracer_fn: &mut TraverseFn) {
+            self.isolation_level.traverse(tracer_fn);
+            self.factory.traverse(tracer_fn);
+        }
+    }
+
     #[derive(FromArgs)]
     struct BackupArgs {
         #[pyarg(any)]
@@ -325,36 +333,48 @@ mod _sqlite {
         sleep: f64,
     }
 
-    #[derive(FromArgs)]
+    unsafe impl Traverse for BackupArgs {
+        fn traverse(&self, tracer_fn: &mut TraverseFn) {
+            self.progress.traverse(tracer_fn);
+            self.name.traverse(tracer_fn);
+        }
+    }
+
+    #[derive(FromArgs, Traverse)]
     struct CreateFunctionArgs {
         #[pyarg(any)]
         name: PyStrRef,
+        #[pytraverse(skip)]
         #[pyarg(any)]
         narg: c_int,
         #[pyarg(any)]
         func: PyObjectRef,
+        #[pytraverse(skip)]
         #[pyarg(named, default)]
         deterministic: bool,
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct CreateAggregateArgs {
         #[pyarg(any)]
         name: PyStrRef,
+        #[pytraverse(skip)]
         #[pyarg(positional)]
         narg: c_int,
         #[pyarg(positional)]
         aggregate_class: PyObjectRef,
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct BlobOpenArgs {
         #[pyarg(positional)]
         table: PyStrRef,
         #[pyarg(positional)]
         column: PyStrRef,
+        #[pytraverse(skip)]
         #[pyarg(positional)]
         row: i64,
+        #[pytraverse(skip)]
         #[pyarg(named, default)]
         readonly: bool,
         #[pyarg(named, default = "vm.ctx.new_str(stringify!(main))")]
@@ -1107,7 +1127,7 @@ mod _sqlite {
         ) -> PyResult<()> {
             let name = name.to_cstring(vm)?;
             let db = self.db_lock(vm)?;
-            let Some(data )= CallbackData::new(callable, vm) else {
+            let Some(data) = CallbackData::new(callable.clone(), vm) else {
                 unsafe {
                     sqlite3_create_collation_v2(db.db, name.as_ptr(), SQLITE_UTF8, null_mut(), None, None);
                 }
@@ -1115,6 +1135,10 @@ mod _sqlite {
             };
             let data = Box::into_raw(Box::new(data));
 
+            if !callable.is_callable() {
+                return Err(vm.new_type_error("parameter must be callable".to_owned()));
+            }
+
             let ret = unsafe {
                 sqlite3_create_collation_v2(
                     db.db,
@@ -1336,20 +1360,24 @@ mod _sqlite {
     }
 
     #[pyattr]
-    #[pyclass(name)]
+    #[pyclass(name, traverse)]
     #[derive(Debug, PyPayload)]
     struct Cursor {
         connection: PyRef<Connection>,
+        #[pytraverse(skip)]
         arraysize: PyAtomic<c_int>,
+        #[pytraverse(skip)]
         row_factory: PyAtomicRef<Option<PyObject>>,
         inner: PyMutex<Option<CursorInner>>,
     }
 
-    #[derive(Debug)]
+    #[derive(Debug, Traverse)]
     struct CursorInner {
         description: Option<PyTupleRef>,
         row_cast_map: Vec<Option<PyObjectRef>>,
+        #[pytraverse(skip)]
         lastrowid: i64,
+        #[pytraverse(skip)]
         rowcount: i64,
         statement: Option<PyRef<Statement>>,
     }
@@ -1789,7 +1817,7 @@ mod _sqlite {
     }
 
     #[pyattr]
-    #[pyclass(name)]
+    #[pyclass(name, traverse)]
     #[derive(Debug, PyPayload)]
     struct Row {
         data: PyTupleRef,
@@ -1918,10 +1946,11 @@ mod _sqlite {
     }
 
     #[pyattr]
-    #[pyclass(name)]
+    #[pyclass(name, traverse)]
     #[derive(Debug, PyPayload)]
     struct Blob {
         connection: PyRef<Connection>,
+        #[pytraverse(skip)]
         inner: PyMutex<Option<BlobInner>>,
     }
 

stdlib/src/ssl.rs

@@ -1,26 +1,17 @@
-use crate::vm::{PyObjectRef, VirtualMachine};
+use crate::vm::{builtins::PyModule, PyRef, VirtualMachine};
 
-pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     // if openssl is vendored, it doesn't know the locations of system certificates
     #[cfg(feature = "ssl-vendor")]
     if let None | Some("0") = option_env!("OPENSSL_NO_VENDOR") {
         openssl_probe::init_ssl_cert_env_vars();
     }
     openssl::init();
-
-    let module = _ssl::make_module(vm);
-    #[cfg(ossl101)]
-    ossl101::extend_module(vm, &module);
-    #[cfg(ossl111)]
-    ossl101::extend_module(vm, &module);
-    #[cfg(windows)]
-    windows::extend_module(vm, &module);
-
-    module
+    _ssl::make_module(vm)
 }
 
 #[allow(non_upper_case_globals)]
-#[pymodule]
+#[pymodule(with(ossl101, windows))]
 mod _ssl {
     use super::bio;
     use crate::{
@@ -797,9 +788,10 @@ mod _ssl {
         }
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct WrapSocketArgs {
         sock: PyRef<PySocket>,
+        #[pytraverse(skip)]
         server_side: bool,
         #[pyarg(any, default)]
         server_hostname: Option<PyStrRef>,
@@ -819,9 +811,11 @@ mod _ssl {
         cadata: Option<Either<PyStrRef, ArgBytesLike>>,
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct LoadCertChainArgs {
+        #[pytraverse(skip)]
         certfile: FsPath,
+        #[pytraverse(skip)]
         #[pyarg(any, optional)]
         keyfile: Option<FsPath>,
         #[pyarg(any, optional)]
@@ -902,11 +896,13 @@ mod _ssl {
     }
 
     #[pyattr]
-    #[pyclass(module = "ssl", name = "_SSLSocket")]
+    #[pyclass(module = "ssl", name = "_SSLSocket", traverse)]
     #[derive(PyPayload)]
     struct PySslSocket {
         ctx: PyRef<PySslContext>,
+        #[pytraverse(skip)]
         stream: PyRwLock<ssl::SslStream<SocketStream>>,
+        #[pytraverse(skip)]
         socket_type: SslServerOrClient,
         server_hostname: Option<PyStrRef>,
         owner: PyRwLock<Option<PyRef<PyWeak>>>,
@@ -1400,9 +1396,21 @@ mod _ssl {
     }
 }
 
+#[cfg(not(ossl101))]
+#[pymodule(sub)]
+mod ossl101 {}
+
+#[cfg(not(ossl111))]
+#[pymodule(sub)]
+mod ossl111 {}
+
+#[cfg(not(windows))]
+#[pymodule(sub)]
+mod windows {}
+
 #[allow(non_upper_case_globals)]
 #[cfg(ossl101)]
-#[pymodule]
+#[pymodule(sub)]
 mod ossl101 {
     #[pyattr]
     use openssl_sys::{
@@ -1413,14 +1421,14 @@ mod ossl101 {
 
 #[allow(non_upper_case_globals)]
 #[cfg(ossl111)]
-#[pymodule]
+#[pymodule(sub)]
 mod ossl111 {
     #[pyattr]
     use openssl_sys::SSL_OP_NO_TLSv1_3 as OP_NO_TLSv1_3;
 }
 
 #[cfg(windows)]
-#[pymodule]
+#[pymodule(sub)]
 mod windows {
     use crate::{
         common::ascii,

stdlib/src/syslog.rs

@@ -95,7 +95,7 @@ mod syslog {
         Ok(())
     }
 
-    #[derive(FromArgs)]
+    #[derive(FromArgs, Traverse)]
     struct SysLogArgs {
         #[pyarg(positional)]
         priority: PyObjectRef,

stdlib/src/unicodedata.rs

@@ -5,11 +5,11 @@
 // spell-checker:ignore nfkc unistr unidata
 
 use crate::vm::{
-    builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyObjectRef, PyPayload, PyResult,
-    VirtualMachine,
+    builtins::PyModule, builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyObjectRef,
+    PyPayload, PyRef, PyResult, VirtualMachine,
 };
 
-pub fn make_module(vm: &VirtualMachine) -> PyObjectRef {
+pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
     let module = unicodedata::make_module(vm);
 
     let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION)

vm/src/builtins/bytearray.rs

@@ -326,7 +326,8 @@ impl PyByteArray {
     fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult {
         let bytes = PyBytesInner::fromhex(string.as_str(), vm)?;
         let bytes = vm.ctx.new_bytes(bytes);
-        PyType::call(&cls, vec![bytes.into()].into(), vm)
+        let args = vec![bytes.into()].into();
+        PyType::call(&cls, args, vm)
     }
 
     #[pymethod]

vm/src/builtins/code.rs

@@ -230,7 +230,7 @@ impl Representable for PyCode {
     }
 }
 
-#[pyclass(with(Py))]
+#[pyclass(with(Representable))]
 impl PyCode {
     #[pyslot]
     fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult {
@@ -418,21 +418,6 @@ impl PyCode {
     }
 }
 
-#[pyclass]
-impl Py<PyCode> {
-    #[pymethod(magic)]
-    fn repr(&self) -> String {
-        let code = &self.code;
-        format!(
-            "<code object {} at {:#x} file {:?}, line {}>",
-            code.obj_name,
-            self.get_id(),
-            code.source_path.as_str(),
-            code.first_line_number
-        )
-    }
-}
-
 impl fmt::Display for PyCode {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         (**self).fmt(f)

vm/src/builtins/descriptor.rs

@@ -32,7 +32,7 @@ pub enum MemberSetter {
     Offset(usize),
 }
 
-pub struct MemberDef {
+pub struct PyMemberDef {
     pub name: String,
     pub kind: MemberKind,
     pub getter: MemberGetter,
@@ -40,7 +40,7 @@ pub struct MemberDef {
     pub doc: Option<String>,
 }
 
-impl MemberDef {
+impl PyMemberDef {
     fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
         match self.getter {
             MemberGetter::Getter(getter) => (getter)(vm, obj),
@@ -64,9 +64,9 @@ impl MemberDef {
     }
 }
 
-impl std::fmt::Debug for MemberDef {
+impl std::fmt::Debug for PyMemberDef {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("MemberDef")
+        f.debug_struct("PyMemberDef")
             .field("name", &self.name)
             .field("kind", &self.kind)
             .field("doc", &self.doc)
@@ -74,14 +74,15 @@ impl std::fmt::Debug for MemberDef {
     }
 }
 
+// PyMemberDescrObject in CPython
 #[pyclass(name = "member_descriptor", module = false)]
 #[derive(Debug)]
-pub struct MemberDescrObject {
+pub struct PyMemberDescriptor {
     pub common: DescrObject,
-    pub member: MemberDef,
+    pub member: PyMemberDef,
 }
 
-impl PyPayload for MemberDescrObject {
+impl PyPayload for PyMemberDescriptor {
     fn class(ctx: &Context) -> &'static Py<PyType> {
         ctx.types.member_descriptor_type
     }
@@ -101,7 +102,7 @@ fn calculate_qualname(descr: &DescrObject, vm: &VirtualMachine) -> PyResult<Opti
 }
 
 #[pyclass(with(GetDescriptor, Constructor, Representable), flags(BASETYPE))]
-impl MemberDescrObject {
+impl PyMemberDescriptor {
     #[pygetset(magic)]
     fn doc(&self) -> Option<String> {
         self.member.doc.to_owned()
@@ -136,7 +137,7 @@ impl MemberDescrObject {
 fn get_slot_from_object(
     obj: PyObjectRef,
     offset: usize,
-    member: &MemberDef,
+    member: &PyMemberDef,
     vm: &VirtualMachine,
 ) -> PyResult {
     let slot = match member.kind {
@@ -158,7 +159,7 @@ fn get_slot_from_object(
 fn set_slot_at_object(
     obj: PyObjectRef,
     offset: usize,
-    member: &MemberDef,
+    member: &PyMemberDef,
     value: PySetterValue,
     vm: &VirtualMachine,
 ) -> PyResult<()> {
@@ -186,9 +187,9 @@ fn set_slot_at_object(
     Ok(())
 }
 
-impl Unconstructible for MemberDescrObject {}
+impl Unconstructible for PyMemberDescriptor {}
 
-impl Representable for MemberDescrObject {
+impl Representable for PyMemberDescriptor {
     #[inline]
     fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
         Ok(format!(
@@ -199,7 +200,7 @@ impl Representable for MemberDescrObject {
     }
 }
 
-impl GetDescriptor for MemberDescrObject {
+impl GetDescriptor for PyMemberDescriptor {
     fn descr_get(
         zelf: PyObjectRef,
         obj: Option<PyObjectRef>,
@@ -218,5 +219,5 @@ impl GetDescriptor for MemberDescrObject {
 
 pub fn init(context: &Context) {
     let member_descriptor_type = &context.types.member_descriptor_type;
-    MemberDescrObject::extend_class(context, member_descriptor_type);
+    PyMemberDescriptor::extend_class(context, member_descriptor_type);
 }

vm/src/builtins/dict.rs

@@ -32,7 +32,7 @@ use std::fmt;
 
 pub type DictContentType = dictdatatype::Dict;
 
-#[pyclass(module = false, name = "dict", unhashable = true)]
+#[pyclass(module = false, name = "dict", unhashable = true, traverse)]
 #[derive(Default)]
 pub struct PyDict {
     entries: DictContentType,

vm/src/builtins/enumerate.rs

@@ -13,9 +13,10 @@ use crate::{
 use num_bigint::BigInt;
 use num_traits::Zero;
 
-#[pyclass(module = false, name = "enumerate")]
+#[pyclass(module = false, name = "enumerate", traverse)]
 #[derive(Debug)]
 pub struct PyEnumerate {
+    #[pytraverse(skip)]
     counter: PyRwLock<BigInt>,
     iterator: PyIter,
 }
@@ -84,7 +85,7 @@ impl IterNext for PyEnumerate {
     }
 }
 
-#[pyclass(module = false, name = "reversed")]
+#[pyclass(module = false, name = "reversed", traverse)]
 #[derive(Debug)]
 pub struct PyReverseSequenceIterator {
     internal: PyMutex<PositionIterInternal<PyObjectRef>>,

vm/src/builtins/filter.rs

@@ -6,7 +6,7 @@ use crate::{
     Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
 };
 
-#[pyclass(module = false, name = "filter")]
+#[pyclass(module = false, name = "filter", traverse)]
 #[derive(Debug)]
 pub struct PyFilter {
     predicate: PyObjectRef,

vm/src/builtins/function.rs

@@ -10,6 +10,7 @@ use crate::common::lock::OnceCell;
 use crate::common::lock::PyMutex;
 use crate::convert::ToPyObject;
 use crate::function::ArgMapping;
+use crate::object::{Traverse, TraverseFn};
 use crate::{
     bytecode,
     class::PyClassImpl,
@@ -25,7 +26,7 @@ use itertools::Itertools;
 #[cfg(feature = "jit")]
 use rustpython_jit::CompiledCode;
 
-#[pyclass(module = false, name = "function")]
+#[pyclass(module = false, name = "function", traverse = "manual")]
 #[derive(Debug)]
 pub struct PyFunction {
     code: PyRef<PyCode>,
@@ -38,6 +39,14 @@ pub struct PyFunction {
     jitted_code: OnceCell<CompiledCode>,
 }
 
+unsafe impl Traverse for PyFunction {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.globals.traverse(tracer_fn);
+        self.closure.traverse(tracer_fn);
+        self.defaults_and_kwdefaults.traverse(tracer_fn);
+    }
+}
+
 impl PyFunction {
     pub(crate) fn new(
         code: PyRef<PyCode>,
@@ -468,7 +477,7 @@ impl Representable for PyFunction {
     }
 }
 
-#[pyclass(module = false, name = "method")]
+#[pyclass(module = false, name = "method", traverse)]
 #[derive(Debug)]
 pub struct PyBoundMethod {
     object: PyObjectRef,
@@ -513,7 +522,7 @@ impl GetAttr for PyBoundMethod {
     }
 }
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 pub struct PyBoundMethodNewArgs {
     #[pyarg(positional)]
     function: PyObjectRef,
@@ -549,25 +558,11 @@ impl PyBoundMethod {
     }
 }
 
-#[pyclass(with(Callable, Comparable, GetAttr, Constructor), flags(HAS_DICT))]
+#[pyclass(
+    with(Callable, Comparable, GetAttr, Constructor, Representable),
+    flags(HAS_DICT)
+)]
 impl PyBoundMethod {
-    #[pymethod(magic)]
-    fn repr(&self, vm: &VirtualMachine) -> PyResult<String> {
-        #[allow(clippy::needless_match)] // False positive on nightly
-        let funcname =
-            if let Some(qname) = vm.get_attribute_opt(self.function.clone(), "__qualname__")? {
-                Some(qname)
-            } else {
-                vm.get_attribute_opt(self.function.clone(), "__name__")?
-            };
-        let funcname: Option<PyStrRef> = funcname.and_then(|o| o.downcast().ok());
-        Ok(format!(
-            "<bound method {} of {}>",
-            funcname.as_ref().map_or("?", |s| s.as_str()),
-            &self.object.repr(vm)?.as_str(),
-        ))
-    }
-
     #[pymethod(magic)]
     fn reduce(
         &self,
@@ -628,7 +623,26 @@ impl PyPayload for PyBoundMethod {
     }
 }
 
-#[pyclass(module = false, name = "cell")]
+impl Representable for PyBoundMethod {
+    #[inline]
+    fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
+        #[allow(clippy::needless_match)] // False positive on nightly
+        let funcname =
+            if let Some(qname) = vm.get_attribute_opt(zelf.function.clone(), "__qualname__")? {
+                Some(qname)
+            } else {
+                vm.get_attribute_opt(zelf.function.clone(), "__name__")?
+            };
+        let funcname: Option<PyStrRef> = funcname.and_then(|o| o.downcast().ok());
+        Ok(format!(
+            "<bound method {} of {}>",
+            funcname.as_ref().map_or("?", |s| s.as_str()),
+            &zelf.object.repr(vm)?.as_str(),
+        ))
+    }
+}
+
+#[pyclass(module = false, name = "cell", traverse)]
 #[derive(Debug, Default)]
 pub(crate) struct PyCell {
     contents: PyMutex<Option<PyObjectRef>>,

vm/src/builtins/iter.rs

@@ -6,6 +6,7 @@ use super::{PyInt, PyTupleRef, PyType};
 use crate::{
     class::PyClassImpl,
     function::ArgCallable,
+    object::{Traverse, TraverseFn},
     protocol::{PyIterReturn, PySequence, PySequenceMethods},
     types::{IterNext, IterNextIterable},
     Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
@@ -24,12 +25,27 @@ pub enum IterStatus<T> {
     Exhausted,
 }
 
+unsafe impl<T: Traverse> Traverse for IterStatus<T> {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        match self {
+            IterStatus::Active(ref r) => r.traverse(tracer_fn),
+            IterStatus::Exhausted => (),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct PositionIterInternal<T> {
     pub status: IterStatus<T>,
     pub position: usize,
 }
 
+unsafe impl<T: Traverse> Traverse for PositionIterInternal<T> {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.status.traverse(tracer_fn)
+    }
+}
+
 impl<T> PositionIterInternal<T> {
     pub fn new(obj: T, position: usize) -> Self {
         Self {
@@ -158,10 +174,11 @@ pub fn builtins_reversed(vm: &VirtualMachine) -> &PyObject {
     INSTANCE.get_or_init(|| vm.builtins.get_attr("reversed", vm).unwrap())
 }
 
-#[pyclass(module = false, name = "iterator")]
+#[pyclass(module = false, name = "iterator", traverse)]
 #[derive(Debug)]
 pub struct PySequenceIterator {
     // cached sequence methods
+    #[pytraverse(skip)]
     seq_methods: &'static PySequenceMethods,
     internal: PyMutex<PositionIterInternal<PyObjectRef>>,
 }
@@ -222,7 +239,7 @@ impl IterNext for PySequenceIterator {
     }
 }
 
-#[pyclass(module = false, name = "callable_iterator")]
+#[pyclass(module = false, name = "callable_iterator", traverse)]
 #[derive(Debug)]
 pub struct PyCallableIterator {
     sentinel: PyObjectRef,

vm/src/builtins/list.rs

@@ -22,7 +22,7 @@ use crate::{
 };
 use std::{fmt, ops::DerefMut};
 
-#[pyclass(module = false, name = "list", unhashable = true)]
+#[pyclass(module = false, name = "list", unhashable = true, traverse)]
 #[derive(Default)]
 pub struct PyList {
     elements: PyRwLock<Vec<PyObjectRef>>,
@@ -86,10 +86,11 @@ impl PyList {
     }
 }
 
-#[derive(FromArgs, Default)]
+#[derive(FromArgs, Default, Traverse)]
 pub(crate) struct SortOptions {
     #[pyarg(named, default)]
     key: Option<PyObjectRef>,
+    #[pytraverse(skip)]
     #[pyarg(named, default = "false")]
     reverse: bool,
 }
@@ -530,7 +531,7 @@ fn do_sort(
     Ok(())
 }
 
-#[pyclass(module = false, name = "list_iterator")]
+#[pyclass(module = false, name = "list_iterator", traverse)]
 #[derive(Debug)]
 pub struct PyListIterator {
     internal: PyMutex<PositionIterInternal<PyListRef>>,
@@ -575,7 +576,7 @@ impl IterNext for PyListIterator {
     }
 }
 
-#[pyclass(module = false, name = "list_reverseiterator")]
+#[pyclass(module = false, name = "list_reverseiterator", traverse)]
 #[derive(Debug)]
 pub struct PyListReverseIterator {
     internal: PyMutex<PositionIterInternal<PyListRef>>,

vm/src/builtins/map.rs

@@ -8,7 +8,7 @@ use crate::{
     Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
 };
 
-#[pyclass(module = false, name = "map")]
+#[pyclass(module = false, name = "map", traverse)]
 #[derive(Debug)]
 pub struct PyMap {
     mapper: PyObjectRef,

vm/src/builtins/mappingproxy.rs

@@ -4,6 +4,7 @@ use crate::{
     class::PyClassImpl,
     convert::ToPyObject,
     function::{ArgMapping, OptionalArg, PyComparisonValue},
+    object::{Traverse, TraverseFn},
     protocol::{PyMapping, PyMappingMethods, PyNumberMethods, PySequenceMethods},
     types::{
         AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp,
@@ -13,7 +14,7 @@ use crate::{
 };
 use once_cell::sync::Lazy;
 
-#[pyclass(module = false, name = "mappingproxy")]
+#[pyclass(module = false, name = "mappingproxy", traverse)]
 #[derive(Debug)]
 pub struct PyMappingProxy {
     mapping: MappingProxyInner,
@@ -25,6 +26,15 @@ enum MappingProxyInner {
     Mapping(ArgMapping),
 }
 
+unsafe impl Traverse for MappingProxyInner {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        match self {
+            MappingProxyInner::Class(ref r) => r.traverse(tracer_fn),
+            MappingProxyInner::Mapping(ref arg) => arg.traverse(tracer_fn),
+        }
+    }
+}
+
 impl PyPayload for PyMappingProxy {
     fn class(ctx: &Context) -> &'static Py<PyType> {
         ctx.types.mappingproxy_type

vm/src/builtins/memory.rs

@@ -33,7 +33,7 @@ use once_cell::sync::Lazy;
 use rustpython_common::lock::PyMutex;
 use std::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range};
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 pub struct PyMemoryViewNewArgs {
     object: PyObjectRef,
 }
@@ -896,7 +896,7 @@ impl Py<PyMemoryView> {
     }
 }
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 struct CastArgs {
     #[pyarg(any)]
     format: PyStrRef,
@@ -1126,7 +1126,7 @@ impl Iterable for PyMemoryView {
 }
 
 #[pyclass(module = false, name = "memory_iterator")]
-#[derive(Debug)]
+#[derive(Debug, Traverse)]
 pub struct PyMemoryViewIterator {
     internal: PyMutex<PositionIterInternal<PyRef<PyMemoryView>>>,
 }

vm/src/builtins/mod.rs

@@ -47,7 +47,7 @@ pub use mappingproxy::PyMappingProxy;
 pub(crate) mod memory;
 pub use memory::PyMemoryView;
 pub(crate) mod module;
-pub use module::PyModule;
+pub use module::{PyModule, PyModuleDef};
 pub(crate) mod namespace;
 pub use namespace::PyNamespace;
 pub(crate) mod object;

vm/src/builtins/module.rs

@@ -5,12 +5,53 @@ use crate::{
     convert::ToPyObject,
     function::FuncArgs,
     types::{GetAttr, Initializer, Representable},
-    AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
+    AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
 };
 
 #[pyclass(module = false, name = "module")]
 #[derive(Debug)]
-pub struct PyModule {}
+pub struct PyModuleDef {
+    // pub index: usize,
+    pub name: &'static PyStrInterned,
+    pub doc: Option<&'static PyStrInterned>,
+    // pub size: isize,
+    // pub methods: &'static [PyMethodDef],
+    pub slots: PyModuleSlots,
+    // traverse: traverseproc
+    // clear: inquiry
+    // free: freefunc
+}
+
+pub type ModuleCreate =
+    fn(&VirtualMachine, &PyObject, &'static PyModuleDef) -> PyResult<PyRef<PyModule>>;
+pub type ModuleExec = fn(&VirtualMachine, &Py<PyModule>) -> PyResult<()>;
+
+#[derive(Default)]
+pub struct PyModuleSlots {
+    pub create: Option<ModuleCreate>,
+    pub exec: Option<ModuleExec>,
+}
+
+impl std::fmt::Debug for PyModuleSlots {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PyModuleSlots")
+            .field("create", &self.create.is_some())
+            .field("exec", &self.exec.is_some())
+            .finish()
+    }
+}
+
+#[allow(clippy::new_without_default)] // avoid Default implementation
+#[pyclass(module = false, name = "module")]
+#[derive(Debug)]
+pub struct PyModule {
+    // PyObject *md_dict;
+    pub def: Option<&'static PyModuleDef>,
+    // state: Any
+    // weaklist
+    // for logging purposes after md_dict is cleared
+    pub name: Option<&'static PyStrInterned>,
+}
 
 impl PyPayload for PyModule {
     fn class(ctx: &Context) -> &'static Py<PyType> {
@@ -26,9 +67,23 @@ pub struct ModuleInitArgs {
 }
 
 impl PyModule {
-    // pub(crate) fn new(d: PyDictRef) -> Self {
-    //     PyModule { dict: d.into() }
-    // }
+    #[allow(clippy::new_without_default)]
+    pub fn new() -> Self {
+        Self {
+            def: None,
+            name: None,
+        }
+    }
+    pub fn from_def(def: &'static PyModuleDef) -> Self {
+        Self {
+            def: Some(def),
+            name: Some(def.name),
+        }
+    }
+    pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py<PyModule>) {
+        let doc = module.def.unwrap().doc.map(|doc| doc.to_owned());
+        module.init_module_dict(module.name.unwrap(), doc, vm);
+    }
 }
 
 impl Py<PyModule> {
@@ -63,13 +118,13 @@ impl Py<PyModule> {
     pub(crate) fn init_module_dict(
         &self,
         name: &'static PyStrInterned,
-        doc: PyObjectRef,
+        doc: Option<PyStrRef>,
         vm: &VirtualMachine,
     ) {
         let dict = self.dict();
         dict.set_item(identifier!(vm, __name__), name.to_object(), vm)
             .expect("Failed to set __name__ on module");
-        dict.set_item(identifier!(vm, __doc__), doc, vm)
+        dict.set_item(identifier!(vm, __doc__), doc.to_pyobject(vm), vm)
             .expect("Failed to set __doc__ on module");
         dict.set_item("__package__", vm.ctx.none(), vm)
             .expect("Failed to set __package__ on module");
@@ -80,7 +135,8 @@ impl Py<PyModule> {
     }
 
     pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult {
-        self.getattr_inner(attr_name.as_pystr(&vm.ctx), vm)
+        let attr_name = attr_name.as_pystr(&vm.ctx);
+        self.getattr_inner(attr_name, vm)
     }
 
     pub fn set_attr<'a>(
@@ -97,7 +153,7 @@ impl Py<PyModule> {
 impl PyModule {
     #[pyslot]
     fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult {
-        PyModule {}.into_ref_with_type(vm, cls).map(Into::into)
+        PyModule::new().into_ref_with_type(vm, cls).map(Into::into)
     }
 
     #[pymethod(magic)]
@@ -120,11 +176,7 @@ impl Initializer for PyModule {
             .slots
             .flags
             .has_feature(crate::types::PyTypeFlags::HAS_DICT));
-        zelf.init_module_dict(
-            vm.ctx.intern_str(args.name.as_str()),
-            args.doc.to_pyobject(vm),
-            vm,
-        );
+        zelf.init_module_dict(vm.ctx.intern_str(args.name.as_str()), args.doc, vm);
         Ok(())
     }
 }

vm/src/builtins/property.rs

@@ -11,7 +11,7 @@ use crate::{
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
 };
 
-#[pyclass(module = false, name = "property")]
+#[pyclass(module = false, name = "property", traverse)]
 #[derive(Debug)]
 pub struct PyProperty {
     getter: PyRwLock<Option<PyObjectRef>>,
@@ -51,7 +51,7 @@ impl GetDescriptor for PyProperty {
         } else if let Some(getter) = zelf.getter.read().as_ref() {
             getter.call((obj,), vm)
         } else {
-            Err(vm.new_attribute_error("unreadable attribute".to_string()))
+            Err(vm.new_attribute_error("property has no getter".to_string()))
         }
     }
 }
@@ -73,14 +73,14 @@ impl PyProperty {
                 if let Some(setter) = zelf.setter.read().as_ref() {
                     setter.call((obj, value), vm).map(drop)
                 } else {
-                    Err(vm.new_attribute_error("can't set attribute".to_owned()))
+                    Err(vm.new_attribute_error("property has no setter".to_owned()))
                 }
             }
             PySetterValue::Delete => {
                 if let Some(deleter) = zelf.deleter.read().as_ref() {
                     deleter.call((obj,), vm).map(drop)
                 } else {
-                    Err(vm.new_attribute_error("can't delete attribute".to_owned()))
+                    Err(vm.new_attribute_error("property has no deleter".to_owned()))
                 }
             }
         }

vm/src/builtins/set.rs

@@ -28,7 +28,7 @@ use std::{fmt, ops::Deref};
 
 pub type SetContentType = dictdatatype::Dict<()>;
 
-#[pyclass(module = false, name = "set", unhashable = true)]
+#[pyclass(module = false, name = "set", unhashable = true, traverse)]
 #[derive(Default)]
 pub struct PySet {
     pub(super) inner: PySetInner,
@@ -151,6 +151,13 @@ pub(super) struct PySetInner {
     content: PyRc<SetContentType>,
 }
 
+unsafe impl crate::object::Traverse for PySetInner {
+    fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) {
+        // FIXME(discord9): Rc means shared ref, so should it be traced?
+        self.content.traverse(tracer_fn)
+    }
+}
+
 impl PySetInner {
     pub(super) fn from_iter<T>(iter: T, vm: &VirtualMachine) -> PyResult<Self>
     where

vm/src/builtins/slice.rs

@@ -12,7 +12,7 @@ use crate::{
 use num_bigint::{BigInt, ToBigInt};
 use num_traits::{One, Signed, Zero};
 
-#[pyclass(module = false, name = "slice", unhashable = true)]
+#[pyclass(module = false, name = "slice", unhashable = true, traverse)]
 #[derive(Debug)]
 pub struct PySlice {
     pub start: Option<PyObjectRef>,
@@ -244,6 +244,7 @@ impl Comparable for PySlice {
         Ok(PyComparisonValue::Implemented(ret))
     }
 }
+
 impl Representable for PySlice {
     #[inline]
     fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {

vm/src/builtins/staticmethod.rs

@@ -8,7 +8,7 @@ use crate::{
     Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
 };
 
-#[pyclass(module = false, name = "staticmethod")]
+#[pyclass(module = false, name = "staticmethod", traverse)]
 #[derive(Debug)]
 pub struct PyStaticMethod {
     pub callable: PyMutex<PyObjectRef>,

vm/src/builtins/str.rs

@@ -15,6 +15,7 @@ use crate::{
     format::{format, format_map},
     function::{ArgIterable, ArgSize, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue},
     intern::PyInterned,
+    object::{Traverse, TraverseFn},
     protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods},
     sequence::SequenceExt,
     sliceable::{SequenceIndex, SliceableSequenceOp},
@@ -186,12 +187,19 @@ impl<'a> AsPyStr<'a> for &'a PyStrInterned {
     }
 }
 
-#[pyclass(module = false, name = "str_iterator")]
+#[pyclass(module = false, name = "str_iterator", traverse = "manual")]
 #[derive(Debug)]
 pub struct PyStrIterator {
     internal: PyMutex<(PositionIterInternal<PyStrRef>, usize)>,
 }
 
+unsafe impl Traverse for PyStrIterator {
+    fn traverse(&self, tracer: &mut TraverseFn) {
+        // No need to worry about deadlock, for inner is a PyStr and can't make ref cycle
+        self.internal.lock().0.traverse(tracer);
+    }
+}
+
 impl PyPayload for PyStrIterator {
     fn class(ctx: &Context) -> &'static Py<PyType> {
         ctx.types.str_iterator_type
@@ -251,7 +259,7 @@ impl IterNext for PyStrIterator {
     }
 }
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 pub struct StrArgs {
     #[pyarg(any, optional)]
     object: OptionalArg<PyObjectRef>,
@@ -310,8 +318,9 @@ impl PyStr {
         Self::new_str_unchecked(bytes, PyStrKind::Ascii)
     }
 
-    pub fn new_ref(s: impl Into<Self>, ctx: &Context) -> PyRef<Self> {
-        PyRef::new_ref(s.into(), ctx.types.str_type.to_owned(), None)
+    pub fn new_ref(zelf: impl Into<Self>, ctx: &Context) -> PyRef<Self> {
+        let zelf = zelf.into();
+        PyRef::new_ref(zelf, ctx.types.str_type.to_owned(), None)
     }
 
     fn new_substr(&self, s: String) -> Self {
@@ -1392,7 +1401,7 @@ impl AsSequence for PyStr {
     }
 }
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 struct EncodeArgs {
     #[pyarg(any, default)]
     encoding: Option<PyStrRef>,
@@ -1456,7 +1465,7 @@ impl ToPyObject for AsciiString {
 
 type SplitArgs = anystr::SplitArgs<PyStrRef>;
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 pub struct FindArgs {
     #[pyarg(positional)]
     sub: PyStrRef,

vm/src/builtins/super.rs

@@ -6,40 +6,74 @@ See also [CPython source code.](https://github.com/python/cpython/blob/50b48572d
 use super::{PyStr, PyType, PyTypeRef};
 use crate::{
     class::PyClassImpl,
-    function::{IntoFuncArgs, OptionalArg},
-    types::{Callable, Constructor, GetAttr, GetDescriptor, Representable},
-    AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
+    common::lock::PyRwLock,
+    function::{FuncArgs, IntoFuncArgs, OptionalArg},
+    types::{Callable, Constructor, GetAttr, GetDescriptor, Initializer, Representable},
+    AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
 };
 
-#[pyclass(module = false, name = "super")]
+#[pyclass(module = false, name = "super", traverse)]
 #[derive(Debug)]
 pub struct PySuper {
+    inner: PyRwLock<PySuperInner>,
+}
+
+#[derive(Debug, Traverse)]
+struct PySuperInner {
     typ: PyTypeRef,
     obj: Option<(PyObjectRef, PyTypeRef)>,
 }
 
+impl PySuperInner {
+    fn new(typ: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> {
+        let obj = if vm.is_none(&obj) {
+            None
+        } else {
+            let obj_type = supercheck(typ.clone(), obj.clone(), vm)?;
+            Some((obj, obj_type))
+        };
+        Ok(Self { typ, obj })
+    }
+}
+
 impl PyPayload for PySuper {
     fn class(ctx: &Context) -> &'static Py<PyType> {
         ctx.types.super_type
     }
 }
 
+impl Constructor for PySuper {
+    type Args = FuncArgs;
+
+    fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult {
+        let obj = PySuper {
+            inner: PyRwLock::new(PySuperInner::new(
+                vm.ctx.types.object_type.to_owned(), // is this correct?
+                vm.ctx.none(),
+                vm,
+            )?),
+        }
+        .into_ref_with_type(vm, cls)?;
+        Ok(obj.into())
+    }
+}
+
 #[derive(FromArgs)]
-pub struct PySuperNewArgs {
+pub struct InitArgs {
     #[pyarg(positional, optional)]
     py_type: OptionalArg<PyTypeRef>,
     #[pyarg(positional, optional)]
     py_obj: OptionalArg<PyObjectRef>,
 }
 
-impl Constructor for PySuper {
-    type Args = PySuperNewArgs;
+impl Initializer for PySuper {
+    type Args = InitArgs;
 
-    fn py_new(
-        cls: PyTypeRef,
+    fn init(
+        zelf: PyRef<Self>,
         Self::Args { py_type, py_obj }: Self::Args,
         vm: &VirtualMachine,
-    ) -> PyResult {
+    ) -> PyResult<()> {
         // Get the type:
         let (typ, obj) = if let OptionalArg::Present(ty) = py_type {
             (ty, py_obj.unwrap_or_none(vm))
@@ -91,44 +125,36 @@ impl Constructor for PySuper {
             (typ, obj)
         };
 
-        PySuper::new(typ, obj, vm)?
-            .into_ref_with_type(vm, cls)
-            .map(Into::into)
+        let mut inner = PySuperInner::new(typ, obj, vm)?;
+        std::mem::swap(&mut inner, &mut zelf.inner.write());
+
+        Ok(())
     }
 }
 
-#[pyclass(with(GetAttr, GetDescriptor, Constructor, Representable))]
+#[pyclass(with(GetAttr, GetDescriptor, Constructor, Initializer, Representable))]
 impl PySuper {
-    fn new(typ: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> {
-        let obj = if vm.is_none(&obj) {
-            None
-        } else {
-            let obj_type = supercheck(typ.clone(), obj.clone(), vm)?;
-            Some((obj, obj_type))
-        };
-        Ok(Self { typ, obj })
-    }
-
     #[pygetset(magic)]
     fn thisclass(&self) -> PyTypeRef {
-        self.typ.clone()
+        self.inner.read().typ.clone()
     }
 
     #[pygetset(magic)]
     fn self_class(&self) -> Option<PyTypeRef> {
-        Some(self.obj.as_ref()?.1.clone())
+        Some(self.inner.read().obj.as_ref()?.1.clone())
     }
 
     #[pygetset]
     fn __self__(&self) -> Option<PyObjectRef> {
-        Some(self.obj.as_ref()?.0.clone())
+        Some(self.inner.read().obj.as_ref()?.0.clone())
     }
 }
 
 impl GetAttr for PySuper {
     fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult {
         let skip = |zelf: &Py<Self>, name| zelf.as_object().generic_getattr(name, vm);
-        let (obj, start_type): (PyObjectRef, PyTypeRef) = match zelf.obj.clone() {
+        let obj = zelf.inner.read().obj.clone();
+        let (obj, start_type): (PyObjectRef, PyTypeRef) = match obj {
             Some(o) => o,
             None => return skip(zelf, name),
         };
@@ -143,7 +169,7 @@ impl GetAttr for PySuper {
             // skip the classes in start_type.mro up to and including zelf.typ
             let mro: Vec<_> = start_type
                 .iter_mro()
-                .skip_while(|cls| !cls.is(&zelf.typ))
+                .skip_while(|cls| !cls.is(&zelf.inner.read().typ))
                 .skip(1) // skip su->type (if any)
                 .collect();
             for cls in mro {
@@ -171,15 +197,26 @@ impl GetDescriptor for PySuper {
         vm: &VirtualMachine,
     ) -> PyResult {
         let (zelf, obj) = Self::_unwrap(&zelf_obj, obj, vm)?;
-        if vm.is_none(&obj) || zelf.obj.is_some() {
+        if vm.is_none(&obj) || zelf.inner.read().obj.is_some() {
             return Ok(zelf_obj);
         }
         let zelf_class = zelf.as_object().class();
         if zelf_class.is(vm.ctx.types.super_type) {
-            Ok(PySuper::new(zelf.typ.clone(), obj, vm)?.into_pyobject(vm))
+            let typ = zelf.inner.read().typ.clone();
+            Ok(PySuper {
+                inner: PyRwLock::new(PySuperInner::new(typ, obj, vm)?),
+            }
+            .into_ref(&vm.ctx)
+            .into())
         } else {
-            let obj = vm.unwrap_or_none(zelf.obj.clone().map(|(o, _)| o));
-            PyType::call(zelf.class(), (zelf.typ.clone(), obj).into_args(vm), vm)
+            let (obj, typ) = {
+                let lock = zelf.inner.read();
+                let obj = lock.obj.as_ref().map(|(o, _)| o.to_owned());
+                let typ = lock.typ.clone();
+                (obj, typ)
+            };
+            let obj = vm.unwrap_or_none(obj);
+            PyType::call(zelf.class(), (typ, obj).into_args(vm), vm)
         }
     }
 }
@@ -187,10 +224,11 @@ impl GetDescriptor for PySuper {
 impl Representable for PySuper {
     #[inline]
     fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
-        let type_name = &zelf.typ.name();
-        let repr = match zelf.obj {
+        let type_name = zelf.inner.read().typ.name().to_owned();
+        let obj = zelf.inner.read().obj.clone();
+        let repr = match obj {
             Some((_, ref ty)) => {
-                format!("<super: <class '{}'>, <{} object>>", type_name, ty.name())
+                format!("<super: <class '{}'>, <{} object>>", &type_name, ty.name())
             }
             None => format!("<super: <class '{type_name}'>, NULL>"),
         };

vm/src/builtins/traceback.rs

@@ -3,12 +3,14 @@ use rustpython_common::lock::PyMutex;
 use super::PyType;
 use crate::{class::PyClassImpl, frame::FrameRef, Context, Py, PyPayload, PyRef};
 
-#[pyclass(module = false, name = "traceback")]
+#[pyclass(module = false, name = "traceback", traverse)]
 #[derive(Debug)]
 pub struct PyTraceback {
     pub next: PyMutex<Option<PyTracebackRef>>,
     pub frame: FrameRef,
+    #[pytraverse(skip)]
     pub lasti: u32,
+    #[pytraverse(skip)]
     pub lineno: usize,
 }
 

vm/src/builtins/tuple.rs

@@ -1,5 +1,6 @@
 use super::{PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef};
 use crate::common::{hash::PyHash, lock::PyMutex};
+use crate::object::{Traverse, TraverseFn};
 use crate::{
     atomic_func,
     class::PyClassImpl,
@@ -21,7 +22,7 @@ use crate::{
 use once_cell::sync::Lazy;
 use std::{fmt, marker::PhantomData};
 
-#[pyclass(module = false, name = "tuple")]
+#[pyclass(module = false, name = "tuple", traverse)]
 pub struct PyTuple {
     elements: Box<[PyObjectRef]>,
 }
@@ -421,7 +422,7 @@ impl Representable for PyTuple {
     }
 }
 
-#[pyclass(module = false, name = "tuple_iterator")]
+#[pyclass(module = false, name = "tuple_iterator", traverse)]
 #[derive(Debug)]
 pub(crate) struct PyTupleIterator {
     internal: PyMutex<PositionIterInternal<PyTupleRef>>,
@@ -479,6 +480,15 @@ pub struct PyTupleTyped<T: TransmuteFromObject> {
     _marker: PhantomData<Vec<T>>,
 }
 
+unsafe impl<T> Traverse for PyTupleTyped<T>
+where
+    T: TransmuteFromObject + Traverse,
+{
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.tuple.traverse(tracer_fn);
+    }
+}
+
 impl<T: TransmuteFromObject> TryFromObject for PyTupleTyped<T> {
     fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
         let tuple = PyTupleRef::try_from_object(vm, obj)?;

vm/src/builtins/type.rs

@@ -5,7 +5,7 @@ use super::{
 use crate::{
     builtins::{
         descriptor::{
-            DescrObject, MemberDef, MemberDescrObject, MemberGetter, MemberKind, MemberSetter,
+            DescrObject, MemberGetter, MemberKind, MemberSetter, PyMemberDef, PyMemberDescriptor,
         },
         function::PyCellRef,
         tuple::{IntoPyTuple, PyTupleTyped},
@@ -20,6 +20,7 @@ use crate::{
     convert::ToPyResult,
     function::{FuncArgs, KwArgs, OptionalArg, PySetterValue},
     identifier,
+    object::{Traverse, TraverseFn},
     protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods},
     types::{AsNumber, Callable, GetAttr, PyTypeFlags, PyTypeSlots, Representable, SetAttr},
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
@@ -29,7 +30,7 @@ use indexmap::{map::Entry, IndexMap};
 use itertools::Itertools;
 use std::{borrow::Borrow, collections::HashSet, fmt, ops::Deref, pin::Pin, ptr::NonNull};
 
-#[pyclass(module = false, name = "type")]
+#[pyclass(module = false, name = "type", traverse = "manual")]
 pub struct PyType {
     pub base: Option<PyTypeRef>,
     pub bases: Vec<PyTypeRef>,
@@ -40,6 +41,20 @@ pub struct PyType {
     pub heaptype_ext: Option<Pin<Box<HeapTypeExt>>>,
 }
 
+unsafe impl crate::object::Traverse for PyType {
+    fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) {
+        self.base.traverse(tracer_fn);
+        self.bases.traverse(tracer_fn);
+        self.mro.traverse(tracer_fn);
+        self.subclasses.traverse(tracer_fn);
+        self.attributes
+            .read_recursive()
+            .iter()
+            .map(|(_, v)| v.traverse(tracer_fn))
+            .count();
+    }
+}
+
 pub struct HeapTypeExt {
     pub name: PyRwLock<PyStrRef>,
     pub slots: Option<PyTupleTyped<PyStrRef>>,
@@ -100,6 +115,12 @@ cfg_if::cfg_if! {
 /// faster and only supports strings as keys.
 pub type PyAttributes = IndexMap<&'static PyStrInterned, PyObjectRef, ahash::RandomState>;
 
+unsafe impl Traverse for PyAttributes {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.values().for_each(|v| v.traverse(tracer_fn));
+    }
+}
+
 impl fmt::Display for PyType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt::Display::fmt(&self.name(), f)
@@ -301,7 +322,8 @@ impl PyType {
         value: V,
         ctx: impl AsRef<Context>,
     ) {
-        let attr_name = ctx.as_ref().intern_str(attr_name);
+        let ctx = ctx.as_ref();
+        let attr_name = ctx.intern_str(attr_name);
         self.set_attr(attr_name, value.into())
     }
 
@@ -753,12 +775,11 @@ impl PyType {
             base.slots.member_count + heaptype_slots.as_ref().map(|x| x.len()).unwrap_or(0);
 
         let flags = PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT;
-        let (slots, heaptype_ext) = unsafe {
-            // # Safety
-            // `slots.name` live long enough because `heaptype_ext` is alive.
+        let (slots, heaptype_ext) = {
             let slots = PyTypeSlots {
                 member_count,
-                ..PyTypeSlots::new(&*(name.as_str() as *const _), flags)
+                flags,
+                ..PyTypeSlots::heap_default()
             };
             let heaptype_ext = HeapTypeExt {
                 name: PyRwLock::new(name),
@@ -783,15 +804,15 @@ impl PyType {
         if let Some(ref slots) = heaptype_slots {
             let mut offset = base_member_count;
             for member in slots.as_slice() {
-                let member_def = MemberDef {
+                let member_def = PyMemberDef {
                     name: member.to_string(),
                     kind: MemberKind::ObjectEx,
                     getter: MemberGetter::Offset(offset),
                     setter: MemberSetter::Offset(offset),
                     doc: None,
                 };
-                let member_descriptor: PyRef<MemberDescrObject> =
-                    vm.ctx.new_pyref(MemberDescrObject {
+                let member_descriptor: PyRef<PyMemberDescriptor> =
+                    vm.ctx.new_pyref(PyMemberDescriptor {
                         common: DescrObject {
                             typ: typ.clone(),
                             name: vm.ctx.intern_str(member.as_str()),

vm/src/builtins/union.rs

@@ -15,7 +15,7 @@ use std::fmt;
 
 const CLS_ATTRS: &[&str] = &["__module__"];
 
-#[pyclass(module = "types", name = "UnionType")]
+#[pyclass(module = "types", name = "UnionType", traverse)]
 pub struct PyUnion {
     args: PyTupleRef,
     parameters: PyTupleRef,

vm/src/builtins/weakproxy.rs

@@ -13,7 +13,7 @@ use crate::{
 };
 use once_cell::sync::Lazy;
 
-#[pyclass(module = false, name = "weakproxy", unhashable = true)]
+#[pyclass(module = false, name = "weakproxy", unhashable = true, traverse)]
 #[derive(Debug)]
 pub struct PyWeakProxy {
     weak: PyRef<PyWeak>,

vm/src/builtins/weakref.rs

@@ -12,7 +12,7 @@ use crate::{
 
 pub use crate::object::PyWeak;
 
-#[derive(FromArgs)]
+#[derive(FromArgs, Traverse)]
 pub struct WeakNewArgs {
     #[pyarg(positional)]
     referent: PyObjectRef,

vm/src/builtins/zip.rs

@@ -9,10 +9,11 @@ use crate::{
 };
 use rustpython_common::atomic::{self, PyAtomic, Radium};
 
-#[pyclass(module = false, name = "zip")]
+#[pyclass(module = false, name = "zip", traverse)]
 #[derive(Debug)]
 pub struct PyZip {
     iterators: Vec<PyIter>,
+    #[pytraverse(skip)]
     strict: PyAtomic<bool>,
 }
 

vm/src/class.rs

@@ -60,6 +60,10 @@ pub trait PyClassDef {
     const DOC: Option<&'static str> = None;
     const BASICSIZE: usize;
     const UNHASHABLE: bool = false;
+
+    // due to restriction of rust trait system, object.__base__ is None
+    // but PyBaseObject::Base will be PyBaseObject.
+    type Base: PyClassDef;
 }
 
 pub trait PyClassImpl: PyClassDef {
@@ -98,14 +102,12 @@ pub trait PyClassImpl: PyClassDef {
                 ctx.new_str(module_name).into(),
             );
         }
-        if class.slots.new.load().is_some() {
-            let bound = PyBoundMethod::new_ref(
-                class.to_owned().into(),
-                ctx.slot_new_wrapper.clone().into(),
-                ctx,
-            );
-            class.set_attr(identifier!(ctx, __new__), bound.into());
-        }
+        let bound_new = PyBoundMethod::new_ref(
+            class.to_owned().into(),
+            ctx.slot_new_wrapper.clone().into(),
+            ctx,
+        );
+        class.set_attr(identifier!(ctx, __new__), bound_new.into());
 
         if class.slots.hash.load().map_or(0, |h| h as usize) == hash_not_implemented as usize {
             class.set_attr(ctx.names.__hash__, ctx.none.clone().into());

vm/src/dictdatatype.rs

@@ -3,15 +3,18 @@
 //! And: https://www.youtube.com/watch?v=p33CVV29OG8
 //! And: http://code.activestate.com/recipes/578375/
 
-use crate::common::{
-    hash,
-    lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard},
-};
 use crate::{
     builtins::{PyInt, PyStr, PyStrInterned, PyStrRef},
     convert::ToPyObject,
     AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine,
 };
+use crate::{
+    common::{
+        hash,
+        lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard},
+    },
+    object::{Traverse, TraverseFn},
+};
 use num_traits::ToPrimitive;
 use std::{fmt, mem::size_of, ops::ControlFlow};
 
@@ -31,6 +34,12 @@ pub struct Dict<T = PyObjectRef> {
     inner: PyRwLock<DictInner<T>>,
 }
 
+unsafe impl<T: Traverse> Traverse for Dict<T> {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.inner.traverse(tracer_fn);
+    }
+}
+
 impl<T> fmt::Debug for Dict<T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("Debug").finish()
@@ -69,6 +78,20 @@ struct DictInner<T> {
     entries: Vec<Option<DictEntry<T>>>,
 }
 
+unsafe impl<T: Traverse> Traverse for DictInner<T> {
+    fn traverse(&self, tracer_fn: &mut TraverseFn) {
+        self.entries
+            .iter()
+            .map(|v| {
+                if let Some(v) = v {
+                    v.key.traverse(tracer_fn);
+                    v.value.traverse(tracer_fn);
+                }
+            })
+            .count();
+    }
+}
+
 impl<T: Clone> Clone for Dict<T> {
     fn clone(&self) -> Self {
         Self {