Fix SHAKE digest length overflow in hashlib (#7125)

Lib/test/test_hashlib.py

@@ -295,8 +295,6 @@ class HashLibTestCase(unittest.TestCase):
                 self.assertIsInstance(h.digest(), bytes)
                 self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_digest_length_overflow(self):
         # See issue #34922
         large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)

crates/stdlib/src/hashlib.rs

@@ -83,9 +83,16 @@ pub mod _hashlib {
     }
 
     impl XofDigestArgs {
+        // Match CPython's SHAKE output guard in Modules/sha3module.c.
+        const MAX_SHAKE_OUTPUT_LENGTH: usize = 1 << 29;
+
         fn length(&self, vm: &VirtualMachine) -> PyResult<usize> {
-            usize::try_from(self.length)
-                .map_err(|_| vm.new_value_error("length must be non-negative"))
+            let length = usize::try_from(self.length)
+                .map_err(|_| vm.new_value_error("length must be non-negative"))?;
+            if length >= Self::MAX_SHAKE_OUTPUT_LENGTH {
+                return Err(vm.new_value_error("length is too large"));
+            }
+            Ok(length)
         }
     }