From 167a999fcb948ffed788d68eeb4ecc0d29856226 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Wed, 28 Oct 2026 01:33:04 -0700 Subject: [PATCH] Support Python 3.15 zlib checksum combination and version deprecation (#9414) * Support Python 3.15 zlib checksum combination and version deprecation Forward checksum combination to the existing zlib-rs backend, retaining CPython checksum masking, conversion warnings, and signed length bounds. Deprecate __version__ through module __getattr__. Sync the canonical test_zlib with scripts/update_lib. Implementation and source review assisted by OpenAI Codex; focused compilation and Clippy passed. Canonical runtime tests remain pending. Assisted-by: OpenAI Codex: model version unavailable * Fix zlib combine ABI conversions for cross-platform Clippy Mark checksum results must-use and make the C unsigned-long and length conversions explicit per ABI. Preserve Windows GNU's 32-bit z_off64_t and large-file length handling without lint suppression. OpenAI Codex assisted with implementation and source review. Standalone common-crate Clippy passed; Windows validation remains with CI. Assisted-by: OpenAI Codex: model version unavailable --------- Co-authored-by: Codex --- Lib/test/test_zlib.py | 120 +++++++++++++++++++++++++- crates/common/src/compression/zlib.rs | 56 ++++++++++++ crates/stdlib/src/zlib.rs | 63 +++++++++++++- 3 files changed, 234 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 4d97fe56f..46c84c55c 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -119,6 +119,114 @@ class ChecksumTestCase(unittest.TestCase): self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam')) +class ChecksumCombineMixin: + """Mixin class for testing checksum combination.""" + + N = 1000 + default_iv: int + + def parse_iv(self, iv): + """Parse an IV value. + + - The default IV is returned if *iv* is None. + - A random IV is returned if *iv* is -1. + - Otherwise, *iv* is returned as is. + """ + if iv is None: + return self.default_iv + if iv == -1: + return random.randint(1, 0x80000000) + return iv + + def checksum(self, data, init=None): + """Compute the checksum of data with a given initial value. + + The *init* value is parsed by ``parse_iv``. + """ + iv = self.parse_iv(init) + return self._checksum(data, iv) + + def _checksum(self, data, init): + raise NotImplementedError + + def combine(self, a, b, blen): + """Combine two checksums together.""" + raise NotImplementedError + + def get_random_data(self, data_len, *, iv=None): + """Get a triplet (data, iv, checksum).""" + data = random.randbytes(data_len) + init = self.parse_iv(iv) + checksum = self.checksum(data, init) + return data, init, checksum + + def test_combine_empty(self): + for _ in range(self.N): + a, iv, checksum = self.get_random_data(32, iv=-1) + res = self.combine(iv, self.checksum(a), len(a)) + self.assertEqual(res, checksum) + + def test_combine_no_iv(self): + for _ in range(self.N): + a, _, chk_a = self.get_random_data(32) + b, _, chk_b = self.get_random_data(64) + res = self.combine(chk_a, chk_b, len(b)) + self.assertEqual(res, self.checksum(a + b)) + + def test_combine_no_iv_invalid_length(self): + a, _, chk_a = self.get_random_data(32) + b, _, chk_b = self.get_random_data(64) + checksum = self.checksum(a + b) + for invalid_len in [1, len(a), 48, len(b) + 1, 191]: + invalid_res = self.combine(chk_a, chk_b, invalid_len) + self.assertNotEqual(invalid_res, checksum) + + self.assertRaises(TypeError, self.combine, 0, 0, "len") + + def test_combine_with_iv(self): + for _ in range(self.N): + a, iv_a, chk_a_with_iv = self.get_random_data(32, iv=-1) + chk_a_no_iv = self.checksum(a) + b, iv_b, chk_b_with_iv = self.get_random_data(64, iv=-1) + chk_b_no_iv = self.checksum(b) + + # We can represent c = COMBINE(CHK(a, iv_a), CHK(b, iv_b)) as: + # + # c = CHK(CHK(b'', iv_a) + CHK(a) + CHK(b'', iv_b) + CHK(b)) + # = COMBINE( + # COMBINE(CHK(b'', iv_a), CHK(a)), + # COMBINE(CHK(b'', iv_b), CHK(b)), + # ) + # = COMBINE(COMBINE(iv_a, CHK(a)), COMBINE(iv_b, CHK(b))) + tmp0 = self.combine(iv_a, chk_a_no_iv, len(a)) + tmp1 = self.combine(iv_b, chk_b_no_iv, len(b)) + expected = self.combine(tmp0, tmp1, len(b)) + checksum = self.combine(chk_a_with_iv, chk_b_with_iv, len(b)) + self.assertEqual(checksum, expected) + + +class CRC32CombineTestCase(ChecksumCombineMixin, unittest.TestCase): + + default_iv = 0 + + def _checksum(self, data, init): + return zlib.crc32(data, init) + + def combine(self, a, b, blen): + return zlib.crc32_combine(a, b, blen) + + +class Adler32CombineTestCase(ChecksumCombineMixin, unittest.TestCase): + + default_iv = 1 + + def _checksum(self, data, init): + return zlib.adler32(data, init) + + def combine(self, a, b, blen): + return zlib.adler32_combine(a, b, blen) + + # Issue #10276 - check that inputs >=4 GiB are handled correctly. class ChecksumBigBufferTestCase(unittest.TestCase): @@ -982,7 +1090,7 @@ class ZlibDecompressorTest(unittest.TestCase): self.assertRaises(EOFError, zlibd.decompress, b"") @support.skip_if_pgo_task - @bigmemtest(size=_4G + 100, memuse=3.3) + @bigmemtest(size=_4G + 100, memuse=4.5) def testDecompress4G(self, size): # "Test zlib._ZlibDecompressor.decompress() with >4GiB input" blocksize = min(10 * 1024 * 1024, size) @@ -1114,5 +1222,15 @@ class CustomInt: return 100 +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(zlib, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/crates/common/src/compression/zlib.rs b/crates/common/src/compression/zlib.rs index 6c34d7cbe..4fbfb503a 100644 --- a/crates/common/src/compression/zlib.rs +++ b/crates/common/src/compression/zlib.rs @@ -28,6 +28,34 @@ pub const MAX_WBITS: i32 = 15; pub const DEF_BUF_SIZE: usize = 16 * 1024; pub const DEF_MEM_LEVEL: u8 = 8; +// CPython uses a large-file off_t except on Windows (always 32-bit) and +// GNU/Hurd, where its configure script disables large-file support. +#[cfg(windows)] +pub type ChecksumLength = i32; +#[cfg(not(any(windows, target_os = "hurd")))] +pub type ChecksumLength = i64; +#[cfg(target_os = "hurd")] +pub type ChecksumLength = libc::off_t; + +#[must_use] +pub fn adler32_combine(adler1: u32, adler2: u32, len2: ChecksumLength) -> u32 { + z_checksum_to_u32(libz_rs_sys::adler32_combine64( + z_checksum_from_u32(adler1), + z_checksum_from_u32(adler2), + z_checksum_length(len2), + )) +} + +// As in zlib, CRC combination requires a nonnegative length. +#[must_use] +pub fn crc32_combine(crc1: u32, crc2: u32, len2: ChecksumLength) -> u32 { + z_checksum_to_u32(libz_rs_sys::crc32_combine64( + z_checksum_from_u32(crc1), + z_checksum_from_u32(crc2), + z_checksum_length(len2), + )) +} + const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; fn valid_level(level: i32) -> bool { @@ -58,6 +86,34 @@ const fn z_size_to_u64(value: core::ffi::c_ulong) -> u64 { } } +const fn z_checksum_length(value: ChecksumLength) -> libz_rs_sys::z_off64_t { + #[cfg(any( + all(windows, not(target_env = "gnu")), + all(target_os = "hurd", target_pointer_width = "32") + ))] + { + value as i64 + } + #[cfg(not(any( + all(windows, not(target_env = "gnu")), + all(target_os = "hurd", target_pointer_width = "32") + )))] + { + value + } +} + +const fn z_checksum_from_u32(value: u32) -> core::ffi::c_ulong { + #[cfg(all(target_pointer_width = "64", not(windows)))] + { + value as u64 + } + #[cfg(any(target_pointer_width = "32", windows))] + { + value + } +} + const fn z_checksum_to_u32(value: core::ffi::c_ulong) -> u32 { #[cfg(all(target_pointer_width = "64", not(windows)))] { diff --git a/crates/stdlib/src/zlib.rs b/crates/stdlib/src/zlib.rs index 0a3082edb..93b6ab934 100644 --- a/crates/stdlib/src/zlib.rs +++ b/crates/stdlib/src/zlib.rs @@ -7,9 +7,9 @@ mod zlib { use crate::compression::{DecompressArgs, DecompressorArgs}; use crate::vm::{ Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyType, PyTypeRef}, + builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyStr, PyType, PyTypeRef}, common::lock::PyMutex, - convert::TryFromBorrowedObject, + convert::{TryFromBorrowedObject, TryFromObject}, function::{ArgBytesLike, ArgIndex, OptionalArg, PySize, PySsize}, object::PyAtomicRef, types::Constructor, @@ -17,6 +17,7 @@ mod zlib { use adler32::RollingAdler32 as Adler32; use alloc::fmt; use core::sync::atomic::{AtomicBool, Ordering}; + use num_traits::ToPrimitive; use rustpython_common::compression::zlib as backend; #[pyattr] @@ -26,8 +27,19 @@ mod zlib { Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, }; - #[pyattr(name = "__version__")] - const __VERSION__: &str = "1.0"; + #[pyfunction] + fn __getattr__(name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if name.downcast_ref::().and_then(|name| name.to_str()) == Some("__version__") { + crate::vm::stdlib::_warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "'__version__' is deprecated and slated for removal in Python 3.20".to_owned(), + 1, + vm, + )?; + return Ok(vm.ctx.new_str("1.0").into()); + } + Err(vm.new_attribute_error(format!("module 'zlib' has no attribute {}", name.repr(vm)?))) + } // We statically link libz-rs, so the compile-time and runtime versions // always match. @@ -84,6 +96,49 @@ mod zlib { crate::binascii::crc32(data, value) } + struct ChecksumValue(u32); + + impl TryFromObject for ChecksumValue { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let value = obj.try_index(vm)?; + if !value + .as_bigint() + .to_i64() + .is_some_and(|value| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(&value)) + { + crate::vm::stdlib::_warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "integer value out of range".to_owned(), + 1, + vm, + )?; + } + Ok(Self(value.as_u32_mask())) + } + } + + #[pyfunction] + fn adler32_combine( + adler1: ChecksumValue, + adler2: ChecksumValue, + len2: PyIntRef, + vm: &VirtualMachine, + ) -> PyResult { + let len2 = len2.try_to_primitive::(vm)?; + Ok(backend::adler32_combine(adler1.0, adler2.0, len2)) + } + + #[pyfunction] + fn crc32_combine( + crc1: ChecksumValue, + crc2: ChecksumValue, + len2: PyIntRef, + vm: &VirtualMachine, + ) -> PyResult { + let len2 = len2.try_to_primitive::(vm)?; + Ok(backend::crc32_combine(crc1.0, crc2.0, len2)) + } + #[derive(FromArgs)] struct PyFuncCompressArgs { #[pyarg(positional)]