From 36cffabeda4ed96f48edc3bab5d2aff40b904488 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Thu, 26 Feb 2026 23:30:59 +0900 Subject: [PATCH] xz2->liblzma and remove liblzma-sys --- Cargo.lock | 44 +- Lib/test/test_lzma.py | 29 -- crates/stdlib/Cargo.toml | 4 +- crates/stdlib/src/compression.rs | 4 + crates/stdlib/src/lzma.rs | 699 +++++++++++++++++++++++++------ 5 files changed, 594 insertions(+), 186 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd0ec6dd1..1f16ec1f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1749,6 +1749,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.16" @@ -1815,17 +1835,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac_address" version = "1.1.8" @@ -3230,9 +3239,10 @@ dependencies = [ "indexmap", "itertools 0.14.0", "libc", + "liblzma", + "liblzma-sys", "libsqlite3-sys", "libz-rs-sys", - "lzma-sys", "mac_address", "malachite-bigint", "md-5", @@ -3291,7 +3301,6 @@ dependencies = [ "x509-cert", "x509-parser", "xml", - "xz2", ] [[package]] @@ -4823,15 +4832,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "zerocopy" version = "0.8.34" diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index efd3c5b0c..a433f0c61 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -80,7 +80,6 @@ class CompressorDecompressorTestCase(unittest.TestCase): lzd.decompress(COMPRESSED_XZ) self.assertRaises(EOFError, lzd.decompress, b"nyan") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument memlimit def test_decompressor_memlimit(self): lzd = LZMADecompressor(memlimit=1024) self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) @@ -101,7 +100,6 @@ class CompressorDecompressorTestCase(unittest.TestCase): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, unused_data) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_auto(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) @@ -109,37 +107,30 @@ class CompressorDecompressorTestCase(unittest.TestCase): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_xz(self): lzd = LZMADecompressor(lzma.FORMAT_XZ) self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_alone(self): lzd = LZMADecompressor(lzma.FORMAT_ALONE) self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_1(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_2(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2) self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_3(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3) self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompressor_raw_4(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_chunks(self): lzd = LZMADecompressor() out = [] @@ -260,14 +251,12 @@ class CompressorDecompressorTestCase(unittest.TestCase): out.append(lzd.decompress(COMPRESSED_XZ[300:])) self.assertEqual(b''.join(out), INPUT) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_unused_data(self): lzd = LZMADecompressor() extra = b"fooblibar" self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64, unused_data=extra) - @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompressor_bad_input(self): lzd = LZMADecompressor() self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) @@ -290,28 +279,24 @@ class CompressorDecompressorTestCase(unittest.TestCase): # Test that LZMACompressor->LZMADecompressor preserves the input data. - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_xz(self): lzc = LZMACompressor() cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_alone(self): lzc = LZMACompressor(lzma.FORMAT_ALONE) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip_raw(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip_raw_empty(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) @@ -322,7 +307,6 @@ class CompressorDecompressorTestCase(unittest.TestCase): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_chunks(self): lzc = LZMACompressor() cdata = [] @@ -333,7 +317,6 @@ class CompressorDecompressorTestCase(unittest.TestCase): lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_roundtrip_empty_chunks(self): lzc = LZMACompressor() cdata = [] @@ -349,7 +332,6 @@ class CompressorDecompressorTestCase(unittest.TestCase): # LZMADecompressor intentionally does not handle concatenated streams. - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' def test_decompressor_multistream(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE, @@ -452,7 +434,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): # Test LZMADecompressor on known-good input data. - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompress_good_input(self): ddata = lzma.decompress(COMPRESSED_XZ) self.assertEqual(ddata, INPUT) @@ -482,7 +463,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self.assertEqual(ddata, INPUT) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_decompress_incomplete_input(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128]) self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128]) @@ -495,7 +475,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128], format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) - @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized def test_decompress_bad_input(self): with self.assertRaises(LZMAError): lzma.decompress(COMPRESSED_BOGUS) @@ -511,7 +490,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): # Test that compress()->decompress() preserves the input data. - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_roundtrip(self): cdata = lzma.compress(INPUT) ddata = lzma.decompress(cdata) @@ -836,7 +814,6 @@ class FileTestCase(unittest.TestCase): f.close() self.assertRaises(ValueError, f.writable) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_read(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: self.assertEqual(f.read(), INPUT) @@ -884,7 +861,6 @@ class FileTestCase(unittest.TestCase): chunks.append(result) self.assertEqual(b"".join(chunks), INPUT) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_read_multistream(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: self.assertEqual(f.read(), INPUT * 5) @@ -1071,7 +1047,6 @@ class FileTestCase(unittest.TestCase): with LZMAFile(BytesIO(), "w") as f: self.assertRaises(ValueError, f.peek) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_iterator(self): with BytesIO(INPUT) as f: lines = f.readlines() @@ -1111,7 +1086,6 @@ class FileTestCase(unittest.TestCase): self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, "Excessive amount of data was decompressed") - @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: @@ -1491,7 +1465,6 @@ class OpenTestCase(unittest.TestCase): with self.assertRaises(ValueError): lzma.open(TESTFN, "rb", newline="\n") - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found. def test_format_and_filters(self): # Test non-default format and filter chain. options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1} @@ -1580,7 +1553,6 @@ class MiscellaneousTestCase(unittest.TestCase): }) self.assertEqual(props, b"]\x00\x00\x80\x00") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: LZMAError not raised def test__decode_filter_properties(self): with self.assertRaises(TypeError): lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes}) @@ -1604,7 +1576,6 @@ class MiscellaneousTestCase(unittest.TestCase): filterspec = lzma._decode_filter_properties(f, b"") self.assertEqual(filterspec, {"id": f}) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 0 arguments, got 1 def test_filter_properties_roundtrip(self): spec1 = lzma._decode_filter_properties( lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index 3ea68d12b..0f16a06e6 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -139,8 +139,8 @@ pkcs8 = { version = "0.10", features = ["encryption", "pkcs5", "pem"], optional [target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] libsqlite3-sys = { version = "0.36", features = ["bundled"], optional = true } -lzma-sys = "0.1" -xz2 = "0.1" +liblzma = "0.4" +liblzma-sys = "0.4" [target.'cfg(windows)'.dependencies] paste = { workspace = true } diff --git a/crates/stdlib/src/compression.rs b/crates/stdlib/src/compression.rs index a857b4e53..8cc48b747 100644 --- a/crates/stdlib/src/compression.rs +++ b/crates/stdlib/src/compression.rs @@ -296,6 +296,10 @@ impl DecompressState { self.eof } + pub fn decompressor(&self) -> &D { + &self.decompress + } + pub fn unused_data(&self) -> PyBytesRef { self.unused_data.clone() } diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index fdd70033a..b98cfb1bf 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore ARMTHUMB +// spell-checker:ignore ARMTHUMB memlimit pub(crate) use _lzma::module_def; @@ -9,53 +9,63 @@ mod _lzma { DecompressError, DecompressState, DecompressStatus, Decompressor, }; use alloc::fmt; - #[pyattr] - use lzma_sys::{ - LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64, - LZMA_CHECK_NONE as CHECK_NONE, LZMA_CHECK_SHA256 as CHECK_SHA256, + use liblzma::stream::{ + Action, Check, Error, Filters, LzmaOptions, MatchFinder, Mode, Status, Stream, + TELL_ANY_CHECK, TELL_NO_CHECK, }; + // lzma_check, lzma_mode, lzma_match_finder have platform-dependent signedness + // (i32 on Windows, u32 elsewhere). Define as fixed-type const to avoid mismatch. #[pyattr] - use lzma_sys::{ + use liblzma_sys::{ LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB, - LZMA_FILTER_IA64 as FILTER_IA64, LZMA_FILTER_LZMA1 as FILTER_LZMA1, - LZMA_FILTER_LZMA2 as FILTER_LZMA2, LZMA_FILTER_POWERPC as FILTER_POWERPC, - LZMA_FILTER_SPARC as FILTER_SPARC, LZMA_FILTER_X86 as FILTER_X86, + LZMA_FILTER_DELTA as FILTER_DELTA, LZMA_FILTER_IA64 as FILTER_IA64, + LZMA_FILTER_LZMA1 as FILTER_LZMA1, LZMA_FILTER_LZMA2 as FILTER_LZMA2, + LZMA_FILTER_POWERPC as FILTER_POWERPC, LZMA_FILTER_SPARC as FILTER_SPARC, + LZMA_FILTER_X86 as FILTER_X86, }; #[pyattr] - use lzma_sys::{ - LZMA_MF_BT2 as MF_BT2, LZMA_MF_BT3 as MF_BT3, LZMA_MF_BT4 as MF_BT4, LZMA_MF_HC3 as MF_HC3, - LZMA_MF_HC4 as MF_HC4, - }; - #[pyattr] - use lzma_sys::{LZMA_MODE_FAST as MODE_FAST, LZMA_MODE_NORMAL as MODE_NORMAL}; - #[pyattr] - use lzma_sys::{ + use liblzma_sys::{ LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME, - LZMA_PRESET_LEVEL_MASK as PRESET_LEVEL_MASK, }; use rustpython_common::lock::PyMutex; - use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyType, PyTypeRef}; + use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyDict, PyType, PyTypeRef}; use rustpython_vm::convert::ToPyException; use rustpython_vm::function::ArgBytesLike; use rustpython_vm::types::Constructor; use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use xz2::stream::{Action, Check, Error, Filters, LzmaOptions, Status, Stream}; - - #[cfg(windows)] - type EnumVal = i32; - #[cfg(not(windows))] - type EnumVal = u32; const BUFSIZ: usize = 8192; - // TODO: can't find this in lzma-sys, but find way not to hardcode this + + // liblzma_sys enum types have platform-dependent signedness; `as _` normalizes to i32 #[pyattr] - const FILTER_DELTA: i32 = 3; + const CHECK_NONE: i32 = liblzma_sys::LZMA_CHECK_NONE as _; + #[pyattr] + const CHECK_CRC32: i32 = liblzma_sys::LZMA_CHECK_CRC32 as _; + #[pyattr] + const CHECK_CRC64: i32 = liblzma_sys::LZMA_CHECK_CRC64 as _; + #[pyattr] + const CHECK_SHA256: i32 = liblzma_sys::LZMA_CHECK_SHA256 as _; #[pyattr] const CHECK_ID_MAX: i32 = 15; #[pyattr] const CHECK_UNKNOWN: i32 = CHECK_ID_MAX + 1; - // the variant ids are hardcoded to be equivalent to the C enum values + #[pyattr] + const MF_HC3: i32 = liblzma_sys::LZMA_MF_HC3 as _; + #[pyattr] + const MF_HC4: i32 = liblzma_sys::LZMA_MF_HC4 as _; + #[pyattr] + const MF_BT2: i32 = liblzma_sys::LZMA_MF_BT2 as _; + #[pyattr] + const MF_BT3: i32 = liblzma_sys::LZMA_MF_BT3 as _; + #[pyattr] + const MF_BT4: i32 = liblzma_sys::LZMA_MF_BT4 as _; + + #[pyattr] + const MODE_FAST: i32 = liblzma_sys::LZMA_MODE_FAST as _; + #[pyattr] + const MODE_NORMAL: i32 = liblzma_sys::LZMA_MODE_NORMAL as _; + enum Format { Auto = 0, Xz = 1, @@ -86,44 +96,106 @@ mod _lzma { vm.new_exception_msg(vm.class("lzma", "LZMAError"), msg.into()) } - #[pyfunction] - fn is_check_supported(check: i32) -> bool { - unsafe { lzma_sys::lzma_check_is_supported(check as _) != 0 } + fn catch_lzma_error(err: Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + match err { + Error::UnsupportedCheck => new_lzma_error("Unsupported integrity check", vm), + Error::Mem => vm.new_memory_error(""), + Error::MemLimit => new_lzma_error("Memory usage limit exceeded", vm), + Error::Format => new_lzma_error("Input format not supported by decoder", vm), + Error::Options => new_lzma_error("Invalid or unsupported options", vm), + Error::Data => new_lzma_error("Corrupt input data", vm), + Error::Program => new_lzma_error("Internal error", vm), + Error::NoCheck => new_lzma_error("Corrupt input data", vm), + } } - // TODO: To implement these we need a function to convert a pyobject to a lzma filter and related structs - #[pyfunction] - fn _encode_filter_properties() -> PyResult<()> { - Ok(()) + fn int_to_check(check: i32) -> Option { + if check == -1 { + return Some(Check::Crc64); + } + match check { + CHECK_NONE => Some(Check::None), + CHECK_CRC32 => Some(Check::Crc32), + CHECK_CRC64 => Some(Check::Crc64), + CHECK_SHA256 => Some(Check::Sha256), + _ => None, + } } - #[pyfunction] - fn _decode_filter_properties(_filter_id: u64, _buffer: ArgBytesLike) -> PyResult<()> { - Ok(()) + fn u32_to_mode(val: u32) -> Option { + match val as i32 { + MODE_FAST => Some(Mode::Fast), + MODE_NORMAL => Some(Mode::Normal), + _ => None, + } } - #[pyattr] - #[pyclass(name = "LZMADecompressor")] - #[derive(PyPayload)] - struct LZMADecompressor { - state: PyMutex>, + fn u32_to_mf(val: u32) -> Option { + match val as i32 { + MF_HC3 => Some(MatchFinder::HashChain3), + MF_HC4 => Some(MatchFinder::HashChain4), + MF_BT2 => Some(MatchFinder::BinaryTree2), + MF_BT3 => Some(MatchFinder::BinaryTree3), + MF_BT4 => Some(MatchFinder::BinaryTree4), + _ => None, + } } - impl Decompressor for Stream { + struct LzmaStream { + stream: Stream, + check: i32, + header_buf: [u8; 8], + header_collected: u8, + track_header: bool, + } + + impl LzmaStream { + fn new(stream: Stream, check: i32, track_header: bool) -> Self { + Self { + stream, + check, + header_buf: [0u8; 8], + header_collected: 0, + track_header, + } + } + } + + impl Decompressor for LzmaStream { type Flush = (); type Status = Status; type Error = Error; fn total_in(&self) -> u64 { - self.total_in() + self.stream.total_in() } + fn decompress_vec( &mut self, input: &[u8], output: &mut Vec, (): Self::Flush, ) -> Result { - self.process_vec(input, output, Action::Run) + if self.track_header && self.header_collected < 8 { + let need = (8 - self.header_collected) as usize; + let n = need.min(input.len()); + self.header_buf[self.header_collected as usize..][..n].copy_from_slice(&input[..n]); + self.header_collected += n as u8; + } + + match self.stream.process_vec(input, output, Action::Run) { + Ok(Status::GetCheck) => { + if self.header_collected >= 8 { + self.check = (self.header_buf[7] & 0x0F) as i32; + } + Ok(Status::Ok) + } + Err(Error::NoCheck) => { + self.check = CHECK_NONE; + Ok(Status::Ok) + } + other => other, + } } } @@ -133,43 +205,426 @@ mod _lzma { } } + fn get_dict_opt_u32( + spec: &PyObjectRef, + key: &str, + vm: &VirtualMachine, + ) -> PyResult> { + let dict = spec.downcast_ref::().ok_or_else(|| { + vm.new_type_error("Filter specifier must be a dict or dict-like object") + })?; + match dict.get_item_opt(key, vm)? { + Some(obj) => Ok(Some(obj.try_into_value::(vm)?)), + None => Ok(None), + } + } + + fn get_dict_opt_u64( + spec: &PyObjectRef, + key: &str, + vm: &VirtualMachine, + ) -> PyResult> { + let dict = spec.downcast_ref::().ok_or_else(|| { + vm.new_type_error("Filter specifier must be a dict or dict-like object") + })?; + match dict.get_item_opt(key, vm)? { + Some(obj) => Ok(Some(obj.try_into_value::(vm)?)), + None => Ok(None), + } + } + + fn parse_filter_spec_lzma(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let preset = get_dict_opt_u32(spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + + let mut opts = LzmaOptions::new_preset(preset) + .map_err(|_| new_lzma_error(format!("Invalid compression preset: {preset}"), vm))?; + + if let Some(v) = get_dict_opt_u32(spec, "dict_size", vm)? { + opts.dict_size(v); + } + if let Some(v) = get_dict_opt_u32(spec, "lc", vm)? { + opts.literal_context_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "lp", vm)? { + opts.literal_position_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "pb", vm)? { + opts.position_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "mode", vm)? { + let mode = u32_to_mode(v) + .ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?; + opts.mode(mode); + } + if let Some(v) = get_dict_opt_u32(spec, "nice_len", vm)? { + opts.nice_len(v); + } + if let Some(v) = get_dict_opt_u32(spec, "mf", vm)? { + let mf = u32_to_mf(v) + .ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?; + opts.match_finder(mf); + } + if let Some(v) = get_dict_opt_u32(spec, "depth", vm)? { + opts.depth(v); + } + + Ok(opts) + } + + fn parse_filter_spec_delta(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let dist = get_dict_opt_u32(spec, "dist", vm)?.unwrap_or(1); + if dist == 0 || dist > 256 { + return Err(vm.new_value_error("Invalid filter specifier for delta filter")); + } + Ok(dist) + } + + fn parse_filter_spec_bcj(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(get_dict_opt_u32(spec, "start_offset", vm)?.unwrap_or(0)) + } + + fn add_bcj_filter( + filters: &mut Filters, + filter_id: u64, + start_offset: u32, + ) -> Result<(), Error> { + if start_offset == 0 { + match filter_id { + FILTER_X86 => { + filters.x86(); + } + FILTER_POWERPC => { + filters.powerpc(); + } + FILTER_IA64 => { + filters.ia64(); + } + FILTER_ARM => { + filters.arm(); + } + FILTER_ARMTHUMB => { + filters.arm_thumb(); + } + FILTER_SPARC => { + filters.sparc(); + } + _ => unreachable!(), + } + Ok(()) + } else { + let props = start_offset.to_le_bytes(); + match filter_id { + FILTER_X86 => { + filters.x86_properties(&props)?; + } + FILTER_POWERPC => { + filters.powerpc_properties(&props)?; + } + FILTER_IA64 => { + filters.ia64_properties(&props)?; + } + FILTER_ARM => { + filters.arm_properties(&props)?; + } + FILTER_ARMTHUMB => { + filters.arm_thumb_properties(&props)?; + } + FILTER_SPARC => { + filters.sparc_properties(&props)?; + } + _ => unreachable!(), + } + Ok(()) + } + } + + fn parse_filter_chain_spec( + filter_specs: Vec, + vm: &VirtualMachine, + ) -> PyResult { + const LZMA_FILTERS_MAX: usize = 4; + if filter_specs.len() > LZMA_FILTERS_MAX { + return Err(new_lzma_error( + format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), + vm, + )); + } + + let mut filters = Filters::new(); + for spec in &filter_specs { + let filter_id = get_dict_opt_u64(spec, "id", vm)? + .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; + + match filter_id { + FILTER_LZMA1 => { + let opts = parse_filter_spec_lzma(spec, vm)?; + filters.lzma1(&opts); + } + FILTER_LZMA2 => { + let opts = parse_filter_spec_lzma(spec, vm)?; + filters.lzma2(&opts); + } + FILTER_DELTA => { + let dist = parse_filter_spec_delta(spec, vm)?; + filters + .delta_properties(&[(dist - 1) as u8]) + .map_err(|e| catch_lzma_error(e, vm))?; + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + let start_offset = parse_filter_spec_bcj(spec, vm)?; + add_bcj_filter(&mut filters, filter_id, start_offset) + .map_err(|e| catch_lzma_error(e, vm))?; + } + _ => { + return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))); + } + } + } + + Ok(filters) + } + + const DEFAULT_LC: u32 = liblzma_sys::LZMA_LC_DEFAULT; + const DEFAULT_LP: u32 = liblzma_sys::LZMA_LP_DEFAULT; + const DEFAULT_PB: u32 = liblzma_sys::LZMA_PB_DEFAULT; + const DICT_POW2: [u8; 10] = [18, 20, 21, 22, 22, 23, 23, 24, 25, 26]; + + fn preset_dict_size(preset: u32) -> u32 { + let level = (preset & liblzma_sys::LZMA_PRESET_LEVEL_MASK) as usize; + if level > 9 { + return 0; + } + 1u32 << DICT_POW2[level] + } + + fn lzma2_dict_size_from_prop(prop: u8) -> u32 { + if prop > 40 { + return u32::MAX; + } + if prop == 40 { + return u32::MAX; + } + let prop = prop as u32; + (2 | (prop & 1)) << (prop / 2 + 11) + } + + fn lzma2_prop_from_dict_size(dict_size: u32) -> u8 { + if dict_size == u32::MAX { + return 40; + } + for i in 0u8..40 { + if lzma2_dict_size_from_prop(i) >= dict_size { + return i; + } + } + 40 + } + + fn encode_lzma1_properties(lc: u32, lp: u32, pb: u32, dict_size: u32) -> Vec { + let mut result = vec![0u8; 5]; + result[0] = ((pb * 5 + lp) * 9 + lc) as u8; + result[1..5].copy_from_slice(&dict_size.to_le_bytes()); + result + } + + fn decode_lzma1_properties(props: &[u8]) -> Option<(u32, u32, u32, u32)> { + if props.len() < 5 { + return None; + } + let mut d = props[0] as u32; + let lc = d % 9; + d /= 9; + let lp = d % 5; + let pb = d / 5; + let dict_size = u32::from_le_bytes([props[1], props[2], props[3], props[4]]); + Some((lc, lp, pb, dict_size)) + } + + fn build_filter_spec( + filter_id: u64, + props: &[u8], + vm: &VirtualMachine, + ) -> PyResult { + let dict = vm.ctx.new_dict(); + dict.set_item("id", vm.new_pyobj(filter_id), vm)?; + + match filter_id { + FILTER_LZMA1 => { + let (lc, lp, pb, dict_size) = decode_lzma1_properties(props) + .ok_or_else(|| new_lzma_error("Invalid or unsupported options", vm))?; + dict.set_item("lc", vm.new_pyobj(lc), vm)?; + dict.set_item("lp", vm.new_pyobj(lp), vm)?; + dict.set_item("pb", vm.new_pyobj(pb), vm)?; + dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?; + } + FILTER_LZMA2 => { + if props.len() != 1 { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + let dict_size = lzma2_dict_size_from_prop(props[0]); + dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?; + } + FILTER_DELTA => { + if props.len() != 1 { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + let dist = props[0] as u32 + 1; + dict.set_item("dist", vm.new_pyobj(dist), vm)?; + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + if props.is_empty() { + // default: no start_offset + } else if props.len() == 4 { + let start_offset = u32::from_le_bytes([props[0], props[1], props[2], props[3]]); + dict.set_item("start_offset", vm.new_pyobj(start_offset), vm)?; + } else { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + } + _ => { + return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))); + } + } + + Ok(dict.into()) + } + + #[pyfunction] + fn is_check_supported(check_id: i32) -> bool { + unsafe { liblzma_sys::lzma_check_is_supported(check_id as _) != 0 } + } + + #[pyfunction] + fn _encode_filter_properties( + filter_spec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult> { + let filter_id = get_dict_opt_u64(&filter_spec, "id", vm)? + .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; + + match filter_id { + FILTER_LZMA1 => { + let preset = + get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + let lc = get_dict_opt_u32(&filter_spec, "lc", vm)?.unwrap_or(DEFAULT_LC); + let lp = get_dict_opt_u32(&filter_spec, "lp", vm)?.unwrap_or(DEFAULT_LP); + let pb = get_dict_opt_u32(&filter_spec, "pb", vm)?.unwrap_or(DEFAULT_PB); + let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)? + .unwrap_or_else(|| preset_dict_size(preset)); + Ok(encode_lzma1_properties(lc, lp, pb, dict_size)) + } + FILTER_LZMA2 => { + let preset = + get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)? + .unwrap_or_else(|| preset_dict_size(preset)); + Ok(vec![lzma2_prop_from_dict_size(dict_size)]) + } + FILTER_DELTA => { + let dist = get_dict_opt_u32(&filter_spec, "dist", vm)?.unwrap_or(1); + if dist == 0 || dist > 256 { + return Err(vm.new_value_error("Invalid filter specifier for delta filter")); + } + Ok(vec![(dist - 1) as u8]) + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + let start_offset = get_dict_opt_u32(&filter_spec, "start_offset", vm)?.unwrap_or(0); + if start_offset == 0 { + Ok(vec![]) + } else { + Ok(start_offset.to_le_bytes().to_vec()) + } + } + _ => Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))), + } + } + + #[pyfunction] + fn _decode_filter_properties( + filter_id: u64, + encoded_props: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult { + let props = encoded_props.borrow_buf(); + build_filter_spec(filter_id, &props, vm) + } + + #[pyattr] + #[pyclass(name = "LZMADecompressor")] + #[derive(PyPayload)] + struct LZMADecompressor { + state: PyMutex>, + } + impl fmt::Debug for LZMADecompressor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_lzma.LZMADecompressor") } } + #[derive(FromArgs)] pub struct LZMADecompressorConstructorArgs { #[pyarg(any, default = FORMAT_AUTO)] format: i32, #[pyarg(any, optional)] - mem_limit: Option, + memlimit: Option, #[pyarg(any, optional)] - filters: Option, + filters: Option>, } impl Constructor for LZMADecompressor { type Args = LZMADecompressorConstructorArgs; fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { - if args.format == FORMAT_RAW && args.mem_limit.is_some() { + if args.format == FORMAT_RAW && args.memlimit.is_some() { return Err(vm.new_value_error("Cannot specify memory limit with FORMAT_RAW")); } - let mem_limit = args.mem_limit.unwrap_or(u64::MAX); - let filters = args.filters.unwrap_or(0); - let stream_result = match args.format { - FORMAT_AUTO => Stream::new_auto_decoder(mem_limit, filters), - FORMAT_XZ => Stream::new_stream_decoder(mem_limit, filters), - FORMAT_ALONE => Stream::new_lzma_decoder(mem_limit), - // TODO: FORMAT_RAW - _ => return Err(new_lzma_error("Invalid format", vm)), + + if args.format == FORMAT_RAW && args.filters.is_none() { + return Err(vm.new_value_error("Must specify filters for FORMAT_RAW")); + } + if args.format != FORMAT_RAW && args.filters.is_some() { + return Err(vm.new_value_error("Cannot specify filters except with FORMAT_RAW")); + } + + let memlimit = args.memlimit.unwrap_or(u64::MAX); + let decoder_flags = TELL_ANY_CHECK | TELL_NO_CHECK; + + let lzma_stream = match args.format { + FORMAT_AUTO => { + let stream = Stream::new_auto_decoder(memlimit, decoder_flags) + .map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_UNKNOWN, true) + } + FORMAT_XZ => { + let stream = Stream::new_stream_decoder(memlimit, decoder_flags) + .map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_UNKNOWN, true) + } + FORMAT_ALONE => { + let stream = + Stream::new_lzma_decoder(memlimit).map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_NONE, false) + } + FORMAT_RAW => { + let filter_specs = args.filters.unwrap(); // safe: checked above + let filters = parse_filter_chain_spec(filter_specs, vm)?; + let stream = + Stream::new_raw_decoder(&filters).map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_NONE, false) + } + _ => { + return Err( + vm.new_value_error(format!("Invalid container format: {}", args.format)) + ); + } }; + Ok(Self { - state: PyMutex::new(DecompressState::new( - stream_result - .map_err(|_| new_lzma_error("Failed to initialize decoder", vm))?, - vm, - )), + state: PyMutex::new(DecompressState::new(lzma_stream, vm)), }) } } @@ -185,11 +640,16 @@ mod _lzma { state .decompress(data, max_length, BUFSIZ, vm) .map_err(|e| match e { - DecompressError::Decompress(err) => new_lzma_error(err.to_string(), vm), + DecompressError::Decompress(err) => catch_lzma_error(err, vm), DecompressError::Eof(err) => err.to_pyexception(vm), }) } + #[pygetset] + fn check(&self) -> i32 { + self.state.lock().decompressor().check + } + #[pygetset] fn eof(&self) -> bool { self.state.lock().eof() @@ -202,12 +662,8 @@ mod _lzma { #[pygetset] fn needs_input(&self) -> bool { - // False if the decompress() method can provide more - // decompressed data before requiring new uncompressed input. self.state.lock().needs_input() } - - // TODO: mro()? } struct CompressorInner { @@ -246,7 +702,7 @@ mod _lzma { ) -> PyResult { self.stream .process_vec(input, output, flush) - .map_err(|_| new_lzma_error("Failed to compress data", vm)) + .map_err(|e| catch_lzma_error(e, vm)) } fn total_in(&mut self) -> usize { @@ -277,36 +733,6 @@ mod _lzma { } } - fn int_to_check(check: i32) -> Option { - if check == -1 { - return Some(Check::None); - } - match check as EnumVal { - CHECK_NONE => Some(Check::None), - CHECK_CRC32 => Some(Check::Crc32), - CHECK_CRC64 => Some(Check::Crc64), - CHECK_SHA256 => Some(Check::Sha256), - _ => None, - } - } - - fn parse_filter_chain_spec( - filter_specs: Vec, - vm: &VirtualMachine, - ) -> PyResult { - // TODO: don't hardcode - const LZMA_FILTERS_MAX: usize = 4; - if filter_specs.len() > LZMA_FILTERS_MAX { - return Err(new_lzma_error( - format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), - vm, - )); - } - let filters = Filters::new(); - for _item in filter_specs {} - Ok(filters) - } - impl LZMACompressor { fn init_xz( check: i32, @@ -315,14 +741,13 @@ mod _lzma { vm: &VirtualMachine, ) -> PyResult { let real_check = - int_to_check(check).ok_or_else(|| vm.new_type_error("Invalid check value"))?; - if let Some(filters) = filters { - let filters = parse_filter_chain_spec(filters, vm)?; - Ok(Stream::new_stream_encoder(&filters, real_check) - .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?) + int_to_check(check).ok_or_else(|| vm.new_value_error("Invalid check value"))?; + if let Some(filter_specs) = filters { + let filters = parse_filter_chain_spec(filter_specs, vm)?; + Stream::new_stream_encoder(&filters, real_check) + .map_err(|e| catch_lzma_error(e, vm)) } else { - Ok(Stream::new_easy_encoder(preset, real_check) - .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?) + Stream::new_easy_encoder(preset, real_check).map_err(|e| catch_lzma_error(e, vm)) } } @@ -332,65 +757,76 @@ mod _lzma { vm: &VirtualMachine, ) -> PyResult { if let Some(_filter_specs) = filter_specs { - Err(new_lzma_error( - "TODO: RUSTPYTHON: LZMA: Alone filter specs", - vm, - )) + // TODO: validate single LZMA1 filter and use its options + let options = LzmaOptions::new_preset(preset).map_err(|_| { + new_lzma_error(format!("Invalid compression preset: {preset}"), vm) + })?; + Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm)) } else { - let options = LzmaOptions::new_preset(preset) - .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?; - let stream = Stream::new_lzma_encoder(&options) - .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?; - Ok(stream) + let options = LzmaOptions::new_preset(preset).map_err(|_| { + new_lzma_error(format!("Invalid compression preset: {preset}"), vm) + })?; + Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm)) } } + + fn init_raw( + filter_specs: Option>, + vm: &VirtualMachine, + ) -> PyResult { + let filter_specs = filter_specs + .ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?; + let filters = parse_filter_chain_spec(filter_specs, vm)?; + Stream::new_raw_encoder(&filters).map_err(|e| catch_lzma_error(e, vm)) + } } #[derive(FromArgs)] pub struct LZMACompressorConstructorArgs { - // format=FORMAT_XZ, check=-1, preset=None, filters=None - // {'format': 3, 'filters': [{'id': 3, 'dist': 2}, {'id': 33, 'preset': 2147483654}]} #[pyarg(any, default = FORMAT_XZ)] format: i32, #[pyarg(any, default = -1)] check: i32, #[pyarg(any, optional)] - preset: Option, + preset: Option, #[pyarg(any, optional)] filters: Option>, - #[pyarg(any, optional)] - _filter_specs: Option>, - #[pyarg(positional, optional)] - preset_obj: Option, } impl Constructor for LZMACompressor { type Args = LZMACompressorConstructorArgs; fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let preset = args.preset.unwrap_or(PRESET_DEFAULT); - #[allow(clippy::unnecessary_cast)] - if args.format != FORMAT_XZ as i32 - && args.check != -1 - && args.check != CHECK_NONE as i32 - { + if args.format != FORMAT_XZ && args.check != -1 && args.check != CHECK_NONE { return Err(new_lzma_error( "Integrity checks are only supported by FORMAT_XZ", vm, )); } - if args.preset_obj.is_some() && args.filters.is_some() { + + if args.preset.is_some() && args.filters.is_some() { return Err(new_lzma_error( "Cannot specify both preset and filter chain", vm, )); } + + let preset: u32 = match &args.preset { + Some(obj) => obj.clone().try_into_value(vm)?, + None => PRESET_DEFAULT, + }; + let stream = match args.format { FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?, FORMAT_ALONE => Self::init_alone(preset, args.filters, vm)?, - // TODO: RAW - _ => return Err(new_lzma_error("Invalid format", vm)), + FORMAT_RAW => Self::init_raw(args.filters, vm)?, + _ => { + return Err( + vm.new_value_error(format!("Invalid container format: {}", args.format)) + ); + } }; + Ok(Self { state: PyMutex::new(CompressState::new(CompressorInner::new(stream))), }) @@ -402,15 +838,12 @@ mod _lzma { #[pymethod] fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult> { let mut state = self.state.lock(); - // TODO: Flush check state.compress(&data.borrow_buf(), vm) } #[pymethod] fn flush(&self, vm: &VirtualMachine) -> PyResult> { - // TODO: flush check let mut state = self.state.lock(); - // TODO: check if action is correct state.flush(Action::Finish, vm) } }