mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

xz2->liblzma and remove liblzma-sys

Browse Source
This commit is contained in:
Jeong, YunWon committed 2026-02-27 17:39:12 +09:00
1 parent 5f86ffd126
commit 5eb061d809
5 files changed
+594 -186
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+22 -22
View File
@@ -1749,6 +1749,26 @@ dependencies = [
"windows-link",
]
[[package]]
name = "liblzma"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899"
dependencies = [
"liblzma-sys",
]
[[package]]
name = "liblzma-sys"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "libm"
version = "0.2.16"
@@ -1815,17 +1835,6 @@ dependencies = [
"twox-hash",
]
[[package]]
name = "lzma-sys"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "mac_address"
version = "1.1.8"
@@ -3230,9 +3239,10 @@ dependencies = [
"indexmap",
"itertools 0.14.0",
"libc",
"liblzma",
"liblzma-sys",
"libsqlite3-sys",
"libz-rs-sys",
"lzma-sys",
"mac_address",
"malachite-bigint",
"md-5",
@@ -3291,7 +3301,6 @@ dependencies = [
"x509-cert",
"x509-parser",
"xml",
"xz2",
]
[[package]]
@@ -4823,15 +4832,6 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a"
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
]
[[package]]
name = "zerocopy"
version = "0.8.34"
Lib/test/test_lzma.py
Vendored
-29
View File
@@ -80,7 +80,6 @@ class CompressorDecompressorTestCase(unittest.TestCase):
lzd.decompress(COMPRESSED_XZ)
self.assertRaises(EOFError, lzd.decompress, b"nyan")
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument memlimit
def test_decompressor_memlimit(self):
lzd = LZMADecompressor(memlimit=1024)
self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
@@ -101,7 +100,6 @@ class CompressorDecompressorTestCase(unittest.TestCase):
self.assertTrue(lzd.eof)
self.assertEqual(lzd.unused_data, unused_data)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_auto(self):
lzd = LZMADecompressor()
self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
@@ -109,37 +107,30 @@ class CompressorDecompressorTestCase(unittest.TestCase):
lzd = LZMADecompressor()
self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_xz(self):
lzd = LZMADecompressor(lzma.FORMAT_XZ)
self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_alone(self):
lzd = LZMADecompressor(lzma.FORMAT_ALONE)
self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompressor_raw_1(self):
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompressor_raw_2(self):
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompressor_raw_3(self):
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompressor_raw_4(self):
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_chunks(self):
lzd = LZMADecompressor()
out = []
@@ -260,14 +251,12 @@ class CompressorDecompressorTestCase(unittest.TestCase):
out.append(lzd.decompress(COMPRESSED_XZ[300:]))
self.assertEqual(b''.join(out), INPUT)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_unused_data(self):
lzd = LZMADecompressor()
extra = b"fooblibar"
self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64,
unused_data=extra)
@unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized
def test_decompressor_bad_input(self):
lzd = LZMADecompressor()
self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1)
@@ -290,28 +279,24 @@ class CompressorDecompressorTestCase(unittest.TestCase):
# Test that LZMACompressor->LZMADecompressor preserves the input data.
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_roundtrip_xz(self):
lzc = LZMACompressor()
cdata = lzc.compress(INPUT) + lzc.flush()
lzd = LZMADecompressor()
self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_roundtrip_alone(self):
lzc = LZMACompressor(lzma.FORMAT_ALONE)
cdata = lzc.compress(INPUT) + lzc.flush()
lzd = LZMADecompressor()
self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format
def test_roundtrip_raw(self):
lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
cdata = lzc.compress(INPUT) + lzc.flush()
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format
def test_roundtrip_raw_empty(self):
lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
cdata = lzc.compress(INPUT)
@@ -322,7 +307,6 @@ class CompressorDecompressorTestCase(unittest.TestCase):
lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_roundtrip_chunks(self):
lzc = LZMACompressor()
cdata = []
@@ -333,7 +317,6 @@ class CompressorDecompressorTestCase(unittest.TestCase):
lzd = LZMADecompressor()
self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_roundtrip_empty_chunks(self):
lzc = LZMACompressor()
cdata = []
@@ -349,7 +332,6 @@ class CompressorDecompressorTestCase(unittest.TestCase):
# LZMADecompressor intentionally does not handle concatenated streams.
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_multistream(self):
lzd = LZMADecompressor()
self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE,
@@ -452,7 +434,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
# Test LZMADecompressor on known-good input data.
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompress_good_input(self):
ddata = lzma.decompress(COMPRESSED_XZ)
self.assertEqual(ddata, INPUT)
@@ -482,7 +463,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
self.assertEqual(ddata, INPUT)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_decompress_incomplete_input(self):
self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128])
self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128])
@@ -495,7 +475,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128],
format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
@unittest.expectedFailure # TODO: RUSTPYTHON; OSError: stream/file format not recognized
def test_decompress_bad_input(self):
with self.assertRaises(LZMAError):
lzma.decompress(COMPRESSED_BOGUS)
@@ -511,7 +490,6 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
# Test that compress()->decompress() preserves the input data.
@unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format
def test_roundtrip(self):
cdata = lzma.compress(INPUT)
ddata = lzma.decompress(cdata)
@@ -836,7 +814,6 @@ class FileTestCase(unittest.TestCase):
f.close()
self.assertRaises(ValueError, f.writable)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_read(self):
with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
self.assertEqual(f.read(), INPUT)
@@ -884,7 +861,6 @@ class FileTestCase(unittest.TestCase):
chunks.append(result)
self.assertEqual(b"".join(chunks), INPUT)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_read_multistream(self):
with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
self.assertEqual(f.read(), INPUT * 5)
@@ -1071,7 +1047,6 @@ class FileTestCase(unittest.TestCase):
with LZMAFile(BytesIO(), "w") as f:
self.assertRaises(ValueError, f.peek)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_iterator(self):
with BytesIO(INPUT) as f:
lines = f.readlines()
@@ -1111,7 +1086,6 @@ class FileTestCase(unittest.TestCase):
self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp,
"Excessive amount of data was decompressed")
@unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format
def test_write(self):
with BytesIO() as dst:
with LZMAFile(dst, "w") as f:
@@ -1491,7 +1465,6 @@ class OpenTestCase(unittest.TestCase):
with self.assertRaises(ValueError):
lzma.open(TESTFN, "rb", newline="\n")
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'list' found.
def test_format_and_filters(self):
# Test non-default format and filter chain.
options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1}
@@ -1580,7 +1553,6 @@ class MiscellaneousTestCase(unittest.TestCase):
})
self.assertEqual(props, b"]\x00\x00\x80\x00")
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: LZMAError not raised
def test__decode_filter_properties(self):
with self.assertRaises(TypeError):
lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes})
@@ -1604,7 +1576,6 @@ class MiscellaneousTestCase(unittest.TestCase):
filterspec = lzma._decode_filter_properties(f, b"")
self.assertEqual(filterspec, {"id": f})
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 0 arguments, got 1
def test_filter_properties_roundtrip(self):
spec1 = lzma._decode_filter_properties(
lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00")
crates/stdlib/Cargo.toml
+2 -2
View File
@@ -139,8 +139,8 @@ pkcs8 = { version = "0.10", features = ["encryption", "pkcs5", "pem"], optional
[target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies]
libsqlite3-sys = { version = "0.36", features = ["bundled"], optional = true }
lzma-sys = "0.1"
xz2 = "0.1"
liblzma = "0.4"
liblzma-sys = "0.4"
[target.'cfg(windows)'.dependencies]
paste = { workspace = true }
crates/stdlib/src/compression.rs
+4
View File
@@ -296,6 +296,10 @@ impl<D: Decompressor> DecompressState<D> {
self.eof
}
pub fn decompressor(&self) -> &D {
&self.decompress
}
pub fn unused_data(&self) -> PyBytesRef {
self.unused_data.clone()
}
crates/stdlib/src/lzma.rs
+566 -133
View File
@@ -1,4 +1,4 @@
// spell-checker:ignore ARMTHUMB
// spell-checker:ignore ARMTHUMB memlimit
pub(crate) use _lzma::module_def;
@@ -9,53 +9,63 @@ mod _lzma {
DecompressError, DecompressState, DecompressStatus, Decompressor,
};
use alloc::fmt;
#[pyattr]
use lzma_sys::{
LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64,
LZMA_CHECK_NONE as CHECK_NONE, LZMA_CHECK_SHA256 as CHECK_SHA256,
use liblzma::stream::{
Action, Check, Error, Filters, LzmaOptions, MatchFinder, Mode, Status, Stream,
TELL_ANY_CHECK, TELL_NO_CHECK,
};
// lzma_check, lzma_mode, lzma_match_finder have platform-dependent signedness
// (i32 on Windows, u32 elsewhere). Define as fixed-type const to avoid mismatch.
#[pyattr]
use lzma_sys::{
use liblzma_sys::{
LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB,
LZMA_FILTER_IA64 as FILTER_IA64, LZMA_FILTER_LZMA1 as FILTER_LZMA1,
LZMA_FILTER_LZMA2 as FILTER_LZMA2, LZMA_FILTER_POWERPC as FILTER_POWERPC,
LZMA_FILTER_SPARC as FILTER_SPARC, LZMA_FILTER_X86 as FILTER_X86,
LZMA_FILTER_DELTA as FILTER_DELTA, LZMA_FILTER_IA64 as FILTER_IA64,
LZMA_FILTER_LZMA1 as FILTER_LZMA1, LZMA_FILTER_LZMA2 as FILTER_LZMA2,
LZMA_FILTER_POWERPC as FILTER_POWERPC, LZMA_FILTER_SPARC as FILTER_SPARC,
LZMA_FILTER_X86 as FILTER_X86,
};
#[pyattr]
use lzma_sys::{
LZMA_MF_BT2 as MF_BT2, LZMA_MF_BT3 as MF_BT3, LZMA_MF_BT4 as MF_BT4, LZMA_MF_HC3 as MF_HC3,
LZMA_MF_HC4 as MF_HC4,
};
#[pyattr]
use lzma_sys::{LZMA_MODE_FAST as MODE_FAST, LZMA_MODE_NORMAL as MODE_NORMAL};
#[pyattr]
use lzma_sys::{
use liblzma_sys::{
LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME,
LZMA_PRESET_LEVEL_MASK as PRESET_LEVEL_MASK,
};
use rustpython_common::lock::PyMutex;
use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyType, PyTypeRef};
use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyDict, PyType, PyTypeRef};
use rustpython_vm::convert::ToPyException;
use rustpython_vm::function::ArgBytesLike;
use rustpython_vm::types::Constructor;
use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine};
use xz2::stream::{Action, Check, Error, Filters, LzmaOptions, Status, Stream};
#[cfg(windows)]
type EnumVal = i32;
#[cfg(not(windows))]
type EnumVal = u32;
const BUFSIZ: usize = 8192;
// TODO: can't find this in lzma-sys, but find way not to hardcode this
// liblzma_sys enum types have platform-dependent signedness; `as _` normalizes to i32
#[pyattr]
const FILTER_DELTA: i32 = 3;
const CHECK_NONE: i32 = liblzma_sys::LZMA_CHECK_NONE as _;
#[pyattr]
const CHECK_CRC32: i32 = liblzma_sys::LZMA_CHECK_CRC32 as _;
#[pyattr]
const CHECK_CRC64: i32 = liblzma_sys::LZMA_CHECK_CRC64 as _;
#[pyattr]
const CHECK_SHA256: i32 = liblzma_sys::LZMA_CHECK_SHA256 as _;
#[pyattr]
const CHECK_ID_MAX: i32 = 15;
#[pyattr]
const CHECK_UNKNOWN: i32 = CHECK_ID_MAX + 1;
// the variant ids are hardcoded to be equivalent to the C enum values
#[pyattr]
const MF_HC3: i32 = liblzma_sys::LZMA_MF_HC3 as _;
#[pyattr]
const MF_HC4: i32 = liblzma_sys::LZMA_MF_HC4 as _;
#[pyattr]
const MF_BT2: i32 = liblzma_sys::LZMA_MF_BT2 as _;
#[pyattr]
const MF_BT3: i32 = liblzma_sys::LZMA_MF_BT3 as _;
#[pyattr]
const MF_BT4: i32 = liblzma_sys::LZMA_MF_BT4 as _;
#[pyattr]
const MODE_FAST: i32 = liblzma_sys::LZMA_MODE_FAST as _;
#[pyattr]
const MODE_NORMAL: i32 = liblzma_sys::LZMA_MODE_NORMAL as _;
enum Format {
Auto = 0,
Xz = 1,
@@ -86,44 +96,106 @@ mod _lzma {
vm.new_exception_msg(vm.class("lzma", "LZMAError"), msg.into())
}
#[pyfunction]
fn is_check_supported(check: i32) -> bool {
unsafe { lzma_sys::lzma_check_is_supported(check as _) != 0 }
fn catch_lzma_error(err: Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
match err {
Error::UnsupportedCheck => new_lzma_error("Unsupported integrity check", vm),
Error::Mem => vm.new_memory_error(""),
Error::MemLimit => new_lzma_error("Memory usage limit exceeded", vm),
Error::Format => new_lzma_error("Input format not supported by decoder", vm),
Error::Options => new_lzma_error("Invalid or unsupported options", vm),
Error::Data => new_lzma_error("Corrupt input data", vm),
Error::Program => new_lzma_error("Internal error", vm),
Error::NoCheck => new_lzma_error("Corrupt input data", vm),
}
}
// TODO: To implement these we need a function to convert a pyobject to a lzma filter and related structs
#[pyfunction]
fn _encode_filter_properties() -> PyResult<()> {
Ok(())
fn int_to_check(check: i32) -> Option<Check> {
if check == -1 {
return Some(Check::Crc64);
}
match check {
CHECK_NONE => Some(Check::None),
CHECK_CRC32 => Some(Check::Crc32),
CHECK_CRC64 => Some(Check::Crc64),
CHECK_SHA256 => Some(Check::Sha256),
_ => None,
}
}
#[pyfunction]
fn _decode_filter_properties(_filter_id: u64, _buffer: ArgBytesLike) -> PyResult<()> {
Ok(())
fn u32_to_mode(val: u32) -> Option<Mode> {
match val as i32 {
MODE_FAST => Some(Mode::Fast),
MODE_NORMAL => Some(Mode::Normal),
_ => None,
}
}
#[pyattr]
#[pyclass(name = "LZMADecompressor")]
#[derive(PyPayload)]
struct LZMADecompressor {
state: PyMutex<DecompressState<Stream>>,
fn u32_to_mf(val: u32) -> Option<MatchFinder> {
match val as i32 {
MF_HC3 => Some(MatchFinder::HashChain3),
MF_HC4 => Some(MatchFinder::HashChain4),
MF_BT2 => Some(MatchFinder::BinaryTree2),
MF_BT3 => Some(MatchFinder::BinaryTree3),
MF_BT4 => Some(MatchFinder::BinaryTree4),
_ => None,
}
}
impl Decompressor for Stream {
struct LzmaStream {
stream: Stream,
check: i32,
header_buf: [u8; 8],
header_collected: u8,
track_header: bool,
}
impl LzmaStream {
fn new(stream: Stream, check: i32, track_header: bool) -> Self {
Self {
stream,
check,
header_buf: [0u8; 8],
header_collected: 0,
track_header,
}
}
}
impl Decompressor for LzmaStream {
type Flush = ();
type Status = Status;
type Error = Error;
fn total_in(&self) -> u64 {
self.total_in()
self.stream.total_in()
}
fn decompress_vec(
&mut self,
input: &[u8],
output: &mut Vec<u8>,
(): Self::Flush,
) -> Result<Self::Status, Self::Error> {
self.process_vec(input, output, Action::Run)
if self.track_header && self.header_collected < 8 {
let need = (8 - self.header_collected) as usize;
let n = need.min(input.len());
self.header_buf[self.header_collected as usize..][..n].copy_from_slice(&input[..n]);
self.header_collected += n as u8;
}
match self.stream.process_vec(input, output, Action::Run) {
Ok(Status::GetCheck) => {
if self.header_collected >= 8 {
self.check = (self.header_buf[7] & 0x0F) as i32;
}
Ok(Status::Ok)
}
Err(Error::NoCheck) => {
self.check = CHECK_NONE;
Ok(Status::Ok)
}
other => other,
}
}
}
@@ -133,43 +205,426 @@ mod _lzma {
}
}
fn get_dict_opt_u32(
spec: &PyObjectRef,
key: &str,
vm: &VirtualMachine,
) -> PyResult<Option<u32>> {
let dict = spec.downcast_ref::<PyDict>().ok_or_else(|| {
vm.new_type_error("Filter specifier must be a dict or dict-like object")
})?;
match dict.get_item_opt(key, vm)? {
Some(obj) => Ok(Some(obj.try_into_value::<u32>(vm)?)),
None => Ok(None),
}
}
fn get_dict_opt_u64(
spec: &PyObjectRef,
key: &str,
vm: &VirtualMachine,
) -> PyResult<Option<u64>> {
let dict = spec.downcast_ref::<PyDict>().ok_or_else(|| {
vm.new_type_error("Filter specifier must be a dict or dict-like object")
})?;
match dict.get_item_opt(key, vm)? {
Some(obj) => Ok(Some(obj.try_into_value::<u64>(vm)?)),
None => Ok(None),
}
}
fn parse_filter_spec_lzma(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<LzmaOptions> {
let preset = get_dict_opt_u32(spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT);
let mut opts = LzmaOptions::new_preset(preset)
.map_err(|_| new_lzma_error(format!("Invalid compression preset: {preset}"), vm))?;
if let Some(v) = get_dict_opt_u32(spec, "dict_size", vm)? {
opts.dict_size(v);
}
if let Some(v) = get_dict_opt_u32(spec, "lc", vm)? {
opts.literal_context_bits(v);
}
if let Some(v) = get_dict_opt_u32(spec, "lp", vm)? {
opts.literal_position_bits(v);
}
if let Some(v) = get_dict_opt_u32(spec, "pb", vm)? {
opts.position_bits(v);
}
if let Some(v) = get_dict_opt_u32(spec, "mode", vm)? {
let mode = u32_to_mode(v)
.ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?;
opts.mode(mode);
}
if let Some(v) = get_dict_opt_u32(spec, "nice_len", vm)? {
opts.nice_len(v);
}
if let Some(v) = get_dict_opt_u32(spec, "mf", vm)? {
let mf = u32_to_mf(v)
.ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?;
opts.match_finder(mf);
}
if let Some(v) = get_dict_opt_u32(spec, "depth", vm)? {
opts.depth(v);
}
Ok(opts)
}
fn parse_filter_spec_delta(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> {
let dist = get_dict_opt_u32(spec, "dist", vm)?.unwrap_or(1);
if dist == 0 || dist > 256 {
return Err(vm.new_value_error("Invalid filter specifier for delta filter"));
}
Ok(dist)
}
fn parse_filter_spec_bcj(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> {
Ok(get_dict_opt_u32(spec, "start_offset", vm)?.unwrap_or(0))
}
fn add_bcj_filter(
filters: &mut Filters,
filter_id: u64,
start_offset: u32,
) -> Result<(), Error> {
if start_offset == 0 {
match filter_id {
FILTER_X86 => {
filters.x86();
}
FILTER_POWERPC => {
filters.powerpc();
}
FILTER_IA64 => {
filters.ia64();
}
FILTER_ARM => {
filters.arm();
}
FILTER_ARMTHUMB => {
filters.arm_thumb();
}
FILTER_SPARC => {
filters.sparc();
}
_ => unreachable!(),
}
Ok(())
} else {
let props = start_offset.to_le_bytes();
match filter_id {
FILTER_X86 => {
filters.x86_properties(&props)?;
}
FILTER_POWERPC => {
filters.powerpc_properties(&props)?;
}
FILTER_IA64 => {
filters.ia64_properties(&props)?;
}
FILTER_ARM => {
filters.arm_properties(&props)?;
}
FILTER_ARMTHUMB => {
filters.arm_thumb_properties(&props)?;
}
FILTER_SPARC => {
filters.sparc_properties(&props)?;
}
_ => unreachable!(),
}
Ok(())
}
}
fn parse_filter_chain_spec(
filter_specs: Vec<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<Filters> {
const LZMA_FILTERS_MAX: usize = 4;
if filter_specs.len() > LZMA_FILTERS_MAX {
return Err(new_lzma_error(
format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"),
vm,
));
}
let mut filters = Filters::new();
for spec in &filter_specs {
let filter_id = get_dict_opt_u64(spec, "id", vm)?
.ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?;
match filter_id {
FILTER_LZMA1 => {
let opts = parse_filter_spec_lzma(spec, vm)?;
filters.lzma1(&opts);
}
FILTER_LZMA2 => {
let opts = parse_filter_spec_lzma(spec, vm)?;
filters.lzma2(&opts);
}
FILTER_DELTA => {
let dist = parse_filter_spec_delta(spec, vm)?;
filters
.delta_properties(&[(dist - 1) as u8])
.map_err(|e| catch_lzma_error(e, vm))?;
}
FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB
| FILTER_SPARC => {
let start_offset = parse_filter_spec_bcj(spec, vm)?;
add_bcj_filter(&mut filters, filter_id, start_offset)
.map_err(|e| catch_lzma_error(e, vm))?;
}
_ => {
return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}")));
}
}
}
Ok(filters)
}
const DEFAULT_LC: u32 = liblzma_sys::LZMA_LC_DEFAULT;
const DEFAULT_LP: u32 = liblzma_sys::LZMA_LP_DEFAULT;
const DEFAULT_PB: u32 = liblzma_sys::LZMA_PB_DEFAULT;
const DICT_POW2: [u8; 10] = [18, 20, 21, 22, 22, 23, 23, 24, 25, 26];
fn preset_dict_size(preset: u32) -> u32 {
let level = (preset & liblzma_sys::LZMA_PRESET_LEVEL_MASK) as usize;
if level > 9 {
return 0;
}
1u32 << DICT_POW2[level]
}
fn lzma2_dict_size_from_prop(prop: u8) -> u32 {
if prop > 40 {
return u32::MAX;
}
if prop == 40 {
return u32::MAX;
}
let prop = prop as u32;
(2 | (prop & 1)) << (prop / 2 + 11)
}
fn lzma2_prop_from_dict_size(dict_size: u32) -> u8 {
if dict_size == u32::MAX {
return 40;
}
for i in 0u8..40 {
if lzma2_dict_size_from_prop(i) >= dict_size {
return i;
}
}
40
}
fn encode_lzma1_properties(lc: u32, lp: u32, pb: u32, dict_size: u32) -> Vec<u8> {
let mut result = vec![0u8; 5];
result[0] = ((pb * 5 + lp) * 9 + lc) as u8;
result[1..5].copy_from_slice(&dict_size.to_le_bytes());
result
}
fn decode_lzma1_properties(props: &[u8]) -> Option<(u32, u32, u32, u32)> {
if props.len() < 5 {
return None;
}
let mut d = props[0] as u32;
let lc = d % 9;
d /= 9;
let lp = d % 5;
let pb = d / 5;
let dict_size = u32::from_le_bytes([props[1], props[2], props[3], props[4]]);
Some((lc, lp, pb, dict_size))
}
fn build_filter_spec(
filter_id: u64,
props: &[u8],
vm: &VirtualMachine,
) -> PyResult<PyObjectRef> {
let dict = vm.ctx.new_dict();
dict.set_item("id", vm.new_pyobj(filter_id), vm)?;
match filter_id {
FILTER_LZMA1 => {
let (lc, lp, pb, dict_size) = decode_lzma1_properties(props)
.ok_or_else(|| new_lzma_error("Invalid or unsupported options", vm))?;
dict.set_item("lc", vm.new_pyobj(lc), vm)?;
dict.set_item("lp", vm.new_pyobj(lp), vm)?;
dict.set_item("pb", vm.new_pyobj(pb), vm)?;
dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?;
}
FILTER_LZMA2 => {
if props.len() != 1 {
return Err(new_lzma_error("Invalid or unsupported options", vm));
}
let dict_size = lzma2_dict_size_from_prop(props[0]);
dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?;
}
FILTER_DELTA => {
if props.len() != 1 {
return Err(new_lzma_error("Invalid or unsupported options", vm));
}
let dist = props[0] as u32 + 1;
dict.set_item("dist", vm.new_pyobj(dist), vm)?;
}
FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB
| FILTER_SPARC => {
if props.is_empty() {
// default: no start_offset
} else if props.len() == 4 {
let start_offset = u32::from_le_bytes([props[0], props[1], props[2], props[3]]);
dict.set_item("start_offset", vm.new_pyobj(start_offset), vm)?;
} else {
return Err(new_lzma_error("Invalid or unsupported options", vm));
}
}
_ => {
return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}")));
}
}
Ok(dict.into())
}
#[pyfunction]
fn is_check_supported(check_id: i32) -> bool {
unsafe { liblzma_sys::lzma_check_is_supported(check_id as _) != 0 }
}
#[pyfunction]
fn _encode_filter_properties(
filter_spec: PyObjectRef,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
let filter_id = get_dict_opt_u64(&filter_spec, "id", vm)?
.ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?;
match filter_id {
FILTER_LZMA1 => {
let preset =
get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT);
let lc = get_dict_opt_u32(&filter_spec, "lc", vm)?.unwrap_or(DEFAULT_LC);
let lp = get_dict_opt_u32(&filter_spec, "lp", vm)?.unwrap_or(DEFAULT_LP);
let pb = get_dict_opt_u32(&filter_spec, "pb", vm)?.unwrap_or(DEFAULT_PB);
let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)?
.unwrap_or_else(|| preset_dict_size(preset));
Ok(encode_lzma1_properties(lc, lp, pb, dict_size))
}
FILTER_LZMA2 => {
let preset =
get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT);
let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)?
.unwrap_or_else(|| preset_dict_size(preset));
Ok(vec![lzma2_prop_from_dict_size(dict_size)])
}
FILTER_DELTA => {
let dist = get_dict_opt_u32(&filter_spec, "dist", vm)?.unwrap_or(1);
if dist == 0 || dist > 256 {
return Err(vm.new_value_error("Invalid filter specifier for delta filter"));
}
Ok(vec![(dist - 1) as u8])
}
FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB
| FILTER_SPARC => {
let start_offset = get_dict_opt_u32(&filter_spec, "start_offset", vm)?.unwrap_or(0);
if start_offset == 0 {
Ok(vec![])
} else {
Ok(start_offset.to_le_bytes().to_vec())
}
}
_ => Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))),
}
}
#[pyfunction]
fn _decode_filter_properties(
filter_id: u64,
encoded_props: ArgBytesLike,
vm: &VirtualMachine,
) -> PyResult<PyObjectRef> {
let props = encoded_props.borrow_buf();
build_filter_spec(filter_id, &props, vm)
}
#[pyattr]
#[pyclass(name = "LZMADecompressor")]
#[derive(PyPayload)]
struct LZMADecompressor {
state: PyMutex<DecompressState<LzmaStream>>,
}
impl fmt::Debug for LZMADecompressor {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "_lzma.LZMADecompressor")
}
}
#[derive(FromArgs)]
pub struct LZMADecompressorConstructorArgs {
#[pyarg(any, default = FORMAT_AUTO)]
format: i32,
#[pyarg(any, optional)]
mem_limit: Option<u64>,
memlimit: Option<u64>,
#[pyarg(any, optional)]
filters: Option<u32>,
filters: Option<Vec<PyObjectRef>>,
}
impl Constructor for LZMADecompressor {
type Args = LZMADecompressorConstructorArgs;
fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
if args.format == FORMAT_RAW && args.mem_limit.is_some() {
if args.format == FORMAT_RAW && args.memlimit.is_some() {
return Err(vm.new_value_error("Cannot specify memory limit with FORMAT_RAW"));
}
let mem_limit = args.mem_limit.unwrap_or(u64::MAX);
let filters = args.filters.unwrap_or(0);
let stream_result = match args.format {
FORMAT_AUTO => Stream::new_auto_decoder(mem_limit, filters),
FORMAT_XZ => Stream::new_stream_decoder(mem_limit, filters),
FORMAT_ALONE => Stream::new_lzma_decoder(mem_limit),
// TODO: FORMAT_RAW
_ => return Err(new_lzma_error("Invalid format", vm)),
if args.format == FORMAT_RAW && args.filters.is_none() {
return Err(vm.new_value_error("Must specify filters for FORMAT_RAW"));
}
if args.format != FORMAT_RAW && args.filters.is_some() {
return Err(vm.new_value_error("Cannot specify filters except with FORMAT_RAW"));
}
let memlimit = args.memlimit.unwrap_or(u64::MAX);
let decoder_flags = TELL_ANY_CHECK | TELL_NO_CHECK;
let lzma_stream = match args.format {
FORMAT_AUTO => {
let stream = Stream::new_auto_decoder(memlimit, decoder_flags)
.map_err(|e| catch_lzma_error(e, vm))?;
LzmaStream::new(stream, CHECK_UNKNOWN, true)
}
FORMAT_XZ => {
let stream = Stream::new_stream_decoder(memlimit, decoder_flags)
.map_err(|e| catch_lzma_error(e, vm))?;
LzmaStream::new(stream, CHECK_UNKNOWN, true)
}
FORMAT_ALONE => {
let stream =
Stream::new_lzma_decoder(memlimit).map_err(|e| catch_lzma_error(e, vm))?;
LzmaStream::new(stream, CHECK_NONE, false)
}
FORMAT_RAW => {
let filter_specs = args.filters.unwrap(); // safe: checked above
let filters = parse_filter_chain_spec(filter_specs, vm)?;
let stream =
Stream::new_raw_decoder(&filters).map_err(|e| catch_lzma_error(e, vm))?;
LzmaStream::new(stream, CHECK_NONE, false)
}
_ => {
return Err(
vm.new_value_error(format!("Invalid container format: {}", args.format))
);
}
};
Ok(Self {
state: PyMutex::new(DecompressState::new(
stream_result
.map_err(|_| new_lzma_error("Failed to initialize decoder", vm))?,
vm,
)),
state: PyMutex::new(DecompressState::new(lzma_stream, vm)),
})
}
}
@@ -185,11 +640,16 @@ mod _lzma {
state
.decompress(data, max_length, BUFSIZ, vm)
.map_err(|e| match e {
DecompressError::Decompress(err) => new_lzma_error(err.to_string(), vm),
DecompressError::Decompress(err) => catch_lzma_error(err, vm),
DecompressError::Eof(err) => err.to_pyexception(vm),
})
}
#[pygetset]
fn check(&self) -> i32 {
self.state.lock().decompressor().check
}
#[pygetset]
fn eof(&self) -> bool {
self.state.lock().eof()
@@ -202,12 +662,8 @@ mod _lzma {
#[pygetset]
fn needs_input(&self) -> bool {
// False if the decompress() method can provide more
// decompressed data before requiring new uncompressed input.
self.state.lock().needs_input()
}
// TODO: mro()?
}
struct CompressorInner {
@@ -246,7 +702,7 @@ mod _lzma {
) -> PyResult<Self::Status> {
self.stream
.process_vec(input, output, flush)
.map_err(|_| new_lzma_error("Failed to compress data", vm))
.map_err(|e| catch_lzma_error(e, vm))
}
fn total_in(&mut self) -> usize {
@@ -277,36 +733,6 @@ mod _lzma {
}
}
fn int_to_check(check: i32) -> Option<Check> {
if check == -1 {
return Some(Check::None);
}
match check as EnumVal {
CHECK_NONE => Some(Check::None),
CHECK_CRC32 => Some(Check::Crc32),
CHECK_CRC64 => Some(Check::Crc64),
CHECK_SHA256 => Some(Check::Sha256),
_ => None,
}
}
fn parse_filter_chain_spec(
filter_specs: Vec<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<Filters> {
// TODO: don't hardcode
const LZMA_FILTERS_MAX: usize = 4;
if filter_specs.len() > LZMA_FILTERS_MAX {
return Err(new_lzma_error(
format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"),
vm,
));
}
let filters = Filters::new();
for _item in filter_specs {}
Ok(filters)
}
impl LZMACompressor {
fn init_xz(
check: i32,
@@ -315,14 +741,13 @@ mod _lzma {
vm: &VirtualMachine,
) -> PyResult<Stream> {
let real_check =
int_to_check(check).ok_or_else(|| vm.new_type_error("Invalid check value"))?;
if let Some(filters) = filters {
let filters = parse_filter_chain_spec(filters, vm)?;
Ok(Stream::new_stream_encoder(&filters, real_check)
.map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
int_to_check(check).ok_or_else(|| vm.new_value_error("Invalid check value"))?;
if let Some(filter_specs) = filters {
let filters = parse_filter_chain_spec(filter_specs, vm)?;
Stream::new_stream_encoder(&filters, real_check)
.map_err(|e| catch_lzma_error(e, vm))
} else {
Ok(Stream::new_easy_encoder(preset, real_check)
.map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
Stream::new_easy_encoder(preset, real_check).map_err(|e| catch_lzma_error(e, vm))
}
}
@@ -332,65 +757,76 @@ mod _lzma {
vm: &VirtualMachine,
) -> PyResult<Stream> {
if let Some(_filter_specs) = filter_specs {
Err(new_lzma_error(
"TODO: RUSTPYTHON: LZMA: Alone filter specs",
vm,
))
// TODO: validate single LZMA1 filter and use its options
let options = LzmaOptions::new_preset(preset).map_err(|_| {
new_lzma_error(format!("Invalid compression preset: {preset}"), vm)
})?;
Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm))
} else {
let options = LzmaOptions::new_preset(preset)
.map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?;
let stream = Stream::new_lzma_encoder(&options)
.map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?;
Ok(stream)
let options = LzmaOptions::new_preset(preset).map_err(|_| {
new_lzma_error(format!("Invalid compression preset: {preset}"), vm)
})?;
Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm))
}
}
fn init_raw(
filter_specs: Option<Vec<PyObjectRef>>,
vm: &VirtualMachine,
) -> PyResult<Stream> {
let filter_specs = filter_specs
.ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?;
let filters = parse_filter_chain_spec(filter_specs, vm)?;
Stream::new_raw_encoder(&filters).map_err(|e| catch_lzma_error(e, vm))
}
}
#[derive(FromArgs)]
pub struct LZMACompressorConstructorArgs {
// format=FORMAT_XZ, check=-1, preset=None, filters=None
// {'format': 3, 'filters': [{'id': 3, 'dist': 2}, {'id': 33, 'preset': 2147483654}]}
#[pyarg(any, default = FORMAT_XZ)]
format: i32,
#[pyarg(any, default = -1)]
check: i32,
#[pyarg(any, optional)]
preset: Option<u32>,
preset: Option<PyObjectRef>,
#[pyarg(any, optional)]
filters: Option<Vec<PyObjectRef>>,
#[pyarg(any, optional)]
_filter_specs: Option<Vec<PyObjectRef>>,
#[pyarg(positional, optional)]
preset_obj: Option<PyObjectRef>,
}
impl Constructor for LZMACompressor {
type Args = LZMACompressorConstructorArgs;
fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
let preset = args.preset.unwrap_or(PRESET_DEFAULT);
#[allow(clippy::unnecessary_cast)]
if args.format != FORMAT_XZ as i32
&& args.check != -1
&& args.check != CHECK_NONE as i32
{
if args.format != FORMAT_XZ && args.check != -1 && args.check != CHECK_NONE {
return Err(new_lzma_error(
"Integrity checks are only supported by FORMAT_XZ",
vm,
));
}
if args.preset_obj.is_some() && args.filters.is_some() {
if args.preset.is_some() && args.filters.is_some() {
return Err(new_lzma_error(
"Cannot specify both preset and filter chain",
vm,
));
}
let preset: u32 = match &args.preset {
Some(obj) => obj.clone().try_into_value(vm)?,
None => PRESET_DEFAULT,
};
let stream = match args.format {
FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?,
FORMAT_ALONE => Self::init_alone(preset, args.filters, vm)?,
// TODO: RAW
_ => return Err(new_lzma_error("Invalid format", vm)),
FORMAT_RAW => Self::init_raw(args.filters, vm)?,
_ => {
return Err(
vm.new_value_error(format!("Invalid container format: {}", args.format))
);
}
};
Ok(Self {
state: PyMutex::new(CompressState::new(CompressorInner::new(stream))),
})
@@ -402,15 +838,12 @@ mod _lzma {
#[pymethod]
fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let mut state = self.state.lock();
// TODO: Flush check
state.compress(&data.borrow_buf(), vm)
}
#[pymethod]
fn flush(&self, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
// TODO: flush check
let mut state = self.state.lock();
// TODO: check if action is correct
state.flush(Action::Finish, vm)
}
}