Merge pull request #5717 from arihant2math/lzma

lzma implementation

Cargo.lock

@@ -1396,6 +1396,17 @@ dependencies = [
  "twox-hash",
 ]
 
+[[package]]
+name = "lzma-sys"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "mac_address"
 version = "1.1.8"
@@ -2467,6 +2478,7 @@ dependencies = [
  "libc",
  "libsqlite3-sys",
  "libz-rs-sys",
+ "lzma-sys",
  "mac_address",
  "malachite-bigint",
  "md-5",
@@ -2512,6 +2524,7 @@ dependencies = [
  "widestring",
  "windows-sys 0.59.0",
  "xml-rs",
+ "xz2",
 ]
 
 [[package]]
@@ -3675,6 +3688,15 @@ version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4"
 
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"

Lib/lzma.py

@@ -0,0 +1,364 @@
+"""Interface to the liblzma compression library.
+
+This module provides a class for reading and writing compressed files,
+classes for incremental (de)compression, and convenience functions for
+one-shot (de)compression.
+
+These classes and functions support both the XZ and legacy LZMA
+container formats, as well as raw compressed data streams.
+"""
+
+__all__ = [
+    "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256",
+    "CHECK_ID_MAX", "CHECK_UNKNOWN",
+    "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64",
+    "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC",
+    "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW",
+    "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4",
+    "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME",
+
+    "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError",
+    "open", "compress", "decompress", "is_check_supported",
+]
+
+import builtins
+import io
+import os
+from _lzma import *
+from _lzma import _encode_filter_properties, _decode_filter_properties
+import _compression
+
+
+# Value 0 no longer used
+_MODE_READ     = 1
+# Value 2 no longer used
+_MODE_WRITE    = 3
+
+
+class LZMAFile(_compression.BaseStream):
+
+    """A file object providing transparent LZMA (de)compression.
+
+    An LZMAFile can act as a wrapper for an existing file object, or
+    refer directly to a named file on disk.
+
+    Note that LZMAFile provides a *binary* file interface - data read
+    is returned as bytes, and data to be written must be given as bytes.
+    """
+
+    def __init__(self, filename=None, mode="r", *,
+                 format=None, check=-1, preset=None, filters=None):
+        """Open an LZMA-compressed file in binary mode.
+
+        filename can be either an actual file name (given as a str,
+        bytes, or PathLike object), in which case the named file is
+        opened, or it can be an existing file object to read from or
+        write to.
+
+        mode can be "r" for reading (default), "w" for (over)writing,
+        "x" for creating exclusively, or "a" for appending. These can
+        equivalently be given as "rb", "wb", "xb" and "ab" respectively.
+
+        format specifies the container format to use for the file.
+        If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the
+        default is FORMAT_XZ.
+
+        check specifies the integrity check to use. This argument can
+        only be used when opening a file for writing. For FORMAT_XZ,
+        the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not
+        support integrity checks - for these formats, check must be
+        omitted, or be CHECK_NONE.
+
+        When opening a file for reading, the *preset* argument is not
+        meaningful, and should be omitted. The *filters* argument should
+        also be omitted, except when format is FORMAT_RAW (in which case
+        it is required).
+
+        When opening a file for writing, the settings used by the
+        compressor can be specified either as a preset compression
+        level (with the *preset* argument), or in detail as a custom
+        filter chain (with the *filters* argument). For FORMAT_XZ and
+        FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset
+        level. For FORMAT_RAW, the caller must always specify a filter
+        chain; the raw compressor does not support preset compression
+        levels.
+
+        preset (if provided) should be an integer in the range 0-9,
+        optionally OR-ed with the constant PRESET_EXTREME.
+
+        filters (if provided) should be a sequence of dicts. Each dict
+        should have an entry for "id" indicating ID of the filter, plus
+        additional entries for options to the filter.
+        """
+        self._fp = None
+        self._closefp = False
+        self._mode = None
+
+        if mode in ("r", "rb"):
+            if check != -1:
+                raise ValueError("Cannot specify an integrity check "
+                                 "when opening a file for reading")
+            if preset is not None:
+                raise ValueError("Cannot specify a preset compression "
+                                 "level when opening a file for reading")
+            if format is None:
+                format = FORMAT_AUTO
+            mode_code = _MODE_READ
+        elif mode in ("w", "wb", "a", "ab", "x", "xb"):
+            if format is None:
+                format = FORMAT_XZ
+            mode_code = _MODE_WRITE
+            self._compressor = LZMACompressor(format=format, check=check,
+                                              preset=preset, filters=filters)
+            self._pos = 0
+        else:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if isinstance(filename, (str, bytes, os.PathLike)):
+            if "b" not in mode:
+                mode += "b"
+            self._fp = builtins.open(filename, mode)
+            self._closefp = True
+            self._mode = mode_code
+        elif hasattr(filename, "read") or hasattr(filename, "write"):
+            self._fp = filename
+            self._mode = mode_code
+        else:
+            raise TypeError("filename must be a str, bytes, file or PathLike object")
+
+        if self._mode == _MODE_READ:
+            raw = _compression.DecompressReader(self._fp, LZMADecompressor,
+                                                trailing_error=LZMAError, format=format, filters=filters)
+            self._buffer = io.BufferedReader(raw)
+
+    def close(self):
+        """Flush and close the file.
+
+        May be called more than once without error. Once the file is
+        closed, any other operation on it will raise a ValueError.
+        """
+        if self.closed:
+            return
+        try:
+            if self._mode == _MODE_READ:
+                self._buffer.close()
+                self._buffer = None
+            elif self._mode == _MODE_WRITE:
+                self._fp.write(self._compressor.flush())
+                self._compressor = None
+        finally:
+            try:
+                if self._closefp:
+                    self._fp.close()
+            finally:
+                self._fp = None
+                self._closefp = False
+
+    @property
+    def closed(self):
+        """True if this file is closed."""
+        return self._fp is None
+
+    @property
+    def name(self):
+        self._check_not_closed()
+        return self._fp.name
+
+    @property
+    def mode(self):
+        return 'wb' if self._mode == _MODE_WRITE else 'rb'
+
+    def fileno(self):
+        """Return the file descriptor for the underlying file."""
+        self._check_not_closed()
+        return self._fp.fileno()
+
+    def seekable(self):
+        """Return whether the file supports seeking."""
+        return self.readable() and self._buffer.seekable()
+
+    def readable(self):
+        """Return whether the file was opened for reading."""
+        self._check_not_closed()
+        return self._mode == _MODE_READ
+
+    def writable(self):
+        """Return whether the file was opened for writing."""
+        self._check_not_closed()
+        return self._mode == _MODE_WRITE
+
+    def peek(self, size=-1):
+        """Return buffered data without advancing the file position.
+
+        Always returns at least one byte of data, unless at EOF.
+        The exact number of bytes returned is unspecified.
+        """
+        self._check_can_read()
+        # Relies on the undocumented fact that BufferedReader.peek() always
+        # returns at least one byte (except at EOF)
+        return self._buffer.peek(size)
+
+    def read(self, size=-1):
+        """Read up to size uncompressed bytes from the file.
+
+        If size is negative or omitted, read until EOF is reached.
+        Returns b"" if the file is already at EOF.
+        """
+        self._check_can_read()
+        return self._buffer.read(size)
+
+    def read1(self, size=-1):
+        """Read up to size uncompressed bytes, while trying to avoid
+        making multiple reads from the underlying stream. Reads up to a
+        buffer's worth of data if size is negative.
+
+        Returns b"" if the file is at EOF.
+        """
+        self._check_can_read()
+        if size < 0:
+            size = io.DEFAULT_BUFFER_SIZE
+        return self._buffer.read1(size)
+
+    def readline(self, size=-1):
+        """Read a line of uncompressed bytes from the file.
+
+        The terminating newline (if present) is retained. If size is
+        non-negative, no more than size bytes will be read (in which
+        case the line may be incomplete). Returns b'' if already at EOF.
+        """
+        self._check_can_read()
+        return self._buffer.readline(size)
+
+    def write(self, data):
+        """Write a bytes object to the file.
+
+        Returns the number of uncompressed bytes written, which is
+        always the length of data in bytes. Note that due to buffering,
+        the file on disk may not reflect the data written until close()
+        is called.
+        """
+        self._check_can_write()
+        if isinstance(data, (bytes, bytearray)):
+            length = len(data)
+        else:
+            # accept any data that supports the buffer protocol
+            data = memoryview(data)
+            length = data.nbytes
+
+        compressed = self._compressor.compress(data)
+        self._fp.write(compressed)
+        self._pos += length
+        return length
+
+    def seek(self, offset, whence=io.SEEK_SET):
+        """Change the file position.
+
+        The new position is specified by offset, relative to the
+        position indicated by whence. Possible values for whence are:
+
+            0: start of stream (default): offset must not be negative
+            1: current stream position
+            2: end of stream; offset must not be positive
+
+        Returns the new file position.
+
+        Note that seeking is emulated, so depending on the parameters,
+        this operation may be extremely slow.
+        """
+        self._check_can_seek()
+        return self._buffer.seek(offset, whence)
+
+    def tell(self):
+        """Return the current file position."""
+        self._check_not_closed()
+        if self._mode == _MODE_READ:
+            return self._buffer.tell()
+        return self._pos
+
+
+def open(filename, mode="rb", *,
+         format=None, check=-1, preset=None, filters=None,
+         encoding=None, errors=None, newline=None):
+    """Open an LZMA-compressed file in binary or text mode.
+
+    filename can be either an actual file name (given as a str, bytes,
+    or PathLike object), in which case the named file is opened, or it
+    can be an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb",
+    "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text
+    mode.
+
+    The format, check, preset and filters arguments specify the
+    compression settings, as for LZMACompressor, LZMADecompressor and
+    LZMAFile.
+
+    For binary mode, this function is equivalent to the LZMAFile
+    constructor: LZMAFile(filename, mode, ...). In this case, the
+    encoding, errors and newline arguments must not be provided.
+
+    For text mode, an LZMAFile object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error
+    handling behavior, and line ending(s).
+
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    lz_mode = mode.replace("t", "")
+    binary_file = LZMAFile(filename, lz_mode, format=format, check=check,
+                           preset=preset, filters=filters)
+
+    if "t" in mode:
+        encoding = io.text_encoding(encoding)
+        return io.TextIOWrapper(binary_file, encoding, errors, newline)
+    else:
+        return binary_file
+
+
+def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None):
+    """Compress a block of data.
+
+    Refer to LZMACompressor's docstring for a description of the
+    optional arguments *format*, *check*, *preset* and *filters*.
+
+    For incremental compression, use an LZMACompressor instead.
+    """
+    comp = LZMACompressor(format, check, preset, filters)
+    return comp.compress(data) + comp.flush()
+
+
+def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
+    """Decompress a block of data.
+
+    Refer to LZMADecompressor's docstring for a description of the
+    optional arguments *format*, *check* and *filters*.
+
+    For incremental decompression, use an LZMADecompressor instead.
+    """
+    results = []
+    while True:
+        decomp = LZMADecompressor(format, memlimit, filters)
+        try:
+            res = decomp.decompress(data)
+        except LZMAError:
+            if results:
+                break  # Leftover data is not a valid LZMA/XZ stream; ignore it.
+            else:
+                raise  # Error on the first iteration; bail out.
+        results.append(res)
+        if not decomp.eof:
+            raise LZMAError("Compressed data ended before the "
+                            "end-of-stream marker was reached")
+        data = decomp.unused_data
+        if not data:
+            break
+    return b"".join(results)

Lib/tarfile.py

@@ -395,6 +395,10 @@ class _Stream:
                     import lzma
                 except ImportError:
                     raise CompressionError("lzma module is not available") from None
+
+                # XXX: RUSTPYTHON; xz is not supported yet
+                raise CompressionError("lzma module is not available") from None
+
                 if mode == "r":
                     self.dbuf = b""
                     self.cmp = lzma.LZMADecompressor()
@@ -1927,6 +1931,9 @@ class TarFile(object):
         except ImportError:
             raise CompressionError("lzma module is not available") from None
 
+        # XXX: RUSTPYTHON; xz is not supported yet
+        raise CompressionError("lzma module is not available") from None
+
         fileobj = LZMAFile(fileobj or name, mode, preset=preset)
 
         try:

Lib/test/support/__init__.py

@@ -498,6 +498,8 @@ def requires_lzma(reason='requires lzma'):
         import lzma
     except ImportError:
         lzma = None
+    # XXX: RUSTPYTHON; xz is not supported yet
+    lzma = None
     return unittest.skipUnless(lzma, reason)
 
 def has_no_debug_ranges():

Lib/test/test_tarfile.py

@@ -31,6 +31,8 @@ try:
     import lzma
 except ImportError:
     lzma = None
+# XXX: RUSTPYTHON; xz is not supported yet
+lzma = None
 
 def sha256sum(data):
     return sha256(data).hexdigest()

Lib/test/test_zoneinfo/test_zoneinfo.py

@@ -1728,6 +1728,8 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase):
                 with self.subTest("filtered", path_var=path_var):
                     self.assertSequenceEqual(tzpath, expected_paths)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_env_variable_relative_paths_warning_location(self):
         path_var = "path/to/somewhere"
 
@@ -1822,6 +1824,8 @@ class TestModule(ZoneInfoTestBase):
         with self.assertRaises(AttributeError):
             self.module.NOATTRIBUTE
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_dir_contains_all(self):
         """dir(self.module) should at least contain everything in __all__."""
         module_all_set = set(self.module.__all__)
@@ -1925,12 +1929,16 @@ class ExtensionBuiltTest(unittest.TestCase):
     rely on these tests as an indication of stable properties of these classes.
     """
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_cache_location(self):
         # The pure Python version stores caches on attributes, but the C
         # extension stores them in C globals (at least for now)
         self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache"))
         self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache"))
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_gc_tracked(self):
         import gc
 

stdlib/Cargo.toml

@@ -112,6 +112,8 @@ foreign-types-shared = { version = "0.1.1", optional = true }
 
 [target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies]
 libsqlite3-sys = { version = "0.28", features = ["bundled"], optional = true }
+lzma-sys = "0.1"
+xz2 = "0.1"
 
 [target.'cfg(windows)'.dependencies]
 junction = { workspace = true }

stdlib/src/bz2.rs

@@ -4,13 +4,13 @@ pub(crate) use _bz2::make_module;
 
 #[pymodule]
 mod _bz2 {
-    use super::super::{
+    use crate::compression::{
         DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor,
     };
-    use crate::common::lock::PyMutex;
     use crate::vm::{
         VirtualMachine,
         builtins::{PyBytesRef, PyTypeRef},
+        common::lock::PyMutex,
         function::{ArgBytesLike, OptionalArg},
         object::{PyPayload, PyResult},
         types::Constructor,
@@ -74,7 +74,7 @@ mod _bz2 {
     impl BZ2Decompressor {
         #[pymethod]
         fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let max_length = args.max_length_negative_is_none();
+            let max_length = args.max_length();
             let data = &*args.data();
 
             let mut state = self.state.lock();

stdlib/src/compression.rs

@@ -1,32 +1,44 @@
 // cspell:ignore chunker
+
+//! internal shared module for compression libraries
+
+use crate::vm::function::{ArgBytesLike, ArgSize, OptionalArg};
 use crate::vm::{
-    VirtualMachine,
+    PyResult, VirtualMachine,
     builtins::{PyBaseExceptionRef, PyBytesRef},
     convert::ToPyException,
-    function::{ArgBytesLike, ArgSize, OptionalArg},
 };
 
+pub const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state";
+// TODO: don't hardcode
+const CHUNKSIZE: usize = u32::MAX as usize;
+
 #[derive(FromArgs)]
-pub(super) struct DecompressArgs {
+pub struct DecompressArgs {
     #[pyarg(positional)]
     data: ArgBytesLike,
     #[pyarg(any, optional)]
-    pub max_length: OptionalArg<ArgSize>,
+    max_length: OptionalArg<ArgSize>,
 }
 
 impl DecompressArgs {
     pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> {
         self.data.borrow_buf()
     }
-    pub fn max_length_negative_is_none(&self) -> Option<usize> {
+    pub fn raw_max_length(&self) -> Option<isize> {
+        self.max_length.into_option().map(|ArgSize { value }| value)
+    }
+
+    // negative is None
+    pub fn max_length(&self) -> Option<usize> {
         self.max_length
             .into_option()
             .and_then(|ArgSize { value }| usize::try_from(value).ok())
     }
 }
 
-pub(super) trait Decompressor {
-    type Flush: FlushKind;
+pub trait Decompressor {
+    type Flush: DecompressFlushKind;
     type Status: DecompressStatus;
     type Error;
 
@@ -42,26 +54,24 @@ pub(super) trait Decompressor {
     }
 }
 
-pub(super) trait DecompressStatus {
+pub trait DecompressStatus {
     fn is_stream_end(&self) -> bool;
 }
 
-pub(super) trait FlushKind: Copy {
+pub trait DecompressFlushKind: Copy {
     const SYNC: Self;
 }
 
-impl FlushKind for () {
+impl DecompressFlushKind for () {
     const SYNC: Self = ();
 }
 
-pub(super) fn flush_sync<T: FlushKind>(_final_chunk: bool) -> T {
+pub fn flush_sync<T: DecompressFlushKind>(_final_chunk: bool) -> T {
     T::SYNC
 }
 
-pub(super) const CHUNKSIZE: usize = u32::MAX as usize;
-
 #[derive(Clone)]
-pub(super) struct Chunker<'a> {
+pub struct Chunker<'a> {
     data1: &'a [u8],
     data2: &'a [u8],
 }
@@ -102,7 +112,7 @@ impl<'a> Chunker<'a> {
     }
 }
 
-pub(super) fn _decompress<D: Decompressor>(
+pub fn _decompress<D: Decompressor>(
     data: &[u8],
     d: &mut D,
     bufsize: usize,
@@ -113,7 +123,7 @@ pub(super) fn _decompress<D: Decompressor>(
     _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush)
 }
 
-pub(super) fn _decompress_chunks<D: Decompressor>(
+pub fn _decompress_chunks<D: Decompressor>(
     data: &mut Chunker<'_>,
     d: &mut D,
     bufsize: usize,
@@ -167,8 +177,104 @@ pub(super) fn _decompress_chunks<D: Decompressor>(
     }
 }
 
+pub trait Compressor {
+    type Status: CompressStatusKind;
+    type Flush: CompressFlushKind;
+    const CHUNKSIZE: usize;
+    const DEF_BUF_SIZE: usize;
+
+    fn compress_vec(
+        &mut self,
+        input: &[u8],
+        output: &mut Vec<u8>,
+        flush: Self::Flush,
+        vm: &VirtualMachine,
+    ) -> PyResult<Self::Status>;
+
+    fn total_in(&mut self) -> usize;
+
+    fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef;
+}
+
+pub trait CompressFlushKind: Copy {
+    const NONE: Self;
+    const FINISH: Self;
+
+    fn to_usize(self) -> usize;
+}
+
+pub trait CompressStatusKind: Copy {
+    const OK: Self;
+    const EOF: Self;
+
+    fn to_usize(self) -> usize;
+}
+
 #[derive(Debug)]
-pub(super) struct DecompressState<D> {
+pub struct CompressState<C: Compressor> {
+    compressor: Option<C>,
+}
+
+impl<C: Compressor> CompressState<C> {
+    pub fn new(compressor: C) -> Self {
+        Self {
+            compressor: Some(compressor),
+        }
+    }
+
+    fn get_compressor(&mut self, vm: &VirtualMachine) -> PyResult<&mut C> {
+        self.compressor
+            .as_mut()
+            .ok_or_else(|| C::new_error(USE_AFTER_FINISH_ERR, vm))
+    }
+
+    pub fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+        let mut buf = Vec::new();
+        let compressor = self.get_compressor(vm)?;
+
+        for mut chunk in data.chunks(C::CHUNKSIZE) {
+            while !chunk.is_empty() {
+                buf.reserve(C::DEF_BUF_SIZE);
+                let prev_in = compressor.total_in();
+                compressor.compress_vec(chunk, &mut buf, C::Flush::NONE, vm)?;
+                let consumed = compressor.total_in() - prev_in;
+                chunk = &chunk[consumed..];
+            }
+        }
+
+        buf.shrink_to_fit();
+        Ok(buf)
+    }
+
+    pub fn flush(&mut self, mode: C::Flush, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+        let mut buf = Vec::new();
+        let compressor = self.get_compressor(vm)?;
+
+        let status = loop {
+            if buf.len() == buf.capacity() {
+                buf.reserve(C::DEF_BUF_SIZE);
+            }
+            let status = compressor.compress_vec(&[], &mut buf, mode, vm)?;
+            if buf.len() != buf.capacity() {
+                break status;
+            }
+        };
+
+        if status.to_usize() == C::Status::EOF.to_usize() {
+            if mode.to_usize() == C::Flush::FINISH.to_usize() {
+                self.compressor = None;
+            } else {
+                return Err(C::new_error("unexpected eof", vm));
+            }
+        }
+
+        buf.shrink_to_fit();
+        Ok(buf)
+    }
+}
+
+#[derive(Debug)]
+pub struct DecompressState<D> {
     decompress: D,
     unused_data: PyBytesRef,
     input_buffer: Vec<u8>,
@@ -248,7 +354,7 @@ impl<D: Decompressor> DecompressState<D> {
     }
 }
 
-pub(super) enum DecompressError<E> {
+pub enum DecompressError<E> {
     Decompress(E),
     Eof(EofError),
 }
@@ -259,7 +365,7 @@ impl<E> From<E> for DecompressError<E> {
     }
 }
 
-pub(super) struct EofError;
+pub struct EofError;
 
 impl ToPyException for EofError {
     fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {

stdlib/src/compression/mod.rs

@@ -1,5 +0,0 @@
-mod generic;
-use generic::*;
-
-pub mod bz2;
-pub mod zlib;

stdlib/src/lib.rs

@@ -16,6 +16,12 @@ mod csv;
 mod dis;
 mod gc;
 
+mod bz2;
+mod compression; // internal module
+#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
+mod lzma;
+mod zlib;
+
 mod blake2;
 mod hashlib;
 mod md5;
@@ -24,7 +30,6 @@ mod sha256;
 mod sha3;
 mod sha512;
 
-mod compression;
 mod json;
 #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))]
 mod locale;
@@ -111,7 +116,7 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "array" => array::make_module,
             "binascii" => binascii::make_module,
             "_bisect" => bisect::make_module,
-            "_bz2" => compression::bz2::make_module,
+            "_bz2" => bz2::make_module,
             "cmath" => cmath::make_module,
             "_contextvars" => contextvars::make_module,
             "_csv" => csv::make_module,
@@ -132,7 +137,7 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_statistics" => statistics::make_module,
             "_struct" => pystruct::make_module,
             "unicodedata" => unicodedata::make_module,
-            "zlib" => compression::zlib::make_module,
+            "zlib" => zlib::make_module,
             "_statistics" => statistics::make_module,
             "_suggestions" => suggestions::make_module,
             // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module,
@@ -150,6 +155,10 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_multiprocessing" => multiprocessing::make_module,
             "_socket" => socket::make_module,
         }
+        #[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
+        {
+            "_lzma" => lzma::make_module,
+        }
         #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))]
         {
             "_sqlite3" => sqlite::make_module,

stdlib/src/lzma.rs

@@ -0,0 +1,395 @@
+// spell-checker:ignore ARMTHUMB
+
+pub(crate) use _lzma::make_module;
+
+#[pymodule]
+mod _lzma {
+    use crate::compression::{
+        CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs,
+        DecompressError, DecompressState, DecompressStatus, Decompressor,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64,
+        LZMA_CHECK_NONE as CHECK_NONE, LZMA_CHECK_SHA256 as CHECK_SHA256,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB,
+        LZMA_FILTER_IA64 as FILTER_IA64, LZMA_FILTER_LZMA1 as FILTER_LZMA1,
+        LZMA_FILTER_LZMA2 as FILTER_LZMA2, LZMA_FILTER_POWERPC as FILTER_POWERPC,
+        LZMA_FILTER_SPARC as FILTER_SPARC, LZMA_FILTER_X86 as FILTER_X86,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_MF_BT2 as MF_BT2, LZMA_MF_BT3 as MF_BT3, LZMA_MF_BT4 as MF_BT4, LZMA_MF_HC3 as MF_HC3,
+        LZMA_MF_HC4 as MF_HC4,
+    };
+    #[pyattr]
+    use lzma_sys::{LZMA_MODE_FAST as MODE_FAST, LZMA_MODE_NORMAL as MODE_NORMAL};
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME,
+        LZMA_PRESET_LEVEL_MASK as PRESET_LEVEL_MASK,
+    };
+    use rustpython_common::lock::PyMutex;
+    use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyTypeRef};
+    use rustpython_vm::convert::ToPyException;
+    use rustpython_vm::function::ArgBytesLike;
+    use rustpython_vm::types::Constructor;
+    use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine};
+    use std::fmt;
+    use xz2::stream::{Action, Check, Error, Filters, Status, Stream};
+
+    #[cfg(windows)]
+    type EnumVal = i32;
+    #[cfg(not(windows))]
+    type EnumVal = u32;
+
+    const BUFSIZ: usize = 8192;
+    // TODO: can't find this in lzma-sys, but find way not to hardcode this
+    #[pyattr]
+    const FILTER_DELTA: i32 = 3;
+    #[pyattr]
+    const CHECK_UNKNOWN: i32 = 16;
+
+    // the variant ids are hardcoded to be equivalent to the C enum values
+    enum Format {
+        Auto = 0,
+        Xz = 1,
+        Alone = 2,
+        Raw = 3,
+    }
+
+    #[pyattr]
+    const FORMAT_AUTO: i32 = Format::Auto as i32;
+    #[pyattr]
+    const FORMAT_XZ: i32 = Format::Xz as i32;
+    #[pyattr]
+    const FORMAT_ALONE: i32 = Format::Alone as i32;
+    #[pyattr]
+    const FORMAT_RAW: i32 = Format::Raw as i32;
+
+    #[pyattr(once, name = "LZMAError")]
+    fn error(vm: &VirtualMachine) -> PyTypeRef {
+        vm.ctx.new_exception_type(
+            "lzma",
+            "LZMAError",
+            Some(vec![vm.ctx.exceptions.exception_type.to_owned()]),
+        )
+    }
+
+    fn new_lzma_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+        vm.new_exception_msg(vm.class("lzma", "LZMAError"), message.into())
+    }
+
+    #[pyfunction]
+    fn is_check_supported(check: i32) -> bool {
+        unsafe { lzma_sys::lzma_check_is_supported(check as _) != 0 }
+    }
+
+    // TODO: To implement these we need a function to convert a pyobject to a lzma filter and related structs
+    #[pyfunction]
+    fn _encode_filter_properties() -> PyResult<()> {
+        Ok(())
+    }
+
+    #[pyfunction]
+    fn _decode_filter_properties(_filter_id: u64, _buffer: ArgBytesLike) -> PyResult<()> {
+        Ok(())
+    }
+
+    #[pyattr]
+    #[pyclass(name = "LZMADecompressor")]
+    #[derive(PyPayload)]
+    struct LZMADecompressor {
+        state: PyMutex<DecompressState<Stream>>,
+    }
+
+    impl Decompressor for Stream {
+        type Flush = ();
+        type Status = Status;
+        type Error = Error;
+
+        fn total_in(&self) -> u64 {
+            self.total_in()
+        }
+        fn decompress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            (): Self::Flush,
+        ) -> Result<Self::Status, Self::Error> {
+            self.process_vec(input, output, Action::Run)
+        }
+    }
+
+    impl DecompressStatus for Status {
+        fn is_stream_end(&self) -> bool {
+            *self == Status::StreamEnd
+        }
+    }
+
+    impl fmt::Debug for LZMADecompressor {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "_lzma.LZMADecompressor")
+        }
+    }
+    #[derive(FromArgs)]
+    pub struct LZMADecompressorConstructorArgs {
+        #[pyarg(any, default = FORMAT_AUTO)]
+        format: i32,
+        #[pyarg(any, optional)]
+        memlimit: Option<u64>,
+        #[pyarg(any, optional)]
+        filters: Option<u32>,
+    }
+
+    impl Constructor for LZMADecompressor {
+        type Args = LZMADecompressorConstructorArgs;
+
+        fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult {
+            let memlimit = args.memlimit.unwrap_or(u64::MAX);
+            let filters = args.filters.unwrap_or(0);
+            let stream_result = match args.format {
+                FORMAT_AUTO => Stream::new_auto_decoder(memlimit, filters),
+                FORMAT_XZ => Stream::new_stream_decoder(memlimit, filters),
+                FORMAT_ALONE => Stream::new_lzma_decoder(memlimit),
+                // TODO: FORMAT_RAW
+                _ => return Err(new_lzma_error("Invalid format", vm)),
+            };
+            Self {
+                state: PyMutex::new(DecompressState::new(
+                    stream_result
+                        .map_err(|_| new_lzma_error("Failed to initialize decoder", vm))?,
+                    vm,
+                )),
+            }
+            .into_ref_with_type(vm, cls)
+            .map(Into::into)
+        }
+    }
+
+    #[pyclass(with(Constructor))]
+    impl LZMADecompressor {
+        #[pymethod]
+        fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            let max_length = args.max_length();
+            let data = &*args.data();
+
+            let mut state = self.state.lock();
+            state
+                .decompress(data, max_length, BUFSIZ, vm)
+                .map_err(|e| match e {
+                    DecompressError::Decompress(err) => vm.new_os_error(err.to_string()),
+                    DecompressError::Eof(err) => err.to_pyexception(vm),
+                })
+        }
+
+        #[pygetset]
+        fn eof(&self) -> bool {
+            self.state.lock().eof()
+        }
+
+        #[pygetset]
+        fn unused_data(&self) -> PyBytesRef {
+            self.state.lock().unused_data()
+        }
+
+        #[pygetset]
+        fn needs_input(&self) -> bool {
+            // False if the decompress() method can provide more
+            // decompressed data before requiring new uncompressed input.
+            self.state.lock().needs_input()
+        }
+
+        // TODO: mro()?
+    }
+
+    struct CompressorInner {
+        stream: Stream,
+    }
+
+    impl CompressStatusKind for Status {
+        const OK: Self = Status::Ok;
+        const EOF: Self = Status::StreamEnd;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl CompressFlushKind for Action {
+        const NONE: Self = Action::Run;
+        const FINISH: Self = Action::Finish;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl Compressor for CompressorInner {
+        type Status = Status;
+        type Flush = Action;
+        const CHUNKSIZE: usize = u32::MAX as usize;
+        const DEF_BUF_SIZE: usize = 16 * 1024;
+
+        fn compress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            flush: Self::Flush,
+            vm: &VirtualMachine,
+        ) -> PyResult<Self::Status> {
+            self.stream
+                .process_vec(input, output, flush)
+                .map_err(|_| new_lzma_error("Failed to compress data", vm))
+        }
+
+        fn total_in(&mut self) -> usize {
+            self.stream.total_in() as usize
+        }
+
+        fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+            new_lzma_error(message, vm)
+        }
+    }
+
+    impl CompressorInner {
+        fn new(stream: Stream) -> Self {
+            Self { stream }
+        }
+    }
+
+    #[pyattr]
+    #[pyclass(name = "LZMACompressor")]
+    #[derive(PyPayload)]
+    struct LZMACompressor {
+        state: PyMutex<CompressState<CompressorInner>>,
+    }
+
+    impl fmt::Debug for LZMACompressor {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "_lzma.LZMACompressor")
+        }
+    }
+
+    fn int_to_check(check: i32) -> Option<Check> {
+        if check == -1 {
+            return Some(Check::None);
+        }
+        match check as EnumVal {
+            CHECK_NONE => Some(Check::None),
+            CHECK_CRC32 => Some(Check::Crc32),
+            CHECK_CRC64 => Some(Check::Crc64),
+            CHECK_SHA256 => Some(Check::Sha256),
+            _ => None,
+        }
+    }
+
+    fn parse_filter_chain_spec(
+        filter_specs: Vec<PyObjectRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<Filters> {
+        // TODO: don't hardcode
+        const LZMA_FILTERS_MAX: usize = 4;
+        if filter_specs.len() > LZMA_FILTERS_MAX {
+            return Err(new_lzma_error(
+                format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"),
+                vm,
+            ));
+        }
+        let filters = Filters::new();
+        for _item in filter_specs {}
+        Ok(filters)
+    }
+
+    impl LZMACompressor {
+        fn init_xz(
+            check: i32,
+            preset: u32,
+            filters: Option<Vec<PyObjectRef>>,
+            vm: &VirtualMachine,
+        ) -> PyResult<Stream> {
+            let real_check = int_to_check(check)
+                .ok_or_else(|| vm.new_type_error("Invalid check value".to_string()))?;
+            if let Some(filters) = filters {
+                let filters = parse_filter_chain_spec(filters, vm)?;
+                Ok(Stream::new_stream_encoder(&filters, real_check)
+                    .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
+            } else {
+                Ok(Stream::new_easy_encoder(preset, real_check)
+                    .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
+            }
+        }
+    }
+
+    #[derive(FromArgs)]
+    pub struct LZMACompressorConstructorArgs {
+        // format=FORMAT_XZ, check=-1, preset=None, filters=None
+        //  {'format': 3, 'filters': [{'id': 3, 'dist': 2}, {'id': 33, 'preset': 2147483654}]}
+        #[pyarg(any, default = FORMAT_XZ)]
+        format: i32,
+        #[pyarg(any, default = -1)]
+        check: i32,
+        #[pyarg(any, optional)]
+        preset: Option<u32>,
+        #[pyarg(any, optional)]
+        filters: Option<Vec<PyObjectRef>>,
+        #[pyarg(any, optional)]
+        _filter_specs: Option<Vec<PyObjectRef>>,
+        #[pyarg(positional, optional)]
+        preset_obj: Option<PyObjectRef>,
+    }
+
+    impl Constructor for LZMACompressor {
+        type Args = LZMACompressorConstructorArgs;
+
+        fn py_new(_cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult {
+            let preset = args.preset.unwrap_or(PRESET_DEFAULT);
+            #[allow(clippy::unnecessary_cast)]
+            if args.format != FORMAT_XZ as i32
+                && args.check != -1
+                && args.check != CHECK_NONE as i32
+            {
+                return Err(new_lzma_error(
+                    "Integrity checks are only supported by FORMAT_XZ",
+                    vm,
+                ));
+            }
+            if args.preset_obj.is_some() && args.filters.is_some() {
+                return Err(new_lzma_error(
+                    "Cannot specify both preset and filter chain",
+                    vm,
+                ));
+            }
+            let stream = match args.format {
+                FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?,
+                // TODO: ALONE AND RAW
+                _ => return Err(new_lzma_error("Invalid format", vm)),
+            };
+            Ok(Self {
+                state: PyMutex::new(CompressState::new(CompressorInner::new(stream))),
+            }
+            .into_pyobject(vm))
+        }
+    }
+
+    #[pyclass(with(Constructor))]
+    impl LZMACompressor {
+        #[pymethod]
+        fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            let mut state = self.state.lock();
+            // TODO: Flush check
+            state.compress(&data.borrow_buf(), vm)
+        }
+
+        #[pymethod]
+        fn flush(&self, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            // TODO: flush check
+            let mut state = self.state.lock();
+            // TODO: check if action is correct
+            state.flush(Action::Finish, vm)
+        }
+    }
+}

stdlib/src/zlib.rs

@@ -1,12 +1,13 @@
-// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz
+// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker
 
 pub(crate) use zlib::make_module;
 
 #[pymodule]
 mod zlib {
-    use super::super::{
-        _decompress, CHUNKSIZE, DecompressArgs, DecompressError, DecompressState, DecompressStatus,
-        Decompressor, FlushKind, flush_sync,
+    use crate::compression::{
+        _decompress, CompressFlushKind, CompressState, CompressStatusKind, Compressor,
+        DecompressArgs, DecompressError, DecompressFlushKind, DecompressState, DecompressStatus,
+        Decompressor, USE_AFTER_FINISH_ERR, flush_sync,
     };
     use crate::vm::{
         PyObject, PyPayload, PyResult, VirtualMachine,
@@ -287,11 +288,10 @@ mod zlib {
 
         #[pymethod]
         fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let max_length: usize = args
-                .max_length
-                .map_or(0, |x| x.value)
-                .try_into()
-                .map_err(|_| vm.new_value_error("must be non-negative".to_owned()))?;
+            let max_length: usize =
+                args.raw_max_length().unwrap_or(0).try_into().map_err(|_| {
+                    vm.new_value_error("max_length must be non-negative".to_owned())
+                })?;
             let max_length = (max_length != 0).then_some(max_length);
             let data = &*args.data();
 
@@ -362,20 +362,20 @@ mod zlib {
             zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap());
         }
         Ok(PyCompress {
-            inner: PyMutex::new(CompressInner::new(compress)),
+            inner: PyMutex::new(CompressState::new(CompressInner::new(compress))),
         })
     }
 
     #[derive(Debug)]
     struct CompressInner {
-        compress: Option<Compress>,
+        compress: Compress,
     }
 
     #[pyattr]
     #[pyclass(name = "Compress")]
     #[derive(Debug, PyPayload)]
     struct PyCompress {
-        inner: PyMutex<CompressInner>,
+        inner: PyMutex<CompressState<CompressInner>>,
     }
 
     #[pyclass]
@@ -408,62 +408,56 @@ mod zlib {
         // }
     }
 
+    const CHUNKSIZE: usize = u32::MAX as usize;
+
     impl CompressInner {
         fn new(compress: Compress) -> Self {
-            Self {
-                compress: Some(compress),
-            }
+            Self { compress }
         }
+    }
 
-        fn get_compress(&mut self, vm: &VirtualMachine) -> PyResult<&mut Compress> {
+    impl CompressStatusKind for Status {
+        const OK: Self = Status::Ok;
+        const EOF: Self = Status::StreamEnd;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl CompressFlushKind for FlushCompress {
+        const NONE: Self = FlushCompress::None;
+        const FINISH: Self = FlushCompress::Finish;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl Compressor for CompressInner {
+        type Status = Status;
+        type Flush = FlushCompress;
+        const CHUNKSIZE: usize = CHUNKSIZE;
+        const DEF_BUF_SIZE: usize = DEF_BUF_SIZE;
+
+        fn compress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            flush: Self::Flush,
+            vm: &VirtualMachine,
+        ) -> PyResult<Self::Status> {
             self.compress
-                .as_mut()
-                .ok_or_else(|| new_zlib_error(USE_AFTER_FINISH_ERR, vm))
+                .compress_vec(input, output, flush)
+                .map_err(|_| new_zlib_error("error while compressing", vm))
         }
 
-        fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let c = self.get_compress(vm)?;
-            let mut buf = Vec::new();
-
-            for mut chunk in data.chunks(CHUNKSIZE) {
-                while !chunk.is_empty() {
-                    buf.reserve(DEF_BUF_SIZE);
-                    let prev_in = c.total_in();
-                    c.compress_vec(chunk, &mut buf, FlushCompress::None)
-                        .map_err(|_| new_zlib_error("error while compressing", vm))?;
-                    let consumed = c.total_in() - prev_in;
-                    chunk = &chunk[consumed as usize..];
-                }
-            }
-
-            buf.shrink_to_fit();
-            Ok(buf)
+        fn total_in(&mut self) -> usize {
+            self.compress.total_in() as usize
         }
 
-        fn flush(&mut self, mode: FlushCompress, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let c = self.get_compress(vm)?;
-            let mut buf = Vec::new();
-
-            let status = loop {
-                if buf.len() == buf.capacity() {
-                    buf.reserve(DEF_BUF_SIZE);
-                }
-                let status = c
-                    .compress_vec(&[], &mut buf, mode)
-                    .map_err(|_| new_zlib_error("error while compressing", vm))?;
-                if buf.len() != buf.capacity() {
-                    break status;
-                }
-            };
-
-            match status {
-                Status::Ok | Status::BufError => {}
-                Status::StreamEnd if mode == FlushCompress::Finish => self.compress = None,
-                Status::StreamEnd => return Err(new_zlib_error("unexpected eof", vm)),
-            }
-
-            buf.shrink_to_fit();
-            Ok(buf)
+        fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+            new_zlib_error(message, vm)
         }
     }
 
@@ -471,8 +465,6 @@ mod zlib {
         vm.new_exception_msg(vm.class("zlib", "error"), message.into())
     }
 
-    const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state";
-
     struct Level(Option<flate2::Compression>);
 
     impl Level {
@@ -520,7 +512,7 @@ mod zlib {
         }
     }
 
-    impl FlushKind for FlushDecompress {
+    impl DecompressFlushKind for FlushDecompress {
         const SYNC: Self = FlushDecompress::Sync;
     }
 
@@ -607,7 +599,7 @@ mod zlib {
 
         #[pymethod]
         fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let max_length = args.max_length_negative_is_none();
+            let max_length = args.max_length();
             let data = &*args.data();
 
             let inner = &mut *self.inner.lock();