_lzma implementation and test marking

Signed-off-by: Ashwin Naren <arihant2math@gmail.com>

Cargo.lock

@@ -237,12 +237,6 @@ version = "1.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540"
 
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
 [[package]]
 name = "bzip2"
 version = "0.5.2"
@@ -636,21 +630,6 @@ dependencies = [
  "target-lexicon",
 ]
 
-[[package]]
-name = "crc"
-version = "3.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
-dependencies = [
- "crc-catalog",
-]
-
-[[package]]
-name = "crc-catalog"
-version = "2.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
-
 [[package]]
 name = "crc32fast"
 version = "1.4.2"
@@ -1418,13 +1397,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "lzma-rs"
-version = "0.3.0"
+name = "lzma-sys"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
 dependencies = [
- "byteorder",
- "crc",
+ "cc",
+ "libc",
+ "pkg-config",
 ]
 
 [[package]]
@@ -2498,7 +2478,7 @@ dependencies = [
  "libc",
  "libsqlite3-sys",
  "libz-rs-sys",
- "lzma-rs",
+ "lzma-sys",
  "mac_address",
  "malachite-bigint",
  "md-5",
@@ -2544,6 +2524,7 @@ dependencies = [
  "widestring",
  "windows-sys 0.59.0",
  "xml-rs",
+ "xz2",
 ]
 
 [[package]]
@@ -3707,6 +3688,15 @@ version = "0.8.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4"
 
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"

Lib/tarfile.py

@@ -392,7 +392,8 @@ class _Stream:
 
             elif comptype == "xz":
                 try:
-                    import lzma
+                    # TODO: RUSTPYTHON remove underscore
+                    import lzma_
                 except ImportError:
                     raise CompressionError("lzma module is not available") from None
                 if mode == "r":
@@ -1923,7 +1924,8 @@ class TarFile(object):
             raise ValueError("mode must be 'r', 'w' or 'x'")
 
         try:
-            from lzma import LZMAFile, LZMAError
+            # TODO: RUSTPYTHON remove underscore
+            from lzma_ import LZMAFile, LZMAError
         except ImportError:
             raise CompressionError("lzma module is not available") from None
 

Lib/test/support/__init__.py

@@ -494,10 +494,12 @@ def requires_bz2(reason='requires bz2'):
     return unittest.skipUnless(bz2, reason)
 
 def requires_lzma(reason='requires lzma'):
-    try:
-        import lzma
-    except ImportError:
-        lzma = None
+    # try:
+        # import lzma
+    # except ImportError:
+        # lzma = None
+    # TODO: RUSTPYTHON
+    lzma = None
     return unittest.skipUnless(lzma, reason)
 
 def has_no_debug_ranges():

Lib/test/test_lzma.py

@@ -22,6 +22,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
 
     # Test error cases.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_simple_bad_args(self):
         self.assertRaises(TypeError, LZMACompressor, [])
         self.assertRaises(TypeError, LZMACompressor, format=3.45)
@@ -62,6 +64,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd.decompress(empty)
         self.assertRaises(EOFError, lzd.decompress, b"quux")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_bad_filter_spec(self):
         self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"])
         self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}])
@@ -78,6 +82,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd.decompress(COMPRESSED_XZ)
         self.assertRaises(EOFError, lzd.decompress, b"nyan")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_memlimit(self):
         lzd = LZMADecompressor(memlimit=1024)
         self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
@@ -98,6 +104,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         self.assertTrue(lzd.eof)
         self.assertEqual(lzd.unused_data, unused_data)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_auto(self):
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
@@ -105,30 +113,44 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_xz(self):
         lzd = LZMADecompressor(lzma.FORMAT_XZ)
         self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_alone(self):
         lzd = LZMADecompressor(lzma.FORMAT_ALONE)
         self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_raw_1(self):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
         self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_raw_2(self):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
         self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_raw_3(self):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
         self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_raw_4(self):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_chunks(self):
         lzd = LZMADecompressor()
         out = []
@@ -141,6 +163,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         self.assertTrue(lzd.eof)
         self.assertEqual(lzd.unused_data, b"")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_chunks_empty(self):
         lzd = LZMADecompressor()
         out = []
@@ -156,6 +180,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         self.assertTrue(lzd.eof)
         self.assertEqual(lzd.unused_data, b"")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_chunks_maxsize(self):
         lzd = LZMADecompressor()
         max_length = 100
@@ -247,12 +273,16 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         out.append(lzd.decompress(COMPRESSED_XZ[300:]))
         self.assertEqual(b''.join(out), INPUT)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_unused_data(self):
         lzd = LZMADecompressor()
         extra = b"fooblibar"
         self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64,
                                 unused_data=extra)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_bad_input(self):
         lzd = LZMADecompressor()
         self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1)
@@ -266,6 +296,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
         self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_bug_28275(self):
         # Test coverage for Issue 28275
         lzd = LZMADecompressor()
@@ -275,24 +307,32 @@ class CompressorDecompressorTestCase(unittest.TestCase):
 
     # Test that LZMACompressor->LZMADecompressor preserves the input data.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_xz(self):
         lzc = LZMACompressor()
         cdata = lzc.compress(INPUT) + lzc.flush()
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_alone(self):
         lzc = LZMACompressor(lzma.FORMAT_ALONE)
         cdata = lzc.compress(INPUT) + lzc.flush()
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_raw(self):
         lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         cdata = lzc.compress(INPUT) + lzc.flush()
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_raw_empty(self):
         lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         cdata = lzc.compress(INPUT)
@@ -303,6 +343,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         self._test_decompressor(lzd, cdata, lzma.CHECK_NONE)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_chunks(self):
         lzc = LZMACompressor()
         cdata = []
@@ -313,6 +355,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip_empty_chunks(self):
         lzc = LZMACompressor()
         cdata = []
@@ -328,6 +372,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
 
     # LZMADecompressor intentionally does not handle concatenated streams.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompressor_multistream(self):
         lzd = LZMADecompressor()
         self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE,
@@ -363,6 +409,8 @@ class CompressorDecompressorTestCase(unittest.TestCase):
 
     # Pickling raises an exception; there's no way to serialize an lzma_stream.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_pickle(self):
         for proto in range(pickle.HIGHEST_PROTOCOL + 1):
             with self.assertRaises(TypeError):
@@ -388,6 +436,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
 
     # Test error cases:
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_bad_args(self):
         self.assertRaises(TypeError, lzma.compress)
         self.assertRaises(TypeError, lzma.compress, [])
@@ -417,6 +467,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
             lzma.decompress(
                 b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_memlimit(self):
         with self.assertRaises(LZMAError):
             lzma.decompress(COMPRESSED_XZ, memlimit=1024)
@@ -429,6 +481,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
 
     # Test LZMADecompressor on known-good input data.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_good_input(self):
         ddata = lzma.decompress(COMPRESSED_XZ)
         self.assertEqual(ddata, INPUT)
@@ -458,6 +512,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
             COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
         self.assertEqual(ddata, INPUT)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_incomplete_input(self):
         self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128])
         self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128])
@@ -470,6 +526,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
         self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128],
                           format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_bad_input(self):
         with self.assertRaises(LZMAError):
             lzma.decompress(COMPRESSED_BOGUS)
@@ -485,6 +543,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
 
     # Test that compress()->decompress() preserves the input data.
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_roundtrip(self):
         cdata = lzma.compress(INPUT)
         ddata = lzma.decompress(cdata)
@@ -510,10 +570,14 @@ class CompressDecompressFunctionTestCase(unittest.TestCase):
 
     # Test robust handling of non-LZMA data following the compressed stream(s).
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_trailing_junk(self):
         ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS)
         self.assertEqual(ddata, INPUT)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_decompress_multistream_trailing_junk(self):
         ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS)
         self.assertEqual(ddata, INPUT * 3)
@@ -550,6 +614,8 @@ class FileTestCase(unittest.TestCase):
             self.assertIsInstance(f, LZMAFile)
             self.assertEqual(f.mode, "wb")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_init_with_PathLike_filename(self):
         filename = FakePath(TESTFN)
         with TempFile(filename, COMPRESSED_XZ):
@@ -630,6 +696,8 @@ class FileTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             LZMAFile(BytesIO(COMPRESSED_XZ), "rw")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_init_bad_check(self):
         with self.assertRaises(TypeError):
             LZMAFile(BytesIO(), "w", check=b"asd")
@@ -667,6 +735,8 @@ class FileTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             LZMAFile(BytesIO(COMPRESSED_XZ), preset=3)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_init_bad_filter_spec(self):
         with self.assertRaises(TypeError):
             LZMAFile(BytesIO(), "w", filters=[b"wobsite"])
@@ -684,6 +754,8 @@ class FileTestCase(unittest.TestCase):
             LZMAFile(BytesIO(), "w",
                      filters=[{"id": lzma.FILTER_X86, "foo": 0}])
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_init_with_preset_and_filters(self):
         with self.assertRaises(ValueError):
             LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW,
@@ -802,6 +874,8 @@ class FileTestCase(unittest.TestCase):
             f.close()
         self.assertRaises(ValueError, f.writable)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_read(self):
         with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
             self.assertEqual(f.read(), INPUT)
@@ -849,6 +923,8 @@ class FileTestCase(unittest.TestCase):
                 chunks.append(result)
             self.assertEqual(b"".join(chunks), INPUT)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_read_multistream(self):
         with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f:
             self.assertEqual(f.read(), INPUT * 5)
@@ -869,10 +945,14 @@ class FileTestCase(unittest.TestCase):
         finally:
             _compression.BUFFER_SIZE = saved_buffer_size
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_read_trailing_junk(self):
         with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f:
             self.assertEqual(f.read(), INPUT)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_read_multistream_trailing_junk(self):
         with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f:
             self.assertEqual(f.read(), INPUT * 5)
@@ -978,6 +1058,8 @@ class FileTestCase(unittest.TestCase):
         with LZMAFile(BytesIO(COMPRESSED_XZ)) as f:
             self.assertRaises(TypeError, f.read, float())
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_read_bad_data(self):
         with LZMAFile(BytesIO(COMPRESSED_BOGUS)) as f:
             self.assertRaises(LZMAError, f.read)
@@ -1035,6 +1117,8 @@ class FileTestCase(unittest.TestCase):
         with LZMAFile(BytesIO(), "w") as f:
             self.assertRaises(ValueError, f.peek)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_iterator(self):
         with BytesIO(INPUT) as f:
             lines = f.readlines()
@@ -1074,6 +1158,8 @@ class FileTestCase(unittest.TestCase):
         self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp,
                              "Excessive amount of data was decompressed")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_write(self):
         with BytesIO() as dst:
             with LZMAFile(dst, "w") as f:
@@ -1342,6 +1428,8 @@ class FileTestCase(unittest.TestCase):
         f.close()
         self.assertRaises(ValueError, f.tell)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_issue21872(self):
         # sometimes decompress data incompletely
 
@@ -1425,6 +1513,8 @@ class OpenTestCase(unittest.TestCase):
             with lzma.open(TESTFN, "rb") as f:
                 self.assertEqual(f.read(), INPUT * 2)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_with_pathlike_filename(self):
         filename = FakePath(TESTFN)
         with TempFile(filename):
@@ -1451,6 +1541,8 @@ class OpenTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             lzma.open(TESTFN, "rb", newline="\n")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_format_and_filters(self):
         # Test non-default format and filter chain.
         options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1}
@@ -1481,6 +1573,8 @@ class OpenTestCase(unittest.TestCase):
             with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f:
                 self.assertEqual(f.read(), "foobar")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_newline(self):
         # Test with explicit newline (universal newline mode disabled).
         text = INPUT.decode("ascii")
@@ -1505,6 +1599,8 @@ class OpenTestCase(unittest.TestCase):
 
 class MiscellaneousTestCase(unittest.TestCase):
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_is_check_supported(self):
         # CHECK_NONE and CHECK_CRC32 should always be supported,
         # regardless of the options liblzma was compiled with.
@@ -1517,6 +1613,8 @@ class MiscellaneousTestCase(unittest.TestCase):
         # This value should not be a valid check ID.
         self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN))
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test__encode_filter_properties(self):
         with self.assertRaises(TypeError):
             lzma._encode_filter_properties(b"not a dict")
@@ -1538,6 +1636,8 @@ class MiscellaneousTestCase(unittest.TestCase):
         })
         self.assertEqual(props, b"]\x00\x00\x80\x00")
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test__decode_filter_properties(self):
         with self.assertRaises(TypeError):
             lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes})
@@ -1561,6 +1661,8 @@ class MiscellaneousTestCase(unittest.TestCase):
             filterspec = lzma._decode_filter_properties(f, b"")
             self.assertEqual(filterspec, {"id": f})
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_filter_properties_roundtrip(self):
         spec1 = lzma._decode_filter_properties(
             lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00")
@@ -2092,4 +2194,4 @@ ISSUE_21872_DAT = (
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

Lib/test/test_tarfile.py

@@ -28,7 +28,9 @@ try:
 except ImportError:
     bz2 = None
 try:
-    import lzma
+    # TODO: RUSTPYTHON
+    lzma = None
+    # import lzma
 except ImportError:
     lzma = None
 

Lib/test/test_zoneinfo/test_zoneinfo.py

@@ -1728,6 +1728,8 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase):
                 with self.subTest("filtered", path_var=path_var):
                     self.assertSequenceEqual(tzpath, expected_paths)
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_env_variable_relative_paths_warning_location(self):
         path_var = "path/to/somewhere"
 
@@ -1822,6 +1824,8 @@ class TestModule(ZoneInfoTestBase):
         with self.assertRaises(AttributeError):
             self.module.NOATTRIBUTE
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_dir_contains_all(self):
         """dir(self.module) should at least contain everything in __all__."""
         module_all_set = set(self.module.__all__)
@@ -1925,12 +1929,16 @@ class ExtensionBuiltTest(unittest.TestCase):
     rely on these tests as an indication of stable properties of these classes.
     """
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_cache_location(self):
         # The pure Python version stores caches on attributes, but the C
         # extension stores them in C globals (at least for now)
         self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache"))
         self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache"))
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_gc_tracked(self):
         import gc
 

stdlib/Cargo.toml

@@ -80,7 +80,6 @@ crc32fast = "1.3.2"
 flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] }
 libz-sys = { package = "libz-rs-sys", version = "0.5" }
 bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] }
-lzma-rs = "0.3"
 
 # tkinter
 tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true }
@@ -113,6 +112,8 @@ foreign-types-shared = { version = "0.1.1", optional = true }
 
 [target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies]
 libsqlite3-sys = { version = "0.28", features = ["bundled"], optional = true }
+lzma-sys = "0.1"
+xz2 = "0.1"
 
 [target.'cfg(windows)'.dependencies]
 junction = { workspace = true }

stdlib/src/compression/bz2.rs

@@ -5,6 +5,9 @@ pub(crate) use _bz2::make_module;
 #[pymodule]
 mod _bz2 {
     use crate::common::lock::PyMutex;
+    use crate::compression::{
+        DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor,
+    };
     use crate::vm::{
         VirtualMachine,
         builtins::{PyBytesRef, PyTypeRef},
@@ -12,9 +15,6 @@ mod _bz2 {
         object::{PyPayload, PyResult},
         types::Constructor,
     };
-    use crate::zlib::{
-        DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor,
-    };
     use bzip2::{Decompress, Status, write::BzEncoder};
     use rustpython_vm::convert::ToPyException;
     use std::{fmt, io::Write};

stdlib/src/compression/lzma.rs

@@ -0,0 +1,395 @@
+// spell-checker:ignore ARMTHUMB
+
+pub(crate) use _lzma::make_module;
+
+#[pymodule]
+mod _lzma {
+    use crate::compression::{
+        CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs,
+        DecompressError, DecompressState, DecompressStatus, Decompressor,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64,
+        LZMA_CHECK_NONE as CHECK_NONE, LZMA_CHECK_SHA256 as CHECK_SHA256,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB,
+        LZMA_FILTER_IA64 as FILTER_IA64, LZMA_FILTER_LZMA1 as FILTER_LZMA1,
+        LZMA_FILTER_LZMA2 as FILTER_LZMA2, LZMA_FILTER_POWERPC as FILTER_POWERPC,
+        LZMA_FILTER_SPARC as FILTER_SPARC, LZMA_FILTER_X86 as FILTER_X86,
+    };
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_MF_BT2 as MF_BT2, LZMA_MF_BT3 as MF_BT3, LZMA_MF_BT4 as MF_BT4, LZMA_MF_HC3 as MF_HC3,
+        LZMA_MF_HC4 as MF_HC4,
+    };
+    #[pyattr]
+    use lzma_sys::{LZMA_MODE_FAST as MODE_FAST, LZMA_MODE_NORMAL as MODE_NORMAL};
+    #[pyattr]
+    use lzma_sys::{
+        LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME,
+        LZMA_PRESET_LEVEL_MASK as PRESET_LEVEL_MASK,
+    };
+    use rustpython_common::lock::PyMutex;
+    use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyTypeRef};
+    use rustpython_vm::convert::ToPyException;
+    use rustpython_vm::function::ArgBytesLike;
+    use rustpython_vm::types::Constructor;
+    use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine};
+    use std::fmt;
+    use xz2::stream::{Action, Check, Error, Filters, Status, Stream};
+
+    #[cfg(windows)]
+    type EnumVal = i32;
+    #[cfg(not(windows))]
+    type EnumVal = u32;
+
+    const BUFSIZ: usize = 8192;
+    // TODO: can't find this in lzma-sys, but find way not to hardcode this
+    #[pyattr]
+    const FILTER_DELTA: i32 = 3;
+    #[pyattr]
+    const CHECK_UNKNOWN: i32 = 16;
+
+    // the variant ids are hardcoded to be equivalent to the C enum values
+    enum Format {
+        Auto = 0,
+        Xz = 1,
+        Alone = 2,
+        Raw = 3,
+    }
+
+    #[pyattr]
+    const FORMAT_AUTO: i32 = Format::Auto as i32;
+    #[pyattr]
+    const FORMAT_XZ: i32 = Format::Xz as i32;
+    #[pyattr]
+    const FORMAT_ALONE: i32 = Format::Alone as i32;
+    #[pyattr]
+    const FORMAT_RAW: i32 = Format::Raw as i32;
+
+    #[pyattr(once, name = "LZMAError")]
+    fn error(vm: &VirtualMachine) -> PyTypeRef {
+        vm.ctx.new_exception_type(
+            "lzma",
+            "LZMAError",
+            Some(vec![vm.ctx.exceptions.exception_type.to_owned()]),
+        )
+    }
+
+    fn new_lzma_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+        vm.new_exception_msg(vm.class("lzma", "LZMAError"), message.into())
+    }
+
+    #[pyfunction]
+    fn is_check_supported(check: i32) -> bool {
+        unsafe { lzma_sys::lzma_check_is_supported(check as _) != 0 }
+    }
+
+    // TODO: To implement these we need a function to convert a pyobject to a lzma filter and related structs
+    #[pyfunction]
+    fn _encode_filter_properties() -> PyResult<()> {
+        Ok(())
+    }
+
+    #[pyfunction]
+    fn _decode_filter_properties(_filter_id: u64, _buffer: ArgBytesLike) -> PyResult<()> {
+        Ok(())
+    }
+
+    #[pyattr]
+    #[pyclass(name = "LZMADecompressor")]
+    #[derive(PyPayload)]
+    struct LZMADecompressor {
+        state: PyMutex<DecompressState<Stream>>,
+    }
+
+    impl Decompressor for Stream {
+        type Flush = ();
+        type Status = Status;
+        type Error = Error;
+
+        fn total_in(&self) -> u64 {
+            self.total_in()
+        }
+        fn decompress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            (): Self::Flush,
+        ) -> Result<Self::Status, Self::Error> {
+            self.process_vec(input, output, Action::Run)
+        }
+    }
+
+    impl DecompressStatus for Status {
+        fn is_stream_end(&self) -> bool {
+            *self == Status::StreamEnd
+        }
+    }
+
+    impl fmt::Debug for LZMADecompressor {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "_lzma.LZMADecompressor")
+        }
+    }
+    #[derive(FromArgs)]
+    pub struct LZMADecompressorConstructorArgs {
+        #[pyarg(any, default = FORMAT_AUTO)]
+        format: i32,
+        #[pyarg(any, optional)]
+        memlimit: Option<u64>,
+        #[pyarg(any, optional)]
+        filters: Option<u32>,
+    }
+
+    impl Constructor for LZMADecompressor {
+        type Args = LZMADecompressorConstructorArgs;
+
+        fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult {
+            let memlimit = args.memlimit.unwrap_or(u64::MAX);
+            let filters = args.filters.unwrap_or(0);
+            let stream_result = match args.format {
+                FORMAT_AUTO => Stream::new_auto_decoder(memlimit, filters),
+                FORMAT_XZ => Stream::new_stream_decoder(memlimit, filters),
+                FORMAT_ALONE => Stream::new_lzma_decoder(memlimit),
+                // TODO: FORMAT_RAW
+                _ => return Err(new_lzma_error("Invalid format", vm)),
+            };
+            Self {
+                state: PyMutex::new(DecompressState::new(
+                    stream_result
+                        .map_err(|_| new_lzma_error("Failed to initialize decoder", vm))?,
+                    vm,
+                )),
+            }
+            .into_ref_with_type(vm, cls)
+            .map(Into::into)
+        }
+    }
+
+    #[pyclass(with(Constructor))]
+    impl LZMADecompressor {
+        #[pymethod]
+        fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            let max_length = args.max_length();
+            let data = &*args.data();
+
+            let mut state = self.state.lock();
+            state
+                .decompress(data, max_length, BUFSIZ, vm)
+                .map_err(|e| match e {
+                    DecompressError::Decompress(err) => vm.new_os_error(err.to_string()),
+                    DecompressError::Eof(err) => err.to_pyexception(vm),
+                })
+        }
+
+        #[pygetset]
+        fn eof(&self) -> bool {
+            self.state.lock().eof()
+        }
+
+        #[pygetset]
+        fn unused_data(&self) -> PyBytesRef {
+            self.state.lock().unused_data()
+        }
+
+        #[pygetset]
+        fn needs_input(&self) -> bool {
+            // False if the decompress() method can provide more
+            // decompressed data before requiring new uncompressed input.
+            self.state.lock().needs_input()
+        }
+
+        // TODO: mro()?
+    }
+
+    struct CompressorInner {
+        stream: Stream,
+    }
+
+    impl CompressStatusKind for Status {
+        const OK: Self = Status::Ok;
+        const EOF: Self = Status::StreamEnd;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl CompressFlushKind for Action {
+        const NONE: Self = Action::Run;
+        const FINISH: Self = Action::Finish;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl Compressor for CompressorInner {
+        type Status = Status;
+        type Flush = Action;
+        const CHUNKSIZE: usize = u32::MAX as usize;
+        const DEF_BUF_SIZE: usize = 16 * 1024;
+
+        fn compress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            flush: Self::Flush,
+            vm: &VirtualMachine,
+        ) -> PyResult<Self::Status> {
+            self.stream
+                .process_vec(input, output, flush)
+                .map_err(|_| new_lzma_error("Failed to compress data", vm))
+        }
+
+        fn total_in(&mut self) -> usize {
+            self.stream.total_in() as usize
+        }
+
+        fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+            new_lzma_error(message, vm)
+        }
+    }
+
+    impl CompressorInner {
+        fn new(stream: Stream) -> Self {
+            Self { stream }
+        }
+    }
+
+    #[pyattr]
+    #[pyclass(name = "LZMACompressor")]
+    #[derive(PyPayload)]
+    struct LZMACompressor {
+        state: PyMutex<CompressState<CompressorInner>>,
+    }
+
+    impl fmt::Debug for LZMACompressor {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, "_lzma.LZMACompressor")
+        }
+    }
+
+    fn int_to_check(check: i32) -> Option<Check> {
+        if check == -1 {
+            return Some(Check::None);
+        }
+        match check as EnumVal {
+            CHECK_NONE => Some(Check::None),
+            CHECK_CRC32 => Some(Check::Crc32),
+            CHECK_CRC64 => Some(Check::Crc64),
+            CHECK_SHA256 => Some(Check::Sha256),
+            _ => None,
+        }
+    }
+
+    fn parse_filter_chain_spec(
+        filter_specs: Vec<PyObjectRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<Filters> {
+        // TODO: don't hardcode
+        const LZMA_FILTERS_MAX: usize = 4;
+        if filter_specs.len() > LZMA_FILTERS_MAX {
+            return Err(new_lzma_error(
+                format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"),
+                vm,
+            ));
+        }
+        let filters = Filters::new();
+        for _item in filter_specs {}
+        Ok(filters)
+    }
+
+    impl LZMACompressor {
+        fn init_xz(
+            check: i32,
+            preset: u32,
+            filters: Option<Vec<PyObjectRef>>,
+            vm: &VirtualMachine,
+        ) -> PyResult<Stream> {
+            let real_check = int_to_check(check)
+                .ok_or_else(|| vm.new_type_error("Invalid check value".to_string()))?;
+            if let Some(filters) = filters {
+                let filters = parse_filter_chain_spec(filters, vm)?;
+                Ok(Stream::new_stream_encoder(&filters, real_check)
+                    .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
+            } else {
+                Ok(Stream::new_easy_encoder(preset, real_check)
+                    .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?)
+            }
+        }
+    }
+
+    #[derive(FromArgs)]
+    pub struct LZMACompressorConstructorArgs {
+        // format=FORMAT_XZ, check=-1, preset=None, filters=None
+        //  {'format': 3, 'filters': [{'id': 3, 'dist': 2}, {'id': 33, 'preset': 2147483654}]}
+        #[pyarg(any, default = FORMAT_XZ)]
+        format: i32,
+        #[pyarg(any, default = -1)]
+        check: i32,
+        #[pyarg(any, optional)]
+        preset: Option<u32>,
+        #[pyarg(any, optional)]
+        filters: Option<Vec<PyObjectRef>>,
+        #[pyarg(any, optional)]
+        _filter_specs: Option<Vec<PyObjectRef>>,
+        #[pyarg(positional, optional)]
+        preset_obj: Option<PyObjectRef>,
+    }
+
+    impl Constructor for LZMACompressor {
+        type Args = LZMACompressorConstructorArgs;
+
+        fn py_new(_cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult {
+            let preset = args.preset.unwrap_or(PRESET_DEFAULT);
+            #[allow(clippy::unnecessary_cast)]
+            if args.format != FORMAT_XZ as i32
+                && args.check != -1
+                && args.check != CHECK_NONE as i32
+            {
+                return Err(new_lzma_error(
+                    "Integrity checks are only supported by FORMAT_XZ",
+                    vm,
+                ));
+            }
+            if args.preset_obj.is_some() && args.filters.is_some() {
+                return Err(new_lzma_error(
+                    "Cannot specify both preset and filter chain",
+                    vm,
+                ));
+            }
+            let stream = match args.format {
+                FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?,
+                // TODO: ALONE AND RAW
+                _ => return Err(new_lzma_error("Invalid format", vm)),
+            };
+            Ok(Self {
+                state: PyMutex::new(CompressState::new(CompressorInner::new(stream))),
+            }
+            .into_pyobject(vm))
+        }
+    }
+
+    #[pyclass(with(Constructor))]
+    impl LZMACompressor {
+        #[pymethod]
+        fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            let mut state = self.state.lock();
+            // TODO: Flush check
+            state.compress(&data.borrow_buf(), vm)
+        }
+
+        #[pymethod]
+        fn flush(&self, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+            // TODO: flush check
+            let mut state = self.state.lock();
+            // TODO: check if action is correct
+            state.flush(Action::Finish, vm)
+        }
+    }
+}

stdlib/src/compression/mod.rs

@@ -0,0 +1,363 @@
+// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker
+
+use crate::vm::function::{ArgBytesLike, ArgSize, OptionalArg};
+use crate::vm::{
+    PyResult, VirtualMachine,
+    builtins::{PyBaseExceptionRef, PyBytesRef},
+    convert::ToPyException,
+};
+pub(crate) mod bz2;
+#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
+pub(crate) mod lzma;
+pub(crate) mod zlib;
+
+pub const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state";
+// TODO: don't hardcode
+const CHUNKSIZE: usize = u32::MAX as usize;
+
+#[derive(Clone)]
+pub struct Chunker<'a> {
+    data1: &'a [u8],
+    data2: &'a [u8],
+}
+impl<'a> Chunker<'a> {
+    pub fn new(data: &'a [u8]) -> Self {
+        Self {
+            data1: data,
+            data2: &[],
+        }
+    }
+    pub fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self {
+        if data1.is_empty() {
+            Self {
+                data1: data2,
+                data2: &[],
+            }
+        } else {
+            Self { data1, data2 }
+        }
+    }
+    pub fn len(&self) -> usize {
+        self.data1.len() + self.data2.len()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.data1.is_empty()
+    }
+    pub fn to_vec(&self) -> Vec<u8> {
+        [self.data1, self.data2].concat()
+    }
+    pub fn chunk(&self) -> &'a [u8] {
+        self.data1.get(..CHUNKSIZE).unwrap_or(self.data1)
+    }
+    pub fn advance(&mut self, consumed: usize) {
+        self.data1 = &self.data1[consumed..];
+        if self.data1.is_empty() {
+            self.data1 = std::mem::take(&mut self.data2);
+        }
+    }
+}
+
+pub fn _decompress_chunks<D: Decompressor>(
+    data: &mut Chunker<'_>,
+    d: &mut D,
+    bufsize: usize,
+    max_length: Option<usize>,
+    calc_flush: impl Fn(bool) -> D::Flush,
+) -> Result<(Vec<u8>, bool), D::Error> {
+    if data.is_empty() {
+        return Ok((Vec::new(), true));
+    }
+    let max_length = max_length.unwrap_or(usize::MAX);
+    let mut buf = Vec::new();
+
+    'outer: loop {
+        let chunk = data.chunk();
+        let flush = calc_flush(chunk.len() == data.len());
+        loop {
+            let additional = std::cmp::min(bufsize, max_length - buf.capacity());
+            if additional == 0 {
+                return Ok((buf, false));
+            }
+            buf.reserve_exact(additional);
+
+            let prev_in = d.total_in();
+            let res = d.decompress_vec(chunk, &mut buf, flush);
+            let consumed = d.total_in() - prev_in;
+
+            data.advance(consumed as usize);
+
+            match res {
+                Ok(status) => {
+                    let stream_end = status.is_stream_end();
+                    if stream_end || data.is_empty() {
+                        // we've reached the end of the stream, we're done
+                        buf.shrink_to_fit();
+                        return Ok((buf, stream_end));
+                    } else if !chunk.is_empty() && consumed == 0 {
+                        // we're gonna need a bigger buffer
+                        continue;
+                    } else {
+                        // next chunk
+                        continue 'outer;
+                    }
+                }
+                Err(e) => {
+                    d.maybe_set_dict(e)?;
+                    // now try the next chunk
+                    continue 'outer;
+                }
+            };
+        }
+    }
+}
+
+pub trait Compressor {
+    type Status: CompressStatusKind;
+    type Flush: CompressFlushKind;
+    const CHUNKSIZE: usize;
+    const DEF_BUF_SIZE: usize;
+
+    fn compress_vec(
+        &mut self,
+        input: &[u8],
+        output: &mut Vec<u8>,
+        flush: Self::Flush,
+        vm: &VirtualMachine,
+    ) -> PyResult<Self::Status>;
+
+    fn total_in(&mut self) -> usize;
+
+    fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef;
+}
+
+pub trait CompressFlushKind: Copy {
+    const NONE: Self;
+    const FINISH: Self;
+
+    fn to_usize(self) -> usize;
+}
+
+pub trait CompressStatusKind: Copy {
+    const OK: Self;
+    const EOF: Self;
+
+    fn to_usize(self) -> usize;
+}
+
+#[derive(Debug)]
+pub struct CompressState<C: Compressor> {
+    compressor: Option<C>,
+}
+
+impl<C: Compressor> CompressState<C> {
+    pub fn new(compressor: C) -> Self {
+        Self {
+            compressor: Some(compressor),
+        }
+    }
+
+    fn get_compressor(&mut self, vm: &VirtualMachine) -> PyResult<&mut C> {
+        self.compressor
+            .as_mut()
+            .ok_or_else(|| C::new_error(USE_AFTER_FINISH_ERR, vm))
+    }
+
+    pub fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+        let mut buf = Vec::new();
+        let compressor = self.get_compressor(vm)?;
+
+        for mut chunk in data.chunks(C::CHUNKSIZE) {
+            while !chunk.is_empty() {
+                buf.reserve(C::DEF_BUF_SIZE);
+                let prev_in = compressor.total_in();
+                compressor.compress_vec(chunk, &mut buf, C::Flush::NONE, vm)?;
+                let consumed = compressor.total_in() - prev_in;
+                chunk = &chunk[consumed..];
+            }
+        }
+
+        buf.shrink_to_fit();
+        Ok(buf)
+    }
+
+    pub fn flush(&mut self, mode: C::Flush, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
+        let mut buf = Vec::new();
+        let compressor = self.get_compressor(vm)?;
+
+        let status = loop {
+            if buf.len() == buf.capacity() {
+                buf.reserve(C::DEF_BUF_SIZE);
+            }
+            let status = compressor.compress_vec(&[], &mut buf, mode, vm)?;
+            if buf.len() != buf.capacity() {
+                break status;
+            }
+        };
+
+        if status.to_usize() == C::Status::EOF.to_usize() {
+            if mode.to_usize() == C::Flush::FINISH.to_usize() {
+                self.compressor = None;
+            } else {
+                return Err(C::new_error("unexpected eof", vm));
+            }
+        }
+
+        buf.shrink_to_fit();
+        Ok(buf)
+    }
+}
+
+pub trait Decompressor {
+    type Flush: DecompressFlushKind;
+    type Status: DecompressStatus;
+    type Error;
+
+    fn total_in(&self) -> u64;
+    fn decompress_vec(
+        &mut self,
+        input: &[u8],
+        output: &mut Vec<u8>,
+        flush: Self::Flush,
+    ) -> Result<Self::Status, Self::Error>;
+    fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> {
+        Err(err)
+    }
+}
+
+pub trait DecompressStatus {
+    fn is_stream_end(&self) -> bool;
+}
+
+pub trait DecompressFlushKind: Copy {
+    const SYNC: Self;
+}
+
+impl DecompressFlushKind for () {
+    const SYNC: Self = ();
+}
+
+pub fn flush_sync<T: DecompressFlushKind>(_final_chunk: bool) -> T {
+    T::SYNC
+}
+
+#[derive(Debug)]
+pub struct DecompressState<D> {
+    decompress: D,
+    unused_data: PyBytesRef,
+    input_buffer: Vec<u8>,
+    eof: bool,
+    needs_input: bool,
+}
+
+impl<D: Decompressor> DecompressState<D> {
+    pub fn new(decompress: D, vm: &VirtualMachine) -> Self {
+        Self {
+            decompress,
+            unused_data: vm.ctx.empty_bytes.clone(),
+            input_buffer: Vec::new(),
+            eof: false,
+            needs_input: true,
+        }
+    }
+
+    pub fn eof(&self) -> bool {
+        self.eof
+    }
+
+    pub fn unused_data(&self) -> PyBytesRef {
+        self.unused_data.clone()
+    }
+
+    pub fn needs_input(&self) -> bool {
+        self.needs_input
+    }
+
+    pub fn decompress(
+        &mut self,
+        data: &[u8],
+        max_length: Option<usize>,
+        bufsize: usize,
+        vm: &VirtualMachine,
+    ) -> Result<Vec<u8>, DecompressError<D::Error>> {
+        if self.eof {
+            return Err(DecompressError::Eof(EofError));
+        }
+
+        let input_buffer = &mut self.input_buffer;
+        let d = &mut self.decompress;
+
+        let mut chunks = Chunker::chain(input_buffer, data);
+
+        let prev_len = chunks.len();
+        let (ret, stream_end) =
+            match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) {
+                Ok((buf, stream_end)) => (Ok(buf), stream_end),
+                Err(err) => (Err(err), false),
+            };
+        let consumed = prev_len - chunks.len();
+
+        self.eof |= stream_end;
+
+        if self.eof {
+            self.needs_input = false;
+            if !chunks.is_empty() {
+                self.unused_data = vm.ctx.new_bytes(chunks.to_vec());
+            }
+        } else if chunks.is_empty() {
+            input_buffer.clear();
+            self.needs_input = true;
+        } else {
+            self.needs_input = false;
+            if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) {
+                input_buffer.clear();
+                input_buffer.extend_from_slice(&data[n_consumed_from_data..]);
+            } else {
+                input_buffer.drain(..consumed);
+                input_buffer.extend_from_slice(data);
+            }
+        }
+
+        ret.map_err(DecompressError::Decompress)
+    }
+}
+
+pub enum DecompressError<E> {
+    Decompress(E),
+    Eof(EofError),
+}
+
+impl<E> From<E> for DecompressError<E> {
+    fn from(err: E) -> Self {
+        Self::Decompress(err)
+    }
+}
+
+pub struct EofError;
+
+impl ToPyException for EofError {
+    fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {
+        vm.new_eof_error("End of stream already reached".to_owned())
+    }
+}
+
+#[derive(FromArgs)]
+pub struct DecompressArgs {
+    #[pyarg(positional)]
+    data: ArgBytesLike,
+    #[pyarg(any, optional)]
+    max_length: OptionalArg<ArgSize>,
+}
+
+impl DecompressArgs {
+    pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> {
+        self.data.borrow_buf()
+    }
+    pub fn raw_max_length(&self) -> Option<isize> {
+        self.max_length.into_option().map(|ArgSize { value }| value)
+    }
+    pub fn max_length(&self) -> Option<usize> {
+        self.max_length
+            .into_option()
+            .and_then(|ArgSize { value }| usize::try_from(value).ok())
+    }
+}

stdlib/src/compression/zlib.rs

@@ -1,11 +1,13 @@
 // spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker
 
-pub(crate) use zlib::{DecompressArgs, make_module};
+pub(crate) use zlib::make_module;
 
 #[pymodule]
 mod zlib {
-    use super::generic::{
-        DecompressError, DecompressState, DecompressStatus, Decompressor, FlushKind, flush_sync,
+    use crate::compression::{
+        _decompress_chunks, Chunker, CompressFlushKind, CompressState, CompressStatusKind,
+        Compressor, DecompressArgs, DecompressError, DecompressFlushKind, DecompressState,
+        DecompressStatus, Decompressor, USE_AFTER_FINISH_ERR, flush_sync,
     };
     use crate::vm::{
         PyObject, PyPayload, PyResult, VirtualMachine,
@@ -144,48 +146,6 @@ mod zlib {
         }
     }
 
-    #[derive(Clone)]
-    pub(crate) struct Chunker<'a> {
-        data1: &'a [u8],
-        data2: &'a [u8],
-    }
-    impl<'a> Chunker<'a> {
-        pub(crate) fn new(data: &'a [u8]) -> Self {
-            Self {
-                data1: data,
-                data2: &[],
-            }
-        }
-        pub(crate) fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self {
-            if data1.is_empty() {
-                Self {
-                    data1: data2,
-                    data2: &[],
-                }
-            } else {
-                Self { data1, data2 }
-            }
-        }
-        pub(crate) fn len(&self) -> usize {
-            self.data1.len() + self.data2.len()
-        }
-        pub(crate) fn is_empty(&self) -> bool {
-            self.data1.is_empty()
-        }
-        pub(crate) fn to_vec(&self) -> Vec<u8> {
-            [self.data1, self.data2].concat()
-        }
-        pub(crate) fn chunk(&self) -> &'a [u8] {
-            self.data1.get(..CHUNKSIZE).unwrap_or(self.data1)
-        }
-        pub(crate) fn advance(&mut self, consumed: usize) {
-            self.data1 = &self.data1[consumed..];
-            if self.data1.is_empty() {
-                self.data1 = std::mem::take(&mut self.data2);
-            }
-        }
-    }
-
     fn _decompress<D: Decompressor>(
         data: &[u8],
         d: &mut D,
@@ -197,60 +157,6 @@ mod zlib {
         _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush)
     }
 
-    pub(super) fn _decompress_chunks<D: Decompressor>(
-        data: &mut Chunker<'_>,
-        d: &mut D,
-        bufsize: usize,
-        max_length: Option<usize>,
-        calc_flush: impl Fn(bool) -> D::Flush,
-    ) -> Result<(Vec<u8>, bool), D::Error> {
-        if data.is_empty() {
-            return Ok((Vec::new(), true));
-        }
-        let max_length = max_length.unwrap_or(usize::MAX);
-        let mut buf = Vec::new();
-
-        'outer: loop {
-            let chunk = data.chunk();
-            let flush = calc_flush(chunk.len() == data.len());
-            loop {
-                let additional = std::cmp::min(bufsize, max_length - buf.capacity());
-                if additional == 0 {
-                    return Ok((buf, false));
-                }
-                buf.reserve_exact(additional);
-
-                let prev_in = d.total_in();
-                let res = d.decompress_vec(chunk, &mut buf, flush);
-                let consumed = d.total_in() - prev_in;
-
-                data.advance(consumed as usize);
-
-                match res {
-                    Ok(status) => {
-                        let stream_end = status.is_stream_end();
-                        if stream_end || data.is_empty() {
-                            // we've reached the end of the stream, we're done
-                            buf.shrink_to_fit();
-                            return Ok((buf, stream_end));
-                        } else if !chunk.is_empty() && consumed == 0 {
-                            // we're gonna need a bigger buffer
-                            continue;
-                        } else {
-                            // next chunk
-                            continue 'outer;
-                        }
-                    }
-                    Err(e) => {
-                        d.maybe_set_dict(e)?;
-                        // now try the next chunk
-                        continue 'outer;
-                    }
-                };
-            }
-        }
-    }
-
     #[derive(FromArgs)]
     struct PyFuncDecompressArgs {
         #[pyarg(positional)]
@@ -393,11 +299,10 @@ mod zlib {
 
         #[pymethod]
         fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let max_length: usize = args
-                .max_length
-                .map_or(0, |x| x.value)
-                .try_into()
-                .map_err(|_| vm.new_value_error("must be non-negative".to_owned()))?;
+            let max_length: usize =
+                args.raw_max_length().unwrap_or(0).try_into().map_err(|_| {
+                    vm.new_value_error("max_length must be non-negative".to_owned())
+                })?;
             let max_length = (max_length != 0).then_some(max_length);
             let data = &*args.data();
 
@@ -434,25 +339,6 @@ mod zlib {
         }
     }
 
-    #[derive(FromArgs)]
-    pub(crate) struct DecompressArgs {
-        #[pyarg(positional)]
-        data: ArgBytesLike,
-        #[pyarg(any, optional)]
-        max_length: OptionalArg<ArgSize>,
-    }
-
-    impl DecompressArgs {
-        pub(crate) fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> {
-            self.data.borrow_buf()
-        }
-        pub(crate) fn max_length(&self) -> Option<usize> {
-            self.max_length
-                .into_option()
-                .and_then(|ArgSize { value }| usize::try_from(value).ok())
-        }
-    }
-
     #[derive(FromArgs)]
     #[allow(dead_code)] // FIXME: use args
     struct CompressobjArgs {
@@ -487,20 +373,20 @@ mod zlib {
             zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap());
         }
         Ok(PyCompress {
-            inner: PyMutex::new(CompressInner::new(compress)),
+            inner: PyMutex::new(CompressState::new(CompressInner::new(compress))),
         })
     }
 
     #[derive(Debug)]
     struct CompressInner {
-        compress: Option<Compress>,
+        compress: Compress,
     }
 
     #[pyattr]
     #[pyclass(name = "Compress")]
     #[derive(Debug, PyPayload)]
     struct PyCompress {
-        inner: PyMutex<CompressInner>,
+        inner: PyMutex<CompressState<CompressInner>>,
     }
 
     #[pyclass]
@@ -537,60 +423,52 @@ mod zlib {
 
     impl CompressInner {
         fn new(compress: Compress) -> Self {
-            Self {
-                compress: Some(compress),
-            }
+            Self { compress }
         }
+    }
 
-        fn get_compress(&mut self, vm: &VirtualMachine) -> PyResult<&mut Compress> {
+    impl CompressStatusKind for Status {
+        const OK: Self = Status::Ok;
+        const EOF: Self = Status::StreamEnd;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl CompressFlushKind for FlushCompress {
+        const NONE: Self = FlushCompress::None;
+        const FINISH: Self = FlushCompress::Finish;
+
+        fn to_usize(self) -> usize {
+            self as usize
+        }
+    }
+
+    impl Compressor for CompressInner {
+        type Status = Status;
+        type Flush = FlushCompress;
+        const CHUNKSIZE: usize = CHUNKSIZE;
+        const DEF_BUF_SIZE: usize = DEF_BUF_SIZE;
+
+        fn compress_vec(
+            &mut self,
+            input: &[u8],
+            output: &mut Vec<u8>,
+            flush: Self::Flush,
+            vm: &VirtualMachine,
+        ) -> PyResult<Self::Status> {
             self.compress
-                .as_mut()
-                .ok_or_else(|| new_zlib_error(USE_AFTER_FINISH_ERR, vm))
+                .compress_vec(input, output, flush)
+                .map_err(|_| new_zlib_error("error while compressing", vm))
         }
 
-        fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let c = self.get_compress(vm)?;
-            let mut buf = Vec::new();
-
-            for mut chunk in data.chunks(CHUNKSIZE) {
-                while !chunk.is_empty() {
-                    buf.reserve(DEF_BUF_SIZE);
-                    let prev_in = c.total_in();
-                    c.compress_vec(chunk, &mut buf, FlushCompress::None)
-                        .map_err(|_| new_zlib_error("error while compressing", vm))?;
-                    let consumed = c.total_in() - prev_in;
-                    chunk = &chunk[consumed as usize..];
-                }
-            }
-
-            buf.shrink_to_fit();
-            Ok(buf)
+        fn total_in(&mut self) -> usize {
+            self.compress.total_in() as usize
         }
 
-        fn flush(&mut self, mode: FlushCompress, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
-            let c = self.get_compress(vm)?;
-            let mut buf = Vec::new();
-
-            let status = loop {
-                if buf.len() == buf.capacity() {
-                    buf.reserve(DEF_BUF_SIZE);
-                }
-                let status = c
-                    .compress_vec(&[], &mut buf, mode)
-                    .map_err(|_| new_zlib_error("error while compressing", vm))?;
-                if buf.len() != buf.capacity() {
-                    break status;
-                }
-            };
-
-            match status {
-                Status::Ok | Status::BufError => {}
-                Status::StreamEnd if mode == FlushCompress::Finish => self.compress = None,
-                Status::StreamEnd => return Err(new_zlib_error("unexpected eof", vm)),
-            }
-
-            buf.shrink_to_fit();
-            Ok(buf)
+        fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef {
+            new_zlib_error(message, vm)
         }
     }
 
@@ -598,8 +476,6 @@ mod zlib {
         vm.new_exception_msg(vm.class("zlib", "error"), message.into())
     }
 
-    const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state";
-
     struct Level(Option<flate2::Compression>);
 
     impl Level {
@@ -647,7 +523,7 @@ mod zlib {
         }
     }
 
-    impl FlushKind for FlushDecompress {
+    impl DecompressFlushKind for FlushDecompress {
         const SYNC: Self = FlushDecompress::Sync;
     }
 
@@ -754,147 +630,3 @@ mod zlib {
         // }
     }
 }
-
-mod generic {
-    use super::zlib::{_decompress_chunks, Chunker};
-    use crate::vm::{
-        VirtualMachine,
-        builtins::{PyBaseExceptionRef, PyBytesRef},
-        convert::ToPyException,
-    };
-
-    pub(crate) trait Decompressor {
-        type Flush: FlushKind;
-        type Status: DecompressStatus;
-        type Error;
-
-        fn total_in(&self) -> u64;
-        fn decompress_vec(
-            &mut self,
-            input: &[u8],
-            output: &mut Vec<u8>,
-            flush: Self::Flush,
-        ) -> Result<Self::Status, Self::Error>;
-        fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> {
-            Err(err)
-        }
-    }
-
-    pub(crate) trait DecompressStatus {
-        fn is_stream_end(&self) -> bool;
-    }
-
-    pub(crate) trait FlushKind: Copy {
-        const SYNC: Self;
-    }
-
-    impl FlushKind for () {
-        const SYNC: Self = ();
-    }
-
-    pub(super) fn flush_sync<T: FlushKind>(_final_chunk: bool) -> T {
-        T::SYNC
-    }
-
-    #[derive(Debug)]
-    pub(crate) struct DecompressState<D> {
-        decompress: D,
-        unused_data: PyBytesRef,
-        input_buffer: Vec<u8>,
-        eof: bool,
-        needs_input: bool,
-    }
-
-    impl<D: Decompressor> DecompressState<D> {
-        pub(crate) fn new(decompress: D, vm: &VirtualMachine) -> Self {
-            Self {
-                decompress,
-                unused_data: vm.ctx.empty_bytes.clone(),
-                input_buffer: Vec::new(),
-                eof: false,
-                needs_input: true,
-            }
-        }
-
-        pub(crate) fn eof(&self) -> bool {
-            self.eof
-        }
-
-        pub(crate) fn unused_data(&self) -> PyBytesRef {
-            self.unused_data.clone()
-        }
-
-        pub(crate) fn needs_input(&self) -> bool {
-            self.needs_input
-        }
-
-        pub(crate) fn decompress(
-            &mut self,
-            data: &[u8],
-            max_length: Option<usize>,
-            bufsize: usize,
-            vm: &VirtualMachine,
-        ) -> Result<Vec<u8>, DecompressError<D::Error>> {
-            if self.eof {
-                return Err(DecompressError::Eof(EofError));
-            }
-
-            let input_buffer = &mut self.input_buffer;
-            let d = &mut self.decompress;
-
-            let mut chunks = Chunker::chain(input_buffer, data);
-
-            let prev_len = chunks.len();
-            let (ret, stream_end) =
-                match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) {
-                    Ok((buf, stream_end)) => (Ok(buf), stream_end),
-                    Err(err) => (Err(err), false),
-                };
-            let consumed = prev_len - chunks.len();
-
-            self.eof |= stream_end;
-
-            if self.eof {
-                self.needs_input = false;
-                if !chunks.is_empty() {
-                    self.unused_data = vm.ctx.new_bytes(chunks.to_vec());
-                }
-            } else if chunks.is_empty() {
-                input_buffer.clear();
-                self.needs_input = true;
-            } else {
-                self.needs_input = false;
-                if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) {
-                    input_buffer.clear();
-                    input_buffer.extend_from_slice(&data[n_consumed_from_data..]);
-                } else {
-                    input_buffer.drain(..consumed);
-                    input_buffer.extend_from_slice(data);
-                }
-            }
-
-            ret.map_err(DecompressError::Decompress)
-        }
-    }
-
-    pub(crate) enum DecompressError<E> {
-        Decompress(E),
-        Eof(EofError),
-    }
-
-    impl<E> From<E> for DecompressError<E> {
-        fn from(err: E) -> Self {
-            Self::Decompress(err)
-        }
-    }
-
-    pub(crate) struct EofError;
-
-    impl ToPyException for EofError {
-        fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {
-            vm.new_eof_error("End of stream already reached".to_owned())
-        }
-    }
-}
-
-pub(crate) use generic::{DecompressError, DecompressState, DecompressStatus, Decompressor};

stdlib/src/lib.rs

@@ -11,6 +11,7 @@ pub mod array;
 mod binascii;
 mod bisect;
 mod cmath;
+mod compression;
 mod contextvars;
 mod csv;
 mod dis;
@@ -37,13 +38,11 @@ mod statistics;
 mod suggestions;
 // TODO: maybe make this an extension module, if we ever get those
 // mod re;
-mod bz2;
 #[cfg(not(target_arch = "wasm32"))]
 pub mod socket;
 #[cfg(all(unix, not(target_os = "redox")))]
 mod syslog;
 mod unicodedata;
-mod zlib;
 
 mod faulthandler;
 #[cfg(any(unix, target_os = "wasi"))]
@@ -83,7 +82,6 @@ mod uuid;
 
 #[cfg(feature = "tkinter")]
 mod tkinter;
-mod lzma;
 
 use rustpython_common as common;
 use rustpython_vm as vm;
@@ -113,7 +111,7 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "array" => array::make_module,
             "binascii" => binascii::make_module,
             "_bisect" => bisect::make_module,
-            "_bz2" => bz2::make_module,
+            "_bz2" => compression::bz2::make_module,
             "cmath" => cmath::make_module,
             "_contextvars" => contextvars::make_module,
             "_csv" => csv::make_module,
@@ -121,7 +119,6 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "faulthandler" => faulthandler::make_module,
             "gc" => gc::make_module,
             "_hashlib" => hashlib::make_module,
-            "_lzma" => lzma::make_module,
             "_sha1" => sha1::make_module,
             "_sha3" => sha3::make_module,
             "_sha256" => sha256::make_module,
@@ -135,7 +132,7 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_statistics" => statistics::make_module,
             "_struct" => pystruct::make_module,
             "unicodedata" => unicodedata::make_module,
-            "zlib" => zlib::make_module,
+            "zlib" => compression::zlib::make_module,
             "_statistics" => statistics::make_module,
             "_suggestions" => suggestions::make_module,
             // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module,
@@ -153,6 +150,10 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_multiprocessing" => multiprocessing::make_module,
             "_socket" => socket::make_module,
         }
+        #[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
+        {
+            "_lzma" => compression::lzma::make_module,
+        }
         #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))]
         {
             "_sqlite3" => sqlite::make_module,

stdlib/src/lzma.rs

@@ -1,6 +0,0 @@
-pub(crate) use _lzma::make_module;
-
-#[pymodule]
-mod _lzma {
-
-}