Update gzip,test_gzip from CPython 3.13

This commit is contained in:
CPython Developers authored and Noa committed 2025-02-26 00:10:24 -06:00
1 parent 8c5602f2fb
commit aba3d5c082
2 files changed
+325 -75

No files matched your search

Vendored
+68 -62
View File
@@ -15,12 +15,16 @@ __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
READ, WRITE = 1, 2
READ = 'rb'
WRITE = 'wb'
_COMPRESS_LEVEL_FAST = 1
_COMPRESS_LEVEL_TRADEOFF = 6
_COMPRESS_LEVEL_BEST = 9
READ_BUFFER_SIZE = 128 * 1024
_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE
def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
encoding=None, errors=None, newline=None):
@@ -118,6 +122,21 @@ class BadGzipFile(OSError):
"""Exception raised in some cases for invalid gzip files."""
class _WriteBufferStream(io.RawIOBase):
"""Minimal object to pass WriteBuffer flushes into GzipFile"""
def __init__(self, gzip_file):
self.gzip_file = gzip_file
def write(self, data):
return self.gzip_file._write_raw(data)
def seekable(self):
return False
def writable(self):
return True
class GzipFile(_compression.BaseStream):
"""The GzipFile class simulates most of the methods of a file object with
the exception of the truncate() method.
@@ -160,9 +179,10 @@ class GzipFile(_compression.BaseStream):
and 9 is slowest and produces the most compression. 0 is no compression
at all. The default is 9.
The mtime argument is an optional numeric timestamp to be written
to the last modification time field in the stream when compressing.
If omitted or None, the current time is used.
The optional mtime argument is the timestamp requested by gzip. The time
is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970.
If mtime is omitted or None, the current time is used. Use mtime = 0
to generate a compressed stream that does not depend on creation time.
"""
@@ -182,6 +202,7 @@ class GzipFile(_compression.BaseStream):
if mode is None:
mode = getattr(fileobj, 'mode', 'rb')
if mode.startswith('r'):
self.mode = READ
raw = _GzipReader(fileobj)
@@ -204,6 +225,9 @@ class GzipFile(_compression.BaseStream):
zlib.DEF_MEM_LEVEL,
0)
self._write_mtime = mtime
self._buffer_size = _WRITE_BUFFER_SIZE
self._buffer = io.BufferedWriter(_WriteBufferStream(self),
buffer_size=self._buffer_size)
else:
raise ValueError("Invalid mode: {!r}".format(mode))
@@ -212,14 +236,6 @@ class GzipFile(_compression.BaseStream):
if self.mode == WRITE:
self._write_gzip_header(compresslevel)
@property
def filename(self):
import warnings
warnings.warn("use the name attribute", DeprecationWarning, 2)
if self.mode == WRITE and self.name[-3:] != ".gz":
return self.name + ".gz"
return self.name
@property
def mtime(self):
"""Last modification time read from stream, or None"""
@@ -237,6 +253,11 @@ class GzipFile(_compression.BaseStream):
self.bufsize = 0
self.offset = 0 # Current file offset for seek(), tell(), etc
def tell(self):
self._check_not_closed()
self._buffer.flush()
return super().tell()
def _write_gzip_header(self, compresslevel):
self.fileobj.write(b'\037\213') # magic header
self.fileobj.write(b'\010') # compression method
@@ -278,6 +299,10 @@ class GzipFile(_compression.BaseStream):
if self.fileobj is None:
raise ValueError("write() on closed GzipFile object")
return self._buffer.write(data)
def _write_raw(self, data):
# Called by our self._buffer underlying WriteBufferStream.
if isinstance(data, (bytes, bytearray)):
length = len(data)
else:
@@ -326,11 +351,11 @@ class GzipFile(_compression.BaseStream):
def close(self):
fileobj = self.fileobj
if fileobj is None:
if fileobj is None or self._buffer.closed:
return
self.fileobj = None
try:
if self.mode == WRITE:
self._buffer.flush()
fileobj.write(self.compress.flush())
write32u(fileobj, self.crc)
# self.size may exceed 2 GiB, or even 4 GiB
@@ -338,6 +363,7 @@ class GzipFile(_compression.BaseStream):
elif self.mode == READ:
self._buffer.close()
finally:
self.fileobj = None
myfileobj = self.myfileobj
if myfileobj:
self.myfileobj = None
@@ -346,6 +372,7 @@ class GzipFile(_compression.BaseStream):
def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
self._check_not_closed()
if self.mode == WRITE:
self._buffer.flush()
# Ensure the compressor's buffer is flushed
self.fileobj.write(self.compress.flush(zlib_mode))
self.fileobj.flush()
@@ -376,6 +403,9 @@ class GzipFile(_compression.BaseStream):
def seek(self, offset, whence=io.SEEK_SET):
if self.mode == WRITE:
self._check_not_closed()
# Flush buffer to ensure validity of self.offset
self._buffer.flush()
if whence != io.SEEK_SET:
if whence == io.SEEK_CUR:
offset = self.offset + offset
@@ -384,10 +414,10 @@ class GzipFile(_compression.BaseStream):
if offset < self.offset:
raise OSError('Negative seek in write mode')
count = offset - self.offset
chunk = b'\0' * 1024
for i in range(count // 1024):
chunk = b'\0' * self._buffer_size
for i in range(count // self._buffer_size):
self.write(chunk)
self.write(b'\0' * (count % 1024))
self.write(b'\0' * (count % self._buffer_size))
elif self.mode == READ:
self._check_not_closed()
return self._buffer.seek(offset, whence)
@@ -454,7 +484,7 @@ def _read_gzip_header(fp):
class _GzipReader(_compression.DecompressReader):
def __init__(self, fp):
super().__init__(_PaddedFile(fp), zlib.decompressobj,
super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor,
wbits=-zlib.MAX_WBITS)
# Set flag indicating start of a new member
self._new_member = True
@@ -502,12 +532,13 @@ class _GzipReader(_compression.DecompressReader):
self._new_member = False
# Read a chunk of data from the file
buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
if self._decompressor.needs_input:
buf = self._fp.read(READ_BUFFER_SIZE)
uncompress = self._decompressor.decompress(buf, size)
else:
uncompress = self._decompressor.decompress(b"", size)
uncompress = self._decompressor.decompress(buf, size)
if self._decompressor.unconsumed_tail != b"":
self._fp.prepend(self._decompressor.unconsumed_tail)
elif self._decompressor.unused_data != b"":
if self._decompressor.unused_data != b"":
# Prepend the already read bytes to the fileobj so they can
# be seen by _read_eof() and _read_gzip_header()
self._fp.prepend(self._decompressor.unused_data)
@@ -518,14 +549,11 @@ class _GzipReader(_compression.DecompressReader):
raise EOFError("Compressed file ended before the "
"end-of-stream marker was reached")
self._add_read_data( uncompress )
self._crc = zlib.crc32(uncompress, self._crc)
self._stream_size += len(uncompress)
self._pos += len(uncompress)
return uncompress
def _add_read_data(self, data):
self._crc = zlib.crc32(data, self._crc)
self._stream_size = self._stream_size + len(data)
def _read_eof(self):
# We've read to the end of the file
# We check that the computed CRC and size of the
@@ -552,43 +580,21 @@ class _GzipReader(_compression.DecompressReader):
self._new_member = True
def _create_simple_gzip_header(compresslevel: int,
mtime = None) -> bytes:
"""
Write a simple gzip header with no extra fields.
:param compresslevel: Compresslevel used to determine the xfl bytes.
:param mtime: The mtime (must support conversion to a 32-bit integer).
:return: A bytes object representing the gzip header.
"""
if mtime is None:
mtime = time.time()
if compresslevel == _COMPRESS_LEVEL_BEST:
xfl = 2
elif compresslevel == _COMPRESS_LEVEL_FAST:
xfl = 4
else:
xfl = 0
# Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
# fields added to header), mtime, xfl and os (255 for unknown OS).
return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0):
"""Compress data in one shot and return the compressed string.
compresslevel sets the compression level in range of 0-9.
mtime can be used to set the modification time. The modification time is
set to the current time by default.
mtime can be used to set the modification time.
The modification time is set to 0 by default, for reproducibility.
"""
if mtime == 0:
# Use zlib as it creates the header with 0 mtime by default.
# This is faster and with less overhead.
return zlib.compress(data, level=compresslevel, wbits=31)
header = _create_simple_gzip_header(compresslevel, mtime)
trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
# Wbits=-15 creates a raw deflate block.
return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
trailer)
# Wbits=31 automatically includes a gzip header and trailer.
gzip_data = zlib.compress(data, level=compresslevel, wbits=31)
if mtime is None:
mtime = time.time()
# Reuse gzip header created by zlib, replace mtime and OS byte for
# consistency.
header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255)
return header + gzip_data[10:]
def decompress(data):
@@ -655,7 +661,7 @@ def main():
f = builtins.open(arg, "rb")
g = open(arg + ".gz", "wb")
while True:
chunk = f.read(io.DEFAULT_BUFFER_SIZE)
chunk = f.read(READ_BUFFER_SIZE)
if not chunk:
break
g.write(chunk)
+257 -13
View File
@@ -5,7 +5,6 @@ import array
import functools
import io
import os
import pathlib
import struct
import sys
import unittest
@@ -16,6 +15,7 @@ from test.support import _4G, bigmemtest, requires_subprocess
from test.support.script_helper import assert_python_ok, assert_python_failure
gzip = import_helper.import_module('gzip')
zlib = import_helper.import_module('zlib')
data1 = b""" int length=DEFAULTALLOC, err = Z_OK;
PyObject *RetVal;
@@ -78,16 +78,18 @@ class TestGzip(BaseTest):
f.close()
def test_write_read_with_pathlike_file(self):
filename = pathlib.Path(self.filename)
filename = os_helper.FakePath(self.filename)
with gzip.GzipFile(filename, 'w') as f:
f.write(data1 * 50)
self.assertIsInstance(f.name, str)
self.assertEqual(f.name, self.filename)
with gzip.GzipFile(filename, 'a') as f:
f.write(data1)
with gzip.GzipFile(filename) as f:
d = f.read()
self.assertEqual(d, data1 * 51)
self.assertIsInstance(f.name, str)
self.assertEqual(f.name, self.filename)
# The following test_write_xy methods test that write accepts
# the corresponding bytes-like object type as input
@@ -471,15 +473,122 @@ class TestGzip(BaseTest):
with io.TextIOWrapper(f, encoding="ascii") as t:
self.assertEqual(t.readlines(), lines)
def test_fileobj_with_name(self):
with open(self.filename, "xb") as raw:
with gzip.GzipFile(fileobj=raw, mode="x") as f:
f.write(b'one')
self.assertEqual(f.name, raw.name)
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, raw.name)
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
with open(self.filename, "wb") as raw:
with gzip.GzipFile(fileobj=raw, mode="w") as f:
f.write(b'two')
self.assertEqual(f.name, raw.name)
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, raw.name)
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
with open(self.filename, "ab") as raw:
with gzip.GzipFile(fileobj=raw, mode="a") as f:
f.write(b'three')
self.assertEqual(f.name, raw.name)
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, raw.name)
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
with open(self.filename, "rb") as raw:
with gzip.GzipFile(fileobj=raw, mode="r") as f:
self.assertEqual(f.read(), b'twothree')
self.assertEqual(f.name, raw.name)
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.mode, gzip.READ)
self.assertIs(f.readable(), True)
self.assertIs(f.writable(), False)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, raw.name)
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.READ)
self.assertIs(f.readable(), True)
self.assertIs(f.writable(), False)
self.assertIs(f.seekable(), True)
def test_fileobj_from_fdopen(self):
# Issue #13781: Opening a GzipFile for writing fails when using a
# fileobj created with os.fdopen().
fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT)
with os.fdopen(fd, "wb") as f:
with gzip.GzipFile(fileobj=f, mode="w") as g:
pass
fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
with os.fdopen(fd, "xb") as raw:
with gzip.GzipFile(fileobj=raw, mode="x") as f:
f.write(b'one')
self.assertEqual(f.name, '')
self.assertEqual(f.fileno(), raw.fileno())
self.assertIs(f.closed, True)
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
with os.fdopen(fd, "wb") as raw:
with gzip.GzipFile(fileobj=raw, mode="w") as f:
f.write(b'two')
self.assertEqual(f.name, '')
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
with os.fdopen(fd, "ab") as raw:
with gzip.GzipFile(fileobj=raw, mode="a") as f:
f.write(b'three')
self.assertEqual(f.name, '')
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
fd = os.open(self.filename, os.O_RDONLY)
with os.fdopen(fd, "rb") as raw:
with gzip.GzipFile(fileobj=raw, mode="r") as f:
self.assertEqual(f.read(), b'twothree')
self.assertEqual(f.name, '')
self.assertEqual(f.fileno(), raw.fileno())
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
def test_fileobj_mode(self):
self.assertEqual(gzip.READ, 'rb')
self.assertEqual(gzip.WRITE, 'wb')
gzip.GzipFile(self.filename, "wb").close()
with open(self.filename, "r+b") as f:
with gzip.GzipFile(fileobj=f, mode='r') as g:
@@ -507,17 +616,69 @@ class TestGzip(BaseTest):
def test_bytes_filename(self):
str_filename = self.filename
try:
bytes_filename = str_filename.encode("ascii")
except UnicodeEncodeError:
self.skipTest("Temporary file name needs to be ASCII")
bytes_filename = os.fsencode(str_filename)
with gzip.GzipFile(bytes_filename, "wb") as f:
f.write(data1 * 50)
self.assertEqual(f.name, bytes_filename)
with gzip.GzipFile(bytes_filename, "rb") as f:
self.assertEqual(f.read(), data1 * 50)
self.assertEqual(f.name, bytes_filename)
# Sanity check that we are actually operating on the right file.
with gzip.GzipFile(str_filename, "rb") as f:
self.assertEqual(f.read(), data1 * 50)
self.assertEqual(f.name, str_filename)
def test_fileobj_without_name(self):
bio = io.BytesIO()
with gzip.GzipFile(fileobj=bio, mode='wb') as f:
f.write(data1 * 50)
self.assertEqual(f.name, '')
self.assertRaises(io.UnsupportedOperation, f.fileno)
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.WRITE)
self.assertIs(f.readable(), False)
self.assertIs(f.writable(), True)
self.assertIs(f.seekable(), True)
bio.seek(0)
with gzip.GzipFile(fileobj=bio, mode='rb') as f:
self.assertEqual(f.read(), data1 * 50)
self.assertEqual(f.name, '')
self.assertRaises(io.UnsupportedOperation, f.fileno)
self.assertEqual(f.mode, gzip.READ)
self.assertIs(f.readable(), True)
self.assertIs(f.writable(), False)
self.assertIs(f.seekable(), True)
self.assertIs(f.closed, False)
self.assertIs(f.closed, True)
self.assertEqual(f.name, '')
self.assertRaises(AttributeError, f.fileno)
self.assertEqual(f.mode, gzip.READ)
self.assertIs(f.readable(), True)
self.assertIs(f.writable(), False)
self.assertIs(f.seekable(), True)
def test_fileobj_and_filename(self):
filename2 = self.filename + 'new'
with (open(self.filename, 'wb') as fileobj,
gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f):
f.write(data1 * 50)
self.assertEqual(f.name, filename2)
with (open(self.filename, 'rb') as fileobj,
gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f):
self.assertEqual(f.read(), data1 * 50)
self.assertEqual(f.name, filename2)
# Sanity check that we are actually operating on the right file.
with gzip.GzipFile(self.filename, 'rb') as f:
self.assertEqual(f.read(), data1 * 50)
self.assertEqual(f.name, self.filename)
def test_decompress_limited(self):
"""Decompressed data buffering should be limited"""
@@ -552,8 +713,18 @@ class TestGzip(BaseTest):
f.read(1) # to set mtime attribute
self.assertEqual(f.mtime, mtime)
def test_compress_mtime_default(self):
# test for gh-125260
datac = gzip.compress(data1, mtime=0)
datac2 = gzip.compress(data1)
self.assertEqual(datac, datac2)
datac3 = gzip.compress(data1, mtime=None)
self.assertNotEqual(datac, datac3)
with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f:
f.read(1) # to set mtime attribute
self.assertGreater(f.mtime, 1)
def test_compress_correct_level(self):
# gzip.compress calls with mtime == 0 take a different code path.
for mtime in (0, 42):
with self.subTest(mtime=mtime):
nocompress = gzip.compress(data1, compresslevel=0, mtime=mtime)
@@ -561,6 +732,17 @@ class TestGzip(BaseTest):
self.assertIn(data1, nocompress)
self.assertNotIn(data1, yescompress)
def test_issue112346(self):
# The OS byte should be 255, this should not change between Python versions.
for mtime in (0, 42):
with self.subTest(mtime=mtime):
compress = gzip.compress(data1, compresslevel=1, mtime=mtime)
self.assertEqual(
struct.unpack("<IxB", compress[4:10]),
(mtime, 255),
"Gzip header does not properly set either mtime or OS byte."
)
def test_decompress(self):
for data in (data1, data2):
buf = io.BytesIO()
@@ -616,6 +798,66 @@ class TestGzip(BaseTest):
self.assertEqual(f.write(q), LENGTH)
self.assertEqual(f.tell(), LENGTH)
def test_flush_flushes_compressor(self):
# See issue GH-105808.
b = io.BytesIO()
message = b"important message here."
with gzip.GzipFile(fileobj=b, mode='w') as f:
f.write(message)
f.flush()
partial_data = b.getvalue()
full_data = b.getvalue()
self.assertEqual(gzip.decompress(full_data), message)
# The partial data should contain the gzip header and the complete
# message, but not the end-of-stream markers (so we can't just
# decompress it directly).
with self.assertRaises(EOFError):
gzip.decompress(partial_data)
d = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
f = io.BytesIO(partial_data)
gzip._read_gzip_header(f)
read_message = d.decompress(f.read())
self.assertEqual(read_message, message)
def test_flush_modes(self):
# Make sure the argument to flush is properly passed to the
# zlib.compressobj; see issue GH-105808.
class FakeCompressor:
def __init__(self):
self.modes = []
def compress(self, data):
return b''
def flush(self, mode=-1):
self.modes.append(mode)
return b''
b = io.BytesIO()
fc = FakeCompressor()
with gzip.GzipFile(fileobj=b, mode='w') as f:
f.compress = fc
f.flush()
f.flush(50)
f.flush(zlib_mode=100)
# The implicit close will also flush the compressor.
expected_modes = [
zlib.Z_SYNC_FLUSH,
50,
100,
-1,
]
self.assertEqual(fc.modes, expected_modes)
def test_write_seek_write(self):
# Make sure that offset is up-to-date before seeking
# See issue GH-108111
b = io.BytesIO()
message = b"important message here."
with gzip.GzipFile(fileobj=b, mode='w') as f:
f.write(message)
f.seek(len(message))
f.write(message)
data = b.getvalue()
self.assertEqual(gzip.decompress(data), message * 2)
class TestOpen(BaseTest):
def test_binary_modes(self):
@@ -646,13 +888,16 @@ class TestOpen(BaseTest):
self.assertEqual(file_data, uncompressed)
def test_pathlike_file(self):
filename = pathlib.Path(self.filename)
filename = os_helper.FakePath(self.filename)
with gzip.open(filename, "wb") as f:
f.write(data1 * 50)
self.assertEqual(f.name, self.filename)
with gzip.open(filename, "ab") as f:
f.write(data1)
self.assertEqual(f.name, self.filename)
with gzip.open(filename) as f:
self.assertEqual(f.read(), data1 * 51)
self.assertEqual(f.name, self.filename)
def test_implicit_binary_modes(self):
# Test implicit binary modes (no "b" or "t" in mode string).
@@ -682,7 +927,6 @@ class TestOpen(BaseTest):
file_data = gzip.decompress(f.read())
self.assertEqual(file_data, uncompressed)
@unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON, windows text mode")
def test_text_modes(self):
uncompressed = data1.decode("ascii") * 50
uncompressed_raw = uncompressed.replace("\n", os.linesep)