diff --git a/Lib/base64.py b/Lib/base64.py index 0f84420d7..fa562f74a 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -4,7 +4,6 @@ # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import struct import binascii @@ -26,6 +25,8 @@ __all__ = [ ] +_NOT_SPECIFIED = ['NOT SPECIFIED'] + bytes_types = (bytes, bytearray) # Types acceptable as binary data def _bytes_from_decode_data(s): @@ -45,44 +46,90 @@ def _bytes_from_decode_data(s): # Base64 encoding/decoding uses binascii -def b64encode(s, altchars=None): +def b64encode(s, altchars=None, *, padded=True, wrapcol=0): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. + + If padded is false, omit padding in the output. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: - assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) - return encoded + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False, + alphabet=alphabet) + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False) -def b64decode(s, altchars=None, validate=False): +def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. + If padded is false, padding in input is not required. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. - If validate is False (the default), characters that are neither in the - normal base-64 alphabet nor the alternative alphabet are discarded prior - to the padding check. If validate is True, these non-alphabet characters - in the input result in a binascii.Error. + If ignorechars is specified, it should be a byte string containing + characters to ignore from the input. The default value of validate is + True if ignorechars is specified, False otherwise. + + If validate is false, characters that are neither in the normal base-64 + alphabet nor the alternative alphabet are discarded prior to the + padding check. If validate is true, these non-alphabet characters in + the input result in a binascii.Error if they are not in ignorechars. For more information about the strict base64 check, see: https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64 """ s = _bytes_from_decode_data(s) + if validate is _NOT_SPECIFIED: + validate = ignorechars is not _NOT_SPECIFIED + badchar = None if altchars is not None: altchars = _bytes_from_decode_data(altchars) - assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) - return binascii.a2b_base64(s, strict_mode=validate) + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + if ignorechars is _NOT_SPECIFIED: + for b in b'+/': + if b not in altchars and b in s: + badchar = b + break + s = s.translate(bytes.maketrans(altchars, b'+/')) + else: + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.a2b_base64(s, strict_mode=validate, + alphabet=alphabet, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if ignorechars is _NOT_SPECIFIED: + ignorechars = b'' + result = binascii.a2b_base64(s, strict_mode=validate, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if badchar is not None: + import warnings + if validate: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=True ' + f'will be an error in future Python versions', + DeprecationWarning, stacklevel=2) + else: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=False ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result def standard_b64encode(s): @@ -103,19 +150,21 @@ def standard_b64decode(s): return b64decode(s) -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') -def urlsafe_b64encode(s): +def urlsafe_b64encode(s, *, padded=True): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. - """ - return b64encode(s).translate(_urlsafe_encode_translation) -def urlsafe_b64decode(s): + If padded is false, omit padding in the output. + """ + return binascii.b2a_base64(s, padded=padded, newline=False, + alphabet=binascii.URLSAFE_BASE64_ALPHABET) + +def urlsafe_b64decode(s, *, padded=False): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result @@ -124,23 +173,46 @@ def urlsafe_b64decode(s): alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. + If padded is false, padding in input is not required. + The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) + badchar = None + for b in b'+/': + if b in s: + badchar = b + break s = s.translate(_urlsafe_decode_translation) - return b64decode(s) + result = binascii.a2b_base64(s, strict_mode=False, padded=padded) + if badchar is not None: + import warnings + warnings.warn(f'invalid character {chr(badchar)!a} in URL-safe Base64 data ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result # Base32 encoding/decoding must be done in Python _B32_ENCODE_DOCSTRING = ''' Encode the bytes-like objects using {encoding} and return a bytes object. + +If padded is false, omit padding in the output. + +If wrapcol is non-zero, insert a newline (b'\\n') character after at most +every wrapcol characters. ''' _B32_DECODE_DOCSTRING = ''' Decode the {encoding} encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + +If padded is false, padding in input is not required. + +ignorechars should be a byte string containing characters to ignore +from the input. {extra_args} The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet @@ -155,108 +227,41 @@ mapped to (when map01 is not None, the digit 0 is always mapped to the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. ''' -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV' -_b32tab2 = {} -_b32rev = {} -def _b32encode(alphabet, s): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32tab2: - b32tab = [bytes((i,)) for i in alphabet] - _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab] - b32tab = None +def b32encode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) +b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + b'\0' * (5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2[alphabet] - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5]) # big endian - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) - # Adjust for any leftover partial quanta - if leftover == 1: - encoded[-6:] = b'======' - elif leftover == 2: - encoded[-4:] = b'====' - elif leftover == 3: - encoded[-3:] = b'===' - elif leftover == 4: - encoded[-1:] = b'=' - return bytes(encoded) - -def _b32decode(alphabet, s, casefold=False, map01=None): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32rev: - _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)} +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) - if len(s) % 8: - raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01 is not None: map01 = _bytes_from_decode_data(map01) - assert len(map01) == 1, repr(map01) s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) - # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev[alphabet] - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5) # big endian - # Process the last, partial quanta - if l % 8 or padchars not in {0, 1, 3, 4, 6}: - raise binascii.Error('Incorrect padding') - if padchars and decoded: - acc <<= 5 * padchars - last = acc.to_bytes(5) # big endian - leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 - decoded[-5:] = last[:leftover] - return bytes(decoded) - - -def b32encode(s): - return _b32encode(_b32alphabet, s) -b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - -def b32decode(s, casefold=False, map01=None): - return _b32decode(_b32alphabet, s, casefold, map01) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) -def b32hexencode(s): - return _b32encode(_b32hexalphabet, s) +def b32hexencode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol, + alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): + s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping - return _b32decode(_b32hexalphabet, s, casefold) + if casefold: + s = s.upper() + return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') @@ -264,62 +269,47 @@ b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', # RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. -def b16encode(s): +def b16encode(s, *, wrapcol=0): """Encode the bytes-like object s using Base16 and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - return binascii.hexlify(s).upper() + if not wrapcol: + return binascii.hexlify(s).upper() + if wrapcol < 0: + raise ValueError('Negative wrapcol') + if wrapcol < 2: + wrapcol = 2 + return binascii.hexlify(s, bytes_per_sep=-(wrapcol//2), sep=b'\n').upper() -def b16decode(s, casefold=False): +def b16decode(s, casefold=False, *, ignorechars=b''): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + ignorechars should be a byte string containing characters to ignore + from the input. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if s.translate(None, delete=b'0123456789ABCDEF'): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) + if not casefold: + s = _bytes_from_decode_data(s) + if not isinstance(ignorechars, bytes): + ignorechars = bytes(memoryview(ignorechars)) + for b in b'abcdef': + if b in s and b not in ignorechars: + raise binascii.Error('Non-base16 digit found') + s = s.translate(None, delete=b'abcdef') + return binascii.unhexlify(s, ignorechars=ignorechars) # # Ascii85 encoding/decoding # - -_a85chars = None -_a85chars2 = None -_A85START = b"<~" -_A85END = b"~>" - -def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): - # Helper function for a85encode and b85encode - if not isinstance(b, bytes_types): - b = memoryview(b).tobytes() - - padding = (-len(b)) % 4 - if padding: - b = b + b'\0' * padding - words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) - - chunks = [b'z' if foldnuls and not word else - b'y' if foldspaces and word == 0x20202020 else - (chars2[word // 614125] + - chars2[word // 85 % 7225] + - chars[word % 85]) - for word in words] - - if padding and not pad: - if chunks[-1] == b'z': - chunks[-1] = chars[0] * 5 - chunks[-1] = chunks[-1][:-padding] - - return b''.join(chunks) - def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): """Encode bytes-like object b using Ascii85 and return a bytes object. @@ -340,31 +330,11 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): with ~>, they must not use a leading <~. """ - global _a85chars, _a85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _a85chars2 is None: - _a85chars = [bytes((i,)) for i in range(33, 118)] - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] + return binascii.b2a_ascii85(b, foldspaces=foldspaces, + adobe=adobe, wrapcol=wrapcol, pad=pad) - result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) - - if adobe: - result = _A85START + result - if wrapcol: - wrapcol = max(2 if adobe else 1, wrapcol) - chunks = [result[i: i + wrapcol] - for i in range(0, len(result), wrapcol)] - if adobe: - if len(chunks[-1]) + 2 > wrapcol: - chunks.append(b'') - result = b'\n'.join(chunks) - if adobe: - result += _A85END - - return result - -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. foldspaces is a flag that specifies whether the 'y' short sequence @@ -380,156 +350,60 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - b = _bytes_from_decode_data(b) - if adobe: - if not b.endswith(_A85END): - raise ValueError( - "Ascii85 encoded byte sequences must end " - "with {!r}".format(_A85END) - ) - if b.startswith(_A85START): - b = b[2:-2] # Strip off start/end markers - else: - b = b[:-2] - # - # We have to go through this stepwise, so as to ignore spaces and handle - # special short sequences - # - packI = struct.Struct('!I').pack - decoded = [] - decoded_append = decoded.append - curr = [] - curr_append = curr.append - curr_clear = curr.clear - for x in b + b'u' * 4: - if b'!'[0] <= x <= b'u'[0]: - curr_append(x) - if len(curr) == 5: - acc = 0 - for x in curr: - acc = 85 * acc + (x - 33) - try: - decoded_append(packI(acc)) - except struct.error: - raise ValueError('Ascii85 overflow') from None - curr_clear() - elif x == b'z'[0]: - if curr: - raise ValueError('z inside Ascii85 5-tuple') - decoded_append(b'\0\0\0\0') - elif foldspaces and x == b'y'[0]: - if curr: - raise ValueError('y inside Ascii85 5-tuple') - decoded_append(b'\x20\x20\x20\x20') - elif x in ignorechars: - # Skip whitespace - continue - else: - raise ValueError('Non-Ascii85 digit found: %c' % x) + return binascii.a2b_ascii85(b, foldspaces=foldspaces, + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) - result = b''.join(decoded) - padding = 4 - len(curr) - if padding: - # Throw away the extra padding - result = result[:-padding] - return result - -# The following code is originally taken (with permission) from Mercurial - -_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") -_b85chars = None -_b85chars2 = None -_b85dec = None - -def b85encode(b, pad=False): +def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + The input is padded with b'\0' so its length is a multiple of 4 bytes before encoding. If pad is true, all the resulting characters are retained in the output, which will always be a multiple of 5 bytes. """ - global _b85chars, _b85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85chars2 is None: - _b85chars = [bytes((i,)) for i in _b85alphabet] - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - return _85encode(b, _b85chars, _b85chars2, pad) + return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) -def b85decode(b): +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - global _b85dec - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85dec is None: - # we don't assign to _b85dec directly to avoid issues when - # multiple threads call this function simultaneously - b85dec_tmp = [None] * 256 - for i, c in enumerate(_b85alphabet): - b85dec_tmp[c] = i - _b85dec = b85dec_tmp + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) - b = _bytes_from_decode_data(b) - padding = (-len(b)) % 5 - b = b + b'~' * padding - out = [] - packI = struct.Struct('!I').pack - for i in range(0, len(b), 5): - chunk = b[i:i + 5] - acc = 0 - try: - for c in chunk: - acc = acc * 85 + _b85dec[c] - except TypeError: - for j, c in enumerate(chunk): - if _b85dec[c] is None: - raise ValueError('bad base85 character at position %d' - % (i + j)) from None - raise - try: - out.append(packI(acc)) - except struct.error: - raise ValueError('base85 overflow in hunk starting at byte %d' - % i) from None +def z85encode(s, pad=False, *, wrapcol=0): + """Encode bytes-like object b in z85 format and return a bytes object. - result = b''.join(out) - if padding: - result = result[:-padding] - return result + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. -_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') -# Translating b85 valid but z85 invalid chars to b'\x00' is required -# to prevent them from being decoded as b85 valid chars. -_z85_b85_decode_diff = b';_`|~' -_z85_decode_translation = bytes.maketrans( - _z85alphabet + _z85_b85_decode_diff, - _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) -) -_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. + """ + return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, + alphabet=binascii.Z85_ALPHABET) -def z85encode(s): - """Encode bytes-like object b in z85 format and return a bytes object.""" - return b85encode(s).translate(_z85_encode_translation) - -def z85decode(s): +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - s = _bytes_from_decode_data(s) - s = s.translate(_z85_decode_translation) - try: - return b85decode(s) - except ValueError as e: - raise ValueError(e.args[0].replace('base85', 'z85')) from None + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it @@ -573,11 +447,10 @@ def encodebytes(s): """Encode a bytestring into a bytes object containing multiple lines of base-64 data.""" _input_type_check(s) - pieces = [] - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) + result = binascii.b2a_base64(s, wrapcol=MAXLINESIZE) + if result == b'\n': + return b'' + return result def decodebytes(s): diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py index 6ba5bb814..6fa013b55 100644 --- a/Lib/test/support/_hypothesis_stubs/__init__.py +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -24,7 +24,13 @@ def given(*_args, **_kwargs): @functools.wraps(f) def test_function(self): for example_args, example_kwargs in examples: - with self.subTest(*example_args, **example_kwargs): + if len(example_args) < 2: + subtest_args = example_args + else: + # subTest takes up to one positional argument. + # When there are more, display them as a tuple + subtest_args = [example_args] + with self.subTest(*subtest_args, **example_kwargs): f(self, *example_args, **example_kwargs) else: diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 9fe69f4f6..868abcfee 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -1,9 +1,11 @@ import unittest import base64 import binascii +import string +import sys import os from array import array -from test.support import cpython_only +from test.support import cpython_only, check_impl_detail from test.support import os_helper from test.support import script_helper from test.support.import_helper import ensure_lazy_imports @@ -14,6 +16,8 @@ class LazyImportTest(unittest.TestCase): def test_lazy_import(self): ensure_lazy_imports("base64", {"re", "getopt"}) +from test.support.hypothesis_helper import hypothesis + class LegacyBase64TestCase(unittest.TestCase): @@ -68,6 +72,13 @@ class LegacyBase64TestCase(unittest.TestCase): eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc') self.check_type_errors(base64.decodebytes) + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_bytes_encode_decode_round_trip(self, payload): + encoded = base64.encodebytes(payload) + decoded = base64.decodebytes(encoded) + self.assertEqual(payload, decoded) + def test_encode(self): eq = self.assertEqual from io import BytesIO, StringIO @@ -96,6 +107,19 @@ class LegacyBase64TestCase(unittest.TestCase): self.assertRaises(TypeError, base64.encode, BytesIO(b'YWJj\n'), StringIO()) self.assertRaises(TypeError, base64.encode, StringIO('YWJj\n'), StringIO()) + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_legacy_encode_decode_round_trip(self, payload): + from io import BytesIO + payload_file_r = BytesIO(payload) + encoded_file_w = BytesIO() + base64.encode(payload_file_r, encoded_file_w) + encoded_file_r = BytesIO(encoded_file_w.getvalue()) + decoded_file_w = BytesIO() + base64.decode(encoded_file_r, decoded_file_w) + decoded = decoded_file_w.getvalue() + self.assertEqual(payload, decoded) + class BaseXYTestCase(unittest.TestCase): @@ -149,6 +173,7 @@ class BaseXYTestCase(unittest.TestCase): b"YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") + # Test with arbitrary alternative characters eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'*$'), b'01a*b$cd') eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=bytearray(b'*$')), @@ -184,6 +209,55 @@ class BaseXYTestCase(unittest.TestCase): b'\xd3V\xbeo\xf7\x1d', b'01a-b_cd') self.check_encode_type_errors(base64.urlsafe_b64encode) + def test_b64encode_padded(self): + b64encode = base64.b64encode + self.assertEqual(b64encode(b'', padded=False), b'') + self.assertEqual(b64encode(b'a', padded=False), b'YQ') + self.assertEqual(b64encode(b'ab', padded=False), b'YWI') + self.assertEqual(b64encode(b'abc', padded=False), b'YWJj') + self.assertEqual(b64encode(b'\xfb', padded=False, altchars=b'-_'), b'-w') + self.assertEqual(b64encode(b'\xfb\xff', padded=False, altchars=b'-_'), + b'-_8') + self.assertEqual(b64encode(b'\xfb\xff\xbf', padded=False, altchars=b'-_'), + b'-_-_') + + urlsafe_b64encode = base64.urlsafe_b64encode + self.assertEqual(urlsafe_b64encode(b'', padded=False), b'') + self.assertEqual(urlsafe_b64encode(b'\xfb', padded=False), b'-w') + self.assertEqual(urlsafe_b64encode(b'\xfb\xff', padded=False), b'-_8') + self.assertEqual(urlsafe_b64encode(b'\xfb\xff\xbf', padded=False), + b'-_-_') + + def _common_test_wrapcol(self, func, data): + eq = self.assertEqual + expected = func(data) + eq(func(data, wrapcol=0), expected) + eq(func(data, wrapcol=80), expected) + eq(func(b'', wrapcol=0), func(b'')) + eq(func(b'', wrapcol=1), func(b'')) + eq(func(data, wrapcol=sys.maxsize), expected) + if check_impl_detail(): + eq(func(data, wrapcol=sys.maxsize*2), expected) + with self.assertRaises(OverflowError): + func(data, wrapcol=2**1000) + with self.assertRaises(ValueError): + func(data, wrapcol=-80) + with self.assertRaises(TypeError): + func(data, wrapcol=80.0) + with self.assertRaises(TypeError): + func(data, wrapcol='80') + if func is not base64.b16encode: + with self.assertRaises(TypeError): + func(data, wrapcol=None) + + def test_b64encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b64encode, b) + eq(base64.b64encode(b, wrapcol=8), b'd3d3LnB5\ndGhvbi5v\ncmc=') + eq(base64.b64encode(b, wrapcol=11), b'd3d3LnB5\ndGhvbi5v\ncmc=') + eq(base64.b64encode(b, wrapcol=1), b'd3d3\nLnB5\ndGhv\nbi5v\ncmc=') + def test_b64decode(self): eq = self.assertEqual @@ -241,29 +315,100 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b64decode(data_str, altchars=altchars), res) eq(base64.b64decode(data, altchars=altchars_str), res) eq(base64.b64decode(data_str, altchars=altchars_str), res) + eq(base64.b64decode(data, altchars=altchars, ignorechars=b'\n'), res) + + eq(base64.b64decode(b'/----', altchars=b'-+', ignorechars=b'/'), b'\xfb\xef\xbe') + eq(base64.b64decode(b'/----', altchars=b'+-', ignorechars=b'/'), b'\xff\xff\xff') + eq(base64.b64decode(b'+----', altchars=b'-/', ignorechars=b'+'), b'\xfb\xef\xbe') + eq(base64.b64decode(b'+----', altchars=b'/-', ignorechars=b'+'), b'\xff\xff\xff') + eq(base64.b64decode(b'+/+/', altchars=b'/+', ignorechars=b''), b'\xff\xef\xfe') + eq(base64.b64decode(b'/+/+', altchars=b'+/', ignorechars=b''), b'\xff\xef\xfe') + + self.assertRaises(ValueError, base64.b64decode, b'', altchars=b'+') + self.assertRaises(ValueError, base64.b64decode, b'', altchars=b'+/-') + self.assertRaises(ValueError, base64.b64decode, '', altchars='+') + self.assertRaises(ValueError, base64.b64decode, '', altchars='+/-') def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') + def test_b64decode_padded(self): + b64decode = base64.b64decode + urlsafe_b64decode = base64.urlsafe_b64decode + def check(data, expected, padded=0): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b64decode(data, padded=False, validate=True) + self.assertEqual(b64decode(data, padded=False, ignorechars=b'='), + expected) + self.assertEqual(urlsafe_b64decode(data, padded=True), expected) + self.assertEqual(urlsafe_b64decode(data, padded=False), expected) + data = data.replace(b'=', b'') + self.assertEqual(b64decode(data, padded=False), expected) + self.assertEqual(b64decode(data, padded=False, validate=True), + expected) + self.assertEqual(urlsafe_b64decode(data), expected) + + check(b'', b'') + check(b'YQ==', b'a') + check(b'YWI=', b'ab') + check(b'YWJj', b'abc') + check(b'Y=WJj', b'abc') + check(b'YW=Jj', b'abc') + check(b'YWJ=j', b'abc') + + with self.assertRaisesRegex(binascii.Error, 'Incorrect padding'): + urlsafe_b64decode(b'YQ', padded=True) + with self.assertRaisesRegex(binascii.Error, 'Incorrect padding'): + urlsafe_b64decode(b'YWI', padded=True) + + def _common_test_ignorechars(self, func): + eq = self.assertEqual + eq(func(b'', ignorechars=b' \n'), b'') + eq(func(b'', ignorechars=b''), b'') + eq(func(b' \n', ignorechars=b' \n'), b'') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b'') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b' ') + with self.assertRaises(binascii.Error): + func(b' \n', ignorechars=b'\n') + with self.assertRaises(TypeError): + func(b'', ignorechars='') + with self.assertRaises(TypeError): + func(b'', ignorechars=[]) + with self.assertRaises(TypeError): + func(b'', ignorechars=None) + def test_b64decode_invalid_chars(self): # issue 1466065: Test some invalid characters. - tests = ((b'%3d==', b'\xdd'), - (b'$3d==', b'\xdd'), - (b'[==', b''), - (b'YW]3=', b'am'), - (b'3{d==', b'\xdd'), - (b'3d}==', b'\xdd'), - (b'@@', b''), - (b'!', b''), - (b"YWJj\n", b"abc"), - (b'YWJj\nYWI=', b'abcab')) + tests = ((b'%3d==', b'\xdd', b'%$'), + (b'$3d==', b'\xdd', b'%$'), + (b'[==', b'', b'[='), + (b'YW]3=', b'am', b']'), + (b'3{d==', b'\xdd', b'{}'), + (b'3d}==', b'\xdd', b'{}'), + (b'@@', b'', b'@!'), + (b'!', b'', b'@!'), + (b"YWJj\n", b"abc", b'\n'), + (b'YWJj\nYWI=', b'abcab', b'\n'), + (b'=YWJj', b'abc', b'='), + (b'Y=WJj', b'abc', b'='), + (b'Y==WJj', b'abc', b'='), + (b'Y===WJj', b'abc', b'='), + (b'YW=Jj', b'abc', b'='), + (b'YWJj=', b'abc', b'='), + (b'YW\nJj', b'abc', b'\n'), + (b'YW\nJj', b'abc', bytearray(b'\n')), + (b'YW\nJj', b'abc', memoryview(b'\n')), + ) funcs = ( base64.b64decode, base64.standard_b64decode, base64.urlsafe_b64decode, ) - for bstr, res in tests: + for bstr, res, ignorechars in tests: for func in funcs: with self.subTest(bstr=bstr, func=func): self.assertEqual(func(bstr), res) @@ -272,14 +417,111 @@ class BaseXYTestCase(unittest.TestCase): base64.b64decode(bstr, validate=True) with self.assertRaises(binascii.Error): base64.b64decode(bstr.decode('ascii'), validate=True) + with self.assertRaises(binascii.Error): + # Even empty ignorechars enables the strict mode. + base64.b64decode(bstr, ignorechars=b'') + r = base64.b64decode(bstr, ignorechars=ignorechars) + self.assertEqual(r, res) - # Normal alphabet characters not discarded when alternative given - res = b'\xfb\xef\xff' - self.assertEqual(base64.b64decode(b'++//', validate=True), res) - self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) - self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) - self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) - self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) + self._common_test_ignorechars(base64.b64decode) + + # Normal alphabet characters will be discarded when alternative given + discarded = ("invalid character %a in Base64 data with %s " + "will be discarded in future Python versions") + error = ("invalid character %a in Base64 data with %s " + "will be an error in future Python versions") + with self.assertWarns(FutureWarning) as cm: + r = base64.b64decode(b'++++', altchars=b'-_') + self.assertEqual(r, b'\xfb\xef\xbe') + self.assertEqual(str(cm.warning), + discarded % ('+', "altchars=b'-_' and validate=False")) + with self.assertWarns(FutureWarning) as cm: + r = base64.b64decode(b'////', altchars=b'-_') + self.assertEqual(r, b'\xff\xff\xff') + self.assertEqual(str(cm.warning), + discarded % ('/', "altchars=b'-_' and validate=False")) + with self.assertWarns(DeprecationWarning) as cm: + r = base64.b64decode(b'++++', altchars=b'-_', validate=True) + self.assertEqual(r, b'\xfb\xef\xbe') + self.assertEqual(str(cm.warning), + error % ('+', "altchars=b'-_' and validate=True")) + with self.assertWarns(DeprecationWarning) as cm: + r = base64.b64decode(b'////', altchars=b'-_', validate=True) + self.assertEqual(r, b'\xff\xff\xff') + self.assertEqual(str(cm.warning), + error % ('/', "altchars=b'-_' and validate=True")) + r = base64.b64decode(b'++++', altchars=b'-_', ignorechars=b'+') + self.assertEqual(r, b'') + r = base64.b64decode(b'////', altchars=b'-_', ignorechars=b'/') + self.assertEqual(r, b'') + r = base64.b64decode(b'++++////', altchars=b'-_', validate=False, ignorechars=b'') + self.assertEqual(r, b'') + with self.assertRaisesRegex(binascii.Error, 'Only base64 data is allowed'): + base64.b64decode(b'////', altchars=b'-_', ignorechars=b'') + with self.assertRaisesRegex(binascii.Error, 'Only base64 data is allowed'): + base64.b64decode(b'++++', altchars=b'-_', ignorechars=b'') + r = base64.b64decode(b'++++YWJj----____', altchars=b'-_', ignorechars=b'+') + self.assertEqual(r, b'abc\xfb\xef\xbe\xff\xff\xff') + r = base64.b64decode(b'////YWJj----____', altchars=b'-_', ignorechars=b'/') + self.assertEqual(r, b'abc\xfb\xef\xbe\xff\xff\xff') + r = base64.b64decode(b'++++,,,,', altchars=b'+,', ignorechars=b'+') + self.assertEqual(r, b'\xfb\xef\xbe\xff\xff\xff') + r = base64.b64decode(b'////YWJj++++,,,,', altchars=b'+,', ignorechars=b'/') + self.assertEqual(r, b'abc\xfb\xef\xbe\xff\xff\xff') + r = base64.b64decode(b'----////', altchars=b'-/', ignorechars=b'/') + self.assertEqual(r, b'\xfb\xef\xbe\xff\xff\xff') + r = base64.b64decode(b'++++YWJj----////', altchars=b'-/', ignorechars=b'+') + self.assertEqual(r, b'abc\xfb\xef\xbe\xff\xff\xff') + + with self.assertWarns(FutureWarning) as cm: + self.assertEqual(base64.urlsafe_b64decode(b'++++'), b'\xfb\xef\xbe') + self.assertEqual(str(cm.warning), + "invalid character '+' in URL-safe Base64 data " + "will be discarded in future Python versions") + with self.assertWarns(FutureWarning) as cm: + self.assertEqual(base64.urlsafe_b64decode(b'////'), b'\xff\xff\xff') + self.assertEqual(str(cm.warning), + "invalid character '/' in URL-safe Base64 data " + "will be discarded in future Python versions") + with self.assertRaises(binascii.Error): + base64.b64decode(b'+/!', altchars=b'-_') + + def _altchars_strategy(): + """Generate 'altchars' for base64 encoding.""" + reserved_chars = (string.digits + string.ascii_letters + "=").encode() + allowed_chars = hypothesis.strategies.sampled_from( + [n for n in range(256) if n not in reserved_chars]) + two_bytes_strategy = hypothesis.strategies.lists( + allowed_chars, min_size=2, max_size=2, unique=True).map(bytes) + return (hypothesis.strategies.none() + | hypothesis.strategies.just(b"_-") + | two_bytes_strategy) + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + altchars=_altchars_strategy(), + validate=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', b"_-", True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', b"_-", False) + def test_b64_encode_decode_round_trip(self, payload, altchars, validate): + encoded = base64.b64encode(payload, altchars=altchars) + decoded = base64.b64decode(encoded, altchars=altchars, + validate=validate) + self.assertEqual(payload, decoded) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_standard_b64_encode_decode_round_trip(self, payload): + encoded = base64.standard_b64encode(payload) + decoded = base64.standard_b64decode(encoded) + self.assertEqual(payload, decoded) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_urlsafe_b64_encode_decode_round_trip(self, payload): + encoded = base64.urlsafe_b64encode(payload) + decoded = base64.urlsafe_b64decode(encoded) + self.assertEqual(payload, decoded) def test_b32encode(self): eq = self.assertEqual @@ -294,6 +536,23 @@ class BaseXYTestCase(unittest.TestCase): self.check_other_types(base64.b32encode, b'abcd', b'MFRGGZA=') self.check_encode_type_errors(base64.b32encode) + def test_b32encode_padded(self): + b32encode = base64.b32encode + self.assertEqual(b32encode(b'', padded=False), b'') + self.assertEqual(b32encode(b'a', padded=False), b'ME') + self.assertEqual(b32encode(b'ab', padded=False), b'MFRA') + self.assertEqual(b32encode(b'abc', padded=False), b'MFRGG') + self.assertEqual(b32encode(b'abcd', padded=False), b'MFRGGZA') + self.assertEqual(b32encode(b'abcde', padded=False), b'MFRGGZDF') + + def test_b32encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b32encode, b) + eq(base64.b32encode(b, wrapcol=16), b'O53XOLTQPF2GQ33O\nFZXXEZY=') + eq(base64.b32encode(b, wrapcol=23), b'O53XOLTQPF2GQ33O\nFZXXEZY=') + eq(base64.b32encode(b, wrapcol=1), b'O53XOLTQ\nPF2GQ33O\nFZXXEZY=') + def test_b32decode(self): eq = self.assertEqual tests = {b'': b'', @@ -329,6 +588,7 @@ class BaseXYTestCase(unittest.TestCase): for data, res in tests.items(): eq(base64.b32decode(data, True), res) + eq(base64.b32decode(data, casefold=True), res) eq(base64.b32decode(data.decode('ascii'), True), res) self.assertRaises(binascii.Error, base64.b32decode, b'me======') @@ -347,6 +607,11 @@ class BaseXYTestCase(unittest.TestCase): self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + self.assertRaises(ValueError, base64.b32decode, b'', map01=b'') + self.assertRaises(ValueError, base64.b32decode, b'', map01=b'LI') + self.assertRaises(TypeError, base64.b32decode, b'', map01=0) + eq(base64.b32decode(b'MLO23456', map01=None), res_L) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456', map01=None) data = b'M1023456' data_str = data.decode('ascii') @@ -362,6 +627,56 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) + def test_b32decode_padded(self): + b32decode = base64.b32decode + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b32decode(data, padded=False) + self.assertEqual(b32decode(data, padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(b32decode(data, padded=False), expected) + + check(b'', b'') + check(b'ME======', b'a') + check(b'MFRA====', b'ab') + check(b'MFRGG===', b'abc') + check(b'MFRGGZA=', b'abcd') + check(b'MFRGGZDF', b'abcde') + check(b'M=FRGGZDF', b'abcde') + check(b'MF=RGGZDF', b'abcde') + check(b'MFR=GGZDF', b'abcde') + check(b'MFRG=GZDF', b'abcde') + check(b'MFRGG=ZDF', b'abcde') + check(b'MFRGGZ=DF', b'abcde') + check(b'MFRGGZD=F', b'abcde') + + def test_b32decode_ignorechars(self): + self._common_test_ignorechars(base64.b32decode) + eq = self.assertEqual + eq(base64.b32decode(b'MFRG\n GZDF\n', ignorechars=b' \n'), b'abcde') + eq(base64.b32decode(b'MFRG\n GZDF\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(base64.b32decode(b'M=======FRGGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MF======RGGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFR=====GGZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRG====GZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGG===ZDF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZ==DF', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZD=F', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRGGZDF=', ignorechars=b'='), b'abcde') + eq(base64.b32decode(b'MFRA======', ignorechars=b'='), b'ab') + + eq(base64.b32decode(b'mfRggzDfmzTQ====', ignorechars=b'mfgz'), + b'\x88\xe7') + eq(base64.b32decode(b'mfRggzDfmzTQ====', casefold=True, ignorechars=b'mfgz'), + b'abcdefg') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', ignorechars=b'01'), b'abcde') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', map01=b'L', ignorechars=b'01'), + b'c\x8a\xb8\xb8\xcb3\xb2\xb1\xb8\xab') + eq(base64.b32decode(b'M0F1R0G1G0Z1D0F1', map01=b'I', ignorechars=b'01'), + b'c\x8a\x88\xb8\xc83\xb2\x81\xb8\xa8') + def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] prefixes = [b'M', b'ME', b'MFRA', b'MFRGG', b'MFRGGZA', b'MFRGGZDF'] @@ -378,6 +693,19 @@ class BaseXYTestCase(unittest.TestCase): with self.assertRaises(binascii.Error): base64.b32decode(data.decode('ascii')) + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans(), + map01=( + hypothesis.strategies.none() + | hypothesis.strategies.binary(min_size=1, max_size=1))) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, None) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, None) + def test_b32_encode_decode_round_trip(self, payload, casefold, map01): + encoded = base64.b32encode(payload) + decoded = base64.b32decode(encoded, casefold=casefold, map01=map01) + self.assertEqual(payload, decoded) + def test_b32hexencode(self): test_cases = [ # to_encode, expected @@ -392,11 +720,21 @@ class BaseXYTestCase(unittest.TestCase): for to_encode, expected in test_cases: with self.subTest(to_decode=to_encode): self.assertEqual(base64.b32hexencode(to_encode), expected) + self.assertEqual(base64.b32hexencode(to_encode, padded=False), + expected.rstrip(b'=')) def test_b32hexencode_other_types(self): self.check_other_types(base64.b32hexencode, b'abcd', b'C5H66P0=') self.check_encode_type_errors(base64.b32hexencode) + def test_b32hexencode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b32hexencode, b) + eq(base64.b32hexencode(b, wrapcol=16), b'ETRNEBJGF5Q6GRRE\n5PNN4PO=') + eq(base64.b32hexencode(b, wrapcol=23), b'ETRNEBJGF5Q6GRRE\n5PNN4PO=') + eq(base64.b32hexencode(b, wrapcol=1), b'ETRNEBJG\nF5Q6GRRE\n5PNN4PO=') + def test_b32hexdecode(self): test_cases = [ # to_decode, expected, casefold @@ -431,6 +769,53 @@ class BaseXYTestCase(unittest.TestCase): self.check_other_types(base64.b32hexdecode, b'C5H66===', b'abc') self.check_decode_type_errors(base64.b32hexdecode) + def test_b32hexdecode_padded(self): + b32hexdecode = base64.b32hexdecode + def check(data, expected): + if b'=' in data: + with self.assertRaisesRegex(binascii.Error, 'Padding not allowed'): + b32hexdecode(data, padded=False) + self.assertEqual(b32hexdecode(data, padded=False, ignorechars=b'='), + expected) + data = data.replace(b'=', b'') + self.assertEqual(b32hexdecode(data, padded=False), expected) + + check(b'', b'') + check(b'C4======', b'a') + check(b'C5H0====', b'ab') + check(b'C5H66===', b'abc') + check(b'C5H66P0=', b'abcd') + check(b'C5H66P35', b'abcde') + check(b'C=5H66P35', b'abcde') + check(b'C5=H66P35', b'abcde') + check(b'C5H=66P35', b'abcde') + check(b'C5H6=6P35', b'abcde') + check(b'C5H66=P35', b'abcde') + check(b'C5H66P=35', b'abcde') + check(b'C5H66P3=5', b'abcde') + + def test_b32hexdecode_ignorechars(self): + self._common_test_ignorechars(base64.b32hexdecode) + eq = self.assertEqual + eq(base64.b32hexdecode(b'C5H6\n 6P35\n', ignorechars=b' \n'), b'abcde') + eq(base64.b32hexdecode(b'C5H6\n 6P35\n', ignorechars=bytearray(b' \n')), b'abcde') + eq(base64.b32hexdecode(b'========C5H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C=======5H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5======H66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H=====66P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H6====6P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66===P35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P==35', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P3=5', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H66P35=', ignorechars=b'='), b'abcde') + eq(base64.b32hexdecode(b'C5H0======', ignorechars=b'='), b'ab') + + eq(base64.b32hexdecode(b'c5h66p35cpjmg===', ignorechars=b'cghjmp'), + b')\x8c2') + eq(base64.b32hexdecode(b'c5h66p35cpjmg===', casefold=True, + ignorechars=b'cghjmp'), + b'abcdefgh') + def test_b32hexdecode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF', b'c4======'] prefixes = [b'M', b'ME', b'MFRA', b'MFRGG', b'MFRGGZA', b'MFRGGZDF'] @@ -447,6 +832,15 @@ class BaseXYTestCase(unittest.TestCase): with self.assertRaises(binascii.Error): base64.b32hexdecode(data.decode('ascii')) + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b32_hexencode_decode_round_trip(self, payload, casefold): + encoded = base64.b32hexencode(payload) + decoded = base64.b32hexdecode(encoded, casefold=casefold) + self.assertEqual(payload, decoded) def test_b16encode(self): eq = self.assertEqual @@ -457,6 +851,14 @@ class BaseXYTestCase(unittest.TestCase): b'0102ABCDEF') self.check_encode_type_errors(base64.b16encode) + def test_b16encode_wrapcol(self): + eq = self.assertEqual + b = b'\x01\x02\xab\xcd\xef' + self._common_test_wrapcol(base64.b16encode, b) + eq(base64.b16encode(b, wrapcol=4), b'0102\nABCD\nEF') + eq(base64.b16encode(b, wrapcol=5), b'0102\nABCD\nEF') + eq(base64.b16encode(b, wrapcol=1), b'01\n02\nAB\nCD\nEF') + def test_b16decode(self): eq = self.assertEqual eq(base64.b16decode(b'0102ABCDEF'), b'\x01\x02\xab\xcd\xef') @@ -484,6 +886,24 @@ class BaseXYTestCase(unittest.TestCase): # Incorrect "padding" self.assertRaises(binascii.Error, base64.b16decode, '010') + def test_b16decode_ignorechars(self): + self._common_test_ignorechars(base64.b16decode) + eq = self.assertEqual + eq(base64.b16decode(b'A B\nC D\n', ignorechars=b' \n'), b'\xab\xcd') + eq(base64.b16decode(b'A B\nC D\n', ignorechars=bytearray(b' \n')), b'\xab\xcd') + eq(base64.b16decode(b'aBcD', ignorechars=b'ac'), b'\xbd') + eq(base64.b16decode(b'aBcD', casefold=True, ignorechars=b'ac'), b'\xab\xcd') + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b16_encode_decode_round_trip(self, payload, casefold): + endoded = base64.b16encode(payload) + decoded = base64.b16decode(endoded, casefold=casefold) + self.assertEqual(payload, decoded) + def test_a85encode(self): eq = self.assertEqual @@ -522,18 +942,33 @@ class BaseXYTestCase(unittest.TestCase): self.assertRaises(TypeError, base64.a85encode, "") - eq(base64.a85encode(b"www.python.org", wrapcol=7, adobe=False), - b'GB\\6`E-\nZP=Df.1\nGEb>') - eq(base64.a85encode(b"\0\0\0\0www.python.org", wrapcol=7, adobe=False), - b'zGB\\6`E\n-ZP=Df.\n1GEb>') - eq(base64.a85encode(b"www.python.org", wrapcol=7, adobe=True), - b'<~GB\\6`\nE-ZP=Df\n.1GEb>\n~>') - eq(base64.a85encode(b' '*8, foldspaces=True, adobe=False), b'yy') eq(base64.a85encode(b' '*7, foldspaces=True, adobe=False), b'y+') + eq(base64.a85encode(b"\0\0\0\0www.python.org", wrapcol=7), + b'zGB\\6`E\n-ZP=Df.\n1GEb>') + eq(base64.a85encode(b, wrapcol=75), b'GB\\6`E-ZP=Df.1GEb>') + eq(base64.a85encode(b, wrapcol=1), + b'G\nB\n\\\n6\n`\nE\n-\nZ\nP\n=\nD\nf\n.\n1\nG\nE\nb\n>') + eq(base64.a85encode(b, wrapcol=7, adobe=True), + b'<~GB\\6`\nE-ZP=Df\n.1GEb>\n~>') + eq(base64.a85encode(b, wrapcol=1), + b'G\nB\n\\\n6\n`\nE\n-\nZ\nP\n=\nD\nf\n.\n1\nG\nE\nb\n>') + eq(base64.a85encode(b, wrapcol=1, adobe=True), + b'<~\nGB\n\\6\n`E\n-Z\nP=\nDf\n.1\nGE\nb>\n~>') + with self.assertRaises(ValueError): + base64.a85encode(b, wrapcol=-7, adobe=True) + eq(base64.a85encode(b'', wrapcol=1, adobe=True), b'<~\n~>') + eq(base64.a85encode(b'', wrapcol=3, adobe=True), b'<~\n~>') + eq(base64.a85encode(b'', wrapcol=4, adobe=True), b'<~~>') + def test_b85encode(self): eq = self.assertEqual @@ -568,11 +1003,20 @@ class BaseXYTestCase(unittest.TestCase): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_b85encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.b85encode, b) + eq(base64.b85encode(b, wrapcol=10), b'cXxL#aCvlS\nZ*DGca%T') + eq(base64.b85encode(b, wrapcol=14), b'cXxL#aCvlS\nZ*DGca%T') + eq(base64.b85encode(b, wrapcol=1), b'cXxL#\naCvlS\nZ*DGc\na%T') + def test_z85encode(self): eq = self.assertEqual tests = { b'': b'', + b'\x86\x4F\xD2\x6F\xB5\x59\xF7\x5B': b'HelloWorld', b'www.python.org': b'CxXl-AcVLsz/dgCA+t', bytes(range(255)): b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" @@ -602,6 +1046,14 @@ class BaseXYTestCase(unittest.TestCase): self.check_other_types(base64.z85encode, b"www.python.org", b'CxXl-AcVLsz/dgCA+t') + def test_z85encode_wrapcol(self): + eq = self.assertEqual + b = b'www.python.org' + self._common_test_wrapcol(base64.z85encode, b) + eq(base64.z85encode(b, wrapcol=10), b'CxXl-AcVLs\nz/dgCA+t') + eq(base64.z85encode(b, wrapcol=14), b'CxXl-AcVLs\nz/dgCA+t') + eq(base64.z85encode(b, wrapcol=1), b'CxXl-\nAcVLs\nz/dgC\nA+t') + def test_a85decode(self): eq = self.assertEqual @@ -733,6 +1185,12 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.a85decode(b'G^+IX'), b"xxxx") eq(base64.a85decode(b'G^+IXGQ7^D'), b"xxxxx\x00\x00\x00") + eq(base64.a85encode(b"\x00", pad=True), b'z') + eq(base64.a85encode(b"\x00"*2, pad=True), b'z') + eq(base64.a85encode(b"\x00"*3, pad=True), b'z') + eq(base64.a85encode(b"\x00"*4, pad=True), b'z') + eq(base64.a85encode(b"\x00"*5, pad=True), b'zz') + def test_b85_padding(self): eq = self.assertEqual @@ -748,6 +1206,21 @@ class BaseXYTestCase(unittest.TestCase): eq(base64.b85decode(b'czAet'), b"xxxx") eq(base64.b85decode(b'czAetcmMzZ'), b"xxxxx\x00\x00\x00") + def test_z85_padding(self): + eq = self.assertEqual + + eq(base64.z85encode(b"x", pad=True), b'CMmZz') + eq(base64.z85encode(b"xx", pad=True), b'CZ6h*') + eq(base64.z85encode(b"xxx", pad=True), b'CZaDk') + eq(base64.z85encode(b"xxxx", pad=True), b'CZaET') + eq(base64.z85encode(b"xxxxx", pad=True), b'CZaETCMmZz') + + eq(base64.z85decode(b'CMmZz'), b"x\x00\x00\x00") + eq(base64.z85decode(b'CZ6h*'), b"xx\x00\x00") + eq(base64.z85decode(b'CZaDk'), b"xxx\x00") + eq(base64.z85decode(b'CZaET'), b"xxxx") + eq(base64.z85decode(b'CZaETCMmZz'), b"xxxxx\x00\x00\x00") + def test_a85decode_errors(self): illegal = (set(range(32)) | set(range(118, 256))) - set(b' \t\n\r\v') for c in illegal: @@ -787,24 +1260,20 @@ class BaseXYTestCase(unittest.TestCase): self.assertEqual(base64.a85decode(b"a b\nc", ignorechars=b" \n"), b'\xc9\x89') - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b"") - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b" ") - with self.assertRaises(ValueError): - base64.a85decode(b"a b\nc", ignorechars=b"\n") - with self.assertRaises(TypeError): - base64.a85decode(b"a b\nc", ignorechars=" \n") - with self.assertRaises(TypeError): - base64.a85decode(b"a b\nc", ignorechars=None) + self._common_test_ignorechars(base64.a85decode) def test_b85decode_errors(self): illegal = list(range(33)) + \ list(b'"\',./:[\\]') + \ list(range(128, 256)) for c in illegal: - with self.assertRaises(ValueError, msg=bytes([c])): - base64.b85decode(b'0000' + bytes([c])) + b = bytes([c]) + with self.assertRaises(ValueError, msg=b): + base64.b85decode(b'0000' + b) + self.assertEqual(base64.b85decode(b'0000' + b, ignorechars=b), + b'\x00\x00\x00') + + self._common_test_ignorechars(base64.b85decode) self.assertRaises(ValueError, base64.b85decode, b'|') self.assertRaises(ValueError, base64.b85decode, b'|N') @@ -817,8 +1286,13 @@ class BaseXYTestCase(unittest.TestCase): list(b'"\',;_`|\\~') + \ list(range(128, 256)) for c in illegal: - with self.assertRaises(ValueError, msg=bytes([c])): - base64.z85decode(b'0000' + bytes([c])) + b = bytes([c]) + with self.assertRaises(ValueError, msg=b): + base64.z85decode(b'0000' + b) + self.assertEqual(base64.z85decode(b'0000' + b, ignorechars=b), + b'\x00\x00\x00') + + self._common_test_ignorechars(base64.z85decode) # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: self.assertRaises(ValueError, base64.z85decode, b'%') @@ -827,6 +1301,61 @@ class BaseXYTestCase(unittest.TestCase): self.assertRaises(ValueError, base64.z85decode, b'%nSc') self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def add_padding(self, payload): + """Add the expected padding for test_?85_encode_decode_round_trip.""" + if len(payload) % 4 != 0: + padding = b"\0" * ((-len(payload)) % 4) + payload = payload + padding + return payload + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + foldspaces=hypothesis.strategies.booleans(), + wrapcol=( + hypothesis.strategies.just(0) + | hypothesis.strategies.integers(1, 1000)), + pad=hypothesis.strategies.booleans(), + adobe=hypothesis.strategies.booleans(), + ) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, 0, False, False) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, 20, True, True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, 0, False, True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, 20, True, False) + def test_a85_encode_decode_round_trip( + self, payload, foldspaces, wrapcol, pad, adobe + ): + encoded = base64.a85encode( + payload, foldspaces=foldspaces, wrapcol=wrapcol, + pad=pad, adobe=adobe, + ) + if wrapcol: + if adobe and wrapcol == 1: + # "adobe" needs wrapcol to be at least 2. + # a85decode quietly uses 2 if 1 is given; it's not worth + # loudly deprecating this behavior. + wrapcol = 2 + for line in encoded.splitlines(keepends=False): + self.assertLessEqual(len(line), wrapcol) + if adobe: + self.assertTrue(encoded.startswith(b'<~')) + self.assertTrue(encoded.endswith(b'~>')) + decoded = base64.a85decode(encoded, foldspaces=foldspaces, adobe=adobe) + if pad: + payload = self.add_padding(payload) + self.assertEqual(payload, decoded) + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + pad=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b85_encode_decode_round_trip(self, payload, pad): + encoded = base64.b85encode(payload, pad=pad) + if pad: + payload = self.add_padding(payload) + decoded = base64.b85decode(encoded) + self.assertEqual(payload, decoded) + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs index 715410f14..d2a4a078a 100644 --- a/crates/stdlib/src/binascii.rs +++ b/crates/stdlib/src/binascii.rs @@ -156,14 +156,24 @@ mod decl { vm.ctx.new_exception_type("binascii", "Incomplete", None) } + #[derive(FromArgs)] + struct HexlifyArgs { + #[pyarg(any)] + data: ArgBytesLike, + #[pyarg(any, optional)] + sep: OptionalArg, + #[pyarg(any, optional)] + bytes_per_sep: OptionalArg, + } + #[pyfunction(name = "b2a_hex")] #[pyfunction] - fn hexlify( - data: ArgBytesLike, - sep: OptionalArg, - bytes_per_sep: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult> { + fn hexlify(args: HexlifyArgs, vm: &VirtualMachine) -> PyResult> { + let HexlifyArgs { + data, + sep, + bytes_per_sep, + } = args; let sep = match sep { OptionalArg::Present(sep) => sep.with_ref(|sep| { let [sep] = sep else {