Merge pull request #4344 from moreal/base64-trailing-bits

Allow trailing bits while decoding base64

Lib/test/test_binascii.py

@@ -4,12 +4,14 @@ import unittest
 import binascii
 import array
 import re
+from test.support import bigmemtest, _1G, _4G, warnings_helper
+
 
 # Note: "*_hex" functions are aliases for "(un)hexlify"
-b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_hqx', 'b2a_qp', 'b2a_uu',
-                 'hexlify', 'rlecode_hqx']
-a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_hqx', 'a2b_qp', 'a2b_uu',
-                 'unhexlify', 'rledecode_hqx']
+b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_qp', 'b2a_uu',
+                 'hexlify']
+a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_qp', 'a2b_uu',
+                 'unhexlify']
 all_functions = a2b_functions + b2a_functions + ['crc32', 'crc_hqx']
 
 
@@ -30,8 +32,6 @@ class BinASCIITest(unittest.TestCase):
         self.assertTrue(issubclass(binascii.Error, Exception))
         self.assertTrue(issubclass(binascii.Incomplete, Exception))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_functions(self):
         # Check presence of all functions
         for name in all_functions:
@@ -52,9 +52,6 @@ class BinASCIITest(unittest.TestCase):
                 res = a2b(self.type2test(a))
             except Exception as err:
                 self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
-            if fb == 'b2a_hqx':
-                # b2a_hqx returns a tuple
-                res, _ = res
             self.assertEqual(res, raw, "{}/{} conversion: "
                              "{!r} != {!r}".format(fb, fa, res, raw))
             self.assertIsInstance(res, bytes)
@@ -117,6 +114,51 @@ class BinASCIITest(unittest.TestCase):
         # empty strings. TBD: shouldn't it raise an exception instead ?
         self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'')
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_base64_strict_mode(self):
+        # Test base64 with strict mode on
+        def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes):
+            with self.assertRaisesRegex(binascii.Error, assert_regex):
+                binascii.a2b_base64(self.type2test(data), strict_mode=True)
+            self.assertEqual(binascii.a2b_base64(self.type2test(data), strict_mode=False),
+                             non_strict_mode_expected_result)
+            self.assertEqual(binascii.a2b_base64(self.type2test(data)),
+                             non_strict_mode_expected_result)
+
+        def assertExcessData(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Excess data', data, non_strict_mode_expected_result)
+
+        def assertNonBase64Data(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Only base64 data', data, non_strict_mode_expected_result)
+
+        def assertLeadingPadding(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Leading padding', data, non_strict_mode_expected_result)
+
+        def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result)
+
+        # Test excess data exceptions
+        assertExcessData(b'ab==a', b'i')
+        assertExcessData(b'ab===', b'i')
+        assertExcessData(b'ab==:', b'i')
+        assertExcessData(b'abc=a', b'i\xb7')
+        assertExcessData(b'abc=:', b'i\xb7')
+        assertExcessData(b'ab==\n', b'i')
+
+        # Test non-base64 data exceptions
+        assertNonBase64Data(b'\nab==', b'i')
+        assertNonBase64Data(b'ab:(){:|:&};:==', b'i')
+        assertNonBase64Data(b'a\nb==', b'i')
+        assertNonBase64Data(b'a\x00b==', b'i')
+
+        # Test malformed padding
+        assertLeadingPadding(b'=', b'')
+        assertLeadingPadding(b'==', b'')
+        assertLeadingPadding(b'===', b'')
+        assertDiscontinuousPadding(b'ab=c=', b'i\xb7')
+        assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b')
+
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
     def test_base64errors(self):
@@ -208,32 +250,6 @@ class BinASCIITest(unittest.TestCase):
 
         self.assertRaises(TypeError, binascii.crc32)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
-    def test_hqx(self):
-        # Perform binhex4 style RLE-compression
-        # Then calculate the hexbin4 binary-to-ASCII translation
-        rle = binascii.rlecode_hqx(self.data)
-        a = binascii.b2a_hqx(self.type2test(rle))
-
-        b, _ = binascii.a2b_hqx(self.type2test(a))
-        res = binascii.rledecode_hqx(b)
-        self.assertEqual(res, self.rawdata)
-
-    def test_rle(self):
-        # test repetition with a repetition longer than the limit of 255
-        data = (b'a' * 100 + b'b' + b'c' * 300)
-
-        encoded = binascii.rlecode_hqx(data)
-        self.assertEqual(encoded,
-                         (b'a\x90d'      # 'a' * 100
-                          b'b'           # 'b'
-                          b'c\x90\xff'   # 'c' * 255
-                          b'c\x90-'))    # 'c' * 45
-
-        decoded = binascii.rledecode_hqx(encoded)
-        self.assertEqual(decoded, data)
-
     def test_hex(self):
         # test hexlification
         s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
@@ -368,8 +384,6 @@ class BinASCIITest(unittest.TestCase):
         self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n')
         self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_empty_string(self):
         # A test for SF bug #1022953.  Make sure SystemError is not raised.
         empty = self.type2test(b'')
@@ -388,7 +402,7 @@ class BinASCIITest(unittest.TestCase):
     @unittest.expectedFailure
     def test_unicode_b2a(self):
         # Unicode strings are not accepted by b2a_* functions.
-        for func in set(all_functions) - set(a2b_functions) | {'rledecode_hqx'}:
+        for func in set(all_functions) - set(a2b_functions):
             try:
                 self.assertRaises(TypeError, getattr(binascii, func), "test")
             except Exception as err:
@@ -403,9 +417,6 @@ class BinASCIITest(unittest.TestCase):
         MAX_ALL = 45
         raw = self.rawdata[:MAX_ALL]
         for fa, fb in zip(a2b_functions, b2a_functions):
-            if fa == 'rledecode_hqx':
-                # Takes non-ASCII data
-                continue
             a2b = getattr(binascii, fa)
             b2a = getattr(binascii, fb)
             try:
@@ -415,10 +426,6 @@ class BinASCIITest(unittest.TestCase):
                 res = a2b(a)
             except Exception as err:
                 self.fail("{}/{} conversion raises {!r}".format(fb, fa, err))
-            if fb == 'b2a_hqx':
-                # b2a_hqx returns a tuple
-                res, _ = res
-                binary_res, _ = binary_res
             self.assertEqual(res, raw, "{}/{} conversion: "
                              "{!r} != {!r}".format(fb, fa, res, raw))
             self.assertEqual(res, binary_res)
@@ -449,6 +456,14 @@ class BytearrayBinASCIITest(BinASCIITest):
 class MemoryviewBinASCIITest(BinASCIITest):
     type2test = memoryview
 
+class ChecksumBigBufferTestCase(unittest.TestCase):
+    """bpo-38256 - check that inputs >=4 GiB are handled correctly."""
+
+    @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
+    def test_big_buffer(self, size):
+        data = b"nyan" * (_1G + 1)
+        self.assertEqual(binascii.crc32(data), 1044521549)
+
 
 if __name__ == "__main__":
     unittest.main()

extra_tests/snippets/stdlib_binascii.py

@@ -49,6 +49,7 @@ assert_equal(
     dec_b64(b"4pii8J+QoyAg4ZaH8J2TpPCdlYrRguKTn/CdlZDwnZWl5Y2Ez4PwnZSrICDimazwn5Gj\n"),
     "☢🐣  ᖇ𝓤𝕊тⓟ𝕐𝕥卄σ𝔫  ♬👣".encode(),
 )
+assert_equal(dec_b64(b"3d=="), b"\xdd")
 
 for exc, expected_name in [
     (binascii.Error, "Error"),

stdlib/src/binascii.rs

@@ -2,8 +2,13 @@ pub(crate) use decl::make_module;
 
 pub(super) use decl::crc32;
 
+pub fn decode<T: AsRef<[u8]>>(input: T) -> Result<Vec<u8>, base64::DecodeError> {
+    base64::decode_config(input, base64::STANDARD.decode_allow_trailing_bits(true))
+}
+
 #[pymodule(name = "binascii")]
 mod decl {
+    use super::decode;
     use crate::vm::{
         builtins::{PyBaseExceptionRef, PyIntRef, PyTypeRef},
         function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg},
@@ -168,7 +173,7 @@ mod decl {
 
         s.with_ref(|b| {
             let decoded = if b.len() % 4 == 0 {
-                base64::decode(b)
+                decode(b)
             } else {
                 Err(base64::DecodeError::InvalidLength)
             };
@@ -181,7 +186,7 @@ mod decl {
                 if buf.len() % 4 != 0 {
                     return Err(base64::DecodeError::InvalidLength);
                 }
-                base64::decode(&buf)
+                decode(&buf)
             })
         })
         .map_err(|err| new_binascii_error(format!("error decoding base64: {err}"), vm))