From d41f40879c3466a27654debb74b5d67f2a86bf85 Mon Sep 17 00:00:00 2001 From: Moreal Date: Mon, 19 Dec 2022 22:25:38 +0900 Subject: [PATCH 1/5] Allow trailing bits while decoding base64 --- stdlib/src/binascii.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index d0bb23752..c9fd34877 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -2,8 +2,13 @@ pub(crate) use decl::make_module; pub(super) use decl::crc32; +pub fn decode>(input: T) -> Result, base64::DecodeError> { + base64::decode_config(input, base64::STANDARD.decode_allow_trailing_bits(true)) +} + #[pymodule(name = "binascii")] mod decl { + use super::decode; use crate::vm::{ builtins::{PyBaseExceptionRef, PyIntRef, PyTypeRef}, function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg}, @@ -168,7 +173,7 @@ mod decl { s.with_ref(|b| { let decoded = if b.len() % 4 == 0 { - base64::decode(b) + decode(b) } else { Err(base64::DecodeError::InvalidLength) }; @@ -181,7 +186,7 @@ mod decl { if buf.len() % 4 != 0 { return Err(base64::DecodeError::InvalidLength); } - base64::decode(&buf) + decode(&buf) }) }) .map_err(|err| new_binascii_error(format!("error decoding base64: {err}"), vm)) From a3b1d2576895069543cc0b093684bc2589090a9c Mon Sep 17 00:00:00 2001 From: Moreal Date: Mon, 19 Dec 2022 23:17:03 +0900 Subject: [PATCH 2/5] Complement binascii extra_tests --- extra_tests/snippets/stdlib_binascii.py | 1 + 1 file changed, 1 insertion(+) diff --git a/extra_tests/snippets/stdlib_binascii.py b/extra_tests/snippets/stdlib_binascii.py index 666f67d51..5dcb5a032 100644 --- a/extra_tests/snippets/stdlib_binascii.py +++ b/extra_tests/snippets/stdlib_binascii.py @@ -49,6 +49,7 @@ assert_equal( dec_b64(b"4pii8J+QoyAg4ZaH8J2TpPCdlYrRguKTn/CdlZDwnZWl5Y2Ez4PwnZSrICDimazwn5Gj\n"), "☢🐣 ᖇ𝓤𝕊тⓟ𝕐𝕥卄σ𝔫 ♬👣".encode(), ) +assert_equal(dec_b64(b"3d=="), b"\xdd") for exc, expected_name in [ (binascii.Error, "Error"), From 9e9c409644634e62c0a49c73a8a75c19ca4ed6a5 Mon Sep 17 00:00:00 2001 From: Moreal Date: Fri, 23 Dec 2022 17:30:53 +0900 Subject: [PATCH 3/5] Update test_binascii.py to CPython 3.11 --- Lib/test/test_binascii.py | 99 +++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 882fb1e9b..365b4d513 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -4,12 +4,14 @@ import unittest import binascii import array import re +from test.support import bigmemtest, _1G, _4G, warnings_helper + # Note: "*_hex" functions are aliases for "(un)hexlify" -b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_hqx', 'b2a_qp', 'b2a_uu', - 'hexlify', 'rlecode_hqx'] -a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_hqx', 'a2b_qp', 'a2b_uu', - 'unhexlify', 'rledecode_hqx'] +b2a_functions = ['b2a_base64', 'b2a_hex', 'b2a_qp', 'b2a_uu', + 'hexlify'] +a2b_functions = ['a2b_base64', 'a2b_hex', 'a2b_qp', 'a2b_uu', + 'unhexlify'] all_functions = a2b_functions + b2a_functions + ['crc32', 'crc_hqx'] @@ -52,9 +54,6 @@ class BinASCIITest(unittest.TestCase): res = a2b(self.type2test(a)) except Exception as err: self.fail("{}/{} conversion raises {!r}".format(fb, fa, err)) - if fb == 'b2a_hqx': - # b2a_hqx returns a tuple - res, _ = res self.assertEqual(res, raw, "{}/{} conversion: " "{!r} != {!r}".format(fb, fa, res, raw)) self.assertIsInstance(res, bytes) @@ -117,6 +116,49 @@ class BinASCIITest(unittest.TestCase): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') + def test_base64_strict_mode(self): + # Test base64 with strict mode on + def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base64(self.type2test(data), strict_mode=True) + self.assertEqual(binascii.a2b_base64(self.type2test(data), strict_mode=False), + non_strict_mode_expected_result) + self.assertEqual(binascii.a2b_base64(self.type2test(data)), + non_strict_mode_expected_result) + + def assertExcessData(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess data', data, non_strict_mode_expected_result) + + def assertNonBase64Data(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Only base64 data', data, non_strict_mode_expected_result) + + def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Leading padding', data, non_strict_mode_expected_result) + + def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + + # Test excess data exceptions + assertExcessData(b'ab==a', b'i') + assertExcessData(b'ab===', b'i') + assertExcessData(b'ab==:', b'i') + assertExcessData(b'abc=a', b'i\xb7') + assertExcessData(b'abc=:', b'i\xb7') + assertExcessData(b'ab==\n', b'i') + + # Test non-base64 data exceptions + assertNonBase64Data(b'\nab==', b'i') + assertNonBase64Data(b'ab:(){:|:&};:==', b'i') + assertNonBase64Data(b'a\nb==', b'i') + assertNonBase64Data(b'a\x00b==', b'i') + + # Test malformed padding + assertLeadingPadding(b'=', b'') + assertLeadingPadding(b'==', b'') + assertLeadingPadding(b'===', b'') + assertDiscontinuousPadding(b'ab=c=', b'i\xb7') + assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + # TODO: RUSTPYTHON @unittest.expectedFailure def test_base64errors(self): @@ -208,32 +250,6 @@ class BinASCIITest(unittest.TestCase): self.assertRaises(TypeError, binascii.crc32) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_hqx(self): - # Perform binhex4 style RLE-compression - # Then calculate the hexbin4 binary-to-ASCII translation - rle = binascii.rlecode_hqx(self.data) - a = binascii.b2a_hqx(self.type2test(rle)) - - b, _ = binascii.a2b_hqx(self.type2test(a)) - res = binascii.rledecode_hqx(b) - self.assertEqual(res, self.rawdata) - - def test_rle(self): - # test repetition with a repetition longer than the limit of 255 - data = (b'a' * 100 + b'b' + b'c' * 300) - - encoded = binascii.rlecode_hqx(data) - self.assertEqual(encoded, - (b'a\x90d' # 'a' * 100 - b'b' # 'b' - b'c\x90\xff' # 'c' * 255 - b'c\x90-')) # 'c' * 45 - - decoded = binascii.rledecode_hqx(encoded) - self.assertEqual(decoded, data) - def test_hex(self): # test hexlification s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' @@ -388,7 +404,7 @@ class BinASCIITest(unittest.TestCase): @unittest.expectedFailure def test_unicode_b2a(self): # Unicode strings are not accepted by b2a_* functions. - for func in set(all_functions) - set(a2b_functions) | {'rledecode_hqx'}: + for func in set(all_functions) - set(a2b_functions): try: self.assertRaises(TypeError, getattr(binascii, func), "test") except Exception as err: @@ -403,9 +419,6 @@ class BinASCIITest(unittest.TestCase): MAX_ALL = 45 raw = self.rawdata[:MAX_ALL] for fa, fb in zip(a2b_functions, b2a_functions): - if fa == 'rledecode_hqx': - # Takes non-ASCII data - continue a2b = getattr(binascii, fa) b2a = getattr(binascii, fb) try: @@ -415,10 +428,6 @@ class BinASCIITest(unittest.TestCase): res = a2b(a) except Exception as err: self.fail("{}/{} conversion raises {!r}".format(fb, fa, err)) - if fb == 'b2a_hqx': - # b2a_hqx returns a tuple - res, _ = res - binary_res, _ = binary_res self.assertEqual(res, raw, "{}/{} conversion: " "{!r} != {!r}".format(fb, fa, res, raw)) self.assertEqual(res, binary_res) @@ -449,6 +458,14 @@ class BytearrayBinASCIITest(BinASCIITest): class MemoryviewBinASCIITest(BinASCIITest): type2test = memoryview +class ChecksumBigBufferTestCase(unittest.TestCase): + """bpo-38256 - check that inputs >=4 GiB are handled correctly.""" + + @bigmemtest(size=_4G + 4, memuse=1, dry_run=False) + def test_big_buffer(self, size): + data = b"nyan" * (_1G + 1) + self.assertEqual(binascii.crc32(data), 1044521549) + if __name__ == "__main__": unittest.main() From 04537b1993035f78aaab94b12edc31e03d10882e Mon Sep 17 00:00:00 2001 From: Moreal Date: Wed, 28 Dec 2022 05:25:23 +0900 Subject: [PATCH 4/5] Mark failed tests on test_binascii yet --- Lib/test/test_binascii.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 365b4d513..c138d55f1 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -116,6 +116,8 @@ class BinASCIITest(unittest.TestCase): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_base64_strict_mode(self): # Test base64 with strict mode on def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): From dc9e958bb3172416e6d3c7747f32778c02436e13 Mon Sep 17 00:00:00 2001 From: Moreal Date: Wed, 28 Dec 2022 05:26:12 +0900 Subject: [PATCH 5/5] Unmark fixed tests on test_binascii --- Lib/test/test_binascii.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index c138d55f1..ef5c2e79e 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -32,8 +32,6 @@ class BinASCIITest(unittest.TestCase): self.assertTrue(issubclass(binascii.Error, Exception)) self.assertTrue(issubclass(binascii.Incomplete, Exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_functions(self): # Check presence of all functions for name in all_functions: @@ -386,8 +384,6 @@ class BinASCIITest(unittest.TestCase): self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n') self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_string(self): # A test for SF bug #1022953. Make sure SystemError is not raised. empty = self.type2test(b'')