mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 10:44:36 +08:00
Implement Python 3.15 base64 APIs (#9085)
* Implement Python 3.15 base64 encoding and decoding Follow CPython 3.15.0 final d220409a13b77de7a6c30abe16ea90db01528edc for base64 options, alphabets, padding and canonical decoding. Keep reverse-table caches in traced module-native state and preserve module-bound callable semantics. Add bounded Python regressions without editing Lib. Assisted-by: OpenAI Codex:model identifier unavailable * Complete binascii codecs for canonical Python 3.15 tests Migrate Lib/test/test_binascii.py unchanged from CPython 3.15.0 final using scripts/update_lib and replace the superseded custom base64 snippet. Implement the base32, Base85, Ascii85 and hex ignorechars APIs required by the canonical suite. Share the existing module-bound original-key alphabet cache across base32/base64/base85 and preserve argument and buffer cleanup semantics. Keep the existing base64 kernels and VM state integration unchanged. Assisted-by: OpenAI Codex:model identifier unavailable
This commit is contained in:
1 parent
597a4221d2
commit
5f3daa5386
9 files changed
+2281
-134
No files matched your search
Generated
-7
@@ -320,12 +320,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -3761,7 +3755,6 @@ name = "rustpython-common"
|
||||
version = "0.6.1"
|
||||
dependencies = [
|
||||
"ascii",
|
||||
"base64",
|
||||
"bitflags 2.13.2",
|
||||
"blake2b_simd",
|
||||
"blake2s_simd",
|
||||
|
||||
@@ -198,7 +198,6 @@ phf = { version = "0.14.0", default-features = false, features = ["macros"]}
|
||||
adler32 = "1.2.0"
|
||||
approx = "0.5.1"
|
||||
ascii = { version = "1.1", default-features = false }
|
||||
base64 = "0.23"
|
||||
blake2b_simd = "1.0.4"
|
||||
blake2s_simd = "1.0.4"
|
||||
bitflags = "2.11.0"
|
||||
|
||||
Vendored
+1141
-40
File diff suppressed because it is too large.
Load diff
@@ -10,7 +10,7 @@ license.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
binascii = ["dep:base64", "dep:crc32fast"]
|
||||
binascii = ["dep:crc32fast"]
|
||||
bz2 = ["std", "dep:bzip2"]
|
||||
cjk-codecs = []
|
||||
inet = ["std"]
|
||||
@@ -39,7 +39,6 @@ rustpython-unicode = { workspace = true }
|
||||
rustpython-wtf8 = { workspace = true }
|
||||
|
||||
ascii = { workspace = true }
|
||||
base64 = { workspace = true, optional = true }
|
||||
bitflags = { workspace = true }
|
||||
blake2b_simd = { workspace = true, optional = true }
|
||||
blake2s_simd = { workspace = true, optional = true }
|
||||
|
||||
+126
-59
@@ -7,7 +7,6 @@
|
||||
//! the caller's responsibility (see `mod.rs`).
|
||||
|
||||
use alloc::{vec, vec::Vec};
|
||||
use base64::Engine;
|
||||
|
||||
/// The base64 pad character `=`.
|
||||
pub const PAD: u8 = 61u8;
|
||||
@@ -31,13 +30,17 @@ pub enum Error {
|
||||
TooLong,
|
||||
/// `a2b_base64`: base64 decoding failed.
|
||||
Base64(Base64DecodeError),
|
||||
/// The base64 output would exceed the maximum Python bytes length.
|
||||
TooMuchBase64Data,
|
||||
/// Reserving the transform output failed.
|
||||
Memory,
|
||||
}
|
||||
|
||||
/// Mirrors CPython's `binascii_a2b_base64_impl` (`Modules/binascii.c`) error
|
||||
/// paths one-for-one, so the caller can build the exact Python error message.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Base64DecodeError {
|
||||
/// A `=` at `quad_pos == 0, i == 0`, strict mode.
|
||||
/// Padding before any decoded output, in strict mode.
|
||||
LeadingPaddingNotAllowed,
|
||||
/// A `=` outside the padding-completion window, strict mode.
|
||||
ExcessPaddingNotAllowed,
|
||||
@@ -51,6 +54,10 @@ pub enum Base64DecodeError {
|
||||
InvalidLastSymbol { index: usize },
|
||||
/// `quad_pos != 0` and the trailing padding does not complete the quad.
|
||||
IncorrectPadding,
|
||||
/// Padding is forbidden when decoding unpadded input in strict mode.
|
||||
PaddingNotAllowed,
|
||||
/// The final symbol contains non-zero unused bits.
|
||||
NonZeroPaddingBits,
|
||||
}
|
||||
|
||||
const fn hex_nibble(n: u8) -> u8 {
|
||||
@@ -230,45 +237,53 @@ pub fn crc_hqx(bytes: &[u8], init: u32) -> u32 {
|
||||
crc
|
||||
}
|
||||
|
||||
/// `a2b_base64`. A line-by-line port of `binascii_a2b_base64_impl`
|
||||
/// (CPython `Modules/binascii.c`): padding characters within the completion
|
||||
/// window (`quad_pos >= 2 && quad_pos + pads <= 4`) are always skipped —
|
||||
/// even in strict mode — and non-strict mode additionally skips *every*
|
||||
/// pad and every non-alphabet byte outside that window. Errors therefore
|
||||
/// only fire in strict mode (mid-loop) or from the two post-loop length
|
||||
/// checks, which apply unconditionally.
|
||||
pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
|
||||
// Converts between ASCII and base-64 characters. The index of a given number yields the
|
||||
// number in ASCII while the value of said index yields the number in base-64. For example
|
||||
// "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0
|
||||
#[rustfmt::skip]
|
||||
const BASE64_TABLE: [i8; 256] = [
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
|
||||
52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */
|
||||
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
|
||||
15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
|
||||
-1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
|
||||
41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1,
|
||||
/// RFC 4648's standard base64 alphabet.
|
||||
pub const BASE64_ALPHABET: &[u8; 64] =
|
||||
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
|
||||
];
|
||||
/// Build the decoder table, with later duplicate symbols taking precedence.
|
||||
#[must_use]
|
||||
pub const fn base64_reverse_table(alphabet: &[u8; 64]) -> [u8; 256] {
|
||||
let mut table = [u8::MAX; 256];
|
||||
let mut index = 0;
|
||||
while index < 64 {
|
||||
table[alphabet[index] as usize] = index as u8;
|
||||
index += 1;
|
||||
}
|
||||
// Padding always takes precedence, even in a custom alphabet.
|
||||
table[PAD as usize] = 64;
|
||||
table
|
||||
}
|
||||
|
||||
let mut decoded: Vec<u8> = Vec::with_capacity(b.len().div_ceil(4) * 3);
|
||||
/// Options for CPython 3.15's base64 decoder.
|
||||
pub struct Base64DecodeOptions<'a> {
|
||||
pub strict_mode: bool,
|
||||
pub padded: bool,
|
||||
pub table: &'a [u8; 256],
|
||||
pub ignored: &'a [bool; 256],
|
||||
pub canonical: bool,
|
||||
}
|
||||
|
||||
let mut quad_pos: u32 = 0; // position in the quad, 0..=3
|
||||
let mut pads: u32 = 0;
|
||||
/// `a2b_base64`, following CPython 3.15's `Modules/binascii.c`.
|
||||
pub fn a2b_base64(b: &[u8], options: Base64DecodeOptions<'_>) -> Result<Vec<u8>, Error> {
|
||||
let Base64DecodeOptions {
|
||||
strict_mode,
|
||||
padded,
|
||||
table,
|
||||
ignored,
|
||||
canonical,
|
||||
} = options;
|
||||
|
||||
let mut decoded: Vec<u8> = Vec::new();
|
||||
decoded
|
||||
.try_reserve_exact(b.len().div_ceil(4) * 3)
|
||||
.map_err(|_| Error::Memory)?;
|
||||
|
||||
let mut quad_pos: usize = 0; // position in the quad, 0..=3
|
||||
let mut pads: usize = 0;
|
||||
let mut left_char: u8 = 0;
|
||||
for (i, &el) in b.iter().enumerate() {
|
||||
if el == PAD {
|
||||
for &el in b {
|
||||
if padded && el == PAD {
|
||||
pads += 1;
|
||||
if quad_pos >= 2 && quad_pos + pads <= 4 {
|
||||
continue;
|
||||
@@ -276,7 +291,7 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
|
||||
// RFC 4648 section-3.3: implementations MAY ignore a pad
|
||||
// character present before the end of the encoded data, and
|
||||
// MAY ignore excess pad characters. Non-strict mode does.
|
||||
if !strict_mode {
|
||||
if !strict_mode || ignored[PAD as usize] {
|
||||
continue;
|
||||
}
|
||||
if quad_pos == 1 {
|
||||
@@ -284,22 +299,26 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
|
||||
// `quad_pos == 1` check below.
|
||||
break;
|
||||
}
|
||||
return Err(Error::Base64(if quad_pos == 0 && i == 0 {
|
||||
return Err(Error::Base64(if quad_pos == 0 && decoded.is_empty() {
|
||||
Base64DecodeError::LeadingPaddingNotAllowed
|
||||
} else {
|
||||
Base64DecodeError::ExcessPaddingNotAllowed
|
||||
}));
|
||||
}
|
||||
|
||||
let binary_char = BASE64_TABLE[el as usize];
|
||||
if binary_char < 0 {
|
||||
if strict_mode {
|
||||
return Err(Error::Base64(Base64DecodeError::OnlyBase64DataAllowed));
|
||||
let binary_char = table[el as usize];
|
||||
if binary_char >= 64 {
|
||||
if strict_mode && !ignored[el as usize] {
|
||||
return Err(Error::Base64(if el == PAD {
|
||||
Base64DecodeError::PaddingNotAllowed
|
||||
} else {
|
||||
Base64DecodeError::OnlyBase64DataAllowed
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if pads > 0 && strict_mode {
|
||||
if pads > 0 && strict_mode && !ignored[PAD as usize] {
|
||||
return Err(Error::Base64(if quad_pos + pads == 4 {
|
||||
Base64DecodeError::ExcessDataAfterPadding
|
||||
} else {
|
||||
@@ -312,21 +331,21 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
|
||||
match quad_pos {
|
||||
0 => {
|
||||
quad_pos = 1;
|
||||
left_char = binary_char as u8;
|
||||
left_char = binary_char;
|
||||
}
|
||||
1 => {
|
||||
quad_pos = 2;
|
||||
decoded.push((left_char << 2) | (binary_char >> 4) as u8);
|
||||
left_char = (binary_char & 0x0f) as u8;
|
||||
decoded.push((left_char << 2) | (binary_char >> 4));
|
||||
left_char = binary_char & 0x0f;
|
||||
}
|
||||
2 => {
|
||||
quad_pos = 3;
|
||||
decoded.push((left_char << 4) | (binary_char >> 2) as u8);
|
||||
left_char = (binary_char & 0x03) as u8;
|
||||
decoded.push((left_char << 4) | (binary_char >> 2));
|
||||
left_char = binary_char & 0x03;
|
||||
}
|
||||
3 => {
|
||||
quad_pos = 0;
|
||||
decoded.push((left_char << 6) | binary_char as u8);
|
||||
decoded.push((left_char << 6) | binary_char);
|
||||
left_char = 0;
|
||||
}
|
||||
_ => unsafe {
|
||||
@@ -344,24 +363,72 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
|
||||
}));
|
||||
}
|
||||
|
||||
if quad_pos != 0 && quad_pos + pads < 4 {
|
||||
if padded && quad_pos != 0 && quad_pos + pads < 4 {
|
||||
return Err(Error::Base64(Base64DecodeError::IncorrectPadding));
|
||||
}
|
||||
|
||||
if canonical && left_char != 0 {
|
||||
return Err(Error::Base64(Base64DecodeError::NonZeroPaddingBits));
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
|
||||
/// `b2a_base64`.
|
||||
#[must_use]
|
||||
pub fn b2a_base64(data: &[u8], newline: bool) -> Vec<u8> {
|
||||
// https://stackoverflow.com/questions/63916821
|
||||
let mut encoded = base64::engine::general_purpose::STANDARD
|
||||
.encode(data)
|
||||
.into_bytes();
|
||||
/// `b2a_base64`, including CPython 3.15's wrapping and custom alphabets.
|
||||
pub fn b2a_base64(
|
||||
data: &[u8],
|
||||
padded: bool,
|
||||
wrapcol: usize,
|
||||
newline: bool,
|
||||
alphabet: &[u8; 64],
|
||||
) -> Result<Vec<u8>, Error> {
|
||||
let pads = (3 - data.len() % 3) % 3;
|
||||
let mut size = data
|
||||
.len()
|
||||
.div_ceil(3)
|
||||
.checked_mul(4)
|
||||
.ok_or(Error::TooMuchBase64Data)?;
|
||||
if !padded {
|
||||
size -= pads;
|
||||
}
|
||||
let width = if wrapcol == 0 {
|
||||
usize::MAX
|
||||
} else {
|
||||
wrapcol.max(4) / 4 * 4
|
||||
};
|
||||
let newlines = size.saturating_sub(1) / width;
|
||||
size = size
|
||||
.checked_add(newlines)
|
||||
.and_then(|n| n.checked_add(usize::from(newline)))
|
||||
.filter(|&n| isize::try_from(n).is_ok())
|
||||
.ok_or(Error::TooMuchBase64Data)?;
|
||||
let mut encoded = Vec::new();
|
||||
encoded.try_reserve_exact(size).map_err(|_| Error::Memory)?;
|
||||
let mut column = 0;
|
||||
let mut emit = |symbol| {
|
||||
if column == width {
|
||||
encoded.push(b'\n');
|
||||
column = 0;
|
||||
}
|
||||
encoded.push(symbol);
|
||||
column += 1;
|
||||
};
|
||||
for chunk in data.chunks(3) {
|
||||
let value = (u32::from(chunk[0]) << 16)
|
||||
| (u32::from(*chunk.get(1).unwrap_or(&0)) << 8)
|
||||
| u32::from(*chunk.get(2).unwrap_or(&0));
|
||||
for index in 0..=chunk.len() {
|
||||
emit(alphabet[((value >> (18 - index * 6)) & 63) as usize]);
|
||||
}
|
||||
if padded {
|
||||
for _ in chunk.len() + 1..4 {
|
||||
emit(PAD);
|
||||
}
|
||||
}
|
||||
}
|
||||
if newline {
|
||||
encoded.push(b'\n');
|
||||
}
|
||||
encoded
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
|
||||
+551
-21
@@ -1,4 +1,6 @@
|
||||
// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode
|
||||
// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode ABCDEFGHIJKLMNPQRSTUVXYZ abcdefhijklmpqr
|
||||
|
||||
mod base_n;
|
||||
|
||||
pub(super) use decl::crc32;
|
||||
pub(crate) use decl::module_def;
|
||||
@@ -8,14 +10,138 @@ use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef};
|
||||
|
||||
#[pymodule(name = "binascii")]
|
||||
mod decl {
|
||||
use super::new_binascii_error;
|
||||
use super::{base_n, new_binascii_error};
|
||||
use crate::vm::{
|
||||
PyResult, VirtualMachine,
|
||||
builtins::PyTypeRef,
|
||||
function::{ArgAsciiBuffer, ArgBytesLike, ArgIndex, OptionalArg},
|
||||
AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine,
|
||||
builtins::{PyBytes, PyBytesRef, PyDict, PyModule, PyTypeRef},
|
||||
function::{
|
||||
ArgAsciiBuffer, ArgBytesLike, ArgIndex, ArgIntoBool, ItemDoc, OptionalArg, PyMethodDef,
|
||||
PyMethodFlags,
|
||||
},
|
||||
};
|
||||
use rustpython_common::binascii;
|
||||
|
||||
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
|
||||
__module_exec(vm, module);
|
||||
module.set_native_state(vm.ctx.new_dict().into());
|
||||
const DECODE: &PyMethodDef = &PyMethodDef::new_const(
|
||||
"a2b_base64",
|
||||
a2b_base64,
|
||||
PyMethodFlags::empty(),
|
||||
ItemDoc::static_text(
|
||||
"a2b_base64($module, data, /, *, strict_mode=<unrepresentable>, padded=True, alphabet=BASE64_ALPHABET, ignorechars=<unrepresentable>, canonical=False)\n--\n\nDecode a line of base64 data.",
|
||||
),
|
||||
);
|
||||
const DECODE32: &PyMethodDef = &PyMethodDef::new_const(
|
||||
"a2b_base32",
|
||||
a2b_base32,
|
||||
PyMethodFlags::empty(),
|
||||
ItemDoc::static_text(
|
||||
"a2b_base32($module, data, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of base32 data.",
|
||||
),
|
||||
);
|
||||
const DECODE85: &PyMethodDef = &PyMethodDef::new_const(
|
||||
"a2b_base85",
|
||||
a2b_base85,
|
||||
PyMethodFlags::empty(),
|
||||
ItemDoc::static_text(
|
||||
"a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of Base85 data.",
|
||||
),
|
||||
);
|
||||
for (name, definition) in [
|
||||
("a2b_base64", DECODE),
|
||||
("a2b_base32", DECODE32),
|
||||
("a2b_base85", DECODE85),
|
||||
] {
|
||||
let function = definition.build_bound_function(&vm.ctx, module.to_owned().into());
|
||||
function
|
||||
.as_object()
|
||||
.set_attr("__module__", vm.ctx.new_str("binascii"), vm)?;
|
||||
module.dict().set_item(name, function.into(), vm)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reverse_table(
|
||||
module: &Py<PyModule>,
|
||||
alphabet: &PyBytesRef,
|
||||
size: usize,
|
||||
padchar: Option<u8>,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<[u8; 256]> {
|
||||
let bytes = alphabet.as_bytes();
|
||||
if bytes.len() != size {
|
||||
return Err(vm.new_value_error(format!("alphabet must have length {size}")));
|
||||
}
|
||||
let cache = module
|
||||
.native_state()
|
||||
.ok_or_else(|| vm.new_runtime_error("Missing binascii state"))?
|
||||
.downcast::<PyDict>()
|
||||
.map_err(|_| vm.new_runtime_error("Broken binascii cache"))?;
|
||||
// The actual bytes object is the key: subclass hash/equality callbacks
|
||||
// are part of CPython's cache lookup and insertion semantics.
|
||||
if let Some(value) = cache.get_item_opt(alphabet.as_object(), vm)? {
|
||||
return value
|
||||
.downcast_ref::<PyBytes>()
|
||||
.and_then(|value| value.as_bytes().try_into().ok())
|
||||
.ok_or_else(|| vm.new_runtime_error("Broken binascii cache"));
|
||||
}
|
||||
let table = base_n::reverse_table(bytes, padchar);
|
||||
cache.set_item(
|
||||
alphabet.as_object(),
|
||||
vm.ctx.new_bytes(table.to_vec()).into(),
|
||||
vm,
|
||||
)?;
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
#[pyattr(name = "CRYPT_ALPHABET")]
|
||||
fn crypt_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx
|
||||
.new_bytes(b"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "UU_ALPHABET")]
|
||||
fn uu_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(
|
||||
b" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_".to_vec(),
|
||||
)
|
||||
}
|
||||
|
||||
#[pyattr(name = "BINHEX_ALPHABET")]
|
||||
fn binhex_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(
|
||||
b"!\"#$%&'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr".to_vec(),
|
||||
)
|
||||
}
|
||||
|
||||
#[pyattr(name = "BASE85_ALPHABET")]
|
||||
fn base85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "ASCII85_ALPHABET")]
|
||||
fn ascii85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "Z85_ALPHABET")]
|
||||
fn z85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "BASE32_ALPHABET")]
|
||||
fn base32_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx
|
||||
.new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "BASE32HEX_ALPHABET")]
|
||||
fn base32hex_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx
|
||||
.new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUV".to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "Error", once)]
|
||||
pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef {
|
||||
vm.ctx.new_exception_type(
|
||||
@@ -54,12 +180,24 @@ mod decl {
|
||||
Ok(data.with_ref(|bytes| binascii::hexlify(bytes, sep, bytes_per_sep)))
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct UnhexlifyArgs {
|
||||
#[pyarg(positional)]
|
||||
hexstr: ArgAsciiBuffer,
|
||||
#[pyarg(named, optional, py_default = "b''")]
|
||||
ignorechars: OptionalArg<ArgBytesLike>,
|
||||
}
|
||||
|
||||
#[pyfunction(name = "a2b_hex")]
|
||||
#[pyfunction]
|
||||
fn unhexlify(hexstr: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
hexstr
|
||||
.with_ref(binascii::unhexlify)
|
||||
.map_err(|e| new_binascii_error(e, vm))
|
||||
fn unhexlify(args: UnhexlifyArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let ignorechars = ignored_argument(&args.ignorechars);
|
||||
let result = args
|
||||
.hexstr
|
||||
.with_ref(|data| base_n::decode_hex(data, &ignorechars));
|
||||
drop(args.hexstr);
|
||||
drop(args.ignorechars);
|
||||
base_n_result(result, vm)
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
@@ -86,30 +224,418 @@ mod decl {
|
||||
data.with_ref(|bytes| binascii::crc_hqx(bytes, crc.into_int_ref().as_u32_mask()))
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct NewlineArg {
|
||||
#[pyarg(named, default = true)]
|
||||
newline: bool,
|
||||
#[pyattr(name = "BASE64_ALPHABET")]
|
||||
fn base64_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx.new_bytes(binascii::BASE64_ALPHABET.to_vec())
|
||||
}
|
||||
|
||||
#[pyattr(name = "URLSAFE_BASE64_ALPHABET")]
|
||||
fn urlsafe_base64_alphabet(vm: &VirtualMachine) -> PyBytesRef {
|
||||
vm.ctx
|
||||
.new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".to_vec())
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct A2bBase64Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgAsciiBuffer,
|
||||
#[pyarg(named, default)]
|
||||
strict_mode: bool,
|
||||
#[pyarg(named, optional)]
|
||||
strict_mode: OptionalArg<ArgIntoBool>,
|
||||
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
|
||||
padded: ArgIntoBool,
|
||||
#[pyarg(named, optional, py_default = "BASE64_ALPHABET")]
|
||||
alphabet: OptionalArg<PyBytesRef>,
|
||||
#[pyarg(named, optional)]
|
||||
ignorechars: OptionalArg<ArgBytesLike>,
|
||||
#[pyarg(named, optional, py_default = "False")]
|
||||
canonical: OptionalArg<PyObjectRef>,
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let A2bBase64Args { data, strict_mode } = args;
|
||||
data.with_ref(|b| binascii::a2b_base64(b, strict_mode))
|
||||
fn a2b_base64(
|
||||
module: PyRef<PyModule>,
|
||||
args: A2bBase64Args,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Vec<u8>> {
|
||||
let A2bBase64Args {
|
||||
data,
|
||||
strict_mode,
|
||||
padded,
|
||||
alphabet,
|
||||
ignorechars,
|
||||
canonical,
|
||||
} = args;
|
||||
let strict_mode = match strict_mode {
|
||||
OptionalArg::Present(value) => value.into_bool(),
|
||||
OptionalArg::Missing => matches!(&ignorechars, OptionalArg::Present(_)),
|
||||
};
|
||||
let result = (|| {
|
||||
// Convert the last Clinic argument while both acquired buffers are alive.
|
||||
let canonical = match canonical {
|
||||
OptionalArg::Present(value) => value.try_to_bool(vm)?,
|
||||
OptionalArg::Missing => false,
|
||||
};
|
||||
const DEFAULT_TABLE: [u8; 256] =
|
||||
binascii::base64_reverse_table(binascii::BASE64_ALPHABET);
|
||||
let table = match &alphabet {
|
||||
OptionalArg::Present(value) => reverse_table(&module, value, 64, Some(b'='), vm)?,
|
||||
OptionalArg::Missing => DEFAULT_TABLE,
|
||||
};
|
||||
let mut ignored = [false; 256];
|
||||
if strict_mode && let OptionalArg::Present(value) = &ignorechars {
|
||||
value.with_ref(|bytes| {
|
||||
for &symbol in bytes {
|
||||
ignored[symbol as usize] = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
data.with_ref(|bytes| {
|
||||
binascii::a2b_base64(
|
||||
bytes,
|
||||
binascii::Base64DecodeOptions {
|
||||
strict_mode,
|
||||
padded: padded.into_bool(),
|
||||
table: &table,
|
||||
ignored: &ignored,
|
||||
canonical,
|
||||
},
|
||||
)
|
||||
})
|
||||
.map_err(|e| new_binascii_error(e, vm))
|
||||
})();
|
||||
// Match Argument Clinic cleanup order, including failures in __bool__.
|
||||
drop(data);
|
||||
drop(ignorechars);
|
||||
result
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct B2aBase64Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgBytesLike,
|
||||
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
|
||||
padded: ArgIntoBool,
|
||||
#[pyarg(named, default = 0)]
|
||||
wrapcol: usize,
|
||||
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
|
||||
newline: ArgIntoBool,
|
||||
#[pyarg(named, optional, py_default = "BASE64_ALPHABET")]
|
||||
alphabet: OptionalArg<ArgBytesLike>,
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec<u8> {
|
||||
data.with_ref(|bytes| binascii::b2a_base64(bytes, newline))
|
||||
fn b2a_base64(args: B2aBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let B2aBase64Args {
|
||||
data,
|
||||
padded,
|
||||
wrapcol,
|
||||
newline,
|
||||
alphabet,
|
||||
} = args;
|
||||
let copy_alphabet = |bytes: &[u8]| -> PyResult<[u8; 64]> {
|
||||
bytes
|
||||
.try_into()
|
||||
.map_err(|_| vm.new_value_error("alphabet must have length 64"))
|
||||
};
|
||||
let alphabet_bytes = match &alphabet {
|
||||
OptionalArg::Present(value) => value.with_ref(copy_alphabet),
|
||||
OptionalArg::Missing => Ok(*binascii::BASE64_ALPHABET),
|
||||
};
|
||||
let result = alphabet_bytes.and_then(|alphabet_bytes| {
|
||||
data.with_ref(|bytes| {
|
||||
binascii::b2a_base64(
|
||||
bytes,
|
||||
padded.into_bool(),
|
||||
wrapcol,
|
||||
newline.into_bool(),
|
||||
&alphabet_bytes,
|
||||
)
|
||||
})
|
||||
.map_err(|e| new_binascii_error(e, vm))
|
||||
});
|
||||
drop(data);
|
||||
drop(alphabet);
|
||||
result
|
||||
}
|
||||
|
||||
fn base_n_result(
|
||||
result: Result<Vec<u8>, base_n::Error>,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Vec<u8>> {
|
||||
result.map_err(|error| match error {
|
||||
base_n::Error::InvalidAlphabet(size) => {
|
||||
vm.new_value_error(format!("alphabet must have length {size}"))
|
||||
}
|
||||
base_n::Error::Codec(message) => vm.new_exception_msg(error_type(vm), message.into()),
|
||||
base_n::Error::Memory => vm.no_memory_error(),
|
||||
})
|
||||
}
|
||||
|
||||
fn encoder_alphabet(
|
||||
argument: &OptionalArg<ArgBytesLike>,
|
||||
default: &[u8],
|
||||
) -> Result<Vec<u8>, base_n::Error> {
|
||||
match argument {
|
||||
OptionalArg::Present(value) => {
|
||||
value.with_ref(|bytes| base_n::copy_alphabet(bytes, default.len()))
|
||||
}
|
||||
OptionalArg::Missing => base_n::copy_alphabet(default, default.len()),
|
||||
}
|
||||
}
|
||||
|
||||
fn ignored_argument(argument: &OptionalArg<ArgBytesLike>) -> [bool; 256] {
|
||||
match argument {
|
||||
OptionalArg::Present(value) => value.with_ref(base_n::ignored),
|
||||
OptionalArg::Missing => [false; 256],
|
||||
}
|
||||
}
|
||||
|
||||
fn canonical_argument(
|
||||
argument: &OptionalArg<PyObjectRef>,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<bool> {
|
||||
match argument {
|
||||
OptionalArg::Present(value) => value.try_to_bool(vm),
|
||||
OptionalArg::Missing => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct B2aBase32Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgBytesLike,
|
||||
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
|
||||
padded: ArgIntoBool,
|
||||
#[pyarg(named, default = 0)]
|
||||
wrapcol: usize,
|
||||
#[pyarg(named, optional, py_default = "BASE32_ALPHABET")]
|
||||
alphabet: OptionalArg<ArgBytesLike>,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct A2bBase32Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgAsciiBuffer,
|
||||
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
|
||||
padded: ArgIntoBool,
|
||||
#[pyarg(named, optional, py_default = "BASE32_ALPHABET")]
|
||||
alphabet: OptionalArg<PyBytesRef>,
|
||||
#[pyarg(named, optional, py_default = "b''")]
|
||||
ignorechars: OptionalArg<ArgBytesLike>,
|
||||
#[pyarg(named, optional, py_default = "False")]
|
||||
canonical: OptionalArg<PyObjectRef>,
|
||||
}
|
||||
|
||||
fn a2b_base32(
|
||||
module: PyRef<PyModule>,
|
||||
args: A2bBase32Args,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Vec<u8>> {
|
||||
let A2bBase32Args {
|
||||
data,
|
||||
padded,
|
||||
alphabet,
|
||||
ignorechars,
|
||||
canonical,
|
||||
} = args;
|
||||
let result = (|| {
|
||||
// This is the last Clinic conversion. Keep acquired exports alive
|
||||
// and reach ordered cleanup even when __bool__ raises.
|
||||
let canonical = canonical_argument(&canonical, vm)?;
|
||||
let table = match &alphabet {
|
||||
OptionalArg::Present(value) => reverse_table(&module, value, 32, Some(b'='), vm)?,
|
||||
OptionalArg::Missing => base_n::reverse_table(base_n::BASE32, Some(b'=')),
|
||||
};
|
||||
let ignore = ignored_argument(&ignorechars);
|
||||
let result = data.with_ref(|data| {
|
||||
base_n::decode32(data, &table, &ignore, padded.into_bool(), canonical)
|
||||
});
|
||||
base_n_result(result, vm)
|
||||
})();
|
||||
// Match Argument Clinic cleanup: release data before auxiliary exports.
|
||||
drop(data);
|
||||
drop(ignorechars);
|
||||
result
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn b2a_base32(args: B2aBase32Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let B2aBase32Args {
|
||||
data,
|
||||
padded,
|
||||
wrapcol,
|
||||
alphabet,
|
||||
} = args;
|
||||
let result = encoder_alphabet(&alphabet, base_n::BASE32).and_then(|alphabet_bytes| {
|
||||
data.with_ref(|data| {
|
||||
base_n::encode32(data, padded.into_bool(), wrapcol, &alphabet_bytes)
|
||||
})
|
||||
});
|
||||
drop(data);
|
||||
drop(alphabet);
|
||||
base_n_result(result, vm)
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct B2aBase85Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgBytesLike,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
pad: ArgIntoBool,
|
||||
#[pyarg(named, default = 0)]
|
||||
wrapcol: usize,
|
||||
#[pyarg(named, optional, py_default = "BASE85_ALPHABET")]
|
||||
alphabet: OptionalArg<ArgBytesLike>,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct A2bBase85Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgAsciiBuffer,
|
||||
#[pyarg(named, optional, py_default = "BASE85_ALPHABET")]
|
||||
alphabet: OptionalArg<PyBytesRef>,
|
||||
#[pyarg(named, optional, py_default = "b''")]
|
||||
ignorechars: OptionalArg<ArgBytesLike>,
|
||||
#[pyarg(named, optional, py_default = "False")]
|
||||
canonical: OptionalArg<PyObjectRef>,
|
||||
}
|
||||
|
||||
fn a2b_base85(
|
||||
module: PyRef<PyModule>,
|
||||
args: A2bBase85Args,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Vec<u8>> {
|
||||
let A2bBase85Args {
|
||||
data,
|
||||
alphabet,
|
||||
ignorechars,
|
||||
canonical,
|
||||
} = args;
|
||||
let result = (|| {
|
||||
// This is the last Clinic conversion. Keep acquired exports alive
|
||||
// and reach ordered cleanup even when __bool__ raises.
|
||||
let canonical = canonical_argument(&canonical, vm)?;
|
||||
let table = match &alphabet {
|
||||
OptionalArg::Present(value) => reverse_table(&module, value, 85, None, vm)?,
|
||||
OptionalArg::Missing => base_n::reverse_table(base_n::BASE85, None),
|
||||
};
|
||||
let ignore = ignored_argument(&ignorechars);
|
||||
let result = data.with_ref(|data| {
|
||||
base_n::decode85(data, &table, &ignore, canonical, false, false, false)
|
||||
});
|
||||
base_n_result(result, vm)
|
||||
})();
|
||||
drop(data);
|
||||
drop(ignorechars);
|
||||
result
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn b2a_base85(args: B2aBase85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let B2aBase85Args {
|
||||
data,
|
||||
pad,
|
||||
wrapcol,
|
||||
alphabet,
|
||||
} = args;
|
||||
let result = encoder_alphabet(&alphabet, base_n::BASE85).and_then(|alphabet_bytes| {
|
||||
data.with_ref(|data| {
|
||||
base_n::encode85(
|
||||
data,
|
||||
pad.into_bool(),
|
||||
wrapcol,
|
||||
&alphabet_bytes,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
})
|
||||
});
|
||||
drop(data);
|
||||
drop(alphabet);
|
||||
base_n_result(result, vm)
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct B2aAscii85Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgBytesLike,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
foldspaces: ArgIntoBool,
|
||||
#[pyarg(named, default = 0)]
|
||||
wrapcol: usize,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
pad: ArgIntoBool,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
adobe: ArgIntoBool,
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
struct A2bAscii85Args {
|
||||
#[pyarg(positional)]
|
||||
data: ArgAsciiBuffer,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
foldspaces: ArgIntoBool,
|
||||
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
|
||||
adobe: ArgIntoBool,
|
||||
#[pyarg(named, optional, py_default = "b''")]
|
||||
ignorechars: OptionalArg<ArgBytesLike>,
|
||||
#[pyarg(named, optional, py_default = "False")]
|
||||
canonical: OptionalArg<PyObjectRef>,
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn a2b_ascii85(args: A2bAscii85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let A2bAscii85Args {
|
||||
data,
|
||||
foldspaces,
|
||||
adobe,
|
||||
ignorechars,
|
||||
canonical,
|
||||
} = args;
|
||||
let result = (|| {
|
||||
// This is the last Clinic conversion. Keep acquired exports alive
|
||||
// and reach ordered cleanup even when __bool__ raises.
|
||||
let canonical = canonical_argument(&canonical, vm)?;
|
||||
let ignore = ignored_argument(&ignorechars);
|
||||
let result = data.with_ref(|data| {
|
||||
base_n::decode85(
|
||||
data,
|
||||
&base_n::reverse_table(base_n::ASCII85, None),
|
||||
&ignore,
|
||||
canonical,
|
||||
true,
|
||||
foldspaces.into_bool(),
|
||||
adobe.into_bool(),
|
||||
)
|
||||
});
|
||||
base_n_result(result, vm)
|
||||
})();
|
||||
drop(data);
|
||||
drop(ignorechars);
|
||||
result
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn b2a_ascii85(args: B2aAscii85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
|
||||
let B2aAscii85Args {
|
||||
data,
|
||||
foldspaces,
|
||||
wrapcol,
|
||||
pad,
|
||||
adobe,
|
||||
} = args;
|
||||
let result = data.with_ref(|data| {
|
||||
base_n::encode85(
|
||||
data,
|
||||
pad.into_bool(),
|
||||
wrapcol,
|
||||
base_n::ASCII85,
|
||||
true,
|
||||
foldspaces.into_bool(),
|
||||
adobe.into_bool(),
|
||||
)
|
||||
});
|
||||
base_n_result(result, vm)
|
||||
}
|
||||
|
||||
#[derive(FromArgs)]
|
||||
@@ -182,6 +708,8 @@ fn new_binascii_error(error: Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
|
||||
Error::TrailingGarbage => "Trailing garbage".to_owned(),
|
||||
Error::TooLong => "At most 45 bytes at once".to_owned(),
|
||||
Error::Base64(e) => base64_message(e),
|
||||
Error::TooMuchBase64Data => "Too much data for base64".to_owned(),
|
||||
Error::Memory => return vm.no_memory_error(),
|
||||
};
|
||||
vm.new_exception_msg(decl::error_type(vm), message.into())
|
||||
}
|
||||
@@ -199,5 +727,7 @@ fn base64_message(error: Base64DecodeError) -> String {
|
||||
"Invalid base64-encoded string: number of data characters ({index}) cannot be 1 more than a multiple of 4"
|
||||
),
|
||||
Base64DecodeError::IncorrectPadding => "Incorrect padding".to_owned(),
|
||||
Base64DecodeError::PaddingNotAllowed => "Padding not allowed".to_owned(),
|
||||
Base64DecodeError::NonZeroPaddingBits => "Non-zero padding bits".to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
// Scalar native equivalents of CPython 3.15 Modules/binascii.c base-N codecs.
|
||||
|
||||
pub(super) const BASE32: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
||||
pub(super) const BASE85: &[u8] =
|
||||
b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
|
||||
pub(super) const ASCII85: &[u8] =
|
||||
b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum Error {
|
||||
InvalidAlphabet(usize),
|
||||
Codec(String),
|
||||
Memory,
|
||||
}
|
||||
|
||||
type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
fn codec(message: impl Into<String>) -> Error {
|
||||
Error::Codec(message.into())
|
||||
}
|
||||
|
||||
fn buffer(size: usize, too_large: &str) -> Result<Vec<u8>> {
|
||||
if size > isize::MAX as usize {
|
||||
return Err(codec(too_large));
|
||||
}
|
||||
let mut result = Vec::new();
|
||||
result.try_reserve_exact(size).map_err(|_| Error::Memory)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn checked_alphabet(alphabet: &[u8], size: usize) -> Result<&[u8]> {
|
||||
if alphabet.len() != size {
|
||||
return Err(Error::InvalidAlphabet(size));
|
||||
}
|
||||
Ok(alphabet)
|
||||
}
|
||||
|
||||
pub(super) fn copy_alphabet(alphabet: &[u8], size: usize) -> Result<Vec<u8>> {
|
||||
checked_alphabet(alphabet, size)?;
|
||||
let mut out = buffer(size, "Too large alphabet")?;
|
||||
out.extend_from_slice(alphabet);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(super) fn reverse_table(alphabet: &[u8], padchar: Option<u8>) -> [u8; 256] {
|
||||
let mut table = [u8::MAX; 256];
|
||||
for (digit, &ch) in alphabet.iter().enumerate() {
|
||||
// Duplicate characters are allowed; their last occurrence wins.
|
||||
table[usize::from(ch)] = digit as u8;
|
||||
}
|
||||
if let Some(padchar) = padchar {
|
||||
table[usize::from(padchar)] = alphabet.len() as u8;
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
pub(super) fn ignored(ignorechars: &[u8]) -> [bool; 256] {
|
||||
let mut table = [false; 256];
|
||||
for &ch in ignorechars {
|
||||
table[usize::from(ch)] = true;
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
fn rounded_width(width: usize, quantum: usize) -> usize {
|
||||
if width == 0 {
|
||||
0
|
||||
} else {
|
||||
width.max(quantum) / quantum * quantum
|
||||
}
|
||||
}
|
||||
|
||||
fn wrapped_size(size: usize, width: usize, newline: bool, too_large: &str) -> Result<usize> {
|
||||
let breaks = if width == 0 || size == 0 {
|
||||
0
|
||||
} else {
|
||||
(size - 1) / width
|
||||
};
|
||||
size.checked_add(breaks)
|
||||
.and_then(|n| n.checked_add(usize::from(newline)))
|
||||
.filter(|n| isize::try_from(*n).is_ok())
|
||||
.ok_or_else(|| codec(too_large))
|
||||
}
|
||||
|
||||
fn wrap(mut input: Vec<u8>, width: usize, newline: bool, too_large: &str) -> Result<Vec<u8>> {
|
||||
let size = wrapped_size(input.len(), width, newline, too_large)?;
|
||||
if width != 0 && input.len() > width {
|
||||
let mut out = buffer(size, too_large)?;
|
||||
for (index, chunk) in input.chunks(width).enumerate() {
|
||||
if index != 0 {
|
||||
out.push(b'\n');
|
||||
}
|
||||
out.extend_from_slice(chunk);
|
||||
}
|
||||
input = out;
|
||||
}
|
||||
if newline {
|
||||
input.try_reserve(1).map_err(|_| Error::Memory)?;
|
||||
input.push(b'\n');
|
||||
}
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
pub(super) fn encode32(
|
||||
data: &[u8],
|
||||
padded: bool,
|
||||
width: usize,
|
||||
alphabet: &[u8],
|
||||
) -> Result<Vec<u8>> {
|
||||
let (size, in_group, out_group, too_large) = (32, 5, 8, "Too much data for base32");
|
||||
let bits = 5;
|
||||
checked_alphabet(alphabet, size)?;
|
||||
let full_size = data
|
||||
.len()
|
||||
.div_ceil(in_group)
|
||||
.checked_mul(out_group)
|
||||
.ok_or_else(|| codec(too_large))?;
|
||||
let pads = (in_group - data.len() % in_group) % in_group * out_group / in_group;
|
||||
let encoded_size = full_size - if padded { 0 } else { pads };
|
||||
let width = rounded_width(width, out_group);
|
||||
wrapped_size(encoded_size, width, false, too_large)?;
|
||||
let mut out = buffer(encoded_size, too_large)?;
|
||||
let mut value = 0u32;
|
||||
let mut available = 0u32;
|
||||
let mask = (1u32 << bits) - 1;
|
||||
for &byte in data {
|
||||
value = (value << 8) | u32::from(byte);
|
||||
available += 8;
|
||||
while available >= bits {
|
||||
available -= bits;
|
||||
out.push(alphabet[((value >> available) & mask) as usize]);
|
||||
}
|
||||
value &= (1u32 << available) - 1;
|
||||
}
|
||||
if available != 0 {
|
||||
out.push(alphabet[((value << (bits - available)) & mask) as usize]);
|
||||
}
|
||||
if padded {
|
||||
out.resize(encoded_size, b'=');
|
||||
}
|
||||
wrap(out, width, false, too_large)
|
||||
}
|
||||
|
||||
pub(super) fn decode32(
|
||||
data: &[u8],
|
||||
table: &[u8; 256],
|
||||
ignorechars: &[bool; 256],
|
||||
padded: bool,
|
||||
canonical: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
let (size, group, decoded_group, name) = (32, 8, 5, "base32");
|
||||
let bits = 5;
|
||||
let ignore = ignorechars;
|
||||
let capacity = data
|
||||
.len()
|
||||
.div_ceil(group)
|
||||
.checked_mul(decoded_group)
|
||||
.ok_or_else(|| codec(format!("Too much {name} data")))?;
|
||||
let mut out = buffer(capacity, &format!("Too much {name} data"))?;
|
||||
let mut pos = 0usize;
|
||||
let mut pads = 0usize;
|
||||
let mut value = 0u32;
|
||||
let mut available = 0u32;
|
||||
for &ch in data {
|
||||
if padded && ch == b'=' {
|
||||
pads += 1;
|
||||
let allowed_pos = matches!(pos, 2 | 4 | 5 | 7);
|
||||
if allowed_pos && pos + pads <= group {
|
||||
continue;
|
||||
}
|
||||
if ignore[usize::from(b'=')] {
|
||||
continue;
|
||||
}
|
||||
if matches!(pos, 1 | 3 | 6) {
|
||||
break;
|
||||
}
|
||||
return Err(codec(if pos == 0 && out.is_empty() {
|
||||
"Leading padding not allowed"
|
||||
} else {
|
||||
"Excess padding not allowed"
|
||||
}));
|
||||
}
|
||||
let digit = table[usize::from(ch)];
|
||||
if usize::from(digit) >= size {
|
||||
if !ignore[usize::from(ch)] {
|
||||
return Err(codec(if ch == b'=' {
|
||||
"Padding not allowed".to_owned()
|
||||
} else {
|
||||
format!("Only {name} data is allowed")
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if pads != 0 && !ignore[usize::from(b'=')] {
|
||||
return Err(codec(if pos + pads == group {
|
||||
"Excess data after padding"
|
||||
} else {
|
||||
"Discontinuous padding not allowed"
|
||||
}));
|
||||
}
|
||||
value = (value << bits) | u32::from(digit);
|
||||
available += bits;
|
||||
if available >= 8 {
|
||||
available -= 8;
|
||||
out.push((value >> available) as u8);
|
||||
value &= (1u32 << available) - 1;
|
||||
}
|
||||
pos += 1;
|
||||
if pos == group {
|
||||
pos = 0;
|
||||
pads = 0;
|
||||
}
|
||||
}
|
||||
if matches!(pos, 1 | 3 | 6) {
|
||||
let count = out.len() / decoded_group * group + pos;
|
||||
return Err(codec(format!(
|
||||
"Invalid base32-encoded string: number of data characters ({count}) cannot be 1, 3, or 6 more than a multiple of 8"
|
||||
)));
|
||||
}
|
||||
if padded && pos != 0 && pos + pads < group {
|
||||
return Err(codec("Incorrect padding"));
|
||||
}
|
||||
if canonical && value != 0 {
|
||||
return Err(codec("Non-zero padding bits"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn digits85(mut value: u32, alphabet: &[u8]) -> [u8; 5] {
|
||||
let mut out = [0; 5];
|
||||
for slot in out.iter_mut().rev() {
|
||||
*slot = alphabet[(value % 85) as usize];
|
||||
value /= 85;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(super) fn encode85(
|
||||
data: &[u8],
|
||||
pad: bool,
|
||||
width: usize,
|
||||
alphabet: &[u8],
|
||||
ascii85: bool,
|
||||
foldspaces: bool,
|
||||
adobe: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
checked_alphabet(alphabet, 85)?;
|
||||
let too_large = if ascii85 {
|
||||
"Too much data for Ascii85"
|
||||
} else {
|
||||
"Too much data for Base85"
|
||||
};
|
||||
let full_size = data
|
||||
.len()
|
||||
.div_ceil(4)
|
||||
.checked_mul(5)
|
||||
.and_then(|n| n.checked_add(if adobe { 4 } else { 0 }))
|
||||
.ok_or_else(|| codec(too_large))?;
|
||||
let size = full_size
|
||||
- if !pad && !data.len().is_multiple_of(4) {
|
||||
4 - data.len() % 4
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let width = if ascii85 {
|
||||
if adobe && width == 1 { 2 } else { width }
|
||||
} else {
|
||||
rounded_width(width, 5)
|
||||
};
|
||||
wrapped_size(size, width, false, too_large)?;
|
||||
let mut out = buffer(size, too_large)?;
|
||||
if adobe {
|
||||
out.extend_from_slice(b"<~");
|
||||
}
|
||||
for chunk in data.chunks(4) {
|
||||
let mut block = [0; 4];
|
||||
block[..chunk.len()].copy_from_slice(chunk);
|
||||
let value = u32::from_be_bytes(block);
|
||||
if ascii85 && value == 0 && (pad || chunk.len() == 4) {
|
||||
out.push(b'z');
|
||||
} else if ascii85 && foldspaces && value == 0x20202020 && chunk.len() == 4 {
|
||||
out.push(b'y');
|
||||
} else {
|
||||
let digits = digits85(value, alphabet);
|
||||
let count = if pad || chunk.len() == 4 {
|
||||
5
|
||||
} else {
|
||||
chunk.len() + 1
|
||||
};
|
||||
out.extend_from_slice(&digits[..count]);
|
||||
}
|
||||
}
|
||||
if adobe {
|
||||
out.extend_from_slice(b"~>");
|
||||
}
|
||||
let mut out = wrap(out, width, false, too_large)?;
|
||||
if adobe && out.len() >= 3 && out[out.len() - 2] == b'\n' {
|
||||
let len = out.len();
|
||||
out[len - 3] = b'\n';
|
||||
out[len - 2] = b'~';
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(super) fn decode85(
|
||||
mut data: &[u8],
|
||||
table: &[u8; 256],
|
||||
ignorechars: &[bool; 256],
|
||||
canonical: bool,
|
||||
ascii85: bool,
|
||||
foldspaces: bool,
|
||||
adobe: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
if adobe {
|
||||
if !data.ends_with(b"~>") {
|
||||
return Err(codec("Ascii85 encoded byte sequences must end with b'~>'"));
|
||||
}
|
||||
data = &data[..data.len() - 2];
|
||||
if data.starts_with(b"<~") {
|
||||
data = &data[2..];
|
||||
}
|
||||
}
|
||||
let ignore = ignorechars;
|
||||
let shortcuts = if ascii85 {
|
||||
data.iter().filter(|ch| matches!(ch, b'y' | b'z')).count()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let size = (data.len() - shortcuts)
|
||||
.div_ceil(5)
|
||||
.checked_mul(4)
|
||||
.and_then(|n| shortcuts.checked_mul(4).and_then(|m| n.checked_add(m)))
|
||||
.ok_or_else(|| codec("Too much Ascii85 data"))?;
|
||||
let mut out = buffer(size, "Too much Ascii85 data")?;
|
||||
let mut value = 0u32;
|
||||
let mut pos = 0usize;
|
||||
let mut from_z = false;
|
||||
let mut index = 0usize;
|
||||
let mut tail_len = 4usize;
|
||||
while index < data.len() || pos != 0 {
|
||||
let (ch, digit) = if index < data.len() {
|
||||
let ch = data[index];
|
||||
(ch, table[usize::from(ch)])
|
||||
} else {
|
||||
tail_len -= 1;
|
||||
(0, 84)
|
||||
};
|
||||
if digit < 85 {
|
||||
value = value
|
||||
.checked_mul(85)
|
||||
.and_then(|n| n.checked_add(u32::from(digit)))
|
||||
.ok_or_else(|| {
|
||||
codec(if ascii85 {
|
||||
"Ascii85 overflow".to_owned()
|
||||
} else {
|
||||
format!("Base85 overflow in hunk starting at byte {}", index / 5 * 5)
|
||||
})
|
||||
})?;
|
||||
pos += 1;
|
||||
} else if ascii85 && (ch == b'z' || (ch == b'y' && foldspaces)) {
|
||||
if pos != 0 {
|
||||
return Err(codec(format!(
|
||||
"'{}' inside Ascii85 5-tuple",
|
||||
char::from(ch)
|
||||
)));
|
||||
}
|
||||
value = if ch == b'y' { 0x20202020 } else { 0 };
|
||||
from_z = ch == b'z';
|
||||
pos = 5;
|
||||
} else if !ignore[usize::from(ch)] {
|
||||
return Err(codec(if ascii85 {
|
||||
format!("Non-Ascii85 digit found: {}", char::from(ch))
|
||||
} else {
|
||||
format!("bad Base85 character at position {index}")
|
||||
}));
|
||||
}
|
||||
index += 1;
|
||||
if pos != 5 {
|
||||
continue;
|
||||
}
|
||||
if tail_len == 0 {
|
||||
return Err(codec(if ascii85 {
|
||||
"Incomplete Ascii85 group"
|
||||
} else {
|
||||
"Incomplete Base85 group"
|
||||
}));
|
||||
}
|
||||
out.extend_from_slice(&value.to_be_bytes()[..tail_len]);
|
||||
if canonical {
|
||||
if ascii85 && tail_len == 4 && value == 0 && !from_z {
|
||||
return Err(codec("Non-canonical encoding, use 'z' for all-zero groups"));
|
||||
}
|
||||
if tail_len < 4 {
|
||||
let pads = 4 - tail_len;
|
||||
let top = value >> (pads * 8) << (pads * 8);
|
||||
let divisor = 85u32.pow(pads as u32);
|
||||
if top / divisor != value / divisor {
|
||||
return Err(codec("Non-zero padding bits"));
|
||||
}
|
||||
}
|
||||
}
|
||||
pos = 0;
|
||||
value = 0;
|
||||
from_z = false;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(super) fn decode_hex(data: &[u8], ignorechars: &[bool; 256]) -> Result<Vec<u8>> {
|
||||
let ignore = ignorechars;
|
||||
let mut out = buffer(data.len() / 2, "Too much hexadecimal data")?;
|
||||
let mut high = None;
|
||||
for &ch in data {
|
||||
let digit = match ch {
|
||||
b'0'..=b'9' => ch - b'0',
|
||||
b'a'..=b'f' => ch - b'a' + 10,
|
||||
b'A'..=b'F' => ch - b'A' + 10,
|
||||
_ if ignore[usize::from(ch)] => continue,
|
||||
_ => return Err(codec("Non-hexadecimal digit found")),
|
||||
};
|
||||
if let Some(hi) = high.take() {
|
||||
out.push((hi << 4) | digit);
|
||||
} else {
|
||||
high = Some(digit);
|
||||
}
|
||||
}
|
||||
if high.is_some() {
|
||||
return Err(codec("Odd number of hexadecimal digits"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -140,7 +140,7 @@ impl Representable for PyNativeFunction {
|
||||
if let Some(bound) = zelf
|
||||
.zelf
|
||||
.as_ref()
|
||||
.filter(|b| !b.class().is(vm.ctx.types.module_type))
|
||||
.filter(|b| !b.class().is_subtype(vm.ctx.types.module_type))
|
||||
{
|
||||
Ok(format!(
|
||||
"<built-in method {} of {} object at {:#x}>",
|
||||
@@ -169,7 +169,7 @@ impl PyNativeFunction {
|
||||
fn __qualname__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult<PyStrRef> {
|
||||
let zelf = zelf.0;
|
||||
let qualname = if let Some(bound) = &zelf.zelf {
|
||||
if bound.class().is(vm.ctx.types.module_type) {
|
||||
if bound.class().is_subtype(vm.ctx.types.module_type) {
|
||||
return Ok(vm.ctx.intern_str(zelf.value.name).to_owned());
|
||||
}
|
||||
let prefix = if bound.class().is(vm.ctx.types.type_type) {
|
||||
|
||||
@@ -82,7 +82,7 @@ impl PyModuleDef {
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(module = false, name = "module")]
|
||||
#[pyclass(module = false, name = "module", traverse = "manual")]
|
||||
// The dict lives in the object extension, not the payload. The offset is
|
||||
// the dict cell at the front of that extension.
|
||||
#[pymember(name = "__dict__", offset = ::rustpython_vm::object::dict_member_offset())]
|
||||
@@ -90,12 +90,26 @@ impl PyModuleDef {
|
||||
pub struct PyModule {
|
||||
// PyObject *md_dict;
|
||||
pub def: Option<&'static PyModuleDef>,
|
||||
// state: Any
|
||||
native_state: crate::object::PyAtomicRef<Option<PyObject>>,
|
||||
// weaklist
|
||||
// for logging purposes after md_dict is cleared
|
||||
pub name: Option<&'static PyStrInterned>,
|
||||
}
|
||||
|
||||
// SAFETY: Native state is the only owned Python reference in the payload.
|
||||
// The module dictionary is visited separately by the object extension.
|
||||
unsafe impl crate::object::Traverse for PyModule {
|
||||
fn traverse(&self, tracer: &mut crate::object::TraverseFn<'_>) {
|
||||
self.native_state.traverse(tracer);
|
||||
}
|
||||
|
||||
fn clear(&mut self, out: &mut Vec<PyObjectRef>) {
|
||||
if let Some(state) = self.native_state.store(None) {
|
||||
out.push(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PyPayload for PyModule {
|
||||
#[inline]
|
||||
fn class(ctx: &Context) -> &'static Py<PyType> {
|
||||
@@ -119,6 +133,7 @@ impl PyModule {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
def: None,
|
||||
native_state: crate::object::PyAtomicRef::new_empty(),
|
||||
name: None,
|
||||
}
|
||||
}
|
||||
@@ -127,10 +142,22 @@ impl PyModule {
|
||||
pub const fn from_def(def: &'static PyModuleDef) -> Self {
|
||||
Self {
|
||||
def: Some(def),
|
||||
native_state: crate::object::PyAtomicRef::new_empty(),
|
||||
name: Some(def.name),
|
||||
}
|
||||
}
|
||||
|
||||
/// Interpreter-local native state, retained and traversed with the module.
|
||||
#[must_use]
|
||||
pub fn native_state(&self) -> Option<PyObjectRef> {
|
||||
self.native_state.load_owned()
|
||||
}
|
||||
|
||||
/// Replace native state without exposing it in the Python module dictionary.
|
||||
pub fn set_native_state(&self, state: PyObjectRef) {
|
||||
drop(self.native_state.store(Some(state)));
|
||||
}
|
||||
|
||||
pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py<Self>) {
|
||||
let doc = module.def.unwrap().doc.map(|doc| doc.to_owned());
|
||||
module.init_dict(module.name.unwrap(), doc, vm);
|
||||
|
||||
Reference in new issue
Block a user