Implement Python 3.15 base64 APIs (#9085)

* Implement Python 3.15 base64 encoding and decoding

Follow CPython 3.15.0 final d220409a13b77de7a6c30abe16ea90db01528edc for base64 options, alphabets, padding and canonical decoding. Keep reverse-table caches in traced module-native state and preserve module-bound callable semantics. Add bounded Python regressions without editing Lib.

Assisted-by: OpenAI Codex:model identifier unavailable

* Complete binascii codecs for canonical Python 3.15 tests

Migrate Lib/test/test_binascii.py unchanged from CPython 3.15.0 final using scripts/update_lib and replace the superseded custom base64 snippet. Implement the base32, Base85, Ascii85 and hex ignorechars APIs required by the canonical suite. Share the existing module-bound original-key alphabet cache across base32/base64/base85 and preserve argument and buffer cleanup semantics. Keep the existing base64 kernels and VM state integration unchanged.

Assisted-by: OpenAI Codex:model identifier unavailable
This commit is contained in:
youknowdot authored and GitHub committed 2026-10-12 16:12:24 +09:00
1 parent 597a4221d2
commit 5f3daa5386
9 files changed
+2281 -134

No files matched your search

Generated
-7
View File
@@ -320,12 +320,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
[[package]]
name = "base64"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64ct"
version = "1.8.3"
@@ -3761,7 +3755,6 @@ name = "rustpython-common"
version = "0.6.1"
dependencies = [
"ascii",
"base64",
"bitflags 2.13.2",
"blake2b_simd",
"blake2s_simd",
-1
View File
@@ -198,7 +198,6 @@ phf = { version = "0.14.0", default-features = false, features = ["macros"]}
adler32 = "1.2.0"
approx = "0.5.1"
ascii = { version = "1.1", default-features = false }
base64 = "0.23"
blake2b_simd = "1.0.4"
blake2s_simd = "1.0.4"
bitflags = "2.11.0"
+1141 -40
View File
File diff suppressed because it is too large. Load diff
+1 -2
View File
@@ -10,7 +10,7 @@ license.workspace = true
[features]
default = ["std"]
binascii = ["dep:base64", "dep:crc32fast"]
binascii = ["dep:crc32fast"]
bz2 = ["std", "dep:bzip2"]
cjk-codecs = []
inet = ["std"]
@@ -39,7 +39,6 @@ rustpython-unicode = { workspace = true }
rustpython-wtf8 = { workspace = true }
ascii = { workspace = true }
base64 = { workspace = true, optional = true }
bitflags = { workspace = true }
blake2b_simd = { workspace = true, optional = true }
blake2s_simd = { workspace = true, optional = true }
+126 -59
View File
@@ -7,7 +7,6 @@
//! the caller's responsibility (see `mod.rs`).
use alloc::{vec, vec::Vec};
use base64::Engine;
/// The base64 pad character `=`.
pub const PAD: u8 = 61u8;
@@ -31,13 +30,17 @@ pub enum Error {
TooLong,
/// `a2b_base64`: base64 decoding failed.
Base64(Base64DecodeError),
/// The base64 output would exceed the maximum Python bytes length.
TooMuchBase64Data,
/// Reserving the transform output failed.
Memory,
}
/// Mirrors CPython's `binascii_a2b_base64_impl` (`Modules/binascii.c`) error
/// paths one-for-one, so the caller can build the exact Python error message.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Base64DecodeError {
/// A `=` at `quad_pos == 0, i == 0`, strict mode.
/// Padding before any decoded output, in strict mode.
LeadingPaddingNotAllowed,
/// A `=` outside the padding-completion window, strict mode.
ExcessPaddingNotAllowed,
@@ -51,6 +54,10 @@ pub enum Base64DecodeError {
InvalidLastSymbol { index: usize },
/// `quad_pos != 0` and the trailing padding does not complete the quad.
IncorrectPadding,
/// Padding is forbidden when decoding unpadded input in strict mode.
PaddingNotAllowed,
/// The final symbol contains non-zero unused bits.
NonZeroPaddingBits,
}
const fn hex_nibble(n: u8) -> u8 {
@@ -230,45 +237,53 @@ pub fn crc_hqx(bytes: &[u8], init: u32) -> u32 {
crc
}
/// `a2b_base64`. A line-by-line port of `binascii_a2b_base64_impl`
/// (CPython `Modules/binascii.c`): padding characters within the completion
/// window (`quad_pos >= 2 && quad_pos + pads <= 4`) are always skipped —
/// even in strict mode — and non-strict mode additionally skips *every*
/// pad and every non-alphabet byte outside that window. Errors therefore
/// only fire in strict mode (mid-loop) or from the two post-loop length
/// checks, which apply unconditionally.
pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
// Converts between ASCII and base-64 characters. The index of a given number yields the
// number in ASCII while the value of said index yields the number in base-64. For example
// "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0
#[rustfmt::skip]
const BASE64_TABLE: [i8; 256] = [
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
-1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1,
/// RFC 4648's standard base64 alphabet.
pub const BASE64_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
];
/// Build the decoder table, with later duplicate symbols taking precedence.
#[must_use]
pub const fn base64_reverse_table(alphabet: &[u8; 64]) -> [u8; 256] {
let mut table = [u8::MAX; 256];
let mut index = 0;
while index < 64 {
table[alphabet[index] as usize] = index as u8;
index += 1;
}
// Padding always takes precedence, even in a custom alphabet.
table[PAD as usize] = 64;
table
}
let mut decoded: Vec<u8> = Vec::with_capacity(b.len().div_ceil(4) * 3);
/// Options for CPython 3.15's base64 decoder.
pub struct Base64DecodeOptions<'a> {
pub strict_mode: bool,
pub padded: bool,
pub table: &'a [u8; 256],
pub ignored: &'a [bool; 256],
pub canonical: bool,
}
let mut quad_pos: u32 = 0; // position in the quad, 0..=3
let mut pads: u32 = 0;
/// `a2b_base64`, following CPython 3.15's `Modules/binascii.c`.
pub fn a2b_base64(b: &[u8], options: Base64DecodeOptions<'_>) -> Result<Vec<u8>, Error> {
let Base64DecodeOptions {
strict_mode,
padded,
table,
ignored,
canonical,
} = options;
let mut decoded: Vec<u8> = Vec::new();
decoded
.try_reserve_exact(b.len().div_ceil(4) * 3)
.map_err(|_| Error::Memory)?;
let mut quad_pos: usize = 0; // position in the quad, 0..=3
let mut pads: usize = 0;
let mut left_char: u8 = 0;
for (i, &el) in b.iter().enumerate() {
if el == PAD {
for &el in b {
if padded && el == PAD {
pads += 1;
if quad_pos >= 2 && quad_pos + pads <= 4 {
continue;
@@ -276,7 +291,7 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
// RFC 4648 section-3.3: implementations MAY ignore a pad
// character present before the end of the encoded data, and
// MAY ignore excess pad characters. Non-strict mode does.
if !strict_mode {
if !strict_mode || ignored[PAD as usize] {
continue;
}
if quad_pos == 1 {
@@ -284,22 +299,26 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
// `quad_pos == 1` check below.
break;
}
return Err(Error::Base64(if quad_pos == 0 && i == 0 {
return Err(Error::Base64(if quad_pos == 0 && decoded.is_empty() {
Base64DecodeError::LeadingPaddingNotAllowed
} else {
Base64DecodeError::ExcessPaddingNotAllowed
}));
}
let binary_char = BASE64_TABLE[el as usize];
if binary_char < 0 {
if strict_mode {
return Err(Error::Base64(Base64DecodeError::OnlyBase64DataAllowed));
let binary_char = table[el as usize];
if binary_char >= 64 {
if strict_mode && !ignored[el as usize] {
return Err(Error::Base64(if el == PAD {
Base64DecodeError::PaddingNotAllowed
} else {
Base64DecodeError::OnlyBase64DataAllowed
}));
}
continue;
}
if pads > 0 && strict_mode {
if pads > 0 && strict_mode && !ignored[PAD as usize] {
return Err(Error::Base64(if quad_pos + pads == 4 {
Base64DecodeError::ExcessDataAfterPadding
} else {
@@ -312,21 +331,21 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
match quad_pos {
0 => {
quad_pos = 1;
left_char = binary_char as u8;
left_char = binary_char;
}
1 => {
quad_pos = 2;
decoded.push((left_char << 2) | (binary_char >> 4) as u8);
left_char = (binary_char & 0x0f) as u8;
decoded.push((left_char << 2) | (binary_char >> 4));
left_char = binary_char & 0x0f;
}
2 => {
quad_pos = 3;
decoded.push((left_char << 4) | (binary_char >> 2) as u8);
left_char = (binary_char & 0x03) as u8;
decoded.push((left_char << 4) | (binary_char >> 2));
left_char = binary_char & 0x03;
}
3 => {
quad_pos = 0;
decoded.push((left_char << 6) | binary_char as u8);
decoded.push((left_char << 6) | binary_char);
left_char = 0;
}
_ => unsafe {
@@ -344,24 +363,72 @@ pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
}));
}
if quad_pos != 0 && quad_pos + pads < 4 {
if padded && quad_pos != 0 && quad_pos + pads < 4 {
return Err(Error::Base64(Base64DecodeError::IncorrectPadding));
}
if canonical && left_char != 0 {
return Err(Error::Base64(Base64DecodeError::NonZeroPaddingBits));
}
Ok(decoded)
}
/// `b2a_base64`.
#[must_use]
pub fn b2a_base64(data: &[u8], newline: bool) -> Vec<u8> {
// https://stackoverflow.com/questions/63916821
let mut encoded = base64::engine::general_purpose::STANDARD
.encode(data)
.into_bytes();
/// `b2a_base64`, including CPython 3.15's wrapping and custom alphabets.
pub fn b2a_base64(
data: &[u8],
padded: bool,
wrapcol: usize,
newline: bool,
alphabet: &[u8; 64],
) -> Result<Vec<u8>, Error> {
let pads = (3 - data.len() % 3) % 3;
let mut size = data
.len()
.div_ceil(3)
.checked_mul(4)
.ok_or(Error::TooMuchBase64Data)?;
if !padded {
size -= pads;
}
let width = if wrapcol == 0 {
usize::MAX
} else {
wrapcol.max(4) / 4 * 4
};
let newlines = size.saturating_sub(1) / width;
size = size
.checked_add(newlines)
.and_then(|n| n.checked_add(usize::from(newline)))
.filter(|&n| isize::try_from(n).is_ok())
.ok_or(Error::TooMuchBase64Data)?;
let mut encoded = Vec::new();
encoded.try_reserve_exact(size).map_err(|_| Error::Memory)?;
let mut column = 0;
let mut emit = |symbol| {
if column == width {
encoded.push(b'\n');
column = 0;
}
encoded.push(symbol);
column += 1;
};
for chunk in data.chunks(3) {
let value = (u32::from(chunk[0]) << 16)
| (u32::from(*chunk.get(1).unwrap_or(&0)) << 8)
| u32::from(*chunk.get(2).unwrap_or(&0));
for index in 0..=chunk.len() {
emit(alphabet[((value >> (18 - index * 6)) & 63) as usize]);
}
if padded {
for _ in chunk.len() + 1..4 {
emit(PAD);
}
}
}
if newline {
encoded.push(b'\n');
}
encoded
Ok(encoded)
}
#[inline]
+551 -21
View File
@@ -1,4 +1,6 @@
// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode
// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode ABCDEFGHIJKLMNPQRSTUVXYZ abcdefhijklmpqr
mod base_n;
pub(super) use decl::crc32;
pub(crate) use decl::module_def;
@@ -8,14 +10,138 @@ use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef};
#[pymodule(name = "binascii")]
mod decl {
use super::new_binascii_error;
use super::{base_n, new_binascii_error};
use crate::vm::{
PyResult, VirtualMachine,
builtins::PyTypeRef,
function::{ArgAsciiBuffer, ArgBytesLike, ArgIndex, OptionalArg},
AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine,
builtins::{PyBytes, PyBytesRef, PyDict, PyModule, PyTypeRef},
function::{
ArgAsciiBuffer, ArgBytesLike, ArgIndex, ArgIntoBool, ItemDoc, OptionalArg, PyMethodDef,
PyMethodFlags,
},
};
use rustpython_common::binascii;
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
module.set_native_state(vm.ctx.new_dict().into());
const DECODE: &PyMethodDef = &PyMethodDef::new_const(
"a2b_base64",
a2b_base64,
PyMethodFlags::empty(),
ItemDoc::static_text(
"a2b_base64($module, data, /, *, strict_mode=<unrepresentable>, padded=True, alphabet=BASE64_ALPHABET, ignorechars=<unrepresentable>, canonical=False)\n--\n\nDecode a line of base64 data.",
),
);
const DECODE32: &PyMethodDef = &PyMethodDef::new_const(
"a2b_base32",
a2b_base32,
PyMethodFlags::empty(),
ItemDoc::static_text(
"a2b_base32($module, data, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of base32 data.",
),
);
const DECODE85: &PyMethodDef = &PyMethodDef::new_const(
"a2b_base85",
a2b_base85,
PyMethodFlags::empty(),
ItemDoc::static_text(
"a2b_base85($module, data, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False)\n--\n\nDecode a line of Base85 data.",
),
);
for (name, definition) in [
("a2b_base64", DECODE),
("a2b_base32", DECODE32),
("a2b_base85", DECODE85),
] {
let function = definition.build_bound_function(&vm.ctx, module.to_owned().into());
function
.as_object()
.set_attr("__module__", vm.ctx.new_str("binascii"), vm)?;
module.dict().set_item(name, function.into(), vm)?;
}
Ok(())
}
fn reverse_table(
module: &Py<PyModule>,
alphabet: &PyBytesRef,
size: usize,
padchar: Option<u8>,
vm: &VirtualMachine,
) -> PyResult<[u8; 256]> {
let bytes = alphabet.as_bytes();
if bytes.len() != size {
return Err(vm.new_value_error(format!("alphabet must have length {size}")));
}
let cache = module
.native_state()
.ok_or_else(|| vm.new_runtime_error("Missing binascii state"))?
.downcast::<PyDict>()
.map_err(|_| vm.new_runtime_error("Broken binascii cache"))?;
// The actual bytes object is the key: subclass hash/equality callbacks
// are part of CPython's cache lookup and insertion semantics.
if let Some(value) = cache.get_item_opt(alphabet.as_object(), vm)? {
return value
.downcast_ref::<PyBytes>()
.and_then(|value| value.as_bytes().try_into().ok())
.ok_or_else(|| vm.new_runtime_error("Broken binascii cache"));
}
let table = base_n::reverse_table(bytes, padchar);
cache.set_item(
alphabet.as_object(),
vm.ctx.new_bytes(table.to_vec()).into(),
vm,
)?;
Ok(table)
}
#[pyattr(name = "CRYPT_ALPHABET")]
fn crypt_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx
.new_bytes(b"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec())
}
#[pyattr(name = "UU_ALPHABET")]
fn uu_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(
b" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_".to_vec(),
)
}
#[pyattr(name = "BINHEX_ALPHABET")]
fn binhex_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(
b"!\"#$%&'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr".to_vec(),
)
}
#[pyattr(name = "BASE85_ALPHABET")]
fn base85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~".to_vec())
}
#[pyattr(name = "ASCII85_ALPHABET")]
fn ascii85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu".to_vec())
}
#[pyattr(name = "Z85_ALPHABET")]
fn z85_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#".to_vec())
}
#[pyattr(name = "BASE32_ALPHABET")]
fn base32_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx
.new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567".to_vec())
}
#[pyattr(name = "BASE32HEX_ALPHABET")]
fn base32hex_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx
.new_bytes(b"0123456789ABCDEFGHIJKLMNOPQRSTUV".to_vec())
}
#[pyattr(name = "Error", once)]
pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef {
vm.ctx.new_exception_type(
@@ -54,12 +180,24 @@ mod decl {
Ok(data.with_ref(|bytes| binascii::hexlify(bytes, sep, bytes_per_sep)))
}
#[derive(FromArgs)]
struct UnhexlifyArgs {
#[pyarg(positional)]
hexstr: ArgAsciiBuffer,
#[pyarg(named, optional, py_default = "b''")]
ignorechars: OptionalArg<ArgBytesLike>,
}
#[pyfunction(name = "a2b_hex")]
#[pyfunction]
fn unhexlify(hexstr: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
hexstr
.with_ref(binascii::unhexlify)
.map_err(|e| new_binascii_error(e, vm))
fn unhexlify(args: UnhexlifyArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let ignorechars = ignored_argument(&args.ignorechars);
let result = args
.hexstr
.with_ref(|data| base_n::decode_hex(data, &ignorechars));
drop(args.hexstr);
drop(args.ignorechars);
base_n_result(result, vm)
}
#[derive(FromArgs)]
@@ -86,30 +224,418 @@ mod decl {
data.with_ref(|bytes| binascii::crc_hqx(bytes, crc.into_int_ref().as_u32_mask()))
}
#[derive(FromArgs)]
struct NewlineArg {
#[pyarg(named, default = true)]
newline: bool,
#[pyattr(name = "BASE64_ALPHABET")]
fn base64_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx.new_bytes(binascii::BASE64_ALPHABET.to_vec())
}
#[pyattr(name = "URLSAFE_BASE64_ALPHABET")]
fn urlsafe_base64_alphabet(vm: &VirtualMachine) -> PyBytesRef {
vm.ctx
.new_bytes(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".to_vec())
}
#[derive(FromArgs)]
struct A2bBase64Args {
#[pyarg(positional)]
data: ArgAsciiBuffer,
#[pyarg(named, default)]
strict_mode: bool,
#[pyarg(named, optional)]
strict_mode: OptionalArg<ArgIntoBool>,
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
padded: ArgIntoBool,
#[pyarg(named, optional, py_default = "BASE64_ALPHABET")]
alphabet: OptionalArg<PyBytesRef>,
#[pyarg(named, optional)]
ignorechars: OptionalArg<ArgBytesLike>,
#[pyarg(named, optional, py_default = "False")]
canonical: OptionalArg<PyObjectRef>,
}
#[pyfunction]
fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let A2bBase64Args { data, strict_mode } = args;
data.with_ref(|b| binascii::a2b_base64(b, strict_mode))
fn a2b_base64(
module: PyRef<PyModule>,
args: A2bBase64Args,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
let A2bBase64Args {
data,
strict_mode,
padded,
alphabet,
ignorechars,
canonical,
} = args;
let strict_mode = match strict_mode {
OptionalArg::Present(value) => value.into_bool(),
OptionalArg::Missing => matches!(&ignorechars, OptionalArg::Present(_)),
};
let result = (|| {
// Convert the last Clinic argument while both acquired buffers are alive.
let canonical = match canonical {
OptionalArg::Present(value) => value.try_to_bool(vm)?,
OptionalArg::Missing => false,
};
const DEFAULT_TABLE: [u8; 256] =
binascii::base64_reverse_table(binascii::BASE64_ALPHABET);
let table = match &alphabet {
OptionalArg::Present(value) => reverse_table(&module, value, 64, Some(b'='), vm)?,
OptionalArg::Missing => DEFAULT_TABLE,
};
let mut ignored = [false; 256];
if strict_mode && let OptionalArg::Present(value) = &ignorechars {
value.with_ref(|bytes| {
for &symbol in bytes {
ignored[symbol as usize] = true;
}
});
}
data.with_ref(|bytes| {
binascii::a2b_base64(
bytes,
binascii::Base64DecodeOptions {
strict_mode,
padded: padded.into_bool(),
table: &table,
ignored: &ignored,
canonical,
},
)
})
.map_err(|e| new_binascii_error(e, vm))
})();
// Match Argument Clinic cleanup order, including failures in __bool__.
drop(data);
drop(ignorechars);
result
}
#[derive(FromArgs)]
struct B2aBase64Args {
#[pyarg(positional)]
data: ArgBytesLike,
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
padded: ArgIntoBool,
#[pyarg(named, default = 0)]
wrapcol: usize,
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
newline: ArgIntoBool,
#[pyarg(named, optional, py_default = "BASE64_ALPHABET")]
alphabet: OptionalArg<ArgBytesLike>,
}
#[pyfunction]
fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec<u8> {
data.with_ref(|bytes| binascii::b2a_base64(bytes, newline))
fn b2a_base64(args: B2aBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let B2aBase64Args {
data,
padded,
wrapcol,
newline,
alphabet,
} = args;
let copy_alphabet = |bytes: &[u8]| -> PyResult<[u8; 64]> {
bytes
.try_into()
.map_err(|_| vm.new_value_error("alphabet must have length 64"))
};
let alphabet_bytes = match &alphabet {
OptionalArg::Present(value) => value.with_ref(copy_alphabet),
OptionalArg::Missing => Ok(*binascii::BASE64_ALPHABET),
};
let result = alphabet_bytes.and_then(|alphabet_bytes| {
data.with_ref(|bytes| {
binascii::b2a_base64(
bytes,
padded.into_bool(),
wrapcol,
newline.into_bool(),
&alphabet_bytes,
)
})
.map_err(|e| new_binascii_error(e, vm))
});
drop(data);
drop(alphabet);
result
}
fn base_n_result(
result: Result<Vec<u8>, base_n::Error>,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
result.map_err(|error| match error {
base_n::Error::InvalidAlphabet(size) => {
vm.new_value_error(format!("alphabet must have length {size}"))
}
base_n::Error::Codec(message) => vm.new_exception_msg(error_type(vm), message.into()),
base_n::Error::Memory => vm.no_memory_error(),
})
}
fn encoder_alphabet(
argument: &OptionalArg<ArgBytesLike>,
default: &[u8],
) -> Result<Vec<u8>, base_n::Error> {
match argument {
OptionalArg::Present(value) => {
value.with_ref(|bytes| base_n::copy_alphabet(bytes, default.len()))
}
OptionalArg::Missing => base_n::copy_alphabet(default, default.len()),
}
}
fn ignored_argument(argument: &OptionalArg<ArgBytesLike>) -> [bool; 256] {
match argument {
OptionalArg::Present(value) => value.with_ref(base_n::ignored),
OptionalArg::Missing => [false; 256],
}
}
fn canonical_argument(
argument: &OptionalArg<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<bool> {
match argument {
OptionalArg::Present(value) => value.try_to_bool(vm),
OptionalArg::Missing => Ok(false),
}
}
#[derive(FromArgs)]
struct B2aBase32Args {
#[pyarg(positional)]
data: ArgBytesLike,
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
padded: ArgIntoBool,
#[pyarg(named, default = 0)]
wrapcol: usize,
#[pyarg(named, optional, py_default = "BASE32_ALPHABET")]
alphabet: OptionalArg<ArgBytesLike>,
}
#[derive(FromArgs)]
struct A2bBase32Args {
#[pyarg(positional)]
data: ArgAsciiBuffer,
#[pyarg(named, default = ArgIntoBool::TRUE, py_default = "True")]
padded: ArgIntoBool,
#[pyarg(named, optional, py_default = "BASE32_ALPHABET")]
alphabet: OptionalArg<PyBytesRef>,
#[pyarg(named, optional, py_default = "b''")]
ignorechars: OptionalArg<ArgBytesLike>,
#[pyarg(named, optional, py_default = "False")]
canonical: OptionalArg<PyObjectRef>,
}
fn a2b_base32(
module: PyRef<PyModule>,
args: A2bBase32Args,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
let A2bBase32Args {
data,
padded,
alphabet,
ignorechars,
canonical,
} = args;
let result = (|| {
// This is the last Clinic conversion. Keep acquired exports alive
// and reach ordered cleanup even when __bool__ raises.
let canonical = canonical_argument(&canonical, vm)?;
let table = match &alphabet {
OptionalArg::Present(value) => reverse_table(&module, value, 32, Some(b'='), vm)?,
OptionalArg::Missing => base_n::reverse_table(base_n::BASE32, Some(b'=')),
};
let ignore = ignored_argument(&ignorechars);
let result = data.with_ref(|data| {
base_n::decode32(data, &table, &ignore, padded.into_bool(), canonical)
});
base_n_result(result, vm)
})();
// Match Argument Clinic cleanup: release data before auxiliary exports.
drop(data);
drop(ignorechars);
result
}
#[pyfunction]
fn b2a_base32(args: B2aBase32Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let B2aBase32Args {
data,
padded,
wrapcol,
alphabet,
} = args;
let result = encoder_alphabet(&alphabet, base_n::BASE32).and_then(|alphabet_bytes| {
data.with_ref(|data| {
base_n::encode32(data, padded.into_bool(), wrapcol, &alphabet_bytes)
})
});
drop(data);
drop(alphabet);
base_n_result(result, vm)
}
#[derive(FromArgs)]
struct B2aBase85Args {
#[pyarg(positional)]
data: ArgBytesLike,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
pad: ArgIntoBool,
#[pyarg(named, default = 0)]
wrapcol: usize,
#[pyarg(named, optional, py_default = "BASE85_ALPHABET")]
alphabet: OptionalArg<ArgBytesLike>,
}
#[derive(FromArgs)]
struct A2bBase85Args {
#[pyarg(positional)]
data: ArgAsciiBuffer,
#[pyarg(named, optional, py_default = "BASE85_ALPHABET")]
alphabet: OptionalArg<PyBytesRef>,
#[pyarg(named, optional, py_default = "b''")]
ignorechars: OptionalArg<ArgBytesLike>,
#[pyarg(named, optional, py_default = "False")]
canonical: OptionalArg<PyObjectRef>,
}
fn a2b_base85(
module: PyRef<PyModule>,
args: A2bBase85Args,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
let A2bBase85Args {
data,
alphabet,
ignorechars,
canonical,
} = args;
let result = (|| {
// This is the last Clinic conversion. Keep acquired exports alive
// and reach ordered cleanup even when __bool__ raises.
let canonical = canonical_argument(&canonical, vm)?;
let table = match &alphabet {
OptionalArg::Present(value) => reverse_table(&module, value, 85, None, vm)?,
OptionalArg::Missing => base_n::reverse_table(base_n::BASE85, None),
};
let ignore = ignored_argument(&ignorechars);
let result = data.with_ref(|data| {
base_n::decode85(data, &table, &ignore, canonical, false, false, false)
});
base_n_result(result, vm)
})();
drop(data);
drop(ignorechars);
result
}
#[pyfunction]
fn b2a_base85(args: B2aBase85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let B2aBase85Args {
data,
pad,
wrapcol,
alphabet,
} = args;
let result = encoder_alphabet(&alphabet, base_n::BASE85).and_then(|alphabet_bytes| {
data.with_ref(|data| {
base_n::encode85(
data,
pad.into_bool(),
wrapcol,
&alphabet_bytes,
false,
false,
false,
)
})
});
drop(data);
drop(alphabet);
base_n_result(result, vm)
}
#[derive(FromArgs)]
struct B2aAscii85Args {
#[pyarg(positional)]
data: ArgBytesLike,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
foldspaces: ArgIntoBool,
#[pyarg(named, default = 0)]
wrapcol: usize,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
pad: ArgIntoBool,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
adobe: ArgIntoBool,
}
#[derive(FromArgs)]
struct A2bAscii85Args {
#[pyarg(positional)]
data: ArgAsciiBuffer,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
foldspaces: ArgIntoBool,
#[pyarg(named, default = ArgIntoBool::FALSE, py_default = "False")]
adobe: ArgIntoBool,
#[pyarg(named, optional, py_default = "b''")]
ignorechars: OptionalArg<ArgBytesLike>,
#[pyarg(named, optional, py_default = "False")]
canonical: OptionalArg<PyObjectRef>,
}
#[pyfunction]
fn a2b_ascii85(args: A2bAscii85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let A2bAscii85Args {
data,
foldspaces,
adobe,
ignorechars,
canonical,
} = args;
let result = (|| {
// This is the last Clinic conversion. Keep acquired exports alive
// and reach ordered cleanup even when __bool__ raises.
let canonical = canonical_argument(&canonical, vm)?;
let ignore = ignored_argument(&ignorechars);
let result = data.with_ref(|data| {
base_n::decode85(
data,
&base_n::reverse_table(base_n::ASCII85, None),
&ignore,
canonical,
true,
foldspaces.into_bool(),
adobe.into_bool(),
)
});
base_n_result(result, vm)
})();
drop(data);
drop(ignorechars);
result
}
#[pyfunction]
fn b2a_ascii85(args: B2aAscii85Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let B2aAscii85Args {
data,
foldspaces,
wrapcol,
pad,
adobe,
} = args;
let result = data.with_ref(|data| {
base_n::encode85(
data,
pad.into_bool(),
wrapcol,
base_n::ASCII85,
true,
foldspaces.into_bool(),
adobe.into_bool(),
)
});
base_n_result(result, vm)
}
#[derive(FromArgs)]
@@ -182,6 +708,8 @@ fn new_binascii_error(error: Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
Error::TrailingGarbage => "Trailing garbage".to_owned(),
Error::TooLong => "At most 45 bytes at once".to_owned(),
Error::Base64(e) => base64_message(e),
Error::TooMuchBase64Data => "Too much data for base64".to_owned(),
Error::Memory => return vm.no_memory_error(),
};
vm.new_exception_msg(decl::error_type(vm), message.into())
}
@@ -199,5 +727,7 @@ fn base64_message(error: Base64DecodeError) -> String {
"Invalid base64-encoded string: number of data characters ({index}) cannot be 1 more than a multiple of 4"
),
Base64DecodeError::IncorrectPadding => "Incorrect padding".to_owned(),
Base64DecodeError::PaddingNotAllowed => "Padding not allowed".to_owned(),
Base64DecodeError::NonZeroPaddingBits => "Non-zero padding bits".to_owned(),
}
}
+431
View File
@@ -0,0 +1,431 @@
// Scalar native equivalents of CPython 3.15 Modules/binascii.c base-N codecs.
pub(super) const BASE32: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
pub(super) const BASE85: &[u8] =
b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
pub(super) const ASCII85: &[u8] =
b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstu";
#[derive(Debug)]
pub(super) enum Error {
InvalidAlphabet(usize),
Codec(String),
Memory,
}
type Result<T> = core::result::Result<T, Error>;
fn codec(message: impl Into<String>) -> Error {
Error::Codec(message.into())
}
fn buffer(size: usize, too_large: &str) -> Result<Vec<u8>> {
if size > isize::MAX as usize {
return Err(codec(too_large));
}
let mut result = Vec::new();
result.try_reserve_exact(size).map_err(|_| Error::Memory)?;
Ok(result)
}
fn checked_alphabet(alphabet: &[u8], size: usize) -> Result<&[u8]> {
if alphabet.len() != size {
return Err(Error::InvalidAlphabet(size));
}
Ok(alphabet)
}
pub(super) fn copy_alphabet(alphabet: &[u8], size: usize) -> Result<Vec<u8>> {
checked_alphabet(alphabet, size)?;
let mut out = buffer(size, "Too large alphabet")?;
out.extend_from_slice(alphabet);
Ok(out)
}
pub(super) fn reverse_table(alphabet: &[u8], padchar: Option<u8>) -> [u8; 256] {
let mut table = [u8::MAX; 256];
for (digit, &ch) in alphabet.iter().enumerate() {
// Duplicate characters are allowed; their last occurrence wins.
table[usize::from(ch)] = digit as u8;
}
if let Some(padchar) = padchar {
table[usize::from(padchar)] = alphabet.len() as u8;
}
table
}
pub(super) fn ignored(ignorechars: &[u8]) -> [bool; 256] {
let mut table = [false; 256];
for &ch in ignorechars {
table[usize::from(ch)] = true;
}
table
}
fn rounded_width(width: usize, quantum: usize) -> usize {
if width == 0 {
0
} else {
width.max(quantum) / quantum * quantum
}
}
fn wrapped_size(size: usize, width: usize, newline: bool, too_large: &str) -> Result<usize> {
let breaks = if width == 0 || size == 0 {
0
} else {
(size - 1) / width
};
size.checked_add(breaks)
.and_then(|n| n.checked_add(usize::from(newline)))
.filter(|n| isize::try_from(*n).is_ok())
.ok_or_else(|| codec(too_large))
}
fn wrap(mut input: Vec<u8>, width: usize, newline: bool, too_large: &str) -> Result<Vec<u8>> {
let size = wrapped_size(input.len(), width, newline, too_large)?;
if width != 0 && input.len() > width {
let mut out = buffer(size, too_large)?;
for (index, chunk) in input.chunks(width).enumerate() {
if index != 0 {
out.push(b'\n');
}
out.extend_from_slice(chunk);
}
input = out;
}
if newline {
input.try_reserve(1).map_err(|_| Error::Memory)?;
input.push(b'\n');
}
Ok(input)
}
pub(super) fn encode32(
data: &[u8],
padded: bool,
width: usize,
alphabet: &[u8],
) -> Result<Vec<u8>> {
let (size, in_group, out_group, too_large) = (32, 5, 8, "Too much data for base32");
let bits = 5;
checked_alphabet(alphabet, size)?;
let full_size = data
.len()
.div_ceil(in_group)
.checked_mul(out_group)
.ok_or_else(|| codec(too_large))?;
let pads = (in_group - data.len() % in_group) % in_group * out_group / in_group;
let encoded_size = full_size - if padded { 0 } else { pads };
let width = rounded_width(width, out_group);
wrapped_size(encoded_size, width, false, too_large)?;
let mut out = buffer(encoded_size, too_large)?;
let mut value = 0u32;
let mut available = 0u32;
let mask = (1u32 << bits) - 1;
for &byte in data {
value = (value << 8) | u32::from(byte);
available += 8;
while available >= bits {
available -= bits;
out.push(alphabet[((value >> available) & mask) as usize]);
}
value &= (1u32 << available) - 1;
}
if available != 0 {
out.push(alphabet[((value << (bits - available)) & mask) as usize]);
}
if padded {
out.resize(encoded_size, b'=');
}
wrap(out, width, false, too_large)
}
pub(super) fn decode32(
data: &[u8],
table: &[u8; 256],
ignorechars: &[bool; 256],
padded: bool,
canonical: bool,
) -> Result<Vec<u8>> {
let (size, group, decoded_group, name) = (32, 8, 5, "base32");
let bits = 5;
let ignore = ignorechars;
let capacity = data
.len()
.div_ceil(group)
.checked_mul(decoded_group)
.ok_or_else(|| codec(format!("Too much {name} data")))?;
let mut out = buffer(capacity, &format!("Too much {name} data"))?;
let mut pos = 0usize;
let mut pads = 0usize;
let mut value = 0u32;
let mut available = 0u32;
for &ch in data {
if padded && ch == b'=' {
pads += 1;
let allowed_pos = matches!(pos, 2 | 4 | 5 | 7);
if allowed_pos && pos + pads <= group {
continue;
}
if ignore[usize::from(b'=')] {
continue;
}
if matches!(pos, 1 | 3 | 6) {
break;
}
return Err(codec(if pos == 0 && out.is_empty() {
"Leading padding not allowed"
} else {
"Excess padding not allowed"
}));
}
let digit = table[usize::from(ch)];
if usize::from(digit) >= size {
if !ignore[usize::from(ch)] {
return Err(codec(if ch == b'=' {
"Padding not allowed".to_owned()
} else {
format!("Only {name} data is allowed")
}));
}
continue;
}
if pads != 0 && !ignore[usize::from(b'=')] {
return Err(codec(if pos + pads == group {
"Excess data after padding"
} else {
"Discontinuous padding not allowed"
}));
}
value = (value << bits) | u32::from(digit);
available += bits;
if available >= 8 {
available -= 8;
out.push((value >> available) as u8);
value &= (1u32 << available) - 1;
}
pos += 1;
if pos == group {
pos = 0;
pads = 0;
}
}
if matches!(pos, 1 | 3 | 6) {
let count = out.len() / decoded_group * group + pos;
return Err(codec(format!(
"Invalid base32-encoded string: number of data characters ({count}) cannot be 1, 3, or 6 more than a multiple of 8"
)));
}
if padded && pos != 0 && pos + pads < group {
return Err(codec("Incorrect padding"));
}
if canonical && value != 0 {
return Err(codec("Non-zero padding bits"));
}
Ok(out)
}
fn digits85(mut value: u32, alphabet: &[u8]) -> [u8; 5] {
let mut out = [0; 5];
for slot in out.iter_mut().rev() {
*slot = alphabet[(value % 85) as usize];
value /= 85;
}
out
}
pub(super) fn encode85(
data: &[u8],
pad: bool,
width: usize,
alphabet: &[u8],
ascii85: bool,
foldspaces: bool,
adobe: bool,
) -> Result<Vec<u8>> {
checked_alphabet(alphabet, 85)?;
let too_large = if ascii85 {
"Too much data for Ascii85"
} else {
"Too much data for Base85"
};
let full_size = data
.len()
.div_ceil(4)
.checked_mul(5)
.and_then(|n| n.checked_add(if adobe { 4 } else { 0 }))
.ok_or_else(|| codec(too_large))?;
let size = full_size
- if !pad && !data.len().is_multiple_of(4) {
4 - data.len() % 4
} else {
0
};
let width = if ascii85 {
if adobe && width == 1 { 2 } else { width }
} else {
rounded_width(width, 5)
};
wrapped_size(size, width, false, too_large)?;
let mut out = buffer(size, too_large)?;
if adobe {
out.extend_from_slice(b"<~");
}
for chunk in data.chunks(4) {
let mut block = [0; 4];
block[..chunk.len()].copy_from_slice(chunk);
let value = u32::from_be_bytes(block);
if ascii85 && value == 0 && (pad || chunk.len() == 4) {
out.push(b'z');
} else if ascii85 && foldspaces && value == 0x20202020 && chunk.len() == 4 {
out.push(b'y');
} else {
let digits = digits85(value, alphabet);
let count = if pad || chunk.len() == 4 {
5
} else {
chunk.len() + 1
};
out.extend_from_slice(&digits[..count]);
}
}
if adobe {
out.extend_from_slice(b"~>");
}
let mut out = wrap(out, width, false, too_large)?;
if adobe && out.len() >= 3 && out[out.len() - 2] == b'\n' {
let len = out.len();
out[len - 3] = b'\n';
out[len - 2] = b'~';
}
Ok(out)
}
pub(super) fn decode85(
mut data: &[u8],
table: &[u8; 256],
ignorechars: &[bool; 256],
canonical: bool,
ascii85: bool,
foldspaces: bool,
adobe: bool,
) -> Result<Vec<u8>> {
if adobe {
if !data.ends_with(b"~>") {
return Err(codec("Ascii85 encoded byte sequences must end with b'~>'"));
}
data = &data[..data.len() - 2];
if data.starts_with(b"<~") {
data = &data[2..];
}
}
let ignore = ignorechars;
let shortcuts = if ascii85 {
data.iter().filter(|ch| matches!(ch, b'y' | b'z')).count()
} else {
0
};
let size = (data.len() - shortcuts)
.div_ceil(5)
.checked_mul(4)
.and_then(|n| shortcuts.checked_mul(4).and_then(|m| n.checked_add(m)))
.ok_or_else(|| codec("Too much Ascii85 data"))?;
let mut out = buffer(size, "Too much Ascii85 data")?;
let mut value = 0u32;
let mut pos = 0usize;
let mut from_z = false;
let mut index = 0usize;
let mut tail_len = 4usize;
while index < data.len() || pos != 0 {
let (ch, digit) = if index < data.len() {
let ch = data[index];
(ch, table[usize::from(ch)])
} else {
tail_len -= 1;
(0, 84)
};
if digit < 85 {
value = value
.checked_mul(85)
.and_then(|n| n.checked_add(u32::from(digit)))
.ok_or_else(|| {
codec(if ascii85 {
"Ascii85 overflow".to_owned()
} else {
format!("Base85 overflow in hunk starting at byte {}", index / 5 * 5)
})
})?;
pos += 1;
} else if ascii85 && (ch == b'z' || (ch == b'y' && foldspaces)) {
if pos != 0 {
return Err(codec(format!(
"'{}' inside Ascii85 5-tuple",
char::from(ch)
)));
}
value = if ch == b'y' { 0x20202020 } else { 0 };
from_z = ch == b'z';
pos = 5;
} else if !ignore[usize::from(ch)] {
return Err(codec(if ascii85 {
format!("Non-Ascii85 digit found: {}", char::from(ch))
} else {
format!("bad Base85 character at position {index}")
}));
}
index += 1;
if pos != 5 {
continue;
}
if tail_len == 0 {
return Err(codec(if ascii85 {
"Incomplete Ascii85 group"
} else {
"Incomplete Base85 group"
}));
}
out.extend_from_slice(&value.to_be_bytes()[..tail_len]);
if canonical {
if ascii85 && tail_len == 4 && value == 0 && !from_z {
return Err(codec("Non-canonical encoding, use 'z' for all-zero groups"));
}
if tail_len < 4 {
let pads = 4 - tail_len;
let top = value >> (pads * 8) << (pads * 8);
let divisor = 85u32.pow(pads as u32);
if top / divisor != value / divisor {
return Err(codec("Non-zero padding bits"));
}
}
}
pos = 0;
value = 0;
from_z = false;
}
Ok(out)
}
pub(super) fn decode_hex(data: &[u8], ignorechars: &[bool; 256]) -> Result<Vec<u8>> {
let ignore = ignorechars;
let mut out = buffer(data.len() / 2, "Too much hexadecimal data")?;
let mut high = None;
for &ch in data {
let digit = match ch {
b'0'..=b'9' => ch - b'0',
b'a'..=b'f' => ch - b'a' + 10,
b'A'..=b'F' => ch - b'A' + 10,
_ if ignore[usize::from(ch)] => continue,
_ => return Err(codec("Non-hexadecimal digit found")),
};
if let Some(hi) = high.take() {
out.push((hi << 4) | digit);
} else {
high = Some(digit);
}
}
if high.is_some() {
return Err(codec("Odd number of hexadecimal digits"));
}
Ok(out)
}
+2 -2
View File
@@ -140,7 +140,7 @@ impl Representable for PyNativeFunction {
if let Some(bound) = zelf
.zelf
.as_ref()
.filter(|b| !b.class().is(vm.ctx.types.module_type))
.filter(|b| !b.class().is_subtype(vm.ctx.types.module_type))
{
Ok(format!(
"<built-in method {} of {} object at {:#x}>",
@@ -169,7 +169,7 @@ impl PyNativeFunction {
fn __qualname__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult<PyStrRef> {
let zelf = zelf.0;
let qualname = if let Some(bound) = &zelf.zelf {
if bound.class().is(vm.ctx.types.module_type) {
if bound.class().is_subtype(vm.ctx.types.module_type) {
return Ok(vm.ctx.intern_str(zelf.value.name).to_owned());
}
let prefix = if bound.class().is(vm.ctx.types.type_type) {
+29 -2
View File
@@ -82,7 +82,7 @@ impl PyModuleDef {
}
}
#[pyclass(module = false, name = "module")]
#[pyclass(module = false, name = "module", traverse = "manual")]
// The dict lives in the object extension, not the payload. The offset is
// the dict cell at the front of that extension.
#[pymember(name = "__dict__", offset = ::rustpython_vm::object::dict_member_offset())]
@@ -90,12 +90,26 @@ impl PyModuleDef {
pub struct PyModule {
// PyObject *md_dict;
pub def: Option<&'static PyModuleDef>,
// state: Any
native_state: crate::object::PyAtomicRef<Option<PyObject>>,
// weaklist
// for logging purposes after md_dict is cleared
pub name: Option<&'static PyStrInterned>,
}
// SAFETY: Native state is the only owned Python reference in the payload.
// The module dictionary is visited separately by the object extension.
unsafe impl crate::object::Traverse for PyModule {
fn traverse(&self, tracer: &mut crate::object::TraverseFn<'_>) {
self.native_state.traverse(tracer);
}
fn clear(&mut self, out: &mut Vec<PyObjectRef>) {
if let Some(state) = self.native_state.store(None) {
out.push(state);
}
}
}
impl PyPayload for PyModule {
#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
@@ -119,6 +133,7 @@ impl PyModule {
pub const fn new() -> Self {
Self {
def: None,
native_state: crate::object::PyAtomicRef::new_empty(),
name: None,
}
}
@@ -127,10 +142,22 @@ impl PyModule {
pub const fn from_def(def: &'static PyModuleDef) -> Self {
Self {
def: Some(def),
native_state: crate::object::PyAtomicRef::new_empty(),
name: Some(def.name),
}
}
/// Interpreter-local native state, retained and traversed with the module.
#[must_use]
pub fn native_state(&self) -> Option<PyObjectRef> {
self.native_state.load_owned()
}
/// Replace native state without exposing it in the Python module dictionary.
pub fn set_native_state(&self, state: PyObjectRef) {
drop(self.native_state.store(Some(state)));
}
pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py<Self>) {
let doc = module.def.unwrap().doc.map(|doc| doc.to_owned());
module.init_dict(module.name.unwrap(), doc, vm);