mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

Allow surrogates in str

Browse Source
This commit is contained in:
Noa committed 2025-03-25 19:05:11 -05:00
1 parent e3a1031081
commit cace112b1a
24 files changed
+2758 -487
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+2
View File
@@ -2338,6 +2338,7 @@ version = "0.4.0"
dependencies = [
"ascii",
"bitflags 2.8.0",
"bstr",
"cfg-if",
"itertools 0.14.0",
"libc",
@@ -2345,6 +2346,7 @@ dependencies = [
"malachite-base",
"malachite-bigint",
"malachite-q",
"memchr",
"num-complex",
"num-traits",
"once_cell",
common/Cargo.toml
+2
View File
@@ -16,12 +16,14 @@ rustpython-literal = { workspace = true }
ascii = { workspace = true }
bitflags = { workspace = true }
bstr = { workspace = true }
cfg-if = { workspace = true }
itertools = { workspace = true }
libc = { workspace = true }
malachite-bigint = { workspace = true }
malachite-q = { workspace = true }
malachite-base = { workspace = true }
memchr = { workspace = true }
num-complex = { workspace = true }
num-traits = { workspace = true }
once_cell = { workspace = true }
common/src/encodings.rs
+12 -10
View File
@@ -1,10 +1,12 @@
use std::ops::Range;
use crate::wtf8::{Wtf8, Wtf8Buf};
pub type EncodeErrorResult<S, B, E> = Result<(EncodeReplace<S, B>, usize), E>;
pub type DecodeErrorResult<S, B, E> = Result<(S, Option<B>, usize), E>;
pub trait StrBuffer: AsRef<str> {
pub trait StrBuffer: AsRef<Wtf8> {
fn is_ascii(&self) -> bool {
self.as_ref().is_ascii()
}
@@ -63,19 +65,19 @@ fn decode_utf8_compatible<E: ErrorHandler, DecodeF, ErrF>(
errors: &E,
decode: DecodeF,
handle_error: ErrF,
) -> Result<(String, usize), E::Error>
) -> Result<(Wtf8Buf, usize), E::Error>
where
DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>,
ErrF: Fn(&[u8], Option<usize>) -> HandleResult<'_>,
{
if data.is_empty() {
return Ok((String::new(), 0));
return Ok((Wtf8Buf::new(), 0));
}
// we need to coerce the lifetime to that of the function body rather than the
// anonymous input lifetime, so that we can assign it data borrowed from data_from_err
let mut data = data;
let mut data_from_err: E::BytesBuf;
let mut out = String::with_capacity(data.len());
let mut out = Wtf8Buf::with_capacity(data.len());
let mut remaining_index = 0;
let mut remaining_data = data;
loop {
@@ -98,7 +100,7 @@ where
err_idx..err_len.map_or_else(|| data.len(), |len| err_idx + len);
let (replace, new_data, restart) =
errors.handle_decode_error(data, err_range, reason)?;
out.push_str(replace.as_ref());
out.push_wtf8(replace.as_ref());
if let Some(new_data) = new_data {
data_from_err = new_data;
data = data_from_err.as_ref();
@@ -130,7 +132,7 @@ pub mod utf8 {
data: &[u8],
errors: &E,
final_decode: bool,
) -> Result<(String, usize), E::Error> {
) -> Result<(Wtf8Buf, usize), E::Error> {
decode_utf8_compatible(
data,
errors,
@@ -218,7 +220,7 @@ pub mod latin_1 {
)?;
match replace {
EncodeReplace::Str(s) => {
if s.as_ref().chars().any(|c| (c as u32) > 255) {
if s.as_ref().code_points().any(|c| c.to_u32() > 255) {
return Err(
errors.error_encoding(full_data, char_range, ERR_REASON)
);
@@ -240,10 +242,10 @@ pub mod latin_1 {
Ok(out)
}
pub fn decode<E: ErrorHandler>(data: &[u8], _errors: &E) -> Result<(String, usize), E::Error> {
pub fn decode<E: ErrorHandler>(data: &[u8], _errors: &E) -> Result<(Wtf8Buf, usize), E::Error> {
let out: String = data.iter().map(|c| *c as char).collect();
let out_len = out.len();
Ok((out, out_len))
Ok((out.into(), out_len))
}
}
@@ -303,7 +305,7 @@ pub mod ascii {
Ok(out)
}
pub fn decode<E: ErrorHandler>(data: &[u8], errors: &E) -> Result<(String, usize), E::Error> {
pub fn decode<E: ErrorHandler>(data: &[u8], errors: &E) -> Result<(Wtf8Buf, usize), E::Error> {
decode_utf8_compatible(
data,
errors,
common/src/lib.rs
+1
View File
@@ -29,6 +29,7 @@ pub mod static_cell;
pub mod str;
#[cfg(windows)]
pub mod windows;
pub mod wtf8;
pub mod vendored {
pub use ascii;
common/src/str.rs
+303 -87
View File
@@ -1,9 +1,9 @@
use crate::{
atomic::{PyAtomic, Radium},
format::CharLen,
hash::PyHash,
};
use ascii::AsciiString;
use crate::atomic::{PyAtomic, Radium};
use crate::format::CharLen;
use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
use ascii::{AsciiChar, AsciiStr, AsciiString};
use core::fmt;
use core::sync::atomic::Ordering::Relaxed;
use std::ops::{Bound, RangeBounds};
#[cfg(not(target_arch = "wasm32"))]
@@ -15,132 +15,313 @@ pub type wchar_t = u32;
/// Utf8 + state.ascii (+ PyUnicode_Kind in future)
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum PyStrKind {
pub enum StrKind {
Ascii,
Utf8,
Wtf8,
}
impl std::ops::BitOr for PyStrKind {
impl std::ops::BitOr for StrKind {
type Output = Self;
fn bitor(self, other: Self) -> Self {
use StrKind::*;
match (self, other) {
(Self::Ascii, Self::Ascii) => Self::Ascii,
_ => Self::Utf8,
(Wtf8, _) | (_, Wtf8) => Wtf8,
(Utf8, _) | (_, Utf8) => Utf8,
(Ascii, Ascii) => Ascii,
}
}
}
impl PyStrKind {
#[inline]
pub fn new_data(self) -> PyStrKindData {
match self {
PyStrKind::Ascii => PyStrKindData::Ascii,
PyStrKind::Utf8 => PyStrKindData::Utf8(Radium::new(usize::MAX)),
impl StrKind {
pub fn is_ascii(&self) -> bool {
matches!(self, Self::Ascii)
}
pub fn is_utf8(&self) -> bool {
matches!(self, Self::Ascii | Self::Utf8)
}
}
pub trait DeduceStrKind {
fn str_kind(&self) -> StrKind;
}
impl DeduceStrKind for str {
fn str_kind(&self) -> StrKind {
if self.is_ascii() {
StrKind::Ascii
} else {
StrKind::Utf8
}
}
}
impl DeduceStrKind for Wtf8 {
fn str_kind(&self) -> StrKind {
if self.is_ascii() {
StrKind::Ascii
} else if self.is_utf8() {
StrKind::Utf8
} else {
StrKind::Wtf8
}
}
}
impl DeduceStrKind for String {
fn str_kind(&self) -> StrKind {
(**self).str_kind()
}
}
impl DeduceStrKind for Wtf8Buf {
fn str_kind(&self) -> StrKind {
(**self).str_kind()
}
}
impl<T: DeduceStrKind + ?Sized> DeduceStrKind for &T {
fn str_kind(&self) -> StrKind {
(**self).str_kind()
}
}
impl<T: DeduceStrKind + ?Sized> DeduceStrKind for Box<T> {
fn str_kind(&self) -> StrKind {
(**self).str_kind()
}
}
#[derive(Debug)]
pub enum PyStrKindData {
Ascii,
// uses usize::MAX as a sentinel for "uncomputed"
Utf8(PyAtomic<usize>),
pub enum PyKindStr<'a> {
Ascii(&'a AsciiStr),
Utf8(&'a str),
Wtf8(&'a Wtf8),
}
impl PyStrKindData {
#[inline]
pub fn kind(&self) -> PyStrKind {
match self {
PyStrKindData::Ascii => PyStrKind::Ascii,
PyStrKindData::Utf8(_) => PyStrKind::Utf8,
}
#[derive(Debug, Clone)]
pub struct StrData {
data: Box<Wtf8>,
kind: StrKind,
len: StrLen,
}
struct StrLen(PyAtomic<usize>);
impl From<usize> for StrLen {
#[inline(always)]
fn from(value: usize) -> Self {
Self(Radium::new(value))
}
}
pub struct BorrowedStr<'a> {
bytes: &'a [u8],
kind: PyStrKindData,
#[allow(dead_code)]
hash: PyAtomic<PyHash>,
}
impl<'a> BorrowedStr<'a> {
/// # Safety
/// `s` have to be an ascii string
#[inline]
pub unsafe fn from_ascii_unchecked(s: &'a [u8]) -> Self {
debug_assert!(s.is_ascii());
Self {
bytes: s,
kind: PyStrKind::Ascii.new_data(),
hash: PyAtomic::<PyHash>::new(0),
}
}
#[inline]
pub fn from_bytes(s: &'a [u8]) -> Self {
let k = if s.is_ascii() {
PyStrKind::Ascii.new_data()
impl fmt::Debug for StrLen {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let len = self.0.load(Relaxed);
if len == usize::MAX {
f.write_str("<uncomputed>")
} else {
PyStrKind::Utf8.new_data()
};
len.fmt(f)
}
}
}
impl StrLen {
#[inline(always)]
fn zero() -> Self {
0usize.into()
}
#[inline(always)]
fn uncomputed() -> Self {
usize::MAX.into()
}
}
impl Clone for StrLen {
fn clone(&self) -> Self {
Self(self.0.load(Relaxed).into())
}
}
impl Default for StrData {
fn default() -> Self {
Self {
bytes: s,
kind: k,
hash: PyAtomic::<PyHash>::new(0),
data: <Box<Wtf8>>::default(),
kind: StrKind::Ascii,
len: StrLen::zero(),
}
}
}
impl From<Box<Wtf8>> for StrData {
fn from(value: Box<Wtf8>) -> Self {
// doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for
// non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532
let kind = value.str_kind();
unsafe { Self::new_str_unchecked(value, kind) }
}
}
impl From<Box<str>> for StrData {
#[inline]
fn from(value: Box<str>) -> Self {
// doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for
// non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532
let kind = value.str_kind();
unsafe { Self::new_str_unchecked(value.into(), kind) }
}
}
impl From<Box<AsciiStr>> for StrData {
#[inline]
fn from(value: Box<AsciiStr>) -> Self {
Self {
len: value.len().into(),
data: value.into(),
kind: StrKind::Ascii,
}
}
}
impl From<AsciiChar> for StrData {
fn from(ch: AsciiChar) -> Self {
AsciiString::from(ch).into_boxed_ascii_str().into()
}
}
impl From<char> for StrData {
fn from(ch: char) -> Self {
if let Ok(ch) = ascii::AsciiChar::from_ascii(ch) {
ch.into()
} else {
Self {
data: ch.to_string().into(),
kind: StrKind::Utf8,
len: 1.into(),
}
}
}
}
impl From<CodePoint> for StrData {
fn from(ch: CodePoint) -> Self {
if let Some(ch) = ch.to_char() {
ch.into()
} else {
Self {
data: Wtf8Buf::from(ch).into(),
kind: StrKind::Wtf8,
len: 1.into(),
}
}
}
}
impl StrData {
/// # Safety
///
/// Given `bytes` must be valid data for given `kind`
pub unsafe fn new_str_unchecked(data: Box<Wtf8>, kind: StrKind) -> Self {
let len = match kind {
StrKind::Ascii => data.len().into(),
_ => StrLen::uncomputed(),
};
Self { data, kind, len }
}
/// # Safety
///
/// `char_len` must be accurate.
pub unsafe fn new_with_char_len(data: Box<Wtf8>, kind: StrKind, char_len: usize) -> Self {
Self {
data,
kind,
len: char_len.into(),
}
}
#[inline]
pub fn as_str(&self) -> &str {
unsafe {
// SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string
std::str::from_utf8_unchecked(self.bytes)
pub fn as_wtf8(&self) -> &Wtf8 {
&self.data
}
#[inline]
pub fn as_str(&self) -> Option<&str> {
self.kind
.is_utf8()
.then(|| unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) })
}
pub fn as_ascii(&self) -> Option<&AsciiStr> {
self.kind
.is_ascii()
.then(|| unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) })
}
pub fn kind(&self) -> StrKind {
self.kind
}
#[inline]
pub fn as_str_kind(&self) -> PyKindStr<'_> {
match self.kind {
StrKind::Ascii => {
PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) })
}
StrKind::Utf8 => {
PyKindStr::Utf8(unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) })
}
StrKind::Wtf8 => PyKindStr::Wtf8(&self.data),
}
}
#[inline]
pub fn len(&self) -> usize {
self.data.len()
}
pub fn is_empty(&self) -> bool {
self.data.is_empty()
}
#[inline]
pub fn char_len(&self) -> usize {
match self.kind {
PyStrKindData::Ascii => self.bytes.len(),
PyStrKindData::Utf8(ref len) => match len.load(core::sync::atomic::Ordering::Relaxed) {
usize::MAX => self._compute_char_len(),
len => len,
},
match self.len.0.load(Relaxed) {
usize::MAX => self._compute_char_len(),
len => len,
}
}
#[cold]
fn _compute_char_len(&self) -> usize {
match self.kind {
PyStrKindData::Utf8(ref char_len) => {
let len = self.as_str().chars().count();
// len cannot be usize::MAX, since vec.capacity() < sys.maxsize
char_len.store(len, core::sync::atomic::Ordering::Relaxed);
len
}
_ => unsafe {
debug_assert!(false); // invalid for non-utf8 strings
std::hint::unreachable_unchecked()
},
let len = if let Some(s) = self.as_str() {
// utf8 chars().count() is optimized
s.chars().count()
} else {
self.data.code_points().count()
};
// len cannot be usize::MAX, since vec.capacity() < sys.maxsize
self.len.0.store(len, Relaxed);
len
}
pub fn nth_char(&self, index: usize) -> CodePoint {
match self.as_str_kind() {
PyKindStr::Ascii(s) => s[index].into(),
PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(),
PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(),
}
}
}
impl std::ops::Deref for BorrowedStr<'_> {
type Target = str;
fn deref(&self) -> &str {
self.as_str()
}
}
impl std::fmt::Display for BorrowedStr<'_> {
impl std::fmt::Display for StrData {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.as_str().fmt(f)
self.data.fmt(f)
}
}
impl CharLen for BorrowedStr<'_> {
impl CharLen for StrData {
fn char_len(&self) -> usize {
self.char_len()
}
@@ -181,6 +362,41 @@ pub fn char_range_end(s: &str, nchars: usize) -> Option<usize> {
Some(i)
}
pub fn try_get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> Option<&Wtf8> {
let mut chars = w.code_points();
let start = match range.start_bound() {
Bound::Included(&i) => i,
Bound::Excluded(&i) => i + 1,
Bound::Unbounded => 0,
};
for _ in 0..start {
chars.next()?;
}
let s = chars.as_wtf8();
let range_len = match range.end_bound() {
Bound::Included(&i) => i + 1 - start,
Bound::Excluded(&i) => i - start,
Bound::Unbounded => return Some(s),
};
codepoint_range_end(s, range_len).map(|end| &s[..end])
}
pub fn get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> &Wtf8 {
try_get_codepoints(w, range).unwrap()
}
#[inline]
pub fn codepoint_range_end(s: &Wtf8, nchars: usize) -> Option<usize> {
let i = match nchars.checked_sub(1) {
Some(last_char_index) => {
let (index, c) = s.code_point_indices().nth(last_char_index)?;
index + c.len_wtf8()
}
None => 0,
};
Some(i)
}
pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> {
if width <= bytes.len() {
bytes.to_vec()
common/src/wtf8/core_char.rs
+113
View File
@@ -0,0 +1,113 @@
//! Unstable functions from [`core::char`]
use core::slice;
pub const MAX_LEN_UTF8: usize = 4;
pub const MAX_LEN_UTF16: usize = 2;
// UTF-8 ranges and tags for encoding characters
const TAG_CONT: u8 = 0b1000_0000;
const TAG_TWO_B: u8 = 0b1100_0000;
const TAG_THREE_B: u8 = 0b1110_0000;
const TAG_FOUR_B: u8 = 0b1111_0000;
const MAX_ONE_B: u32 = 0x80;
const MAX_TWO_B: u32 = 0x800;
const MAX_THREE_B: u32 = 0x10000;
#[inline]
#[must_use]
pub const fn len_utf8(code: u32) -> usize {
match code {
..MAX_ONE_B => 1,
..MAX_TWO_B => 2,
..MAX_THREE_B => 3,
_ => 4,
}
}
#[inline]
#[must_use]
const fn len_utf16(code: u32) -> usize {
if (code & 0xFFFF) == code { 1 } else { 2 }
}
/// Encodes a raw `u32` value as UTF-8 into the provided byte buffer,
/// and then returns the subslice of the buffer that contains the encoded character.
///
/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range.
/// (Creating a `char` in the surrogate range is UB.)
/// The result is valid [generalized UTF-8] but not valid UTF-8.
///
/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8
///
/// # Panics
///
/// Panics if the buffer is not large enough.
/// A buffer of length four is large enough to encode any `char`.
#[doc(hidden)]
#[inline]
pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] {
let len = len_utf8(code);
match (len, &mut *dst) {
(1, [a, ..]) => {
*a = code as u8;
}
(2, [a, b, ..]) => {
*a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
*b = (code & 0x3F) as u8 | TAG_CONT;
}
(3, [a, b, c, ..]) => {
*a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
*b = (code >> 6 & 0x3F) as u8 | TAG_CONT;
*c = (code & 0x3F) as u8 | TAG_CONT;
}
(4, [a, b, c, d, ..]) => {
*a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
*b = (code >> 12 & 0x3F) as u8 | TAG_CONT;
*c = (code >> 6 & 0x3F) as u8 | TAG_CONT;
*d = (code & 0x3F) as u8 | TAG_CONT;
}
_ => {
panic!(
"encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}",
dst_len = dst.len(),
)
}
};
// SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds.
unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) }
}
/// Encodes a raw `u32` value as UTF-16 into the provided `u16` buffer,
/// and then returns the subslice of the buffer that contains the encoded character.
///
/// Unlike `char::encode_utf16`, this method also handles codepoints in the surrogate range.
/// (Creating a `char` in the surrogate range is UB.)
///
/// # Panics
///
/// Panics if the buffer is not large enough.
/// A buffer of length 2 is large enough to encode any `char`.
#[doc(hidden)]
#[inline]
pub fn encode_utf16_raw(mut code: u32, dst: &mut [u16]) -> &mut [u16] {
let len = len_utf16(code);
match (len, &mut *dst) {
(1, [a, ..]) => {
*a = code as u16;
}
(2, [a, b, ..]) => {
code -= 0x1_0000;
*a = (code >> 10) as u16 | 0xD800;
*b = (code & 0x3FF) as u16 | 0xDC00;
}
_ => {
panic!(
"encode_utf16: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}",
dst_len = dst.len(),
)
}
};
// SAFETY: `<&mut [u16]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds.
unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) }
}
common/src/wtf8/core_str.rs
+113
View File
@@ -0,0 +1,113 @@
//! Operations related to UTF-8 validation.
//!
//! Copied from `core::str::validations`
/// Returns the initial codepoint accumulator for the first byte.
/// The first byte is special, only want bottom 5 bits for width 2, 4 bits
/// for width 3, and 3 bits for width 4.
#[inline]
const fn utf8_first_byte(byte: u8, width: u32) -> u32 {
(byte & (0x7F >> width)) as u32
}
/// Returns the value of `ch` updated with continuation byte `byte`.
#[inline]
const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 {
(ch << 6) | (byte & CONT_MASK) as u32
}
/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the
/// bits `10`).
#[inline]
pub(super) const fn utf8_is_cont_byte(byte: u8) -> bool {
(byte as i8) < -64
}
/// Reads the next code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
///
/// # Safety
///
/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string
#[inline]
pub unsafe fn next_code_point<'a, I: Iterator<Item = &'a u8>>(bytes: &mut I) -> Option<u32> {
// Decode UTF-8
let x = *bytes.next()?;
if x < 128 {
return Some(x as u32);
}
// Multibyte case follows
// Decode from a byte combination out of: [[[x y] z] w]
// NOTE: Performance is sensitive to the exact formulation here
let init = utf8_first_byte(x, 2);
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let y = unsafe { *bytes.next().unwrap_unchecked() };
let mut ch = utf8_acc_cont_byte(init, y);
if x >= 0xE0 {
// [[x y z] w] case
// 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let z = unsafe { *bytes.next().unwrap_unchecked() };
let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z);
ch = init << 12 | y_z;
if x >= 0xF0 {
// [x y z w] case
// use only the lower 3 bits of `init`
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let w = unsafe { *bytes.next().unwrap_unchecked() };
ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w);
}
}
Some(ch)
}
/// Reads the last code point out of a byte iterator (assuming a
/// UTF-8-like encoding).
///
/// # Safety
///
/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string
#[inline]
pub unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option<u32>
where
I: DoubleEndedIterator<Item = &'a u8>,
{
// Decode UTF-8
let w = match *bytes.next_back()? {
next_byte if next_byte < 128 => return Some(next_byte as u32),
back_byte => back_byte,
};
// Multibyte case follows
// Decode from a byte combination out of: [x [y [z w]]]
let mut ch;
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let z = unsafe { *bytes.next_back().unwrap_unchecked() };
ch = utf8_first_byte(z, 2);
if utf8_is_cont_byte(z) {
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let y = unsafe { *bytes.next_back().unwrap_unchecked() };
ch = utf8_first_byte(y, 3);
if utf8_is_cont_byte(y) {
// SAFETY: `bytes` produces an UTF-8-like string,
// so the iterator must produce a value here.
let x = unsafe { *bytes.next_back().unwrap_unchecked() };
ch = utf8_first_byte(x, 4);
ch = utf8_acc_cont_byte(ch, y);
}
ch = utf8_acc_cont_byte(ch, z);
}
ch = utf8_acc_cont_byte(ch, w);
Some(ch)
}
/// Mask of the value bits of a continuation byte.
const CONT_MASK: u8 = 0b0011_1111;
common/src/wtf8/mod.rs
+1347
View File
File diff suppressed because it is too large. Load diff
stdlib/src/pystruct.rs
+14 -17
View File
@@ -27,23 +27,20 @@ pub(crate) mod _struct {
fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
// CPython turns str to bytes but we do reversed way here
// The only performance difference is this transition cost
let fmt = match_class! {
match obj {
s @ PyStr => if s.is_ascii() {
Some(s)
} else {
None
},
b @ PyBytes => if b.is_ascii() {
Some(unsafe {
PyStr::new_ascii_unchecked(b.as_bytes().to_vec())
}.into_ref(&vm.ctx))
} else {
None
},
other => return Err(vm.new_type_error(format!("Struct() argument 1 must be a str or bytes object, not {}", other.class().name()))),
}
}.ok_or_else(|| vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned()))?;
let fmt = match_class!(match obj {
s @ PyStr => s.is_ascii().then_some(s),
b @ PyBytes => ascii::AsciiStr::from_ascii(&b)
.ok()
.map(|s| vm.ctx.new_str(s)),
other =>
return Err(vm.new_type_error(format!(
"Struct() argument 1 must be a str or bytes object, not {}",
other.class().name()
))),
})
.ok_or_else(|| {
vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned())
})?;
Ok(IntoStructFormatBytes(fmt))
}
}
stdlib/src/unicodedata.rs
+33 -16
View File
@@ -65,6 +65,7 @@ mod unicodedata {
function::OptionalArg,
};
use itertools::Itertools;
use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
use ucd::{Codepoint, EastAsianWidth};
use unic_char_property::EnumeratedCharProperty;
use unic_normal::StrNormalForm;
@@ -84,14 +85,23 @@ mod unicodedata {
Self { unic_version }
}
fn check_age(&self, c: char) -> bool {
Age::of(c).is_some_and(|age| age.actual() <= self.unic_version)
fn check_age(&self, c: CodePoint) -> bool {
c.to_char()
.is_none_or(|c| Age::of(c).is_some_and(|age| age.actual() <= self.unic_version))
}
fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<Option<char>> {
let c = character.as_str().chars().exactly_one().map_err(|_| {
vm.new_type_error("argument must be an unicode character, not str".to_owned())
})?;
fn extract_char(
&self,
character: PyStrRef,
vm: &VirtualMachine,
) -> PyResult<Option<CodePoint>> {
let c = character
.as_wtf8()
.code_points()
.exactly_one()
.map_err(|_| {
vm.new_type_error("argument must be an unicode character, not str".to_owned())
})?;
Ok(self.check_age(c).then_some(c))
}
@@ -103,7 +113,10 @@ mod unicodedata {
fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
Ok(self
.extract_char(character, vm)?
.map_or(GeneralCategory::Unassigned, GeneralCategory::of)
.map_or(GeneralCategory::Unassigned, |c| {
c.to_char()
.map_or(GeneralCategory::Surrogate, GeneralCategory::of)
})
.abbr_name()
.to_owned())
}
@@ -111,7 +124,7 @@ mod unicodedata {
#[pymethod]
fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
if let Some(character) = unicode_names2::character(name.as_str()) {
if self.check_age(character) {
if self.check_age(character.into()) {
return Ok(character.to_string());
}
}
@@ -129,7 +142,7 @@ mod unicodedata {
if let Some(c) = c {
if self.check_age(c) {
if let Some(name) = unicode_names2::name(c) {
if let Some(name) = c.to_char().and_then(unicode_names2::name) {
return Ok(vm.ctx.new_str(name.to_string()).into());
}
}
@@ -144,7 +157,10 @@ mod unicodedata {
vm: &VirtualMachine,
) -> PyResult<&'static str> {
let bidi = match self.extract_char(character, vm)? {
Some(c) => BidiClass::of(c).abbr_name(),
Some(c) => c
.to_char()
.map_or(BidiClass::LeftToRight, BidiClass::of)
.abbr_name(),
None => "",
};
Ok(bidi)
@@ -159,19 +175,20 @@ mod unicodedata {
) -> PyResult<&'static str> {
Ok(self
.extract_char(character, vm)?
.and_then(|c| c.to_char())
.map_or(EastAsianWidth::Neutral, |c| c.east_asian_width())
.abbr_name())
}
#[pymethod]
fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<String> {
fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<Wtf8Buf> {
use super::NormalizeForm::*;
let text = unistr.as_str();
let text = unistr.as_wtf8();
let normalized_text = match form {
Nfc => text.nfc().collect::<String>(),
Nfkc => text.nfkc().collect::<String>(),
Nfd => text.nfd().collect::<String>(),
Nfkd => text.nfkd().collect::<String>(),
Nfc => text.map_utf8(|s| s.nfc()).collect(),
Nfkc => text.map_utf8(|s| s.nfkc()).collect(),
Nfd => text.map_utf8(|s| s.nfd()).collect(),
Nfkd => text.map_utf8(|s| s.nfkd()).collect(),
};
Ok(normalized_text)
}
vm/src/anystr.rs
+66 -60
View File
@@ -7,28 +7,13 @@ use crate::{
use num_traits::{cast::ToPrimitive, sign::Signed};
#[derive(FromArgs)]
pub struct SplitArgs<T: TryFromObject + AnyStrWrapper> {
pub struct SplitArgs<T: TryFromObject> {
#[pyarg(any, default)]
sep: Option<T>,
#[pyarg(any, default = "-1")]
maxsplit: isize,
}
impl<T: TryFromObject + AnyStrWrapper> SplitArgs<T> {
pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option<T>, isize)> {
let sep = if let Some(s) = self.sep {
let sep = s.as_ref();
if sep.is_empty() {
return Err(vm.new_value_error("empty separator".to_owned()));
}
Some(s)
} else {
None
};
Ok((sep, self.maxsplit))
}
}
#[derive(FromArgs)]
pub struct SplitLinesArgs {
#[pyarg(any, default = "false")]
@@ -132,9 +117,9 @@ impl StringRange for std::ops::Range<usize> {
}
}
pub trait AnyStrWrapper {
type Str: ?Sized + AnyStr;
fn as_ref(&self) -> &Self::Str;
pub trait AnyStrWrapper<S: AnyStr + ?Sized> {
fn as_ref(&self) -> Option<&S>;
fn is_empty(&self) -> bool;
}
pub trait AnyStrContainer<S>
@@ -146,15 +131,18 @@ where
fn push_str(&mut self, s: &S);
}
pub trait AnyStr {
type Char: Copy;
type Container: AnyStrContainer<Self> + Extend<Self::Char>;
pub trait AnyChar: Copy {
fn is_lowercase(self) -> bool;
fn is_uppercase(self) -> bool;
fn bytes_len(self) -> usize;
}
fn element_bytes_len(c: Self::Char) -> usize;
pub trait AnyStr {
type Char: AnyChar;
type Container: AnyStrContainer<Self> + Extend<Self::Char>;
fn to_container(&self) -> Self::Container;
fn as_bytes(&self) -> &[u8];
fn chars(&self) -> impl Iterator<Item = char>;
fn elements(&self) -> impl Iterator<Item = Self::Char>;
fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self;
// FIXME: get_chars is expensive for str
@@ -172,29 +160,35 @@ pub trait AnyStr {
new
}
fn py_split<T, SP, SN, SW, R>(
fn py_split<T, SP, SN, SW>(
&self,
args: SplitArgs<T>,
vm: &VirtualMachine,
full_obj: impl FnOnce() -> PyObjectRef,
split: SP,
splitn: SN,
splitw: SW,
) -> PyResult<Vec<R>>
) -> PyResult<Vec<PyObjectRef>>
where
T: TryFromObject + AnyStrWrapper<Str = Self>,
SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<R>,
SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<R>,
SW: Fn(&Self, isize, &VirtualMachine) -> Vec<R>,
T: TryFromObject + AnyStrWrapper<Self>,
SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<PyObjectRef>,
SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<PyObjectRef>,
SW: Fn(&Self, isize, &VirtualMachine) -> Vec<PyObjectRef>,
{
let (sep, maxsplit) = args.get_value(vm)?;
let splits = if let Some(pattern) = sep {
if maxsplit < 0 {
split(self, pattern.as_ref(), vm)
if args.sep.as_ref().is_some_and(|sep| sep.is_empty()) {
return Err(vm.new_value_error("empty separator".to_owned()));
}
let splits = if let Some(pattern) = args.sep {
let Some(pattern) = pattern.as_ref() else {
return Ok(vec![full_obj()]);
};
if args.maxsplit < 0 {
split(self, pattern, vm)
} else {
splitn(self, pattern.as_ref(), (maxsplit + 1) as usize, vm)
splitn(self, pattern, (args.maxsplit + 1) as usize, vm)
}
} else {
splitw(self, maxsplit, vm)
splitw(self, args.maxsplit, vm)
};
Ok(splits)
}
@@ -242,13 +236,19 @@ pub trait AnyStr {
func_default: FD,
) -> &'a Self
where
S: AnyStrWrapper<Str = Self>,
S: AnyStrWrapper<Self>,
FC: Fn(&'a Self, &Self) -> &'a Self,
FD: Fn(&'a Self) -> &'a Self,
{
let chars = chars.flatten();
match chars {
Some(chars) => func_chars(self, chars.as_ref()),
Some(chars) => {
if let Some(chars) = chars.as_ref() {
func_chars(self, chars)
} else {
self
}
}
None => func_default(self),
}
}
@@ -281,7 +281,7 @@ pub trait AnyStr {
fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container {
let mut u = Self::Container::with_capacity(
(left + right) * Self::element_bytes_len(fillchar) + self.bytes_len(),
(left + right) * fillchar.bytes_len() + self.bytes_len(),
);
u.extend(std::iter::repeat(fillchar).take(left));
u.push_str(self);
@@ -305,19 +305,17 @@ pub trait AnyStr {
fn py_join(
&self,
mut iter: impl std::iter::Iterator<
Item = PyResult<impl AnyStrWrapper<Str = Self> + TryFromObject>,
>,
mut iter: impl std::iter::Iterator<Item = PyResult<impl AnyStrWrapper<Self> + TryFromObject>>,
) -> PyResult<Self::Container> {
let mut joined = if let Some(elem) = iter.next() {
elem?.as_ref().to_container()
elem?.as_ref().unwrap().to_container()
} else {
return Ok(Self::Container::new());
};
for elem in iter {
let elem = elem?;
joined.push_str(self);
joined.push_str(elem.as_ref());
joined.push_str(elem.as_ref().unwrap());
}
Ok(joined)
}
@@ -403,25 +401,33 @@ pub trait AnyStr {
rustpython_common::str::zfill(self.as_bytes(), width)
}
fn py_iscase<F, G>(&self, is_case: F, is_opposite: G) -> bool
where
F: Fn(char) -> bool,
G: Fn(char) -> bool,
{
// Unified form of CPython functions:
// _Py_bytes_islower
// Py_bytes_isupper
// unicode_islower_impl
// unicode_isupper_impl
let mut cased = false;
for c in self.chars() {
if is_opposite(c) {
// Unified form of CPython functions:
// _Py_bytes_islower
// Py_bytes_isupper
// unicode_islower_impl
// unicode_isupper_impl
fn py_islower(&self) -> bool {
let mut lower = false;
for c in self.elements() {
if c.is_uppercase() {
return false;
} else if !cased && is_case(c) {
cased = true
} else if !lower && c.is_lowercase() {
lower = true
}
}
cased
lower
}
fn py_isupper(&self) -> bool {
let mut upper = false;
for c in self.elements() {
if c.is_lowercase() {
return false;
} else if !upper && c.is_uppercase() {
upper = true
}
}
upper
}
}
vm/src/builtins/genericalias.rs
+15 -18
View File
@@ -124,7 +124,7 @@ impl PyGenericAlias {
Ok(format!(
"{}[{}]",
repr_item(self.origin.clone().into(), vm)?,
if self.args.len() == 0 {
if self.args.is_empty() {
"()".to_owned()
} else {
self.args
@@ -261,23 +261,20 @@ fn subs_tvars(
.and_then(|sub_params| {
PyTupleRef::try_from_object(vm, sub_params)
.ok()
.and_then(|sub_params| {
if sub_params.len() > 0 {
let sub_args = sub_params
.iter()
.map(|arg| {
if let Some(idx) = tuple_index(params, arg) {
argitems[idx].clone()
} else {
arg.clone()
}
})
.collect::<Vec<_>>();
let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into();
Some(obj.get_item(&*sub_args, vm))
} else {
None
}
.filter(|sub_params| !sub_params.is_empty())
.map(|sub_params| {
let sub_args = sub_params
.iter()
.map(|arg| {
if let Some(idx) = tuple_index(params, arg) {
argitems[idx].clone()
} else {
arg.clone()
}
})
.collect::<Vec<_>>();
let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into();
obj.get_item(&*sub_args, vm)
})
})
.unwrap_or(Ok(obj))
vm/src/builtins/object.rs
+1 -1
View File
@@ -159,7 +159,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine)
slots.set_item(name.as_str(), value, vm).unwrap();
}
if slots.len() > 0 {
if !slots.is_empty() {
return (state, slots).to_pyresult(vm);
}
}
vm/src/builtins/set.rs
+1 -3
View File
@@ -784,9 +784,7 @@ impl Initializer for PySet {
type Args = OptionalArg<PyObjectRef>;
fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> {
if zelf.len() > 0 {
zelf.clear();
}
zelf.clear();
if let OptionalArg::Present(it) = iterable {
zelf.update(PosArgs::new(vec![it]), vm)?;
}
vm/src/builtins/str.rs
+673 -221
View File
File diff suppressed because it is too large. Load diff
vm/src/builtins/union.rs
+1 -1
View File
@@ -204,7 +204,7 @@ impl PyUnion {
vm,
)?;
let mut res;
if new_args.len() == 0 {
if new_args.is_empty() {
res = make_union(&new_args, vm);
} else {
res = new_args.fast_getitem(0);
vm/src/bytesinner.rs
+22 -16
View File
@@ -355,13 +355,11 @@ impl PyBytesInner {
}
pub fn islower(&self) -> bool {
self.elements
.py_iscase(char::is_lowercase, char::is_uppercase)
self.elements.py_islower()
}
pub fn isupper(&self) -> bool {
self.elements
.py_iscase(char::is_uppercase, char::is_lowercase)
self.elements.py_isupper()
}
pub fn isspace(&self) -> bool {
@@ -654,6 +652,7 @@ impl PyBytesInner {
let elements = self.elements.py_split(
options,
vm,
|| convert(&self.elements, vm),
|v, s, vm| v.split_str(s).map(|v| convert(v, vm)).collect(),
|v, s, n, vm| v.splitn_str(n, s).map(|v| convert(v, vm)).collect(),
|v, n, vm| v.py_split_whitespace(n, |v| convert(v, vm)),
@@ -673,6 +672,7 @@ impl PyBytesInner {
let mut elements = self.elements.py_split(
options,
vm,
|| convert(&self.elements, vm),
|v, s, vm| v.rsplit_str(s).map(|v| convert(v, vm)).collect(),
|v, s, n, vm| v.rsplitn_str(n, s).map(|v| convert(v, vm)).collect(),
|v, n, vm| v.py_rsplit_whitespace(n, |v| convert(v, vm)),
@@ -998,10 +998,12 @@ pub trait ByteOr: ToPrimitive {
impl ByteOr for BigInt {}
impl AnyStrWrapper for PyBytesInner {
type Str = [u8];
fn as_ref(&self) -> &[u8] {
&self.elements
impl AnyStrWrapper<[u8]> for PyBytesInner {
fn as_ref(&self) -> Option<&[u8]> {
Some(&self.elements)
}
fn is_empty(&self) -> bool {
self.elements.is_empty()
}
}
@@ -1021,14 +1023,22 @@ impl AnyStrContainer<[u8]> for Vec<u8> {
const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b];
impl anystr::AnyChar for u8 {
fn is_lowercase(self) -> bool {
self.is_ascii_lowercase()
}
fn is_uppercase(self) -> bool {
self.is_ascii_uppercase()
}
fn bytes_len(self) -> usize {
1
}
}
impl AnyStr for [u8] {
type Char = u8;
type Container = Vec<u8>;
fn element_bytes_len(_: u8) -> usize {
1
}
fn to_container(&self) -> Self::Container {
self.to_vec()
}
@@ -1037,10 +1047,6 @@ impl AnyStr for [u8] {
self
}
fn chars(&self) -> impl Iterator<Item = char> {
bstr::ByteSlice::chars(self)
}
fn elements(&self) -> impl Iterator<Item = u8> {
self.iter().copied()
}
vm/src/codecs.rs
+13 -10
View File
@@ -1,3 +1,5 @@
use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
use crate::{
AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine,
builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef},
@@ -570,10 +572,11 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb
return Err(err.downcast().unwrap());
}
let s_after_start =
crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or("");
crate::common::str::try_get_codepoints(s.as_wtf8(), range.start..).unwrap_or_default();
let num_chars = range.len();
let mut out: Vec<u8> = Vec::with_capacity(num_chars * 4);
for c in s_after_start.chars().take(num_chars).map(|x| x as u32) {
for c in s_after_start.code_points().take(num_chars) {
let c = c.to_u32();
if !(0xd800..=0xdfff).contains(&c) {
// Not a surrogate, fail with original exception
return Err(err.downcast().unwrap());
@@ -671,7 +674,7 @@ fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyOb
}
Ok((
vm.new_pyobj(format!("\\x{c:x?}")),
vm.new_pyobj(CodePoint::from_u32(c).unwrap()),
range.start + byte_length,
))
} else {
@@ -683,11 +686,11 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py
if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) {
let range = extract_unicode_error_range(&err, vm)?;
let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?;
let s_after_start =
crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or("");
let s_after_start = crate::common::str::try_get_codepoints(object.as_wtf8(), range.start..)
.unwrap_or_default();
let mut out: Vec<u8> = Vec::with_capacity(range.len());
for ch in s_after_start.chars().take(range.len()) {
let ch = ch as u32;
for ch in s_after_start.code_points().take(range.len()) {
let ch = ch.to_u32();
if !(0xdc80..=0xdcff).contains(&ch) {
// Not a UTF-8b surrogate, fail with original exception
return Err(err.downcast().unwrap());
@@ -702,14 +705,14 @@ fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(Py
let object = PyBytesRef::try_from_object(vm, object)?;
let p = &object.as_bytes()[range.clone()];
let mut consumed = 0;
let mut replace = String::with_capacity(4 * range.len());
let mut replace = Wtf8Buf::with_capacity(4 * range.len());
while consumed < 4 && consumed < range.len() {
let c = p[consumed] as u32;
let c = p[consumed] as u16;
// Refuse to escape ASCII bytes
if c < 128 {
break;
}
write!(replace, "#{}", 0xdc00 + c).unwrap();
replace.push(CodePoint::from(0xdc00 + c));
consumed += 1;
}
if consumed == 0 {
vm/src/frame.rs
+4 -3
View File
@@ -21,6 +21,7 @@ use crate::{
};
use indexmap::IndexMap;
use itertools::Itertools;
use rustpython_common::wtf8::Wtf8Buf;
#[cfg(feature = "threading")]
use std::sync::atomic;
use std::{fmt, iter::zip};
@@ -697,8 +698,8 @@ impl ExecutingFrame<'_> {
.pop_multiple(size.get(arg) as usize)
.as_slice()
.iter()
.map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref())
.collect::<String>();
.map(|pyobj| pyobj.payload::<PyStr>().unwrap())
.collect::<Wtf8Buf>();
let str_obj = vm.ctx.new_str(s);
self.push_value(str_obj.into());
Ok(None)
@@ -1468,7 +1469,7 @@ impl ExecutingFrame<'_> {
let kwarg_names = kwarg_names
.as_slice()
.iter()
.map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref().to_owned());
.map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_str().to_owned());
FuncArgs::with_kwargs_names(args, kwarg_names)
}
vm/src/import.rs
+5 -5
View File
@@ -204,15 +204,15 @@ fn remove_importlib_frames_inner(
// TODO: This function should do nothing on verbose mode.
// TODO: Fix this function after making PyTraceback.next mutable
pub fn remove_importlib_frames(
vm: &VirtualMachine,
exc: &PyBaseExceptionRef,
) -> PyBaseExceptionRef {
pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyBaseExceptionRef) {
if vm.state.settings.verbose != 0 {
return;
}
let always_trim = exc.fast_isinstance(vm.ctx.exceptions.import_error);
if let Some(tb) = exc.traceback() {
let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0;
exc.set_traceback(trimmed_tb);
}
exc.clone()
}
vm/src/stdlib/builtins.rs
+12 -16
View File
@@ -31,7 +31,9 @@ mod builtins {
stdlib::sys,
types::PyComparisonOp,
};
use itertools::Itertools;
use num_traits::{Signed, ToPrimitive};
use rustpython_common::wtf8::CodePoint;
#[cfg(not(feature = "rustpython-compiler"))]
const CODEGEN_NOT_SUPPORTED: &str =
@@ -85,13 +87,13 @@ mod builtins {
}
#[pyfunction]
fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<String> {
fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<CodePoint> {
let value = i
.try_to_primitive::<isize>(vm)?
.to_u32()
.and_then(char::from_u32)
.and_then(CodePoint::from_u32)
.ok_or_else(|| vm.new_value_error("chr() arg not in range(0x110000)".to_owned()))?;
Ok(value.to_string())
Ok(value)
}
#[derive(FromArgs)]
@@ -618,21 +620,15 @@ mod builtins {
}
Ok(u32::from(bytes[0]))
}),
Either::B(string) => {
let string = string.as_str();
let string_len = string.chars().count();
if string_len != 1 {
return Err(vm.new_type_error(format!(
Either::B(string) => match string.as_wtf8().code_points().exactly_one() {
Ok(character) => Ok(character.to_u32()),
Err(_) => {
let string_len = string.char_len();
Err(vm.new_type_error(format!(
"ord() expected a character, but string of length {string_len} found"
)));
)))
}
match string.chars().next() {
Some(character) => Ok(character as u32),
None => Err(vm.new_type_error(
"ord() could not guess the integer representing this character".to_owned(),
)),
}
}
},
}
}
vm/src/stdlib/codecs.rs
+3 -1
View File
@@ -2,6 +2,8 @@ pub(crate) use _codecs::make_module;
#[pymodule]
mod _codecs {
use rustpython_common::wtf8::Wtf8Buf;
use crate::common::encodings;
use crate::{
AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine,
@@ -257,7 +259,7 @@ mod _codecs {
}
}
type DecodeResult = PyResult<(String, usize)>;
type DecodeResult = PyResult<(Wtf8Buf, usize)>;
#[derive(FromArgs)]
struct DecodeArgs {
vm/src/vm/interpreter.rs
+1 -1
View File
@@ -163,7 +163,7 @@ mod tests {
let b = vm.new_pyobj(4_i32);
let res = vm._mul(&a, &b).unwrap();
let value = res.payload::<PyStr>().unwrap();
assert_eq!(value.as_ref(), "Hello Hello Hello Hello ")
assert_eq!(value.as_str(), "Hello Hello Hello Hello ")
})
}
}
vm/src/vm/mod.rs
+1 -1
View File
@@ -610,7 +610,7 @@ impl VirtualMachine {
let from_list = from_list.to_pyobject(self);
import_func
.call((module.to_owned(), globals, locals, from_list, level), self)
.map_err(|exc| import::remove_importlib_frames(self, &exc))
.inspect_err(|exc| import::remove_importlib_frames(self, exc))
}
}
}