mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

Merge format and literal back into this repo (and update malachite) (#5618)

Browse Source 
* Merge format and literal back into this repo

* Update format and literal to work

* Update malachite

* Remove RustPython/Parser from Cargo.toml
This commit is contained in:
Noa authored and GitHub committed 2025-03-23 09:27:13 +09:00
1 parent eaf4cdf5e1
commit c585678ec9
22 files changed
+3240 -73
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+22 -39
View File
@@ -787,6 +787,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foreign-types"
version = "0.3.2"
@@ -896,15 +902,15 @@ name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
dependencies = [
"foldhash",
]
[[package]]
name = "heck"
@@ -1255,24 +1261,13 @@ dependencies = [
"libc",
]
[[package]]
name = "malachite"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5"
dependencies = [
"malachite-base",
"malachite-nz",
"malachite-q",
]
[[package]]
name = "malachite-base"
version = "0.4.22"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb"
checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713"
dependencies = [
"hashbrown 0.14.5",
"hashbrown 0.15.2",
"itertools 0.11.0",
"libm",
"ryu",
@@ -1280,12 +1275,13 @@ dependencies = [
[[package]]
name = "malachite-bigint"
version = "0.2.3"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8"
checksum = "7b1b1fec8b370139968919a5b77071c94b282eaba3da1cf179ae5299060d4e75"
dependencies = [
"derive_more",
"malachite",
"malachite-base",
"malachite-nz",
"num-integer",
"num-traits",
"paste",
@@ -1293,9 +1289,9 @@ dependencies = [
[[package]]
name = "malachite-nz"
version = "0.4.22"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7"
checksum = "175263cd5b846c552b9afb9d4b03ca465b4ad10717d789cad7dac24441c4fcbe"
dependencies = [
"itertools 0.11.0",
"libm",
@@ -1304,9 +1300,9 @@ dependencies = [
[[package]]
name = "malachite-q"
version = "0.4.22"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191"
checksum = "5261ba8feb1ad20cddab3d625af28206c663c08014b2e5c5f9bd54b0f230234f"
dependencies = [
"itertools 0.11.0",
"malachite-base",
@@ -2152,7 +2148,7 @@ dependencies = [
"parking_lot",
"radium",
"rand 0.9.0",
"rustpython-format",
"rustpython-literal",
"siphasher 0.3.11",
"volatile",
"widestring",
@@ -2229,18 +2225,6 @@ dependencies = [
"once_cell",
]
[[package]]
name = "rustpython-format"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"bitflags 2.8.0",
"itertools 0.11.0",
"malachite-bigint",
"num-traits",
"rustpython-literal",
]
[[package]]
name = "rustpython-jit"
version = "0.4.0"
@@ -2259,12 +2243,12 @@ dependencies = [
[[package]]
name = "rustpython-literal"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"hexf-parse",
"is-macro",
"lexical-parse-float",
"num-traits",
"rand 0.9.0",
"unic-ucd-category",
]
@@ -2416,7 +2400,6 @@ dependencies = [
"rustpython-compiler-core",
"rustpython-compiler-source",
"rustpython-derive",
"rustpython-format",
"rustpython-jit",
"rustpython-literal",
"rustpython-sre_engine",
Cargo.toml
+5 -19
View File
@@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies =
[workspace]
resolver = "2"
members = [
"compiler", "compiler/core", "compiler/codegen", "compiler/source",
"compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
"wasm/lib",
]
@@ -116,6 +116,7 @@ rustpython-common = { path = "common", version = "0.4.0" }
rustpython-derive = { path = "derive", version = "0.4.0" }
rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
rustpython-jit = { path = "jit", version = "0.4.0" }
rustpython-literal = { path = "compiler/literal", version = "0.4.0" }
rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
rustpython-pylib = { path = "pylib", version = "0.4.0" }
rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
@@ -127,21 +128,6 @@ ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0
ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
# rustpython-literal = { version = "0.4.0" }
# rustpython-parser-core = { version = "0.4.0" }
# rustpython-parser = { version = "0.4.0" }
# rustpython-ast = { version = "0.4.0" }
# rustpython-format= { version = "0.4.0" }
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-parser-core = { path = "../RustPython-parser/core" }
# rustpython-parser = { path = "../RustPython-parser/parser" }
# rustpython-ast = { path = "../RustPython-parser/ast" }
# rustpython-format = { path = "../RustPython-parser/format" }
ahash = "0.8.11"
ascii = "1.1"
@@ -163,9 +149,9 @@ junction = "1.2.0"
libc = "0.2.169"
log = "0.4.25"
nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
malachite-bigint = "0.2.3"
malachite-q = "0.4.22"
malachite-base = "0.4.22"
malachite-bigint = "0.5"
malachite-q = "0.5"
malachite-base = "0.5"
memchr = "2.7.4"
num-complex = "0.4.6"
num-integer = "0.1.46"
common/Cargo.toml
+2 -2
View File
@@ -12,7 +12,7 @@ license.workspace = true
threading = ["parking_lot"]
[dependencies]
rustpython-format = { workspace = true }
rustpython-literal = { workspace = true }
ascii = { workspace = true }
bitflags = { workspace = true }
@@ -45,4 +45,4 @@ windows-sys = { workspace = true, features = [
] }
[lints]
workspace = true
workspace = true
common/src/cformat.rs
+1044
View File
File diff suppressed because it is too large. Load diff
common/src/format.rs
+1328
View File
File diff suppressed because it is too large. Load diff
common/src/lib.rs
+2
View File
@@ -9,6 +9,7 @@ pub use macros::*;
pub mod atomic;
pub mod borrow;
pub mod boxvec;
pub mod cformat;
pub mod cmp;
#[cfg(any(unix, windows, target_os = "wasi"))]
pub mod crt_fd;
@@ -16,6 +17,7 @@ pub mod encodings;
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
pub mod fileutils;
pub mod float_ops;
pub mod format;
pub mod hash;
pub mod int;
pub mod linked_list;
common/src/str.rs
+1 -1
View File
@@ -1,9 +1,9 @@
use crate::{
atomic::{PyAtomic, Radium},
format::CharLen,
hash::PyHash,
};
use ascii::AsciiString;
use rustpython_format::CharLen;
use std::ops::{Bound, RangeBounds};
#[cfg(not(target_arch = "wasm32"))]
compiler/literal/Cargo.toml
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "rustpython-literal"
description = "Common literal handling utilities mostly useful for unparse and repr."
edition = { workspace = true }
version = { workspace = true }
authors = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
rust-version = { workspace = true }
[dependencies]
hexf-parse = "0.2.1"
is-macro.workspace = true
lexical-parse-float = { version = "0.8.0", features = ["format"] }
num-traits = { workspace = true }
unic-ucd-category = "0.9"
[dev-dependencies]
rand = { workspace = true }
compiler/literal/src/char.rs
+15
View File
@@ -0,0 +1,15 @@
use unic_ucd_category::GeneralCategory;
/// According to python following categories aren't printable:
/// * Cc (Other, Control)
/// * Cf (Other, Format)
/// * Cs (Other, Surrogate)
/// * Co (Other, Private Use)
/// * Cn (Other, Not Assigned)
/// * Zl Separator, Line ('\u2028', LINE SEPARATOR)
/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
/// * Zs (Separator, Space) other than ASCII space('\x20').
pub fn is_printable(c: char) -> bool {
let cat = GeneralCategory::of(c);
!(cat.is_other() || cat.is_separator())
}
compiler/literal/src/escape.rs
+441
View File
@@ -0,0 +1,441 @@
#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)]
pub enum Quote {
Single,
Double,
}
impl Quote {
#[inline]
pub const fn swap(self) -> Quote {
match self {
Quote::Single => Quote::Double,
Quote::Double => Quote::Single,
}
}
#[inline]
pub const fn to_byte(&self) -> u8 {
match self {
Quote::Single => b'\'',
Quote::Double => b'"',
}
}
#[inline]
pub const fn to_char(&self) -> char {
match self {
Quote::Single => '\'',
Quote::Double => '"',
}
}
}
pub struct EscapeLayout {
pub quote: Quote,
pub len: Option<usize>,
}
pub trait Escape {
fn source_len(&self) -> usize;
fn layout(&self) -> &EscapeLayout;
fn changed(&self) -> bool {
self.layout().len != Some(self.source_len())
}
fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
if self.changed() {
self.write_body_slow(formatter)
} else {
self.write_source(formatter)
}
}
}
/// Returns the outer quotes to use and the number of quotes that need to be
/// escaped.
pub(crate) const fn choose_quote(
single_count: usize,
double_count: usize,
preferred_quote: Quote,
) -> (Quote, usize) {
let (primary_count, secondary_count) = match preferred_quote {
Quote::Single => (single_count, double_count),
Quote::Double => (double_count, single_count),
};
// always use primary unless we have primary but no secondary
let use_secondary = primary_count > 0 && secondary_count == 0;
if use_secondary {
(preferred_quote.swap(), secondary_count)
} else {
(preferred_quote, primary_count)
}
}
pub struct UnicodeEscape<'a> {
source: &'a str,
layout: EscapeLayout,
}
impl<'a> UnicodeEscape<'a> {
#[inline]
pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self {
let layout = EscapeLayout { quote, len: None };
Self { source, layout }
}
#[inline]
pub fn with_preferred_quote(source: &'a str, quote: Quote) -> Self {
let layout = Self::repr_layout(source, quote);
Self { source, layout }
}
#[inline]
pub fn new_repr(source: &'a str) -> Self {
Self::with_preferred_quote(source, Quote::Single)
}
#[inline]
pub fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> {
StrRepr(self)
}
}
pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>);
impl StrRepr<'_, '_> {
pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
let quote = self.0.layout().quote.to_char();
formatter.write_char(quote)?;
self.0.write_body(formatter)?;
formatter.write_char(quote)
}
pub fn to_string(&self) -> Option<String> {
let mut s = String::with_capacity(self.0.layout().len?);
self.write(&mut s).unwrap();
Some(s)
}
}
impl std::fmt::Display for StrRepr<'_, '_> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.write(formatter)
}
}
impl UnicodeEscape<'_> {
const REPR_RESERVED_LEN: usize = 2; // for quotes
pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout {
Self::output_layout_with_checker(source, preferred_quote, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize)
})
}
fn output_layout_with_checker(
source: &str,
preferred_quote: Quote,
length_add: impl Fn(usize, usize) -> Option<usize>,
) -> EscapeLayout {
let mut out_len = Self::REPR_RESERVED_LEN;
let mut single_count = 0;
let mut double_count = 0;
for ch in source.chars() {
let incr = match ch {
'\'' => {
single_count += 1;
1
}
'"' => {
double_count += 1;
1
}
c => Self::escaped_char_len(c),
};
let Some(new_len) = length_add(out_len, incr) else {
#[cold]
fn stop(
single_count: usize,
double_count: usize,
preferred_quote: Quote,
) -> EscapeLayout {
EscapeLayout {
quote: choose_quote(single_count, double_count, preferred_quote).0,
len: None,
}
}
return stop(single_count, double_count, preferred_quote);
};
out_len = new_len;
}
let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
// we'll be adding backslashes in front of the existing inner quotes
let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
return EscapeLayout { quote, len: None };
};
EscapeLayout {
quote,
len: Some(out_len - Self::REPR_RESERVED_LEN),
}
}
fn escaped_char_len(ch: char) -> usize {
match ch {
'\\' | '\t' | '\r' | '\n' => 2,
ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH
ch if ch.is_ascii() => 1,
ch if crate::char::is_printable(ch) => {
// max = std::cmp::max(ch, max);
ch.len_utf8()
}
ch if (ch as u32) < 0x100 => 4, // \xHH
ch if (ch as u32) < 0x10000 => 6, // \uHHHH
_ => 10, // \uHHHHHHHH
}
}
fn write_char(
ch: char,
quote: Quote,
formatter: &mut impl std::fmt::Write,
) -> std::fmt::Result {
match ch {
'\n' => formatter.write_str("\\n"),
'\t' => formatter.write_str("\\t"),
'\r' => formatter.write_str("\\r"),
// these 2 branches *would* be handled below, but we shouldn't have to do a
// unicodedata lookup just for ascii characters
'\x20'..='\x7e' => {
// printable ascii range
if ch == quote.to_char() || ch == '\\' {
formatter.write_char('\\')?;
}
formatter.write_char(ch)
}
ch if ch.is_ascii() => {
write!(formatter, "\\x{:02x}", ch as u8)
}
ch if crate::char::is_printable(ch) => formatter.write_char(ch),
'\0'..='\u{ff}' => {
write!(formatter, "\\x{:02x}", ch as u32)
}
'\0'..='\u{ffff}' => {
write!(formatter, "\\u{:04x}", ch as u32)
}
_ => {
write!(formatter, "\\U{:08x}", ch as u32)
}
}
}
}
impl Escape for UnicodeEscape<'_> {
fn source_len(&self) -> usize {
self.source.len()
}
fn layout(&self) -> &EscapeLayout {
&self.layout
}
fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
formatter.write_str(self.source)
}
#[cold]
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
for ch in self.source.chars() {
Self::write_char(ch, self.layout().quote, formatter)?;
}
Ok(())
}
}
pub struct AsciiEscape<'a> {
source: &'a [u8],
layout: EscapeLayout,
}
impl<'a> AsciiEscape<'a> {
#[inline]
pub fn new(source: &'a [u8], layout: EscapeLayout) -> Self {
Self { source, layout }
}
#[inline]
pub fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self {
let layout = EscapeLayout { quote, len: None };
Self { source, layout }
}
#[inline]
pub fn with_preferred_quote(source: &'a [u8], quote: Quote) -> Self {
let layout = Self::repr_layout(source, quote);
Self { source, layout }
}
#[inline]
pub fn new_repr(source: &'a [u8]) -> Self {
Self::with_preferred_quote(source, Quote::Single)
}
#[inline]
pub fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> {
BytesRepr(self)
}
}
impl AsciiEscape<'_> {
pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout {
Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize)
})
}
pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout {
Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| {
Some((a as isize).checked_add(b as isize)? as usize)
})
}
fn output_layout_with_checker(
source: &[u8],
preferred_quote: Quote,
reserved_len: usize,
length_add: impl Fn(usize, usize) -> Option<usize>,
) -> EscapeLayout {
let mut out_len = reserved_len;
let mut single_count = 0;
let mut double_count = 0;
for ch in source.iter() {
let incr = match ch {
b'\'' => {
single_count += 1;
1
}
b'"' => {
double_count += 1;
1
}
c => Self::escaped_char_len(*c),
};
let Some(new_len) = length_add(out_len, incr) else {
#[cold]
fn stop(
single_count: usize,
double_count: usize,
preferred_quote: Quote,
) -> EscapeLayout {
EscapeLayout {
quote: choose_quote(single_count, double_count, preferred_quote).0,
len: None,
}
}
return stop(single_count, double_count, preferred_quote);
};
out_len = new_len;
}
let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
// we'll be adding backslashes in front of the existing inner quotes
let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
return EscapeLayout { quote, len: None };
};
EscapeLayout {
quote,
len: Some(out_len - reserved_len),
}
}
fn escaped_char_len(ch: u8) -> usize {
match ch {
b'\\' | b'\t' | b'\r' | b'\n' => 2,
0x20..=0x7e => 1,
_ => 4, // \xHH
}
}
fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
match ch {
b'\t' => formatter.write_str("\\t"),
b'\n' => formatter.write_str("\\n"),
b'\r' => formatter.write_str("\\r"),
0x20..=0x7e => {
// printable ascii range
if ch == quote.to_byte() || ch == b'\\' {
formatter.write_char('\\')?;
}
formatter.write_char(ch as char)
}
ch => write!(formatter, "\\x{ch:02x}"),
}
}
}
impl Escape for AsciiEscape<'_> {
fn source_len(&self) -> usize {
self.source.len()
}
fn layout(&self) -> &EscapeLayout {
&self.layout
}
fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
formatter.write_str(unsafe {
// SAFETY: this function must be called only when source is printable ascii characters
std::str::from_utf8_unchecked(self.source)
})
}
#[cold]
fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
for ch in self.source.iter() {
Self::write_char(*ch, self.layout().quote, formatter)?;
}
Ok(())
}
}
pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>);
impl BytesRepr<'_, '_> {
pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
let quote = self.0.layout().quote.to_char();
formatter.write_char('b')?;
formatter.write_char(quote)?;
self.0.write_body(formatter)?;
formatter.write_char(quote)
}
pub fn to_string(&self) -> Option<String> {
let mut s = String::with_capacity(self.0.layout().len?);
self.write(&mut s).unwrap();
Some(s)
}
}
impl std::fmt::Display for BytesRepr<'_, '_> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.write(formatter)
}
}
#[cfg(test)]
mod unicode_escape_tests {
use super::*;
#[test]
fn changed() {
fn test(s: &str) -> bool {
UnicodeEscape::new_repr(s).changed()
}
assert!(!test("hello"));
assert!(!test("'hello'"));
assert!(!test("\"hello\""));
assert!(test("'\"hello"));
assert!(test("hello\n"));
}
}
compiler/literal/src/float.rs
+342
View File
@@ -0,0 +1,342 @@
use crate::format::Case;
use num_traits::{Float, Zero};
use std::f64;
pub fn parse_str(literal: &str) -> Option<f64> {
parse_inner(literal.trim().as_bytes())
}
fn strip_underlines(literal: &[u8]) -> Option<Vec<u8>> {
let mut prev = b'\0';
let mut dup = Vec::<u8>::new();
for p in literal {
if *p == b'_' {
// Underscores are only allowed after digits.
if !prev.is_ascii_digit() {
return None;
}
} else {
dup.push(*p);
// Underscores are only allowed before digits.
if prev == b'_' && !p.is_ascii_digit() {
return None;
}
}
prev = *p;
}
// Underscores are not allowed at the end.
if prev == b'_' {
return None;
}
Some(dup)
}
pub fn parse_bytes(literal: &[u8]) -> Option<f64> {
parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace()))
}
fn trim_slice<T>(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] {
let mut it = v.iter();
// it.take_while_ref(&mut trim).for_each(drop);
// hmm.. `&mut slice::Iter<_>` is not `Clone`
// it.by_ref().rev().take_while_ref(&mut trim).for_each(drop);
while it.clone().next().is_some_and(&mut trim) {
it.next();
}
while it.clone().next_back().is_some_and(&mut trim) {
it.next_back();
}
it.as_slice()
}
fn parse_inner(literal: &[u8]) -> Option<f64> {
use lexical_parse_float::{
FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL,
};
// Use custom function for underline handling for now.
// For further information see https://github.com/Alexhuszagh/rust-lexical/issues/96.
let stripped = strip_underlines(literal)?;
// lexical-core's format::PYTHON_STRING is inaccurate
const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL)
.no_special(false)
.build();
f64::from_lexical_with_options::<PYTHON_STRING>(&stripped, &Options::new()).ok()
}
pub fn is_integer(v: f64) -> bool {
(v - v.round()).abs() < f64::EPSILON
}
fn format_nan(case: Case) -> String {
let nan = match case {
Case::Lower => "nan",
Case::Upper => "NAN",
};
nan.to_string()
}
fn format_inf(case: Case) -> String {
let inf = match case {
Case::Lower => "inf",
Case::Upper => "INF",
};
inf.to_string()
}
pub fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str {
match (precision, alternate_form) {
(0, true) => ".",
_ => "",
}
}
pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String {
match magnitude {
magnitude if magnitude.is_finite() => {
let point = decimal_point_or_empty(precision, alternate_form);
format!("{magnitude:.precision$}{point}")
}
magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(),
}
}
// Formats floats into Python style exponent notation, by first formatting in Rust style
// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`).
pub fn format_exponent(
precision: usize,
magnitude: f64,
case: Case,
alternate_form: bool,
) -> String {
match magnitude {
magnitude if magnitude.is_finite() => {
let r_exp = format!("{magnitude:.precision$e}");
let mut parts = r_exp.splitn(2, 'e');
let base = parts.next().unwrap();
let exponent = parts.next().unwrap().parse::<i64>().unwrap();
let e = match case {
Case::Lower => 'e',
Case::Upper => 'E',
};
let point = decimal_point_or_empty(precision, alternate_form);
format!("{base}{point}{e}{exponent:+#03}")
}
magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(),
}
}
/// If s represents a floating point value, trailing zeros and a possibly trailing
/// decimal point will be removed.
/// This function does NOT work with decimal commas.
fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String {
if !alternate_form && s.contains('.') {
// only truncate floating point values when not in alternate form
let s = remove_trailing_zeros(s);
remove_trailing_decimal_point(s)
} else {
s
}
}
fn remove_trailing_zeros(s: String) -> String {
let mut s = s;
while s.ends_with('0') {
s.pop();
}
s
}
fn remove_trailing_decimal_point(s: String) -> String {
let mut s = s;
if s.ends_with('.') {
s.pop();
}
s
}
pub fn format_general(
precision: usize,
magnitude: f64,
case: Case,
alternate_form: bool,
always_shows_fract: bool,
) -> String {
match magnitude {
magnitude if magnitude.is_finite() => {
let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude);
let mut parts = r_exp.splitn(2, 'e');
let base = parts.next().unwrap();
let exponent = parts.next().unwrap().parse::<i64>().unwrap();
if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) {
let e = match case {
Case::Lower => 'e',
Case::Upper => 'E',
};
let magnitude = format!("{:.*}", precision + 1, base);
let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form);
format!("{base}{point}{e}{exponent:+#03}")
} else {
let precision = ((precision as i64) - 1 - exponent) as usize;
let magnitude = format!("{magnitude:.precision$}");
let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
let point = decimal_point_or_empty(precision, alternate_form);
format!("{base}{point}")
}
}
magnitude if magnitude.is_nan() => format_nan(case),
magnitude if magnitude.is_infinite() => format_inf(case),
_ => "".to_string(),
}
}
// TODO: rewrite using format_general
pub fn to_string(value: f64) -> String {
let lit = format!("{value:e}");
if let Some(position) = lit.find('e') {
let significand = &lit[..position];
let exponent = &lit[position + 1..];
let exponent = exponent.parse::<i32>().unwrap();
if exponent < 16 && exponent > -5 {
if is_integer(value) {
format!("{value:.1?}")
} else {
value.to_string()
}
} else {
format!("{significand}e{exponent:+#03}")
}
} else {
let mut s = value.to_string();
s.make_ascii_lowercase();
s
}
}
pub fn from_hex(s: &str) -> Option<f64> {
if let Ok(f) = hexf_parse::parse_hexf64(s, false) {
return Some(f);
}
match s.to_ascii_lowercase().as_str() {
"nan" | "+nan" | "-nan" => Some(f64::NAN),
"inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY),
"-inf" | "-infinity" => Some(f64::NEG_INFINITY),
value => {
let mut hex = String::with_capacity(value.len());
let has_0x = value.contains("0x");
let has_p = value.contains('p');
let has_dot = value.contains('.');
let mut start = 0;
if !has_0x && value.starts_with('-') {
hex.push_str("-0x");
start += 1;
} else if !has_0x {
hex.push_str("0x");
if value.starts_with('+') {
start += 1;
}
}
for (index, ch) in value.chars().enumerate() {
if ch == 'p' {
if has_dot {
hex.push('p');
} else {
hex.push_str(".p");
}
} else if index >= start {
hex.push(ch);
}
}
if !has_p && has_dot {
hex.push_str("p0");
} else if !has_p && !has_dot {
hex.push_str(".p0")
}
hexf_parse::parse_hexf64(hex.as_str(), false).ok()
}
}
}
pub fn to_hex(value: f64) -> String {
let (mantissa, exponent, sign) = value.integer_decode();
let sign_fmt = if sign < 0 { "-" } else { "" };
match value {
value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"),
value if value.is_infinite() => format!("{sign_fmt}inf"),
value if value.is_nan() => "nan".to_owned(),
_ => {
const BITS: i16 = 52;
const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff;
format!(
"{}{:#x}.{:013x}p{:+}",
sign_fmt,
mantissa >> BITS,
mantissa & FRACT_MASK,
exponent + BITS
)
}
}
}
#[test]
fn test_to_hex() {
use rand::Rng;
for _ in 0..20000 {
let bytes = rand::rng().random::<u64>();
let f = f64::from_bits(bytes);
if !f.is_finite() {
continue;
}
let hex = to_hex(f);
// println!("{} -> {}", f, hex);
let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap();
// println!(" -> {}", roundtrip);
assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip);
}
}
#[test]
fn test_remove_trailing_zeros() {
assert!(remove_trailing_zeros(String::from("100")) == *"1");
assert!(remove_trailing_zeros(String::from("100.00")) == *"100.");
// leave leading zeros untouched
assert!(remove_trailing_zeros(String::from("001")) == *"001");
// leave strings untouched if they don't end with 0
assert!(remove_trailing_zeros(String::from("101")) == *"101");
}
#[test]
fn test_remove_trailing_decimal_point() {
assert!(remove_trailing_decimal_point(String::from("100.")) == *"100");
assert!(remove_trailing_decimal_point(String::from("1.")) == *"1");
// leave leading decimal points untouched
assert!(remove_trailing_decimal_point(String::from(".5")) == *".5");
}
#[test]
fn test_maybe_remove_trailing_redundant_chars() {
assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100.");
assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100");
assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1");
assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10");
// don't truncate integers
assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000");
}
compiler/literal/src/format.rs
+5
View File
@@ -0,0 +1,5 @@
#[derive(Debug, PartialEq, Eq, Clone, Copy, is_macro::Is, Hash)]
pub enum Case {
Lower,
Upper,
}
compiler/literal/src/lib.rs
+4
View File
@@ -0,0 +1,4 @@
pub mod char;
pub mod escape;
pub mod float;
pub mod format;
deny.toml
+1 -1
View File
@@ -242,4 +242,4 @@ unknown-git = "warn"
# if not specified. If it is specified but empty, no registries are allowed.
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# List of URLs for allowed Git repositories
allow-git = ["https://github.com/RustPython/Parser.git", "https://github.com/RustPython/__doc__"]
allow-git = ["https://github.com/RustPython/__doc__"]
vm/Cargo.toml
-1
View File
@@ -42,7 +42,6 @@ rustpython-compiler-core = { workspace = true }
rustpython-compiler-source = { workspace = true }
# rustpython-parser-core = { workspace = true }
rustpython-literal = { workspace = true }
rustpython-format = { workspace = true }
rustpython-sre_engine = { workspace = true }
ascii = { workspace = true }
vm/src/builtins/bool.rs
+1 -1
View File
@@ -1,4 +1,5 @@
use super::{PyInt, PyStrRef, PyType, PyTypeRef};
use crate::common::format::FormatSpec;
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject,
VirtualMachine,
@@ -11,7 +12,6 @@ use crate::{
};
use malachite_bigint::Sign;
use num_traits::Zero;
use rustpython_format::FormatSpec;
use std::fmt::{Debug, Formatter};
impl ToPyObject for bool {
vm/src/builtins/float.rs
+1 -2
View File
@@ -7,7 +7,7 @@ use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
TryFromBorrowedObject, TryFromObject, VirtualMachine,
class::PyClassImpl,
common::{float_ops, hash},
common::{float_ops, format::FormatSpec, hash},
convert::{IntoPyException, ToPyObject, ToPyResult},
function::{
ArgBytesLike, OptionalArg, OptionalOption,
@@ -21,7 +21,6 @@ use malachite_bigint::{BigInt, ToBigInt};
use num_complex::Complex64;
use num_traits::{Signed, ToPrimitive, Zero};
use rustpython_common::int::float_to_ratio;
use rustpython_format::FormatSpec;
#[pyclass(module = false, name = "float")]
#[derive(Debug, Copy, Clone, PartialEq)]
vm/src/builtins/int.rs
+1 -1
View File
@@ -6,6 +6,7 @@ use crate::{
bytesinner::PyBytesInner,
class::PyClassImpl,
common::{
format::FormatSpec,
hash,
int::{bigint_to_finite_float, bytes_to_int, true_div},
},
@@ -20,7 +21,6 @@ use crate::{
use malachite_bigint::{BigInt, Sign};
use num_integer::Integer;
use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero};
use rustpython_format::FormatSpec;
use std::fmt;
use std::ops::{Neg, Not};
vm/src/builtins/str.rs
+1 -1
View File
@@ -31,10 +31,10 @@ use once_cell::sync::Lazy;
use rustpython_common::{
ascii,
atomic::{self, PyAtomic, Radium},
format::{FormatSpec, FormatString, FromTemplate},
hash,
lock::PyMutex,
};
use rustpython_format::{FormatSpec, FormatString, FromTemplate};
use std::{char, fmt, ops::Range, string::ToString};
use unic_ucd_bidi::BidiClass;
use unic_ucd_category::GeneralCategory;
vm/src/cformat.rs
+1 -1
View File
@@ -1,6 +1,7 @@
//! Implementation of Printf-Style string formatting
//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting).
use crate::common::cformat::*;
use crate::{
AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine,
builtins::{
@@ -12,7 +13,6 @@ use crate::{
};
use itertools::Itertools;
use num_traits::cast::ToPrimitive;
use rustpython_format::cformat::*;
use std::str::FromStr;
fn spec_format_bytes(
vm/src/format.rs
+1 -1
View File
@@ -6,7 +6,7 @@ use crate::{
stdlib::builtins,
};
use rustpython_format::*;
use crate::common::format::*;
impl IntoPyException for FormatSpecError {
fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef {
vm/src/stdlib/string.rs
+3 -3
View File
@@ -6,15 +6,15 @@ pub(crate) use _string::make_module;
#[pymodule]
mod _string {
use crate::common::ascii;
use crate::common::format::{
FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
};
use crate::{
PyObjectRef, PyResult, VirtualMachine,
builtins::{PyList, PyStrRef},
convert::ToPyException,
convert::ToPyObject,
};
use rustpython_format::{
FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
};
use std::mem;
fn create_format_part(