Merge format and literal back into this repo (and update malachite) (#5618)

* Merge format and literal back into this repo

* Update format and literal to work

* Update malachite

* Remove RustPython/Parser from Cargo.toml

Cargo.lock

@@ -787,6 +787,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "foreign-types"
 version = "0.3.2"
@@ -896,15 +902,15 @@ name = "hashbrown"
 version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-dependencies = [
- "ahash",
-]
 
 [[package]]
 name = "hashbrown"
 version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+dependencies = [
+ "foldhash",
+]
 
 [[package]]
 name = "heck"
@@ -1255,24 +1261,13 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "malachite"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5"
-dependencies = [
- "malachite-base",
- "malachite-nz",
- "malachite-q",
-]
-
 [[package]]
 name = "malachite-base"
-version = "0.4.22"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb"
+checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713"
 dependencies = [
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
  "itertools 0.11.0",
  "libm",
  "ryu",
@@ -1280,12 +1275,13 @@ dependencies = [
 
 [[package]]
 name = "malachite-bigint"
-version = "0.2.3"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8"
+checksum = "7b1b1fec8b370139968919a5b77071c94b282eaba3da1cf179ae5299060d4e75"
 dependencies = [
  "derive_more",
- "malachite",
+ "malachite-base",
+ "malachite-nz",
  "num-integer",
  "num-traits",
  "paste",
@@ -1293,9 +1289,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-nz"
-version = "0.4.22"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7"
+checksum = "175263cd5b846c552b9afb9d4b03ca465b4ad10717d789cad7dac24441c4fcbe"
 dependencies = [
  "itertools 0.11.0",
  "libm",
@@ -1304,9 +1300,9 @@ dependencies = [
 
 [[package]]
 name = "malachite-q"
-version = "0.4.22"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191"
+checksum = "5261ba8feb1ad20cddab3d625af28206c663c08014b2e5c5f9bd54b0f230234f"
 dependencies = [
  "itertools 0.11.0",
  "malachite-base",
@@ -2152,7 +2148,7 @@ dependencies = [
  "parking_lot",
  "radium",
  "rand 0.9.0",
- "rustpython-format",
+ "rustpython-literal",
  "siphasher 0.3.11",
  "volatile",
  "widestring",
@@ -2229,18 +2225,6 @@ dependencies = [
  "once_cell",
 ]
 
-[[package]]
-name = "rustpython-format"
-version = "0.4.0"
-source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
-dependencies = [
- "bitflags 2.8.0",
- "itertools 0.11.0",
- "malachite-bigint",
- "num-traits",
- "rustpython-literal",
-]
-
 [[package]]
 name = "rustpython-jit"
 version = "0.4.0"
@@ -2259,12 +2243,12 @@ dependencies = [
 [[package]]
 name = "rustpython-literal"
 version = "0.4.0"
-source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
 dependencies = [
  "hexf-parse",
  "is-macro",
  "lexical-parse-float",
  "num-traits",
+ "rand 0.9.0",
  "unic-ucd-category",
 ]
 
@@ -2416,7 +2400,6 @@ dependencies = [
  "rustpython-compiler-core",
  "rustpython-compiler-source",
  "rustpython-derive",
- "rustpython-format",
  "rustpython-jit",
  "rustpython-literal",
  "rustpython-sre_engine",

Cargo.toml

@@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies =
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/core", "compiler/codegen", "compiler/source",
+    "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source",
     ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
     "wasm/lib",
 ]
@@ -116,6 +116,7 @@ rustpython-common = { path = "common", version = "0.4.0" }
 rustpython-derive = { path = "derive", version = "0.4.0" }
 rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" }
 rustpython-jit = { path = "jit", version = "0.4.0" }
+rustpython-literal = { path = "compiler/literal", version = "0.4.0" }
 rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" }
 rustpython-pylib = { path = "pylib", version = "0.4.0" }
 rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
@@ -127,21 +128,6 @@ ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0
 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
 ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" }
-# rustpython-literal = { version = "0.4.0" }
-# rustpython-parser-core = { version = "0.4.0" }
-# rustpython-parser = { version = "0.4.0" }
-# rustpython-ast = { version = "0.4.0" }
-# rustpython-format= { version = "0.4.0" }
-rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
-# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
-# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
-# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
-rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
-# rustpython-literal = { path = "../RustPython-parser/literal" }
-# rustpython-parser-core = { path = "../RustPython-parser/core" }
-# rustpython-parser = { path = "../RustPython-parser/parser" }
-# rustpython-ast = { path = "../RustPython-parser/ast" }
-# rustpython-format = { path = "../RustPython-parser/format" }
 
 ahash = "0.8.11"
 ascii = "1.1"
@@ -163,9 +149,9 @@ junction = "1.2.0"
 libc = "0.2.169"
 log = "0.4.25"
 nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
-malachite-bigint = "0.2.3"
+malachite-bigint = "0.5"
-malachite-q = "0.4.22"
+malachite-q = "0.5"
-malachite-base = "0.4.22"
+malachite-base = "0.5"
 memchr = "2.7.4"
 num-complex = "0.4.6"
 num-integer = "0.1.46"

common/Cargo.toml

@@ -12,7 +12,7 @@ license.workspace = true
 threading = ["parking_lot"]
 
 [dependencies]
-rustpython-format = { workspace = true }
+rustpython-literal = { workspace = true }
 
 ascii = { workspace = true }
 bitflags = { workspace = true }
@@ -45,4 +45,4 @@ windows-sys = { workspace = true, features = [
 ] }
 
 [lints]
-workspace = true
+workspace = true

common/src/lib.rs

@@ -9,6 +9,7 @@ pub use macros::*;
 pub mod atomic;
 pub mod borrow;
 pub mod boxvec;
+pub mod cformat;
 pub mod cmp;
 #[cfg(any(unix, windows, target_os = "wasi"))]
 pub mod crt_fd;
@@ -16,6 +17,7 @@ pub mod encodings;
 #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
 pub mod fileutils;
 pub mod float_ops;
+pub mod format;
 pub mod hash;
 pub mod int;
 pub mod linked_list;

common/src/str.rs

@@ -1,9 +1,9 @@
 use crate::{
     atomic::{PyAtomic, Radium},
+    format::CharLen,
     hash::PyHash,
 };
 use ascii::AsciiString;
-use rustpython_format::CharLen;
 use std::ops::{Bound, RangeBounds};
 
 #[cfg(not(target_arch = "wasm32"))]

compiler/literal/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "rustpython-literal"
+description = "Common literal handling utilities mostly useful for unparse and repr."
+edition = { workspace = true }
+version = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+hexf-parse = "0.2.1"
+is-macro.workspace = true
+lexical-parse-float = { version = "0.8.0", features = ["format"] }
+num-traits = { workspace = true }
+unic-ucd-category = "0.9"
+
+[dev-dependencies]
+rand = { workspace = true }

compiler/literal/src/char.rs

@@ -0,0 +1,15 @@
+use unic_ucd_category::GeneralCategory;
+
+/// According to python following categories aren't printable:
+/// * Cc (Other, Control)
+/// * Cf (Other, Format)
+/// * Cs (Other, Surrogate)
+/// * Co (Other, Private Use)
+/// * Cn (Other, Not Assigned)
+/// * Zl Separator, Line ('\u2028', LINE SEPARATOR)
+/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
+/// * Zs (Separator, Space) other than ASCII space('\x20').
+pub fn is_printable(c: char) -> bool {
+    let cat = GeneralCategory::of(c);
+    !(cat.is_other() || cat.is_separator())
+}

compiler/literal/src/escape.rs

@@ -0,0 +1,441 @@
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)]
+pub enum Quote {
+    Single,
+    Double,
+}
+
+impl Quote {
+    #[inline]
+    pub const fn swap(self) -> Quote {
+        match self {
+            Quote::Single => Quote::Double,
+            Quote::Double => Quote::Single,
+        }
+    }
+
+    #[inline]
+    pub const fn to_byte(&self) -> u8 {
+        match self {
+            Quote::Single => b'\'',
+            Quote::Double => b'"',
+        }
+    }
+
+    #[inline]
+    pub const fn to_char(&self) -> char {
+        match self {
+            Quote::Single => '\'',
+            Quote::Double => '"',
+        }
+    }
+}
+
+pub struct EscapeLayout {
+    pub quote: Quote,
+    pub len: Option<usize>,
+}
+
+pub trait Escape {
+    fn source_len(&self) -> usize;
+    fn layout(&self) -> &EscapeLayout;
+    fn changed(&self) -> bool {
+        self.layout().len != Some(self.source_len())
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
+    fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        if self.changed() {
+            self.write_body_slow(formatter)
+        } else {
+            self.write_source(formatter)
+        }
+    }
+}
+
+/// Returns the outer quotes to use and the number of quotes that need to be
+/// escaped.
+pub(crate) const fn choose_quote(
+    single_count: usize,
+    double_count: usize,
+    preferred_quote: Quote,
+) -> (Quote, usize) {
+    let (primary_count, secondary_count) = match preferred_quote {
+        Quote::Single => (single_count, double_count),
+        Quote::Double => (double_count, single_count),
+    };
+
+    // always use primary unless we have primary but no secondary
+    let use_secondary = primary_count > 0 && secondary_count == 0;
+    if use_secondary {
+        (preferred_quote.swap(), secondary_count)
+    } else {
+        (preferred_quote, primary_count)
+    }
+}
+
+pub struct UnicodeEscape<'a> {
+    source: &'a str,
+    layout: EscapeLayout,
+}
+
+impl<'a> UnicodeEscape<'a> {
+    #[inline]
+    pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self {
+        let layout = EscapeLayout { quote, len: None };
+        Self { source, layout }
+    }
+    #[inline]
+    pub fn with_preferred_quote(source: &'a str, quote: Quote) -> Self {
+        let layout = Self::repr_layout(source, quote);
+        Self { source, layout }
+    }
+    #[inline]
+    pub fn new_repr(source: &'a str) -> Self {
+        Self::with_preferred_quote(source, Quote::Single)
+    }
+    #[inline]
+    pub fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> {
+        StrRepr(self)
+    }
+}
+
+pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>);
+
+impl StrRepr<'_, '_> {
+    pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        let quote = self.0.layout().quote.to_char();
+        formatter.write_char(quote)?;
+        self.0.write_body(formatter)?;
+        formatter.write_char(quote)
+    }
+
+    pub fn to_string(&self) -> Option<String> {
+        let mut s = String::with_capacity(self.0.layout().len?);
+        self.write(&mut s).unwrap();
+        Some(s)
+    }
+}
+
+impl std::fmt::Display for StrRepr<'_, '_> {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.write(formatter)
+    }
+}
+
+impl UnicodeEscape<'_> {
+    const REPR_RESERVED_LEN: usize = 2; // for quotes
+
+    pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout {
+        Self::output_layout_with_checker(source, preferred_quote, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    fn output_layout_with_checker(
+        source: &str,
+        preferred_quote: Quote,
+        length_add: impl Fn(usize, usize) -> Option<usize>,
+    ) -> EscapeLayout {
+        let mut out_len = Self::REPR_RESERVED_LEN;
+        let mut single_count = 0;
+        let mut double_count = 0;
+
+        for ch in source.chars() {
+            let incr = match ch {
+                '\'' => {
+                    single_count += 1;
+                    1
+                }
+                '"' => {
+                    double_count += 1;
+                    1
+                }
+                c => Self::escaped_char_len(c),
+            };
+            let Some(new_len) = length_add(out_len, incr) else {
+                #[cold]
+                fn stop(
+                    single_count: usize,
+                    double_count: usize,
+                    preferred_quote: Quote,
+                ) -> EscapeLayout {
+                    EscapeLayout {
+                        quote: choose_quote(single_count, double_count, preferred_quote).0,
+                        len: None,
+                    }
+                }
+                return stop(single_count, double_count, preferred_quote);
+            };
+            out_len = new_len;
+        }
+
+        let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
+        // we'll be adding backslashes in front of the existing inner quotes
+        let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
+            return EscapeLayout { quote, len: None };
+        };
+
+        EscapeLayout {
+            quote,
+            len: Some(out_len - Self::REPR_RESERVED_LEN),
+        }
+    }
+
+    fn escaped_char_len(ch: char) -> usize {
+        match ch {
+            '\\' | '\t' | '\r' | '\n' => 2,
+            ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH
+            ch if ch.is_ascii() => 1,
+            ch if crate::char::is_printable(ch) => {
+                // max = std::cmp::max(ch, max);
+                ch.len_utf8()
+            }
+            ch if (ch as u32) < 0x100 => 4,   // \xHH
+            ch if (ch as u32) < 0x10000 => 6, // \uHHHH
+            _ => 10,                          // \uHHHHHHHH
+        }
+    }
+
+    fn write_char(
+        ch: char,
+        quote: Quote,
+        formatter: &mut impl std::fmt::Write,
+    ) -> std::fmt::Result {
+        match ch {
+            '\n' => formatter.write_str("\\n"),
+            '\t' => formatter.write_str("\\t"),
+            '\r' => formatter.write_str("\\r"),
+            // these 2 branches *would* be handled below, but we shouldn't have to do a
+            // unicodedata lookup just for ascii characters
+            '\x20'..='\x7e' => {
+                // printable ascii range
+                if ch == quote.to_char() || ch == '\\' {
+                    formatter.write_char('\\')?;
+                }
+                formatter.write_char(ch)
+            }
+            ch if ch.is_ascii() => {
+                write!(formatter, "\\x{:02x}", ch as u8)
+            }
+            ch if crate::char::is_printable(ch) => formatter.write_char(ch),
+            '\0'..='\u{ff}' => {
+                write!(formatter, "\\x{:02x}", ch as u32)
+            }
+            '\0'..='\u{ffff}' => {
+                write!(formatter, "\\u{:04x}", ch as u32)
+            }
+            _ => {
+                write!(formatter, "\\U{:08x}", ch as u32)
+            }
+        }
+    }
+}
+
+impl Escape for UnicodeEscape<'_> {
+    fn source_len(&self) -> usize {
+        self.source.len()
+    }
+
+    fn layout(&self) -> &EscapeLayout {
+        &self.layout
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        formatter.write_str(self.source)
+    }
+
+    #[cold]
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        for ch in self.source.chars() {
+            Self::write_char(ch, self.layout().quote, formatter)?;
+        }
+        Ok(())
+    }
+}
+
+pub struct AsciiEscape<'a> {
+    source: &'a [u8],
+    layout: EscapeLayout,
+}
+
+impl<'a> AsciiEscape<'a> {
+    #[inline]
+    pub fn new(source: &'a [u8], layout: EscapeLayout) -> Self {
+        Self { source, layout }
+    }
+    #[inline]
+    pub fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self {
+        let layout = EscapeLayout { quote, len: None };
+        Self { source, layout }
+    }
+    #[inline]
+    pub fn with_preferred_quote(source: &'a [u8], quote: Quote) -> Self {
+        let layout = Self::repr_layout(source, quote);
+        Self { source, layout }
+    }
+    #[inline]
+    pub fn new_repr(source: &'a [u8]) -> Self {
+        Self::with_preferred_quote(source, Quote::Single)
+    }
+    #[inline]
+    pub fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> {
+        BytesRepr(self)
+    }
+}
+
+impl AsciiEscape<'_> {
+    pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout {
+        Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout {
+        Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    fn output_layout_with_checker(
+        source: &[u8],
+        preferred_quote: Quote,
+        reserved_len: usize,
+        length_add: impl Fn(usize, usize) -> Option<usize>,
+    ) -> EscapeLayout {
+        let mut out_len = reserved_len;
+        let mut single_count = 0;
+        let mut double_count = 0;
+
+        for ch in source.iter() {
+            let incr = match ch {
+                b'\'' => {
+                    single_count += 1;
+                    1
+                }
+                b'"' => {
+                    double_count += 1;
+                    1
+                }
+                c => Self::escaped_char_len(*c),
+            };
+            let Some(new_len) = length_add(out_len, incr) else {
+                #[cold]
+                fn stop(
+                    single_count: usize,
+                    double_count: usize,
+                    preferred_quote: Quote,
+                ) -> EscapeLayout {
+                    EscapeLayout {
+                        quote: choose_quote(single_count, double_count, preferred_quote).0,
+                        len: None,
+                    }
+                }
+                return stop(single_count, double_count, preferred_quote);
+            };
+            out_len = new_len;
+        }
+
+        let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
+        // we'll be adding backslashes in front of the existing inner quotes
+        let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
+            return EscapeLayout { quote, len: None };
+        };
+
+        EscapeLayout {
+            quote,
+            len: Some(out_len - reserved_len),
+        }
+    }
+
+    fn escaped_char_len(ch: u8) -> usize {
+        match ch {
+            b'\\' | b'\t' | b'\r' | b'\n' => 2,
+            0x20..=0x7e => 1,
+            _ => 4, // \xHH
+        }
+    }
+
+    fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        match ch {
+            b'\t' => formatter.write_str("\\t"),
+            b'\n' => formatter.write_str("\\n"),
+            b'\r' => formatter.write_str("\\r"),
+            0x20..=0x7e => {
+                // printable ascii range
+                if ch == quote.to_byte() || ch == b'\\' {
+                    formatter.write_char('\\')?;
+                }
+                formatter.write_char(ch as char)
+            }
+            ch => write!(formatter, "\\x{ch:02x}"),
+        }
+    }
+}
+
+impl Escape for AsciiEscape<'_> {
+    fn source_len(&self) -> usize {
+        self.source.len()
+    }
+
+    fn layout(&self) -> &EscapeLayout {
+        &self.layout
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        formatter.write_str(unsafe {
+            // SAFETY: this function must be called only when source is printable ascii characters
+            std::str::from_utf8_unchecked(self.source)
+        })
+    }
+
+    #[cold]
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        for ch in self.source.iter() {
+            Self::write_char(*ch, self.layout().quote, formatter)?;
+        }
+        Ok(())
+    }
+}
+
+pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>);
+
+impl BytesRepr<'_, '_> {
+    pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        let quote = self.0.layout().quote.to_char();
+        formatter.write_char('b')?;
+        formatter.write_char(quote)?;
+        self.0.write_body(formatter)?;
+        formatter.write_char(quote)
+    }
+
+    pub fn to_string(&self) -> Option<String> {
+        let mut s = String::with_capacity(self.0.layout().len?);
+        self.write(&mut s).unwrap();
+        Some(s)
+    }
+}
+
+impl std::fmt::Display for BytesRepr<'_, '_> {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.write(formatter)
+    }
+}
+
+#[cfg(test)]
+mod unicode_escape_tests {
+    use super::*;
+
+    #[test]
+    fn changed() {
+        fn test(s: &str) -> bool {
+            UnicodeEscape::new_repr(s).changed()
+        }
+        assert!(!test("hello"));
+        assert!(!test("'hello'"));
+        assert!(!test("\"hello\""));
+
+        assert!(test("'\"hello"));
+        assert!(test("hello\n"));
+    }
+}

compiler/literal/src/float.rs

@@ -0,0 +1,342 @@
+use crate::format::Case;
+use num_traits::{Float, Zero};
+use std::f64;
+
+pub fn parse_str(literal: &str) -> Option<f64> {
+    parse_inner(literal.trim().as_bytes())
+}
+
+fn strip_underlines(literal: &[u8]) -> Option<Vec<u8>> {
+    let mut prev = b'\0';
+    let mut dup = Vec::<u8>::new();
+    for p in literal {
+        if *p == b'_' {
+            // Underscores are only allowed after digits.
+            if !prev.is_ascii_digit() {
+                return None;
+            }
+        } else {
+            dup.push(*p);
+            // Underscores are only allowed before digits.
+            if prev == b'_' && !p.is_ascii_digit() {
+                return None;
+            }
+        }
+        prev = *p;
+    }
+
+    // Underscores are not allowed at the end.
+    if prev == b'_' {
+        return None;
+    }
+
+    Some(dup)
+}
+
+pub fn parse_bytes(literal: &[u8]) -> Option<f64> {
+    parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace()))
+}
+
+fn trim_slice<T>(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] {
+    let mut it = v.iter();
+    // it.take_while_ref(&mut trim).for_each(drop);
+    // hmm.. `&mut slice::Iter<_>` is not `Clone`
+    // it.by_ref().rev().take_while_ref(&mut trim).for_each(drop);
+    while it.clone().next().is_some_and(&mut trim) {
+        it.next();
+    }
+    while it.clone().next_back().is_some_and(&mut trim) {
+        it.next_back();
+    }
+    it.as_slice()
+}
+
+fn parse_inner(literal: &[u8]) -> Option<f64> {
+    use lexical_parse_float::{
+        FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL,
+    };
+
+    // Use custom function for underline handling for now.
+    // For further information see https://github.com/Alexhuszagh/rust-lexical/issues/96.
+    let stripped = strip_underlines(literal)?;
+
+    // lexical-core's format::PYTHON_STRING is inaccurate
+    const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL)
+        .no_special(false)
+        .build();
+    f64::from_lexical_with_options::<PYTHON_STRING>(&stripped, &Options::new()).ok()
+}
+
+pub fn is_integer(v: f64) -> bool {
+    (v - v.round()).abs() < f64::EPSILON
+}
+
+fn format_nan(case: Case) -> String {
+    let nan = match case {
+        Case::Lower => "nan",
+        Case::Upper => "NAN",
+    };
+
+    nan.to_string()
+}
+
+fn format_inf(case: Case) -> String {
+    let inf = match case {
+        Case::Lower => "inf",
+        Case::Upper => "INF",
+    };
+
+    inf.to_string()
+}
+
+pub fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str {
+    match (precision, alternate_form) {
+        (0, true) => ".",
+        _ => "",
+    }
+}
+
+pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let point = decimal_point_or_empty(precision, alternate_form);
+            format!("{magnitude:.precision$}{point}")
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+// Formats floats into Python style exponent notation, by first formatting in Rust style
+// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`).
+pub fn format_exponent(
+    precision: usize,
+    magnitude: f64,
+    case: Case,
+    alternate_form: bool,
+) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let r_exp = format!("{magnitude:.precision$e}");
+            let mut parts = r_exp.splitn(2, 'e');
+            let base = parts.next().unwrap();
+            let exponent = parts.next().unwrap().parse::<i64>().unwrap();
+            let e = match case {
+                Case::Lower => 'e',
+                Case::Upper => 'E',
+            };
+            let point = decimal_point_or_empty(precision, alternate_form);
+            format!("{base}{point}{e}{exponent:+#03}")
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+/// If s represents a floating point value, trailing zeros and a possibly trailing
+/// decimal point will be removed.
+/// This function does NOT work with decimal commas.
+fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String {
+    if !alternate_form && s.contains('.') {
+        // only truncate floating point values when not in alternate form
+        let s = remove_trailing_zeros(s);
+        remove_trailing_decimal_point(s)
+    } else {
+        s
+    }
+}
+
+fn remove_trailing_zeros(s: String) -> String {
+    let mut s = s;
+    while s.ends_with('0') {
+        s.pop();
+    }
+    s
+}
+
+fn remove_trailing_decimal_point(s: String) -> String {
+    let mut s = s;
+    if s.ends_with('.') {
+        s.pop();
+    }
+    s
+}
+
+pub fn format_general(
+    precision: usize,
+    magnitude: f64,
+    case: Case,
+    alternate_form: bool,
+    always_shows_fract: bool,
+) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude);
+            let mut parts = r_exp.splitn(2, 'e');
+            let base = parts.next().unwrap();
+            let exponent = parts.next().unwrap().parse::<i64>().unwrap();
+            if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) {
+                let e = match case {
+                    Case::Lower => 'e',
+                    Case::Upper => 'E',
+                };
+                let magnitude = format!("{:.*}", precision + 1, base);
+                let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
+                let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form);
+                format!("{base}{point}{e}{exponent:+#03}")
+            } else {
+                let precision = ((precision as i64) - 1 - exponent) as usize;
+                let magnitude = format!("{magnitude:.precision$}");
+                let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
+                let point = decimal_point_or_empty(precision, alternate_form);
+                format!("{base}{point}")
+            }
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+// TODO: rewrite using format_general
+pub fn to_string(value: f64) -> String {
+    let lit = format!("{value:e}");
+    if let Some(position) = lit.find('e') {
+        let significand = &lit[..position];
+        let exponent = &lit[position + 1..];
+        let exponent = exponent.parse::<i32>().unwrap();
+        if exponent < 16 && exponent > -5 {
+            if is_integer(value) {
+                format!("{value:.1?}")
+            } else {
+                value.to_string()
+            }
+        } else {
+            format!("{significand}e{exponent:+#03}")
+        }
+    } else {
+        let mut s = value.to_string();
+        s.make_ascii_lowercase();
+        s
+    }
+}
+
+pub fn from_hex(s: &str) -> Option<f64> {
+    if let Ok(f) = hexf_parse::parse_hexf64(s, false) {
+        return Some(f);
+    }
+    match s.to_ascii_lowercase().as_str() {
+        "nan" | "+nan" | "-nan" => Some(f64::NAN),
+        "inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY),
+        "-inf" | "-infinity" => Some(f64::NEG_INFINITY),
+        value => {
+            let mut hex = String::with_capacity(value.len());
+            let has_0x = value.contains("0x");
+            let has_p = value.contains('p');
+            let has_dot = value.contains('.');
+            let mut start = 0;
+
+            if !has_0x && value.starts_with('-') {
+                hex.push_str("-0x");
+                start += 1;
+            } else if !has_0x {
+                hex.push_str("0x");
+                if value.starts_with('+') {
+                    start += 1;
+                }
+            }
+
+            for (index, ch) in value.chars().enumerate() {
+                if ch == 'p' {
+                    if has_dot {
+                        hex.push('p');
+                    } else {
+                        hex.push_str(".p");
+                    }
+                } else if index >= start {
+                    hex.push(ch);
+                }
+            }
+
+            if !has_p && has_dot {
+                hex.push_str("p0");
+            } else if !has_p && !has_dot {
+                hex.push_str(".p0")
+            }
+
+            hexf_parse::parse_hexf64(hex.as_str(), false).ok()
+        }
+    }
+}
+
+pub fn to_hex(value: f64) -> String {
+    let (mantissa, exponent, sign) = value.integer_decode();
+    let sign_fmt = if sign < 0 { "-" } else { "" };
+    match value {
+        value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"),
+        value if value.is_infinite() => format!("{sign_fmt}inf"),
+        value if value.is_nan() => "nan".to_owned(),
+        _ => {
+            const BITS: i16 = 52;
+            const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff;
+            format!(
+                "{}{:#x}.{:013x}p{:+}",
+                sign_fmt,
+                mantissa >> BITS,
+                mantissa & FRACT_MASK,
+                exponent + BITS
+            )
+        }
+    }
+}
+
+#[test]
+fn test_to_hex() {
+    use rand::Rng;
+    for _ in 0..20000 {
+        let bytes = rand::rng().random::<u64>();
+        let f = f64::from_bits(bytes);
+        if !f.is_finite() {
+            continue;
+        }
+        let hex = to_hex(f);
+        // println!("{} -> {}", f, hex);
+        let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap();
+        // println!("  -> {}", roundtrip);
+        assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip);
+    }
+}
+
+#[test]
+fn test_remove_trailing_zeros() {
+    assert!(remove_trailing_zeros(String::from("100")) == *"1");
+    assert!(remove_trailing_zeros(String::from("100.00")) == *"100.");
+
+    // leave leading zeros untouched
+    assert!(remove_trailing_zeros(String::from("001")) == *"001");
+
+    // leave strings untouched if they don't end with 0
+    assert!(remove_trailing_zeros(String::from("101")) == *"101");
+}
+
+#[test]
+fn test_remove_trailing_decimal_point() {
+    assert!(remove_trailing_decimal_point(String::from("100.")) == *"100");
+    assert!(remove_trailing_decimal_point(String::from("1.")) == *"1");
+
+    // leave leading decimal points untouched
+    assert!(remove_trailing_decimal_point(String::from(".5")) == *".5");
+}
+
+#[test]
+fn test_maybe_remove_trailing_redundant_chars() {
+    assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100.");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10");
+
+    // don't truncate integers
+    assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000");
+}

compiler/literal/src/format.rs

@@ -0,0 +1,5 @@
+#[derive(Debug, PartialEq, Eq, Clone, Copy, is_macro::Is, Hash)]
+pub enum Case {
+    Lower,
+    Upper,
+}

compiler/literal/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod char;
+pub mod escape;
+pub mod float;
+pub mod format;

deny.toml

@@ -242,4 +242,4 @@ unknown-git = "warn"
 # if not specified. If it is specified but empty, no registries are allowed.
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 # List of URLs for allowed Git repositories
-allow-git = ["https://github.com/RustPython/Parser.git", "https://github.com/RustPython/__doc__"]
+allow-git = ["https://github.com/RustPython/__doc__"]

vm/Cargo.toml

@@ -42,7 +42,6 @@ rustpython-compiler-core = { workspace = true }
 rustpython-compiler-source = { workspace = true }
 # rustpython-parser-core = { workspace = true }
 rustpython-literal = { workspace = true }
-rustpython-format = { workspace = true }
 rustpython-sre_engine = { workspace = true }
 
 ascii = { workspace = true }

vm/src/builtins/bool.rs

@@ -1,4 +1,5 @@
 use super::{PyInt, PyStrRef, PyType, PyTypeRef};
+use crate::common::format::FormatSpec;
 use crate::{
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject,
     VirtualMachine,
@@ -11,7 +12,6 @@ use crate::{
 };
 use malachite_bigint::Sign;
 use num_traits::Zero;
-use rustpython_format::FormatSpec;
 use std::fmt::{Debug, Formatter};
 
 impl ToPyObject for bool {

vm/src/builtins/float.rs

@@ -7,7 +7,7 @@ use crate::{
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
     TryFromBorrowedObject, TryFromObject, VirtualMachine,
     class::PyClassImpl,
-    common::{float_ops, hash},
+    common::{float_ops, format::FormatSpec, hash},
     convert::{IntoPyException, ToPyObject, ToPyResult},
     function::{
         ArgBytesLike, OptionalArg, OptionalOption,
@@ -21,7 +21,6 @@ use malachite_bigint::{BigInt, ToBigInt};
 use num_complex::Complex64;
 use num_traits::{Signed, ToPrimitive, Zero};
 use rustpython_common::int::float_to_ratio;
-use rustpython_format::FormatSpec;
 
 #[pyclass(module = false, name = "float")]
 #[derive(Debug, Copy, Clone, PartialEq)]

vm/src/builtins/int.rs

@@ -6,6 +6,7 @@ use crate::{
     bytesinner::PyBytesInner,
     class::PyClassImpl,
     common::{
+        format::FormatSpec,
         hash,
         int::{bigint_to_finite_float, bytes_to_int, true_div},
     },
@@ -20,7 +21,6 @@ use crate::{
 use malachite_bigint::{BigInt, Sign};
 use num_integer::Integer;
 use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero};
-use rustpython_format::FormatSpec;
 use std::fmt;
 use std::ops::{Neg, Not};
 

vm/src/builtins/str.rs

@@ -31,10 +31,10 @@ use once_cell::sync::Lazy;
 use rustpython_common::{
     ascii,
     atomic::{self, PyAtomic, Radium},
+    format::{FormatSpec, FormatString, FromTemplate},
     hash,
     lock::PyMutex,
 };
-use rustpython_format::{FormatSpec, FormatString, FromTemplate};
 use std::{char, fmt, ops::Range, string::ToString};
 use unic_ucd_bidi::BidiClass;
 use unic_ucd_category::GeneralCategory;

vm/src/cformat.rs

@@ -1,6 +1,7 @@
 //! Implementation of Printf-Style string formatting
 //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting).
 
+use crate::common::cformat::*;
 use crate::{
     AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine,
     builtins::{
@@ -12,7 +13,6 @@ use crate::{
 };
 use itertools::Itertools;
 use num_traits::cast::ToPrimitive;
-use rustpython_format::cformat::*;
 use std::str::FromStr;
 
 fn spec_format_bytes(

vm/src/format.rs

@@ -6,7 +6,7 @@ use crate::{
     stdlib::builtins,
 };
 
-use rustpython_format::*;
+use crate::common::format::*;
 
 impl IntoPyException for FormatSpecError {
     fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef {

vm/src/stdlib/string.rs

@@ -6,15 +6,15 @@ pub(crate) use _string::make_module;
 #[pymodule]
 mod _string {
     use crate::common::ascii;
+    use crate::common::format::{
+        FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
+    };
     use crate::{
         PyObjectRef, PyResult, VirtualMachine,
         builtins::{PyList, PyStrRef},
         convert::ToPyException,
         convert::ToPyObject,
     };
-    use rustpython_format::{
-        FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
-    };
     use std::mem;
 
     fn create_format_part(