From c585678ec925f2da6fdf224eb8ced93d65c82a90 Mon Sep 17 00:00:00 2001 From: Noa Date: Sat, 22 Mar 2025 19:27:13 -0500 Subject: [PATCH] Merge format and literal back into this repo (and update malachite) (#5618) * Merge format and literal back into this repo * Update format and literal to work * Update malachite * Remove RustPython/Parser from Cargo.toml --- Cargo.lock | 61 +- Cargo.toml | 24 +- common/Cargo.toml | 4 +- common/src/cformat.rs | 1044 +++++++++++++++++++++++++ common/src/format.rs | 1328 ++++++++++++++++++++++++++++++++ common/src/lib.rs | 2 + common/src/str.rs | 2 +- compiler/literal/Cargo.toml | 19 + compiler/literal/src/char.rs | 15 + compiler/literal/src/escape.rs | 441 +++++++++++ compiler/literal/src/float.rs | 342 ++++++++ compiler/literal/src/format.rs | 5 + compiler/literal/src/lib.rs | 4 + deny.toml | 2 +- vm/Cargo.toml | 1 - vm/src/builtins/bool.rs | 2 +- vm/src/builtins/float.rs | 3 +- vm/src/builtins/int.rs | 2 +- vm/src/builtins/str.rs | 2 +- vm/src/cformat.rs | 2 +- vm/src/format.rs | 2 +- vm/src/stdlib/string.rs | 6 +- 22 files changed, 3240 insertions(+), 73 deletions(-) create mode 100644 common/src/cformat.rs create mode 100644 common/src/format.rs create mode 100644 compiler/literal/Cargo.toml create mode 100644 compiler/literal/src/char.rs create mode 100644 compiler/literal/src/escape.rs create mode 100644 compiler/literal/src/float.rs create mode 100644 compiler/literal/src/format.rs create mode 100644 compiler/literal/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c8106856e..94ae260f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,6 +787,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -896,15 +902,15 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash", +] [[package]] name = "heck" @@ -1255,24 +1261,13 @@ dependencies = [ "libc", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - [[package]] name = "malachite-base" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", "itertools 0.11.0", "libm", "ryu", @@ -1280,12 +1275,13 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.2.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" +checksum = "7b1b1fec8b370139968919a5b77071c94b282eaba3da1cf179ae5299060d4e75" dependencies = [ "derive_more", - "malachite", + "malachite-base", + "malachite-nz", "num-integer", "num-traits", "paste", @@ -1293,9 +1289,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" +checksum = "175263cd5b846c552b9afb9d4b03ca465b4ad10717d789cad7dac24441c4fcbe" dependencies = [ "itertools 0.11.0", "libm", @@ -1304,9 +1300,9 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.4.22" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" +checksum = "5261ba8feb1ad20cddab3d625af28206c663c08014b2e5c5f9bd54b0f230234f" dependencies = [ "itertools 0.11.0", "malachite-base", @@ -2152,7 +2148,7 @@ dependencies = [ "parking_lot", "radium", "rand 0.9.0", - "rustpython-format", + "rustpython-literal", "siphasher 0.3.11", "volatile", "widestring", @@ -2229,18 +2225,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "rustpython-format" -version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" -dependencies = [ - "bitflags 2.8.0", - "itertools 0.11.0", - "malachite-bigint", - "num-traits", - "rustpython-literal", -] - [[package]] name = "rustpython-jit" version = "0.4.0" @@ -2259,12 +2243,12 @@ dependencies = [ [[package]] name = "rustpython-literal" version = "0.4.0" -source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23" dependencies = [ "hexf-parse", "is-macro", "lexical-parse-float", "num-traits", + "rand 0.9.0", "unic-ucd-category", ] @@ -2416,7 +2400,6 @@ dependencies = [ "rustpython-compiler-core", "rustpython-compiler-source", "rustpython-derive", - "rustpython-format", "rustpython-jit", "rustpython-literal", "rustpython-sre_engine", diff --git a/Cargo.toml b/Cargo.toml index 469f895c7..1da72c9dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = [workspace] resolver = "2" members = [ - "compiler", "compiler/core", "compiler/codegen", "compiler/source", + "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source", ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wasm/lib", ] @@ -116,6 +116,7 @@ rustpython-common = { path = "common", version = "0.4.0" } rustpython-derive = { path = "derive", version = "0.4.0" } rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" } rustpython-jit = { path = "jit", version = "0.4.0" } +rustpython-literal = { path = "compiler/literal", version = "0.4.0" } rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" } rustpython-pylib = { path = "pylib", version = "0.4.0" } rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" } @@ -127,21 +128,6 @@ ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -# rustpython-literal = { version = "0.4.0" } -# rustpython-parser-core = { version = "0.4.0" } -# rustpython-parser = { version = "0.4.0" } -# rustpython-ast = { version = "0.4.0" } -# rustpython-format= { version = "0.4.0" } -rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" } -# rustpython-literal = { path = "../RustPython-parser/literal" } -# rustpython-parser-core = { path = "../RustPython-parser/core" } -# rustpython-parser = { path = "../RustPython-parser/parser" } -# rustpython-ast = { path = "../RustPython-parser/ast" } -# rustpython-format = { path = "../RustPython-parser/format" } ahash = "0.8.11" ascii = "1.1" @@ -163,9 +149,9 @@ junction = "1.2.0" libc = "0.2.169" log = "0.4.25" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.2.3" -malachite-q = "0.4.22" -malachite-base = "0.4.22" +malachite-bigint = "0.5" +malachite-q = "0.5" +malachite-base = "0.5" memchr = "2.7.4" num-complex = "0.4.6" num-integer = "0.1.46" diff --git a/common/Cargo.toml b/common/Cargo.toml index 8ba07f4be..589170064 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -12,7 +12,7 @@ license.workspace = true threading = ["parking_lot"] [dependencies] -rustpython-format = { workspace = true } +rustpython-literal = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } @@ -45,4 +45,4 @@ windows-sys = { workspace = true, features = [ ] } [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/common/src/cformat.rs b/common/src/cformat.rs new file mode 100644 index 000000000..cd282eb4c --- /dev/null +++ b/common/src/cformat.rs @@ -0,0 +1,1044 @@ +//! Implementation of Printf-Style string formatting +//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use bitflags::bitflags; +use malachite_bigint::{BigInt, Sign}; +use num_traits::Signed; +use rustpython_literal::{float, format::Case}; +use std::{ + cmp, fmt, + iter::{Enumerate, Peekable}, + str::FromStr, +}; + +#[derive(Debug, PartialEq)] +pub enum CFormatErrorType { + UnmatchedKeyParentheses, + MissingModuloSign, + UnsupportedFormatChar(char), + IncompleteFormat, + IntTooBig, + // Unimplemented, +} + +// also contains how many chars the parsing function consumed +pub type ParsingError = (CFormatErrorType, usize); + +#[derive(Debug, PartialEq)] +pub struct CFormatError { + pub typ: CFormatErrorType, // FIXME + pub index: usize, +} + +impl fmt::Display for CFormatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use CFormatErrorType::*; + match self.typ { + UnmatchedKeyParentheses => write!(f, "incomplete format key"), + IncompleteFormat => write!(f, "incomplete format"), + UnsupportedFormatChar(c) => write!( + f, + "unsupported format character '{}' ({:#x}) at index {}", + c, c as u32, self.index + ), + IntTooBig => write!(f, "width/precision too big"), + _ => write!(f, "unexpected error parsing format string"), + } + } +} + +pub type CFormatConversion = super::format::FormatConversion; + +#[derive(Debug, PartialEq)] +pub enum CNumberType { + Decimal, + Octal, + Hex(Case), +} + +#[derive(Debug, PartialEq)] +pub enum CFloatType { + Exponent(Case), + PointDecimal(Case), + General(Case), +} + +#[derive(Debug, PartialEq)] +pub enum CFormatType { + Number(CNumberType), + Float(CFloatType), + Character, + String(CFormatConversion), +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPrecision { + Quantity(CFormatQuantity), + Dot, +} + +impl From for CFormatPrecision { + fn from(quantity: CFormatQuantity) -> Self { + CFormatPrecision::Quantity(quantity) + } +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct CConversionFlags: u32 { + const ALTERNATE_FORM = 0b0000_0001; + const ZERO_PAD = 0b0000_0010; + const LEFT_ADJUST = 0b0000_0100; + const BLANK_SIGN = 0b0000_1000; + const SIGN_CHAR = 0b0001_0000; + } +} + +impl CConversionFlags { + #[inline] + pub fn sign_string(&self) -> &'static str { + if self.contains(CConversionFlags::SIGN_CHAR) { + "+" + } else if self.contains(CConversionFlags::BLANK_SIGN) { + " " + } else { + "" + } + } +} + +#[derive(Debug, PartialEq)] +pub enum CFormatQuantity { + Amount(usize), + FromValuesTuple, +} + +#[derive(Debug, PartialEq)] +pub struct CFormatSpec { + pub mapping_key: Option, + pub flags: CConversionFlags, + pub min_field_width: Option, + pub precision: Option, + pub format_type: CFormatType, + pub format_char: char, + // chars_consumed: usize, +} + +impl FromStr for CFormatSpec { + type Err = ParsingError; + + fn from_str(text: &str) -> Result { + let mut chars = text.chars().enumerate().peekable(); + if chars.next().map(|x| x.1) != Some('%') { + return Err((CFormatErrorType::MissingModuloSign, 1)); + } + + CFormatSpec::parse(&mut chars) + } +} + +pub type ParseIter = Peekable>; + +impl CFormatSpec { + pub fn parse(iter: &mut ParseIter) -> Result + where + T: Into + Copy, + I: Iterator, + { + let mapping_key = parse_spec_mapping_key(iter)?; + let flags = parse_flags(iter); + let min_field_width = parse_quantity(iter)?; + let precision = parse_precision(iter)?; + consume_length(iter); + let (format_type, format_char) = parse_format_type(iter)?; + + Ok(CFormatSpec { + mapping_key, + flags, + min_field_width, + precision, + format_type, + format_char, + }) + } + + fn compute_fill_string(fill_char: char, fill_chars_needed: usize) -> String { + (0..fill_chars_needed) + .map(|_| fill_char) + .collect::() + } + + fn fill_string( + &self, + string: String, + fill_char: char, + num_prefix_chars: Option, + ) -> String { + let mut num_chars = string.chars().count(); + if let Some(num_prefix_chars) = num_prefix_chars { + num_chars += num_prefix_chars; + } + let num_chars = num_chars; + + let width = match &self.min_field_width { + Some(CFormatQuantity::Amount(width)) => cmp::max(width, &num_chars), + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + format!("{string}{fill_string}") + } else { + format!("{fill_string}{string}") + } + } else { + string + } + } + + fn fill_string_with_precision(&self, string: String, fill_char: char) -> String { + let num_chars = string.chars().count(); + + let width = match &self.precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(width))) => { + cmp::max(width, &num_chars) + } + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + // Don't left-adjust if precision-filling: that will always be prepending 0s to %d + // arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with + // the 0-filled string as the string param. + format!("{fill_string}{string}") + } else { + string + } + } + + fn format_string_with_precision( + &self, + string: String, + precision: Option<&CFormatPrecision>, + ) -> String { + // truncate if needed + let string = match precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) + if string.chars().count() > *precision => + { + string.chars().take(*precision).collect::() + } + Some(CFormatPrecision::Dot) => { + // truncate to 0 + String::new() + } + _ => string, + }; + self.fill_string(string, ' ', None) + } + + #[inline] + pub fn format_string(&self, string: String) -> String { + self.format_string_with_precision(string, self.precision.as_ref()) + } + + #[inline] + pub fn format_char(&self, ch: char) -> String { + self.format_string_with_precision( + ch.to_string(), + Some(&(CFormatQuantity::Amount(1).into())), + ) + } + + pub fn format_bytes(&self, bytes: &[u8]) -> Vec { + let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) = + self.precision + { + &bytes[..cmp::min(bytes.len(), precision)] + } else { + bytes + }; + if let Some(CFormatQuantity::Amount(width)) = self.min_field_width { + let fill = cmp::max(0, width - bytes.len()); + let mut v = Vec::with_capacity(bytes.len() + fill); + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + v.extend_from_slice(bytes); + v.append(&mut vec![b' '; fill]); + } else { + v.append(&mut vec![b' '; fill]); + v.extend_from_slice(bytes); + } + v + } else { + bytes.to_vec() + } + } + + pub fn format_number(&self, num: &BigInt) -> String { + use CNumberType::*; + let magnitude = num.abs(); + let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { + match self.format_type { + CFormatType::Number(Octal) => "0o", + CFormatType::Number(Hex(Case::Lower)) => "0x", + CFormatType::Number(Hex(Case::Upper)) => "0X", + _ => "", + } + } else { + "" + }; + + let magnitude_string: String = match self.format_type { + CFormatType::Number(Decimal) => magnitude.to_str_radix(10), + CFormatType::Number(Octal) => magnitude.to_str_radix(8), + CFormatType::Number(Hex(Case::Lower)) => magnitude.to_str_radix(16), + CFormatType::Number(Hex(Case::Upper)) => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + result + } + _ => unreachable!(), // Should not happen because caller has to make sure that this is a number + }; + + let sign_string = match num.sign() { + Sign::Minus => "-", + _ => self.flags.sign_string(), + }; + + let padded_magnitude_string = self.fill_string_with_precision(magnitude_string, '0'); + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' // '-' overrides the '0' conversion if both are given + }; + let signed_prefix = format!("{sign_string}{prefix}"); + format!( + "{}{}", + signed_prefix, + self.fill_string( + padded_magnitude_string, + fill_char, + Some(signed_prefix.chars().count()), + ), + ) + } else { + self.fill_string( + format!("{sign_string}{prefix}{padded_magnitude_string}"), + ' ', + None, + ) + } + } + + pub fn format_float(&self, num: f64) -> String { + let sign_string = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + self.flags.sign_string() + }; + + let precision = match &self.precision { + Some(CFormatPrecision::Quantity(quantity)) => match quantity { + CFormatQuantity::Amount(amount) => *amount, + CFormatQuantity::FromValuesTuple => 6, + }, + Some(CFormatPrecision::Dot) => 0, + None => 6, + }; + + let magnitude_string = match &self.format_type { + CFormatType::Float(CFloatType::PointDecimal(case)) => { + let magnitude = num.abs(); + float::format_fixed( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ) + } + CFormatType::Float(CFloatType::Exponent(case)) => { + let magnitude = num.abs(); + float::format_exponent( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ) + } + CFormatType::Float(CFloatType::General(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + let magnitude = num.abs(); + float::format_general( + precision, + magnitude, + *case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + false, + ) + } + _ => unreachable!(), + }; + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' + }; + format!( + "{}{}", + sign_string, + self.fill_string( + magnitude_string, + fill_char, + Some(sign_string.chars().count()), + ) + ) + } else { + self.fill_string(format!("{sign_string}{magnitude_string}"), ' ', None) + } + } +} + +fn parse_spec_mapping_key(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(index, c)) = iter.peek() { + if c.into() == '(' { + iter.next().unwrap(); + return match parse_text_inside_parentheses(iter) { + Some(key) => Ok(Some(key)), + None => Err((CFormatErrorType::UnmatchedKeyParentheses, index)), + }; + } + } + Ok(None) +} + +fn parse_flags(iter: &mut ParseIter) -> CConversionFlags +where + T: Into + Copy, + I: Iterator, +{ + let mut flags = CConversionFlags::empty(); + while let Some(&(_, c)) = iter.peek() { + let flag = match c.into() { + '#' => CConversionFlags::ALTERNATE_FORM, + '0' => CConversionFlags::ZERO_PAD, + '-' => CConversionFlags::LEFT_ADJUST, + ' ' => CConversionFlags::BLANK_SIGN, + '+' => CConversionFlags::SIGN_CHAR, + _ => break, + }; + iter.next().unwrap(); + flags |= flag; + } + flags +} + +fn consume_length(iter: &mut ParseIter) +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + let c = c.into(); + if c == 'h' || c == 'l' || c == 'L' { + iter.next().unwrap(); + } + } +} + +fn parse_format_type(iter: &mut ParseIter) -> Result<(CFormatType, char), ParsingError> +where + T: Into, + I: Iterator, +{ + use CFloatType::*; + use CNumberType::*; + let (index, c) = match iter.next() { + Some((index, c)) => (index, c.into()), + None => { + return Err(( + CFormatErrorType::IncompleteFormat, + iter.peek().map(|x| x.0).unwrap_or(0), + )); + } + }; + let format_type = match c { + 'd' | 'i' | 'u' => CFormatType::Number(Decimal), + 'o' => CFormatType::Number(Octal), + 'x' => CFormatType::Number(Hex(Case::Lower)), + 'X' => CFormatType::Number(Hex(Case::Upper)), + 'e' => CFormatType::Float(Exponent(Case::Lower)), + 'E' => CFormatType::Float(Exponent(Case::Upper)), + 'f' => CFormatType::Float(PointDecimal(Case::Lower)), + 'F' => CFormatType::Float(PointDecimal(Case::Upper)), + 'g' => CFormatType::Float(General(Case::Lower)), + 'G' => CFormatType::Float(General(Case::Upper)), + 'c' => CFormatType::Character, + 'r' => CFormatType::String(CFormatConversion::Repr), + 's' => CFormatType::String(CFormatConversion::Str), + 'b' => CFormatType::String(CFormatConversion::Bytes), + 'a' => CFormatType::String(CFormatConversion::Ascii), + _ => return Err((CFormatErrorType::UnsupportedFormatChar(c), index)), + }; + Ok((format_type, c)) +} + +fn parse_quantity(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + let c: char = c.into(); + if c == '*' { + iter.next().unwrap(); + return Ok(Some(CFormatQuantity::FromValuesTuple)); + } + if let Some(i) = c.to_digit(10) { + let mut num = i as i32; + iter.next().unwrap(); + while let Some(&(index, c)) = iter.peek() { + if let Some(i) = c.into().to_digit(10) { + num = num + .checked_mul(10) + .and_then(|num| num.checked_add(i as i32)) + .ok_or((CFormatErrorType::IntTooBig, index))?; + iter.next().unwrap(); + } else { + break; + } + } + return Ok(Some(CFormatQuantity::Amount(num.unsigned_abs() as usize))); + } + } + Ok(None) +} + +fn parse_precision(iter: &mut ParseIter) -> Result, ParsingError> +where + T: Into + Copy, + I: Iterator, +{ + if let Some(&(_, c)) = iter.peek() { + if c.into() == '.' { + iter.next().unwrap(); + let quantity = parse_quantity(iter)?; + let precision = quantity.map_or(CFormatPrecision::Dot, CFormatPrecision::Quantity); + return Ok(Some(precision)); + } + } + Ok(None) +} + +fn parse_text_inside_parentheses(iter: &mut ParseIter) -> Option +where + T: Into, + I: Iterator, +{ + let mut counter: i32 = 1; + let mut contained_text = String::new(); + loop { + let (_, c) = iter.next()?; + let c = c.into(); + match c { + _ if c == '(' => { + counter += 1; + } + _ if c == ')' => { + counter -= 1; + } + _ => (), + } + + if counter > 0 { + contained_text.push(c); + } else { + break; + } + } + + Some(contained_text) +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPart { + Literal(T), + Spec(CFormatSpec), +} + +impl CFormatPart { + #[inline] + pub fn is_specifier(&self) -> bool { + matches!(self, CFormatPart::Spec(_)) + } + + #[inline] + pub fn has_key(&self) -> bool { + match self { + CFormatPart::Spec(s) => s.mapping_key.is_some(), + _ => false, + } + } +} + +#[derive(Debug, PartialEq)] +pub struct CFormatStrOrBytes { + parts: Vec<(usize, CFormatPart)>, +} + +impl CFormatStrOrBytes { + pub fn check_specifiers(&self) -> Option<(usize, bool)> { + let mut count = 0; + let mut mapping_required = false; + for (_, part) in &self.parts { + if part.is_specifier() { + let has_key = part.has_key(); + if count == 0 { + mapping_required = has_key; + } else if mapping_required != has_key { + return None; + } + count += 1; + } + } + Some((count, mapping_required)) + } + + #[inline] + pub fn iter(&self) -> impl Iterator)> { + self.parts.iter() + } + + #[inline] + pub fn iter_mut(&mut self) -> impl Iterator)> { + self.parts.iter_mut() + } +} + +pub type CFormatBytes = CFormatStrOrBytes>; + +impl CFormatBytes { + pub fn parse>(iter: &mut ParseIter) -> Result { + let mut parts = vec![]; + let mut literal = vec![]; + let mut part_index = 0; + while let Some((index, c)) = iter.next() { + if c == b'%' { + if let Some(&(_, second)) = iter.peek() { + if second == b'%' { + iter.next().unwrap(); + literal.push(b'%'); + continue; + } else { + if !literal.is_empty() { + parts.push(( + part_index, + CFormatPart::Literal(std::mem::take(&mut literal)), + )); + } + let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { + typ: err.0, + index: err.1, + })?; + parts.push((index, CFormatPart::Spec(spec))); + if let Some(&(index, _)) = iter.peek() { + part_index = index; + } + } + } else { + return Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: index + 1, + }); + } + } else { + literal.push(c); + } + } + if !literal.is_empty() { + parts.push((part_index, CFormatPart::Literal(literal))); + } + Ok(Self { parts }) + } + + pub fn parse_from_bytes(bytes: &[u8]) -> Result { + let mut iter = bytes.iter().cloned().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +pub type CFormatString = CFormatStrOrBytes; + +impl FromStr for CFormatString { + type Err = CFormatError; + + fn from_str(text: &str) -> Result { + let mut iter = text.chars().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +impl CFormatString { + pub(crate) fn parse>( + iter: &mut ParseIter, + ) -> Result { + let mut parts = vec![]; + let mut literal = String::new(); + let mut part_index = 0; + while let Some((index, c)) = iter.next() { + if c == '%' { + if let Some(&(_, second)) = iter.peek() { + if second == '%' { + iter.next().unwrap(); + literal.push('%'); + continue; + } else { + if !literal.is_empty() { + parts.push(( + part_index, + CFormatPart::Literal(std::mem::take(&mut literal)), + )); + } + let spec = CFormatSpec::parse(iter).map_err(|err| CFormatError { + typ: err.0, + index: err.1, + })?; + parts.push((index, CFormatPart::Spec(spec))); + if let Some(&(index, _)) = iter.peek() { + part_index = index; + } + } + } else { + return Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: index + 1, + }); + } + } else { + literal.push(c); + } + } + if !literal.is_empty() { + parts.push((part_index, CFormatPart::Literal(literal))); + } + Ok(Self { parts }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + "%10s" + .parse::() + .unwrap() + .format_string("test".to_owned()), + " test".to_owned() + ); + assert_eq!( + "%-10s" + .parse::() + .unwrap() + .format_string("test".to_owned()), + "test ".to_owned() + ); + assert_eq!( + "%#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + " 0x1337".to_owned() + ); + assert_eq!( + "%-#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_key() { + let expected = Ok(CFormatSpec { + mapping_key: Some("amount".to_owned()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!("%(amount)d".parse::(), expected); + + let expected = Ok(CFormatSpec { + mapping_key: Some("m((u(((l((((ti))))p)))l))e".to_owned()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!( + "%(m((u(((l((((ti))))p)))l))e)d".parse::(), + expected + ); + } + + #[test] + fn test_format_parse_key_fail() { + assert_eq!( + "%(aged".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnmatchedKeyParentheses, + index: 1 + }) + ); + } + + #[test] + fn test_format_parse_type_fail() { + assert_eq!( + "Hello %n".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnsupportedFormatChar('n'), + index: 7 + }) + ); + } + + #[test] + fn test_incomplete_format_fail() { + assert_eq!( + "Hello %".parse::(), + Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: 7 + }) + ); + } + + #[test] + fn test_parse_flags() { + let expected = Ok(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + min_field_width: Some(CFormatQuantity::Amount(10)), + precision: None, + mapping_key: None, + flags: CConversionFlags::all(), + }); + let parsed = "% 0 -+++###10d".parse::(); + assert_eq!(parsed, expected); + assert_eq!( + parsed.unwrap().format_number(&BigInt::from(12)), + "+12 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_string() { + assert_eq!( + "%5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + " Hell".to_owned() + ); + assert_eq!( + "%-5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + "Hell ".to_owned() + ); + assert_eq!( + "%.s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + "".to_owned() + ); + assert_eq!( + "%5.s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_owned()), + " ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_unicode_string() { + assert_eq!( + "%.2s" + .parse::() + .unwrap() + .format_string("❤❤❤❤❤❤❤❤".to_owned()), + "❤❤".to_owned() + ); + } + + #[test] + fn test_parse_and_format_number() { + assert_eq!( + "%5d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "%05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%.5d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%+05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "+0027".to_owned() + ); + assert_eq!( + "%-d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "%08x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "00001337".to_owned() + ); + assert_eq!( + "%#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x00001337".to_owned() + ); + assert_eq!( + "%-#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_float() { + assert_eq!( + "%f".parse::().unwrap().format_float(1.2345), + "1.234500" + ); + assert_eq!( + "%.2f".parse::().unwrap().format_float(1.2345), + "1.23" + ); + assert_eq!( + "%.f".parse::().unwrap().format_float(1.2345), + "1" + ); + assert_eq!( + "%+.f".parse::().unwrap().format_float(1.2345), + "+1" + ); + assert_eq!( + "%+f".parse::().unwrap().format_float(1.2345), + "+1.234500" + ); + assert_eq!( + "% f".parse::().unwrap().format_float(1.2345), + " 1.234500" + ); + assert_eq!( + "%f".parse::().unwrap().format_float(-1.2345), + "-1.234500" + ); + assert_eq!( + "%f".parse::() + .unwrap() + .format_float(1.2345678901), + "1.234568" + ); + } + + #[test] + fn test_format_parse() { + let fmt = "Hello, my name is %s and I'm %d years old"; + let expected = Ok(CFormatString { + parts: vec![ + (0, CFormatPart::Literal("Hello, my name is ".to_owned())), + ( + 18, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::String(CFormatConversion::Str), + format_char: 's', + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (20, CFormatPart::Literal(" and I'm ".to_owned())), + ( + 29, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (31, CFormatPart::Literal(" years old".to_owned())), + ], + }); + let result = fmt.parse::(); + assert_eq!( + result, expected, + "left = {result:#?} \n\n\n right = {expected:#?}" + ); + } +} diff --git a/common/src/format.rs b/common/src/format.rs new file mode 100644 index 000000000..1c41df38c --- /dev/null +++ b/common/src/format.rs @@ -0,0 +1,1328 @@ +use itertools::{Itertools, PeekingNext}; +use malachite_bigint::{BigInt, Sign}; +use num_traits::FromPrimitive; +use num_traits::{Signed, cast::ToPrimitive}; +use rustpython_literal::float; +use rustpython_literal::format::Case; +use std::ops::Deref; +use std::{cmp, str::FromStr}; + +trait FormatParse { + fn parse(text: &str) -> (Option, &str) + where + Self: Sized; +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatConversion { + Str, + Repr, + Ascii, + Bytes, +} + +impl FormatParse for FormatConversion { + fn parse(text: &str) -> (Option, &str) { + let Some(conversion) = Self::from_string(text) else { + return (None, text); + }; + let mut chars = text.chars(); + chars.next(); // Consume the bang + chars.next(); // Consume one r,s,a char + (Some(conversion), chars.as_str()) + } +} + +impl FormatConversion { + pub fn from_char(c: char) -> Option { + match c { + 's' => Some(FormatConversion::Str), + 'r' => Some(FormatConversion::Repr), + 'a' => Some(FormatConversion::Ascii), + 'b' => Some(FormatConversion::Bytes), + _ => None, + } + } + + fn from_string(text: &str) -> Option { + let mut chars = text.chars(); + if chars.next() != Some('!') { + return None; + } + + FormatConversion::from_char(chars.next()?) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatAlign { + Left, + Right, + AfterSign, + Center, +} + +impl FormatAlign { + fn from_char(c: char) -> Option { + match c { + '<' => Some(FormatAlign::Left), + '>' => Some(FormatAlign::Right), + '=' => Some(FormatAlign::AfterSign), + '^' => Some(FormatAlign::Center), + _ => None, + } + } +} + +impl FormatParse for FormatAlign { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + if let Some(maybe_align) = chars.next().and_then(Self::from_char) { + (Some(maybe_align), chars.as_str()) + } else { + (None, text) + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatSign { + Plus, + Minus, + MinusOrSpace, +} + +impl FormatParse for FormatSign { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('-') => (Some(Self::Minus), chars.as_str()), + Some('+') => (Some(Self::Plus), chars.as_str()), + Some(' ') => (Some(Self::MinusOrSpace), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatGrouping { + Comma, + Underscore, +} + +impl FormatParse for FormatGrouping { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('_') => (Some(Self::Underscore), chars.as_str()), + Some(',') => (Some(Self::Comma), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatType { + String, + Binary, + Character, + Decimal, + Octal, + Number(Case), + Hex(Case), + Exponent(Case), + GeneralFormat(Case), + FixedPoint(Case), + Percentage, +} + +impl From<&FormatType> for char { + fn from(from: &FormatType) -> char { + match from { + FormatType::String => 's', + FormatType::Binary => 'b', + FormatType::Character => 'c', + FormatType::Decimal => 'd', + FormatType::Octal => 'o', + FormatType::Number(Case::Lower) => 'n', + FormatType::Number(Case::Upper) => 'N', + FormatType::Hex(Case::Lower) => 'x', + FormatType::Hex(Case::Upper) => 'X', + FormatType::Exponent(Case::Lower) => 'e', + FormatType::Exponent(Case::Upper) => 'E', + FormatType::GeneralFormat(Case::Lower) => 'g', + FormatType::GeneralFormat(Case::Upper) => 'G', + FormatType::FixedPoint(Case::Lower) => 'f', + FormatType::FixedPoint(Case::Upper) => 'F', + FormatType::Percentage => '%', + } + } +} + +impl FormatParse for FormatType { + fn parse(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('s') => (Some(Self::String), chars.as_str()), + Some('b') => (Some(Self::Binary), chars.as_str()), + Some('c') => (Some(Self::Character), chars.as_str()), + Some('d') => (Some(Self::Decimal), chars.as_str()), + Some('o') => (Some(Self::Octal), chars.as_str()), + Some('n') => (Some(Self::Number(Case::Lower)), chars.as_str()), + Some('N') => (Some(Self::Number(Case::Upper)), chars.as_str()), + Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_str()), + Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_str()), + Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_str()), + Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_str()), + Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_str()), + Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_str()), + Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_str()), + Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_str()), + Some('%') => (Some(Self::Percentage), chars.as_str()), + _ => (None, text), + } + } +} + +#[derive(Debug, PartialEq)] +pub struct FormatSpec { + conversion: Option, + fill: Option, + align: Option, + sign: Option, + alternate_form: bool, + width: Option, + grouping_option: Option, + precision: Option, + format_type: Option, +} + +fn get_num_digits(text: &str) -> usize { + for (index, character) in text.char_indices() { + if !character.is_ascii_digit() { + return index; + } + } + text.len() +} + +fn parse_fill_and_align(text: &str) -> (Option, Option, &str) { + let char_indices: Vec<(usize, char)> = text.char_indices().take(3).collect(); + if char_indices.is_empty() { + (None, None, text) + } else if char_indices.len() == 1 { + let (maybe_align, remaining) = FormatAlign::parse(text); + (None, maybe_align, remaining) + } else { + let (maybe_align, remaining) = FormatAlign::parse(&text[char_indices[1].0..]); + if maybe_align.is_some() { + (Some(char_indices[0].1), maybe_align, remaining) + } else { + let (only_align, only_align_remaining) = FormatAlign::parse(text); + (None, only_align, only_align_remaining) + } + } +} + +fn parse_number(text: &str) -> Result<(Option, &str), FormatSpecError> { + let num_digits: usize = get_num_digits(text); + if num_digits == 0 { + return Ok((None, text)); + } + if let Ok(num) = text[..num_digits].parse::() { + Ok((Some(num), &text[num_digits..])) + } else { + // NOTE: this condition is different from CPython + Err(FormatSpecError::DecimalDigitsTooMany) + } +} + +fn parse_alternate_form(text: &str) -> (bool, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('#') => (true, chars.as_str()), + _ => (false, text), + } +} + +fn parse_zero(text: &str) -> (bool, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('0') => (true, chars.as_str()), + _ => (false, text), + } +} + +fn parse_precision(text: &str) -> Result<(Option, &str), FormatSpecError> { + let mut chars = text.chars(); + Ok(match chars.next() { + Some('.') => { + let (size, remaining) = parse_number(chars.as_str())?; + if let Some(size) = size { + if size > i32::MAX as usize { + return Err(FormatSpecError::PrecisionTooBig); + } + (Some(size), remaining) + } else { + (None, text) + } + } + _ => (None, text), + }) +} + +impl FormatSpec { + pub fn parse(text: &str) -> Result { + // get_integer in CPython + let (conversion, text) = FormatConversion::parse(text); + let (mut fill, mut align, text) = parse_fill_and_align(text); + let (sign, text) = FormatSign::parse(text); + let (alternate_form, text) = parse_alternate_form(text); + let (zero, text) = parse_zero(text); + let (width, text) = parse_number(text)?; + let (grouping_option, text) = FormatGrouping::parse(text); + let (precision, text) = parse_precision(text)?; + let (format_type, text) = FormatType::parse(text); + if !text.is_empty() { + return Err(FormatSpecError::InvalidFormatSpecifier); + } + + if zero && fill.is_none() { + fill.replace('0'); + align = align.or(Some(FormatAlign::AfterSign)); + } + + Ok(FormatSpec { + conversion, + fill, + align, + sign, + alternate_form, + width, + grouping_option, + precision, + format_type, + }) + } + + fn compute_fill_string(fill_char: char, fill_chars_needed: i32) -> String { + (0..fill_chars_needed) + .map(|_| fill_char) + .collect::() + } + + fn add_magnitude_separators_for_char( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + // Don't add separators to the floating decimal point of numbers + let mut parts = magnitude_str.splitn(2, '.'); + let magnitude_int_str = parts.next().unwrap().to_string(); + let dec_digit_cnt = magnitude_str.len() as i32 - magnitude_int_str.len() as i32; + let int_digit_cnt = disp_digit_cnt - dec_digit_cnt; + let mut result = FormatSpec::separate_integer(magnitude_int_str, inter, sep, int_digit_cnt); + if let Some(part) = parts.next() { + result.push_str(&format!(".{part}")) + } + result + } + + fn separate_integer( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + let magnitude_len = magnitude_str.len() as i32; + let offset = (disp_digit_cnt % (inter + 1) == 0) as i32; + let disp_digit_cnt = disp_digit_cnt + offset; + let pad_cnt = disp_digit_cnt - magnitude_len; + let sep_cnt = disp_digit_cnt / (inter + 1); + let diff = pad_cnt - sep_cnt; + if pad_cnt > 0 && diff > 0 { + // separate with 0 padding + let padding = "0".repeat(diff as usize); + let padded_num = format!("{padding}{magnitude_str}"); + FormatSpec::insert_separator(padded_num, inter, sep, sep_cnt) + } else { + // separate without padding + let sep_cnt = (magnitude_len - 1) / inter; + FormatSpec::insert_separator(magnitude_str, inter, sep, sep_cnt) + } + } + + fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String { + let magnitude_len = magnitude_str.len() as i32; + for i in 1..sep_cnt + 1 { + magnitude_str.insert((magnitude_len - inter * i) as usize, sep); + } + magnitude_str + } + + fn validate_format(&self, default_format_type: FormatType) -> Result<(), FormatSpecError> { + let format_type = self.format_type.as_ref().unwrap_or(&default_format_type); + match (&self.grouping_option, format_type) { + ( + Some(FormatGrouping::Comma), + FormatType::String + | FormatType::Character + | FormatType::Binary + | FormatType::Octal + | FormatType::Hex(_) + | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat(',', ch)) + } + ( + Some(FormatGrouping::Underscore), + FormatType::String | FormatType::Character | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat('_', ch)) + } + _ => Ok(()), + } + } + + fn get_separator_interval(&self) -> usize { + match self.format_type { + Some(FormatType::Binary | FormatType::Octal | FormatType::Hex(_)) => 4, + Some(FormatType::Decimal | FormatType::Number(_) | FormatType::FixedPoint(_)) => 3, + None => 3, + _ => panic!("Separators only valid for numbers!"), + } + } + + fn add_magnitude_separators(&self, magnitude_str: String, prefix: &str) -> String { + match &self.grouping_option { + Some(fg) => { + let sep = match fg { + FormatGrouping::Comma => ',', + FormatGrouping::Underscore => '_', + }; + let inter = self.get_separator_interval().try_into().unwrap(); + let magnitude_len = magnitude_str.len(); + let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; + let disp_digit_cnt = cmp::max(width, magnitude_len as i32); + FormatSpec::add_magnitude_separators_for_char( + magnitude_str, + inter, + sep, + disp_digit_cnt, + ) + } + None => magnitude_str, + } + } + + pub fn format_bool(&self, input: bool) -> Result { + let x = u8::from(input); + match &self.format_type { + Some( + FormatType::Binary + | FormatType::Decimal + | FormatType::Octal + | FormatType::Number(Case::Lower) + | FormatType::Hex(_) + | FormatType::GeneralFormat(_) + | FormatType::Character, + ) => self.format_int(&BigInt::from_u8(x).unwrap()), + Some(FormatType::Exponent(_) | FormatType::FixedPoint(_) | FormatType::Percentage) => { + self.format_float(x as f64) + } + None => { + let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); + Ok(first_letter.collect::() + &input.to_string()[1..]) + } + _ => Err(FormatSpecError::InvalidFormatSpecifier), + } + } + + pub fn format_float(&self, num: f64) -> Result { + self.validate_format(FormatType::FixedPoint(Case::Lower))?; + let precision = self.precision.unwrap_or(6); + let magnitude = num.abs(); + let raw_magnitude_str: Result = match &self.format_type { + Some(FormatType::FixedPoint(case)) => Ok(float::format_fixed( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Decimal) + | Some(FormatType::Binary) + | Some(FormatType::Octal) + | Some(FormatType::Hex(_)) + | Some(FormatType::String) + | Some(FormatType::Character) + | Some(FormatType::Number(Case::Upper)) => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "float")) + } + Some(FormatType::GeneralFormat(case)) | Some(FormatType::Number(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + Ok(float::format_general( + precision, + magnitude, + *case, + self.alternate_form, + false, + )) + } + Some(FormatType::Exponent(case)) => Ok(float::format_exponent( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Percentage) => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan%".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf%".to_owned()), + _ => { + let result = format!("{:.*}", precision, magnitude * 100.0); + let point = float::decimal_point_or_empty(precision, self.alternate_form); + Ok(format!("{result}{point}%")) + } + }, + None => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf".to_owned()), + _ => match self.precision { + Some(precision) => Ok(float::format_general( + precision, + magnitude, + Case::Lower, + self.alternate_form, + true, + )), + None => Ok(float::to_string(magnitude)), + }, + }, + }; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } + }; + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) + } + + #[inline] + fn format_int_radix(&self, magnitude: BigInt, radix: u32) -> Result { + match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => Ok(magnitude.to_str_radix(radix)), + } + } + + pub fn format_int(&self, num: &BigInt) -> Result { + self.validate_format(FormatType::Decimal)?; + let magnitude = num.abs(); + let prefix = if self.alternate_form { + match self.format_type { + Some(FormatType::Binary) => "0b", + Some(FormatType::Octal) => "0o", + Some(FormatType::Hex(Case::Lower)) => "0x", + Some(FormatType::Hex(Case::Upper)) => "0X", + _ => "", + } + } else { + "" + }; + let raw_magnitude_str = match self.format_type { + Some(FormatType::Binary) => self.format_int_radix(magnitude, 2), + Some(FormatType::Decimal) => self.format_int_radix(magnitude, 10), + Some(FormatType::Octal) => self.format_int_radix(magnitude, 8), + Some(FormatType::Hex(Case::Lower)) => self.format_int_radix(magnitude, 16), + Some(FormatType::Hex(Case::Upper)) => match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + Ok(result) + } + }, + Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), + Some(FormatType::Number(Case::Upper)) => { + Err(FormatSpecError::UnknownFormatCode('N', "int")) + } + Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")), + Some(FormatType::Character) => match (self.sign, self.alternate_form) { + (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), + (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), + (_, _) => match num.to_u32() { + Some(n) if n <= 0x10ffff => Ok(std::char::from_u32(n).unwrap().to_string()), + Some(_) | None => Err(FormatSpecError::CodeNotInRange), + }, + }, + Some(FormatType::GeneralFormat(_)) + | Some(FormatType::FixedPoint(_)) + | Some(FormatType::Exponent(_)) + | Some(FormatType::Percentage) => match num.to_f64() { + Some(float) => return self.format_float(float), + _ => Err(FormatSpecError::UnableToConvert), + }, + None => self.format_int_radix(magnitude, 10), + }?; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = match num.sign() { + Sign::Minus => "-", + _ => match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + }, + }; + let sign_prefix = format!("{sign_str}{prefix}"); + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, &sign_prefix); + self.format_sign_and_align( + &AsciiStr::new(&magnitude_str), + &sign_prefix, + FormatAlign::Right, + ) + } + + pub fn format_string(&self, s: &T) -> Result + where + T: CharLen + Deref, + { + self.validate_format(FormatType::String)?; + match self.format_type { + Some(FormatType::String) | None => self + .format_sign_and_align(s, "", FormatAlign::Left) + .map(|mut value| { + if let Some(precision) = self.precision { + value.truncate(precision); + } + value + }), + _ => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "str")) + } + } + } + + fn format_sign_and_align( + &self, + magnitude_str: &T, + sign_str: &str, + default_align: FormatAlign, + ) -> Result + where + T: CharLen + Deref, + { + let align = self.align.unwrap_or(default_align); + + let num_chars = magnitude_str.char_len(); + let fill_char = self.fill.unwrap_or(' '); + let fill_chars_needed: i32 = self.width.map_or(0, |w| { + cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32)) + }); + + let magnitude_str = magnitude_str.deref(); + Ok(match align { + FormatAlign::Left => format!( + "{}{}{}", + sign_str, + magnitude_str, + FormatSpec::compute_fill_string(fill_char, fill_chars_needed) + ), + FormatAlign::Right => format!( + "{}{}{}", + FormatSpec::compute_fill_string(fill_char, fill_chars_needed), + sign_str, + magnitude_str + ), + FormatAlign::AfterSign => format!( + "{}{}{}", + sign_str, + FormatSpec::compute_fill_string(fill_char, fill_chars_needed), + magnitude_str + ), + FormatAlign::Center => { + let left_fill_chars_needed = fill_chars_needed / 2; + let right_fill_chars_needed = fill_chars_needed - left_fill_chars_needed; + let left_fill_string = + FormatSpec::compute_fill_string(fill_char, left_fill_chars_needed); + let right_fill_string = + FormatSpec::compute_fill_string(fill_char, right_fill_chars_needed); + format!("{left_fill_string}{sign_str}{magnitude_str}{right_fill_string}") + } + }) + } +} + +pub trait CharLen { + /// Returns the number of characters in the text + fn char_len(&self) -> usize; +} + +struct AsciiStr<'a> { + inner: &'a str, +} + +impl<'a> AsciiStr<'a> { + fn new(inner: &'a str) -> Self { + Self { inner } + } +} + +impl CharLen for AsciiStr<'_> { + fn char_len(&self) -> usize { + self.inner.len() + } +} + +impl Deref for AsciiStr<'_> { + type Target = str; + fn deref(&self) -> &Self::Target { + self.inner + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatSpecError { + DecimalDigitsTooMany, + PrecisionTooBig, + InvalidFormatSpecifier, + UnspecifiedFormat(char, char), + UnknownFormatCode(char, &'static str), + PrecisionNotAllowed, + NotAllowed(&'static str), + UnableToConvert, + CodeNotInRange, + NotImplemented(char, &'static str), +} + +#[derive(Debug, PartialEq)] +pub enum FormatParseError { + UnmatchedBracket, + MissingStartBracket, + UnescapedStartBracketInLiteral, + InvalidFormatSpecifier, + UnknownConversion, + EmptyAttribute, + MissingRightBracket, + InvalidCharacterAfterRightBracket, +} + +impl FromStr for FormatSpec { + type Err = FormatSpecError; + fn from_str(s: &str) -> Result { + FormatSpec::parse(s) + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldNamePart { + Attribute(String), + Index(usize), + StringIndex(String), +} + +impl FieldNamePart { + fn parse_part( + chars: &mut impl PeekingNext, + ) -> Result, FormatParseError> { + chars + .next() + .map(|ch| match ch { + '.' => { + let mut attribute = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + attribute.push(ch); + } + if attribute.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else { + Ok(FieldNamePart::Attribute(attribute)) + } + } + '[' => { + let mut index = String::new(); + for ch in chars { + if ch == ']' { + return if index.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else if let Ok(index) = index.parse::() { + Ok(FieldNamePart::Index(index)) + } else { + Ok(FieldNamePart::StringIndex(index)) + }; + } + index.push(ch); + } + Err(FormatParseError::MissingRightBracket) + } + _ => Err(FormatParseError::InvalidCharacterAfterRightBracket), + }) + .transpose() + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldType { + Auto, + Index(usize), + Keyword(String), +} + +#[derive(Debug, PartialEq)] +pub struct FieldName { + pub field_type: FieldType, + pub parts: Vec, +} + +impl FieldName { + pub fn parse(text: &str) -> Result { + let mut chars = text.chars().peekable(); + let mut first = String::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + first.push(ch); + } + + let field_type = if first.is_empty() { + FieldType::Auto + } else if let Ok(index) = first.parse::() { + FieldType::Index(index) + } else { + FieldType::Keyword(first) + }; + + let mut parts = Vec::new(); + while let Some(part) = FieldNamePart::parse_part(&mut chars)? { + parts.push(part) + } + + Ok(FieldName { field_type, parts }) + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatPart { + Field { + field_name: String, + conversion_spec: Option, + format_spec: String, + }, + Literal(String), +} + +#[derive(Debug, PartialEq)] +pub struct FormatString { + pub format_parts: Vec, +} + +impl FormatString { + fn parse_literal_single(text: &str) -> Result<(char, &str), FormatParseError> { + let mut chars = text.chars(); + // This should never be called with an empty str + let first_char = chars.next().unwrap(); + // isn't this detectable only with bytes operation? + if first_char == '{' || first_char == '}' { + let maybe_next_char = chars.next(); + // if we see a bracket, it has to be escaped by doubling up to be in a literal + return if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char { + Err(FormatParseError::UnescapedStartBracketInLiteral) + } else { + Ok((first_char, chars.as_str())) + }; + } + Ok((first_char, chars.as_str())) + } + + fn parse_literal(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + let mut cur_text = text; + let mut result_string = String::new(); + while !cur_text.is_empty() { + match FormatString::parse_literal_single(cur_text) { + Ok((next_char, remaining)) => { + result_string.push(next_char); + cur_text = remaining; + } + Err(err) => { + return if !result_string.is_empty() { + Ok((FormatPart::Literal(result_string), cur_text)) + } else { + Err(err) + }; + } + } + } + Ok((FormatPart::Literal(result_string), "")) + } + + fn parse_part_in_brackets(text: &str) -> Result { + let mut chars = text.chars().peekable(); + + let mut left = String::new(); + let mut right = String::new(); + + let mut split = false; + let mut selected = &mut left; + let mut inside_brackets = false; + + while let Some(char) = chars.next() { + if char == '[' { + inside_brackets = true; + + selected.push(char); + + while let Some(next_char) = chars.next() { + selected.push(next_char); + + if next_char == ']' { + inside_brackets = false; + break; + } + if chars.peek().is_none() { + return Err(FormatParseError::MissingRightBracket); + } + } + } else if char == ':' && !split && !inside_brackets { + split = true; + selected = &mut right; + } else { + selected.push(char); + } + } + + // before the comma is a keyword or arg index, after the comma is maybe a spec. + let arg_part: &str = &left; + + let format_spec = if split { right } else { String::new() }; + + // left can still be the conversion (!r, !s, !a) + let parts: Vec<&str> = arg_part.splitn(2, '!').collect(); + // before the bang is a keyword or arg index, after the comma is maybe a conversion spec. + let arg_part = parts[0]; + + let conversion_spec = parts + .get(1) + .map(|conversion| { + // conversions are only every one character + conversion + .chars() + .exactly_one() + .map_err(|_| FormatParseError::UnknownConversion) + }) + .transpose()?; + + Ok(FormatPart::Field { + field_name: arg_part.to_owned(), + conversion_spec, + format_spec, + }) + } + + fn parse_spec(text: &str) -> Result<(FormatPart, &str), FormatParseError> { + let mut nested = false; + let mut end_bracket_pos = None; + let mut left = String::new(); + + // There may be one layer nesting brackets in spec + for (idx, c) in text.char_indices() { + if idx == 0 { + if c != '{' { + return Err(FormatParseError::MissingStartBracket); + } + } else if c == '{' { + if nested { + return Err(FormatParseError::InvalidFormatSpecifier); + } else { + nested = true; + left.push(c); + continue; + } + } else if c == '}' { + if nested { + nested = false; + left.push(c); + continue; + } else { + end_bracket_pos = Some(idx); + break; + } + } else { + left.push(c); + } + } + if let Some(pos) = end_bracket_pos { + let (_, right) = text.split_at(pos); + let format_part = FormatString::parse_part_in_brackets(&left)?; + Ok((format_part, &right[1..])) + } else { + Err(FormatParseError::UnmatchedBracket) + } + } +} + +pub trait FromTemplate<'a>: Sized { + type Err; + fn from_str(s: &'a str) -> Result; +} + +impl<'a> FromTemplate<'a> for FormatString { + type Err = FormatParseError; + + fn from_str(text: &'a str) -> Result { + let mut cur_text: &str = text; + let mut parts: Vec = Vec::new(); + while !cur_text.is_empty() { + // Try to parse both literals and bracketed format parts until we + // run out of text + cur_text = FormatString::parse_literal(cur_text) + .or_else(|_| FormatString::parse_spec(cur_text)) + .map(|(part, new_text)| { + parts.push(part); + new_text + })?; + } + Ok(FormatString { + format_parts: parts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + parse_fill_and_align(" <"), + (Some(' '), Some(FormatAlign::Left), "") + ); + assert_eq!( + parse_fill_and_align(" <22"), + (Some(' '), Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align("<22"), + (None, Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align(" ^^"), + (Some(' '), Some(FormatAlign::Center), "^") + ); + assert_eq!( + parse_fill_and_align("==="), + (Some('='), Some(FormatAlign::AfterSign), "=") + ); + } + + #[test] + fn test_width_only() { + let expected = Ok(FormatSpec { + conversion: None, + fill: None, + align: None, + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("33"), expected); + } + + #[test] + fn test_fill_and_width() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'), + align: Some(FormatAlign::Right), + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("<>33"), expected); + } + + #[test] + fn test_all() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'), + align: Some(FormatAlign::Right), + sign: Some(FormatSign::Minus), + alternate_form: true, + width: Some(23), + grouping_option: Some(FormatGrouping::Comma), + precision: Some(11), + format_type: Some(FormatType::Binary), + }); + assert_eq!(FormatSpec::parse("<>-#23,.11b"), expected); + } + + fn format_bool(text: &str, value: bool) -> Result { + FormatSpec::parse(text).and_then(|spec| spec.format_bool(value)) + } + + #[test] + fn test_format_bool() { + assert_eq!(format_bool("b", true), Ok("1".to_owned())); + assert_eq!(format_bool("b", false), Ok("0".to_owned())); + assert_eq!(format_bool("d", true), Ok("1".to_owned())); + assert_eq!(format_bool("d", false), Ok("0".to_owned())); + assert_eq!(format_bool("o", true), Ok("1".to_owned())); + assert_eq!(format_bool("o", false), Ok("0".to_owned())); + assert_eq!(format_bool("n", true), Ok("1".to_owned())); + assert_eq!(format_bool("n", false), Ok("0".to_owned())); + assert_eq!(format_bool("x", true), Ok("1".to_owned())); + assert_eq!(format_bool("x", false), Ok("0".to_owned())); + assert_eq!(format_bool("X", true), Ok("1".to_owned())); + assert_eq!(format_bool("X", false), Ok("0".to_owned())); + assert_eq!(format_bool("g", true), Ok("1".to_owned())); + assert_eq!(format_bool("g", false), Ok("0".to_owned())); + assert_eq!(format_bool("G", true), Ok("1".to_owned())); + assert_eq!(format_bool("G", false), Ok("0".to_owned())); + assert_eq!(format_bool("c", true), Ok("\x01".to_owned())); + assert_eq!(format_bool("c", false), Ok("\x00".to_owned())); + assert_eq!(format_bool("e", true), Ok("1.000000e+00".to_owned())); + assert_eq!(format_bool("e", false), Ok("0.000000e+00".to_owned())); + assert_eq!(format_bool("E", true), Ok("1.000000E+00".to_owned())); + assert_eq!(format_bool("E", false), Ok("0.000000E+00".to_owned())); + assert_eq!(format_bool("f", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("f", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("F", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("F", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("%", true), Ok("100.000000%".to_owned())); + assert_eq!(format_bool("%", false), Ok("0.000000%".to_owned())); + } + + #[test] + fn test_format_int() { + assert_eq!( + FormatSpec::parse("d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10".to_owned()) + ); + assert_eq!( + FormatSpec::parse("b") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10000".to_owned()) + ); + assert_eq!( + FormatSpec::parse("o") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("20".to_owned()) + ); + assert_eq!( + FormatSpec::parse("+d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("+16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("^ 5d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Minus, b"\x10")), + Ok(" -16 ".to_owned()) + ); + assert_eq!( + FormatSpec::parse("0>+#10x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("00000+0x10".to_owned()) + ); + } + + #[test] + fn test_format_int_sep() { + let spec = FormatSpec::parse(",").expect(""); + assert_eq!(spec.grouping_option, Some(FormatGrouping::Comma)); + assert_eq!( + spec.format_int(&BigInt::from_str("1234567890123456789012345678").unwrap()), + Ok("1,234,567,890,123,456,789,012,345,678".to_owned()) + ); + } + + #[test] + fn test_format_parse() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("abcd".to_owned()), + FormatPart::Field { + field_name: "1".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + FormatPart::Literal(":".to_owned()), + FormatPart::Field { + field_name: "key".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + ], + }); + + assert_eq!(FormatString::from_str("abcd{1}:{key}"), expected); + } + + #[test] + fn test_format_parse_multi_byte_char() { + assert!(FormatString::from_str("{a:%ЫйЯЧ}").is_ok()); + } + + #[test] + fn test_format_parse_fail() { + assert_eq!( + FormatString::from_str("{s"), + Err(FormatParseError::UnmatchedBracket) + ); + } + + #[test] + fn test_square_brackets_inside_format() { + assert_eq!( + FormatString::from_str("{[:123]}"), + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "[:123]".to_owned(), + conversion_spec: None, + format_spec: "".to_owned(), + }], + }), + ); + + assert_eq!(FormatString::from_str("{asdf[:123]asdf}"), { + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "asdf[:123]asdf".to_owned(), + conversion_spec: None, + format_spec: "".to_owned(), + }], + }) + }); + + assert_eq!(FormatString::from_str("{[1234}"), { + Err(FormatParseError::MissingRightBracket) + }); + } + + #[test] + fn test_format_parse_escape() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("{".to_owned()), + FormatPart::Field { + field_name: "key".to_owned(), + conversion_spec: None, + format_spec: String::new(), + }, + FormatPart::Literal("}ddfe".to_owned()), + ], + }); + + assert_eq!(FormatString::from_str("{{{key}}}ddfe"), expected); + } + + #[test] + fn test_format_invalid_specification() { + assert_eq!( + FormatSpec::parse("%3"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse(".2fa"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("ds"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("x+"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("b4"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("o!"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("d "), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + } + + #[test] + fn test_parse_field_name() { + assert_eq!( + FieldName::parse(""), + Ok(FieldName { + field_type: FieldType::Auto, + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("0"), + Ok(FieldName { + field_type: FieldType::Index(0), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key"), + Ok(FieldName { + field_type: FieldType::Keyword("key".to_owned()), + parts: Vec::new(), + }) + ); + assert_eq!( + FieldName::parse("key.attr[0][string]"), + Ok(FieldName { + field_type: FieldType::Keyword("key".to_owned()), + parts: vec![ + FieldNamePart::Attribute("attr".to_owned()), + FieldNamePart::Index(0), + FieldNamePart::StringIndex("string".to_owned()) + ], + }) + ); + assert_eq!( + FieldName::parse("key.."), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key[]"), + Err(FormatParseError::EmptyAttribute) + ); + assert_eq!( + FieldName::parse("key["), + Err(FormatParseError::MissingRightBracket) + ); + assert_eq!( + FieldName::parse("key[0]after"), + Err(FormatParseError::InvalidCharacterAfterRightBracket) + ); + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index cf2286c1c..760ba8c55 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -9,6 +9,7 @@ pub use macros::*; pub mod atomic; pub mod borrow; pub mod boxvec; +pub mod cformat; pub mod cmp; #[cfg(any(unix, windows, target_os = "wasi"))] pub mod crt_fd; @@ -16,6 +17,7 @@ pub mod encodings; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] pub mod fileutils; pub mod float_ops; +pub mod format; pub mod hash; pub mod int; pub mod linked_list; diff --git a/common/src/str.rs b/common/src/str.rs index b4f7a1a63..89d2381d3 100644 --- a/common/src/str.rs +++ b/common/src/str.rs @@ -1,9 +1,9 @@ use crate::{ atomic::{PyAtomic, Radium}, + format::CharLen, hash::PyHash, }; use ascii::AsciiString; -use rustpython_format::CharLen; use std::ops::{Bound, RangeBounds}; #[cfg(not(target_arch = "wasm32"))] diff --git a/compiler/literal/Cargo.toml b/compiler/literal/Cargo.toml new file mode 100644 index 000000000..de7e89c0b --- /dev/null +++ b/compiler/literal/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "rustpython-literal" +description = "Common literal handling utilities mostly useful for unparse and repr." +edition = { workspace = true } +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +hexf-parse = "0.2.1" +is-macro.workspace = true +lexical-parse-float = { version = "0.8.0", features = ["format"] } +num-traits = { workspace = true } +unic-ucd-category = "0.9" + +[dev-dependencies] +rand = { workspace = true } diff --git a/compiler/literal/src/char.rs b/compiler/literal/src/char.rs new file mode 100644 index 000000000..cd64f6dfa --- /dev/null +++ b/compiler/literal/src/char.rs @@ -0,0 +1,15 @@ +use unic_ucd_category::GeneralCategory; + +/// According to python following categories aren't printable: +/// * Cc (Other, Control) +/// * Cf (Other, Format) +/// * Cs (Other, Surrogate) +/// * Co (Other, Private Use) +/// * Cn (Other, Not Assigned) +/// * Zl Separator, Line ('\u2028', LINE SEPARATOR) +/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) +/// * Zs (Separator, Space) other than ASCII space('\x20'). +pub fn is_printable(c: char) -> bool { + let cat = GeneralCategory::of(c); + !(cat.is_other() || cat.is_separator()) +} diff --git a/compiler/literal/src/escape.rs b/compiler/literal/src/escape.rs new file mode 100644 index 000000000..ba8e3ecf1 --- /dev/null +++ b/compiler/literal/src/escape.rs @@ -0,0 +1,441 @@ +#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)] +pub enum Quote { + Single, + Double, +} + +impl Quote { + #[inline] + pub const fn swap(self) -> Quote { + match self { + Quote::Single => Quote::Double, + Quote::Double => Quote::Single, + } + } + + #[inline] + pub const fn to_byte(&self) -> u8 { + match self { + Quote::Single => b'\'', + Quote::Double => b'"', + } + } + + #[inline] + pub const fn to_char(&self) -> char { + match self { + Quote::Single => '\'', + Quote::Double => '"', + } + } +} + +pub struct EscapeLayout { + pub quote: Quote, + pub len: Option, +} + +pub trait Escape { + fn source_len(&self) -> usize; + fn layout(&self) -> &EscapeLayout; + fn changed(&self) -> bool { + self.layout().len != Some(self.source_len()) + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; + fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + if self.changed() { + self.write_body_slow(formatter) + } else { + self.write_source(formatter) + } + } +} + +/// Returns the outer quotes to use and the number of quotes that need to be +/// escaped. +pub(crate) const fn choose_quote( + single_count: usize, + double_count: usize, + preferred_quote: Quote, +) -> (Quote, usize) { + let (primary_count, secondary_count) = match preferred_quote { + Quote::Single => (single_count, double_count), + Quote::Double => (double_count, single_count), + }; + + // always use primary unless we have primary but no secondary + let use_secondary = primary_count > 0 && secondary_count == 0; + if use_secondary { + (preferred_quote.swap(), secondary_count) + } else { + (preferred_quote, primary_count) + } +} + +pub struct UnicodeEscape<'a> { + source: &'a str, + layout: EscapeLayout, +} + +impl<'a> UnicodeEscape<'a> { + #[inline] + pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a str, quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a str) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> { + StrRepr(self) + } +} + +pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>); + +impl StrRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl std::fmt::Display for StrRepr<'_, '_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.write(formatter) + } +} + +impl UnicodeEscape<'_> { + const REPR_RESERVED_LEN: usize = 2; // for quotes + + pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &str, + preferred_quote: Quote, + length_add: impl Fn(usize, usize) -> Option, + ) -> EscapeLayout { + let mut out_len = Self::REPR_RESERVED_LEN; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source.chars() { + let incr = match ch { + '\'' => { + single_count += 1; + 1 + } + '"' => { + double_count += 1; + 1 + } + c => Self::escaped_char_len(c), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - Self::REPR_RESERVED_LEN), + } + } + + fn escaped_char_len(ch: char) -> usize { + match ch { + '\\' | '\t' | '\r' | '\n' => 2, + ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH + ch if ch.is_ascii() => 1, + ch if crate::char::is_printable(ch) => { + // max = std::cmp::max(ch, max); + ch.len_utf8() + } + ch if (ch as u32) < 0x100 => 4, // \xHH + ch if (ch as u32) < 0x10000 => 6, // \uHHHH + _ => 10, // \uHHHHHHHH + } + } + + fn write_char( + ch: char, + quote: Quote, + formatter: &mut impl std::fmt::Write, + ) -> std::fmt::Result { + match ch { + '\n' => formatter.write_str("\\n"), + '\t' => formatter.write_str("\\t"), + '\r' => formatter.write_str("\\r"), + // these 2 branches *would* be handled below, but we shouldn't have to do a + // unicodedata lookup just for ascii characters + '\x20'..='\x7e' => { + // printable ascii range + if ch == quote.to_char() || ch == '\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch) + } + ch if ch.is_ascii() => { + write!(formatter, "\\x{:02x}", ch as u8) + } + ch if crate::char::is_printable(ch) => formatter.write_char(ch), + '\0'..='\u{ff}' => { + write!(formatter, "\\x{:02x}", ch as u32) + } + '\0'..='\u{ffff}' => { + write!(formatter, "\\u{:04x}", ch as u32) + } + _ => { + write!(formatter, "\\U{:08x}", ch as u32) + } + } + } +} + +impl Escape for UnicodeEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + formatter.write_str(self.source) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + for ch in self.source.chars() { + Self::write_char(ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct AsciiEscape<'a> { + source: &'a [u8], + layout: EscapeLayout, +} + +impl<'a> AsciiEscape<'a> { + #[inline] + pub fn new(source: &'a [u8], layout: EscapeLayout) -> Self { + Self { source, layout } + } + #[inline] + pub fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a [u8]) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> { + BytesRepr(self) + } +} + +impl AsciiEscape<'_> { + pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout { + Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &[u8], + preferred_quote: Quote, + reserved_len: usize, + length_add: impl Fn(usize, usize) -> Option, + ) -> EscapeLayout { + let mut out_len = reserved_len; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source.iter() { + let incr = match ch { + b'\'' => { + single_count += 1; + 1 + } + b'"' => { + double_count += 1; + 1 + } + c => Self::escaped_char_len(*c), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - reserved_len), + } + } + + fn escaped_char_len(ch: u8) -> usize { + match ch { + b'\\' | b'\t' | b'\r' | b'\n' => 2, + 0x20..=0x7e => 1, + _ => 4, // \xHH + } + } + + fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + match ch { + b'\t' => formatter.write_str("\\t"), + b'\n' => formatter.write_str("\\n"), + b'\r' => formatter.write_str("\\r"), + 0x20..=0x7e => { + // printable ascii range + if ch == quote.to_byte() || ch == b'\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch as char) + } + ch => write!(formatter, "\\x{ch:02x}"), + } + } +} + +impl Escape for AsciiEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + formatter.write_str(unsafe { + // SAFETY: this function must be called only when source is printable ascii characters + std::str::from_utf8_unchecked(self.source) + }) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + for ch in self.source.iter() { + Self::write_char(*ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>); + +impl BytesRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char('b')?; + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl std::fmt::Display for BytesRepr<'_, '_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.write(formatter) + } +} + +#[cfg(test)] +mod unicode_escape_tests { + use super::*; + + #[test] + fn changed() { + fn test(s: &str) -> bool { + UnicodeEscape::new_repr(s).changed() + } + assert!(!test("hello")); + assert!(!test("'hello'")); + assert!(!test("\"hello\"")); + + assert!(test("'\"hello")); + assert!(test("hello\n")); + } +} diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs new file mode 100644 index 000000000..a58a3f48a --- /dev/null +++ b/compiler/literal/src/float.rs @@ -0,0 +1,342 @@ +use crate::format::Case; +use num_traits::{Float, Zero}; +use std::f64; + +pub fn parse_str(literal: &str) -> Option { + parse_inner(literal.trim().as_bytes()) +} + +fn strip_underlines(literal: &[u8]) -> Option> { + let mut prev = b'\0'; + let mut dup = Vec::::new(); + for p in literal { + if *p == b'_' { + // Underscores are only allowed after digits. + if !prev.is_ascii_digit() { + return None; + } + } else { + dup.push(*p); + // Underscores are only allowed before digits. + if prev == b'_' && !p.is_ascii_digit() { + return None; + } + } + prev = *p; + } + + // Underscores are not allowed at the end. + if prev == b'_' { + return None; + } + + Some(dup) +} + +pub fn parse_bytes(literal: &[u8]) -> Option { + parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace())) +} + +fn trim_slice(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] { + let mut it = v.iter(); + // it.take_while_ref(&mut trim).for_each(drop); + // hmm.. `&mut slice::Iter<_>` is not `Clone` + // it.by_ref().rev().take_while_ref(&mut trim).for_each(drop); + while it.clone().next().is_some_and(&mut trim) { + it.next(); + } + while it.clone().next_back().is_some_and(&mut trim) { + it.next_back(); + } + it.as_slice() +} + +fn parse_inner(literal: &[u8]) -> Option { + use lexical_parse_float::{ + FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL, + }; + + // Use custom function for underline handling for now. + // For further information see https://github.com/Alexhuszagh/rust-lexical/issues/96. + let stripped = strip_underlines(literal)?; + + // lexical-core's format::PYTHON_STRING is inaccurate + const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL) + .no_special(false) + .build(); + f64::from_lexical_with_options::(&stripped, &Options::new()).ok() +} + +pub fn is_integer(v: f64) -> bool { + (v - v.round()).abs() < f64::EPSILON +} + +fn format_nan(case: Case) -> String { + let nan = match case { + Case::Lower => "nan", + Case::Upper => "NAN", + }; + + nan.to_string() +} + +fn format_inf(case: Case) -> String { + let inf = match case { + Case::Lower => "inf", + Case::Upper => "INF", + }; + + inf.to_string() +} + +pub fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str { + match (precision, alternate_form) { + (0, true) => ".", + _ => "", + } +} + +pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let point = decimal_point_or_empty(precision, alternate_form); + format!("{magnitude:.precision$}{point}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// Formats floats into Python style exponent notation, by first formatting in Rust style +// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`). +pub fn format_exponent( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{magnitude:.precision$e}"); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::().unwrap(); + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +/// If s represents a floating point value, trailing zeros and a possibly trailing +/// decimal point will be removed. +/// This function does NOT work with decimal commas. +fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String { + if !alternate_form && s.contains('.') { + // only truncate floating point values when not in alternate form + let s = remove_trailing_zeros(s); + remove_trailing_decimal_point(s) + } else { + s + } +} + +fn remove_trailing_zeros(s: String) -> String { + let mut s = s; + while s.ends_with('0') { + s.pop(); + } + s +} + +fn remove_trailing_decimal_point(s: String) -> String { + let mut s = s; + if s.ends_with('.') { + s.pop(); + } + s +} + +pub fn format_general( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, + always_shows_fract: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::().unwrap(); + if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) { + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let magnitude = format!("{:.*}", precision + 1, base); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } else { + let precision = ((precision as i64) - 1 - exponent) as usize; + let magnitude = format!("{magnitude:.precision$}"); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}") + } + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// TODO: rewrite using format_general +pub fn to_string(value: f64) -> String { + let lit = format!("{value:e}"); + if let Some(position) = lit.find('e') { + let significand = &lit[..position]; + let exponent = &lit[position + 1..]; + let exponent = exponent.parse::().unwrap(); + if exponent < 16 && exponent > -5 { + if is_integer(value) { + format!("{value:.1?}") + } else { + value.to_string() + } + } else { + format!("{significand}e{exponent:+#03}") + } + } else { + let mut s = value.to_string(); + s.make_ascii_lowercase(); + s + } +} + +pub fn from_hex(s: &str) -> Option { + if let Ok(f) = hexf_parse::parse_hexf64(s, false) { + return Some(f); + } + match s.to_ascii_lowercase().as_str() { + "nan" | "+nan" | "-nan" => Some(f64::NAN), + "inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY), + "-inf" | "-infinity" => Some(f64::NEG_INFINITY), + value => { + let mut hex = String::with_capacity(value.len()); + let has_0x = value.contains("0x"); + let has_p = value.contains('p'); + let has_dot = value.contains('.'); + let mut start = 0; + + if !has_0x && value.starts_with('-') { + hex.push_str("-0x"); + start += 1; + } else if !has_0x { + hex.push_str("0x"); + if value.starts_with('+') { + start += 1; + } + } + + for (index, ch) in value.chars().enumerate() { + if ch == 'p' { + if has_dot { + hex.push('p'); + } else { + hex.push_str(".p"); + } + } else if index >= start { + hex.push(ch); + } + } + + if !has_p && has_dot { + hex.push_str("p0"); + } else if !has_p && !has_dot { + hex.push_str(".p0") + } + + hexf_parse::parse_hexf64(hex.as_str(), false).ok() + } + } +} + +pub fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"), + value if value.is_infinite() => format!("{sign_fmt}inf"), + value if value.is_nan() => "nan".to_owned(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}{:#x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::rng().random::(); + let f = f64::from_bits(bytes); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } +} + +#[test] +fn test_remove_trailing_zeros() { + assert!(remove_trailing_zeros(String::from("100")) == *"1"); + assert!(remove_trailing_zeros(String::from("100.00")) == *"100."); + + // leave leading zeros untouched + assert!(remove_trailing_zeros(String::from("001")) == *"001"); + + // leave strings untouched if they don't end with 0 + assert!(remove_trailing_zeros(String::from("101")) == *"101"); +} + +#[test] +fn test_remove_trailing_decimal_point() { + assert!(remove_trailing_decimal_point(String::from("100.")) == *"100"); + assert!(remove_trailing_decimal_point(String::from("1.")) == *"1"); + + // leave leading decimal points untouched + assert!(remove_trailing_decimal_point(String::from(".5")) == *".5"); +} + +#[test] +fn test_maybe_remove_trailing_redundant_chars() { + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100."); + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100"); + assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1"); + assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10"); + + // don't truncate integers + assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000"); +} diff --git a/compiler/literal/src/format.rs b/compiler/literal/src/format.rs new file mode 100644 index 000000000..4ce21ad78 --- /dev/null +++ b/compiler/literal/src/format.rs @@ -0,0 +1,5 @@ +#[derive(Debug, PartialEq, Eq, Clone, Copy, is_macro::Is, Hash)] +pub enum Case { + Lower, + Upper, +} diff --git a/compiler/literal/src/lib.rs b/compiler/literal/src/lib.rs new file mode 100644 index 000000000..9b9620573 --- /dev/null +++ b/compiler/literal/src/lib.rs @@ -0,0 +1,4 @@ +pub mod char; +pub mod escape; +pub mod float; +pub mod format; diff --git a/deny.toml b/deny.toml index 4063d1fec..85513455f 100644 --- a/deny.toml +++ b/deny.toml @@ -242,4 +242,4 @@ unknown-git = "warn" # if not specified. If it is specified but empty, no registries are allowed. allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories -allow-git = ["https://github.com/RustPython/Parser.git", "https://github.com/RustPython/__doc__"] +allow-git = ["https://github.com/RustPython/__doc__"] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 49b0883bb..559390d27 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -42,7 +42,6 @@ rustpython-compiler-core = { workspace = true } rustpython-compiler-source = { workspace = true } # rustpython-parser-core = { workspace = true } rustpython-literal = { workspace = true } -rustpython-format = { workspace = true } rustpython-sre_engine = { workspace = true } ascii = { workspace = true } diff --git a/vm/src/builtins/bool.rs b/vm/src/builtins/bool.rs index 93f5f4f1d..006fc4d1e 100644 --- a/vm/src/builtins/bool.rs +++ b/vm/src/builtins/bool.rs @@ -1,4 +1,5 @@ use super::{PyInt, PyStrRef, PyType, PyTypeRef}; +use crate::common::format::FormatSpec; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, @@ -11,7 +12,6 @@ use crate::{ }; use malachite_bigint::Sign; use num_traits::Zero; -use rustpython_format::FormatSpec; use std::fmt::{Debug, Formatter}; impl ToPyObject for bool { diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs index e33c25cb5..b4601fbb9 100644 --- a/vm/src/builtins/float.rs +++ b/vm/src/builtins/float.rs @@ -7,7 +7,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, class::PyClassImpl, - common::{float_ops, hash}, + common::{float_ops, format::FormatSpec, hash}, convert::{IntoPyException, ToPyObject, ToPyResult}, function::{ ArgBytesLike, OptionalArg, OptionalOption, @@ -21,7 +21,6 @@ use malachite_bigint::{BigInt, ToBigInt}; use num_complex::Complex64; use num_traits::{Signed, ToPrimitive, Zero}; use rustpython_common::int::float_to_ratio; -use rustpython_format::FormatSpec; #[pyclass(module = false, name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index db9c5ea4e..aa9613e9d 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -6,6 +6,7 @@ use crate::{ bytesinner::PyBytesInner, class::PyClassImpl, common::{ + format::FormatSpec, hash, int::{bigint_to_finite_float, bytes_to_int, true_div}, }, @@ -20,7 +21,6 @@ use crate::{ use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; -use rustpython_format::FormatSpec; use std::fmt; use std::ops::{Neg, Not}; diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index 3446a3d54..554fa1de7 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -31,10 +31,10 @@ use once_cell::sync::Lazy; use rustpython_common::{ ascii, atomic::{self, PyAtomic, Radium}, + format::{FormatSpec, FormatString, FromTemplate}, hash, lock::PyMutex, }; -use rustpython_format::{FormatSpec, FormatString, FromTemplate}; use std::{char, fmt, ops::Range, string::ToString}; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 6e14034d0..a11c06a25 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,6 +1,7 @@ //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use crate::common::cformat::*; use crate::{ AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, builtins::{ @@ -12,7 +13,6 @@ use crate::{ }; use itertools::Itertools; use num_traits::cast::ToPrimitive; -use rustpython_format::cformat::*; use std::str::FromStr; fn spec_format_bytes( diff --git a/vm/src/format.rs b/vm/src/format.rs index 7e9bb5426..1dbfd4677 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -6,7 +6,7 @@ use crate::{ stdlib::builtins, }; -use rustpython_format::*; +use crate::common::format::*; impl IntoPyException for FormatSpecError { fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 8a8a18273..3c399e1c3 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -6,15 +6,15 @@ pub(crate) use _string::make_module; #[pymodule] mod _string { use crate::common::ascii; + use crate::common::format::{ + FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, + }; use crate::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyList, PyStrRef}, convert::ToPyException, convert::ToPyObject, }; - use rustpython_format::{ - FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, - }; use std::mem; fn create_format_part(