diff --git a/.cspell.json b/.cspell.json index af2f1401d..f173874d4 100644 --- a/.cspell.json +++ b/.cspell.json @@ -66,11 +66,13 @@ "deoptimize", "emscripten", "excs", + "fdigits", "flufl", "fnfe", "fsdefault", "ifexp", "implicits", + "inity", "interps", "jitted", "jitting", diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 0938c89cb..a514111c1 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -1262,7 +1262,6 @@ class HexFloatTestCase(FloatsAreIdenticalMixin, unittest.TestCase): self.identical(got, expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: invalid hexadecimal floating-point string def test_from_hex(self): MIN = self.MIN MAX = self.MAX diff --git a/crates/common/src/float_ops.rs b/crates/common/src/float_ops.rs index ca1c716a9..4f117e80c 100644 --- a/crates/common/src/float_ops.rs +++ b/crates/common/src/float_ops.rs @@ -279,6 +279,414 @@ pub fn round_float_digits(x: f64, ndigits: i32) -> Option { Some(result) } +/// Error from [`from_hex`], mapping to the exception the caller should raise. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HexFloatError { + /// ValueError "invalid hexadecimal floating-point string" + Invalid, + /// ValueError "hexadecimal string too long to convert" + TooLong, + /// OverflowError "hexadecimal value too large to represent as a float" + Overflow, +} + +const DBL_MANT_DIG: i64 = 53; +const DBL_MIN_EXP: i64 = -1021; +const DBL_MAX_EXP: i64 = 1024; + +/// Read byte at `i`, returning `None` past the end so that digit/sign/space +/// scans stop at the string boundary. +#[inline] +fn byte_at(bytes: &[u8], i: usize) -> Option { + bytes.get(i).copied() +} + +/// '0'-'9' -> 0..9, 'a'-'f'/'A'-'F' -> 10..15, else `None`. +#[inline] +const fn hex_from_char(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +/// Peek at byte `i` and decode it as a hex digit, or `None` if it is out of +/// range or not a hex digit. +#[inline] +fn hex_digit_at(bytes: &[u8], i: usize) -> Option { + byte_at(bytes, i).and_then(hex_from_char) +} + +/// `t` must be an ASCII-lowercase literal. Returns true if every byte of `t` +/// matched case-insensitively starting at `s`. +fn case_insensitive_match(bytes: &[u8], s: usize, t: &[u8]) -> bool { + let mut si = s; + let mut ti = 0; + while ti < t.len() && byte_at(bytes, si).is_some_and(|b| b.to_ascii_lowercase() == t[ti]) { + si += 1; + ti += 1; + } + ti == t.len() +} + +/// Returns `Some((value, endptr))` when the text at `p` parses as inf/nan, +/// otherwise `None`. +fn parse_inf_or_nan(bytes: &[u8], p: usize) -> Option<(f64, usize)> { + let mut s = p; + let mut negate = false; + if byte_at(bytes, s) == Some(b'-') { + negate = true; + s += 1; + } else if byte_at(bytes, s) == Some(b'+') { + s += 1; + } + if case_insensitive_match(bytes, s, b"inf") { + s += 3; + if case_insensitive_match(bytes, s, b"inity") { + s += 5; + } + let value = if negate { + f64::NEG_INFINITY + } else { + f64::INFINITY + }; + Some((value, s)) + } else if case_insensitive_match(bytes, s, b"nan") { + s += 3; + let value = if negate { + f64::from_bits(0xfff8_0000_0000_0000) + } else { + f64::from_bits(0x7ff8_0000_0000_0000) + }; + Some((value, s)) + } else { + None + } +} + +/// Correctly-rounded scalbn. Every call site scales an already-representable +/// value, so the result is exact. +const fn ldexp(x: f64, mut n: i32) -> f64 { + let x1p1023 = f64::from_bits(0x7fe0000000000000); + let x1p53 = f64::from_bits(0x4340000000000000); + let x1p_1022 = f64::from_bits(0x0010000000000000); + let mut y = x; + if n > 1023 { + y *= x1p1023; + n -= 1023; + if n > 1023 { + y *= x1p1023; + n -= 1023; + if n > 1023 { + n = 1023; + } + } + } else if n < -1022 { + y *= x1p_1022 * x1p53; + n += 1022 - 53; + if n < -1022 { + y *= x1p_1022 * x1p53; + n += 1022 - 53; + if n < -1022 { + n = -1022; + } + } + } + y * f64::from_bits(((0x3ff + n) as u64) << 52) +} + +/// Parse the already-validated `[+-]?[0-9]+` slice `bytes[start..end]` as base-10 +/// signed, saturating to i64::MIN/MAX on overflow like strtol. +fn strtol_saturating(bytes: &[u8], start: usize, end: usize) -> i64 { + let mut i = start; + let mut neg = false; + if i < end && (bytes[i] == b'+' || bytes[i] == b'-') { + neg = bytes[i] == b'-'; + i += 1; + } + let mut val: i64 = 0; + let mut overflowed = false; + while i < end { + let d = (bytes[i] - b'0') as i64; + match val.checked_mul(10).and_then(|v| v.checked_add(d)) { + Some(v) => val = v, + None => { + overflowed = true; + break; + } + } + i += 1; + } + if overflowed { + if neg { i64::MIN } else { i64::MAX } + } else if neg { + -val + } else { + val + } +} + +/// Parse a hexadecimal floating-point string (the `float.fromhex` grammar). +/// +/// The raw string is consumed as-is: leading and trailing whitespace are handled +/// internally using the ASCII space set, so callers must not trim first. +pub fn from_hex(s: &str) -> Result { + let bytes = s.as_bytes(); + let s_end = bytes.len(); + + let mut negate = false; + let mut idx = 0usize; + let mut x; + + // leading whitespace + while byte_at(bytes, idx).is_some_and(rustpython_wtf8::is_py_ascii_whitespace) { + idx += 1; + } + + // infinities and nans + if let Some((value, end)) = parse_inf_or_nan(bytes, idx) { + idx = end; + return finish_hex(bytes, s_end, idx, negate, value); + } + + // optional sign + if byte_at(bytes, idx) == Some(b'-') { + idx += 1; + negate = true; + } else if byte_at(bytes, idx) == Some(b'+') { + idx += 1; + } + + // [0x] + let s_store = idx; + if byte_at(bytes, idx) == Some(b'0') { + idx += 1; + if matches!(byte_at(bytes, idx), Some(b'x' | b'X')) { + idx += 1; + } else { + idx = s_store; + } + } + + // coefficient: [. ] + let coeff_start = idx; + while hex_digit_at(bytes, idx).is_some() { + idx += 1; + } + let s_store = idx; + let coeff_end = if byte_at(bytes, idx) == Some(b'.') { + idx += 1; + while hex_digit_at(bytes, idx).is_some() { + idx += 1; + } + idx - 1 + } else { + idx + }; + + // ndigits = total # of hex digits; fdigits = # after point + let ndigits_total = (coeff_end - coeff_start) as i64; + let fdigits = (coeff_end - s_store) as i64; + if ndigits_total == 0 { + return Err(HexFloatError::Invalid); + } + let insane_bound = core::cmp::min( + DBL_MIN_EXP - DBL_MANT_DIG - i64::MIN / 2, + i64::MAX / 2 + 1 - DBL_MAX_EXP, + ) / 4; + if ndigits_total > insane_bound { + return Err(HexFloatError::TooLong); + } + + // [p ] + let exp = if matches!(byte_at(bytes, idx), Some(b'p' | b'P')) { + idx += 1; + let exp_start = idx; + if matches!(byte_at(bytes, idx), Some(b'-' | b'+')) { + idx += 1; + } + if !matches!(byte_at(bytes, idx), Some(b'0'..=b'9')) { + return Err(HexFloatError::Invalid); + } + idx += 1; + while matches!(byte_at(bytes, idx), Some(b'0'..=b'9')) { + idx += 1; + } + strtol_saturating(bytes, exp_start, idx) + } else { + 0 + }; + + // HEX_DIGIT(j): jth hex digit counting from the least significant. + let hex_digit = |j: i64| -> i32 { + let byte_idx = if j < fdigits { + coeff_end as i64 - j + } else { + coeff_end as i64 - 1 - j + }; + hex_digit_at(bytes, byte_idx as usize).expect("hex digit within coefficient") as i32 + }; + + // Discard leading zeros, and catch extreme overflow and underflow. + let mut ndigits = ndigits_total; + while ndigits > 0 && hex_digit(ndigits - 1) == 0 { + ndigits -= 1; + } + if ndigits == 0 || exp < i64::MIN / 2 { + x = 0.0; + return finish_hex(bytes, s_end, idx, negate, x); + } + if exp > i64::MAX / 2 { + return Err(HexFloatError::Overflow); + } + + // Adjust exponent for fractional part. + let exp = exp - 4 * fdigits; + + // top_exp = 1 more than exponent of most significant bit of coefficient. + let mut top_exp = exp + 4 * (ndigits - 1); + let mut digit = hex_digit(ndigits - 1); + while digit != 0 { + top_exp += 1; + digit /= 2; + } + + // catch almost all nonextreme cases of overflow and underflow here + if top_exp < DBL_MIN_EXP - DBL_MANT_DIG { + x = 0.0; + return finish_hex(bytes, s_end, idx, negate, x); + } + if top_exp > DBL_MAX_EXP { + return Err(HexFloatError::Overflow); + } + + // lsb = exponent of least significant bit of the rounded value. + let lsb = core::cmp::max(top_exp, DBL_MIN_EXP) - DBL_MANT_DIG; + + x = 0.0; + if exp >= lsb { + // no rounding required + let mut i = ndigits - 1; + while i >= 0 { + x = 16.0 * x + hex_digit(i) as f64; + i -= 1; + } + x = ldexp(x, exp as i32); + return finish_hex(bytes, s_end, idx, negate, x); + } + + // rounding required. key_digit is the index of the hex digit + // containing the first bit to be rounded away. + let half_eps: i32 = 1 << ((lsb - exp - 1) % 4) as i32; + let key_digit = (lsb - exp - 1) / 4; + let mut i = ndigits - 1; + while i > key_digit { + x = 16.0 * x + hex_digit(i) as f64; + i -= 1; + } + let digit = hex_digit(key_digit); + x = 16.0 * x + (digit & (16 - 2 * half_eps)) as f64; + + // round-half-even + if (digit & half_eps) != 0 { + let round_up = if (digit & (3 * half_eps - 1)) != 0 + || (half_eps == 8 && key_digit + 1 < ndigits && (hex_digit(key_digit + 1) & 1) != 0) + { + true + } else { + let mut r = false; + let mut i = key_digit - 1; + while i >= 0 { + if hex_digit(i) != 0 { + r = true; + break; + } + i -= 1; + } + r + }; + if round_up { + x += (2 * half_eps) as f64; + if top_exp == DBL_MAX_EXP && x == ldexp((2 * half_eps) as f64, DBL_MANT_DIG as i32) { + // overflow corner case + return Err(HexFloatError::Overflow); + } + } + } + x = ldexp(x, (exp + 4 * key_digit) as i32); + + finish_hex(bytes, s_end, idx, negate, x) +} + +/// Skip trailing whitespace, require the whole string was consumed, and apply +/// the sign. +fn finish_hex( + bytes: &[u8], + s_end: usize, + mut idx: usize, + negate: bool, + x: f64, +) -> Result { + while byte_at(bytes, idx).is_some_and(rustpython_wtf8::is_py_ascii_whitespace) { + idx += 1; + } + if idx != s_end { + return Err(HexFloatError::Invalid); + } + Ok(if negate { -x } else { x }) +} + +#[cfg(test)] +mod from_hex_tests { + use super::{HexFloatError, from_hex}; + + fn bits(s: &str) -> u64 { + from_hex(s).unwrap().to_bits() + } + + #[test] + fn from_hex_exact_bits() { + assert_eq!(bits("0x1p-1074"), 0x0000000000000001); + assert_eq!(bits("0x1.fffffffffffffp+1023"), 0x7fefffffffffffff); + // round-half-even ties + assert_eq!(bits("0x1.00000000000008p0"), 0x3ff0000000000000); + assert_eq!(bits("0x1.00000000000018p0"), 0x3ff0000000000002); + assert_eq!(bits("-0x1p0"), 0xbff0000000000000); + assert_eq!(bits("0x0p0"), 0x0000000000000000); + assert_eq!(bits("-0x0p0"), 0x8000000000000000); + } + + #[test] + fn from_hex_inf_nan() { + assert_eq!(bits("inf"), 0x7ff0000000000000); + assert_eq!(bits("-inf"), 0xfff0000000000000); + assert_eq!(bits("Infinity"), 0x7ff0000000000000); + + let n = from_hex("nan").unwrap(); + assert!(n.is_nan()); + assert_eq!(n.to_bits(), 0x7ff8000000000000); + let neg = from_hex("-nan").unwrap(); + assert!(neg.is_nan()); + assert_eq!(neg.to_bits(), 0xfff8000000000000); + } + + #[test] + fn from_hex_whitespace() { + assert_eq!(bits(" 0x1p0 "), 0x3ff0000000000000); + assert_eq!(bits("\t0x1p0\n"), 0x3ff0000000000000); + } + + #[test] + fn from_hex_errors() { + assert_eq!(from_hex("0x1p1024"), Err(HexFloatError::Overflow)); + assert_eq!(from_hex("0x1z"), Err(HexFloatError::Invalid)); + assert_eq!(from_hex(""), Err(HexFloatError::Invalid)); + assert_eq!(from_hex("0x1 p0"), Err(HexFloatError::Invalid)); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 6468c6d0c..5deeab67e 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -341,10 +341,6 @@ const CONT_MASK: u8 = 0b0011_1111; const UNDERSCORE: u32 = '_' as u32; -const fn is_py_ascii_whitespace(b: u8) -> bool { - matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') -} - #[inline] pub(crate) fn is_word(ch: u32) -> bool { ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) @@ -352,7 +348,7 @@ pub(crate) fn is_word(ch: u32) -> bool { #[inline] pub(crate) fn is_space(ch: u32) -> bool { - u8::try_from(ch).is_ok_and(is_py_ascii_whitespace) + u8::try_from(ch).is_ok_and(rustpython_wtf8::is_py_ascii_whitespace) } #[inline] diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 36846640e..1c861b14f 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -399,8 +399,16 @@ impl PyFloat { #[pyclassmethod] fn fromhex(cls: PyTypeRef, string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { - let result = crate::literal::float::from_hex(string.as_str().trim()) - .ok_or_else(|| vm.new_value_error("invalid hexadecimal floating-point string"))?; + use float_ops::HexFloatError; + let result = float_ops::from_hex(string.as_str()).map_err(|e| match e { + HexFloatError::Overflow => { + vm.new_overflow_error("hexadecimal value too large to represent as a float") + } + HexFloatError::TooLong => vm.new_value_error("hexadecimal string too long to convert"), + HexFloatError::Invalid => { + vm.new_value_error("invalid hexadecimal floating-point string") + } + })?; PyType::call(&cls, vec![vm.ctx.new_float(result).into()].into(), vm) } diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index 08518deb1..9cfe4d760 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -9,6 +9,7 @@ use crate::{ byte::bytes_from_object, cformat::cformat_bytes, common::hash, + common::wtf8::is_py_ascii_whitespace, function::{ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue}, literal::escape::Escape, protocol::PyBuffer, @@ -1207,10 +1208,6 @@ pub(crate) fn bytes_to_hex( } } -pub(crate) const fn is_py_ascii_whitespace(b: u8) -> bool { - matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') -} - /// ASCII-only title casing. /// /// This is purposely naive as is CPython's implementation. diff --git a/crates/wtf8/src/lib.rs b/crates/wtf8/src/lib.rs index b31ed1cf0..2167b6b04 100644 --- a/crates/wtf8/src/lib.rs +++ b/crates/wtf8/src/lib.rs @@ -1347,6 +1347,13 @@ pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); } +/// True for the ASCII bytes Python treats as whitespace in numeric parsing +/// (`\t \n \x0b \x0c \r` and space). +#[must_use] +pub const fn is_py_ascii_whitespace(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r' | b' ') +} + /// Iterator for the code points of a WTF-8 string. /// /// Created with the method `.code_points()`.