Implement round-half-even float.fromhex in common float_ops (#8234)

* Add round-half-even hex float parser to common float_ops

Add float_ops::from_hex returning Result<f64, HexFloatError>, which parses
the coefficient and exponent, rounds round-half-even over the full exponent
range, and detects overflow. Route float.fromhex through it: raise
OverflowError for values too large to represent and ValueError for invalid
input. Remove the expectedFailure on test_from_hex.

Assisted-by: Claude

* Address review: Option-returning byte helpers, const fns, drop test_ prefix

Assisted-by: Claude

* Whitelist fdigits and inity in cspell

Assisted-by: Claude

* Home is_py_ascii_whitespace in rustpython-wtf8; share with common and sre_engine

Assisted-by: Claude

* Use shared wtf8 is_py_ascii_whitespace in bytes_inner

Remove the private copy of is_py_ascii_whitespace in bytes_inner.rs and
import the shared rustpython_common::wtf8::is_py_ascii_whitespace.

Assisted-by: Claude

.cspell.json

@@ -66,11 +66,13 @@
     "deoptimize",
     "emscripten",
     "excs",
+    "fdigits",
     "flufl",
     "fnfe",
     "fsdefault",
     "ifexp",
     "implicits",
+    "inity",
     "interps",
     "jitted",
     "jitting",

Lib/test/test_float.py

@@ -1262,7 +1262,6 @@ class HexFloatTestCase(FloatsAreIdenticalMixin, unittest.TestCase):
                     self.identical(got, expected)
 
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; ValueError: invalid hexadecimal floating-point string
     def test_from_hex(self):
         MIN = self.MIN
         MAX = self.MAX

crates/common/src/float_ops.rs

@@ -279,6 +279,414 @@ pub fn round_float_digits(x: f64, ndigits: i32) -> Option<f64> {
     Some(result)
 }
 
+/// Error from [`from_hex`], mapping to the exception the caller should raise.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HexFloatError {
+    /// ValueError "invalid hexadecimal floating-point string"
+    Invalid,
+    /// ValueError "hexadecimal string too long to convert"
+    TooLong,
+    /// OverflowError "hexadecimal value too large to represent as a float"
+    Overflow,
+}
+
+const DBL_MANT_DIG: i64 = 53;
+const DBL_MIN_EXP: i64 = -1021;
+const DBL_MAX_EXP: i64 = 1024;
+
+/// Read byte at `i`, returning `None` past the end so that digit/sign/space
+/// scans stop at the string boundary.
+#[inline]
+fn byte_at(bytes: &[u8], i: usize) -> Option<u8> {
+    bytes.get(i).copied()
+}
+
+/// '0'-'9' -> 0..9, 'a'-'f'/'A'-'F' -> 10..15, else `None`.
+#[inline]
+const fn hex_from_char(c: u8) -> Option<u8> {
+    match c {
+        b'0'..=b'9' => Some(c - b'0'),
+        b'a'..=b'f' => Some(c - b'a' + 10),
+        b'A'..=b'F' => Some(c - b'A' + 10),
+        _ => None,
+    }
+}
+
+/// Peek at byte `i` and decode it as a hex digit, or `None` if it is out of
+/// range or not a hex digit.
+#[inline]
+fn hex_digit_at(bytes: &[u8], i: usize) -> Option<u8> {
+    byte_at(bytes, i).and_then(hex_from_char)
+}
+
+/// `t` must be an ASCII-lowercase literal. Returns true if every byte of `t`
+/// matched case-insensitively starting at `s`.
+fn case_insensitive_match(bytes: &[u8], s: usize, t: &[u8]) -> bool {
+    let mut si = s;
+    let mut ti = 0;
+    while ti < t.len() && byte_at(bytes, si).is_some_and(|b| b.to_ascii_lowercase() == t[ti]) {
+        si += 1;
+        ti += 1;
+    }
+    ti == t.len()
+}
+
+/// Returns `Some((value, endptr))` when the text at `p` parses as inf/nan,
+/// otherwise `None`.
+fn parse_inf_or_nan(bytes: &[u8], p: usize) -> Option<(f64, usize)> {
+    let mut s = p;
+    let mut negate = false;
+    if byte_at(bytes, s) == Some(b'-') {
+        negate = true;
+        s += 1;
+    } else if byte_at(bytes, s) == Some(b'+') {
+        s += 1;
+    }
+    if case_insensitive_match(bytes, s, b"inf") {
+        s += 3;
+        if case_insensitive_match(bytes, s, b"inity") {
+            s += 5;
+        }
+        let value = if negate {
+            f64::NEG_INFINITY
+        } else {
+            f64::INFINITY
+        };
+        Some((value, s))
+    } else if case_insensitive_match(bytes, s, b"nan") {
+        s += 3;
+        let value = if negate {
+            f64::from_bits(0xfff8_0000_0000_0000)
+        } else {
+            f64::from_bits(0x7ff8_0000_0000_0000)
+        };
+        Some((value, s))
+    } else {
+        None
+    }
+}
+
+/// Correctly-rounded scalbn. Every call site scales an already-representable
+/// value, so the result is exact.
+const fn ldexp(x: f64, mut n: i32) -> f64 {
+    let x1p1023 = f64::from_bits(0x7fe0000000000000);
+    let x1p53 = f64::from_bits(0x4340000000000000);
+    let x1p_1022 = f64::from_bits(0x0010000000000000);
+    let mut y = x;
+    if n > 1023 {
+        y *= x1p1023;
+        n -= 1023;
+        if n > 1023 {
+            y *= x1p1023;
+            n -= 1023;
+            if n > 1023 {
+                n = 1023;
+            }
+        }
+    } else if n < -1022 {
+        y *= x1p_1022 * x1p53;
+        n += 1022 - 53;
+        if n < -1022 {
+            y *= x1p_1022 * x1p53;
+            n += 1022 - 53;
+            if n < -1022 {
+                n = -1022;
+            }
+        }
+    }
+    y * f64::from_bits(((0x3ff + n) as u64) << 52)
+}
+
+/// Parse the already-validated `[+-]?[0-9]+` slice `bytes[start..end]` as base-10
+/// signed, saturating to i64::MIN/MAX on overflow like strtol.
+fn strtol_saturating(bytes: &[u8], start: usize, end: usize) -> i64 {
+    let mut i = start;
+    let mut neg = false;
+    if i < end && (bytes[i] == b'+' || bytes[i] == b'-') {
+        neg = bytes[i] == b'-';
+        i += 1;
+    }
+    let mut val: i64 = 0;
+    let mut overflowed = false;
+    while i < end {
+        let d = (bytes[i] - b'0') as i64;
+        match val.checked_mul(10).and_then(|v| v.checked_add(d)) {
+            Some(v) => val = v,
+            None => {
+                overflowed = true;
+                break;
+            }
+        }
+        i += 1;
+    }
+    if overflowed {
+        if neg { i64::MIN } else { i64::MAX }
+    } else if neg {
+        -val
+    } else {
+        val
+    }
+}
+
+/// Parse a hexadecimal floating-point string (the `float.fromhex` grammar).
+///
+/// The raw string is consumed as-is: leading and trailing whitespace are handled
+/// internally using the ASCII space set, so callers must not trim first.
+pub fn from_hex(s: &str) -> Result<f64, HexFloatError> {
+    let bytes = s.as_bytes();
+    let s_end = bytes.len();
+
+    let mut negate = false;
+    let mut idx = 0usize;
+    let mut x;
+
+    // leading whitespace
+    while byte_at(bytes, idx).is_some_and(rustpython_wtf8::is_py_ascii_whitespace) {
+        idx += 1;
+    }
+
+    // infinities and nans
+    if let Some((value, end)) = parse_inf_or_nan(bytes, idx) {
+        idx = end;
+        return finish_hex(bytes, s_end, idx, negate, value);
+    }
+
+    // optional sign
+    if byte_at(bytes, idx) == Some(b'-') {
+        idx += 1;
+        negate = true;
+    } else if byte_at(bytes, idx) == Some(b'+') {
+        idx += 1;
+    }
+
+    // [0x]
+    let s_store = idx;
+    if byte_at(bytes, idx) == Some(b'0') {
+        idx += 1;
+        if matches!(byte_at(bytes, idx), Some(b'x' | b'X')) {
+            idx += 1;
+        } else {
+            idx = s_store;
+        }
+    }
+
+    // coefficient: <integer> [. <fraction>]
+    let coeff_start = idx;
+    while hex_digit_at(bytes, idx).is_some() {
+        idx += 1;
+    }
+    let s_store = idx;
+    let coeff_end = if byte_at(bytes, idx) == Some(b'.') {
+        idx += 1;
+        while hex_digit_at(bytes, idx).is_some() {
+            idx += 1;
+        }
+        idx - 1
+    } else {
+        idx
+    };
+
+    // ndigits = total # of hex digits; fdigits = # after point
+    let ndigits_total = (coeff_end - coeff_start) as i64;
+    let fdigits = (coeff_end - s_store) as i64;
+    if ndigits_total == 0 {
+        return Err(HexFloatError::Invalid);
+    }
+    let insane_bound = core::cmp::min(
+        DBL_MIN_EXP - DBL_MANT_DIG - i64::MIN / 2,
+        i64::MAX / 2 + 1 - DBL_MAX_EXP,
+    ) / 4;
+    if ndigits_total > insane_bound {
+        return Err(HexFloatError::TooLong);
+    }
+
+    // [p <exponent>]
+    let exp = if matches!(byte_at(bytes, idx), Some(b'p' | b'P')) {
+        idx += 1;
+        let exp_start = idx;
+        if matches!(byte_at(bytes, idx), Some(b'-' | b'+')) {
+            idx += 1;
+        }
+        if !matches!(byte_at(bytes, idx), Some(b'0'..=b'9')) {
+            return Err(HexFloatError::Invalid);
+        }
+        idx += 1;
+        while matches!(byte_at(bytes, idx), Some(b'0'..=b'9')) {
+            idx += 1;
+        }
+        strtol_saturating(bytes, exp_start, idx)
+    } else {
+        0
+    };
+
+    // HEX_DIGIT(j): jth hex digit counting from the least significant.
+    let hex_digit = |j: i64| -> i32 {
+        let byte_idx = if j < fdigits {
+            coeff_end as i64 - j
+        } else {
+            coeff_end as i64 - 1 - j
+        };
+        hex_digit_at(bytes, byte_idx as usize).expect("hex digit within coefficient") as i32
+    };
+
+    // Discard leading zeros, and catch extreme overflow and underflow.
+    let mut ndigits = ndigits_total;
+    while ndigits > 0 && hex_digit(ndigits - 1) == 0 {
+        ndigits -= 1;
+    }
+    if ndigits == 0 || exp < i64::MIN / 2 {
+        x = 0.0;
+        return finish_hex(bytes, s_end, idx, negate, x);
+    }
+    if exp > i64::MAX / 2 {
+        return Err(HexFloatError::Overflow);
+    }
+
+    // Adjust exponent for fractional part.
+    let exp = exp - 4 * fdigits;
+
+    // top_exp = 1 more than exponent of most significant bit of coefficient.
+    let mut top_exp = exp + 4 * (ndigits - 1);
+    let mut digit = hex_digit(ndigits - 1);
+    while digit != 0 {
+        top_exp += 1;
+        digit /= 2;
+    }
+
+    // catch almost all nonextreme cases of overflow and underflow here
+    if top_exp < DBL_MIN_EXP - DBL_MANT_DIG {
+        x = 0.0;
+        return finish_hex(bytes, s_end, idx, negate, x);
+    }
+    if top_exp > DBL_MAX_EXP {
+        return Err(HexFloatError::Overflow);
+    }
+
+    // lsb = exponent of least significant bit of the rounded value.
+    let lsb = core::cmp::max(top_exp, DBL_MIN_EXP) - DBL_MANT_DIG;
+
+    x = 0.0;
+    if exp >= lsb {
+        // no rounding required
+        let mut i = ndigits - 1;
+        while i >= 0 {
+            x = 16.0 * x + hex_digit(i) as f64;
+            i -= 1;
+        }
+        x = ldexp(x, exp as i32);
+        return finish_hex(bytes, s_end, idx, negate, x);
+    }
+
+    // rounding required. key_digit is the index of the hex digit
+    // containing the first bit to be rounded away.
+    let half_eps: i32 = 1 << ((lsb - exp - 1) % 4) as i32;
+    let key_digit = (lsb - exp - 1) / 4;
+    let mut i = ndigits - 1;
+    while i > key_digit {
+        x = 16.0 * x + hex_digit(i) as f64;
+        i -= 1;
+    }
+    let digit = hex_digit(key_digit);
+    x = 16.0 * x + (digit & (16 - 2 * half_eps)) as f64;
+
+    // round-half-even
+    if (digit & half_eps) != 0 {
+        let round_up = if (digit & (3 * half_eps - 1)) != 0
+            || (half_eps == 8 && key_digit + 1 < ndigits && (hex_digit(key_digit + 1) & 1) != 0)
+        {
+            true
+        } else {
+            let mut r = false;
+            let mut i = key_digit - 1;
+            while i >= 0 {
+                if hex_digit(i) != 0 {
+                    r = true;
+                    break;
+                }
+                i -= 1;
+            }
+            r
+        };
+        if round_up {
+            x += (2 * half_eps) as f64;
+            if top_exp == DBL_MAX_EXP && x == ldexp((2 * half_eps) as f64, DBL_MANT_DIG as i32) {
+                // overflow corner case
+                return Err(HexFloatError::Overflow);
+            }
+        }
+    }
+    x = ldexp(x, (exp + 4 * key_digit) as i32);
+
+    finish_hex(bytes, s_end, idx, negate, x)
+}
+
+/// Skip trailing whitespace, require the whole string was consumed, and apply
+/// the sign.
+fn finish_hex(
+    bytes: &[u8],
+    s_end: usize,
+    mut idx: usize,
+    negate: bool,
+    x: f64,
+) -> Result<f64, HexFloatError> {
+    while byte_at(bytes, idx).is_some_and(rustpython_wtf8::is_py_ascii_whitespace) {
+        idx += 1;
+    }
+    if idx != s_end {
+        return Err(HexFloatError::Invalid);
+    }
+    Ok(if negate { -x } else { x })
+}
+
+#[cfg(test)]
+mod from_hex_tests {
+    use super::{HexFloatError, from_hex};
+
+    fn bits(s: &str) -> u64 {
+        from_hex(s).unwrap().to_bits()
+    }
+
+    #[test]
+    fn from_hex_exact_bits() {
+        assert_eq!(bits("0x1p-1074"), 0x0000000000000001);
+        assert_eq!(bits("0x1.fffffffffffffp+1023"), 0x7fefffffffffffff);
+        // round-half-even ties
+        assert_eq!(bits("0x1.00000000000008p0"), 0x3ff0000000000000);
+        assert_eq!(bits("0x1.00000000000018p0"), 0x3ff0000000000002);
+        assert_eq!(bits("-0x1p0"), 0xbff0000000000000);
+        assert_eq!(bits("0x0p0"), 0x0000000000000000);
+        assert_eq!(bits("-0x0p0"), 0x8000000000000000);
+    }
+
+    #[test]
+    fn from_hex_inf_nan() {
+        assert_eq!(bits("inf"), 0x7ff0000000000000);
+        assert_eq!(bits("-inf"), 0xfff0000000000000);
+        assert_eq!(bits("Infinity"), 0x7ff0000000000000);
+
+        let n = from_hex("nan").unwrap();
+        assert!(n.is_nan());
+        assert_eq!(n.to_bits(), 0x7ff8000000000000);
+        let neg = from_hex("-nan").unwrap();
+        assert!(neg.is_nan());
+        assert_eq!(neg.to_bits(), 0xfff8000000000000);
+    }
+
+    #[test]
+    fn from_hex_whitespace() {
+        assert_eq!(bits("  0x1p0  "), 0x3ff0000000000000);
+        assert_eq!(bits("\t0x1p0\n"), 0x3ff0000000000000);
+    }
+
+    #[test]
+    fn from_hex_errors() {
+        assert_eq!(from_hex("0x1p1024"), Err(HexFloatError::Overflow));
+        assert_eq!(from_hex("0x1z"), Err(HexFloatError::Invalid));
+        assert_eq!(from_hex(""), Err(HexFloatError::Invalid));
+        assert_eq!(from_hex("0x1 p0"), Err(HexFloatError::Invalid));
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/sre_engine/src/string.rs

@@ -341,10 +341,6 @@ const CONT_MASK: u8 = 0b0011_1111;
 
 const UNDERSCORE: u32 = '_' as u32;
 
-const fn is_py_ascii_whitespace(b: u8) -> bool {
-    matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
-}
-
 #[inline]
 pub(crate) fn is_word(ch: u32) -> bool {
     ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
@@ -352,7 +348,7 @@ pub(crate) fn is_word(ch: u32) -> bool {
 
 #[inline]
 pub(crate) fn is_space(ch: u32) -> bool {
-    u8::try_from(ch).is_ok_and(is_py_ascii_whitespace)
+    u8::try_from(ch).is_ok_and(rustpython_wtf8::is_py_ascii_whitespace)
 }
 
 #[inline]

crates/vm/src/builtins/float.rs

@@ -399,8 +399,16 @@ impl PyFloat {
 
     #[pyclassmethod]
     fn fromhex(cls: PyTypeRef, string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult {
-        let result = crate::literal::float::from_hex(string.as_str().trim())
-            .ok_or_else(|| vm.new_value_error("invalid hexadecimal floating-point string"))?;
+        use float_ops::HexFloatError;
+        let result = float_ops::from_hex(string.as_str()).map_err(|e| match e {
+            HexFloatError::Overflow => {
+                vm.new_overflow_error("hexadecimal value too large to represent as a float")
+            }
+            HexFloatError::TooLong => vm.new_value_error("hexadecimal string too long to convert"),
+            HexFloatError::Invalid => {
+                vm.new_value_error("invalid hexadecimal floating-point string")
+            }
+        })?;
         PyType::call(&cls, vec![vm.ctx.new_float(result).into()].into(), vm)
     }
 

crates/vm/src/bytes_inner.rs

@@ -9,6 +9,7 @@ use crate::{
     byte::bytes_from_object,
     cformat::cformat_bytes,
     common::hash,
+    common::wtf8::is_py_ascii_whitespace,
     function::{ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue},
     literal::escape::Escape,
     protocol::PyBuffer,
@@ -1207,10 +1208,6 @@ pub(crate) fn bytes_to_hex(
     }
 }
 
-pub(crate) const fn is_py_ascii_whitespace(b: u8) -> bool {
-    matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
-}
-
 /// ASCII-only title casing.
 ///
 /// This is purposely naive as is CPython's implementation.

crates/wtf8/src/lib.rs

@@ -1347,6 +1347,13 @@ pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! {
     panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary");
 }
 
+/// True for the ASCII bytes Python treats as whitespace in numeric parsing
+/// (`\t \n \x0b \x0c \r` and space).
+#[must_use]
+pub const fn is_py_ascii_whitespace(b: u8) -> bool {
+    matches!(b, b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r' | b' ')
+}
+
 /// Iterator for the code points of a WTF-8 string.
 ///
 /// Created with the method `.code_points()`.