Use current locale for byte regular-expression matching (#9042)

* Use current locale for byte regular-expression matching

Extract the Python 3.14-compatible LOCALE matching paths from #8954, keeping host callbacks outside the no_std engine and preserving the C-locale fallback. Existing canonical regex tests cover locale changes and compiled patterns. Keep expected-failure markers until patched native execution proves they are obsolete; omit duplicate snippet tests and Python 3.15 prefixmatch additions.

Assisted-by: OpenAI Codex:model identifier unavailable

* Enable the repaired locale caching regression

The exact patched native interpreter passes the existing locale flag and compiled-pattern tests and unexpectedly passes locale caching. Remove only the obsolete expected-failure decorator; preserve all canonical assertions and platform skips.

Assisted-by: OpenAI Codex:model identifier unavailable

Lib/test/test_re.py

@@ -2179,7 +2179,6 @@ class ReTests(unittest.TestCase):
         # with ignore case.
         self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3))
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true
     @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390")
     @unittest.skipIf(sys.platform.startswith("sunos"),
                      "test doesn't work on Solaris, gh-91214")

crates/host_env/src/locale.rs

@@ -3,6 +3,53 @@ use core::{ffi::CStr, ptr};
 
 pub use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME};
 
+// CPython uses the wide-character APIs after btowc() on Apple platforms.
+// Darwin's wint_t is int; libc does not expose these declarations.
+#[cfg(target_vendor = "apple")]
+unsafe extern "C" {
+    fn btowc(ch: libc::c_int) -> libc::c_int;
+    fn iswalnum(ch: libc::c_int) -> libc::c_int;
+    fn towlower(ch: libc::c_int) -> libc::c_int;
+    fn towupper(ch: libc::c_int) -> libc::c_int;
+}
+
+/// Classify a byte using the current LC_CTYPE locale.
+pub fn byte_isalnum(ch: u8) -> bool {
+    // ctype functions require EOF or an unsigned-char value, never a signed byte.
+    #[cfg(not(target_vendor = "apple"))]
+    unsafe {
+        libc::isalnum(ch.into()) != 0
+    }
+    #[cfg(target_vendor = "apple")]
+    unsafe {
+        iswalnum(btowc(ch.into())) != 0
+    }
+}
+
+/// Lowercase a byte using the current LC_CTYPE locale.
+pub fn byte_tolower(ch: u8) -> u32 {
+    #[cfg(not(target_vendor = "apple"))]
+    unsafe {
+        libc::tolower(ch.into()) as u32
+    }
+    #[cfg(target_vendor = "apple")]
+    unsafe {
+        towlower(btowc(ch.into())) as u32
+    }
+}
+
+/// Uppercase a byte using the current LC_CTYPE locale.
+pub fn byte_toupper(ch: u8) -> u32 {
+    #[cfg(not(target_vendor = "apple"))]
+    unsafe {
+        libc::toupper(ch.into()) as u32
+    }
+    #[cfg(target_vendor = "apple")]
+    unsafe {
+        towupper(btowc(ch.into())) as u32
+    }
+}
+
 #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
 pub use libc::LC_MESSAGES;
 

crates/sre_engine/src/engine.rs

@@ -1,11 +1,11 @@
 // good luck to those that follow; here be dragons
 
 use crate::string::{
-    is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space,
-    is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode,
+    is_digit, is_linebreak, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, is_uni_word,
+    is_word, lower_ascii, lower_unicode, upper_unicode,
 };
 
-use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
+use super::{Locale, MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
 use alloc::{vec, vec::Vec};
 use core::{convert::TryFrom, ptr::null};
 use optional::Optioned;
@@ -18,6 +18,7 @@ pub struct Request<'a, S> {
     pub pattern_codes: &'a [u32],
     pub match_all: bool,
     pub must_advance: bool,
+    pub locale: Locale,
 }
 
 impl<'a, S: StrDrive> Request<'a, S> {
@@ -38,6 +39,7 @@ impl<'a, S: StrDrive> Request<'a, S> {
             pattern_codes,
             match_all,
             must_advance: false,
+            locale: Locale::default(),
         }
     }
 }
@@ -674,20 +676,24 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
                         }
                         SreOpcode::CATEGORY => {
                             let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap();
-                            if ctx.at_end(req) || !category(cat_code, ctx.peek_char::<S>()) {
+                            if ctx.at_end(req)
+                                || !category(cat_code, ctx.peek_char::<S>(), req.locale)
+                            {
                                 break 'result false;
                             }
                             ctx.skip_code(2);
                             ctx.advance_char::<S>();
                         }
-                        SreOpcode::IN => general_op_in!(charset),
+                        SreOpcode::IN => general_op_in!(|set, c| charset(set, c, req.locale)),
                         SreOpcode::IN_IGNORE => {
-                            general_op_in!(|set, c| charset(set, lower_ascii(c)))
+                            general_op_in!(|set, c| charset(set, lower_ascii(c), req.locale))
                         }
                         SreOpcode::IN_UNI_IGNORE => {
-                            general_op_in!(|set, c| charset(set, lower_unicode(c)))
+                            general_op_in!(|set, c| charset(set, lower_unicode(c), req.locale))
+                        }
+                        SreOpcode::IN_LOC_IGNORE => {
+                            general_op_in!(|set, c| charset_loc_ignore(set, c, req.locale))
                         }
-                        SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore),
                         SreOpcode::MARK => {
                             state
                                 .marks
@@ -837,13 +843,17 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
                         SreOpcode::NOT_LITERAL_UNI_IGNORE => {
                             general_op_literal!(|code, c| code != lower_unicode(c))
                         }
-                        SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore),
+                        SreOpcode::LITERAL_LOC_IGNORE => {
+                            general_op_literal!(|code, c| char_loc_ignore(code, c, req.locale))
+                        }
                         SreOpcode::NOT_LITERAL_LOC_IGNORE => {
-                            general_op_literal!(|code, c| !char_loc_ignore(code, c))
+                            general_op_literal!(|code, c| !char_loc_ignore(code, c, req.locale))
                         }
                         SreOpcode::GROUPREF => general_op_groupref!(|x| x),
                         SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii),
-                        SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate),
+                        SreOpcode::GROUPREF_LOC_IGNORE => {
+                            general_op_groupref!(|c| req.locale.lower(c))
+                        }
                         SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode),
                         SreOpcode::GROUPREF_EXISTS => {
                             let (group_start, group_end) =
@@ -1042,7 +1052,7 @@ fn search_info_charset<S: StrDrive>(
     req.must_advance = false;
 
     loop {
-        while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>()) {
+        while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>(), req.locale) {
             ctx.advance_char::<S>();
         }
         if ctx.at_end(req) {
@@ -1226,27 +1236,27 @@ fn at<S: StrDrive>(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode)
         }
         SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req),
         SreAtCode::END_STRING => ctx.at_end(req),
-        SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word),
-        SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word),
+        SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, |ch| req.locale.is_word(ch)),
+        SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, |ch| req.locale.is_word(ch)),
         SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word),
         SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word),
     }
 }
 
-fn char_loc_ignore(code: u32, c: u32) -> bool {
-    code == c || code == lower_locate(c) || code == upper_locate(c)
+fn char_loc_ignore(code: u32, c: u32, locale: Locale) -> bool {
+    code == c || code == locale.lower(c) || code == locale.upper(c)
 }
 
-fn charset_loc_ignore(set: &[u32], c: u32) -> bool {
-    let lo = lower_locate(c);
-    if charset(set, c) {
+fn charset_loc_ignore(set: &[u32], c: u32, locale: Locale) -> bool {
+    let lo = locale.lower(c);
+    if charset(set, lo, locale) {
         return true;
     }
-    let up = upper_locate(c);
-    up != lo && charset(set, up)
+    let up = locale.upper(c);
+    up != lo && charset(set, up, locale)
 }
 
-fn category(cat_code: SreCatCode, c: u32) -> bool {
+fn category(cat_code: SreCatCode, c: u32, locale: Locale) -> bool {
     match cat_code {
         SreCatCode::DIGIT => is_digit(c),
         SreCatCode::NOT_DIGIT => !is_digit(c),
@@ -1256,8 +1266,8 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
         SreCatCode::NOT_WORD => !is_word(c),
         SreCatCode::LINEBREAK => is_linebreak(c),
         SreCatCode::NOT_LINEBREAK => !is_linebreak(c),
-        SreCatCode::LOC_WORD => is_loc_word(c),
-        SreCatCode::LOC_NOT_WORD => !is_loc_word(c),
+        SreCatCode::LOC_WORD => locale.is_word(c),
+        SreCatCode::LOC_NOT_WORD => !locale.is_word(c),
         SreCatCode::UNI_DIGIT => is_uni_digit(c),
         SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c),
         SreCatCode::UNI_SPACE => is_uni_space(c),
@@ -1269,7 +1279,7 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
     }
 }
 
-fn charset(set: &[u32], ch: u32) -> bool {
+fn charset(set: &[u32], ch: u32, locale: Locale) -> bool {
     /* check if character is a member of the given set */
     let mut ok = true;
     let mut i = 0;
@@ -1292,7 +1302,7 @@ fn charset(set: &[u32], ch: u32) -> bool {
                         break;
                     }
                 };
-                if category(cat_code, ch) {
+                if category(cat_code, ch, locale) {
                     return ok;
                 }
                 i += 2;
@@ -1382,7 +1392,8 @@ fn _count<S: StrDrive>(
             ctx.skip_char::<S>(max_count);
         }
         SreOpcode::IN => {
-            while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>())
+            while ctx.cursor.position < end
+                && charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>(), req.locale)
             {
                 ctx.advance_char::<S>();
             }
@@ -1400,10 +1411,14 @@ fn _count<S: StrDrive>(
             general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c));
         }
         SreOpcode::LITERAL_LOC_IGNORE => {
-            general_count_literal(req, ctx, end, char_loc_ignore);
+            general_count_literal(req, ctx, end, |code, c| {
+                char_loc_ignore(code, c, req.locale)
+            });
         }
         SreOpcode::NOT_LITERAL_LOC_IGNORE => {
-            general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c));
+            general_count_literal(req, ctx, end, |code, c| {
+                !char_loc_ignore(code, c, req.locale)
+            });
         }
         SreOpcode::LITERAL_UNI_IGNORE => {
             general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c));

crates/sre_engine/src/lib.rs

@@ -8,7 +8,7 @@ pub mod string;
 
 pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode};
 pub use engine::{Request, SearchIter, State};
-pub use string::{StrDrive, StringCursor};
+pub use string::{Locale, StrDrive, StringCursor};
 
 pub const CODESIZE: usize = 4;
 

crates/sre_engine/src/string.rs

@@ -391,15 +391,44 @@ pub(crate) fn is_digit(ch: u32) -> bool {
     u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit())
 }
 
-#[inline]
-pub(crate) fn is_loc_alnum(ch: u32) -> bool {
-    // FIXME: Ignore the locales
-    u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
+/// Byte classification and case conversion in the current locale.
+///
+/// Callbacks keep host APIs outside this `no_std` engine. They are called while
+/// matching, rather than cached in a compiled pattern, so a locale change also
+/// affects existing patterns and scanners. Without host callbacks, matching uses
+/// the ASCII rules of the C locale.
+#[derive(Debug, Clone, Copy)]
+pub struct Locale {
+    pub is_alnum: fn(u8) -> bool,
+    pub to_lower: fn(u8) -> u32,
+    pub to_upper: fn(u8) -> u32,
 }
 
-#[inline]
-pub(crate) fn is_loc_word(ch: u32) -> bool {
-    ch == UNDERSCORE || is_loc_alnum(ch)
+impl Default for Locale {
+    fn default() -> Self {
+        Self {
+            is_alnum: |ch| ch.is_ascii_alphanumeric(),
+            to_lower: |ch| ch.to_ascii_lowercase() as u32,
+            to_upper: |ch| ch.to_ascii_uppercase() as u32,
+        }
+    }
+}
+
+impl Locale {
+    #[inline]
+    pub(crate) fn is_word(self, ch: u32) -> bool {
+        ch == UNDERSCORE || u8::try_from(ch).is_ok_and(self.is_alnum)
+    }
+
+    #[inline]
+    pub(crate) fn lower(self, ch: u32) -> u32 {
+        u8::try_from(ch).map_or(ch, self.to_lower)
+    }
+
+    #[inline]
+    pub(crate) fn upper(self, ch: u32) -> u32 {
+        u8::try_from(ch).map_or(ch, self.to_upper)
+    }
 }
 
 #[inline]
@@ -413,18 +442,6 @@ pub fn lower_ascii(ch: u32) -> u32 {
     u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32)
 }
 
-#[inline]
-pub(crate) fn lower_locate(ch: u32) -> u32 {
-    // FIXME: Ignore the locales
-    lower_ascii(ch)
-}
-
-#[inline]
-pub(crate) fn upper_locate(ch: u32) -> u32 {
-    // FIXME: Ignore the locales
-    u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32)
-}
-
 #[inline]
 pub(crate) fn is_uni_digit(ch: u32) -> bool {
     // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL),

crates/vm/src/stdlib/_sre.rs

@@ -59,7 +59,17 @@ mod _sre {
         fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef;
 
         fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> {
-            Request::new(self, start, end, &pattern.code, false)
+            let request = Request::new(self, start, end, &pattern.code, false);
+            #[cfg(all(feature = "host_env", any(unix, windows)))]
+            let request = Request {
+                locale: rustpython_sre_engine::Locale {
+                    is_alnum: rustpython_host_env::locale::byte_isalnum,
+                    to_lower: rustpython_host_env::locale::byte_tolower,
+                    to_upper: rustpython_host_env::locale::byte_toupper,
+                },
+                ..request
+            };
+            request
         }
     }