Fix isalnum/isalpha to use Unicode general category checks; fix regex \\w for Mn characters

Agent-Logs-Url: https://github.com/RustPython/RustPython/sessions/6f573a91-8811-486c-933d-7ba9a9067643

Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com>

Cargo.lock

@@ -3249,6 +3249,7 @@ dependencies = [
  "num_enum",
  "optional",
  "rustpython-wtf8",
+ "unic-ucd-category",
 ]
 
 [[package]]

crates/sre_engine/Cargo.toml

@@ -16,6 +16,7 @@ harness = false
 
 [dependencies]
 rustpython-wtf8 = { workspace = true }
+unic-ucd-category = { workspace = true }
 num_enum = { workspace = true }
 bitflags = { workspace = true }
 optional = { workspace = true }

crates/sre_engine/src/string.rs

@@ -1,4 +1,5 @@
 use rustpython_wtf8::Wtf8;
+use unic_ucd_category::GeneralCategory;
 
 #[derive(Debug, Clone, Copy)]
 pub struct StringCursor {
@@ -441,9 +442,20 @@ pub(crate) const fn is_uni_linebreak(ch: u32) -> bool {
 }
 #[inline]
 pub(crate) fn is_uni_alnum(ch: u32) -> bool {
-    // TODO: check with cpython
     char::try_from(ch)
-        .map(|x| x.is_alphanumeric())
+        .map(|c| {
+            matches!(
+                GeneralCategory::of(c),
+                GeneralCategory::UppercaseLetter
+                    | GeneralCategory::LowercaseLetter
+                    | GeneralCategory::TitlecaseLetter
+                    | GeneralCategory::ModifierLetter
+                    | GeneralCategory::OtherLetter
+                    | GeneralCategory::DecimalNumber
+                    | GeneralCategory::LetterNumber
+                    | GeneralCategory::OtherNumber
+            )
+        })
         .unwrap_or(false)
 }
 #[inline]

crates/vm/src/builtins/str.rs

@@ -945,7 +945,20 @@ impl PyStr {
 
     #[pymethod]
     fn isalnum(&self) -> bool {
-        !self.data.is_empty() && self.char_all(char::is_alphanumeric)
+        !self.data.is_empty()
+            && self.char_all(|c| {
+                matches!(
+                    GeneralCategory::of(c),
+                    GeneralCategory::UppercaseLetter
+                        | GeneralCategory::LowercaseLetter
+                        | GeneralCategory::TitlecaseLetter
+                        | GeneralCategory::ModifierLetter
+                        | GeneralCategory::OtherLetter
+                        | GeneralCategory::DecimalNumber
+                        | GeneralCategory::LetterNumber
+                        | GeneralCategory::OtherNumber
+                )
+            })
     }
 
     #[pymethod]
@@ -1056,7 +1069,17 @@ impl PyStr {
 
     #[pymethod]
     fn isalpha(&self) -> bool {
-        !self.data.is_empty() && self.char_all(char::is_alphabetic)
+        !self.data.is_empty()
+            && self.char_all(|c| {
+                matches!(
+                    GeneralCategory::of(c),
+                    GeneralCategory::UppercaseLetter
+                        | GeneralCategory::LowercaseLetter
+                        | GeneralCategory::TitlecaseLetter
+                        | GeneralCategory::ModifierLetter
+                        | GeneralCategory::OtherLetter
+                )
+            })
     }
 
     #[pymethod]

extra_tests/snippets/builtin_str_unicode.py

@@ -38,3 +38,12 @@ assert b"xn--pythn-mua.org.".decode("idna") == "pyth\xf6n.org."
 # TODO: add east_asian_width and mirrored
 # assert unicodedata.ucd_3_2_0.east_asian_width('\u231a') == 'N'
 # assert not unicodedata.ucd_3_2_0.mirrored("\u0f3a")
+
+# U+0345 COMBINING GREEK YPOGEGRAMMENI (category Mn) should not be alphanumeric.
+# CPython's isalpha/isalnum use Unicode letter categories (Lu/Ll/Lt/Lm/Lo),
+# not the broader Unicode Alphabetic derived property.
+assert not "\u0345".isalpha(), "isalpha should not match Mn category characters"
+assert not "\u0345".isalnum(), "isalnum should not match Mn category characters"
+
+import re
+assert not re.match(r"\w", "\u0345"), r"\w should not match U+0345 (category Mn)"