fix: Handle char expansion in islower, isupper (#7583)

* fix: Handle char expansion in islower, isupper

Closes: #7526

`py_islower` and `py_isupper` need to handle expansions for letter
casing. Comparing chars directly can miss edge cases in certain
languages. Unfortunately, like the last PR, this allocates to handle
potential expansions.

I also had to add `icu_casemap` as a dependency.
RustPython is already using parts of icu4x so this doesn't add many
transitive dependencies.

* Ensure islower/isupper handles strs without chars

This fixes a regression mentioned by CodeRabbit. I also figured out how
to check a string's case without allocation using Unicode properties.
Thus, this commit removes `icu_casemap` again. `icu_casemap` and my old
solution is required for a robust case check, but it seems like the
current code is fine for Python.

Cargo.lock

@@ -1505,12 +1505,13 @@ dependencies = [
 
 [[package]]
 name = "icu_collections"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
 dependencies = [
  "displaydoc",
  "potential_utf",
+ "utf8_iter",
  "yoke",
  "zerofrom",
  "zerovec",
@@ -1518,9 +1519,9 @@ dependencies = [
 
 [[package]]
 name = "icu_locale_core"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
 dependencies = [
  "displaydoc",
  "litemap",
@@ -1531,9 +1532,9 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
 dependencies = [
  "icu_collections",
  "icu_normalizer_data",
@@ -1548,15 +1549,15 @@ dependencies = [
 
 [[package]]
 name = "icu_normalizer_data"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
 
 [[package]]
 name = "icu_properties"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
 dependencies = [
  "icu_collections",
  "icu_locale_core",
@@ -1568,15 +1569,15 @@ dependencies = [
 
 [[package]]
 name = "icu_properties_data"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
 
 [[package]]
 name = "icu_provider"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
 dependencies = [
  "displaydoc",
  "icu_locale_core",
@@ -4054,9 +4055,9 @@ checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00"
 
 [[package]]
 name = "tinystr"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
 dependencies = [
  "displaydoc",
  "zerovec",
@@ -4905,9 +4906,9 @@ checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a"
 
 [[package]]
 name = "yoke"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -4916,9 +4917,9 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4989,9 +4990,9 @@ dependencies = [
 
 [[package]]
 name = "zerotrie"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
 dependencies = [
  "displaydoc",
  "yoke",
@@ -5000,9 +5001,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
 dependencies = [
  "yoke",
  "zerofrom",
@@ -5011,9 +5012,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec-derive"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",

crates/vm/src/anystr.rs

@@ -4,6 +4,10 @@ use crate::{
     convert::TryFromBorrowedObject,
     function::OptionalOption,
 };
+use icu_properties::{
+    CodePointSetData,
+    props::{Alphabetic, ChangesWhenLowercased, ChangesWhenUppercased},
+};
 use num_traits::{cast::ToPrimitive, sign::Signed};
 
 use core::ops::Range;
@@ -130,8 +134,6 @@ where
 }
 
 pub trait AnyChar: Copy {
-    fn is_lowercase(self) -> bool;
-    fn is_uppercase(self) -> bool;
     fn bytes_len(self) -> usize;
 }
 
@@ -407,12 +409,16 @@ pub trait AnyStr {
     //  _Py_bytes_islower
     //  unicode_islower_impl
     fn py_islower(&self) -> bool {
+        let case_change = CodePointSetData::new::<ChangesWhenLowercased>();
+        let alphabetic = CodePointSetData::new::<Alphabetic>();
         let mut lower = false;
-        for c in self.elements() {
-            if c.is_uppercase() {
+        for chunk in self.as_bytes().utf8_chunks().map(|c| c.valid()) {
+            if chunk.chars().any(|c| case_change.contains(c)) {
                 return false;
-            } else if !lower && c.is_lowercase() {
-                lower = true
+            }
+
+            if !lower && chunk.chars().any(|c| alphabetic.contains(c)) {
+                lower = true;
             }
         }
         lower
@@ -422,12 +428,16 @@ pub trait AnyStr {
     //   Py_bytes_isupper
     //  unicode_isupper_impl
     fn py_isupper(&self) -> bool {
+        let case_change = CodePointSetData::new::<ChangesWhenUppercased>();
+        let alphabetic = CodePointSetData::new::<Alphabetic>();
         let mut upper = false;
-        for c in self.elements() {
-            if c.is_lowercase() {
+        for chunk in self.as_bytes().utf8_chunks().map(|c| c.valid()) {
+            if chunk.chars().any(|c| case_change.contains(c)) {
                 return false;
-            } else if !upper && c.is_uppercase() {
-                upper = true
+            }
+
+            if !upper && chunk.chars().any(|c| alphabetic.contains(c)) {
+                upper = true;
             }
         }
         upper

crates/vm/src/builtins/str.rs

@@ -2228,14 +2228,6 @@ impl AnyStrContainer<str> for String {
 }
 
 impl anystr::AnyChar for char {
-    fn is_lowercase(self) -> bool {
-        self.is_lowercase()
-    }
-
-    fn is_uppercase(self) -> bool {
-        self.is_uppercase()
-    }
-
     fn bytes_len(self) -> usize {
         self.len_utf8()
     }
@@ -2341,12 +2333,6 @@ impl AnyStrContainer<Wtf8> for Wtf8Buf {
 }
 
 impl anystr::AnyChar for CodePoint {
-    fn is_lowercase(self) -> bool {
-        self.is_char_and(char::is_lowercase)
-    }
-    fn is_uppercase(self) -> bool {
-        self.is_char_and(char::is_uppercase)
-    }
     fn bytes_len(self) -> usize {
         self.len_wtf8()
     }
@@ -2459,14 +2445,6 @@ impl AnyStrContainer<AsciiStr> for AsciiString {
 }
 
 impl anystr::AnyChar for ascii::AsciiChar {
-    fn is_lowercase(self) -> bool {
-        self.is_lowercase()
-    }
-
-    fn is_uppercase(self) -> bool {
-        self.is_uppercase()
-    }
-
     fn bytes_len(self) -> usize {
         1
     }

crates/vm/src/bytes_inner.rs

@@ -1031,14 +1031,6 @@ impl AnyStrContainer<[u8]> for Vec<u8> {
 const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b];
 
 impl anystr::AnyChar for u8 {
-    fn is_lowercase(self) -> bool {
-        self.is_ascii_lowercase()
-    }
-
-    fn is_uppercase(self) -> bool {
-        self.is_ascii_uppercase()
-    }
-
     fn bytes_len(self) -> usize {
         1
     }

extra_tests/snippets/builtin_str.py

@@ -69,6 +69,8 @@ assert not a.isdigit()
 assert not a.isdecimal()
 assert not a.isnumeric()
 assert a.istitle()
+assert "\u1c89".istitle()
+# assert "DZ".title() == "Dz"
 assert a.isalpha()
 
 s = "1 2 3"
@@ -220,6 +222,14 @@ assert (
 assert "abc\t12345\txyz".expandtabs() == "abc     12345   xyz"
 assert "-".join(["1", "2", "3"]) == "1-2-3"
 assert "HALLO".isupper()
+assert not "123".isupper()
+assert not "123".islower()
+assert not "\U0001f431".isupper()
+assert not "\U0001f431".islower()
+assert "\U0001f431 CAT".isupper()
+assert "\U0001f431 cat".islower()
+assert "\u0295".islower()
+assert "\u1c89".isupper()
 assert "hello, my name is".partition("my ") == ("hello, ", "my ", "name is")
 assert "hello".partition("is") == ("hello", "", "")
 assert "hello, my name is".rpartition("is") == ("hello, my name ", "is", "")
@@ -236,6 +246,8 @@ assert not "😂".isidentifier()
 assert not "123".isidentifier()
 
 assert "Σίσυφος".swapcase() == "σΊΣΥΦΟΣ"
+assert "\u0295".swapcase() == "\u0295"
+assert "\u1c89".swapcase() == "\u1c8a"
 
 # String Formatting
 assert "{} {}".format(1, 2) == "1 2"