unicodedata: Fix name, unmask test_name (#8599)

This commit fixes most of UCD's names. The two main fixes are:

* Handling chars missing in 3.2.0 by checking for membership #8548
* Handling names not present in unicodenames

AI disclosure: I used AI for the initial research of this patch to
determine why the Tangut characters are missing. AI also assisted with a
bug fix where I neglected to parse the Tangut u32 as hex. I wrote all of
the code.

Assisted-by: GPT 5.6 Luna

Lib/test/test_unicodedata.py

@@ -131,7 +131,6 @@ class BaseUnicodeFunctionsTest:
         result = h.hexdigest()
         self.assertEqual(result, self.expectedchecksum)
 
-    @unittest.expectedFailure   # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
     def test_name(self):
         name = self.db.name
         self.assertRaises(ValueError, name, '\0')
@@ -732,7 +731,6 @@ class UnicodeFunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
                         'ebfc9dd281c2226998fd435744dd2e9321899beb')
 
     @requires_resource('network')
-    @unittest.expectedFailure   # TODO: RUSTPYTHON; AssertionError: None != 'TANGUT IDEOGRAPH-17000'
     def test_all_names(self):
         TESTDATAFILE = "DerivedName.txt"
         testdata = download_test_data_file(TESTDATAFILE)
@@ -1141,10 +1139,6 @@ class Unicode_3_2_0_FunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
     def test_normalization(self):
         return super().test_normalization()
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; AssertionError: 'LATIN SMALL LETTER D WITH CURL' != None
-    def test_name(self):
-        return super().test_name()
-
     @unittest.expectedSuccess  # TODO: RUSTPYTHON
     def test_combining(self):
         return super().test_combining()

crates/stdlib/src/unicodedata.rs

@@ -102,6 +102,7 @@ mod unicodedata {
         fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> {
             if let Some(name_str) = name.to_str()
                 && let Some(character) = unicode_core::lookup_character(name_str)
+                && self.inner.membership(character)
             {
                 return Ok(character.to_string());
             }
@@ -119,11 +120,12 @@ mod unicodedata {
             default: OptionalArg<PyObjectRef>,
             vm: &VirtualMachine,
         ) -> PyResult {
-            if let Some(name) = self
-                .extract_char(character, vm)?
-                .to_char()
-                .and_then(unicode_core::character_name)
-            {
+            if let Some(name) = self.extract_char(character, vm)?.to_char().and_then(|ch| {
+                self.inner
+                    .membership(ch)
+                    .then(|| unicode_core::character_name(ch))
+                    .flatten()
+            }) {
                 return Ok(vm.ctx.new_str(name).into());
             }
             default.ok_or_else(|| vm.new_value_error("no such name"))

crates/unicode/Cargo.toml

@@ -20,6 +20,7 @@ writeable = { workspace = true }
 
 [build-dependencies]
 icu_properties = { workspace = true }
+unicode_names2 = { workspace = true }
 
 [lints]
 workspace = true

crates/unicode/build.rs

@@ -194,6 +194,7 @@ fn full_data_parsers_latest() {
     let reader = UnicodeLineReader::from_file_name("UnicodeData.txt", true);
 
     let mut decomp_lines = Vec::new();
+    let mut algo_names = Vec::new();
     for line in reader {
         let decomp_field = line.field(
             NonZeroUsize::new(5).unwrap(),
@@ -205,9 +206,21 @@ fn full_data_parsers_latest() {
         if !decomp_field.is_empty() {
             decomp_lines.push((line.start, decomp_field.to_owned()));
         }
+
+        let name_field = line.field(
+            NonZeroUsize::new(1).unwrap(),
+            Some(&format!(
+                "field 1 (name) missing from UnicodeData.txt: {}",
+                line.line
+            )),
+        );
+        if name_field.ends_with("First>") || name_field.ends_with("Last>") {
+            algo_names.push((line.start, name_field.into()));
+        }
     }
 
     generate_decomp(decomp_lines);
+    generate_algo_names(algo_names);
 }
 
 fn generate_decomp(decomp_lines: Vec<(u32, String)>) {
@@ -287,6 +300,46 @@ fn generate_decomp(decomp_lines: Vec<(u32, String)>) {
     write_slice_pairs(&mut writer, "DECOMP_UPDATES", "(u32, u32)", &mut values);
 }
 
+fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
+    const TANGUT: &str = "<Tangut Ideograph";
+    const TANGUT_SUP: &str = "<Tangut Ideograph Supplement";
+
+    let (names, []) = names.as_chunks::<2>() else {
+        panic!("Uneven algorithmic names: {names:#?}");
+    };
+
+    let mut values = Vec::new();
+    for &[(start, ref raw_start), (end, ref raw_end)] in names {
+        // Surrogate and private use ranges are algorithmic names.
+        if (0xD800..=0xDFFF).contains(&start)
+            || (0xE000..=0xF8FF).contains(&start)
+            || (0xF0000..=0xFFFFD).contains(&start)
+            || (0x100000..=0x10FFFD).contains(&start)
+        {
+            continue;
+        };
+        // Skip existing as there are only a few unimplemented algorithmic names.
+        if unicode_names2::name(char::from_u32(start).unwrap()).is_none() {
+            match (raw_start.split_once(','), raw_end.split_once(',')) {
+                (Some((TANGUT, _)), Some((TANGUT, _)))
+                | (Some((TANGUT_SUP, _)), Some((TANGUT_SUP, _))) => {
+                    values.push((start, end, "AlgorithmicName::TangutIdeograph"));
+                }
+                (None, None) => panic!("Unexpected name parsed:\n{raw_start}\n{raw_end}"),
+                _ => panic!("Unhandled algorithmic name:\n{raw_start}\n{raw_end}"),
+            }
+        }
+    }
+
+    let mut writer = open_writer("algo_names.rs");
+    write_slice_display(
+        &mut writer,
+        "ALGO_NAMES",
+        "(u32, u32, AlgorithmicName)",
+        &mut values,
+    );
+}
+
 /// Drive parsers that require the full 3.2.0 data.
 ///
 /// As the full data set is HUGE, it's more efficient to parse it once then delegate to subparsers.

crates/unicode/src/data.rs

@@ -19,6 +19,7 @@ use icu_properties::props::{
 };
 use rustpython_wtf8::CodePoint;
 
+include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
 include!(concat!(env!("OUT_DIR"), "/generated/bidi_class_3_2.rs"));
 include!(concat!(env!("OUT_DIR"), "/generated/binary_props_3_2.rs"));
 include!(concat!(
@@ -75,6 +76,37 @@ impl DecompositionType {
     }
 }
 
+#[derive(Clone, Copy, PartialEq)]
+enum AlgorithmicName {
+    TangutIdeograph,
+}
+
+impl AlgorithmicName {
+    const fn name_base(self) -> &'static str {
+        match self {
+            Self::TangutIdeograph => "TANGUT IDEOGRAPH",
+        }
+    }
+
+    fn to_name(self, ch: char) -> String {
+        match self {
+            Self::TangutIdeograph => format!("{}-{:0X}", self.name_base(), ch as u32),
+        }
+    }
+
+    fn check_name(search_name: &str) -> Option<char> {
+        if let Some(without_base) = search_name.strip_prefix(Self::TangutIdeograph.name_base()) {
+            let cp = u32::from_str_radix(without_base.strip_prefix('-')?.trim(), 16).ok()?;
+            let ch = char::from_u32(cp)?;
+            if lookup_table(ALGO_NAMES, ch)? == Self::TangutIdeograph {
+                return Some(ch);
+            }
+        }
+
+        None
+    }
+}
+
 fn lookup_table<T: Copy>(table: &[(u32, u32, T)], ch: char) -> Option<T> {
     let ch = ch as u32;
     table
@@ -91,7 +123,10 @@ fn lookup_table<T: Copy>(table: &[(u32, u32, T)], ch: char) -> Option<T> {
         .map(|i| table[i].2)
 }
 
-fn membership_3_2(ch: u32) -> bool {
+#[cold]
+#[inline(never)]
+#[must_use]
+pub fn membership_3_2(ch: u32) -> bool {
     MEMBERSHIP_3_2
         .binary_search_by(|&(start, end)| {
             if ch > end {
@@ -136,12 +171,17 @@ pub fn unicode_version() -> String {
 }
 
 /// Look up a character by its Unicode name (`unicodedata.lookup`).
-pub use unicode_names2::character as lookup_character;
+#[must_use]
+pub fn lookup_character(search_name: &str) -> Option<char> {
+    unicode_names2::character(search_name).or_else(|| AlgorithmicName::check_name(search_name))
+}
 
 /// The Unicode name of `ch` (`unicodedata.name`), if any.
 #[must_use]
 pub fn character_name(ch: char) -> Option<String> {
-    unicode_names2::name(ch).map(|name| name.to_string())
+    unicode_names2::name(ch)
+        .map(|name| name.to_string())
+        .or_else(|| lookup_table(ALGO_NAMES, ch).map(|v| v.to_name(ch)))
 }
 
 /// A view over the Unicode character database at a fixed version.
@@ -159,6 +199,17 @@ impl Ucd {
         Self { modern }
     }
 
+    #[must_use]
+    pub fn membership(self, ch: char) -> bool {
+        if !self.modern {
+            cold_path();
+            membership_3_2(ch as u32)
+        } else {
+            // Rust chars always "exist" for modern Unicode or else they wouldn't be chars.
+            true
+        }
+    }
+
     #[must_use]
     pub fn category(&self, c: CodePoint) -> &'static str {
         let Some(c) = c.to_char() else {