diff --git a/crates/unicode/build.rs b/crates/unicode/build.rs index 92a771baf..19561d817 100644 --- a/crates/unicode/build.rs +++ b/crates/unicode/build.rs @@ -2,14 +2,18 @@ extern crate alloc; -use core::num::NonZeroUsize; +use core::{ + fmt::{Debug, Display}, + iter::Iterator, + num::NonZeroUsize, +}; use alloc::collections::{BTreeMap, BTreeSet}; use std::{ env, fs::{self, File}, - io::{self, BufRead, BufReader, BufWriter, Write}, + io::{BufRead, BufReader, BufWriter, Lines, Write}, path::{Path, PathBuf}, thread, }; @@ -18,191 +22,320 @@ use icu_properties::props::{ BidiClass, EnumeratedProperty, GeneralCategory, NamedEnumeratedProperty, NumericType, }; -fn generate_unicode_3_2() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_3_2.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let base = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32"); - - write_derived( - &base, - "DerivedGeneralCategory-3.2.0.txt", - "GENERAL_CATEGORY", - "(u32, u32, GeneralCategory)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_general(id); - if id != GeneralCategory::Unassigned { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, GeneralCategory::{id:?}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedEastAsianWidth-3.2.0.txt", - "EAST_ASIAN_WIDTH", - "(u32, u32, EastAsianWidth)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_eaw(id); - if id != "EastAsianWidth::Neutral" { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedBidiClass-3.2.0.txt", - "BIDI_CLASS_DIFF", - "(u32, u32, BidiClass)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id = parse_bidi(id); - for i in start..=end { - let legacy = BidiClass::try_from_str(id.rsplit_once("::").unwrap().1) - .expect("Unicode data contains valid variants"); - let modern = char::from_u32(i).map(BidiClass::for_char); - - if Some(legacy) != modern { - return Some((start, end, id)); - } - } - - None - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - write!(writer, "];").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedBinaryProperties-3.2.0.txt", - "BIDI_MIRRORED", - "(u32, u32)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - assert_eq!( - "Bidi_Mirrored", - id.trim(), - "DerivedBinaryProperties-3.2.0 only has Bidi_Mirrored" - ); - Some((start, end)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _)| *start); - writeln!(writer, "{values:?};").unwrap(); - }, - ); - - write_derived( - &base, - "DerivedCombiningClass-3.2.0.txt", - "COMBINING_CLASS", - "(u32, u32, CanonicalCombiningClass)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, end, id, _| { - let id: u8 = id.parse().unwrap(); - if id == 0 { - return None; - } - Some((start, end, id)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!( - writer, - "({start}, {end}, CanonicalCombiningClass::from_icu4c_value({id}))," - ) - .unwrap(); - } - writeln!(writer, "];").unwrap(); - }, - ); +/// Iterator over Unicode data file lines. +struct UnicodeLineReader { + reader: Lines>, } -fn generate_numeric_type() { +impl UnicodeLineReader { + fn new(reader: BufReader) -> Self { + let reader = reader.lines(); + Self { reader } + } + + fn from_file_name(file_name: &str, modern: bool) -> Self { + Self::new(open_reader(file_name, modern)) + } + + fn next_line_raw(&mut self) -> Option> { + self.reader.find_map(|line| { + let line = line.unwrap(); + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + None + } else { + Some(line.to_owned().into_boxed_str()) + } + }) + } +} + +impl Iterator for UnicodeLineReader { + type Item = UnicodeLine; + + fn next(&mut self) -> Option { + let line = self.next_line_raw()?; + + let mut fields = line.split(';'); + let range = fields.next().expect("Unicode data is missing a char range"); + let (start, end) = match range.split_once("..") { + Some((left, right)) => { + let start = u32::from_str_radix(left.trim(), 16).unwrap(); + let end = u32::from_str_radix(right.trim(), 16).unwrap(); + (start, end) + } + None => { + let start = u32::from_str_radix(range.trim(), 16).unwrap(); + (start, start) + } + }; + + Some(UnicodeLine { start, end, line }) + } +} + +struct UnicodeLine { + start: u32, + end: u32, + line: Box, +} + +impl UnicodeLine { + /// Retrieve a field string from a line of Unicode data. + fn field(&self, n: NonZeroUsize, msg: Option<&str>) -> &str { + let field = + self.line.split(';').nth(n.get()).unwrap_or_else(|| { + panic!("{}", msg.unwrap_or("Unicode data is missing a property")) + }); + // The field may have a comment so strip that out + field.split_once('#').map_or(field, |(left, _)| left).trim() + } +} + +/// Helper to write a vector of condensed start, end char pairs and the associated ICU prop. +/// +/// The values are impl Display because Debug would break the representation. For example, strs +/// would be wrapped in quotation marks which is wrong. +fn write_slice_display( + writer: &mut impl Write, + static_name: &str, + array_type: &str, + values: &mut [(u32, u32, impl Display)], +) { + write_slice_pre(writer, static_name, array_type); + values.sort_unstable_by_key(|(start, _, _)| *start); + for (start, end, id) in values { + write!(writer, "({start}, {end}, {id}),").unwrap(); + } + write_slice_post(writer); +} + +fn write_slice_debug( + writer: &mut impl Write, + static_name: &str, + array_type: &str, + values: &mut [(u32, u32, impl Debug)], +) { + write_slice_pre(writer, static_name, array_type); + values.sort_unstable_by_key(|(start, _, _)| *start); + for (start, end, id) in values { + write!(writer, "({start}, {end}, {id:?}),").unwrap(); + } + write_slice_post(writer); +} + +/// Helper to write a vector of start, end char pairs. +/// +/// See: [`write_slice_display`]. +fn write_slice_pairs( + writer: &mut impl Write, + static_name: &str, + array_type: &str, + values: &mut [(u32, u32)], +) { + write_slice_pre(writer, static_name, array_type); + values.sort_unstable_by_key(|(start, _)| *start); + for (start, end) in values { + write!(writer, "({start}, {end}),").unwrap(); + } + write_slice_post(writer); +} + +/// Helper to write a pre-sorted slice of strs. +/// +/// See: [`write_slice_display`] +fn write_slice_unordered( + writer: &mut impl Write, + static_name: &str, + array_type: &str, + values: &mut [impl Display], +) { + write_slice_pre(writer, static_name, array_type); + for v in values { + write!(writer, "{v},").unwrap(); + } + write_slice_post(writer); +} + +fn write_slice_pre(writer: &mut impl Write, static_name: &str, array_type: &str) { + write!(writer, "static {static_name}: &[{array_type}] = &[").unwrap(); +} + +fn write_slice_post(writer: &mut impl Write) { + writeln!(writer, "];").unwrap(); +} + +fn open_writer(file_name: &str) -> BufWriter { let path = PathBuf::from(env::var("OUT_DIR").unwrap()) .join("generated") - .join("unicode_num_type.rs"); + .join(file_name); fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); + BufWriter::new(File::create(&path).unwrap()) +} - let base = Path::new(env!("CARGO_MANIFEST_DIR")) +fn open_reader(file_name: &str, modern: bool) -> BufReader { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) .join("unicode") - .join("ucd32"); + .join(if modern { "latest" } else { "ucd32" }) + .join(file_name); - write_derived( - &base, - "DerivedNumericType-3.2.0.txt", - "NUMERIC_TYPE_DIFF", - "(u32, u32, NumericType)", - NonZeroUsize::new(1).unwrap(), + BufReader::new(File::open(&path).unwrap_or_else(|e| { + panic!( + "{e}: vendored Unicode data file should exist: {}", + path.display() + ) + })) +} + +/// Drive parsers that require the latest Unicode data. +/// +/// The full data is huge so it's ideal to parse it only once. +fn full_data_parsers_latest() { + let reader = UnicodeLineReader::from_file_name("UnicodeData.txt", true); + + let mut decomp_lines = Vec::new(); + for line in reader { + let decomp_field = line.field( + NonZeroUsize::new(5).unwrap(), + Some(&format!( + "field 5 missing from UnicodeData.txt: {}", + line.line + )), + ); + if !decomp_field.is_empty() { + decomp_lines.push((line.start, decomp_field.to_owned())); + } + } + + generate_decomp(decomp_lines); +} + +fn generate_decomp(decomp_lines: Vec<(u32, String)>) { + let mut writer = open_writer("decomp.rs"); + + // NOTE: + // This ONLY parses compatibility decomposition because Python exposes the tags. The tags are + // the "", "", et cetera bits before the decomposition. Thus, we can save space + // by using icu4x's CanonicalDecomposer for non-compatibility decomposition. + let mut decomp_ranges = Vec::new(); + let mut values = Vec::new(); + for (start, value) in decomp_lines { + // We're building a sparse array. Most characters don't decompose, so we don't + // need to literally store a row for each char. + assert!( + !value.is_empty(), + "Decomp field shouldn't be empty at this point" + ); + + let Some((dtype, decomp)) = value.split_once('>').map(|(dtype, decomp)| { + let dtype = dtype + .strip_prefix('<') + .unwrap_or_else(|| panic!("Compatibility decomp; expected \n\tgot: {value}")); + ( + parse_decomp_type(dtype), + decomp + .split_whitespace() + .map(|s| u32::from_str_radix(s, 16).unwrap()), + ) + }) else { + continue; + }; + + decomp_ranges.extend(decomp); + let end = decomp_ranges.len(); + + let value = format!("({start}, DecompositionType::{dtype:?}, {end})"); + values.push(value); + } + + write_slice_unordered( &mut writer, - |start, end, id, _| { - let id = parse_numeric_type_str(id); - let differs = (start..=end).any(|c| match char::from_u32(c) { - Some(c) => { - let modern = parse_numeric_type_val(NumericType::for_char(c)); - modern != id - } - None => true, - }); - - if differs { - Some((start, end, id)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(start, _, _)| *start); - write!(writer, "[").unwrap(); - for (start, end, id) in values { - write!(writer, "({start}, {end}, {id}),").unwrap(); - } - writeln!(writer, "];").unwrap(); - }, + "DECOMP_COMPAT", + "(u32, DecompositionType, usize)", + &mut values, ); + write_slice_unordered(&mut writer, "DECOMP_RANGE", "u32", &mut decomp_ranges); + + // Normalization corrections is super small - only a handful chars at the time of writing. + let reader = UnicodeLineReader::from_file_name("NormalizationCorrections.txt", true); + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 2 missing from NormalizationCorrections: {}", + line.line + )), + ); + let original = u32::from_str_radix(field.trim(), 16).unwrap_or_else(|e| { + panic!("field 2 of decomp corrections should be a char in hex: {field} {e}") + }); + let version = line.field( + NonZeroUsize::new(3).unwrap(), + Some(&format!( + "field 4 of decomp corrections should be a UCD version: {}", + line.line + )), + ); + + // `version` = when the char was updated. Therefore, we use the incorrect chars past + // 3.2.0 but skip the chars fixed in 3.2.0 because they'll already be right. + if version != "3.2.0" { + values.push((line.start, original)) + } + } + write_slice_pairs(&mut writer, "DECOMP_UPDATES", "(u32, u32)", &mut values); +} + +/// Drive parsers that require the full 3.2.0 data. +/// +/// As the full data set is HUGE, it's more efficient to parse it once then delegate to subparsers. +fn full_data_parsers_3_2() { + let reader = UnicodeLineReader::from_file_name("UnicodeData-3.2.0.txt", false); + + // `DerivedNumericValues` writes the value rounded to a few digits, so the + // fraction `UnicodeData` field 8 spells out is what the value comes from. + // A character `UnicodeData` leaves out takes its value from Unihan, where + // the rounded field is a whole number and loses nothing. + let mut ucd32_fractions = BTreeMap::new(); + + // Parse membership from the full data. Unfortunately, this is largely uncompressed. + let mut membership_set = BTreeSet::new(); + let mut range_membership = Vec::new(); + + for line in reader { + // 3.2.0 membership + let name = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (char name) missing from UnicodeData-3.2.0.txt: {}", + line.line + )), + ); + if name.ends_with("First>") | name.ends_with("Last>") { + // Some lines (literally 20) are compressed ranges, so we have to handle those separately + range_membership.push(line.start); + } else { + membership_set.insert((line.start, line.end)); + } + + // Fractions + let numeric = line.field( + NonZeroUsize::new(8).unwrap(), + Some(&format!( + "field 8 (fraction) missing from UnicodeData-3.2.0.txt: {}", + line.line + )), + ); + if !numeric.is_empty() { + // start == end because the full data doesn't list code ranges. + ucd32_fractions.insert(line.start, parse_numeric_value(numeric)); + } + } + + // Now delegate to parsers that need the full data. + generate_membership_3_2(membership_set, range_membership); + generate_numeric_value(ucd32_fractions); } /// Generate a compressed array of Unicode 3.2 membership. @@ -211,34 +344,7 @@ fn generate_numeric_type() { /// default to the latest Unicode if a character exists in 3.2 but isn't different. Membership /// is needed because diffs aren't enough - a character may be absent in 3.2 which is different /// than returning a default. -fn generate_membership_3_2() { - let path = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32") - .join("UnicodeData-3.2.0.txt"); - let reader = BufReader::new(File::open(path).unwrap()); - - // Parse membership from the first data. Unfortunately, this is largely uncompressed. - let mut membership_set = BTreeSet::new(); - // TODO: Oh my, is this hacky and ugly... - let mut range_membership = Vec::new(); - parse_unicode_3_2( - reader, - NonZeroUsize::new(1).unwrap(), - &mut io::empty(), - |start, end, value, _| { - if value.ends_with("First>") | value.ends_with("Last>") { - // Some lines (literally 20) are compressed ranges, so we have to handle those separately - range_membership.push(start); - } else { - membership_set.insert((start, end)); - } - - Option::<()>::None - }, - |_writer, _values| {}, - ); - +fn generate_membership_3_2(membership_set: BTreeSet<(u32, u32)>, range_membership: Vec) { // Second pass. Compress the ranges. let mut iter = membership_set.iter(); let &(mut start_prev, mut end_prev) = iter.next().unwrap(); @@ -256,290 +362,274 @@ fn generate_membership_3_2() { membership.push((start_prev, end_prev)); let (chunks, &[]) = range_membership.as_chunks::<2>() else { - panic!("Range membership is always in pairs") + panic!("Range membership is always in pairs ('First>' and 'Last>'"); }; for &chunk in chunks { membership.push(chunk.into()); } - membership.sort_unstable_by_key(|&(start, _)| start); - - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("membership_3_2.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - writeln!(writer, "static MEMBERSHIP_3_2: &[(u32, u32)] = &").unwrap(); - write!(writer, "{membership:?};").unwrap(); + let mut writer = open_writer("membership_3_2.rs"); + write_slice_pairs(&mut writer, "MEMBERSHIP_3_2", "(u32, u32)", &mut membership); } -/// Read a numeric value written either on its own or as `numerator/denominator`. -fn parse_numeric_value(text: &str) -> f64 { - let text = text.trim(); - let (numerator, denominator) = text.split_once('/').unwrap_or((text, "1")); - let numerator: f64 = numerator - .trim() - .parse() - .expect("Unicode data contains valid properties"); - let denominator: f64 = denominator - .trim() - .parse() - .expect("Unicode data contains valid properties"); - numerator / denominator -} +fn generate_numeric_value(ucd32_fractions: BTreeMap) { + let mut ucd32_diffs = BTreeMap::new(); + let numeric_32 = UnicodeLineReader::from_file_name("DerivedNumericValues-3.2.0.txt", false); + for line in numeric_32 { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (numeric representation) missing from DerivedNumericValues-3.2.0.txt: {}", + line.line + )), + ); + let value = ucd32_fractions + .get(&line.start) + .copied() + .unwrap_or_else(|| parse_numeric_value(field)); -fn generate_numeric_value() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_numeric_value.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let ucd32 = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("ucd32"); - // `DerivedNumericValues` writes the value rounded to a few digits, so the - // fraction `UnicodeData` field 8 spells out is what the value comes from. - // A character `UnicodeData` leaves out takes its value from Unihan, where - // the rounded field is a whole number and loses nothing. - let mut ucd32_fractions = BTreeMap::new(); - for line in BufReader::new(File::open(ucd32.join("UnicodeData-3.2.0.txt")).unwrap()) - .lines() - .map(Result::unwrap) - { - let mut fields = line.split(';'); - let Some(code) = fields.next() else { continue }; - let Some(numeric) = fields.nth(7) else { - continue; - }; - if !numeric.is_empty() { - let code = u32::from_str_radix(code.trim(), 16).unwrap(); - ucd32_fractions.insert(code, parse_numeric_value(numeric)); - } + ucd32_diffs.insert((line.start, line.end), value); } - let mut ucd32_diffs = BTreeMap::new(); - let numeric_32 = - BufReader::new(File::open(ucd32.join("DerivedNumericValues-3.2.0.txt")).unwrap()); - parse_unicode_3_2( - numeric_32, - NonZeroUsize::new(1).unwrap(), - &mut io::empty(), - |start, end, value, _| { - let value = ucd32_fractions - .get(&start) - .copied() - .unwrap_or_else(|| parse_numeric_value(value)); - ucd32_diffs.insert((start, end), value); - Option::<()>::None - }, - |_writer, _values| {}, - ); + let ucd_latest = UnicodeLineReader::from_file_name("DerivedNumericValues.txt", true); + let mut values_latest = Vec::new(); + for line in ucd_latest { + // Field 3 holds the fraction; field 1 rounds it to a few digits. + let field = line.field( + NonZeroUsize::new(3).unwrap(), + Some(&format!( + "field 3 (fraction) missing from DerivedNumericValues.txt: {}", + line.line + )), + ); + let value = parse_numeric_value(field); - let ucd_latest = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("latest"); + if ucd32_diffs + .get(&(line.start, line.end)) + .is_some_and(|old_v| *old_v == value) + { + ucd32_diffs.remove(&(line.start, line.end)); + } - write_derived( - &ucd_latest, - "DerivedNumericValues.txt", + values_latest.push((line.start, line.end, value)); + } + + let mut writer = open_writer("numeric_value_3_2.rs"); + write_slice_debug( + &mut writer, "NUMERIC_VALUES", "(u32, u32, f64)", - // Field 3 holds the fraction; field 1 rounds it to a few digits. - NonZeroUsize::new(3).unwrap(), - &mut writer, - |start, end, value, _| { - let value = parse_numeric_value(value); - - if ucd32_diffs - .get(&(start, end)) - .is_some_and(|old_v| *old_v == value) - { - ucd32_diffs.remove(&(start, end)); - } - - Some((start, end, value)) - }, - |writer, mut values| { - values.sort_unstable_by_key(|(ch, _, _)| *ch); - writeln!(writer, "{values:?};").unwrap(); - }, + &mut values_latest, ); - - // TODO: More flexible parser - writeln!( - writer, - "static NUMERIC_VALUES_DIFF: &[(u32, u32, f64)] = &[" - ) - .unwrap(); - for ((start, end), value) in ucd32_diffs { - write!(writer, "({start}, {end}, {value:?}),").unwrap(); - } - writeln!(writer, "];").unwrap(); -} - -fn generate_unicode_latest() { - let path = PathBuf::from(env::var("OUT_DIR").unwrap()) - .join("generated") - .join("unicode_latest.rs"); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut writer = BufWriter::new(File::create(&path).unwrap()); - - let base = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("unicode") - .join("latest"); - - // NOTE: - // This ONLY parses compatibility decomposition because Python exposes the tags. The tags are - // the "", "", et cetera bits before the decomposition. Thus, we can save space - // by using icu4x's CanonicalDecomposer for non-compatibility decomposition. - let mut decomp_ranges = Vec::new(); - write_derived( - &base, - "UnicodeData.txt", - "DECOMP_COMPAT", - "(u32, DecompositionType, usize)", - NonZeroUsize::new(5).unwrap(), + let mut ucd32_diffs: Vec<_> = ucd32_diffs + .into_iter() + .map(|((start, end), value)| (start, end, value)) + .collect(); + write_slice_debug( &mut writer, - |start, _end, value, _| { - // We're building a sparse array. Most characters don't decompose, so we don't - // need to literally store a row for each char. - if value.is_empty() { - return None; - } - - let (dtype, decomp) = value.split_once('>').map(|(dtype, decomp)| { - let dtype = dtype.strip_prefix('<').unwrap_or_else(|| { - panic!("Compatibility decomp; expected \n\tgot: {value}") - }); - ( - parse_decomp_type(dtype), - decomp - .split_whitespace() - .map(|s| u32::from_str_radix(s, 16).unwrap()), - ) - })?; - - decomp_ranges.extend(decomp); - let end = decomp_ranges.len(); - - Some((start, dtype, end)) - }, - |writer, values| { - // UnicodeData.txt should already be sorted - write!(writer, "[").unwrap(); - for (start, dtype, end) in values { - write!(writer, "({start}, DecompositionType::{dtype:?}, {end}),").unwrap(); - } - writeln!(writer, "];").unwrap(); - }, - ); - - writeln!(writer, "static DECOMP_RANGE: &[u32] = &{decomp_ranges:?};").unwrap(); - - // Normalization corrections is super small - only a handful chars at the time of writing. - write_derived( - &base, - "NormalizationCorrections.txt", - "DECOMP_UPDATES", - "(u32, u32)", - NonZeroUsize::new(1).unwrap(), - &mut writer, - |start, _end, value, line| { - let original = u32::from_str_radix(value.trim(), 16).unwrap_or_else(|e| { - panic!("field 2 of decomp corrections should be a char in hex: {value} {e}") - }); - let version = line - .rsplit(';') - .next() - .unwrap_or_else(|| { - panic!("field 4 of decomp corrections should be a UCD version: {line}") - }) - .split_once('#') - .unwrap() - .0 - .trim(); - - // `version` = when the char was updated. Therefore, we use the incorrect chars past - // 3.2.0 but skip the chars fixed in 3.2.0 because they'll already be right. - if version != "3.2.0" { - Some((start, original)) - } else { - None - } - }, - |writer, mut values| { - values.sort_unstable_by_key(|(c, _)| *c); - write!(writer, "{values:?};").unwrap(); - }, + "NUMERIC_VALUES_DIFF", + "(u32, u32, f64)", + &mut ucd32_diffs, ); } -#[expect(clippy::too_many_arguments)] -fn write_derived( - base: &Path, - file_name: &str, - static_name: &str, - array_type: &str, - field: NonZeroUsize, - writer: &mut W, - parse: P, - write_vec: FW, -) where - W: Write, - P: FnMut(u32, u32, &str, &str) -> Option, - FW: FnMut(&mut W, Vec), -{ - let path = base.join(file_name); - let reader = BufReader::new(File::open(path).unwrap()); - writeln!(writer, "static {static_name}: &[{array_type}] = &").unwrap(); - parse_unicode_3_2(reader, field, writer, parse, write_vec); -} +fn general_category_3_2() { + let mut writer = open_writer("gen_cat_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedGeneralCategory-3.2.0.txt", false); -/// Parse Unicode 3.2.0 property files. -fn parse_unicode_3_2( - reader: impl BufRead, - field: NonZeroUsize, - writer: &mut W, - mut parse: P, - mut write_vec: FW, -) where - W: Write, - P: FnMut(u32, u32, &str, &str) -> Option, - FW: FnMut(&mut W, Vec), -{ - let mut parsed = Vec::new(); - - for line in reader.lines().map(Result::unwrap) { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - - let mut fields = line.split(';'); - let range = fields.next().expect("Unicode data is missing a char range"); - let id = fields - .nth(field.get().saturating_sub(1)) - .expect("Unicode data is missing a property"); - let (start, end) = match range.split_once("..") { - Some((left, right)) => { - let start = u32::from_str_radix(left.trim(), 16).unwrap(); - let end = u32::from_str_radix(right.trim(), 16).unwrap(); - (start, end) - } - None => { - let start = u32::from_str_radix(range.trim(), 16).unwrap(); - (start, start) - } - }; - - let id = id.split_once('#').map_or(id, |(left, _)| left).trim(); - if let Some(val) = parse(start, end, id, line) { - parsed.push(val); + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (category) missing from DerivedGeneralCategory-3.2.0.txt: {}", + line.line + )), + ); + let id = parse_general(field); + if id != GeneralCategory::Unassigned { + let value = format!("GeneralCategory::{id:?}"); + values.push((line.start, line.end, value)); } } - write_vec(writer, parsed); + + write_slice_display( + &mut writer, + "GENERAL_CATEGORY", + "(u32, u32, GeneralCategory)", + &mut values, + ); +} + +fn east_asian_width_3_2() { + let mut writer = open_writer("eaw_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedEastAsianWidth-3.2.0.txt", false); + + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (east Asian width class) missing from DerivedEastAsianWidth-3.2.0.txt: {}", + line.line + )), + ); + let id = parse_eaw(field); + if id != "EastAsianWidth::Neutral" { + values.push((line.start, line.end, id)); + } + } + + write_slice_display( + &mut writer, + "EAST_ASIAN_WIDTH", + "(u32, u32, EastAsianWidth)", + &mut values, + ); +} + +fn bidi_class_3_2() { + let mut writer = open_writer("bidi_class_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedBidiClass-3.2.0.txt", false); + + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (BIDI class) missing from DerivedBidiClass-3.2.0.txt: {}", + line.line + )), + ); + let id = parse_bidi(field); + for i in line.start..=line.end { + let legacy = BidiClass::try_from_str(id.rsplit_once("::").unwrap().1) + .expect("Unicode data contains valid variants"); + let modern = char::from_u32(i).map(BidiClass::for_char); + + if Some(legacy) != modern { + values.push((line.start, line.end, id)); + break; + } + } + } + + write_slice_display( + &mut writer, + "BIDI_CLASS_DIFF", + "(u32, u32, BidiClass)", + &mut values, + ); +} + +fn binary_props_3_2() { + let mut writer = open_writer("binary_props_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedBinaryProperties-3.2.0.txt", false); + + let mut values = Vec::new(); + for line in reader { + let id = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (binary property) missing from DerivedBinaryProperties-3.2.0.txt: {}", + line.line + )), + ); + assert_eq!( + "Bidi_Mirrored", + id.trim(), + "DerivedBinaryProperties-3.2.0 only has Bidi_Mirrored" + ); + values.push((line.start, line.end)); + } + + write_slice_pairs(&mut writer, "BIDI_MIRRORED", "(u32, u32)", &mut values); +} + +fn combining_class_3_2() { + let mut writer = open_writer("combining_class_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedCombiningClass-3.2.0.txt", false); + + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (combining class) missing from DerivedCombiningClass-3.2.0.txt: {}", + line.line + )), + ); + let id: u8 = field.parse().unwrap(); + if id != 0 { + let value = format!("CanonicalCombiningClass::from_icu4c_value({id})"); + values.push((line.start, line.end, value)); + } + } + + write_slice_display( + &mut writer, + "COMBINING_CLASS", + "(u32, u32, CanonicalCombiningClass)", + &mut values, + ); +} + +fn generate_numeric_type_3_2() { + let mut writer = open_writer("num_type_3_2.rs"); + let reader = UnicodeLineReader::from_file_name("DerivedNumericType-3.2.0.txt", false); + + let mut values = Vec::new(); + for line in reader { + let field = line.field( + NonZeroUsize::new(1).unwrap(), + Some(&format!( + "field 1 (numeric type) missing from DerivedNumericType-3.2.0.txt: {}", + line.line + )), + ); + let id = parse_numeric_type_str(field); + let differs = (line.start..=line.end).any(|c| match char::from_u32(c) { + Some(c) => { + let modern = parse_numeric_type_val(NumericType::for_char(c)); + modern != id + } + None => true, + }); + + if differs { + values.push((line.start, line.end, id)); + } + } + + write_slice_display( + &mut writer, + "NUMERIC_TYPE_DIFF", + "(u32, u32, NumericType)", + &mut values, + ); +} + +/// Run each UCD parser in parallel. +fn drive_parsers() { + let parsers = [ + full_data_parsers_latest, + full_data_parsers_3_2, + general_category_3_2, + east_asian_width_3_2, + bidi_class_3_2, + binary_props_3_2, + combining_class_3_2, + generate_numeric_type_3_2, + ]; + + let mut handles = Vec::with_capacity(parsers.len()); + for parser in parsers { + handles.push(thread::spawn(parser)); + } + + for handle in handles { + handle.join().unwrap(); + } } fn parse_general(id: &str) -> GeneralCategory { @@ -684,18 +774,24 @@ fn parse_decomp_type(id: &str) -> DecompositionType { } } +/// Read a numeric value written either on its own or as `numerator/denominator`. +fn parse_numeric_value(text: &str) -> f64 { + let text = text.trim(); + let (numerator, denominator) = text.split_once('/').unwrap_or((text, "1")); + let numerator: f64 = numerator + .trim() + .parse() + .expect("Unicode data contains valid properties"); + let denominator: f64 = denominator + .trim() + .parse() + .expect("Unicode data contains valid properties"); + numerator / denominator +} + fn main() { println!("cargo:rerun-if-changed=unicode/ucd32"); println!("cargo:rerun-if-changed=unicode/latest"); - let t_32 = thread::spawn(generate_unicode_3_2); - let t_32_membership = thread::spawn(generate_membership_3_2); - let t_numeric_type = thread::spawn(generate_numeric_type); - let t_numeric_value = thread::spawn(generate_numeric_value); - let t_latest = thread::spawn(generate_unicode_latest); - t_32.join().unwrap(); - t_32_membership.join().unwrap(); - t_numeric_type.join().unwrap(); - t_numeric_value.join().unwrap(); - t_latest.join().unwrap(); + drive_parsers(); } diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index 15e76bb17..73102ada4 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -19,14 +19,18 @@ use icu_properties::props::{ }; use rustpython_wtf8::CodePoint; -include!(concat!(env!("OUT_DIR"), "/generated/membership_3_2.rs")); -include!(concat!(env!("OUT_DIR"), "/generated/unicode_3_2.rs")); -include!(concat!(env!("OUT_DIR"), "/generated/unicode_latest.rs")); -include!(concat!(env!("OUT_DIR"), "/generated/unicode_num_type.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/bidi_class_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/binary_props_3_2.rs")); include!(concat!( env!("OUT_DIR"), - "/generated/unicode_numeric_value.rs" + "/generated/combining_class_3_2.rs" )); +include!(concat!(env!("OUT_DIR"), "/generated/decomp.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/eaw_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/gen_cat_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/membership_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/numeric_value_3_2.rs")); +include!(concat!(env!("OUT_DIR"), "/generated/num_type_3_2.rs")); #[derive(Clone, Copy)] enum DecompositionType {