From 98211a34dee7aa9eb837752f4a5ada20053e9440 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Thu, 26 Dec 2019 13:29:20 -0600 Subject: [PATCH] Try to cut down on the # of unicode crates we use --- Cargo.lock | 3 +-- parser/Cargo.toml | 4 ++-- parser/src/lexer.rs | 13 +++--------- vm/Cargo.toml | 3 ++- vm/src/obj/objstr.rs | 47 +++++++++++++++++++++----------------------- 5 files changed, 30 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57059c632..6b6278406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1286,7 +1286,7 @@ dependencies = [ "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-ident 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "wtf8 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1354,7 +1354,6 @@ dependencies = [ "unic 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-casing 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode_names2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-bindgen 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 047714c9d..0d6da2b66 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -17,6 +17,6 @@ log="0.4.1" regex = "1" num-bigint = "0.2" num-traits = "0.2" -unicode-xid = "0.2.0" -unic-emoji-char = "0.9.0" +unic-emoji-char = "0.9" +unic-ucd-ident = "0.9" wtf8 = "0.0.3" diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index a51bc3cf9..2a7011967 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -2,9 +2,6 @@ //! //! This means source code is translated into separate tokens. -extern crate unic_emoji_char; -extern crate unicode_xid; - pub use super::token::Tok; use crate::error::{LexicalError, LexicalErrorType}; use crate::location::Location; @@ -15,8 +12,7 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; use unic_emoji_char::is_emoji_presentation; -use unicode_xid::UnicodeXID; -use wtf8; +use unic_ucd_ident::{is_xid_continue, is_xid_start}; #[derive(Clone, Copy, PartialEq, Debug, Default)] struct IndentationLevel { @@ -658,17 +654,14 @@ where } fn is_identifier_start(&self, c: char) -> bool { - match c { - '_' => true, - c => UnicodeXID::is_xid_start(c), - } + c == '_' || is_xid_start(c) } fn is_identifier_continuation(&self) -> bool { if let Some(c) = self.chr0 { match c { '_' | '0'..='9' => true, - c => UnicodeXID::is_xid_continue(c), + c => is_xid_continue(c), } } else { false diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 62f87ca8e..068cf0f0e 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -46,7 +46,6 @@ rustc_version_runtime = "0.1.*" statrs = "0.12.0" caseless = "0.2.1" chrono = { version = "=0.4.9", features = ["wasmbind"] } -unicode-xid = "0.2.0" lazy_static = "^1.0.1" lexical = "4" itertools = "0.8" @@ -55,6 +54,8 @@ hexf-parse = "0.1.0" indexmap = "1.0.2" crc = "^1.0.0" unicode_names2 = "0.3" +# TODO: use unic for this; needed for title case: +# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 unicode-casing = "0.1" unic = "0.9" unic-common = "0.9" diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 0e0f3fb22..063e18152 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,6 +1,3 @@ -extern crate unicode_categories; -extern crate unicode_xid; - use std::cell::Cell; use std::char; use std::fmt; @@ -10,10 +7,10 @@ use std::str::FromStr; use std::string::ToString; use num_traits::ToPrimitive; +use unic::ucd::category::GeneralCategory; +use unic::ucd::ident::{is_xid_continue, is_xid_start}; use unic::ucd::is_cased; use unicode_casing::CharExt; -use unicode_categories::UnicodeCategories; -use unicode_xid::UnicodeXID; use super::objbytes::{PyBytes, PyBytesRef}; use super::objdict::PyDict; @@ -366,16 +363,7 @@ impl PyString { formatted.push_str(&format!("\\x{:02x}", c as u32)); } else if c.is_ascii() { formatted.push(c); - } else if c.is_other() || c.is_separator() { - // According to python following categories aren't printable: - // * Cc (Other, Control) - // * Cf (Other, Format) - // * Cs (Other, Surrogate) - // * Co (Other, Private Use) - // * Cn (Other, Not Assigned) - // * Zl Separator, Line ('\u2028', LINE SEPARATOR) - // * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) - // * Zs (Separator, Space) other than ASCII space('\x20'). + } else if !char_is_printable(c) { let code = c as u32; let escaped = if code < 0xff { format!("\\U{:02x}", code) @@ -742,10 +730,9 @@ impl PyString { /// * Zs (Separator, Space) other than ASCII space('\x20'). #[pymethod] fn isprintable(&self, _vm: &VirtualMachine) -> bool { - self.value.chars().all(|c| match c { - '\u{0020}' => true, - _ => !(c.is_other_control() | c.is_separator()), - }) + self.value + .chars() + .all(|c| c == '\u{0020}' || char_is_printable(c)) } // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty @@ -1094,13 +1081,9 @@ impl PyString { #[pymethod] fn isidentifier(&self, _vm: &VirtualMachine) -> bool { let mut chars = self.value.chars(); - let is_identifier_start = match chars.next() { - Some('_') => true, - Some(c) => UnicodeXID::is_xid_start(c), - None => false, - }; + let is_identifier_start = chars.next().map_or(false, |c| c == '_' || is_xid_start(c)); // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(UnicodeXID::is_xid_continue) + is_identifier_start && chars.all(is_xid_continue) } // https://docs.python.org/3/library/stdtypes.html#str.translate @@ -1706,6 +1689,20 @@ fn adjust_indices( } } +// According to python following categories aren't printable: +// * Cc (Other, Control) +// * Cf (Other, Format) +// * Cs (Other, Surrogate) +// * Co (Other, Private Use) +// * Cn (Other, Not Assigned) +// * Zl Separator, Line ('\u2028', LINE SEPARATOR) +// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) +// * Zs (Separator, Space) other than ASCII space('\x20'). +fn char_is_printable(c: char) -> bool { + let cat = GeneralCategory::of(c); + !(cat.is_other() || cat.is_separator()) +} + #[cfg(test)] mod tests { use super::*;