From ca11e7dea79ed413138ee85acdd06b1070801ae6 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 24 Mar 2019 20:16:38 +0100 Subject: [PATCH] Add support for emoji as name. --- Cargo.lock | 42 ++++++++++++++++++++++++++++++++++++++++++ parser/Cargo.toml | 1 + parser/src/lexer.rs | 24 +++++++++++++++++------- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57dadb57b..6193772c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,6 +760,7 @@ dependencies = [ "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1044,6 +1045,42 @@ name = "ucd-util" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-emoji-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicode-normalization" version = "0.1.7" @@ -1364,6 +1401,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +"checksum unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" +"checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" +"checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" +"checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index 7545a27d4..f2f8e5a6b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -15,3 +15,4 @@ regex="0.2.2" num-bigint = "0.2" num-traits = "0.2" unicode-xid = "0.1.0" +unic-emoji-char = "0.9.0" diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index db8fee2c7..defa5e232 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1,6 +1,7 @@ //! This module takes care of lexing python source text. This means source //! code is translated into separate tokens. +extern crate unic_emoji_char; extern crate unicode_xid; pub use super::token::Tok; @@ -9,6 +10,7 @@ use num_traits::Num; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; +use unic_emoji_char::is_emoji_presentation; use unicode_xid::UnicodeXID; #[derive(Clone, Copy, PartialEq, Debug)] @@ -545,7 +547,7 @@ where fn is_identifier_start(&self, c: char) -> bool { match c { - 'a'..='z' | 'A'..='Z' | '_' => true, + '_' => true, c => UnicodeXID::is_xid_start(c), } } @@ -553,7 +555,7 @@ where fn is_identifier_continuation(&self) -> bool { if let Some(c) = self.chr0 { match c { - 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => true, + '_' | '0'..='9' => true, c => UnicodeXID::is_xid_continue(c), } } else { @@ -705,6 +707,18 @@ where // First check identifier: if self.is_identifier_start(c) { return Some(self.lex_identifier()); + } else if is_emoji_presentation(c) { + let tok_start = self.get_pos(); + self.next_char(); + let tok_end = self.get_pos(); + println!("Emoji: {}", c); + return Some(Ok(( + tok_start, + Tok::Name { + name: c.to_string(), + }, + tok_end, + ))); } else { match c { '0'..='9' => return Some(self.lex_number()), @@ -801,11 +815,7 @@ where } _ => { let tok_end = self.get_pos(); - return Some(Ok(( - tok_start, - Tok::DoubleSlash, - tok_end, - ))); + return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); } } }