mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-29 03:04:38 +08:00
Code Activity

Add support for emoji as name.

Browse Source
This commit is contained in:
Windel Bouwman committed 2019-03-24 20:16:38 +01:00
1 parent d0c4fcb2fa
commit ca11e7dea7
3 files changed
+60 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+42
View File
@@ -760,6 +760,7 @@ dependencies = [
"num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -1044,6 +1045,42 @@ name = "ucd-util"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unic-char-property"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unic-char-range"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unic-common"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unic-emoji-char"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unic-ucd-version"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-normalization"
version = "0.1.7"
@@ -1364,6 +1401,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
"checksum unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
"checksum unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
"checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
"checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
"checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"
"checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1"
"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
parser/Cargo.toml
+1
View File
@@ -15,3 +15,4 @@ regex="0.2.2"
num-bigint = "0.2"
num-traits = "0.2"
unicode-xid = "0.1.0"
unic-emoji-char = "0.9.0"
parser/src/lexer.rs
+17 -7
View File
@@ -1,6 +1,7 @@
//! This module takes care of lexing python source text. This means source
//! code is translated into separate tokens.
extern crate unic_emoji_char;
extern crate unicode_xid;
pub use super::token::Tok;
@@ -9,6 +10,7 @@ use num_traits::Num;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::str::FromStr;
use unic_emoji_char::is_emoji_presentation;
use unicode_xid::UnicodeXID;
#[derive(Clone, Copy, PartialEq, Debug)]
@@ -545,7 +547,7 @@ where
fn is_identifier_start(&self, c: char) -> bool {
match c {
'a'..='z' | 'A'..='Z' | '_' => true,
'_' => true,
c => UnicodeXID::is_xid_start(c),
}
}
@@ -553,7 +555,7 @@ where
fn is_identifier_continuation(&self) -> bool {
if let Some(c) = self.chr0 {
match c {
'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => true,
'_' | '0'..='9' => true,
c => UnicodeXID::is_xid_continue(c),
}
} else {
@@ -705,6 +707,18 @@ where
// First check identifier:
if self.is_identifier_start(c) {
return Some(self.lex_identifier());
} else if is_emoji_presentation(c) {
let tok_start = self.get_pos();
self.next_char();
let tok_end = self.get_pos();
println!("Emoji: {}", c);
return Some(Ok((
tok_start,
Tok::Name {
name: c.to_string(),
},
tok_end,
)));
} else {
match c {
'0'..='9' => return Some(self.lex_number()),
@@ -801,11 +815,7 @@ where
}
_ => {
let tok_end = self.get_pos();
return Some(Ok((
tok_start,
Tok::DoubleSlash,
tok_end,
)));
return Some(Ok((tok_start, Tok::DoubleSlash, tok_end)));
}
}
}