From 51ba929fc630fb07a526b840c18418c47ca8ab9c Mon Sep 17 00:00:00 2001 From: Dimitris Fasarakis Hilliard Date: Sun, 29 Jan 2023 19:53:00 +0200 Subject: [PATCH 1/5] Match on ascii start/continuation characters before calling functions. --- compiler/parser/src/lexer.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 1c124f627..981115d82 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -534,12 +534,15 @@ where } fn is_identifier_start(&self, c: char) -> bool { - c == '_' || is_xid_start(c) + match c { + 'a'..='z' | 'A'..='Z' | '_' => true, + _ => is_xid_start(c), + } } fn is_identifier_continuation(&self) -> bool { match self.window[0] { - Some('_' | '0'..='9') => true, + Some('a'..='z' | 'A'..='Z' | '_' | '0'..='9') => true, Some(c) => is_xid_continue(c), _ => false, } From ecac89521b94268c5d9f8480ca5072491ac945c0 Mon Sep 17 00:00:00 2001 From: Dimitris Fasarakis Hilliard Date: Sun, 29 Jan 2023 20:08:21 +0200 Subject: [PATCH 2/5] Don't call is_emoji_presentation for each invocation of consume_normal --- compiler/parser/src/lexer.rs | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 981115d82..3cb282079 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -689,24 +689,12 @@ where fn consume_normal(&mut self) -> Result<(), LexicalError> { // Check if we have some character: if let Some(c) = self.window[0] { - // First check identifier: - if self.is_identifier_start(c) { - let identifier = self.lex_identifier()?; - self.emit(identifier); - } else if is_emoji_presentation(c) { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.emit(( - tok_start, - Tok::Name { - name: c.to_string(), - }, - tok_end, - )); - } else { - self.consume_character(c)?; - } + if self.is_identifier_start(c) { + let identifier = self.lex_identifier()?; + self.emit(identifier); + } else { + self.consume_character(c)?; + } } else { // We reached end of file. let tok_pos = self.get_pos(); @@ -1112,13 +1100,25 @@ where }); } } - _ => { - let c = self.next_char(); - return Err(LexicalError { - error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, - location: self.get_pos(), - }); + if is_emoji_presentation(c) { + let tok_start = self.get_pos(); + self.next_char(); + let tok_end = self.get_pos(); + self.emit(( + tok_start, + Tok::Name { + name: c.to_string(), + }, + tok_end, + )); + } else { + let c = self.next_char(); + return Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, + location: self.get_pos(), + }); + } } // Ignore all the rest.. } From 7edf3c86c34574e960d473c8a568048c9dae2273 Mon Sep 17 00:00:00 2001 From: Dimitris Fasarakis Hilliard Date: Sun, 29 Jan 2023 20:20:07 +0200 Subject: [PATCH 3/5] Add initial capacities, use u32s for indents/spaces. --- compiler/parser/src/lexer.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 3cb282079..653ef032c 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -18,8 +18,8 @@ use unic_ucd_ident::{is_xid_continue, is_xid_start}; #[derive(Clone, Copy, PartialEq, Debug, Default)] struct IndentationLevel { - tabs: usize, - spaces: usize, + tabs: u32, + spaces: u32, } impl IndentationLevel { @@ -225,7 +225,8 @@ where at_begin_of_line: true, nesting: 0, indentations: Indentations::default(), - pending: Vec::new(), + // Usually we have less than 5 tokens pending. + pending: Vec::with_capacity(5), location: start, window: CharWindow::new(input), }; @@ -257,13 +258,13 @@ where }; let start_pos = self.get_pos(); - let mut name = String::new(); + let mut name = String::with_capacity(8); while self.is_identifier_continuation() { name.push(self.next_char().unwrap()); } let end_pos = self.get_pos(); - if let Some(tok) = KEYWORDS.get(name.as_str()) { + if let Some(tok) = KEYWORDS.get(&name) { Ok((start_pos, tok.clone(), end_pos)) } else { Ok((start_pos, Tok::Name { name }, end_pos)) @@ -464,7 +465,7 @@ where self.next_char(); } let quote_char = self.next_char().unwrap(); - let mut string_content = String::new(); + let mut string_content = String::with_capacity(5); // If the next two characters are also the quote character, then we have a triple-quoted // string; consume those two characters and ensure that we require a triple-quote to close @@ -567,8 +568,8 @@ where /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character. fn eat_indentation(&mut self) -> Result { // Determine indentation: - let mut spaces: usize = 0; - let mut tabs: usize = 0; + let mut spaces: u32 = 0; + let mut tabs: u32 = 0; loop { match self.window[0] { Some(' ') => { @@ -1127,7 +1128,7 @@ where fn eat_single_char(&mut self, ty: Tok) { let tok_start = self.get_pos(); - self.next_char().unwrap(); + self.next_char(); let tok_end = self.get_pos(); self.emit((tok_start, ty, tok_end)); } From 0fcac14fd795dccf6b1c71ea608324b1f8195c7b Mon Sep 17 00:00:00 2001 From: Dimitris Fasarakis Hilliard Date: Mon, 30 Jan 2023 21:42:59 +0200 Subject: [PATCH 4/5] Eat for comma. --- compiler/parser/src/lexer.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 653ef032c..61bff3702 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -690,12 +690,12 @@ where fn consume_normal(&mut self) -> Result<(), LexicalError> { // Check if we have some character: if let Some(c) = self.window[0] { - if self.is_identifier_start(c) { - let identifier = self.lex_identifier()?; - self.emit(identifier); - } else { - self.consume_character(c)?; - } + if self.is_identifier_start(c) { + let identifier = self.lex_identifier()?; + self.emit(identifier); + } else { + self.consume_character(c)?; + } } else { // We reached end of file. let tok_pos = self.get_pos(); @@ -1039,10 +1039,7 @@ where } } ',' => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.emit((tok_start, Tok::Comma, tok_end)); + self.eat_single_char(Tok::Comma); } '.' => { if let Some('0'..='9') = self.window[1] { @@ -1102,7 +1099,7 @@ where } } _ => { - if is_emoji_presentation(c) { + if is_emoji_presentation(c) { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); From 5025113da0a035389f20fd35f2386bd7c13bfd6d Mon Sep 17 00:00:00 2001 From: Dimitris Fasarakis Hilliard Date: Wed, 1 Feb 2023 12:17:33 +0200 Subject: [PATCH 5/5] Hint that the unwrap should always succeed. --- compiler/parser/src/lexer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 61bff3702..5eeaf885d 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -1125,7 +1125,11 @@ where fn eat_single_char(&mut self, ty: Tok) { let tok_start = self.get_pos(); - self.next_char(); + self.next_char().unwrap_or_else(|| unsafe { + // SAFETY: eat_single_char has been called only after a character has been read + // from the window, so the window is guaranteed to be non-empty. + std::hint::unreachable_unchecked() + }); let tok_end = self.get_pos(); self.emit((tok_start, ty, tok_end)); }