From 8c199ee2f2d880bb881848f104909d812171794c Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 21 Jul 2019 13:05:10 +0200
Subject: [PATCH 1/2] Refactor the lexer code to bring some more sanity and
 reduce function size.

---
 parser/src/error.rs  |  13 +-
 parser/src/lexer.rs  | 979 ++++++++++++++++++++++---------------------
 parser/src/parser.rs |   2 +-
 3 files changed, 515 insertions(+), 479 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 129692d2d..f0d4c8bcd 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -1,6 +1,5 @@
 //! Define internal parse error types
 //! The goal is to provide a matching and a safe error API, maksing errors from LALR
-use lalrpop_util::ParseError as InnerError;
 use lalrpop_util::ParseError as LalrpopError;
 
 use crate::location::Location;
@@ -106,23 +105,23 @@ pub enum ParseErrorType {
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
-    fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
+impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
+    fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
         match err {
             // TODO: Are there cases where this isn't an EOF?
-            InnerError::InvalidToken { location } => ParseError {
+            LalrpopError::InvalidToken { location } => ParseError {
                 error: ParseErrorType::EOF,
                 location,
             },
-            InnerError::ExtraToken { token } => ParseError {
+            LalrpopError::ExtraToken { token } => ParseError {
                 error: ParseErrorType::ExtraToken(token.1),
                 location: token.0,
             },
-            InnerError::User { error } => ParseError {
+            LalrpopError::User { error } => ParseError {
                 error: ParseErrorType::Lexical(error.error),
                 location: error.location,
             },
-            InnerError::UnrecognizedToken { token, expected } => {
+            LalrpopError::UnrecognizedToken { token, expected } => {
                 match token {
                     Some(tok) => ParseError {
                         error: ParseErrorType::UnrecognizedToken(tok.1, expected),
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index d59bec99b..f9e622e1a 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -53,7 +53,7 @@ pub struct Lexer<T: Iterator<Item = char>> {
     at_begin_of_line: bool,
     nesting: usize, // Amount of parenthesis
     indentation_stack: Vec<IndentationLevel>,
-    pending: Vec<LexResult>,
+    pending: Vec<Spanned>,
     chr0: Option<char>,
     chr1: Option<char>,
     location: Location,
@@ -297,6 +297,7 @@ where
         }
     }
 
+    /// Numeric lexing. The feast can start!
     fn lex_number(&mut self) -> LexResult {
         let start_pos = self.get_pos();
         if self.chr0 == Some('0') {
@@ -323,6 +324,7 @@ where
         }
     }
 
+    /// Lex a hex/octal/decimal/binary number without a decimal point.
     fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult {
         let mut value_text = String::new();
 
@@ -410,8 +412,8 @@ where
         }
     }
 
+    /// Skip everything until end of line
     fn lex_comment(&mut self) {
-        // Skip everything until end of line
         self.next_char();
         loop {
             match self.chr0 {
@@ -534,14 +536,11 @@ where
                             break;
                         }
                     } else {
-                        if c == '\n' {
-                            if !triple_quoted {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::StringError,
-                                    location: self.get_pos(),
-                                });
-                            }
-                            self.new_line();
+                        if c == '\n' && !triple_quoted {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::StringError,
+                                location: self.get_pos(),
+                            });
                         }
                         string_content.push(c);
                     }
@@ -626,25 +625,8 @@ where
         }
     }
 
-    fn next_char(&mut self) -> Option<char> {
-        let c = self.chr0;
-        let nxt = self.chars.next();
-        self.chr0 = self.chr1;
-        self.chr1 = nxt;
-        self.location.go_right();
-        c
-    }
-
-    fn get_pos(&self) -> Location {
-        self.location.clone()
-    }
-
-    fn new_line(&mut self) {
-        self.location.newline();
-    }
-
     /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
-    fn determine_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
+    fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
         // Determine indentation:
         let mut spaces: usize = 0;
         let mut tabs: usize = 0;
@@ -688,7 +670,6 @@ where
                 Some('\n') => {
                     // Empty line!
                     self.next_char();
-                    self.new_line();
                     spaces = 0;
                     tabs = 0;
                 }
@@ -701,472 +682,528 @@ where
         Ok(IndentationLevel { spaces, tabs })
     }
 
-    #[allow(clippy::cognitive_complexity)]
+    /// This is the main entry point. Call this function to retrieve the next token.
+    /// This function is used by the iterator implementation.
     fn inner_next(&mut self) -> LexResult {
-        if !self.pending.is_empty() {
-            return self.pending.remove(0);
-        }
-
-        // top loop
-        loop {
+        // top loop, keep on processing, until we have something pending.
+        while self.pending.is_empty() {
             // Detect indentation levels
             if self.at_begin_of_line {
                 self.at_begin_of_line = false;
+                self.handle_indentations()?;
+            }
 
-                let indentation_level = self.determine_indentation()?;
+            self.consume_normal()?;
+        }
 
-                if self.nesting == 0 {
-                    // Determine indent or dedent:
-                    let current_indentation = *self.indentation_stack.last().unwrap();
-                    let ordering = indentation_level.compare_strict(&current_indentation);
-                    match ordering {
-                        Some(Ordering::Equal) => {
-                            // Same same
-                        }
-                        Some(Ordering::Greater) => {
-                            // New indentation level:
-                            self.indentation_stack.push(indentation_level);
-                            let tok_start = self.get_pos();
-                            let tok_end = tok_start.clone();
-                            return Ok((tok_start, Tok::Indent, tok_end));
-                        }
-                        Some(Ordering::Less) => {
-                            // One or more dedentations
-                            // Pop off other levels until col is found:
+        Ok(self.pending.remove(0))
+    }
 
-                            loop {
-                                let ordering = indentation_level
-                                    .compare_strict(self.indentation_stack.last().unwrap());
-                                match ordering {
-                                    Some(Ordering::Less) => {
-                                        self.indentation_stack.pop();
-                                        let tok_start = self.get_pos();
-                                        let tok_end = tok_start.clone();
-                                        self.pending.push(Ok((tok_start, Tok::Dedent, tok_end)));
-                                    }
-                                    None => {
-                                        return Err(LexicalError {
-                                            error: LexicalErrorType::OtherError("inconsistent use of tabs and spaces in indentation".to_string()),
-                                            location: self.get_pos(),
-                                        });
-                                    }
-                                    _ => {
-                                        break;
-                                    }
-                                };
+    fn handle_indentations(&mut self) -> Result<(), LexicalError> {
+        let indentation_level = self.eat_indentation()?;
+
+        if self.nesting == 0 {
+            // Determine indent or dedent:
+            let current_indentation = *self.indentation_stack.last().unwrap();
+            let ordering = indentation_level.compare_strict(&current_indentation);
+            match ordering {
+                Some(Ordering::Equal) => {
+                    // Same same
+                }
+                Some(Ordering::Greater) => {
+                    // New indentation level:
+                    self.indentation_stack.push(indentation_level);
+                    let tok_start = self.get_pos();
+                    let tok_end = tok_start.clone();
+                    self.emit((tok_start, Tok::Indent, tok_end));
+                }
+                Some(Ordering::Less) => {
+                    // One or more dedentations
+                    // Pop off other levels until col is found:
+
+                    loop {
+                        let ordering = indentation_level
+                            .compare_strict(self.indentation_stack.last().unwrap());
+                        match ordering {
+                            Some(Ordering::Less) => {
+                                self.indentation_stack.pop();
+                                let tok_start = self.get_pos();
+                                let tok_end = tok_start.clone();
+                                self.emit((tok_start, Tok::Dedent, tok_end));
                             }
-
-                            if indentation_level != *self.indentation_stack.last().unwrap() {
-                                // TODO: handle wrong indentations
+                            None => {
                                 return Err(LexicalError {
                                     error: LexicalErrorType::OtherError(
-                                        "Non matching indentation levels!".to_string(),
+                                        "inconsistent use of tabs and spaces in indentation"
+                                            .to_string(),
                                     ),
                                     location: self.get_pos(),
                                 });
                             }
+                            _ => {
+                                break;
+                            }
+                        };
+                    }
 
-                            return self.pending.remove(0);
-                        }
-                        None => {
-                            return Err(LexicalError {
-                                error: LexicalErrorType::OtherError(
-                                    "inconsistent use of tabs and spaces in indentation"
-                                        .to_string(),
-                                ),
-                                location: self.get_pos(),
-                            });
-                        }
+                    if indentation_level != *self.indentation_stack.last().unwrap() {
+                        // TODO: handle wrong indentations
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError(
+                                "Non matching indentation levels!".to_string(),
+                            ),
+                            location: self.get_pos(),
+                        });
                     }
                 }
-            }
-
-            // Check if we have some character:
-            if let Some(c) = self.chr0 {
-                // First check identifier:
-                if self.is_identifier_start(c) {
-                    return self.lex_identifier();
-                } else if is_emoji_presentation(c) {
-                    let tok_start = self.get_pos();
-                    self.next_char();
-                    let tok_end = self.get_pos();
-                    return Ok((
-                        tok_start,
-                        Tok::Name {
-                            name: c.to_string(),
-                        },
-                        tok_end,
-                    ));
-                } else {
-                    match c {
-                        '0'..='9' => return self.lex_number(),
-                        '#' => {
-                            self.lex_comment();
-                            continue;
-                        }
-                        '"' => {
-                            return self.lex_string(false, false, false, false);
-                        }
-                        '\'' => {
-                            return self.lex_string(false, false, false, false);
-                        }
-                        '=' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::EqEqual, tok_end));
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Equal, tok_end));
-                                }
-                            }
-                        }
-                        '+' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::PlusEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::Plus, tok_end));
-                            }
-                        }
-                        '*' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::StarEqual, tok_end));
-                                }
-                                Some('*') => {
-                                    self.next_char();
-                                    match self.chr0 {
-                                        Some('=') => {
-                                            self.next_char();
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::DoubleStarEqual, tok_end));
-                                        }
-                                        _ => {
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::DoubleStar, tok_end));
-                                        }
-                                    }
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Star, tok_end));
-                                }
-                            }
-                        }
-                        '/' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::SlashEqual, tok_end));
-                                }
-                                Some('/') => {
-                                    self.next_char();
-                                    match self.chr0 {
-                                        Some('=') => {
-                                            self.next_char();
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::DoubleSlashEqual, tok_end));
-                                        }
-                                        _ => {
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::DoubleSlash, tok_end));
-                                        }
-                                    }
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Slash, tok_end));
-                                }
-                            }
-                        }
-                        '%' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::PercentEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::Percent, tok_end));
-                            }
-                        }
-                        '|' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::VbarEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::Vbar, tok_end));
-                            }
-                        }
-                        '^' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::CircumflexEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::CircumFlex, tok_end));
-                            }
-                        }
-                        '&' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::AmperEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::Amper, tok_end));
-                            }
-                        }
-                        '-' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::MinusEqual, tok_end));
-                                }
-                                Some('>') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Rarrow, tok_end));
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Minus, tok_end));
-                                }
-                            }
-                        }
-                        '@' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::AtEqual, tok_end));
-                            } else {
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::At, tok_end));
-                            }
-                        }
-                        '!' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            if let Some('=') = self.chr0 {
-                                self.next_char();
-                                let tok_end = self.get_pos();
-                                return Ok((tok_start, Tok::NotEqual, tok_end));
-                            } else {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::UnrecognizedToken { tok: '!' },
-                                    location: tok_start,
-                                });
-                            }
-                        }
-                        '~' => {
-                            return self.eat_single_char(Tok::Tilde);
-                        }
-                        '(' => {
-                            let result = self.eat_single_char(Tok::Lpar);
-                            self.nesting += 1;
-                            return result;
-                        }
-                        ')' => {
-                            let result = self.eat_single_char(Tok::Rpar);
-                            if self.nesting == 0 {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::NestingError,
-                                    location: self.get_pos(),
-                                });
-                            }
-                            self.nesting -= 1;
-                            return result;
-                        }
-                        '[' => {
-                            let result = self.eat_single_char(Tok::Lsqb);
-                            self.nesting += 1;
-                            return result;
-                        }
-                        ']' => {
-                            let result = self.eat_single_char(Tok::Rsqb);
-                            if self.nesting == 0 {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::NestingError,
-                                    location: self.get_pos(),
-                                });
-                            }
-                            self.nesting -= 1;
-                            return result;
-                        }
-                        '{' => {
-                            let result = self.eat_single_char(Tok::Lbrace);
-                            self.nesting += 1;
-                            return result;
-                        }
-                        '}' => {
-                            let result = self.eat_single_char(Tok::Rbrace);
-                            if self.nesting == 0 {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::NestingError,
-                                    location: self.get_pos(),
-                                });
-                            }
-                            self.nesting -= 1;
-                            return result;
-                        }
-                        ':' => {
-                            return self.eat_single_char(Tok::Colon);
-                        }
-                        ';' => {
-                            return self.eat_single_char(Tok::Semi);
-                        }
-                        '<' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('<') => {
-                                    self.next_char();
-                                    match self.chr0 {
-                                        Some('=') => {
-                                            self.next_char();
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::LeftShiftEqual, tok_end));
-                                        }
-                                        _ => {
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::LeftShift, tok_end));
-                                        }
-                                    }
-                                }
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::LessEqual, tok_end));
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Less, tok_end));
-                                }
-                            }
-                        }
-                        '>' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            match self.chr0 {
-                                Some('>') => {
-                                    self.next_char();
-                                    match self.chr0 {
-                                        Some('=') => {
-                                            self.next_char();
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::RightShiftEqual, tok_end));
-                                        }
-                                        _ => {
-                                            let tok_end = self.get_pos();
-                                            return Ok((tok_start, Tok::RightShift, tok_end));
-                                        }
-                                    }
-                                }
-                                Some('=') => {
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::GreaterEqual, tok_end));
-                                }
-                                _ => {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Greater, tok_end));
-                                }
-                            }
-                        }
-                        ',' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            let tok_end = self.get_pos();
-                            return Ok((tok_start, Tok::Comma, tok_end));
-                        }
-                        '.' => {
-                            if let Some('0'..='9') = self.chr1 {
-                                return self.lex_number();
-                            } else {
-                                let tok_start = self.get_pos();
-                                self.next_char();
-                                if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) {
-                                    self.next_char();
-                                    self.next_char();
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Ellipsis, tok_end));
-                                } else {
-                                    let tok_end = self.get_pos();
-                                    return Ok((tok_start, Tok::Dot, tok_end));
-                                }
-                            }
-                        }
-                        '\n' => {
-                            let tok_start = self.get_pos();
-                            self.next_char();
-                            let tok_end = self.get_pos();
-                            self.new_line();
-
-                            // Depending on the nesting level, we emit newline or not:
-                            if self.nesting == 0 {
-                                self.at_begin_of_line = true;
-                                return Ok((tok_start, Tok::Newline, tok_end));
-                            } else {
-                                continue;
-                            }
-                        }
-                        ' ' => {
-                            // Skip whitespaces
-                            self.next_char();
-                            continue;
-                        }
-                        _ => {
-                            let c = self.next_char();
-                            return Err(LexicalError {
-                                error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
-                                location: self.get_pos(),
-                            });
-                        } // Ignore all the rest..
-                    }
+                None => {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError(
+                            "inconsistent use of tabs and spaces in indentation".to_string(),
+                        ),
+                        location: self.get_pos(),
+                    });
                 }
-            } else {
-                let tok_pos = self.get_pos();
-                return Ok((tok_pos.clone(), Tok::EndOfFile, tok_pos));
             }
         }
+
+        Ok(())
     }
 
-    fn eat_single_char(&mut self, ty: Tok) -> LexResult {
+    /// Take a look at the next character, if any, and decide upon the next steps.
+    fn consume_normal(&mut self) -> Result<(), LexicalError> {
+        // Check if we have some character:
+        if let Some(c) = self.chr0 {
+            // First check identifier:
+            if self.is_identifier_start(c) {
+                let identifier = self.lex_identifier()?;
+                self.emit(identifier);
+            } else if is_emoji_presentation(c) {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+                self.emit((
+                    tok_start,
+                    Tok::Name {
+                        name: c.to_string(),
+                    },
+                    tok_end,
+                ));
+            } else {
+                self.consume_character(c)?;
+            }
+        } else {
+            // We reached end of file.
+            let tok_pos = self.get_pos();
+
+            // First of all, we need all nestings to be finished.
+            if self.nesting > 0 {
+                return Err(LexicalError {
+                    error: LexicalErrorType::NestingError,
+                    location: tok_pos,
+                });
+            }
+
+            /*
+                        // Next, insert a trailing newline, if required.
+                        if !self.at_begin_of_line {
+                            self.emit((tok_pos.clone(), Tok::Newline, tok_pos.clone()));
+                        }
+
+                        // Next, flush the indentation stack to zero.
+                        while !self.indentation_stack.is_empty() {
+                            self.indentation_stack.pop();
+                            self.emit((tok_pos.clone(), Tok::Dedent, tok_pos.clone()));
+                        }
+            */
+            self.emit((tok_pos.clone(), Tok::EndOfFile, tok_pos));
+        }
+
+        Ok(())
+    }
+
+    /// Okay, we are facing a weird character, what is it? Determine that.
+    fn consume_character(&mut self, c: char) -> Result<(), LexicalError> {
+        match c {
+            '0'..='9' => {
+                let number = self.lex_number()?;
+                self.emit(number);
+            }
+            '#' => {
+                self.lex_comment();
+            }
+            '"' | '\'' => {
+                let string = self.lex_string(false, false, false, false)?;
+                self.emit(string);
+            }
+            '=' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::EqEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Equal, tok_end));
+                    }
+                }
+            }
+            '+' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::PlusEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Plus, tok_end));
+                }
+            }
+            '*' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::StarEqual, tok_end));
+                    }
+                    Some('*') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleStarEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleStar, tok_end));
+                            }
+                        }
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Star, tok_end));
+                    }
+                }
+            }
+            '/' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::SlashEqual, tok_end));
+                    }
+                    Some('/') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleSlashEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleSlash, tok_end));
+                            }
+                        }
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Slash, tok_end));
+                    }
+                }
+            }
+            '%' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::PercentEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Percent, tok_end));
+                }
+            }
+            '|' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::VbarEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Vbar, tok_end));
+                }
+            }
+            '^' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::CircumflexEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::CircumFlex, tok_end));
+                }
+            }
+            '&' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::AmperEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Amper, tok_end));
+                }
+            }
+            '-' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::MinusEqual, tok_end));
+                    }
+                    Some('>') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Rarrow, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Minus, tok_end));
+                    }
+                }
+            }
+            '@' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::AtEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::At, tok_end));
+                }
+            }
+            '!' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::NotEqual, tok_end));
+                } else {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::UnrecognizedToken { tok: '!' },
+                        location: tok_start,
+                    });
+                }
+            }
+            '~' => {
+                self.eat_single_char(Tok::Tilde);
+            }
+            '(' => {
+                self.eat_single_char(Tok::Lpar);
+                self.nesting += 1;
+            }
+            ')' => {
+                self.eat_single_char(Tok::Rpar);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            '[' => {
+                self.eat_single_char(Tok::Lsqb);
+                self.nesting += 1;
+            }
+            ']' => {
+                self.eat_single_char(Tok::Rsqb);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            '{' => {
+                self.eat_single_char(Tok::Lbrace);
+                self.nesting += 1;
+            }
+            '}' => {
+                self.eat_single_char(Tok::Rbrace);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            ':' => {
+                self.eat_single_char(Tok::Colon);
+            }
+            ';' => {
+                self.eat_single_char(Tok::Semi);
+            }
+            '<' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('<') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::LeftShiftEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::LeftShift, tok_end));
+                            }
+                        }
+                    }
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::LessEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Less, tok_end));
+                    }
+                }
+            }
+            '>' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('>') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::RightShiftEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::RightShift, tok_end));
+                            }
+                        }
+                    }
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::GreaterEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Greater, tok_end));
+                    }
+                }
+            }
+            ',' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+                self.emit((tok_start, Tok::Comma, tok_end));
+            }
+            '.' => {
+                if let Some('0'..='9') = self.chr1 {
+                    let number = self.lex_number()?;
+                    self.emit(number);
+                } else {
+                    let tok_start = self.get_pos();
+                    self.next_char();
+                    if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) {
+                        self.next_char();
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Ellipsis, tok_end));
+                    } else {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Dot, tok_end));
+                    }
+                }
+            }
+            '\n' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+
+                // Depending on the nesting level, we emit newline or not:
+                if self.nesting == 0 {
+                    self.at_begin_of_line = true;
+                    self.emit((tok_start, Tok::Newline, tok_end));
+                }
+            }
+            ' ' => {
+                // Skip whitespaces
+                self.next_char();
+            }
+            _ => {
+                let c = self.next_char();
+                return Err(LexicalError {
+                    error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
+                    location: self.get_pos(),
+                });
+            } // Ignore all the rest..
+        }
+
+        Ok(())
+    }
+
+    fn eat_single_char(&mut self, ty: Tok) {
         let tok_start = self.get_pos();
-        self.next_char();
+        self.next_char().unwrap();
         let tok_end = self.get_pos();
-        Ok((tok_start, ty, tok_end))
+        self.emit((tok_start, ty, tok_end));
+    }
+
+    /// Helper function to go to the next character coming up.
+    fn next_char(&mut self) -> Option<char> {
+        let c = self.chr0;
+        let nxt = self.chars.next();
+        self.chr0 = self.chr1;
+        self.chr1 = nxt;
+        if c == Some('\n') {
+            self.location.newline();
+        } else {
+            self.location.go_right();
+        }
+        c
+    }
+
+    /// Helper function to retrieve the current position.
+    fn get_pos(&self) -> Location {
+        self.location.clone()
+    }
+
+    /// Helper function to emit a lexed token to the queue of tokens.
+    fn emit(&mut self, spanned: Spanned) {
+        self.pending.push(spanned);
     }
 }
 
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index a3ca48c7a..4a88ae816 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -121,7 +121,7 @@ mod tests {
 
     #[test]
     fn test_parse_empty() {
-        let parse_ast = parse_program(&String::from("\n"));
+        let parse_ast = parse_program(&String::from(""));
         assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] }))
     }
 

From e922a6a952f4d00f83c458bd23f5f72840d3db6f Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 21 Jul 2019 14:00:55 +0200
Subject: [PATCH 2/2] Fix trailing newline issue #17

---
 parser/src/lexer.rs       | 130 +++++++++++++++++++++-----------------
 parser/src/parser.rs      |  14 ++--
 parser/src/python.lalrpop |   2 +-
 src/main.rs               |   4 +-
 vm/src/builtins.rs        |   7 --
 vm/src/eval.rs            |   2 +-
 wasm/lib/src/vm_class.rs  |   3 +-
 7 files changed, 82 insertions(+), 80 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index f9e622e1a..edf5e219f 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -16,34 +16,45 @@ use unic_emoji_char::is_emoji_presentation;
 use unicode_xid::UnicodeXID;
 use wtf8;
 
-#[derive(Clone, Copy, PartialEq, Debug)]
+#[derive(Clone, Copy, PartialEq, Debug, Default)]
 struct IndentationLevel {
     tabs: usize,
     spaces: usize,
 }
 
 impl IndentationLevel {
-    fn new() -> IndentationLevel {
-        IndentationLevel { tabs: 0, spaces: 0 }
-    }
-    fn compare_strict(&self, other: &IndentationLevel) -> Option<Ordering> {
+    fn compare_strict(
+        &self,
+        other: &IndentationLevel,
+        location: Location,
+    ) -> Result<Ordering, LexicalError> {
         // We only know for sure that we're smaller or bigger if tabs
         // and spaces both differ in the same direction. Otherwise we're
         // dependent on the size of tabs.
         if self.tabs < other.tabs {
             if self.spaces <= other.spaces {
-                Some(Ordering::Less)
+                Ok(Ordering::Less)
             } else {
-                None
+                Err(LexicalError {
+                    location,
+                    error: LexicalErrorType::OtherError(
+                        "inconsistent use of tabs and spaces in indentation".to_string(),
+                    ),
+                })
             }
         } else if self.tabs > other.tabs {
             if self.spaces >= other.spaces {
-                Some(Ordering::Greater)
+                Ok(Ordering::Greater)
             } else {
-                None
+                Err(LexicalError {
+                    location,
+                    error: LexicalErrorType::OtherError(
+                        "inconsistent use of tabs and spaces in indentation".to_string(),
+                    ),
+                })
             }
         } else {
-            Some(self.spaces.cmp(&other.spaces))
+            Ok(self.spaces.cmp(&other.spaces))
         }
     }
 }
@@ -234,7 +245,7 @@ where
             chars: input,
             at_begin_of_line: true,
             nesting: 0,
-            indentation_stack: vec![IndentationLevel::new()],
+            indentation_stack: vec![Default::default()],
             pending: Vec::new(),
             chr0: None,
             location: Location::new(0, 0),
@@ -704,65 +715,49 @@ where
 
         if self.nesting == 0 {
             // Determine indent or dedent:
-            let current_indentation = *self.indentation_stack.last().unwrap();
-            let ordering = indentation_level.compare_strict(&current_indentation);
+            let current_indentation = self.indentation_stack.last().unwrap();
+            let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;
             match ordering {
-                Some(Ordering::Equal) => {
+                Ordering::Equal => {
                     // Same same
                 }
-                Some(Ordering::Greater) => {
+                Ordering::Greater => {
                     // New indentation level:
                     self.indentation_stack.push(indentation_level);
                     let tok_start = self.get_pos();
                     let tok_end = tok_start.clone();
                     self.emit((tok_start, Tok::Indent, tok_end));
                 }
-                Some(Ordering::Less) => {
+                Ordering::Less => {
                     // One or more dedentations
                     // Pop off other levels until col is found:
 
                     loop {
+                        let current_indentation = self.indentation_stack.last().unwrap();
                         let ordering = indentation_level
-                            .compare_strict(self.indentation_stack.last().unwrap());
+                            .compare_strict(current_indentation, self.get_pos())?;
                         match ordering {
-                            Some(Ordering::Less) => {
+                            Ordering::Less => {
                                 self.indentation_stack.pop();
                                 let tok_start = self.get_pos();
                                 let tok_end = tok_start.clone();
                                 self.emit((tok_start, Tok::Dedent, tok_end));
                             }
-                            None => {
+                            Ordering::Equal => {
+                                // We arrived at proper level of indentation.
+                                break;
+                            }
+                            Ordering::Greater => {
+                                // TODO: handle wrong indentations
                                 return Err(LexicalError {
                                     error: LexicalErrorType::OtherError(
-                                        "inconsistent use of tabs and spaces in indentation"
-                                            .to_string(),
+                                        "Non matching indentation levels!".to_string(),
                                     ),
                                     location: self.get_pos(),
                                 });
                             }
-                            _ => {
-                                break;
-                            }
                         };
                     }
-
-                    if indentation_level != *self.indentation_stack.last().unwrap() {
-                        // TODO: handle wrong indentations
-                        return Err(LexicalError {
-                            error: LexicalErrorType::OtherError(
-                                "Non matching indentation levels!".to_string(),
-                            ),
-                            location: self.get_pos(),
-                        });
-                    }
-                }
-                None => {
-                    return Err(LexicalError {
-                        error: LexicalErrorType::OtherError(
-                            "inconsistent use of tabs and spaces in indentation".to_string(),
-                        ),
-                        location: self.get_pos(),
-                    });
                 }
             }
         }
@@ -804,18 +799,18 @@ where
                 });
             }
 
-            /*
-                        // Next, insert a trailing newline, if required.
-                        if !self.at_begin_of_line {
-                            self.emit((tok_pos.clone(), Tok::Newline, tok_pos.clone()));
-                        }
+            // Next, insert a trailing newline, if required.
+            if !self.at_begin_of_line {
+                self.at_begin_of_line = true;
+                self.emit((tok_pos.clone(), Tok::Newline, tok_pos.clone()));
+            }
+
+            // Next, flush the indentation stack to zero.
+            while self.indentation_stack.len() > 1 {
+                self.indentation_stack.pop();
+                self.emit((tok_pos.clone(), Tok::Dedent, tok_pos.clone()));
+            }
 
-                        // Next, flush the indentation stack to zero.
-                        while !self.indentation_stack.is_empty() {
-                            self.indentation_stack.pop();
-                            self.emit((tok_pos.clone(), Tok::Dedent, tok_pos.clone()));
-                        }
-            */
             self.emit((tok_pos.clone(), Tok::EndOfFile, tok_pos));
         }
 
@@ -1325,7 +1320,8 @@ mod tests {
                 Tok::String {
                     value: "\\".to_string(),
                     is_fstring: false,
-                }
+                },
+                Tok::Newline,
             ]
         );
     }
@@ -1358,6 +1354,7 @@ mod tests {
                     real: 0.0,
                     imag: 2.2,
                 },
+                Tok::Newline,
             ]
         );
     }
@@ -1369,7 +1366,7 @@ mod tests {
             fn $name() {
                 let source = String::from(format!(r"99232  # {}", $eol));
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
             }
             )*
         }
@@ -1395,6 +1392,7 @@ mod tests {
                         Tok::Int { value: BigInt::from(123) },
                         Tok::Newline,
                         Tok::Int { value: BigInt::from(456) },
+                        Tok::Newline,
                     ]
                 )
             }
@@ -1430,6 +1428,7 @@ mod tests {
                 Tok::Int {
                     value: BigInt::from(0)
                 },
+                Tok::Newline,
             ]
         );
     }
@@ -1457,6 +1456,7 @@ mod tests {
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
                         Tok::Dedent,
+                        Tok::Newline,
                     ]
                 );
             }
@@ -1501,6 +1501,7 @@ mod tests {
                         Tok::Newline,
                         Tok::Dedent,
                         Tok::Dedent,
+                        Tok::Newline,
                     ]
                 );
             }
@@ -1539,6 +1540,7 @@ mod tests {
                         Tok::Newline,
                         Tok::Dedent,
                         Tok::Dedent,
+                        Tok::Newline,
                     ]
                 );
             }
@@ -1578,6 +1580,7 @@ mod tests {
                         Tok::Int { value: BigInt::from(2) },
                         Tok::Rsqb,
                         Tok::Newline,
+                        Tok::Newline,
                     ]
                 );
             }
@@ -1603,6 +1606,7 @@ mod tests {
                 Tok::DoubleSlashEqual,
                 Tok::Slash,
                 Tok::Slash,
+                Tok::Newline,
             ]
         );
     }
@@ -1642,6 +1646,7 @@ mod tests {
                     value: String::from("raw\'"),
                     is_fstring: false,
                 },
+                Tok::Newline,
             ]
         );
     }
@@ -1660,6 +1665,7 @@ mod tests {
                             value: String::from("abcdef"),
                             is_fstring: false,
                         },
+                        Tok::Newline,
                     ]
                 )
             }
@@ -1674,26 +1680,32 @@ mod tests {
     }
 
     #[test]
-    fn test_byte() {
+    fn test_single_quoted_byte() {
         // single quote
         let all = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
         let source = String::from(all);
         let tokens = lex_source(&source);
         let res = (0..=255).collect::<Vec<u8>>();
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
+    }
 
+    #[test]
+    fn test_double_quoted_byte() {
         // double quote
         let all = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
         let source = String::from(all);
         let tokens = lex_source(&source);
         let res = (0..=255).collect::<Vec<u8>>();
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
+    }
 
+    #[test]
+    fn test_escape_char_in_byte_literal() {
         // backslash doesnt escape
         let all = r##"b"omkmok\Xaa""##;
         let source = String::from(all);
         let tokens = lex_source(&source);
         let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97];
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }]);
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
     }
 }
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 4a88ae816..60d61be22 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -127,7 +127,7 @@ mod tests {
 
     #[test]
     fn test_parse_print_hello() {
-        let source = String::from("print('Hello world')\n");
+        let source = String::from("print('Hello world')");
         let parse_ast = parse_program(&source).unwrap();
         assert_eq!(
             parse_ast,
@@ -151,7 +151,7 @@ mod tests {
 
     #[test]
     fn test_parse_print_2() {
-        let source = String::from("print('Hello world', 2)\n");
+        let source = String::from("print('Hello world', 2)");
         let parse_ast = parse_program(&source).unwrap();
         assert_eq!(
             parse_ast,
@@ -175,7 +175,7 @@ mod tests {
 
     #[test]
     fn test_parse_kwargs() {
-        let source = String::from("my_func('positional', keyword=2)\n");
+        let source = String::from("my_func('positional', keyword=2)");
         let parse_ast = parse_program(&source).unwrap();
         assert_eq!(
             parse_ast,
@@ -202,7 +202,7 @@ mod tests {
 
     #[test]
     fn test_parse_if_elif_else() {
-        let source = String::from("if 1: 10\nelif 2: 20\nelse: 30\n");
+        let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
         let parse_ast = parse_statement(&source).unwrap();
         assert_eq!(
             parse_ast,
@@ -226,7 +226,7 @@ mod tests {
 
     #[test]
     fn test_parse_lambda() {
-        let source = String::from("lambda x, y: x * y\n"); // lambda(x, y): x * y");
+        let source = String::from("lambda x, y: x * y"); // lambda(x, y): x * y");
         let parse_ast = parse_statement(&source);
         assert_eq!(
             parse_ast,
@@ -265,7 +265,7 @@ mod tests {
 
     #[test]
     fn test_parse_tuples() {
-        let source = String::from("a, b = 4, 5\n");
+        let source = String::from("a, b = 4, 5");
 
         assert_eq!(
             parse_statement(&source),
@@ -292,7 +292,7 @@ mod tests {
     #[test]
     fn test_parse_class() {
         let source = String::from(
-            "class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass\n",
+            "class Foo(A, B):\n def __init__(self):\n  pass\n def method_with_default(self, arg='default'):\n  pass",
         );
         assert_eq!(
             parse_statement(&source),
diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop
index d08efc041..3ea502f7c 100644
--- a/parser/src/python.lalrpop
+++ b/parser/src/python.lalrpop
@@ -21,7 +21,7 @@ grammar;
 pub Top: ast::Top = {
     StartProgram <p:Program> => ast::Top::Program(p),
     StartStatement <s:Statement> => ast::Top::Statement(s),
-    StartExpression <e:Test> => ast::Top::Expression(e),
+    StartExpression <e:Test> ("\n")* => ast::Top::Expression(e),
 };
 
 Program: ast::Program = {
diff --git a/src/main.rs b/src/main.rs
index 1da922848..d4a0f2c4a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -298,11 +298,9 @@ fn handle_exception<T>(vm: &VirtualMachine, result: PyResult<T>) {
     }
 }
 
-fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult<()> {
+fn run_command(vm: &VirtualMachine, source: String) -> PyResult<()> {
     debug!("Running command {}", source);
 
-    // This works around https://github.com/RustPython/RustPython/issues/17
-    source.push('\n');
     _run_string(vm, &source, "<stdin>".to_string())?;
     Ok(())
 }
diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs
index 17abf16b4..ec71ac5fa 100644
--- a/vm/src/builtins.rs
+++ b/vm/src/builtins.rs
@@ -108,9 +108,6 @@ fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult<PyCodeRef
         Either::B(bytes) => str::from_utf8(&bytes).unwrap().to_string(),
     };
 
-    // TODO: fix this newline bug:
-    let source = format!("{}\n", source);
-
     let mode = get_compile_mode(vm, &args.mode.value)?;
 
     vm.compile(&source, &mode, args.filename.value.to_string())
@@ -161,8 +158,6 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
     } else if objtype::isinstance(source, &vm.ctx.str_type()) {
         let mode = compile::Mode::Eval;
         let source = objstr::get_value(source);
-        // TODO: fix this newline bug:
-        let source = format!("{}\n", source);
         vm.compile(&source, &mode, "<string>".to_string())
             .map_err(|err| vm.new_syntax_error(&err))?
     } else {
@@ -190,8 +185,6 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult {
     let code_obj = if objtype::isinstance(source, &vm.ctx.str_type()) {
         let mode = compile::Mode::Exec;
         let source = objstr::get_value(source);
-        // TODO: fix this newline bug:
-        let source = format!("{}\n", source);
         vm.compile(&source, &mode, "<string>".to_string())
             .map_err(|err| vm.new_syntax_error(&err))?
     } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) {
diff --git a/vm/src/eval.rs b/vm/src/eval.rs
index a55a3e03e..40f084df7 100644
--- a/vm/src/eval.rs
+++ b/vm/src/eval.rs
@@ -31,7 +31,7 @@ mod tests {
 
     #[test]
     fn test_print_42() {
-        let source = String::from("print('Hello world')\n");
+        let source = String::from("print('Hello world')");
         let vm: VirtualMachine = Default::default();
         let vars = vm.new_scope_with_builtins();
         let _result = eval(&vm, &source, vars, "<unittest>");
diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs
index 8ba80c694..75b110730 100644
--- a/wasm/lib/src/vm_class.rs
+++ b/wasm/lib/src/vm_class.rs
@@ -271,13 +271,12 @@ impl WASMVirtualMachine {
         })?
     }
 
-    fn run(&self, mut source: String, mode: compile::Mode) -> Result<JsValue, JsValue> {
+    fn run(&self, source: String, mode: compile::Mode) -> Result<JsValue, JsValue> {
         self.assert_valid()?;
         self.with_unchecked(
             |StoredVirtualMachine {
                  ref vm, ref scope, ..
              }| {
-                source.push('\n');
                 let code = vm.compile(&source, &mode, "<wasm>".to_string());
                 let code = code.map_err(|err| {
                     let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));