Merge pull request #1698 from palaviv/fix-line-continue

fix line continuation parsing

parser/src/error.rs

@@ -27,6 +27,7 @@ pub enum LexicalErrorType {
     DuplicateKeywordArgumentError,
     UnrecognizedToken { tok: char },
     FStringError(FStringErrorType),
+    LineContinuationError,
     OtherError(String),
 }
 
@@ -55,6 +56,9 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::UnrecognizedToken { tok } => {
                 write!(f, "Got unexpected token {}", tok)
             }
+            LexicalErrorType::LineContinuationError => {
+                write!(f, "unexpected character after line continuation character")
+            }
             LexicalErrorType::OtherError(msg) => write!(f, "{}", msg),
         }
     }

parser/src/lexer.rs

@@ -117,8 +117,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 
 pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator<Item = LexResult> + 'a {
     let nlh = NewlineHandler::new(source.chars());
-    let lch = LineContinuationHandler::new(nlh);
-    Lexer::new(lch)
+    Lexer::new(nlh)
 }
 
 // The newline handler is an iterator which collapses different newline
@@ -178,61 +177,6 @@ where
     }
 }
 
-// Glues \ and \n into a single line:
-pub struct LineContinuationHandler<T: Iterator<Item = char>> {
-    source: T,
-    chr0: Option<char>,
-    chr1: Option<char>,
-}
-
-impl<T> LineContinuationHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    pub fn new(source: T) -> Self {
-        let mut nlh = LineContinuationHandler {
-            source,
-            chr0: None,
-            chr1: None,
-        };
-        nlh.shift();
-        nlh.shift();
-        nlh
-    }
-
-    fn shift(&mut self) -> Option<char> {
-        let result = self.chr0;
-        self.chr0 = self.chr1;
-        self.chr1 = self.source.next();
-        result
-    }
-}
-
-impl<T> Iterator for LineContinuationHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    type Item = char;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // Collapse \r\n into \n
-        loop {
-            if self.chr0 == Some('\\') && self.chr1 == Some('\n') {
-                // Skip backslash and newline
-                self.shift();
-                self.shift();
-            // Idea: insert trailing newline here:
-            // } else if self.chr0 != Some('\n') && self.chr1.is_none() {
-            //     self.chr1 = Some('\n');
-            } else {
-                break;
-            }
-        }
-
-        self.shift()
-    }
-}
-
 impl<T> Lexer<T>
 where
     T: Iterator<Item = char>,
@@ -1201,6 +1145,18 @@ where
                     self.next_char();
                 }
             }
+            '\\' => {
+                self.next_char();
+                if let Some('\n') = self.chr0 {
+                    self.next_char();
+                } else {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::LineContinuationError,
+                        location: self.get_pos(),
+                    });
+                }
+            }
+
             _ => {
                 let c = self.next_char();
                 return Err(LexicalError {

tests/snippets/funky_syntax.py

@@ -1,3 +1,4 @@
+import traceback
 
 a = 2
 b = 2 + 4 if a < 5 else 'boe'
@@ -12,3 +13,10 @@ e = lambda x: 1 if x else 0
 assert e(True) == 1
 assert e(False) == 0
 
+try:
+	a = "aaaa" + \
+		"bbbb"
+	1/0
+except ZeroDivisionError as ex:
+	tb = traceback.extract_tb(ex.__traceback__)
+	assert tb[0].lineno == 19

tests/snippets/invalid_syntax.py

@@ -51,3 +51,12 @@ with assert_raises(TabError):
 
 with assert_raises(SyntaxError):
     compile('0xX', 'test.py', 'exec')
+
+
+src = """
+"aaaa" \a
+"bbbb"
+"""
+
+with assert_raises(SyntaxError):
+    compile(src, 'test.py', 'exec')