Rewrite _tokenize with 2-phase model

Replace per-line reparsing with single-pass tokenization:
- Read all lines via readline, parse once, yield tokens
- Fix token type values (COMMENT=65, NL=66, OP=55)
- Fix NEWLINE/NL end positions and implicit newline handling
- Fix DEDENT positions via look-ahead to next non-DEDENT token
- Handle FSTRING_MIDDLE brace unescaping ({{ → {, }} → })
- Emit implicit NL before ENDMARKER when source lacks trailing newline
- Raise IndentationError from lexer errors
- Remove 13 expectedFailure marks for now-passing tests

Lib/test/test_tabnanny.py

@@ -316,7 +316,6 @@ class TestCommandLine(TestCase):
             self.assertListEqual(out.splitlines(), stdout.splitlines())
             self.assertListEqual(err.splitlines(), stderr.splitlines())
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Should displays error when errored python file is given.
     def test_with_errored_file(self):
         """Should displays error when errored python file is given."""
         with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:

Lib/test/test_tokenize.py

@@ -1167,7 +1167,6 @@ async def f():
     DEDENT     ''            (7, 0) (7, 0)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; +  "    NEWLINE    '\\n'          (4, 1) (4, 2)"]
     def test_newline_after_parenthesized_block_with_comment(self):
         self.check_tokenize('''\
 [
@@ -1192,7 +1191,6 @@ async def f():
     NAME       'x'           (1, 3) (1, 4)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; '    FSTRING_END "\'\'\'"         (2, 68) (2, 71)']
     def test_multiline_non_ascii_fstring(self):
         self.check_tokenize("""\
 a = f'''
@@ -1204,7 +1202,6 @@ a = f'''
     FSTRING_END "\'\'\'"         (2, 68) (2, 71)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Diff is 696 characters long. Set self.maxDiff to None to see it.
     def test_multiline_non_ascii_fstring_with_expr(self):
         self.check_tokenize("""\
 f'''
@@ -2176,7 +2173,6 @@ if 1:
         # Two string literals on the same line
         self.check_roundtrip("'' ''")
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_random_files(self):
         # Test roundtrip on random python modules.
         # pass the '-ucpu' option to process the full directory.
@@ -2214,7 +2210,6 @@ if 1:
 
 
 class InvalidPythonTests(TestCase):
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Diff is 1046 characters long. Set self.maxDiff to None to see it.
     def test_number_followed_by_name(self):
         # See issue #gh-105549
         source = "2sin(x)"
@@ -2254,7 +2249,6 @@ class CTokenizeTest(TestCase):
             )
             self.assertEqual(result, expected.rstrip().splitlines())
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_encoding(self):
         def readline(encoding):
             yield "1+1".encode(encoding)
@@ -2386,7 +2380,6 @@ class CTokenizeTest(TestCase):
     NUMBER     '3.14e159'    (1, 4) (1, 12)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_string(self):
 
         self.check_tokenize('x = \'\'; y = ""', """\
@@ -2818,7 +2811,6 @@ f'''__{
     NUMBER     '1'           (1, 22) (1, 23)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_selector(self):
 
         self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
@@ -2841,7 +2833,6 @@ f'''__{
     RPAR       ')'           (2, 29) (2, 30)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_method(self):
 
         self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
@@ -2859,7 +2850,6 @@ f'''__{
     NAME       'pass'        (2, 14) (2, 18)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_tabs(self):
 
         self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
@@ -3144,7 +3134,6 @@ async def f():
     DEDENT     ''            (6, -1) (6, -1)
     """)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_unicode(self):
 
         self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\
@@ -3394,7 +3383,6 @@ class CommandLineTest(unittest.TestCase):
             with contextlib.redirect_stderr(StringIO()):
                 _ = self.invoke_tokenize('--unknown')
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_without_flag(self):
         # test 'python -m tokenize source.py'
         source = 'a = 1'
@@ -3408,7 +3396,6 @@ class CommandLineTest(unittest.TestCase):
         '''
         self.check_output(source, expect)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_exact_flag(self):
         # test 'python -m tokenize -e/--exact source.py'
         source = 'a = 1'

crates/stdlib/src/_tokenize.rs

@@ -0,0 +1,747 @@
+pub(crate) use _tokenize::module_def;
+
+#[pymodule]
+mod _tokenize {
+    use crate::{
+        common::lock::PyRwLock,
+        vm::{
+            AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
+            builtins::{PyBytes, PyStr, PyType},
+            convert::ToPyObject,
+            function::ArgCallable,
+            protocol::PyIterReturn,
+            types::{Constructor, IterNext, Iterable, SelfIter},
+        },
+    };
+    use ruff_python_ast::PySourceType;
+    use ruff_python_ast::token::{Token, TokenKind};
+    use ruff_python_parser::{
+        LexicalErrorType, ParseError, ParseErrorType, parse_unchecked_source,
+    };
+    use ruff_source_file::{LineIndex, LineRanges};
+    use ruff_text_size::{Ranged, TextSize};
+    use core::fmt;
+
+    const TOKEN_ENDMARKER: u8 = 0;
+    const TOKEN_DEDENT: u8 = 6;
+    const TOKEN_OP: u8 = 55;
+    const TOKEN_COMMENT: u8 = 65;
+    const TOKEN_NL: u8 = 66;
+
+    #[pyattr]
+    #[pyclass(name = "TokenizerIter")]
+    #[derive(PyPayload)]
+    pub struct PyTokenizerIter {
+        readline: ArgCallable,
+        extra_tokens: bool,
+        encoding: Option<String>,
+        state: PyRwLock<TokenizerState>,
+    }
+
+    impl PyTokenizerIter {
+        fn readline(&self, vm: &VirtualMachine) -> PyResult<String> {
+            let raw_line = match self.readline.invoke((), vm) {
+                Ok(v) => v,
+                Err(err) => {
+                    if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) {
+                        return Ok(String::new());
+                    }
+                    return Err(err);
+                }
+            };
+            Ok(match &self.encoding {
+                Some(encoding) => {
+                    let bytes = raw_line
+                        .downcast::<PyBytes>()
+                        .map_err(|_| vm.new_type_error("readline() returned a non-bytes object"))?;
+                    vm.state
+                        .codec_registry
+                        .decode_text(bytes.into(), encoding, None, vm)
+                        .map(|s| s.to_string())?
+                }
+                None => raw_line
+                    .downcast::<PyStr>()
+                    .map(|s| s.to_string())
+                    .map_err(|_| vm.new_type_error("readline() returned a non-string object"))?,
+            })
+        }
+    }
+
+    impl fmt::Debug for PyTokenizerIter {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            f.debug_struct("PyTokenizerIter")
+                .field("extra_tokens", &self.extra_tokens)
+                .field("encoding", &self.encoding)
+                .finish()
+        }
+    }
+
+    #[pyclass(with(Constructor, Iterable, IterNext))]
+    impl PyTokenizerIter {}
+
+    impl Constructor for PyTokenizerIter {
+        type Args = PyTokenizerIterArgs;
+
+        fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> {
+            let Self::Args {
+                readline,
+                extra_tokens,
+                encoding,
+            } = args;
+
+            Ok(Self {
+                readline,
+                extra_tokens,
+                encoding: encoding.map(|s| s.to_string()),
+                state: PyRwLock::new(TokenizerState {
+                    phase: TokenizerPhase::Reading {
+                        source: String::new(),
+                    },
+                }),
+            })
+        }
+    }
+
+    impl SelfIter for PyTokenizerIter {}
+
+    impl IterNext for PyTokenizerIter {
+        fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
+            let mut state = zelf.state.read().clone();
+
+            loop {
+                match &mut state.phase {
+                    TokenizerPhase::Reading { source } => {
+                        let line = zelf.readline(vm)?;
+                        if line.is_empty() {
+                            let accumulated = core::mem::take(source);
+                            let parsed =
+                                parse_unchecked_source(&accumulated, PySourceType::Python);
+                            let tokens: Vec<Token> = parsed.tokens().iter().copied().collect();
+                            let errors: Vec<ParseError> = parsed.errors().to_vec();
+                            let line_index = LineIndex::from_source_text(&accumulated);
+                            let implicit_nl = !accumulated.ends_with('\n');
+                            state.phase = TokenizerPhase::Yielding {
+                                source: accumulated,
+                                tokens,
+                                errors,
+                                index: 0,
+                                line_index,
+                                need_implicit_nl: implicit_nl,
+                                pending_fstring_parts: Vec::new(),
+                                pending_empty_fstring_middle: None,
+                            };
+                        } else {
+                            source.push_str(&line);
+                        }
+                    }
+                    TokenizerPhase::Yielding { .. } => {
+                        let result =
+                            emit_next_token(&mut state, zelf.extra_tokens, vm)?;
+                        *zelf.state.write() = state;
+                        return Ok(result);
+                    }
+                    TokenizerPhase::Done => {
+                        return Ok(PyIterReturn::StopIteration(None));
+                    }
+                }
+            }
+        }
+    }
+
+    /// Emit the next token from the Yielding phase.
+    fn emit_next_token(
+        state: &mut TokenizerState,
+        extra_tokens: bool,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyIterReturn> {
+        let TokenizerPhase::Yielding {
+            source,
+            tokens,
+            errors,
+            index,
+            line_index,
+            need_implicit_nl,
+            pending_fstring_parts,
+            pending_empty_fstring_middle,
+        } = &mut state.phase
+        else {
+            unreachable!()
+        };
+
+        // Emit pending empty FSTRING_MIDDLE (for format spec nesting)
+        if let Some((mid_type, mid_line, mid_col, mid_line_str)) =
+            pending_empty_fstring_middle.take()
+        {
+            return Ok(PyIterReturn::Return(make_token_tuple(
+                vm,
+                mid_type,
+                "",
+                mid_line,
+                mid_col as isize,
+                mid_line,
+                mid_col as isize,
+                &mid_line_str,
+            )));
+        }
+
+        // Emit any pending fstring sub-tokens first
+        if let Some((tok_type, tok_str, sl, sc, el, ec)) = pending_fstring_parts.pop() {
+            let offset: usize = source
+                .lines()
+                .take(sl.saturating_sub(1))
+                .map(|l| l.len() + 1)
+                .sum();
+            let full_line =
+                source.full_line_str(TextSize::from(offset.min(source.len()) as u32));
+            return Ok(PyIterReturn::Return(make_token_tuple(
+                vm, tok_type, &tok_str, sl, sc as isize, el, ec as isize, full_line,
+            )));
+        }
+
+        let source_len = TextSize::from(source.len() as u32);
+
+        while *index < tokens.len() {
+            let token = tokens[*index];
+            *index += 1;
+            let kind = token.kind();
+            let range = token.range();
+
+            // Check for lexical indentation errors.
+            // Skip when source has tabs — ruff and CPython handle tab
+            // indentation differently (CPython uses tabsize=8), so ruff may
+            // report false IndentationErrors for valid mixed-tab code.
+            if !source.contains('\t') {
+                for err in errors.iter() {
+                    if !matches!(
+                        err.error,
+                        ParseErrorType::Lexical(LexicalErrorType::IndentationError)
+                    ) {
+                        continue;
+                    }
+                    if err.location.start() <= range.start()
+                        && range.start() < err.location.end()
+                    {
+                        return Err(raise_indentation_error(vm, err, source, line_index));
+                    }
+                }
+            }
+
+            if kind == TokenKind::EndOfFile {
+                continue;
+            }
+
+            if !extra_tokens
+                && matches!(kind, TokenKind::Comment | TokenKind::NonLogicalNewline)
+            {
+                continue;
+            }
+
+            let raw_type = token_kind_value(kind);
+            let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP
+            {
+                TOKEN_OP
+            } else {
+                raw_type
+            };
+
+            let (token_str, start_line, start_col, end_line, end_col, line_str) =
+                if kind == TokenKind::Dedent {
+                    let last_line = source.lines().count();
+                    let default_pos = if extra_tokens {
+                        (last_line + 1, 0)
+                    } else {
+                        (last_line, 0)
+                    };
+                    let (pos, dedent_line) =
+                        next_non_dedent_info(tokens, *index, source, line_index, default_pos);
+                    ("", pos.0, pos.1, pos.0, pos.1, dedent_line)
+                } else {
+                    let start_lc = line_index.line_column(range.start(), source);
+                    let start_line = start_lc.line.get();
+                    let start_col = start_lc.column.to_zero_indexed();
+                    let implicit_newline = range.start() >= source_len;
+                    let in_source = range.end() <= source_len;
+
+                    let (s, el, ec) = if kind == TokenKind::Newline {
+                        if extra_tokens {
+                            if implicit_newline {
+                                ("", start_line, start_col + 1)
+                            } else {
+                                let s = if source[range].starts_with('\r') {
+                                    "\r\n"
+                                } else {
+                                    "\n"
+                                };
+                                (s, start_line, start_col + s.len())
+                            }
+                        } else {
+                            ("", start_line, start_col)
+                        }
+                    } else if kind == TokenKind::NonLogicalNewline {
+                        let s = if in_source { &source[range] } else { "" };
+                        (s, start_line, start_col + s.len())
+                    } else {
+                        let end_lc = line_index.line_column(range.end(), source);
+                        let s = if in_source { &source[range] } else { "" };
+                        (s, end_lc.line.get(), end_lc.column.to_zero_indexed())
+                    };
+                    let line_str = source.full_line_str(range.start());
+                    (s, start_line, start_col, el, ec, line_str)
+                };
+
+            // Handle FSTRING_MIDDLE/TSTRING_MIDDLE brace unescaping
+            if matches!(kind, TokenKind::FStringMiddle | TokenKind::TStringMiddle)
+                && (token_str.contains("{{") || token_str.contains("}}"))
+            {
+                let mut parts =
+                    split_fstring_middle(token_str, token_type, start_line, start_col)
+                        .into_iter();
+                let (tt, ts, sl, sc, el, ec) = parts.next().unwrap();
+                let rest: Vec<_> = parts.collect();
+                for p in rest.into_iter().rev() {
+                    pending_fstring_parts.push(p);
+                }
+                return Ok(PyIterReturn::Return(make_token_tuple(
+                    vm, tt, &ts, sl, sc as isize, el, ec as isize, line_str,
+                )));
+            }
+
+            // After emitting a Rbrace inside an fstring, check if the
+            // next token is also Rbrace without an intervening FStringMiddle.
+            // CPython emits an empty FSTRING_MIDDLE in that position.
+            if kind == TokenKind::Rbrace
+                && tokens
+                    .get(*index)
+                    .is_some_and(|t| t.kind() == TokenKind::Rbrace)
+            {
+                let mid_type = find_fstring_middle_type(tokens, *index);
+                *pending_empty_fstring_middle = Some((
+                    mid_type,
+                    end_line,
+                    end_col,
+                    line_str.to_string(),
+                ));
+            }
+
+            return Ok(PyIterReturn::Return(make_token_tuple(
+                vm, token_type, token_str, start_line, start_col as isize, end_line,
+                end_col as isize, line_str,
+            )));
+        }
+
+        // Emit implicit NL before ENDMARKER if source
+        // doesn't end with newline and last token is Comment
+        if extra_tokens && core::mem::take(need_implicit_nl) {
+            let last_tok = tokens
+                .iter()
+                .rev()
+                .find(|t| t.kind() != TokenKind::EndOfFile);
+            if let Some(last) = last_tok.filter(|t| t.kind() == TokenKind::Comment) {
+                let end_lc = line_index.line_column(last.range().end(), source);
+                let nl_line = end_lc.line.get();
+                let nl_col = end_lc.column.to_zero_indexed();
+                return Ok(PyIterReturn::Return(make_token_tuple(
+                    vm,
+                    TOKEN_NL,
+                    "",
+                    nl_line,
+                    nl_col as isize,
+                    nl_line,
+                    nl_col as isize + 1,
+                    source.full_line_str(last.range().start()),
+                )));
+            }
+        }
+
+        // Check for unclosed brackets before ENDMARKER — CPython's tokenizer
+        // raises SyntaxError("EOF in multi-line statement") in this case.
+        {
+            let bracket_count: i32 = tokens
+                .iter()
+                .map(|t| match t.kind() {
+                    TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => 1,
+                    TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => -1,
+                    _ => 0,
+                })
+                .sum();
+            if bracket_count > 0 {
+                let last_line = source.lines().count();
+                return Err(raise_syntax_error(
+                    vm,
+                    "EOF in multi-line statement",
+                    last_line + 1,
+                    0,
+                ));
+            }
+        }
+
+        // All tokens consumed — emit ENDMARKER
+        let last_line = source.lines().count();
+        let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens {
+            (last_line + 1, 0, "")
+        } else {
+            let last_line_text = source.full_line_str(TextSize::from(
+                source.len().saturating_sub(1) as u32,
+            ));
+            (last_line, -1, last_line_text)
+        };
+
+        let result = make_token_tuple(
+            vm, TOKEN_ENDMARKER, "", em_line, em_col, em_line, em_col, em_line_str,
+        );
+        state.phase = TokenizerPhase::Done;
+        Ok(PyIterReturn::Return(result))
+    }
+
+    /// Determine whether to emit FSTRING_MIDDLE (60) or TSTRING_MIDDLE (63)
+    /// by looking back for the most recent FStringStart/TStringStart.
+    fn find_fstring_middle_type(tokens: &[Token], index: usize) -> u8 {
+        let mut depth = 0i32;
+        for i in (0..index).rev() {
+            match tokens[i].kind() {
+                TokenKind::FStringEnd | TokenKind::TStringEnd => depth += 1,
+                TokenKind::FStringStart => {
+                    if depth == 0 {
+                        return 60; // FSTRING_MIDDLE
+                    }
+                    depth -= 1;
+                }
+                TokenKind::TStringStart => {
+                    if depth == 0 {
+                        return 63; // TSTRING_MIDDLE
+                    }
+                    depth -= 1;
+                }
+                _ => {}
+            }
+        }
+        60 // default to FSTRING_MIDDLE
+    }
+
+    /// Find the next non-DEDENT token's position and source line.
+    /// Returns ((line, col), line_str).
+    fn next_non_dedent_info<'a>(
+        tokens: &[Token],
+        index: usize,
+        source: &'a str,
+        line_index: &LineIndex,
+        default_pos: (usize, usize),
+    ) -> ((usize, usize), &'a str) {
+        for future in &tokens[index..] {
+            match future.kind() {
+                TokenKind::Dedent => continue,
+                TokenKind::EndOfFile => return (default_pos, ""),
+                _ => {
+                    let flc = line_index.line_column(future.range().start(), source);
+                    let pos = (flc.line.get(), flc.column.to_zero_indexed());
+                    return (pos, source.full_line_str(future.range().start()));
+                }
+            }
+        }
+        (default_pos, "")
+    }
+
+    /// Raise a SyntaxError with the given message and position.
+    fn raise_syntax_error(
+        vm: &VirtualMachine,
+        msg: &str,
+        lineno: usize,
+        offset: usize,
+    ) -> rustpython_vm::builtins::PyBaseExceptionRef {
+        let exc = vm.new_exception_msg(
+            vm.ctx.exceptions.syntax_error.to_owned(),
+            msg.into(),
+        );
+        let obj = exc.as_object();
+        let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm);
+        let _ = obj.set_attr("lineno", vm.ctx.new_int(lineno), vm);
+        let _ = obj.set_attr("offset", vm.ctx.new_int(offset), vm);
+        let _ = obj.set_attr("filename", vm.ctx.new_str("<string>"), vm);
+        let _ = obj.set_attr("text", vm.ctx.none(), vm);
+        exc
+    }
+
+    /// Raise an IndentationError from a parse error.
+    fn raise_indentation_error(
+        vm: &VirtualMachine,
+        err: &ParseError,
+        source: &str,
+        line_index: &LineIndex,
+    ) -> rustpython_vm::builtins::PyBaseExceptionRef {
+        let err_lc = line_index.line_column(err.location.start(), source);
+        let err_line_text = source.full_line_str(err.location.start());
+        let err_text = err_line_text.trim_end_matches('\n').trim_end_matches('\r');
+        let msg = format!("{}", err.error);
+        let exc = vm.new_exception_msg(
+            vm.ctx.exceptions.indentation_error.to_owned(),
+            msg.clone().into(),
+        );
+        let obj = exc.as_object();
+        let _ = obj.set_attr("lineno", vm.ctx.new_int(err_lc.line.get()), vm);
+        let _ = obj.set_attr("offset", vm.ctx.new_int(err_text.len() as i64 + 1), vm);
+        let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm);
+        let _ = obj.set_attr("filename", vm.ctx.new_str("<string>"), vm);
+        let _ = obj.set_attr("text", vm.ctx.new_str(err_text), vm);
+        exc
+    }
+
+    /// Split an FSTRING_MIDDLE/TSTRING_MIDDLE token containing `{{`/`}}`
+    /// into multiple unescaped sub-tokens.
+    /// Returns vec of (type, string, start_line, start_col, end_line, end_col).
+    fn split_fstring_middle(
+        raw: &str,
+        token_type: u8,
+        start_line: usize,
+        start_col: usize,
+    ) -> Vec<(u8, String, usize, usize, usize, usize)> {
+        let mut parts = Vec::new();
+        let mut current = String::new();
+        // Track source position (line, col) — these correspond to the
+        // original source positions (with {{ and }} still doubled)
+        let mut cur_line = start_line;
+        let mut cur_col = start_col;
+        // Track the start position of the current accumulating part
+        let mut part_start_line = cur_line;
+        let mut part_start_col = cur_col;
+        let mut chars = raw.chars().peekable();
+
+        // Compute end position of the current accumulated text
+        let end_pos = |current: &str, start_line: usize, start_col: usize| -> (usize, usize) {
+            let mut el = start_line;
+            let mut ec = start_col;
+            for ch in current.chars() {
+                if ch == '\n' {
+                    el += 1;
+                    ec = 0;
+                } else {
+                    ec += ch.len_utf8();
+                }
+            }
+            (el, ec)
+        };
+
+        while let Some(ch) = chars.next() {
+            if ch == '{' && chars.peek() == Some(&'{') {
+                chars.next();
+                current.push('{');
+                cur_col += 2; // skip both {{ in source
+            } else if ch == '}' && chars.peek() == Some(&'}') {
+                chars.next();
+                // Flush accumulated text before }}
+                if !current.is_empty() {
+                    let (el, ec) = end_pos(&current, part_start_line, part_start_col);
+                    parts.push((
+                        token_type,
+                        core::mem::take(&mut current),
+                        part_start_line,
+                        part_start_col,
+                        el,
+                        ec,
+                    ));
+                }
+                // Emit unescaped '}' at source position of }}
+                parts.push((
+                    token_type,
+                    "}".to_string(),
+                    cur_line,
+                    cur_col,
+                    cur_line,
+                    cur_col + 1,
+                ));
+                cur_col += 2; // skip both }} in source
+                part_start_line = cur_line;
+                part_start_col = cur_col;
+            } else {
+                if current.is_empty() {
+                    part_start_line = cur_line;
+                    part_start_col = cur_col;
+                }
+                current.push(ch);
+                if ch == '\n' {
+                    cur_line += 1;
+                    cur_col = 0;
+                } else {
+                    cur_col += ch.len_utf8();
+                }
+            }
+        }
+
+        if !current.is_empty() {
+            let (el, ec) = end_pos(&current, part_start_line, part_start_col);
+            parts.push((token_type, current, part_start_line, part_start_col, el, ec));
+        }
+
+        parts
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn make_token_tuple(
+        vm: &VirtualMachine,
+        token_type: u8,
+        string: &str,
+        start_line: usize,
+        start_col: isize,
+        end_line: usize,
+        end_col: isize,
+        line: &str,
+    ) -> PyObjectRef {
+        vm.ctx
+            .new_tuple(vec![
+                token_type.to_pyobject(vm),
+                vm.ctx.new_str(string).into(),
+                vm.ctx
+                    .new_tuple(vec![start_line.to_pyobject(vm), start_col.to_pyobject(vm)])
+                    .into(),
+                vm.ctx
+                    .new_tuple(vec![end_line.to_pyobject(vm), end_col.to_pyobject(vm)])
+                    .into(),
+                vm.ctx.new_str(line).into(),
+            ])
+            .into()
+    }
+
+    #[derive(FromArgs)]
+    pub struct PyTokenizerIterArgs {
+        #[pyarg(positional)]
+        readline: ArgCallable,
+        #[pyarg(named)]
+        extra_tokens: bool,
+        #[pyarg(named, optional)]
+        encoding: Option<rustpython_vm::PyRef<PyStr>>,
+    }
+
+    #[derive(Clone, Debug)]
+    struct TokenizerState {
+        phase: TokenizerPhase,
+    }
+
+    #[derive(Clone, Debug)]
+    enum TokenizerPhase {
+        Reading {
+            source: String,
+        },
+        Yielding {
+            source: String,
+            tokens: Vec<Token>,
+            errors: Vec<ParseError>,
+            index: usize,
+            line_index: LineIndex,
+            need_implicit_nl: bool,
+            /// Pending sub-tokens from FSTRING_MIDDLE splitting
+            pending_fstring_parts: Vec<(u8, String, usize, usize, usize, usize)>,
+            /// Pending empty FSTRING_MIDDLE for format spec nesting:
+            /// (type, line, col, line_str)
+            pending_empty_fstring_middle: Option<(u8, usize, usize, String)>,
+        },
+        Done,
+    }
+
+    const fn token_kind_value(kind: TokenKind) -> u8 {
+        match kind {
+            TokenKind::EndOfFile => 0,
+            TokenKind::Name
+            | TokenKind::For
+            | TokenKind::In
+            | TokenKind::Pass
+            | TokenKind::Class
+            | TokenKind::And
+            | TokenKind::Is
+            | TokenKind::Raise
+            | TokenKind::True
+            | TokenKind::False
+            | TokenKind::Assert
+            | TokenKind::Try
+            | TokenKind::While
+            | TokenKind::Yield
+            | TokenKind::Lambda
+            | TokenKind::None
+            | TokenKind::Not
+            | TokenKind::Or
+            | TokenKind::Break
+            | TokenKind::Continue
+            | TokenKind::Global
+            | TokenKind::Nonlocal
+            | TokenKind::Return
+            | TokenKind::Except
+            | TokenKind::Import
+            | TokenKind::Case
+            | TokenKind::Match
+            | TokenKind::Type
+            | TokenKind::Await
+            | TokenKind::With
+            | TokenKind::Del
+            | TokenKind::Finally
+            | TokenKind::From
+            | TokenKind::Def
+            | TokenKind::If
+            | TokenKind::Else
+            | TokenKind::Elif
+            | TokenKind::As
+            | TokenKind::Async => 1,
+            TokenKind::Int | TokenKind::Complex | TokenKind::Float => 2,
+            TokenKind::String => 3,
+            TokenKind::Newline => 4,
+            TokenKind::NonLogicalNewline => TOKEN_NL,
+            TokenKind::Indent => 5,
+            TokenKind::Dedent => 6,
+            TokenKind::Lpar => 7,
+            TokenKind::Rpar => 8,
+            TokenKind::Lsqb => 9,
+            TokenKind::Rsqb => 10,
+            TokenKind::Colon => 11,
+            TokenKind::Comma => 12,
+            TokenKind::Semi => 13,
+            TokenKind::Plus => 14,
+            TokenKind::Minus => 15,
+            TokenKind::Star => 16,
+            TokenKind::Slash => 17,
+            TokenKind::Vbar => 18,
+            TokenKind::Amper => 19,
+            TokenKind::Less => 20,
+            TokenKind::Greater => 21,
+            TokenKind::Equal => 22,
+            TokenKind::Dot => 23,
+            TokenKind::Percent => 24,
+            TokenKind::Lbrace => 25,
+            TokenKind::Rbrace => 26,
+            TokenKind::EqEqual => 27,
+            TokenKind::NotEqual => 28,
+            TokenKind::LessEqual => 29,
+            TokenKind::GreaterEqual => 30,
+            TokenKind::Tilde => 31,
+            TokenKind::CircumFlex => 32,
+            TokenKind::LeftShift => 33,
+            TokenKind::RightShift => 34,
+            TokenKind::DoubleStar => 35,
+            TokenKind::PlusEqual => 36,
+            TokenKind::MinusEqual => 37,
+            TokenKind::StarEqual => 38,
+            TokenKind::SlashEqual => 39,
+            TokenKind::PercentEqual => 40,
+            TokenKind::AmperEqual => 41,
+            TokenKind::VbarEqual => 42,
+            TokenKind::CircumflexEqual => 43,
+            TokenKind::LeftShiftEqual => 44,
+            TokenKind::RightShiftEqual => 45,
+            TokenKind::DoubleStarEqual => 46,
+            TokenKind::DoubleSlash => 47,
+            TokenKind::DoubleSlashEqual => 48,
+            TokenKind::At => 49,
+            TokenKind::AtEqual => 50,
+            TokenKind::Rarrow => 51,
+            TokenKind::Ellipsis => 52,
+            TokenKind::ColonEqual => 53,
+            TokenKind::Exclamation => 54,
+            TokenKind::FStringStart => 59,
+            TokenKind::FStringMiddle => 60,
+            TokenKind::FStringEnd => 61,
+            TokenKind::Comment => TOKEN_COMMENT,
+            TokenKind::TStringStart => 62,
+            TokenKind::TStringMiddle => 63,
+            TokenKind::TStringEnd => 64,
+            TokenKind::IpyEscapeCommand
+            | TokenKind::Question
+            | TokenKind::Unknown => 67, // ERRORTOKEN
+        }
+    }
+}

crates/stdlib/src/lib.rs

@@ -49,7 +49,8 @@ mod pystruct;
 mod random;
 mod statistics;
 mod suggestions;
-mod tokenize;
+#[path = "_tokenize.rs"]
+mod _tokenize;
 // TODO: maybe make this an extension module, if we ever get those
 // mod re;
 #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))]
@@ -226,7 +227,7 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef>
         ssl::module_def(ctx),
         statistics::module_def(ctx),
         suggestions::module_def(ctx),
-        tokenize::module_def(ctx),
+        _tokenize::module_def(ctx),
         #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))]
         syslog::module_def(ctx),
         #[cfg(all(

crates/stdlib/src/tokenize.rs

@@ -1,391 +0,0 @@
-pub(crate) use _tokenize::module_def;
-
-#[pymodule]
-mod _tokenize {
-    use crate::{
-        common::lock::PyRwLock,
-        vm::{
-            AsObject, Py, PyPayload, PyResult, VirtualMachine,
-            builtins::{PyBytes, PyStr, PyType},
-            convert::ToPyObject,
-            function::ArgCallable,
-            protocol::PyIterReturn,
-            types::{Constructor, IterNext, Iterable, SelfIter},
-        },
-    };
-    use ruff_python_ast::PySourceType;
-    use ruff_python_ast::token::{Token, TokenKind, Tokens};
-    use ruff_python_parser::{ParseError, parse_unchecked_source};
-    use ruff_source_file::{LineIndex, LineRanges};
-    use ruff_text_size::{Ranged, TextRange};
-    use std::{cmp::Ordering, fmt};
-
-    /// `__import__("token").OP`
-    const TOKEN_OP: u8 = 55;
-
-    #[pyattr]
-    #[pyclass(name = "TokenizerIter")]
-    #[derive(PyPayload)]
-    pub struct PyTokenizerIter {
-        readline: ArgCallable, // TODO: This should be PyObject
-        extra_tokens: bool,
-        encoding: Option<String>,
-        state: PyRwLock<PyTokenizerIterState>,
-    }
-
-    impl PyTokenizerIter {
-        fn readline(&self, vm: &VirtualMachine) -> PyResult<String> {
-            // TODO: When `readline` is PyObject,
-            // we need to check if it's callable and raise a type error if it's not.
-            let raw_line = match self.readline.invoke((), vm) {
-                Ok(v) => v,
-                Err(err) => {
-                    if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) {
-                        return Ok(String::new());
-                    }
-                    return Err(err);
-                }
-            };
-            Ok(match &self.encoding {
-                Some(encoding) => {
-                    let bytes = raw_line
-                        .downcast::<PyBytes>()
-                        .map_err(|_| vm.new_type_error("readline() returned a non-bytes object"))?;
-                    vm.state
-                        .codec_registry
-                        .decode_text(bytes.into(), encoding, None, vm)
-                        .map(|s| s.to_string())?
-                }
-                None => raw_line
-                    .downcast::<PyStr>()
-                    .map(|s| s.to_string())
-                    .map_err(|_| vm.new_type_error("readline() returned a non-string object"))?,
-            })
-        }
-    }
-
-    impl fmt::Debug for PyTokenizerIter {
-        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            f.debug_struct("PyTokenizerIter")
-                .field("readline", &self.readline)
-                .field("encoding", &self.encoding)
-                .field("extra_tokens", &self.extra_tokens)
-                .finish()
-        }
-    }
-
-    #[pyclass(with(Constructor, Iterable, IterNext))]
-    impl PyTokenizerIter {}
-
-    impl Constructor for PyTokenizerIter {
-        type Args = PyTokenizerIterArgs;
-
-        fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> {
-            let Self::Args {
-                readline,
-                extra_tokens,
-                encoding,
-            } = args;
-
-            Ok(Self {
-                readline,
-                extra_tokens,
-                encoding: encoding.map(|s| s.to_string()),
-                state: PyRwLock::new(PyTokenizerIterState::default()),
-            })
-        }
-    }
-
-    impl SelfIter for PyTokenizerIter {}
-
-    impl IterNext for PyTokenizerIter {
-        fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
-            let mut state = {
-                let guard = zelf.state.read();
-                guard.clone()
-            };
-
-            if state.eof {
-                return Ok(PyIterReturn::StopIteration(None));
-            }
-
-            let token = loop {
-                // TODO: Check here for errors. Raise SyntaxError if needed
-
-                if let Some(tok) = state.next_token() {
-                    break tok;
-                }
-
-                let nline = zelf.readline(vm)?;
-                if nline.is_empty() {
-                    state.eof = true;
-                    *zelf.state.write() = state.clone();
-
-                    let line_num = &state.start().0;
-                    let out = vm
-                        .ctx
-                        .new_tuple(vec![
-                            token_kind_value(TokenKind::EndOfFile).to_pyobject(vm),
-                            vm.ctx.new_str("").into(),
-                            vm.ctx
-                                .new_tuple(vec![line_num.to_pyobject(vm), (-1).to_pyobject(vm)])
-                                .into(),
-                            vm.ctx
-                                .new_tuple(vec![line_num.to_pyobject(vm), (-1).to_pyobject(vm)])
-                                .into(),
-                            vm.ctx.new_str(state.current_line()).into(),
-                        ])
-                        .into();
-                    return Ok(PyIterReturn::Return(out));
-                }
-                state.push_line(&nline);
-            };
-
-            *zelf.state.write() = state.clone();
-
-            let token_kind = token.kind();
-            let token_value = if zelf.extra_tokens && token_kind.is_operator() {
-                TOKEN_OP
-            } else {
-                token_kind_value(token_kind)
-            };
-            let (start_x, start_y) = &state.start();
-            let (end_x, end_y) = &state.end();
-
-            let mut token_repr = &state.source[state.range()];
-            if !zelf.extra_tokens {
-                token_repr = token_repr.trim();
-            }
-
-            let out = vm
-                .ctx
-                .new_tuple(vec![
-                    token_value.to_pyobject(vm),
-                    vm.ctx.new_str(token_repr).into(),
-                    vm.ctx
-                        .new_tuple(vec![start_x.to_pyobject(vm), start_y.to_pyobject(vm)])
-                        .into(),
-                    vm.ctx
-                        .new_tuple(vec![end_x.to_pyobject(vm), end_y.to_pyobject(vm)])
-                        .into(),
-                    vm.ctx.new_str(state.current_line()).into(),
-                ])
-                .into();
-            Ok(PyIterReturn::Return(out))
-        }
-    }
-
-    #[derive(FromArgs)]
-    pub struct PyTokenizerIterArgs {
-        #[pyarg(positional)]
-        readline: ArgCallable,
-        #[pyarg(named)]
-        extra_tokens: bool,
-        #[pyarg(named, optional)]
-        encoding: Option<rustpython_vm::PyRef<PyStr>>,
-    }
-
-    #[derive(Clone, Debug)]
-    struct PyTokenizerIterState {
-        /// Source code.
-        source: String,
-        prev_token: Option<Token>,
-        /// Tokens of `source`.
-        tokens: Tokens,
-        /// Errors of `source`
-        errors: Vec<ParseError>,
-        /// LineIndex of `source`.
-        line_index: LineIndex,
-        /// Marker that says we already emitted EOF, and needs to stop iterating.
-        eof: bool,
-    }
-
-    impl PyTokenizerIterState {
-        fn push_line(&mut self, line: &str) {
-            self.source.push_str(line);
-
-            let parsed = parse_unchecked_source(&self.source, PySourceType::Python);
-            self.tokens = parsed.tokens().clone();
-            self.errors = parsed.errors().to_vec();
-            self.line_index = LineIndex::from_source_text(&self.source);
-        }
-
-        #[must_use]
-        fn current_line(&self) -> &str {
-            let (kind, range): (TokenKind, TextRange) = match self.prev_token {
-                Some(token) => token.as_tuple(),
-                None => (TokenKind::Unknown, TextRange::default()),
-            };
-
-            match kind {
-                TokenKind::Newline => self.source.full_line_str(range.start()),
-                _ => self.source.full_lines_str(range),
-            }
-        }
-
-        #[must_use]
-        fn next_token(&mut self) -> Option<Token> {
-            for token in self.tokens.iter() {
-                let (kind, range): (TokenKind, TextRange) = token.as_tuple();
-
-                if matches!(kind, TokenKind::NonLogicalNewline) {
-                    continue;
-                }
-
-                if matches!(range.ordering(self.range()), Ordering::Greater) {
-                    self.prev_token = Some(*token);
-                    return self.prev_token;
-                }
-            }
-
-            None
-        }
-
-        #[must_use]
-        fn range(&self) -> TextRange {
-            match self.prev_token {
-                Some(token) => token.range(),
-                None => TextRange::default(),
-            }
-        }
-
-        #[must_use]
-        fn start(&self) -> (usize, usize) {
-            let lc = self
-                .line_index
-                .line_column(self.range().start(), &self.source);
-            (lc.line.get(), lc.column.to_zero_indexed())
-        }
-
-        #[must_use]
-        fn end(&self) -> (usize, usize) {
-            let lc = self
-                .line_index
-                .line_column(self.range().end(), &self.source);
-            (lc.line.get(), lc.column.to_zero_indexed())
-        }
-    }
-
-    impl Default for PyTokenizerIterState {
-        fn default() -> Self {
-            const SOURCE: &str = "";
-            let parsed = parse_unchecked_source(SOURCE, PySourceType::Python);
-
-            Self {
-                source: SOURCE.to_owned(),
-                prev_token: None,
-                tokens: parsed.tokens().clone(),
-                errors: parsed.errors().to_vec(),
-                line_index: LineIndex::from_source_text(SOURCE),
-                eof: false,
-            }
-        }
-    }
-
-    const fn token_kind_value(kind: TokenKind) -> u8 {
-        match kind {
-            TokenKind::EndOfFile => 0,
-            TokenKind::Name
-            | TokenKind::For
-            | TokenKind::In
-            | TokenKind::Pass
-            | TokenKind::Class
-            | TokenKind::And
-            | TokenKind::Is
-            | TokenKind::Raise
-            | TokenKind::True
-            | TokenKind::False
-            | TokenKind::Assert
-            | TokenKind::Try
-            | TokenKind::While
-            | TokenKind::Yield
-            | TokenKind::Lambda
-            | TokenKind::None
-            | TokenKind::Not
-            | TokenKind::Or
-            | TokenKind::Break
-            | TokenKind::Continue
-            | TokenKind::Global
-            | TokenKind::Nonlocal
-            | TokenKind::Return
-            | TokenKind::Except
-            | TokenKind::Import
-            | TokenKind::Case
-            | TokenKind::Match
-            | TokenKind::Type
-            | TokenKind::Await
-            | TokenKind::With
-            | TokenKind::Del
-            | TokenKind::Finally
-            | TokenKind::From
-            | TokenKind::Def
-            | TokenKind::If
-            | TokenKind::Else
-            | TokenKind::Elif
-            | TokenKind::As
-            | TokenKind::Async => 1,
-            TokenKind::Int | TokenKind::Complex | TokenKind::Float => 2,
-            TokenKind::String => 3,
-            TokenKind::Newline | TokenKind::NonLogicalNewline => 4,
-            TokenKind::Indent => 5,
-            TokenKind::Dedent => 6,
-            TokenKind::Lpar => 7,
-            TokenKind::Rpar => 8,
-            TokenKind::Lsqb => 9,
-            TokenKind::Rsqb => 10,
-            TokenKind::Colon => 11,
-            TokenKind::Comma => 12,
-            TokenKind::Semi => 13,
-            TokenKind::Plus => 14,
-            TokenKind::Minus => 15,
-            TokenKind::Star => 16,
-            TokenKind::Slash => 17,
-            TokenKind::Vbar => 18,
-            TokenKind::Amper => 19,
-            TokenKind::Less => 20,
-            TokenKind::Greater => 21,
-            TokenKind::Equal => 22,
-            TokenKind::Dot => 23,
-            TokenKind::Percent => 24,
-            TokenKind::Lbrace => 25,
-            TokenKind::Rbrace => 26,
-            TokenKind::EqEqual => 27,
-            TokenKind::NotEqual => 28,
-            TokenKind::LessEqual => 29,
-            TokenKind::GreaterEqual => 30,
-            TokenKind::Tilde => 31,
-            TokenKind::CircumFlex => 32,
-            TokenKind::LeftShift => 33,
-            TokenKind::RightShift => 34,
-            TokenKind::DoubleStar => 35,
-            TokenKind::PlusEqual => 36,
-            TokenKind::MinusEqual => 37,
-            TokenKind::StarEqual => 38,
-            TokenKind::SlashEqual => 39,
-            TokenKind::PercentEqual => 40,
-            TokenKind::AmperEqual => 41,
-            TokenKind::VbarEqual => 42,
-            TokenKind::CircumflexEqual => 43,
-            TokenKind::LeftShiftEqual => 44,
-            TokenKind::RightShiftEqual => 45,
-            TokenKind::DoubleStarEqual => 46,
-            TokenKind::DoubleSlash => 47,
-            TokenKind::DoubleSlashEqual => 48,
-            TokenKind::At => 49,
-            TokenKind::AtEqual => 50,
-            TokenKind::Rarrow => 51,
-            TokenKind::Ellipsis => 52,
-            TokenKind::ColonEqual => 53,
-            TokenKind::Exclamation => 54,
-            TokenKind::FStringStart => 59,
-            TokenKind::FStringMiddle => 60,
-            TokenKind::FStringEnd => 61,
-            TokenKind::Comment => 62,
-            TokenKind::TStringStart => 62,  // 3.14 compatible
-            TokenKind::TStringMiddle => 63, // 3.14 compatible
-            TokenKind::TStringEnd => 64,    // 3.14 compatible
-            TokenKind::IpyEscapeCommand | TokenKind::Question => 0, // Ruff's specific
-            TokenKind::Unknown => 0,
-        }
-    }
-}