From 55737ede635ba35bcc60e61420bb3a5a0b5295ec Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 9 Mar 2026 11:55:31 +0900 Subject: [PATCH] Rewrite _tokenize with 2-phase model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace per-line reparsing with single-pass tokenization: - Read all lines via readline, parse once, yield tokens - Fix token type values (COMMENT=65, NL=66, OP=55) - Fix NEWLINE/NL end positions and implicit newline handling - Fix DEDENT positions via look-ahead to next non-DEDENT token - Handle FSTRING_MIDDLE brace unescaping ({{ → {, }} → }) - Emit implicit NL before ENDMARKER when source lacks trailing newline - Raise IndentationError from lexer errors - Remove 13 expectedFailure marks for now-passing tests --- Lib/test/test_tabnanny.py | 1 - Lib/test/test_tokenize.py | 13 - crates/stdlib/src/_tokenize.rs | 747 +++++++++++++++++++++++++++++++++ crates/stdlib/src/lib.rs | 5 +- crates/stdlib/src/tokenize.rs | 391 ----------------- 5 files changed, 750 insertions(+), 407 deletions(-) create mode 100644 crates/stdlib/src/_tokenize.rs delete mode 100644 crates/stdlib/src/tokenize.rs diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index 372be9eb8..d7a77eb26 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -316,7 +316,6 @@ class TestCommandLine(TestCase): self.assertListEqual(out.splitlines(), stdout.splitlines()) self.assertListEqual(err.splitlines(), stderr.splitlines()) - @unittest.expectedFailure # TODO: RUSTPYTHON; Should displays error when errored python file is given. def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index c10f80a72..394a87c36 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1167,7 +1167,6 @@ async def f(): DEDENT '' (7, 0) (7, 0) """) - @unittest.expectedFailure # TODO: RUSTPYTHON; + " NEWLINE '\\n' (4, 1) (4, 2)"] def test_newline_after_parenthesized_block_with_comment(self): self.check_tokenize('''\ [ @@ -1192,7 +1191,6 @@ async def f(): NAME 'x' (1, 3) (1, 4) """) - @unittest.expectedFailure # TODO: RUSTPYTHON; ' FSTRING_END "\'\'\'" (2, 68) (2, 71)'] def test_multiline_non_ascii_fstring(self): self.check_tokenize("""\ a = f''' @@ -1204,7 +1202,6 @@ a = f''' FSTRING_END "\'\'\'" (2, 68) (2, 71) """) - @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 696 characters long. Set self.maxDiff to None to see it. def test_multiline_non_ascii_fstring_with_expr(self): self.check_tokenize("""\ f''' @@ -2176,7 +2173,6 @@ if 1: # Two string literals on the same line self.check_roundtrip("'' ''") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_random_files(self): # Test roundtrip on random python modules. # pass the '-ucpu' option to process the full directory. @@ -2214,7 +2210,6 @@ if 1: class InvalidPythonTests(TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 1046 characters long. Set self.maxDiff to None to see it. def test_number_followed_by_name(self): # See issue #gh-105549 source = "2sin(x)" @@ -2254,7 +2249,6 @@ class CTokenizeTest(TestCase): ) self.assertEqual(result, expected.rstrip().splitlines()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) @@ -2386,7 +2380,6 @@ class CTokenizeTest(TestCase): NUMBER '3.14e159' (1, 4) (1, 12) """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string(self): self.check_tokenize('x = \'\'; y = ""', """\ @@ -2818,7 +2811,6 @@ f'''__{ NUMBER '1' (1, 22) (1, 23) """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_selector(self): self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\ @@ -2841,7 +2833,6 @@ f'''__{ RPAR ')' (2, 29) (2, 30) """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_method(self): self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ @@ -2859,7 +2850,6 @@ f'''__{ NAME 'pass' (2, 14) (2, 18) """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tabs(self): self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ @@ -3144,7 +3134,6 @@ async def f(): DEDENT '' (6, -1) (6, -1) """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode(self): self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\ @@ -3394,7 +3383,6 @@ class CommandLineTest(unittest.TestCase): with contextlib.redirect_stderr(StringIO()): _ = self.invoke_tokenize('--unknown') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_without_flag(self): # test 'python -m tokenize source.py' source = 'a = 1' @@ -3408,7 +3396,6 @@ class CommandLineTest(unittest.TestCase): ''' self.check_output(source, expect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_exact_flag(self): # test 'python -m tokenize -e/--exact source.py' source = 'a = 1' diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs new file mode 100644 index 000000000..13e40ff12 --- /dev/null +++ b/crates/stdlib/src/_tokenize.rs @@ -0,0 +1,747 @@ +pub(crate) use _tokenize::module_def; + +#[pymodule] +mod _tokenize { + use crate::{ + common::lock::PyRwLock, + vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyBytes, PyStr, PyType}, + convert::ToPyObject, + function::ArgCallable, + protocol::PyIterReturn, + types::{Constructor, IterNext, Iterable, SelfIter}, + }, + }; + use ruff_python_ast::PySourceType; + use ruff_python_ast::token::{Token, TokenKind}; + use ruff_python_parser::{ + LexicalErrorType, ParseError, ParseErrorType, parse_unchecked_source, + }; + use ruff_source_file::{LineIndex, LineRanges}; + use ruff_text_size::{Ranged, TextSize}; + use core::fmt; + + const TOKEN_ENDMARKER: u8 = 0; + const TOKEN_DEDENT: u8 = 6; + const TOKEN_OP: u8 = 55; + const TOKEN_COMMENT: u8 = 65; + const TOKEN_NL: u8 = 66; + + #[pyattr] + #[pyclass(name = "TokenizerIter")] + #[derive(PyPayload)] + pub struct PyTokenizerIter { + readline: ArgCallable, + extra_tokens: bool, + encoding: Option, + state: PyRwLock, + } + + impl PyTokenizerIter { + fn readline(&self, vm: &VirtualMachine) -> PyResult { + let raw_line = match self.readline.invoke((), vm) { + Ok(v) => v, + Err(err) => { + if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + return Ok(String::new()); + } + return Err(err); + } + }; + Ok(match &self.encoding { + Some(encoding) => { + let bytes = raw_line + .downcast::() + .map_err(|_| vm.new_type_error("readline() returned a non-bytes object"))?; + vm.state + .codec_registry + .decode_text(bytes.into(), encoding, None, vm) + .map(|s| s.to_string())? + } + None => raw_line + .downcast::() + .map(|s| s.to_string()) + .map_err(|_| vm.new_type_error("readline() returned a non-string object"))?, + }) + } + } + + impl fmt::Debug for PyTokenizerIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyTokenizerIter") + .field("extra_tokens", &self.extra_tokens) + .field("encoding", &self.encoding) + .finish() + } + } + + #[pyclass(with(Constructor, Iterable, IterNext))] + impl PyTokenizerIter {} + + impl Constructor for PyTokenizerIter { + type Args = PyTokenizerIterArgs; + + fn py_new(_cls: &Py, args: Self::Args, _vm: &VirtualMachine) -> PyResult { + let Self::Args { + readline, + extra_tokens, + encoding, + } = args; + + Ok(Self { + readline, + extra_tokens, + encoding: encoding.map(|s| s.to_string()), + state: PyRwLock::new(TokenizerState { + phase: TokenizerPhase::Reading { + source: String::new(), + }, + }), + }) + } + } + + impl SelfIter for PyTokenizerIter {} + + impl IterNext for PyTokenizerIter { + fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let mut state = zelf.state.read().clone(); + + loop { + match &mut state.phase { + TokenizerPhase::Reading { source } => { + let line = zelf.readline(vm)?; + if line.is_empty() { + let accumulated = core::mem::take(source); + let parsed = + parse_unchecked_source(&accumulated, PySourceType::Python); + let tokens: Vec = parsed.tokens().iter().copied().collect(); + let errors: Vec = parsed.errors().to_vec(); + let line_index = LineIndex::from_source_text(&accumulated); + let implicit_nl = !accumulated.ends_with('\n'); + state.phase = TokenizerPhase::Yielding { + source: accumulated, + tokens, + errors, + index: 0, + line_index, + need_implicit_nl: implicit_nl, + pending_fstring_parts: Vec::new(), + pending_empty_fstring_middle: None, + }; + } else { + source.push_str(&line); + } + } + TokenizerPhase::Yielding { .. } => { + let result = + emit_next_token(&mut state, zelf.extra_tokens, vm)?; + *zelf.state.write() = state; + return Ok(result); + } + TokenizerPhase::Done => { + return Ok(PyIterReturn::StopIteration(None)); + } + } + } + } + } + + /// Emit the next token from the Yielding phase. + fn emit_next_token( + state: &mut TokenizerState, + extra_tokens: bool, + vm: &VirtualMachine, + ) -> PyResult { + let TokenizerPhase::Yielding { + source, + tokens, + errors, + index, + line_index, + need_implicit_nl, + pending_fstring_parts, + pending_empty_fstring_middle, + } = &mut state.phase + else { + unreachable!() + }; + + // Emit pending empty FSTRING_MIDDLE (for format spec nesting) + if let Some((mid_type, mid_line, mid_col, mid_line_str)) = + pending_empty_fstring_middle.take() + { + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + mid_type, + "", + mid_line, + mid_col as isize, + mid_line, + mid_col as isize, + &mid_line_str, + ))); + } + + // Emit any pending fstring sub-tokens first + if let Some((tok_type, tok_str, sl, sc, el, ec)) = pending_fstring_parts.pop() { + let offset: usize = source + .lines() + .take(sl.saturating_sub(1)) + .map(|l| l.len() + 1) + .sum(); + let full_line = + source.full_line_str(TextSize::from(offset.min(source.len()) as u32)); + return Ok(PyIterReturn::Return(make_token_tuple( + vm, tok_type, &tok_str, sl, sc as isize, el, ec as isize, full_line, + ))); + } + + let source_len = TextSize::from(source.len() as u32); + + while *index < tokens.len() { + let token = tokens[*index]; + *index += 1; + let kind = token.kind(); + let range = token.range(); + + // Check for lexical indentation errors. + // Skip when source has tabs — ruff and CPython handle tab + // indentation differently (CPython uses tabsize=8), so ruff may + // report false IndentationErrors for valid mixed-tab code. + if !source.contains('\t') { + for err in errors.iter() { + if !matches!( + err.error, + ParseErrorType::Lexical(LexicalErrorType::IndentationError) + ) { + continue; + } + if err.location.start() <= range.start() + && range.start() < err.location.end() + { + return Err(raise_indentation_error(vm, err, source, line_index)); + } + } + } + + if kind == TokenKind::EndOfFile { + continue; + } + + if !extra_tokens + && matches!(kind, TokenKind::Comment | TokenKind::NonLogicalNewline) + { + continue; + } + + let raw_type = token_kind_value(kind); + let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP + { + TOKEN_OP + } else { + raw_type + }; + + let (token_str, start_line, start_col, end_line, end_col, line_str) = + if kind == TokenKind::Dedent { + let last_line = source.lines().count(); + let default_pos = if extra_tokens { + (last_line + 1, 0) + } else { + (last_line, 0) + }; + let (pos, dedent_line) = + next_non_dedent_info(tokens, *index, source, line_index, default_pos); + ("", pos.0, pos.1, pos.0, pos.1, dedent_line) + } else { + let start_lc = line_index.line_column(range.start(), source); + let start_line = start_lc.line.get(); + let start_col = start_lc.column.to_zero_indexed(); + let implicit_newline = range.start() >= source_len; + let in_source = range.end() <= source_len; + + let (s, el, ec) = if kind == TokenKind::Newline { + if extra_tokens { + if implicit_newline { + ("", start_line, start_col + 1) + } else { + let s = if source[range].starts_with('\r') { + "\r\n" + } else { + "\n" + }; + (s, start_line, start_col + s.len()) + } + } else { + ("", start_line, start_col) + } + } else if kind == TokenKind::NonLogicalNewline { + let s = if in_source { &source[range] } else { "" }; + (s, start_line, start_col + s.len()) + } else { + let end_lc = line_index.line_column(range.end(), source); + let s = if in_source { &source[range] } else { "" }; + (s, end_lc.line.get(), end_lc.column.to_zero_indexed()) + }; + let line_str = source.full_line_str(range.start()); + (s, start_line, start_col, el, ec, line_str) + }; + + // Handle FSTRING_MIDDLE/TSTRING_MIDDLE brace unescaping + if matches!(kind, TokenKind::FStringMiddle | TokenKind::TStringMiddle) + && (token_str.contains("{{") || token_str.contains("}}")) + { + let mut parts = + split_fstring_middle(token_str, token_type, start_line, start_col) + .into_iter(); + let (tt, ts, sl, sc, el, ec) = parts.next().unwrap(); + let rest: Vec<_> = parts.collect(); + for p in rest.into_iter().rev() { + pending_fstring_parts.push(p); + } + return Ok(PyIterReturn::Return(make_token_tuple( + vm, tt, &ts, sl, sc as isize, el, ec as isize, line_str, + ))); + } + + // After emitting a Rbrace inside an fstring, check if the + // next token is also Rbrace without an intervening FStringMiddle. + // CPython emits an empty FSTRING_MIDDLE in that position. + if kind == TokenKind::Rbrace + && tokens + .get(*index) + .is_some_and(|t| t.kind() == TokenKind::Rbrace) + { + let mid_type = find_fstring_middle_type(tokens, *index); + *pending_empty_fstring_middle = Some(( + mid_type, + end_line, + end_col, + line_str.to_string(), + )); + } + + return Ok(PyIterReturn::Return(make_token_tuple( + vm, token_type, token_str, start_line, start_col as isize, end_line, + end_col as isize, line_str, + ))); + } + + // Emit implicit NL before ENDMARKER if source + // doesn't end with newline and last token is Comment + if extra_tokens && core::mem::take(need_implicit_nl) { + let last_tok = tokens + .iter() + .rev() + .find(|t| t.kind() != TokenKind::EndOfFile); + if let Some(last) = last_tok.filter(|t| t.kind() == TokenKind::Comment) { + let end_lc = line_index.line_column(last.range().end(), source); + let nl_line = end_lc.line.get(); + let nl_col = end_lc.column.to_zero_indexed(); + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + TOKEN_NL, + "", + nl_line, + nl_col as isize, + nl_line, + nl_col as isize + 1, + source.full_line_str(last.range().start()), + ))); + } + } + + // Check for unclosed brackets before ENDMARKER — CPython's tokenizer + // raises SyntaxError("EOF in multi-line statement") in this case. + { + let bracket_count: i32 = tokens + .iter() + .map(|t| match t.kind() { + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => 1, + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => -1, + _ => 0, + }) + .sum(); + if bracket_count > 0 { + let last_line = source.lines().count(); + return Err(raise_syntax_error( + vm, + "EOF in multi-line statement", + last_line + 1, + 0, + )); + } + } + + // All tokens consumed — emit ENDMARKER + let last_line = source.lines().count(); + let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens { + (last_line + 1, 0, "") + } else { + let last_line_text = source.full_line_str(TextSize::from( + source.len().saturating_sub(1) as u32, + )); + (last_line, -1, last_line_text) + }; + + let result = make_token_tuple( + vm, TOKEN_ENDMARKER, "", em_line, em_col, em_line, em_col, em_line_str, + ); + state.phase = TokenizerPhase::Done; + Ok(PyIterReturn::Return(result)) + } + + /// Determine whether to emit FSTRING_MIDDLE (60) or TSTRING_MIDDLE (63) + /// by looking back for the most recent FStringStart/TStringStart. + fn find_fstring_middle_type(tokens: &[Token], index: usize) -> u8 { + let mut depth = 0i32; + for i in (0..index).rev() { + match tokens[i].kind() { + TokenKind::FStringEnd | TokenKind::TStringEnd => depth += 1, + TokenKind::FStringStart => { + if depth == 0 { + return 60; // FSTRING_MIDDLE + } + depth -= 1; + } + TokenKind::TStringStart => { + if depth == 0 { + return 63; // TSTRING_MIDDLE + } + depth -= 1; + } + _ => {} + } + } + 60 // default to FSTRING_MIDDLE + } + + /// Find the next non-DEDENT token's position and source line. + /// Returns ((line, col), line_str). + fn next_non_dedent_info<'a>( + tokens: &[Token], + index: usize, + source: &'a str, + line_index: &LineIndex, + default_pos: (usize, usize), + ) -> ((usize, usize), &'a str) { + for future in &tokens[index..] { + match future.kind() { + TokenKind::Dedent => continue, + TokenKind::EndOfFile => return (default_pos, ""), + _ => { + let flc = line_index.line_column(future.range().start(), source); + let pos = (flc.line.get(), flc.column.to_zero_indexed()); + return (pos, source.full_line_str(future.range().start())); + } + } + } + (default_pos, "") + } + + /// Raise a SyntaxError with the given message and position. + fn raise_syntax_error( + vm: &VirtualMachine, + msg: &str, + lineno: usize, + offset: usize, + ) -> rustpython_vm::builtins::PyBaseExceptionRef { + let exc = vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + msg.into(), + ); + let obj = exc.as_object(); + let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm); + let _ = obj.set_attr("lineno", vm.ctx.new_int(lineno), vm); + let _ = obj.set_attr("offset", vm.ctx.new_int(offset), vm); + let _ = obj.set_attr("filename", vm.ctx.new_str(""), vm); + let _ = obj.set_attr("text", vm.ctx.none(), vm); + exc + } + + /// Raise an IndentationError from a parse error. + fn raise_indentation_error( + vm: &VirtualMachine, + err: &ParseError, + source: &str, + line_index: &LineIndex, + ) -> rustpython_vm::builtins::PyBaseExceptionRef { + let err_lc = line_index.line_column(err.location.start(), source); + let err_line_text = source.full_line_str(err.location.start()); + let err_text = err_line_text.trim_end_matches('\n').trim_end_matches('\r'); + let msg = format!("{}", err.error); + let exc = vm.new_exception_msg( + vm.ctx.exceptions.indentation_error.to_owned(), + msg.clone().into(), + ); + let obj = exc.as_object(); + let _ = obj.set_attr("lineno", vm.ctx.new_int(err_lc.line.get()), vm); + let _ = obj.set_attr("offset", vm.ctx.new_int(err_text.len() as i64 + 1), vm); + let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm); + let _ = obj.set_attr("filename", vm.ctx.new_str(""), vm); + let _ = obj.set_attr("text", vm.ctx.new_str(err_text), vm); + exc + } + + /// Split an FSTRING_MIDDLE/TSTRING_MIDDLE token containing `{{`/`}}` + /// into multiple unescaped sub-tokens. + /// Returns vec of (type, string, start_line, start_col, end_line, end_col). + fn split_fstring_middle( + raw: &str, + token_type: u8, + start_line: usize, + start_col: usize, + ) -> Vec<(u8, String, usize, usize, usize, usize)> { + let mut parts = Vec::new(); + let mut current = String::new(); + // Track source position (line, col) — these correspond to the + // original source positions (with {{ and }} still doubled) + let mut cur_line = start_line; + let mut cur_col = start_col; + // Track the start position of the current accumulating part + let mut part_start_line = cur_line; + let mut part_start_col = cur_col; + let mut chars = raw.chars().peekable(); + + // Compute end position of the current accumulated text + let end_pos = |current: &str, start_line: usize, start_col: usize| -> (usize, usize) { + let mut el = start_line; + let mut ec = start_col; + for ch in current.chars() { + if ch == '\n' { + el += 1; + ec = 0; + } else { + ec += ch.len_utf8(); + } + } + (el, ec) + }; + + while let Some(ch) = chars.next() { + if ch == '{' && chars.peek() == Some(&'{') { + chars.next(); + current.push('{'); + cur_col += 2; // skip both {{ in source + } else if ch == '}' && chars.peek() == Some(&'}') { + chars.next(); + // Flush accumulated text before }} + if !current.is_empty() { + let (el, ec) = end_pos(¤t, part_start_line, part_start_col); + parts.push(( + token_type, + core::mem::take(&mut current), + part_start_line, + part_start_col, + el, + ec, + )); + } + // Emit unescaped '}' at source position of }} + parts.push(( + token_type, + "}".to_string(), + cur_line, + cur_col, + cur_line, + cur_col + 1, + )); + cur_col += 2; // skip both }} in source + part_start_line = cur_line; + part_start_col = cur_col; + } else { + if current.is_empty() { + part_start_line = cur_line; + part_start_col = cur_col; + } + current.push(ch); + if ch == '\n' { + cur_line += 1; + cur_col = 0; + } else { + cur_col += ch.len_utf8(); + } + } + } + + if !current.is_empty() { + let (el, ec) = end_pos(¤t, part_start_line, part_start_col); + parts.push((token_type, current, part_start_line, part_start_col, el, ec)); + } + + parts + } + + #[allow(clippy::too_many_arguments)] + fn make_token_tuple( + vm: &VirtualMachine, + token_type: u8, + string: &str, + start_line: usize, + start_col: isize, + end_line: usize, + end_col: isize, + line: &str, + ) -> PyObjectRef { + vm.ctx + .new_tuple(vec![ + token_type.to_pyobject(vm), + vm.ctx.new_str(string).into(), + vm.ctx + .new_tuple(vec![start_line.to_pyobject(vm), start_col.to_pyobject(vm)]) + .into(), + vm.ctx + .new_tuple(vec![end_line.to_pyobject(vm), end_col.to_pyobject(vm)]) + .into(), + vm.ctx.new_str(line).into(), + ]) + .into() + } + + #[derive(FromArgs)] + pub struct PyTokenizerIterArgs { + #[pyarg(positional)] + readline: ArgCallable, + #[pyarg(named)] + extra_tokens: bool, + #[pyarg(named, optional)] + encoding: Option>, + } + + #[derive(Clone, Debug)] + struct TokenizerState { + phase: TokenizerPhase, + } + + #[derive(Clone, Debug)] + enum TokenizerPhase { + Reading { + source: String, + }, + Yielding { + source: String, + tokens: Vec, + errors: Vec, + index: usize, + line_index: LineIndex, + need_implicit_nl: bool, + /// Pending sub-tokens from FSTRING_MIDDLE splitting + pending_fstring_parts: Vec<(u8, String, usize, usize, usize, usize)>, + /// Pending empty FSTRING_MIDDLE for format spec nesting: + /// (type, line, col, line_str) + pending_empty_fstring_middle: Option<(u8, usize, usize, String)>, + }, + Done, + } + + const fn token_kind_value(kind: TokenKind) -> u8 { + match kind { + TokenKind::EndOfFile => 0, + TokenKind::Name + | TokenKind::For + | TokenKind::In + | TokenKind::Pass + | TokenKind::Class + | TokenKind::And + | TokenKind::Is + | TokenKind::Raise + | TokenKind::True + | TokenKind::False + | TokenKind::Assert + | TokenKind::Try + | TokenKind::While + | TokenKind::Yield + | TokenKind::Lambda + | TokenKind::None + | TokenKind::Not + | TokenKind::Or + | TokenKind::Break + | TokenKind::Continue + | TokenKind::Global + | TokenKind::Nonlocal + | TokenKind::Return + | TokenKind::Except + | TokenKind::Import + | TokenKind::Case + | TokenKind::Match + | TokenKind::Type + | TokenKind::Await + | TokenKind::With + | TokenKind::Del + | TokenKind::Finally + | TokenKind::From + | TokenKind::Def + | TokenKind::If + | TokenKind::Else + | TokenKind::Elif + | TokenKind::As + | TokenKind::Async => 1, + TokenKind::Int | TokenKind::Complex | TokenKind::Float => 2, + TokenKind::String => 3, + TokenKind::Newline => 4, + TokenKind::NonLogicalNewline => TOKEN_NL, + TokenKind::Indent => 5, + TokenKind::Dedent => 6, + TokenKind::Lpar => 7, + TokenKind::Rpar => 8, + TokenKind::Lsqb => 9, + TokenKind::Rsqb => 10, + TokenKind::Colon => 11, + TokenKind::Comma => 12, + TokenKind::Semi => 13, + TokenKind::Plus => 14, + TokenKind::Minus => 15, + TokenKind::Star => 16, + TokenKind::Slash => 17, + TokenKind::Vbar => 18, + TokenKind::Amper => 19, + TokenKind::Less => 20, + TokenKind::Greater => 21, + TokenKind::Equal => 22, + TokenKind::Dot => 23, + TokenKind::Percent => 24, + TokenKind::Lbrace => 25, + TokenKind::Rbrace => 26, + TokenKind::EqEqual => 27, + TokenKind::NotEqual => 28, + TokenKind::LessEqual => 29, + TokenKind::GreaterEqual => 30, + TokenKind::Tilde => 31, + TokenKind::CircumFlex => 32, + TokenKind::LeftShift => 33, + TokenKind::RightShift => 34, + TokenKind::DoubleStar => 35, + TokenKind::PlusEqual => 36, + TokenKind::MinusEqual => 37, + TokenKind::StarEqual => 38, + TokenKind::SlashEqual => 39, + TokenKind::PercentEqual => 40, + TokenKind::AmperEqual => 41, + TokenKind::VbarEqual => 42, + TokenKind::CircumflexEqual => 43, + TokenKind::LeftShiftEqual => 44, + TokenKind::RightShiftEqual => 45, + TokenKind::DoubleStarEqual => 46, + TokenKind::DoubleSlash => 47, + TokenKind::DoubleSlashEqual => 48, + TokenKind::At => 49, + TokenKind::AtEqual => 50, + TokenKind::Rarrow => 51, + TokenKind::Ellipsis => 52, + TokenKind::ColonEqual => 53, + TokenKind::Exclamation => 54, + TokenKind::FStringStart => 59, + TokenKind::FStringMiddle => 60, + TokenKind::FStringEnd => 61, + TokenKind::Comment => TOKEN_COMMENT, + TokenKind::TStringStart => 62, + TokenKind::TStringMiddle => 63, + TokenKind::TStringEnd => 64, + TokenKind::IpyEscapeCommand + | TokenKind::Question + | TokenKind::Unknown => 67, // ERRORTOKEN + } + } +} diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index 04aa623d1..4c06eea9e 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -49,7 +49,8 @@ mod pystruct; mod random; mod statistics; mod suggestions; -mod tokenize; +#[path = "_tokenize.rs"] +mod _tokenize; // TODO: maybe make this an extension module, if we ever get those // mod re; #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] @@ -226,7 +227,7 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> ssl::module_def(ctx), statistics::module_def(ctx), suggestions::module_def(ctx), - tokenize::module_def(ctx), + _tokenize::module_def(ctx), #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] syslog::module_def(ctx), #[cfg(all( diff --git a/crates/stdlib/src/tokenize.rs b/crates/stdlib/src/tokenize.rs deleted file mode 100644 index 33667a203..000000000 --- a/crates/stdlib/src/tokenize.rs +++ /dev/null @@ -1,391 +0,0 @@ -pub(crate) use _tokenize::module_def; - -#[pymodule] -mod _tokenize { - use crate::{ - common::lock::PyRwLock, - vm::{ - AsObject, Py, PyPayload, PyResult, VirtualMachine, - builtins::{PyBytes, PyStr, PyType}, - convert::ToPyObject, - function::ArgCallable, - protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, SelfIter}, - }, - }; - use ruff_python_ast::PySourceType; - use ruff_python_ast::token::{Token, TokenKind, Tokens}; - use ruff_python_parser::{ParseError, parse_unchecked_source}; - use ruff_source_file::{LineIndex, LineRanges}; - use ruff_text_size::{Ranged, TextRange}; - use std::{cmp::Ordering, fmt}; - - /// `__import__("token").OP` - const TOKEN_OP: u8 = 55; - - #[pyattr] - #[pyclass(name = "TokenizerIter")] - #[derive(PyPayload)] - pub struct PyTokenizerIter { - readline: ArgCallable, // TODO: This should be PyObject - extra_tokens: bool, - encoding: Option, - state: PyRwLock, - } - - impl PyTokenizerIter { - fn readline(&self, vm: &VirtualMachine) -> PyResult { - // TODO: When `readline` is PyObject, - // we need to check if it's callable and raise a type error if it's not. - let raw_line = match self.readline.invoke((), vm) { - Ok(v) => v, - Err(err) => { - if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) { - return Ok(String::new()); - } - return Err(err); - } - }; - Ok(match &self.encoding { - Some(encoding) => { - let bytes = raw_line - .downcast::() - .map_err(|_| vm.new_type_error("readline() returned a non-bytes object"))?; - vm.state - .codec_registry - .decode_text(bytes.into(), encoding, None, vm) - .map(|s| s.to_string())? - } - None => raw_line - .downcast::() - .map(|s| s.to_string()) - .map_err(|_| vm.new_type_error("readline() returned a non-string object"))?, - }) - } - } - - impl fmt::Debug for PyTokenizerIter { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PyTokenizerIter") - .field("readline", &self.readline) - .field("encoding", &self.encoding) - .field("extra_tokens", &self.extra_tokens) - .finish() - } - } - - #[pyclass(with(Constructor, Iterable, IterNext))] - impl PyTokenizerIter {} - - impl Constructor for PyTokenizerIter { - type Args = PyTokenizerIterArgs; - - fn py_new(_cls: &Py, args: Self::Args, _vm: &VirtualMachine) -> PyResult { - let Self::Args { - readline, - extra_tokens, - encoding, - } = args; - - Ok(Self { - readline, - extra_tokens, - encoding: encoding.map(|s| s.to_string()), - state: PyRwLock::new(PyTokenizerIterState::default()), - }) - } - } - - impl SelfIter for PyTokenizerIter {} - - impl IterNext for PyTokenizerIter { - fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let mut state = { - let guard = zelf.state.read(); - guard.clone() - }; - - if state.eof { - return Ok(PyIterReturn::StopIteration(None)); - } - - let token = loop { - // TODO: Check here for errors. Raise SyntaxError if needed - - if let Some(tok) = state.next_token() { - break tok; - } - - let nline = zelf.readline(vm)?; - if nline.is_empty() { - state.eof = true; - *zelf.state.write() = state.clone(); - - let line_num = &state.start().0; - let out = vm - .ctx - .new_tuple(vec![ - token_kind_value(TokenKind::EndOfFile).to_pyobject(vm), - vm.ctx.new_str("").into(), - vm.ctx - .new_tuple(vec![line_num.to_pyobject(vm), (-1).to_pyobject(vm)]) - .into(), - vm.ctx - .new_tuple(vec![line_num.to_pyobject(vm), (-1).to_pyobject(vm)]) - .into(), - vm.ctx.new_str(state.current_line()).into(), - ]) - .into(); - return Ok(PyIterReturn::Return(out)); - } - state.push_line(&nline); - }; - - *zelf.state.write() = state.clone(); - - let token_kind = token.kind(); - let token_value = if zelf.extra_tokens && token_kind.is_operator() { - TOKEN_OP - } else { - token_kind_value(token_kind) - }; - let (start_x, start_y) = &state.start(); - let (end_x, end_y) = &state.end(); - - let mut token_repr = &state.source[state.range()]; - if !zelf.extra_tokens { - token_repr = token_repr.trim(); - } - - let out = vm - .ctx - .new_tuple(vec![ - token_value.to_pyobject(vm), - vm.ctx.new_str(token_repr).into(), - vm.ctx - .new_tuple(vec![start_x.to_pyobject(vm), start_y.to_pyobject(vm)]) - .into(), - vm.ctx - .new_tuple(vec![end_x.to_pyobject(vm), end_y.to_pyobject(vm)]) - .into(), - vm.ctx.new_str(state.current_line()).into(), - ]) - .into(); - Ok(PyIterReturn::Return(out)) - } - } - - #[derive(FromArgs)] - pub struct PyTokenizerIterArgs { - #[pyarg(positional)] - readline: ArgCallable, - #[pyarg(named)] - extra_tokens: bool, - #[pyarg(named, optional)] - encoding: Option>, - } - - #[derive(Clone, Debug)] - struct PyTokenizerIterState { - /// Source code. - source: String, - prev_token: Option, - /// Tokens of `source`. - tokens: Tokens, - /// Errors of `source` - errors: Vec, - /// LineIndex of `source`. - line_index: LineIndex, - /// Marker that says we already emitted EOF, and needs to stop iterating. - eof: bool, - } - - impl PyTokenizerIterState { - fn push_line(&mut self, line: &str) { - self.source.push_str(line); - - let parsed = parse_unchecked_source(&self.source, PySourceType::Python); - self.tokens = parsed.tokens().clone(); - self.errors = parsed.errors().to_vec(); - self.line_index = LineIndex::from_source_text(&self.source); - } - - #[must_use] - fn current_line(&self) -> &str { - let (kind, range): (TokenKind, TextRange) = match self.prev_token { - Some(token) => token.as_tuple(), - None => (TokenKind::Unknown, TextRange::default()), - }; - - match kind { - TokenKind::Newline => self.source.full_line_str(range.start()), - _ => self.source.full_lines_str(range), - } - } - - #[must_use] - fn next_token(&mut self) -> Option { - for token in self.tokens.iter() { - let (kind, range): (TokenKind, TextRange) = token.as_tuple(); - - if matches!(kind, TokenKind::NonLogicalNewline) { - continue; - } - - if matches!(range.ordering(self.range()), Ordering::Greater) { - self.prev_token = Some(*token); - return self.prev_token; - } - } - - None - } - - #[must_use] - fn range(&self) -> TextRange { - match self.prev_token { - Some(token) => token.range(), - None => TextRange::default(), - } - } - - #[must_use] - fn start(&self) -> (usize, usize) { - let lc = self - .line_index - .line_column(self.range().start(), &self.source); - (lc.line.get(), lc.column.to_zero_indexed()) - } - - #[must_use] - fn end(&self) -> (usize, usize) { - let lc = self - .line_index - .line_column(self.range().end(), &self.source); - (lc.line.get(), lc.column.to_zero_indexed()) - } - } - - impl Default for PyTokenizerIterState { - fn default() -> Self { - const SOURCE: &str = ""; - let parsed = parse_unchecked_source(SOURCE, PySourceType::Python); - - Self { - source: SOURCE.to_owned(), - prev_token: None, - tokens: parsed.tokens().clone(), - errors: parsed.errors().to_vec(), - line_index: LineIndex::from_source_text(SOURCE), - eof: false, - } - } - } - - const fn token_kind_value(kind: TokenKind) -> u8 { - match kind { - TokenKind::EndOfFile => 0, - TokenKind::Name - | TokenKind::For - | TokenKind::In - | TokenKind::Pass - | TokenKind::Class - | TokenKind::And - | TokenKind::Is - | TokenKind::Raise - | TokenKind::True - | TokenKind::False - | TokenKind::Assert - | TokenKind::Try - | TokenKind::While - | TokenKind::Yield - | TokenKind::Lambda - | TokenKind::None - | TokenKind::Not - | TokenKind::Or - | TokenKind::Break - | TokenKind::Continue - | TokenKind::Global - | TokenKind::Nonlocal - | TokenKind::Return - | TokenKind::Except - | TokenKind::Import - | TokenKind::Case - | TokenKind::Match - | TokenKind::Type - | TokenKind::Await - | TokenKind::With - | TokenKind::Del - | TokenKind::Finally - | TokenKind::From - | TokenKind::Def - | TokenKind::If - | TokenKind::Else - | TokenKind::Elif - | TokenKind::As - | TokenKind::Async => 1, - TokenKind::Int | TokenKind::Complex | TokenKind::Float => 2, - TokenKind::String => 3, - TokenKind::Newline | TokenKind::NonLogicalNewline => 4, - TokenKind::Indent => 5, - TokenKind::Dedent => 6, - TokenKind::Lpar => 7, - TokenKind::Rpar => 8, - TokenKind::Lsqb => 9, - TokenKind::Rsqb => 10, - TokenKind::Colon => 11, - TokenKind::Comma => 12, - TokenKind::Semi => 13, - TokenKind::Plus => 14, - TokenKind::Minus => 15, - TokenKind::Star => 16, - TokenKind::Slash => 17, - TokenKind::Vbar => 18, - TokenKind::Amper => 19, - TokenKind::Less => 20, - TokenKind::Greater => 21, - TokenKind::Equal => 22, - TokenKind::Dot => 23, - TokenKind::Percent => 24, - TokenKind::Lbrace => 25, - TokenKind::Rbrace => 26, - TokenKind::EqEqual => 27, - TokenKind::NotEqual => 28, - TokenKind::LessEqual => 29, - TokenKind::GreaterEqual => 30, - TokenKind::Tilde => 31, - TokenKind::CircumFlex => 32, - TokenKind::LeftShift => 33, - TokenKind::RightShift => 34, - TokenKind::DoubleStar => 35, - TokenKind::PlusEqual => 36, - TokenKind::MinusEqual => 37, - TokenKind::StarEqual => 38, - TokenKind::SlashEqual => 39, - TokenKind::PercentEqual => 40, - TokenKind::AmperEqual => 41, - TokenKind::VbarEqual => 42, - TokenKind::CircumflexEqual => 43, - TokenKind::LeftShiftEqual => 44, - TokenKind::RightShiftEqual => 45, - TokenKind::DoubleStarEqual => 46, - TokenKind::DoubleSlash => 47, - TokenKind::DoubleSlashEqual => 48, - TokenKind::At => 49, - TokenKind::AtEqual => 50, - TokenKind::Rarrow => 51, - TokenKind::Ellipsis => 52, - TokenKind::ColonEqual => 53, - TokenKind::Exclamation => 54, - TokenKind::FStringStart => 59, - TokenKind::FStringMiddle => 60, - TokenKind::FStringEnd => 61, - TokenKind::Comment => 62, - TokenKind::TStringStart => 62, // 3.14 compatible - TokenKind::TStringMiddle => 63, // 3.14 compatible - TokenKind::TStringEnd => 64, // 3.14 compatible - TokenKind::IpyEscapeCommand | TokenKind::Question => 0, // Ruff's specific - TokenKind::Unknown => 0, - } - } -}