From 8c73f3cb30d16fc356acdd51171fd9e1aeeec69b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 25 Jan 2026 16:54:11 +0900 Subject: [PATCH] emit RESUME --- Lib/test/test_compile.py | 2 - Lib/test/test_peepholer.py | 1 - crates/codegen/src/compile.rs | 78 ++++++----- crates/codegen/src/ir.rs | 126 +++++++++++++++--- ...thon_codegen__compile__tests__if_ands.snap | 17 +-- ...hon_codegen__compile__tests__if_mixed.snap | 21 +-- ...ython_codegen__compile__tests__if_ors.snap | 17 +-- ...pile__tests__nested_double_async_with.snap | 12 +- 8 files changed, 187 insertions(+), 87 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 629ab19f5..5f86cf38f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -152,8 +152,6 @@ if 1: pass""" compile(s, "", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure # This test is probably specific to CPython and may not generalize # to other implementations. We are trying to ensure that when # the first line of code starts after 256, correct line numbers diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 87ebc3e39..8594ed875 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -147,7 +147,6 @@ class TestTranforms(BytecodeTestCase): self.assertNotInBytecode(code, 'UNPACK_SEQUENCE') self.check_lnotab(code) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_folding_of_tuples_of_constants(self): for line, elem in ( ('a = 1,2,3', (1, 2, 3)), diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 9d04b8396..8ab0e6223 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -997,12 +997,6 @@ impl Compiler { key: usize, // In RustPython, we use the index in symbol_table_stack as key lineno: u32, ) -> CompileResult<()> { - // Create location - let location = SourceLocation { - line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), - character_offset: OneIndexed::MIN, - }; - // Allocate a new compiler unit // In Rust, we'll create the structure directly @@ -1148,40 +1142,50 @@ impl Compiler { self.set_qualname(); } - // Emit RESUME instruction - let _resume_loc = if scope_type == CompilerScope::Module { - // Module scope starts with lineno 0 - SourceLocation { - line: OneIndexed::MIN, - character_offset: OneIndexed::MIN, - } - } else { - location - }; + // Emit RESUME (handles async preamble and module lineno 0) + // CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module + self.emit_resume_for_scope(scope_type, lineno); - // Set the source range for the RESUME instruction - // For now, just use an empty range at the beginning - self.current_source_range = TextRange::default(); + Ok(()) + } + /// Emit RESUME instruction with proper handling for async preamble and module lineno. + /// codegen_enter_scope equivalent for RESUME emission. + fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) { // For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME if scope_type == CompilerScope::AsyncFunction { emit!(self, Instruction::ReturnGenerator); emit!(self, Instruction::PopTop); } - emit!( - self, - Instruction::Resume { - arg: bytecode::ResumeType::AtFuncStart as u32 + // CPython: LOCATION(lineno, lineno, 0, 0) + // Module scope: loc.lineno = 0 (before the first line) + let lineno_override = if scope_type == CompilerScope::Module { + Some(0) + } else { + None + }; + + // Use lineno for location (col = 0 as in CPython) + let location = SourceLocation { + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::MIN, // col = 0 + }; + let end_location = location; // end_lineno = lineno, end_col = 0 + let except_handler = self.current_except_handler(); + + self.current_block().instructions.push(ir::InstructionInfo { + instr: Instruction::Resume { + arg: OpArgMarker::marker(), } - ); - - if scope_type == CompilerScope::Module { - // This would be loc.lineno = -1 in CPython - // We handle this differently in RustPython - } - - Ok(()) + .into(), + arg: OpArg(bytecode::ResumeType::AtFuncStart as u32), + target: BlockIdx::NULL, + location, + end_location, + except_handler, + lineno_override, + }); } fn push_output( @@ -1750,6 +1754,8 @@ impl Compiler { self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + let (doc, statements) = split_doc(&body.body, &self.opts); if let Some(value) = doc { self.emit_load_const(ConstantData::Str { @@ -1795,6 +1801,8 @@ impl Compiler { self.future_annotations = symbol_table.future_annotations; self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + // Handle annotations based on future_annotations flag if Self::find_ann(body) { if self.future_annotations { @@ -1858,6 +1866,7 @@ impl Compiler { symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); self.compile_statements(body)?; @@ -1887,6 +1896,8 @@ impl Compiler { symbol_table: SymbolTable, ) -> CompileResult<()> { self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + self.compile_expression(&expression.body)?; self.emit_return_value(); Ok(()) @@ -2631,7 +2642,7 @@ impl Compiler { // Get the current symbol table let key = self.symbol_table_stack.len() - 1; - let lineno = expr.range().start().to_u32(); + let lineno = self.get_source_line_number().get().to_u32(); // Enter scope with the type parameter name self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?; @@ -7764,6 +7775,7 @@ impl Compiler { location, end_location, except_handler, + lineno_override: None, }); } @@ -7788,8 +7800,6 @@ impl Compiler { } fn emit_load_const(&mut self, constant: ConstantData) { - // Always add to co_consts and emit LOAD_CONST. - // Optimization pass converts LOAD_CONST to LOAD_SMALL_INT where appropriate. let idx = self.arg_constant(constant); self.emit_arg(idx, |idx| Instruction::LoadConst { idx }) } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 3fe8f2e37..fa662e185 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -14,6 +14,15 @@ use rustpython_compiler_core::{ varint::{write_signed_varint, write_varint}, }; +/// Location info for linetable generation (allows line 0 for RESUME) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LineTableLocation { + line: i32, + end_line: i32, + col: i32, + end_col: i32, +} + /// Metadata for a code unit // = _PyCompile_CodeUnitMetadata #[derive(Clone, Debug)] @@ -94,6 +103,8 @@ pub struct InstructionInfo { pub location: SourceLocation, pub end_location: SourceLocation, pub except_handler: Option, + /// Override line number for linetable (e.g., line 0 for module RESUME) + pub lineno_override: Option, } /// Exception handler information for an instruction. @@ -161,11 +172,16 @@ impl CodeInfo { mut self, opts: &crate::compile::CompileOpts, ) -> crate::InternalResult { - // Always fold tuple constants and convert to LOAD_SMALL_INT - // 1. fold_tuple_of_constants - fold constant sequences into tuples - // 2. maybe_instr_make_load_smallint - convert LOAD_CONST to LOAD_SMALL_INT + // Always fold tuple constants self.fold_tuple_constants(); - self.convert_to_load_small_int(); + // Python only applies LOAD_SMALL_INT conversion to module-level code + // (not inside functions). Module code lacks OPTIMIZED flag. + // Note: RustPython incorrectly sets NEWLOCALS on modules, so only check OPTIMIZED + let is_module_level = !self.flags.contains(CodeFlags::OPTIMIZED); + if is_module_level { + self.convert_to_load_small_int(); + } + self.remove_unused_consts(); self.remove_nops(); if opts.optimize > 0 { @@ -209,6 +225,7 @@ impl CodeInfo { let mut instructions = Vec::new(); let mut locations = Vec::new(); + let mut linetable_locations: Vec = Vec::new(); // convert_pseudo_ops: instructions before the main loop for block in blocks @@ -315,6 +332,16 @@ impl CodeInfo { (info.location, info.end_location), info.arg.instr_size(), )); + // Collect linetable locations with lineno_override support + let lt_loc = LineTableLocation { + line: info + .lineno_override + .unwrap_or_else(|| info.location.line.get() as i32), + end_line: info.end_location.line.get() as i32, + col: info.location.character_offset.to_zero_indexed() as i32, + end_col: info.end_location.character_offset.to_zero_indexed() as i32, + }; + linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) @@ -330,12 +357,13 @@ impl CodeInfo { } instructions.clear(); - locations.clear() + locations.clear(); + linetable_locations.clear(); } - // Generate linetable from locations + // Generate linetable from linetable_locations (supports line 0 for RESUME) let linetable = generate_linetable( - &locations, + &linetable_locations, first_line_number.get() as i32, opts.debug_ranges, ); @@ -578,12 +606,72 @@ impl CodeInfo { }; // Check if it's in small int range: -5 to 256 (_PY_IS_SMALL_INT) - if let Some(small) = value.to_i32() { - if (-5..=256).contains(&small) { - // Convert to LOAD_CONST to LOAD_SMALL_INT - instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into(); - // The arg is the i32 value stored as u32 (two's complement) - instr.arg = OpArg(small as u32); + if let Some(small) = value.to_i32().filter(|v| (-5..=256).contains(v)) { + // Convert LOAD_CONST to LOAD_SMALL_INT + instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into(); + // The arg is the i32 value stored as u32 (two's complement) + instr.arg = OpArg(small as u32); + } + } + } + } + + /// Remove constants that are no longer referenced by LOAD_CONST instructions. + /// remove_unused_consts + fn remove_unused_consts(&mut self) { + let nconsts = self.metadata.consts.len(); + if nconsts == 0 { + return; + } + + // Mark used constants + // The first constant (index 0) is always kept (may be docstring) + let mut used = vec![false; nconsts]; + used[0] = true; + + for block in &self.blocks { + for instr in &block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let idx = instr.arg.0 as usize; + if idx < nconsts { + used[idx] = true; + } + } + } + } + + // Check if any constants can be removed + let n_used: usize = used.iter().filter(|&&u| u).count(); + if n_used == nconsts { + return; // Nothing to remove + } + + // Build old_to_new index mapping + let mut old_to_new = vec![0usize; nconsts]; + let mut new_idx = 0usize; + for (old_idx, &is_used) in used.iter().enumerate() { + if is_used { + old_to_new[old_idx] = new_idx; + new_idx += 1; + } + } + + // Build new consts list + let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect(); + self.metadata.consts.clear(); + for (old_idx, constant) in old_consts.into_iter().enumerate() { + if used[old_idx] { + self.metadata.consts.insert(constant); + } + } + + // Update LOAD_CONST instruction arguments + for block in &mut self.blocks { + for instr in &mut block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let old_idx = instr.arg.0 as usize; + if old_idx < nconsts { + instr.arg = OpArg(old_to_new[old_idx] as u32); } } } @@ -759,7 +847,7 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator + ' /// Generate Python 3.11+ format linetable from source locations fn generate_linetable( - locations: &[(SourceLocation, SourceLocation)], + locations: &[LineTableLocation], first_line: i32, debug_ranges: bool, ) -> Box<[u8]> { @@ -774,7 +862,7 @@ fn generate_linetable( let mut i = 0; while i < locations.len() { - let (loc, end_loc) = &locations[i]; + let loc = &locations[i]; // Count consecutive instructions with the same location let mut length = 1; @@ -787,8 +875,8 @@ fn generate_linetable( let entry_length = length.min(8); // Get line information - let line = loc.line.get() as i32; - let end_line = end_loc.line.get() as i32; + let line = loc.line; + let end_line = loc.end_line; let line_delta = line - prev_line; let end_line_delta = end_line - line; @@ -808,8 +896,8 @@ fn generate_linetable( } // Get column information (only when debug_ranges is enabled) - let col = loc.character_offset.to_zero_indexed() as i32; - let end_col = end_loc.character_offset.to_zero_indexed() as i32; + let col = loc.col; + let end_col = loc.end_col; // Choose the appropriate encoding based on line delta and column info if line_delta == 0 && end_line_delta == 0 { diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap index fb052cb26..26583f5da 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap @@ -2,12 +2,13 @@ source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" --- - 1 0 LOAD_CONST (True) - 1 POP_JUMP_IF_FALSE (6) - 2 LOAD_CONST (False) - 3 POP_JUMP_IF_FALSE (6) - 4 LOAD_CONST (False) - 5 POP_JUMP_IF_FALSE (6) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (7) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_FALSE (7) + 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (7) - 2 >> 6 LOAD_CONST (None) - 7 RETURN_VALUE + 2 >> 7 LOAD_CONST (None) + 8 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap index 021f88e3d..21976b257 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap @@ -2,14 +2,15 @@ source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" --- - 1 0 LOAD_CONST (True) - 1 POP_JUMP_IF_FALSE (4) - 2 LOAD_CONST (False) - 3 POP_JUMP_IF_TRUE (8) - >> 4 LOAD_CONST (False) - 5 POP_JUMP_IF_FALSE (8) - 6 LOAD_CONST (True) - 7 POP_JUMP_IF_FALSE (8) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (5) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_TRUE (9) + >> 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (9) + 7 LOAD_CONST (True) + 8 POP_JUMP_IF_FALSE (9) - 2 >> 8 LOAD_CONST (None) - 9 RETURN_VALUE + 2 >> 9 LOAD_CONST (None) + 10 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap index 2096e8837..e1d41377d 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap @@ -2,12 +2,13 @@ source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" --- - 1 0 LOAD_CONST (True) - 1 POP_JUMP_IF_TRUE (6) - 2 LOAD_CONST (False) - 3 POP_JUMP_IF_TRUE (6) - 4 LOAD_CONST (False) - 5 POP_JUMP_IF_FALSE (6) + 1 0 RESUME (0) + 1 LOAD_CONST (True) + 2 POP_JUMP_IF_TRUE (7) + 3 LOAD_CONST (False) + 4 POP_JUMP_IF_TRUE (7) + 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (7) - 2 >> 6 LOAD_CONST (None) - 7 RETURN_VALUE + 2 >> 7 LOAD_CONST (None) + 8 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap index 4cfe9a92e..534f8bd34 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -2,7 +2,9 @@ source: crates/codegen/src/compile.rs expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" --- - 3 0 LOAD_CONST (): 1 0 RUSTPYTHON_PLACEHOLDER + 1 0 RESUME (0) + + 3 1 LOAD_CONST (): 1 0 RUSTPYTHON_PLACEHOLDER 1 POP_TOP 2 RESUME (0) @@ -157,7 +159,7 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 143 LOAD_CONST (None) 144 RETURN_VALUE - 1 MAKE_FUNCTION - 2 STORE_NAME (0, test) - 3 LOAD_CONST (None) - 4 RETURN_VALUE + 2 MAKE_FUNCTION + 3 STORE_NAME (0, test) + 4 LOAD_CONST (None) + 5 RETURN_VALUE