emit RESUME

Lib/test/test_compile.py

@@ -152,8 +152,6 @@ if 1:
         pass"""
         compile(s, "<string>", "exec")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     # This test is probably specific to CPython and may not generalize
     # to other implementations.  We are trying to ensure that when
     # the first line of code starts after 256, correct line numbers

Lib/test/test_peepholer.py

@@ -147,7 +147,6 @@ class TestTranforms(BytecodeTestCase):
                 self.assertNotInBytecode(code, 'UNPACK_SEQUENCE')
                 self.check_lnotab(code)
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_folding_of_tuples_of_constants(self):
         for line, elem in (
             ('a = 1,2,3', (1, 2, 3)),

crates/codegen/src/compile.rs

@@ -997,12 +997,6 @@ impl Compiler {
         key: usize, // In RustPython, we use the index in symbol_table_stack as key
         lineno: u32,
     ) -> CompileResult<()> {
-        // Create location
-        let location = SourceLocation {
-            line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN),
-            character_offset: OneIndexed::MIN,
-        };
-
         // Allocate a new compiler unit
 
         // In Rust, we'll create the structure directly
@@ -1148,40 +1142,50 @@ impl Compiler {
             self.set_qualname();
         }
 
-        // Emit RESUME instruction
-        let _resume_loc = if scope_type == CompilerScope::Module {
-            // Module scope starts with lineno 0
-            SourceLocation {
-                line: OneIndexed::MIN,
-                character_offset: OneIndexed::MIN,
-            }
-        } else {
-            location
-        };
+        // Emit RESUME (handles async preamble and module lineno 0)
+        // CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module
+        self.emit_resume_for_scope(scope_type, lineno);
 
-        // Set the source range for the RESUME instruction
-        // For now, just use an empty range at the beginning
-        self.current_source_range = TextRange::default();
+        Ok(())
+    }
 
+    /// Emit RESUME instruction with proper handling for async preamble and module lineno.
+    /// codegen_enter_scope equivalent for RESUME emission.
+    fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) {
         // For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME
         if scope_type == CompilerScope::AsyncFunction {
             emit!(self, Instruction::ReturnGenerator);
             emit!(self, Instruction::PopTop);
         }
 
-        emit!(
-            self,
-            Instruction::Resume {
-                arg: bytecode::ResumeType::AtFuncStart as u32
+        // CPython: LOCATION(lineno, lineno, 0, 0)
+        // Module scope: loc.lineno = 0 (before the first line)
+        let lineno_override = if scope_type == CompilerScope::Module {
+            Some(0)
+        } else {
+            None
+        };
+
+        // Use lineno for location (col = 0 as in CPython)
+        let location = SourceLocation {
+            line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN),
+            character_offset: OneIndexed::MIN, // col = 0
+        };
+        let end_location = location; // end_lineno = lineno, end_col = 0
+        let except_handler = self.current_except_handler();
+
+        self.current_block().instructions.push(ir::InstructionInfo {
+            instr: Instruction::Resume {
+                arg: OpArgMarker::marker(),
             }
-        );
-
-        if scope_type == CompilerScope::Module {
-            // This would be loc.lineno = -1 in CPython
-            // We handle this differently in RustPython
-        }
-
-        Ok(())
+            .into(),
+            arg: OpArg(bytecode::ResumeType::AtFuncStart as u32),
+            target: BlockIdx::NULL,
+            location,
+            end_location,
+            except_handler,
+            lineno_override,
+        });
     }
 
     fn push_output(
@@ -1750,6 +1754,8 @@ impl Compiler {
         self.future_annotations = symbol_table.future_annotations;
         self.symbol_table_stack.push(symbol_table);
 
+        self.emit_resume_for_scope(CompilerScope::Module, 1);
+
         let (doc, statements) = split_doc(&body.body, &self.opts);
         if let Some(value) = doc {
             self.emit_load_const(ConstantData::Str {
@@ -1795,6 +1801,8 @@ impl Compiler {
         self.future_annotations = symbol_table.future_annotations;
         self.symbol_table_stack.push(symbol_table);
 
+        self.emit_resume_for_scope(CompilerScope::Module, 1);
+
         // Handle annotations based on future_annotations flag
         if Self::find_ann(body) {
             if self.future_annotations {
@@ -1858,6 +1866,7 @@ impl Compiler {
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
+        self.emit_resume_for_scope(CompilerScope::Module, 1);
 
         self.compile_statements(body)?;
 
@@ -1887,6 +1896,8 @@ impl Compiler {
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
+        self.emit_resume_for_scope(CompilerScope::Module, 1);
+
         self.compile_expression(&expression.body)?;
         self.emit_return_value();
         Ok(())
@@ -2631,7 +2642,7 @@ impl Compiler {
 
         // Get the current symbol table
         let key = self.symbol_table_stack.len() - 1;
-        let lineno = expr.range().start().to_u32();
+        let lineno = self.get_source_line_number().get().to_u32();
 
         // Enter scope with the type parameter name
         self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?;
@@ -7764,6 +7775,7 @@ impl Compiler {
             location,
             end_location,
             except_handler,
+            lineno_override: None,
         });
     }
 
@@ -7788,8 +7800,6 @@ impl Compiler {
     }
 
     fn emit_load_const(&mut self, constant: ConstantData) {
-        // Always add to co_consts and emit LOAD_CONST.
-        // Optimization pass converts LOAD_CONST to LOAD_SMALL_INT where appropriate.
         let idx = self.arg_constant(constant);
         self.emit_arg(idx, |idx| Instruction::LoadConst { idx })
     }

crates/codegen/src/ir.rs

@@ -14,6 +14,15 @@ use rustpython_compiler_core::{
     varint::{write_signed_varint, write_varint},
 };
 
+/// Location info for linetable generation (allows line 0 for RESUME)
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct LineTableLocation {
+    line: i32,
+    end_line: i32,
+    col: i32,
+    end_col: i32,
+}
+
 /// Metadata for a code unit
 // = _PyCompile_CodeUnitMetadata
 #[derive(Clone, Debug)]
@@ -94,6 +103,8 @@ pub struct InstructionInfo {
     pub location: SourceLocation,
     pub end_location: SourceLocation,
     pub except_handler: Option<ExceptHandlerInfo>,
+    /// Override line number for linetable (e.g., line 0 for module RESUME)
+    pub lineno_override: Option<i32>,
 }
 
 /// Exception handler information for an instruction.
@@ -161,11 +172,16 @@ impl CodeInfo {
         mut self,
         opts: &crate::compile::CompileOpts,
     ) -> crate::InternalResult<CodeObject> {
-        // Always fold tuple constants and convert to LOAD_SMALL_INT
-        // 1. fold_tuple_of_constants - fold constant sequences into tuples
-        // 2. maybe_instr_make_load_smallint - convert LOAD_CONST to LOAD_SMALL_INT
+        // Always fold tuple constants
         self.fold_tuple_constants();
-        self.convert_to_load_small_int();
+        // Python only applies LOAD_SMALL_INT conversion to module-level code
+        // (not inside functions). Module code lacks OPTIMIZED flag.
+        // Note: RustPython incorrectly sets NEWLOCALS on modules, so only check OPTIMIZED
+        let is_module_level = !self.flags.contains(CodeFlags::OPTIMIZED);
+        if is_module_level {
+            self.convert_to_load_small_int();
+        }
+        self.remove_unused_consts();
         self.remove_nops();
 
         if opts.optimize > 0 {
@@ -209,6 +225,7 @@ impl CodeInfo {
 
         let mut instructions = Vec::new();
         let mut locations = Vec::new();
+        let mut linetable_locations: Vec<LineTableLocation> = Vec::new();
 
         // convert_pseudo_ops: instructions before the main loop
         for block in blocks
@@ -315,6 +332,16 @@ impl CodeInfo {
                         (info.location, info.end_location),
                         info.arg.instr_size(),
                     ));
+                    // Collect linetable locations with lineno_override support
+                    let lt_loc = LineTableLocation {
+                        line: info
+                            .lineno_override
+                            .unwrap_or_else(|| info.location.line.get() as i32),
+                        end_line: info.end_location.line.get() as i32,
+                        col: info.location.character_offset.to_zero_indexed() as i32,
+                        end_col: info.end_location.character_offset.to_zero_indexed() as i32,
+                    };
+                    linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size()));
                     instructions.extend(
                         extras
                             .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte))
@@ -330,12 +357,13 @@ impl CodeInfo {
             }
 
             instructions.clear();
-            locations.clear()
+            locations.clear();
+            linetable_locations.clear();
         }
 
-        // Generate linetable from locations
+        // Generate linetable from linetable_locations (supports line 0 for RESUME)
         let linetable = generate_linetable(
-            &locations,
+            &linetable_locations,
             first_line_number.get() as i32,
             opts.debug_ranges,
         );
@@ -578,12 +606,72 @@ impl CodeInfo {
                 };
 
                 // Check if it's in small int range: -5 to 256 (_PY_IS_SMALL_INT)
-                if let Some(small) = value.to_i32() {
-                    if (-5..=256).contains(&small) {
-                        // Convert to LOAD_CONST to LOAD_SMALL_INT
-                        instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into();
-                        // The arg is the i32 value stored as u32 (two's complement)
-                        instr.arg = OpArg(small as u32);
+                if let Some(small) = value.to_i32().filter(|v| (-5..=256).contains(v)) {
+                    // Convert LOAD_CONST to LOAD_SMALL_INT
+                    instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into();
+                    // The arg is the i32 value stored as u32 (two's complement)
+                    instr.arg = OpArg(small as u32);
+                }
+            }
+        }
+    }
+
+    /// Remove constants that are no longer referenced by LOAD_CONST instructions.
+    /// remove_unused_consts
+    fn remove_unused_consts(&mut self) {
+        let nconsts = self.metadata.consts.len();
+        if nconsts == 0 {
+            return;
+        }
+
+        // Mark used constants
+        // The first constant (index 0) is always kept (may be docstring)
+        let mut used = vec![false; nconsts];
+        used[0] = true;
+
+        for block in &self.blocks {
+            for instr in &block.instructions {
+                if let Some(Instruction::LoadConst { .. }) = instr.instr.real() {
+                    let idx = instr.arg.0 as usize;
+                    if idx < nconsts {
+                        used[idx] = true;
+                    }
+                }
+            }
+        }
+
+        // Check if any constants can be removed
+        let n_used: usize = used.iter().filter(|&&u| u).count();
+        if n_used == nconsts {
+            return; // Nothing to remove
+        }
+
+        // Build old_to_new index mapping
+        let mut old_to_new = vec![0usize; nconsts];
+        let mut new_idx = 0usize;
+        for (old_idx, &is_used) in used.iter().enumerate() {
+            if is_used {
+                old_to_new[old_idx] = new_idx;
+                new_idx += 1;
+            }
+        }
+
+        // Build new consts list
+        let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect();
+        self.metadata.consts.clear();
+        for (old_idx, constant) in old_consts.into_iter().enumerate() {
+            if used[old_idx] {
+                self.metadata.consts.insert(constant);
+            }
+        }
+
+        // Update LOAD_CONST instruction arguments
+        for block in &mut self.blocks {
+            for instr in &mut block.instructions {
+                if let Some(Instruction::LoadConst { .. }) = instr.instr.real() {
+                    let old_idx = instr.arg.0 as usize;
+                    if old_idx < nconsts {
+                        instr.arg = OpArg(old_to_new[old_idx] as u32);
                     }
                 }
             }
@@ -759,7 +847,7 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '
 
 /// Generate Python 3.11+ format linetable from source locations
 fn generate_linetable(
-    locations: &[(SourceLocation, SourceLocation)],
+    locations: &[LineTableLocation],
     first_line: i32,
     debug_ranges: bool,
 ) -> Box<[u8]> {
@@ -774,7 +862,7 @@ fn generate_linetable(
     let mut i = 0;
 
     while i < locations.len() {
-        let (loc, end_loc) = &locations[i];
+        let loc = &locations[i];
 
         // Count consecutive instructions with the same location
         let mut length = 1;
@@ -787,8 +875,8 @@ fn generate_linetable(
             let entry_length = length.min(8);
 
             // Get line information
-            let line = loc.line.get() as i32;
-            let end_line = end_loc.line.get() as i32;
+            let line = loc.line;
+            let end_line = loc.end_line;
             let line_delta = line - prev_line;
             let end_line_delta = end_line - line;
 
@@ -808,8 +896,8 @@ fn generate_linetable(
             }
 
             // Get column information (only when debug_ranges is enabled)
-            let col = loc.character_offset.to_zero_indexed() as i32;
-            let end_col = end_loc.character_offset.to_zero_indexed() as i32;
+            let col = loc.col;
+            let end_col = loc.end_col;
 
             // Choose the appropriate encoding based on line delta and column info
             if line_delta == 0 && end_line_delta == 0 {

crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap

@@ -2,12 +2,13 @@
 source: crates/codegen/src/compile.rs
 expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
 ---
-  1           0 LOAD_CONST           (True)
-              1 POP_JUMP_IF_FALSE    (6)
-              2 LOAD_CONST           (False)
-              3 POP_JUMP_IF_FALSE    (6)
-              4 LOAD_CONST           (False)
-              5 POP_JUMP_IF_FALSE    (6)
+  1           0 RESUME               (0)
+              1 LOAD_CONST           (True)
+              2 POP_JUMP_IF_FALSE    (7)
+              3 LOAD_CONST           (False)
+              4 POP_JUMP_IF_FALSE    (7)
+              5 LOAD_CONST           (False)
+              6 POP_JUMP_IF_FALSE    (7)
 
-  2     >>    6 LOAD_CONST           (None)
-              7 RETURN_VALUE
+  2     >>    7 LOAD_CONST           (None)
+              8 RETURN_VALUE

crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap

@@ -2,14 +2,15 @@
 source: crates/codegen/src/compile.rs
 expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
 ---
-  1           0 LOAD_CONST           (True)
-              1 POP_JUMP_IF_FALSE    (4)
-              2 LOAD_CONST           (False)
-              3 POP_JUMP_IF_TRUE     (8)
-        >>    4 LOAD_CONST           (False)
-              5 POP_JUMP_IF_FALSE    (8)
-              6 LOAD_CONST           (True)
-              7 POP_JUMP_IF_FALSE    (8)
+  1           0 RESUME               (0)
+              1 LOAD_CONST           (True)
+              2 POP_JUMP_IF_FALSE    (5)
+              3 LOAD_CONST           (False)
+              4 POP_JUMP_IF_TRUE     (9)
+        >>    5 LOAD_CONST           (False)
+              6 POP_JUMP_IF_FALSE    (9)
+              7 LOAD_CONST           (True)
+              8 POP_JUMP_IF_FALSE    (9)
 
-  2     >>    8 LOAD_CONST           (None)
-              9 RETURN_VALUE
+  2     >>    9 LOAD_CONST           (None)
+             10 RETURN_VALUE

crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap

@@ -2,12 +2,13 @@
 source: crates/codegen/src/compile.rs
 expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
 ---
-  1           0 LOAD_CONST           (True)
-              1 POP_JUMP_IF_TRUE     (6)
-              2 LOAD_CONST           (False)
-              3 POP_JUMP_IF_TRUE     (6)
-              4 LOAD_CONST           (False)
-              5 POP_JUMP_IF_FALSE    (6)
+  1           0 RESUME               (0)
+              1 LOAD_CONST           (True)
+              2 POP_JUMP_IF_TRUE     (7)
+              3 LOAD_CONST           (False)
+              4 POP_JUMP_IF_TRUE     (7)
+              5 LOAD_CONST           (False)
+              6 POP_JUMP_IF_FALSE    (7)
 
-  2     >>    6 LOAD_CONST           (None)
-              7 RETURN_VALUE
+  2     >>    7 LOAD_CONST           (None)
+              8 RETURN_VALUE

crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap

@@ -2,7 +2,9 @@
 source: crates/codegen/src/compile.rs
 expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n        with self.subTest(type=type(stop_exc)):\n            try:\n                async with egg():\n                    raise stop_exc\n            except Exception as ex:\n                self.assertIs(ex, stop_exc)\n            else:\n                self.fail(f'{stop_exc} was suppressed')\n\")"
 ---
-  3           0 LOAD_CONST           (<code object test at ??? file "source_path", line 1>):  1               0 RUSTPYTHON_PLACEHOLDER
+  1           0 RESUME               (0)
+
+  3           1 LOAD_CONST           (<code object test at ??? file "source_path", line 1>):  1               0 RUSTPYTHON_PLACEHOLDER
                   1 POP_TOP
                   2 RESUME               (0)
 
@@ -157,7 +159,7 @@ expression: "compile_exec(\"\\\nasync def test():\n    for stop_exc in (StopIter
                 143 LOAD_CONST           (None)
                 144 RETURN_VALUE
 
-              1 MAKE_FUNCTION
-              2 STORE_NAME           (0, test)
-              3 LOAD_CONST           (None)
-              4 RETURN_VALUE
+              2 MAKE_FUNCTION
+              3 STORE_NAME           (0, test)
+              4 LOAD_CONST           (None)
+              5 RETURN_VALUE