From c8e9bd65521092286cd1a20d52af4ff383688fdb Mon Sep 17 00:00:00 2001 From: dot Date: Mon, 12 Oct 2026 03:37:40 +0900 Subject: [PATCH] Retain stable runtime code-field owners Represent runtime constants with one immutable tuple owner and copy-on-write compiler views. Finalize code fields with Context, retain public reader field objects, and preserve canonical normalization without mutating published tuples. Add owner-lifetime, reader-hook and Python identity coverage. Focused checks, Clippy, native tests and finite compiler capture verified. Full integration and marshal byte parity remain pending. Assisted-by: OpenAI Codex:model identifier unavailable --- crates/compiler-core/src/bytecode.rs | 56 +++- crates/compiler-core/src/marshal.rs | 168 +++++++++- crates/vm/src/builtins/code.rs | 305 ++++++++++++++---- crates/vm/src/stdlib/marshal.rs | 19 ++ crates/vm/src/vm/vm_new.rs | 6 +- .../snippets/stable_code_fields_315.py | 154 +++++++++ tools/opcode_metadata/STABLE_CODE_FIELDS.md | 58 ++++ 7 files changed, 679 insertions(+), 87 deletions(-) create mode 100644 extra_tests/snippets/stable_code_fields_315.py create mode 100644 tools/opcode_metadata/STABLE_CODE_FIELDS.md diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index c5f396718..08cfc880c 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -286,12 +286,16 @@ impl PyCodeLocationInfoKind { pub trait Constant: Sized + Clone { type Name: AsRef; + /// An immutable owner whose element view remains valid while borrowed. + type TupleOwner: Clone + AsRef<[Self]>; + /// Transforms the given Constant to a BorrowedConstant fn borrow_constant(&self) -> BorrowedConstant<'_, Self>; } impl Constant for ConstantData { type Name = String; + type TupleOwner = Box<[Self]>; fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { match self { @@ -373,19 +377,59 @@ impl ConstantBag for BasicBag { } #[derive(Clone)] -pub struct Constants(Box<[C]>); +pub struct Constants(ConstantStorage); + +#[derive(Clone)] +enum ConstantStorage { + Boxed(Box<[C]>), + Tuple(C::TupleOwner), +} + +impl Constants { + pub fn from_tuple(owner: C::TupleOwner) -> Self { + Self(ConstantStorage::Tuple(owner)) + } + + pub fn tuple_owner(&self) -> Option<&C::TupleOwner> { + match &self.0 { + ConstantStorage::Tuple(owner) => Some(owner), + ConstantStorage::Boxed(_) => None, + } + } + + /// Consume boxed elements without adding another lasting element owner. + /// An immutable tuple must be copied when its elements are taken out. + pub fn into_boxed(self) -> Box<[C]> { + match self.0 { + ConstantStorage::Boxed(elements) => elements, + ConstantStorage::Tuple(owner) => owner.as_ref().into(), + } + } +} impl Deref for Constants { type Target = [C]; fn deref(&self) -> &Self::Target { - &self.0 + match &self.0 { + ConstantStorage::Boxed(elements) => elements, + ConstantStorage::Tuple(owner) => owner.as_ref(), + } } } impl DerefMut for Constants { fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 + if let ConstantStorage::Tuple(owner) = &self.0 { + // Allocate and clone before releasing the old owner. No published + // tuple is ever mutated, even when it has only one current owner. + let elements = owner.as_ref().into(); + self.0 = ConstantStorage::Boxed(elements); + } + match &mut self.0 { + ConstantStorage::Boxed(elements) => elements, + ConstantStorage::Tuple(_) => unreachable!(), + } } } @@ -393,19 +437,19 @@ impl Index for Constants { type Output = C; fn index(&self, consti: oparg::ConstIdx) -> &Self::Output { - &self.0[consti.as_usize()] + &self.deref()[consti.as_usize()] } } impl IndexMut for Constants { fn index_mut(&mut self, consti: oparg::ConstIdx) -> &mut Self::Output { - &mut self.0[consti.as_usize()] + &mut self.deref_mut()[consti.as_usize()] } } impl FromIterator for Constants { fn from_iter>(iter: T) -> Self { - Self(iter.into_iter().collect()) + Self(ConstantStorage::Boxed(iter.into_iter().collect())) } } diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 1615c1d6a..357b173f2 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -554,6 +554,18 @@ fn read_const_value( Ok(value) } +/// Actual surviving code-field objects from the public marshal stream. +/// Execution bytecode is deliberately excluded: exported co_code is a separate +/// lazy cache, not the input byte object's retained identity. +pub struct CodeFieldObjects { + pub constants: V, + pub names: V, + pub localsplus_names: V, + pub localsplus_kinds: V, + pub linetable: V, + pub exceptiontable: V, +} + pub trait MarshalBag: Copy { type Value: Clone; type ConstantBag: ConstantBag; @@ -624,6 +636,18 @@ pub trait MarshalBag: Copy { self.make_code_with_constants(code, constants) } + /// Preserve actual field owners in runtime bags. Existing overrides keep + /// their original hook chain through the default implementation. + fn make_code_with_fields( + &self, + code: CodeObject<::Constant>, + constants: Vec, + code_bytes: Vec, + _fields: CodeFieldObjects, + ) -> Result { + self.make_code_with_constants_and_bytes(code, constants, code_bytes) + } + fn make_stop_iter(&self) -> Result; fn make_list(&self, it: impl Iterator) -> Result; @@ -958,17 +982,20 @@ fn deserialize_code_value_inner( .into_iter() .collect(); - let read_strings = - |rdr: &mut R, refs: &mut Vec>| -> Result> { - let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?; - bag.tuple_elements_from_value(&tuple) - .ok_or(MarshalError::BadType)? - .iter() - .map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType)) - .collect() - }; - let names_raw = read_strings(rdr, refs)?; - let localsplusnames = read_strings(rdr, refs)?; + let read_strings = |rdr: &mut R, + refs: &mut Vec>| + -> Result<(Bag::Value, Vec)> { + let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let names = bag + .tuple_elements_from_value(&tuple) + .ok_or(MarshalError::BadType)? + .iter() + .map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType)) + .collect::>>()?; + Ok((tuple, names)) + }; + let (names_value, names_raw) = read_strings(rdr, refs)?; + let (localsplusnames_value, localsplusnames) = read_strings(rdr, refs)?; let kinds_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; let localspluskinds = bag @@ -1050,7 +1077,19 @@ fn deserialize_code_value_inner( linetable, exceptiontable, }; - bag.make_code_with_constants_and_bytes(code, constant_values, code_bytes) + bag.make_code_with_fields( + code, + constant_values, + code_bytes, + CodeFieldObjects { + constants: consts_value, + names: names_value, + localsplus_names: localsplusnames_value, + localsplus_kinds: kinds_value, + linetable: linetable_value, + exceptiontable: exceptiontable_value, + }, + ) } fn deserialize_value_typed( @@ -1988,3 +2027,108 @@ mod tests { assert_eq!(code.constants.len(), 2); } } + +#[cfg(test)] +mod field_hook_tests { + use super::*; + use core::cell::Cell; + + #[derive(Clone, Copy)] + struct HookBag<'a>(&'a Cell); + + impl MarshalBag for HookBag<'_> { + type Value = (); + type ConstantBag = BasicBag; + fn make_bool(&self, _: bool) {} + fn make_none(&self) {} + fn make_ellipsis(&self) {} + fn make_float(&self, _: f64) {} + fn make_complex(&self, _: Complex64) {} + fn make_str(&self, _: &Wtf8) {} + fn make_bytes(&self, _: &[u8]) {} + fn make_int(&self, _: BigInt) {} + fn make_tuple(&self, _: impl Iterator) {} + fn make_code(&self, _: CodeObject) -> Result<()> { + assert_eq!(self.0.get(), 3); + self.0.set(7); + Ok(()) + } + fn make_code_with_constants(&self, code: CodeObject, constants: Vec<()>) -> Result<()> { + assert_eq!(self.0.get(), 1); + assert_eq!(constants.len(), 2); + self.0.set(3); + self.make_code(code) + } + fn make_code_with_constants_and_bytes( + &self, + code: CodeObject, + constants: Vec<()>, + bytes: Vec, + ) -> Result<()> { + assert_eq!(self.0.get(), 0); + assert_eq!(bytes, [1, 2]); + self.0.set(1); + self.make_code_with_constants(code, constants) + } + fn make_stop_iter(&self) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_list(&self, _: impl Iterator) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_set(&self, _: impl Iterator) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_frozenset(&self, _: impl Iterator) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_dict(&self, _: impl Iterator) -> Result<()> { + Err(MarshalError::BadType) + } + fn constant_bag(self) -> BasicBag { + BasicBag + } + } + + #[test] + fn actual_fields_default_preserves_existing_hook_chain() { + let code = CodeObject { + instructions: CodeUnits::try_from(&[][..]).unwrap(), + locations: Box::new([]), + flags: CodeFlags::empty(), + posonlyarg_count: 0, + arg_count: 0, + kwonlyarg_count: 0, + source_path: "code_storage.py".into(), + first_line_number: None, + max_stackdepth: 1, + obj_name: "sample".into(), + qualname: "sample".into(), + constants: core::iter::empty().collect(), + names: Box::new([]), + varnames: Box::new([]), + cellvars: Box::new([]), + freevars: Box::new([]), + localspluskinds: Box::new([]), + linetable: Box::new([]), + exceptiontable: Box::new([]), + }; + let calls = Cell::new(0); + HookBag(&calls) + .make_code_with_fields( + code, + alloc::vec![(), ()], + alloc::vec![1, 2], + CodeFieldObjects { + constants: (), + names: (), + localsplus_names: (), + localsplus_kinds: (), + linetable: (), + exceptiontable: (), + }, + ) + .unwrap(); + assert_eq!(calls.get(), 7); + } +} diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 8511f45c0..e8becd4ec 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -154,11 +154,11 @@ pub struct ReplaceArgs { #[pyarg(named, optional)] co_firstlineno: OptionalArg, #[pyarg(named, optional)] - co_consts: OptionalArg>, + co_consts: OptionalArg, #[pyarg(named, optional)] co_name: OptionalArg, #[pyarg(named, optional)] - co_names: OptionalArg>, + co_names: OptionalArg, #[pyarg(named, optional)] co_flags: OptionalArg, #[pyarg(named, optional)] @@ -256,7 +256,22 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { }) } +/// Immutable tuple backing for runtime constants. Construction is private so +/// the only exposed slice always borrows a real tuple owner. +#[derive(Clone)] +pub struct LiteralTupleOwner(PyTupleRef); + +impl AsRef<[Literal]> for LiteralTupleOwner { + fn as_ref(&self) -> &[Literal] { + let elements = self.0.as_slice(); + // SAFETY: Literal is transparent over PyObjectRef and accepts every + // Python object. The immutable slice cannot outlive this tuple owner. + unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) } + } +} + impl Constant for Literal { + type TupleOwner = LiteralTupleOwner; type Name = &'static PyStrInterned; fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { borrow_obj_constant(&self.0) @@ -410,6 +425,14 @@ pub struct PyCode { /// Derived once so frame-local proxy operations do not repeatedly scan /// merged cell variables. localsplus_names: Box<[&'static PyStrInterned]>, + names_tuple: PyTupleRef, + localsplus_names_tuple: PyTupleRef, + localsplus_kinds_bytes: PyBytesRef, + linetable_bytes: PyBytesRef, + exceptiontable_bytes: PyBytesRef, + varnames_tuple: OnceCell, + cellvars_tuple: OnceCell, + freevars_tuple: OnceCell, #[pymember(name = "co_filename")] source_path: AtomicPtr, #[pymember(name = "co_nlocals")] @@ -523,7 +546,19 @@ impl PyCode { /// The input must not be cloned from an already instrumented PyCode: /// generic monitoring wrappers need side tables that CodeObject does not /// carry. Python `replace` reconstructs from canonical bytes instead. - pub fn new(mut code: CodeObject) -> Self { + pub fn new(mut code: CodeObject, ctx: &Context) -> Self { + if code.constants.tuple_owner().is_none() { + let boxed = core::mem::replace(&mut code.constants, core::iter::empty().collect()); + let tuple = ctx.new_tuple( + boxed + .into_boxed() + .into_vec() + .into_iter() + .map(|x| x.0) + .collect(), + ); + code.constants = bytecode::Constants::from_tuple(LiteralTupleOwner(tuple)); + } code.instructions.normalize_get_iter(code.flags); let canonical_code = code.instructions.original_bytes().into_boxed_slice(); let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned; @@ -561,8 +596,26 @@ impl PyCode { | Instruction::InstrumentedEndAsyncFor ) }); + let names_tuple = ctx.new_tuple(code.names.iter().map(|name| name.to_object()).collect()); + let localsplus_names_tuple = ctx.new_tuple( + localsplus_names + .iter() + .map(|name| name.to_object()) + .collect(), + ); + let localsplus_kinds_bytes = ctx.new_bytes(code.localspluskinds.to_vec()); + let linetable_bytes = ctx.new_bytes(code.linetable.to_vec()); + let exceptiontable_bytes = ctx.new_bytes(code.exceptiontable.to_vec()); Self { code, + names_tuple, + localsplus_names_tuple, + localsplus_kinds_bytes, + linetable_bytes, + exceptiontable_bytes, + varnames_tuple: OnceCell::new(), + cellvars_tuple: OnceCell::new(), + freevars_tuple: OnceCell::new(), canonical_code, code_bytes: OnceCell::new(), localsplus_names, @@ -575,6 +628,65 @@ impl PyCode { } } + /// Attach validated public-reader fields before publication. Superseded + /// boxed constants are dropped before the returned code can be observed. + pub(crate) fn from_marshaled_fields( + mut code: CodeObject, + fields: rustpython_compiler_core::marshal::CodeFieldObjects, + ctx: &Context, + ) -> Result { + use rustpython_compiler_core::marshal::MarshalError; + let constants = fields + .constants + .downcast::() + .map_err(|_| MarshalError::BadType)?; + let names = fields + .names + .downcast::() + .map_err(|_| MarshalError::BadType)?; + let locals = fields + .localsplus_names + .downcast::() + .map_err(|_| MarshalError::BadType)?; + let kinds = fields + .localsplus_kinds + .downcast::() + .map_err(|_| MarshalError::BadType)?; + let lines = fields + .linetable + .downcast::() + .map_err(|_| MarshalError::BadType)?; + let exceptions = fields + .exceptiontable + .downcast::() + .map_err(|_| MarshalError::BadType)?; + code.constants = bytecode::Constants::from_tuple(LiteralTupleOwner(constants)); + let mut result = Self::new(code, ctx); + result.retain_normalized_names(names, Some(locals)); + result.localsplus_kinds_bytes = kinds; + result.linetable_bytes = lines; + result.exceptiontable_bytes = exceptions; + Ok(result) + } + + fn retain_normalized_names(&mut self, names: PyTupleRef, locals: Option) { + let matches = |tuple: &PyTupleRef, canonical: &[&'static PyStrInterned]| { + tuple.len() == canonical.len() + && tuple + .iter() + .zip(canonical) + .all(|(item, name)| item.is(*name)) + }; + if matches(&names, &self.code.names) { + self.names_tuple = names; + } + if let Some(locals) = locals + && matches(&locals, &self.localsplus_names) + { + self.localsplus_names_tuple = locals; + } + } + #[inline(always)] pub(crate) fn localsplus_name(&self, index: usize) -> &'static PyStrInterned { // Bytecode operands and frame-proxy indices are validated against @@ -596,7 +708,11 @@ impl PyCode { } pub fn new_ref_with_bag(vm: &VirtualMachine, code: CodeObject) -> PyRef { - PyRef::new_ref(Self::new(code), vm.ctx.types.code_type.to_owned(), None) + PyRef::new_ref( + Self::new(code, &vm.ctx), + vm.ctx.types.code_type.to_owned(), + None, + ) } pub fn new_ref_from_bytecode(vm: &VirtualMachine, code: bytecode::CodeObject) -> PyRef { @@ -900,16 +1016,7 @@ impl Constructor for PyCode { let instructions = CodeUnits::try_from(bytecode_bytes) .map_err(|e| vm.new_value_error(format!("invalid bytecode: {e}")))?; - // Convert constants - let constants = args - .consts - .as_slice() - .iter() - .map(|obj| { - // Convert PyObject to Literal constant. For now, just wrap it - Literal(obj.clone()) - }) - .collect(); + let constants = bytecode::Constants::from_tuple(LiteralTupleOwner(args.consts)); // Create locations (start and end pairs) let row = if args.firstlineno > 0 { @@ -971,7 +1078,10 @@ impl Constructor for PyCode { exceptiontable: args.exceptiontable.as_bytes().to_vec().into_boxed_slice(), }; - Ok(Self::new(code)) + let mut result = Self::new(code, &vm.ctx); + result.linetable_bytes = args.linetable; + result.exceptiontable_bytes = args.exceptiontable; + Ok(result) } } @@ -989,12 +1099,12 @@ impl PyCode { impl Py { #[pygetset] pub fn co_cellvars(&self, vm: &VirtualMachine) -> PyTupleRef { - let cellvars = self - .cellvars - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(cellvars) + self.cellvars_tuple + .get_or_init(|| { + vm.ctx + .new_tuple(self.cellvars.iter().map(|name| name.to_object()).collect()) + }) + .clone() } #[pygetset] @@ -1003,27 +1113,33 @@ impl Py { } #[pygetset] - fn co_consts(&self, vm: &VirtualMachine) -> PyTupleRef { - let consts = self.code.constants.iter().map(|x| x.0.clone()).collect(); - vm.ctx.new_tuple(consts) + fn co_consts(&self) -> PyTupleRef { + self.code + .constants + .tuple_owner() + .expect("finalized constants") + .0 + .clone() } #[pygetset] - fn co_names(&self, vm: &VirtualMachine) -> PyTupleRef { - let names = self - .code - .names - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(names) + fn co_names(&self) -> PyTupleRef { + self.names_tuple.clone() } #[pygetset] pub fn co_varnames(&self, vm: &VirtualMachine) -> PyTupleRef { - let varnames = self.code.varnames.iter().map(|s| s.to_object()).collect(); - vm.ctx.new_tuple(varnames) + self.varnames_tuple + .get_or_init(|| { + vm.ctx.new_tuple( + self.code + .varnames + .iter() + .map(|name| name.to_object()) + .collect(), + ) + }) + .clone() } #[pygetset] @@ -1047,26 +1163,27 @@ impl Py { #[pygetset] pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef { - let names = self - .code - .freevars - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(names) + self.freevars_tuple + .get_or_init(|| { + vm.ctx.new_tuple( + self.code + .freevars + .iter() + .map(|name| name.to_object()) + .collect(), + ) + }) + .clone() } #[pygetset] - pub fn co_linetable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { - // Return the actual linetable from the code object - vm.ctx.new_bytes(self.code.linetable.to_vec()) + pub fn co_linetable(&self) -> PyBytesRef { + self.linetable_bytes.clone() } #[pygetset] - pub fn co_exceptiontable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { - // Return the actual exception table from the code object - vm.ctx.new_bytes(self.code.exceptiontable.to_vec()) + pub fn co_exceptiontable(&self) -> PyBytesRef { + self.exceptiontable_bytes.clone() } // spell-checker: ignore lnotab @@ -1398,7 +1515,13 @@ impl Py { let constants = match co_consts { OptionalArg::Present(constants) => constants, - OptionalArg::Missing => self.code.constants.iter().map(|x| x.0.clone()).collect(), + OptionalArg::Missing => self + .code + .constants + .tuple_owner() + .expect("finalized constants") + .0 + .clone(), }; let obj_name = match co_name { @@ -1408,13 +1531,7 @@ impl Py { let names = match co_names { OptionalArg::Present(names) => names, - OptionalArg::Missing => self - .code - .names - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(), + OptionalArg::Missing => self.names_tuple.clone(), }; let flags = CodeFlags::from_bits_truncate(match co_flags { @@ -1503,15 +1620,13 @@ impl Py { // Handle linetable and exceptiontable let linetable = match co_linetable { - OptionalArg::Present(linetable) => linetable.as_bytes().to_vec().into_boxed_slice(), - OptionalArg::Missing => self.code.linetable.clone(), + OptionalArg::Present(linetable) => linetable, + OptionalArg::Missing => self.linetable_bytes.clone(), }; let exceptiontable = match co_exceptiontable { - OptionalArg::Present(exceptiontable) => { - exceptiontable.as_bytes().to_vec().into_boxed_slice() - } - OptionalArg::Missing => self.code.exceptiontable.clone(), + OptionalArg::Present(exceptiontable) => exceptiontable, + OptionalArg::Missing => self.exceptiontable_bytes.clone(), }; let new_code = CodeObject { @@ -1529,17 +1644,21 @@ impl Py { // FIXME: invalid locations. Actually locations is a duplication of linetable. // It can be removed once we move every other code to use linetable only. locations: self.code.locations.clone(), - constants: constants.into_iter().map(Literal).collect(), - names: intern_all(names, "co_names")?, + constants: bytecode::Constants::from_tuple(LiteralTupleOwner(constants)), + names: intern_all(names.as_slice().to_vec(), "co_names")?, varnames, cellvars, freevars, localspluskinds, - linetable, - exceptiontable, + linetable: linetable.as_bytes().into(), + exceptiontable: exceptiontable.as_bytes().into(), }; - Ok(PyCode::new(new_code)) + let mut result = PyCode::new(new_code, &vm.ctx); + result.retain_normalized_names(names, None); + result.linetable_bytes = linetable; + result.exceptiontable_bytes = exceptiontable; + Ok(result) } #[pymethod] @@ -1661,3 +1780,53 @@ impl<'a> LineTableReader<'a> { pub(crate) fn init(ctx: &'static Context) { PyCode::extend_class(ctx, ctx.types.code_type); } + +#[cfg(test)] +mod storage_tests { + use super::*; + + #[test] + fn tuple_constants_share_owner_and_copy_on_write() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let child: PyObjectRef = vm.ctx.new_list(vec![]).into(); + assert_eq!(child.strong_count(), 1); + let owner = vm.ctx.new_tuple(vec![child.clone()]); + let mut constants = bytecode::Constants::from_tuple(LiteralTupleOwner(owner.clone())); + assert_eq!(child.strong_count(), 2); + let shared = constants.clone(); + assert_eq!(child.strong_count(), 2); + constants[bytecode::oparg::ConstIdx::from_u32(0)] = Literal(vm.ctx.none()); + assert!(constants.tuple_owner().is_none()); + assert!(shared[bytecode::oparg::ConstIdx::from_u32(0)].0.is(&child)); + assert!(owner.as_slice()[0].is(&child)); + assert_eq!(child.strong_count(), 2); + drop(shared); + drop(owner); + assert_eq!(child.strong_count(), 1); + }); + } + + #[cfg(feature = "compiler")] + #[test] + fn published_code_has_one_constant_owner_representation() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let compiled = vm + .compile("pass", crate::compiler::Mode::Exec, "code_storage.py") + .unwrap(); + let child: PyObjectRef = vm.ctx.new_list(vec![]).into(); + let mut raw = compiled.code.clone(); + raw.constants = core::iter::once(Literal(child.clone())).collect(); + assert_eq!(child.strong_count(), 2); + let code = PyCode::new(raw, &vm.ctx); + assert_eq!(child.strong_count(), 2); + let owner = code.code.constants.tuple_owner().unwrap().0.clone(); + let cloned = code.code.clone(); + assert_eq!(child.strong_count(), 2); + drop(code); + drop(cloned); + assert_eq!(child.strong_count(), 2); + drop(owner); + assert_eq!(child.strong_count(), 1); + }); + } +} diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 2624e9c0e..7aedd1a57 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -602,6 +602,25 @@ mod decl { } Ok(crate::builtins::PyCode::new_ref_with_bag(self.vm, code).into()) } + fn make_code_with_fields( + &self, + code: CodeObject, + constants: Vec, + _code_bytes: Vec, + fields: marshal::CodeFieldObjects, + ) -> Result { + if !self.allow_code { + return Err(self.remember_python_error( + self.vm + .new_value_error("unmarshalling code objects is disallowed"), + )); + } + // The actual tuple in fields is the surviving owner. Discard the + // compatibility channel's temporary child references first. + drop(constants); + let code = PyCode::from_marshaled_fields(code, fields, &self.vm.ctx)?; + Ok(code.into_ref(&self.vm.ctx).into()) + } fn make_stop_iter(&self) -> Result { Ok(self.vm.ctx.exceptions.stop_iteration.to_owned().into()) } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 1a1696b71..1ce273644 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -56,7 +56,11 @@ impl VirtualMachine { pub fn new_code(&self, code: impl IntoCodeObject) -> PyRef { let code = code.into_code_object(self); - PyRef::new_ref(PyCode::new(code), self.ctx.types.code_type.to_owned(), None) + PyRef::new_ref( + PyCode::new(code, &self.ctx), + self.ctx.types.code_type.to_owned(), + None, + ) } pub fn new_tuple(&self, value: impl IntoPyTuple) -> PyTupleRef { diff --git a/extra_tests/snippets/stable_code_fields_315.py b/extra_tests/snippets/stable_code_fields_315.py new file mode 100644 index 000000000..637c79fad --- /dev/null +++ b/extra_tests/snippets/stable_code_fields_315.py @@ -0,0 +1,154 @@ +"""Stable surviving code fields and sole constants tuple ownership.""" + +import gc +import marshal +import sys +import types +import weakref + +SOURCE = "def sample(a):\n try:\n return a + marker\n except TypeError:\n return 0\n" +FILENAME = "stable_code_fields.py" +base = compile(SOURCE, FILENAME, "exec").co_consts[0] +for name in ( + "co_consts", + "co_names", + "co_varnames", + "co_freevars", + "co_cellvars", + "co_linetable", + "co_exceptiontable", + "co_code", +): + value = getattr(base, name) + assert getattr(base, name) is value + +replacement = base.replace() +for name in ("co_consts", "co_names", "co_linetable", "co_exceptiontable"): + assert getattr(replacement, name) is getattr(base, name) +assert replacement.co_code == base.co_code +assert replacement.co_code is not base.co_code +assert replacement.co_varnames == base.co_varnames +assert replacement.co_varnames is not base.co_varnames + + +class Child: + pass + + +child = Child() +weak = weakref.ref(child) +values = (child, [1, 2], slice(1, 5, 2), frozenset((3, 4))) +owned = base.replace(co_consts=values) +assert owned.co_consts is values +assert owned.co_consts[0] is child +assert owned.co_consts[2] is values[2] +assert owned.co_consts[3] is values[3] +clone = owned.replace() +assert clone.co_consts is values +del child, owned, values +gc.collect() +assert weak() is not None +del clone +gc.collect() +assert weak() is None + +# The public constructor copies validated names, retains its constants and +# surviving byte tables, and independently creates its exported code cache. +input_code = bytes(bytearray(base.co_code)) +input_names = tuple(list(base.co_names)) +constructed = types.CodeType( + base.co_argcount, + base.co_posonlyargcount, + base.co_kwonlyargcount, + base.co_nlocals, + base.co_stacksize, + base.co_flags, + input_code, + base.co_consts, + input_names, + base.co_varnames, + FILENAME, + base.co_name, + base.co_qualname, + base.co_firstlineno, + base.co_linetable, + base.co_exceptiontable, + base.co_freevars, + base.co_cellvars, +) +assert constructed.co_consts is base.co_consts +assert constructed.co_names == input_names +assert constructed.co_names is not input_names +assert constructed.co_linetable is base.co_linetable +assert constructed.co_exceptiontable is base.co_exceptiontable +assert constructed.co_code == input_code +assert constructed.co_code is not input_code + +# Retain the known normalization gap. Rust never mutates a published caller +# tuple; when a member needs canonical interning it creates a normalized copy. +raw_name = "".join(("mar", "ker")) +assert raw_name is not base.co_names[0] +names = (raw_name,) +normalized = base.replace(co_names=names) +assert normalized.co_names[0] is base.co_names[0] +if sys.implementation.name == "rustpython": + assert normalized.co_names is not names + assert names[0] is raw_name +else: + assert normalized.co_names is names + assert names[0] is base.co_names[0] + +outer = compile( + "def outer(a):\n def inner():\n return a\n return inner\n", + FILENAME, + "exec", +).co_consts[0] +inner = next(value for value in outer.co_consts if isinstance(value, types.CodeType)) +assert outer.co_cellvars == ("a",) +assert outer.co_cellvars is outer.co_cellvars +assert inner.co_freevars == ("a",) +assert inner.co_freevars is inner.co_freevars + +# Whole unchanged streams emitted by the pinned final oracle. Both field-first +# and code-first ordering exercise reader reference identity; the current Rust +# writer is deliberately not used to create the expected aliases. +FIELD_FIRST = bytes.fromhex( + "2905a901e900000000a902da066d61726b6572da09547970654572726f72f3230000008100f002030511d80f" + "1094368d7ad00819f8dc0b14f400010511da0f10f003010511faf310000000830c100090091f039a011f039e" + "011f03e30100000000000000000000000400000003000000f34400000080000000190055005b000000000000" + "0000002a000000000000000000000021001e005b0200000000000000000600630500001a001d001b005d0021" + "00680039031b006801720000000072020000002901da0161730100000026da15737461626c655f636f64655f" + "6669656c64732e7079da0673616d706c65720b0000000100000072050000007206000000" +) +CODE_FIRST = bytes.fromhex( + "2905e30100000000000000000000000400000003000000f34400000080000000190055005b00000000000000" + "00002a000000000000000000000021001e005b0200000000000000000600630500001a001d001b005d002100" + "680039031b006801a901e900000000a902da066d61726b6572da09547970654572726f722901da0161730100" + "000026da15737461626c655f636f64655f6669656c64732e7079da0673616d706c65720900000001000000f3" + "230000008100f002030511d80f1094368d7ad00819f8dc0b14f400010511da0f10f003010511faf310000000" + "830c100090091f039a011f039e011f0372020000007204000000720a000000720b000000" +) +for order, payload in (("field_first", FIELD_FIRST), ("code_first", CODE_FIRST)): + decoded = marshal.loads(payload) + if order == "field_first": + consts, names, lines, exceptions, code = decoded + else: + code, consts, names, lines, exceptions = decoded + assert code.co_consts is consts + assert code.co_names is names + assert code.co_linetable is lines + assert code.co_exceptiontable is exceptions + assert code.co_code == base.co_code + assert code.co_code is code.co_code + function = types.FunctionType(code, {"marker": 10}) + assert function(2) == 12 + assert function("x") == 0 + +# Final CodeType.replace accepts tuple fields, not arbitrary iterables. +for field in ("co_consts", "co_names"): + try: + base.replace(**{field: []}) + except TypeError: + pass + else: + raise AssertionError(field + " accepted a list") diff --git a/tools/opcode_metadata/STABLE_CODE_FIELDS.md b/tools/opcode_metadata/STABLE_CODE_FIELDS.md new file mode 100644 index 000000000..1cad59af7 --- /dev/null +++ b/tools/opcode_metadata/STABLE_CODE_FIELDS.md @@ -0,0 +1,58 @@ +# Stable code fields: storage-only follow-on + +This stage follows canonical code storage commit6746df059. It does not change the +marshal writer, reference flags, native call ownership, permanent string interner +or filename handling. Full serialization parity remains unresolved. + +Runtime `Constants` owns one immutable Python tuple. It does not keep a +second boxed list of owning child references beside a getter cache. The compiler +representation remains boxed. Cloning a runtime constant pool shares the tuple; +mutable access copies the elements before replacing the owner. Published tuples +are never mutated. The owner-bound Literal slice uses the existing transparent +layout over PyObjectRef and never calls the slice/frozenset constant adapter. + +Surviving names, locals-plus names/kinds and line/exception tables have stable +Python owners. Derived variable/cell/free-name tuples are separate lazy caches. +Constructor names are copied after validation; replace retains unchanged names. +Public marshal decode retains actual field objects when normalized name member +identities permit it, otherwise uses a safe normalized copy. That copy has a +known tuple-identity difference from CPython's in-place name normalization. +Existing constant interning behavior is preserved, not expanded. + +Decoder input co_code bytes are not retained as the exported co_code object. +That export remains A0's independent lazy cache of canonical normalized bytes. +The new reader hook defaults to the existing bytes/constants/code hook chain. +Runtime decoding drops temporary child vectors and replaced boxes before code +publication. Reader allow_code, type, depth and incomplete-reference checks stay +in force. + +## Rust source API migration + +- `PyCode::new(code, &Context)` now requires the creating runtime's context so + final tuple/metadata objects exist before publication. Callers must still pass + pristine instruction streams, as required by A0. +- Implementations of compiler-core `Constant` supply `TupleOwner: Clone + + AsRef<[Self]>`. BasicBag uses boxed constants; the runtime uses a private + validated tuple owner. Downstream bags must provide an immutable owner-bound + element view. No Python object type is introduced into compiler-core. +- Public getters for co_linetable/co_exceptiontable no longer require a VM + argument; they return retained Python bytes owners. +- A new `MarshalBag::make_code_with_fields` is additive. Its default delegates + `make_code_with_constants_and_bytes`, preserving earlier customization hooks. + +The checked-in Python test retains full certified streams in both alias orders, +checks identity and execution, and explicitly preserves the normalization gap. +Rust tests verify actual child-owner counts, COW isolation, publication and +reader-hook ordering. No test count is used by production serialization. + +## Integration limits + +The existing untracked PyCode policy and permanent-intern ownership contracts +are unchanged. Pinned CPython only tracks code objects in Py_GIL_DISABLED +builds; the certified GIL-enabled oracle is untracked too. This stage does not claim a general cycle-collection repair or mortal +interning parity. Python child ownership moves to the tuple edge and ordinary +Rust drops release it; a future traversal change must account for that real edge. +Compilation against current main after #9080 is a separate integration gate even +though the source audit found no direct overlap and transparent ownership remains. +Full workspace/CAPI/release and the canonical metadata dependencies remain +required before a general integration/CI-green claim.