mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Retain stable runtime code-field owners
Represent runtime constants with one immutable tuple owner and copy-on-write compiler views. Finalize code fields with Context, retain public reader field objects, and preserve canonical normalization without mutating published tuples. Add owner-lifetime, reader-hook and Python identity coverage. Focused checks, Clippy, native tests and finite compiler capture verified. Full integration and marshal byte parity remain pending. Assisted-by: OpenAI Codex:model identifier unavailable
This commit is contained in:
1 parent
6746df059e
commit
c8e9bd6552
7 files changed
+679
-87
No files matched your search
@@ -286,12 +286,16 @@ impl PyCodeLocationInfoKind {
|
||||
pub trait Constant: Sized + Clone {
|
||||
type Name: AsRef<str>;
|
||||
|
||||
/// An immutable owner whose element view remains valid while borrowed.
|
||||
type TupleOwner: Clone + AsRef<[Self]>;
|
||||
|
||||
/// Transforms the given Constant to a BorrowedConstant
|
||||
fn borrow_constant(&self) -> BorrowedConstant<'_, Self>;
|
||||
}
|
||||
|
||||
impl Constant for ConstantData {
|
||||
type Name = String;
|
||||
type TupleOwner = Box<[Self]>;
|
||||
|
||||
fn borrow_constant(&self) -> BorrowedConstant<'_, Self> {
|
||||
match self {
|
||||
@@ -373,19 +377,59 @@ impl ConstantBag for BasicBag {
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Constants<C: Constant>(Box<[C]>);
|
||||
pub struct Constants<C: Constant>(ConstantStorage<C>);
|
||||
|
||||
#[derive(Clone)]
|
||||
enum ConstantStorage<C: Constant> {
|
||||
Boxed(Box<[C]>),
|
||||
Tuple(C::TupleOwner),
|
||||
}
|
||||
|
||||
impl<C: Constant> Constants<C> {
|
||||
pub fn from_tuple(owner: C::TupleOwner) -> Self {
|
||||
Self(ConstantStorage::Tuple(owner))
|
||||
}
|
||||
|
||||
pub fn tuple_owner(&self) -> Option<&C::TupleOwner> {
|
||||
match &self.0 {
|
||||
ConstantStorage::Tuple(owner) => Some(owner),
|
||||
ConstantStorage::Boxed(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume boxed elements without adding another lasting element owner.
|
||||
/// An immutable tuple must be copied when its elements are taken out.
|
||||
pub fn into_boxed(self) -> Box<[C]> {
|
||||
match self.0 {
|
||||
ConstantStorage::Boxed(elements) => elements,
|
||||
ConstantStorage::Tuple(owner) => owner.as_ref().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Constant> Deref for Constants<C> {
|
||||
type Target = [C];
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
match &self.0 {
|
||||
ConstantStorage::Boxed(elements) => elements,
|
||||
ConstantStorage::Tuple(owner) => owner.as_ref(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Constant> DerefMut for Constants<C> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
if let ConstantStorage::Tuple(owner) = &self.0 {
|
||||
// Allocate and clone before releasing the old owner. No published
|
||||
// tuple is ever mutated, even when it has only one current owner.
|
||||
let elements = owner.as_ref().into();
|
||||
self.0 = ConstantStorage::Boxed(elements);
|
||||
}
|
||||
match &mut self.0 {
|
||||
ConstantStorage::Boxed(elements) => elements,
|
||||
ConstantStorage::Tuple(_) => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,19 +437,19 @@ impl<C: Constant> Index<oparg::ConstIdx> for Constants<C> {
|
||||
type Output = C;
|
||||
|
||||
fn index(&self, consti: oparg::ConstIdx) -> &Self::Output {
|
||||
&self.0[consti.as_usize()]
|
||||
&self.deref()[consti.as_usize()]
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Constant> IndexMut<oparg::ConstIdx> for Constants<C> {
|
||||
fn index_mut(&mut self, consti: oparg::ConstIdx) -> &mut Self::Output {
|
||||
&mut self.0[consti.as_usize()]
|
||||
&mut self.deref_mut()[consti.as_usize()]
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Constant> FromIterator<C> for Constants<C> {
|
||||
fn from_iter<T: IntoIterator<Item = C>>(iter: T) -> Self {
|
||||
Self(iter.into_iter().collect())
|
||||
Self(ConstantStorage::Boxed(iter.into_iter().collect()))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -554,6 +554,18 @@ fn read_const_value<R: Read, Bag: ConstantBag>(
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Actual surviving code-field objects from the public marshal stream.
|
||||
/// Execution bytecode is deliberately excluded: exported co_code is a separate
|
||||
/// lazy cache, not the input byte object's retained identity.
|
||||
pub struct CodeFieldObjects<V> {
|
||||
pub constants: V,
|
||||
pub names: V,
|
||||
pub localsplus_names: V,
|
||||
pub localsplus_kinds: V,
|
||||
pub linetable: V,
|
||||
pub exceptiontable: V,
|
||||
}
|
||||
|
||||
pub trait MarshalBag: Copy {
|
||||
type Value: Clone;
|
||||
type ConstantBag: ConstantBag;
|
||||
@@ -624,6 +636,18 @@ pub trait MarshalBag: Copy {
|
||||
self.make_code_with_constants(code, constants)
|
||||
}
|
||||
|
||||
/// Preserve actual field owners in runtime bags. Existing overrides keep
|
||||
/// their original hook chain through the default implementation.
|
||||
fn make_code_with_fields(
|
||||
&self,
|
||||
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
|
||||
constants: Vec<Self::Value>,
|
||||
code_bytes: Vec<u8>,
|
||||
_fields: CodeFieldObjects<Self::Value>,
|
||||
) -> Result<Self::Value> {
|
||||
self.make_code_with_constants_and_bytes(code, constants, code_bytes)
|
||||
}
|
||||
|
||||
fn make_stop_iter(&self) -> Result<Self::Value>;
|
||||
|
||||
fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
|
||||
@@ -958,17 +982,20 @@ fn deserialize_code_value_inner<R: Read, Bag: MarshalBag>(
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let read_strings =
|
||||
|rdr: &mut R, refs: &mut Vec<Option<Bag::Value>>| -> Result<Vec<alloc::string::String>> {
|
||||
let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?;
|
||||
bag.tuple_elements_from_value(&tuple)
|
||||
.ok_or(MarshalError::BadType)?
|
||||
.iter()
|
||||
.map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType))
|
||||
.collect()
|
||||
};
|
||||
let names_raw = read_strings(rdr, refs)?;
|
||||
let localsplusnames = read_strings(rdr, refs)?;
|
||||
let read_strings = |rdr: &mut R,
|
||||
refs: &mut Vec<Option<Bag::Value>>|
|
||||
-> Result<(Bag::Value, Vec<alloc::string::String>)> {
|
||||
let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?;
|
||||
let names = bag
|
||||
.tuple_elements_from_value(&tuple)
|
||||
.ok_or(MarshalError::BadType)?
|
||||
.iter()
|
||||
.map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok((tuple, names))
|
||||
};
|
||||
let (names_value, names_raw) = read_strings(rdr, refs)?;
|
||||
let (localsplusnames_value, localsplusnames) = read_strings(rdr, refs)?;
|
||||
|
||||
let kinds_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
|
||||
let localspluskinds = bag
|
||||
@@ -1050,7 +1077,19 @@ fn deserialize_code_value_inner<R: Read, Bag: MarshalBag>(
|
||||
linetable,
|
||||
exceptiontable,
|
||||
};
|
||||
bag.make_code_with_constants_and_bytes(code, constant_values, code_bytes)
|
||||
bag.make_code_with_fields(
|
||||
code,
|
||||
constant_values,
|
||||
code_bytes,
|
||||
CodeFieldObjects {
|
||||
constants: consts_value,
|
||||
names: names_value,
|
||||
localsplus_names: localsplusnames_value,
|
||||
localsplus_kinds: kinds_value,
|
||||
linetable: linetable_value,
|
||||
exceptiontable: exceptiontable_value,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
|
||||
@@ -1988,3 +2027,108 @@ mod tests {
|
||||
assert_eq!(code.constants.len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod field_hook_tests {
|
||||
use super::*;
|
||||
use core::cell::Cell;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct HookBag<'a>(&'a Cell<u8>);
|
||||
|
||||
impl MarshalBag for HookBag<'_> {
|
||||
type Value = ();
|
||||
type ConstantBag = BasicBag;
|
||||
fn make_bool(&self, _: bool) {}
|
||||
fn make_none(&self) {}
|
||||
fn make_ellipsis(&self) {}
|
||||
fn make_float(&self, _: f64) {}
|
||||
fn make_complex(&self, _: Complex64) {}
|
||||
fn make_str(&self, _: &Wtf8) {}
|
||||
fn make_bytes(&self, _: &[u8]) {}
|
||||
fn make_int(&self, _: BigInt) {}
|
||||
fn make_tuple(&self, _: impl Iterator<Item = ()>) {}
|
||||
fn make_code(&self, _: CodeObject) -> Result<()> {
|
||||
assert_eq!(self.0.get(), 3);
|
||||
self.0.set(7);
|
||||
Ok(())
|
||||
}
|
||||
fn make_code_with_constants(&self, code: CodeObject, constants: Vec<()>) -> Result<()> {
|
||||
assert_eq!(self.0.get(), 1);
|
||||
assert_eq!(constants.len(), 2);
|
||||
self.0.set(3);
|
||||
self.make_code(code)
|
||||
}
|
||||
fn make_code_with_constants_and_bytes(
|
||||
&self,
|
||||
code: CodeObject,
|
||||
constants: Vec<()>,
|
||||
bytes: Vec<u8>,
|
||||
) -> Result<()> {
|
||||
assert_eq!(self.0.get(), 0);
|
||||
assert_eq!(bytes, [1, 2]);
|
||||
self.0.set(1);
|
||||
self.make_code_with_constants(code, constants)
|
||||
}
|
||||
fn make_stop_iter(&self) -> Result<()> {
|
||||
Err(MarshalError::BadType)
|
||||
}
|
||||
fn make_list(&self, _: impl Iterator<Item = ()>) -> Result<()> {
|
||||
Err(MarshalError::BadType)
|
||||
}
|
||||
fn make_set(&self, _: impl Iterator<Item = ()>) -> Result<()> {
|
||||
Err(MarshalError::BadType)
|
||||
}
|
||||
fn make_frozenset(&self, _: impl Iterator<Item = ()>) -> Result<()> {
|
||||
Err(MarshalError::BadType)
|
||||
}
|
||||
fn make_dict(&self, _: impl Iterator<Item = ((), ())>) -> Result<()> {
|
||||
Err(MarshalError::BadType)
|
||||
}
|
||||
fn constant_bag(self) -> BasicBag {
|
||||
BasicBag
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actual_fields_default_preserves_existing_hook_chain() {
|
||||
let code = CodeObject {
|
||||
instructions: CodeUnits::try_from(&[][..]).unwrap(),
|
||||
locations: Box::new([]),
|
||||
flags: CodeFlags::empty(),
|
||||
posonlyarg_count: 0,
|
||||
arg_count: 0,
|
||||
kwonlyarg_count: 0,
|
||||
source_path: "code_storage.py".into(),
|
||||
first_line_number: None,
|
||||
max_stackdepth: 1,
|
||||
obj_name: "sample".into(),
|
||||
qualname: "sample".into(),
|
||||
constants: core::iter::empty().collect(),
|
||||
names: Box::new([]),
|
||||
varnames: Box::new([]),
|
||||
cellvars: Box::new([]),
|
||||
freevars: Box::new([]),
|
||||
localspluskinds: Box::new([]),
|
||||
linetable: Box::new([]),
|
||||
exceptiontable: Box::new([]),
|
||||
};
|
||||
let calls = Cell::new(0);
|
||||
HookBag(&calls)
|
||||
.make_code_with_fields(
|
||||
code,
|
||||
alloc::vec![(), ()],
|
||||
alloc::vec![1, 2],
|
||||
CodeFieldObjects {
|
||||
constants: (),
|
||||
names: (),
|
||||
localsplus_names: (),
|
||||
localsplus_kinds: (),
|
||||
linetable: (),
|
||||
exceptiontable: (),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(calls.get(), 7);
|
||||
}
|
||||
}
|
||||
+237
-68
@@ -154,11 +154,11 @@ pub struct ReplaceArgs {
|
||||
#[pyarg(named, optional)]
|
||||
co_firstlineno: OptionalArg<u32>,
|
||||
#[pyarg(named, optional)]
|
||||
co_consts: OptionalArg<Vec<PyObjectRef>>,
|
||||
co_consts: OptionalArg<PyTupleRef>,
|
||||
#[pyarg(named, optional)]
|
||||
co_name: OptionalArg<PyStrRef>,
|
||||
#[pyarg(named, optional)]
|
||||
co_names: OptionalArg<Vec<PyObjectRef>>,
|
||||
co_names: OptionalArg<PyTupleRef>,
|
||||
#[pyarg(named, optional)]
|
||||
co_flags: OptionalArg<u32>,
|
||||
#[pyarg(named, optional)]
|
||||
@@ -256,7 +256,22 @@ fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Immutable tuple backing for runtime constants. Construction is private so
|
||||
/// the only exposed slice always borrows a real tuple owner.
|
||||
#[derive(Clone)]
|
||||
pub struct LiteralTupleOwner(PyTupleRef);
|
||||
|
||||
impl AsRef<[Literal]> for LiteralTupleOwner {
|
||||
fn as_ref(&self) -> &[Literal] {
|
||||
let elements = self.0.as_slice();
|
||||
// SAFETY: Literal is transparent over PyObjectRef and accepts every
|
||||
// Python object. The immutable slice cannot outlive this tuple owner.
|
||||
unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Constant for Literal {
|
||||
type TupleOwner = LiteralTupleOwner;
|
||||
type Name = &'static PyStrInterned;
|
||||
fn borrow_constant(&self) -> BorrowedConstant<'_, Self> {
|
||||
borrow_obj_constant(&self.0)
|
||||
@@ -410,6 +425,14 @@ pub struct PyCode {
|
||||
/// Derived once so frame-local proxy operations do not repeatedly scan
|
||||
/// merged cell variables.
|
||||
localsplus_names: Box<[&'static PyStrInterned]>,
|
||||
names_tuple: PyTupleRef,
|
||||
localsplus_names_tuple: PyTupleRef,
|
||||
localsplus_kinds_bytes: PyBytesRef,
|
||||
linetable_bytes: PyBytesRef,
|
||||
exceptiontable_bytes: PyBytesRef,
|
||||
varnames_tuple: OnceCell<PyTupleRef>,
|
||||
cellvars_tuple: OnceCell<PyTupleRef>,
|
||||
freevars_tuple: OnceCell<PyTupleRef>,
|
||||
#[pymember(name = "co_filename")]
|
||||
source_path: AtomicPtr<PyStrInterned>,
|
||||
#[pymember(name = "co_nlocals")]
|
||||
@@ -523,7 +546,19 @@ impl PyCode {
|
||||
/// The input must not be cloned from an already instrumented PyCode:
|
||||
/// generic monitoring wrappers need side tables that CodeObject does not
|
||||
/// carry. Python `replace` reconstructs from canonical bytes instead.
|
||||
pub fn new(mut code: CodeObject) -> Self {
|
||||
pub fn new(mut code: CodeObject, ctx: &Context) -> Self {
|
||||
if code.constants.tuple_owner().is_none() {
|
||||
let boxed = core::mem::replace(&mut code.constants, core::iter::empty().collect());
|
||||
let tuple = ctx.new_tuple(
|
||||
boxed
|
||||
.into_boxed()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.map(|x| x.0)
|
||||
.collect(),
|
||||
);
|
||||
code.constants = bytecode::Constants::from_tuple(LiteralTupleOwner(tuple));
|
||||
}
|
||||
code.instructions.normalize_get_iter(code.flags);
|
||||
let canonical_code = code.instructions.original_bytes().into_boxed_slice();
|
||||
let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned;
|
||||
@@ -561,8 +596,26 @@ impl PyCode {
|
||||
| Instruction::InstrumentedEndAsyncFor
|
||||
)
|
||||
});
|
||||
let names_tuple = ctx.new_tuple(code.names.iter().map(|name| name.to_object()).collect());
|
||||
let localsplus_names_tuple = ctx.new_tuple(
|
||||
localsplus_names
|
||||
.iter()
|
||||
.map(|name| name.to_object())
|
||||
.collect(),
|
||||
);
|
||||
let localsplus_kinds_bytes = ctx.new_bytes(code.localspluskinds.to_vec());
|
||||
let linetable_bytes = ctx.new_bytes(code.linetable.to_vec());
|
||||
let exceptiontable_bytes = ctx.new_bytes(code.exceptiontable.to_vec());
|
||||
Self {
|
||||
code,
|
||||
names_tuple,
|
||||
localsplus_names_tuple,
|
||||
localsplus_kinds_bytes,
|
||||
linetable_bytes,
|
||||
exceptiontable_bytes,
|
||||
varnames_tuple: OnceCell::new(),
|
||||
cellvars_tuple: OnceCell::new(),
|
||||
freevars_tuple: OnceCell::new(),
|
||||
canonical_code,
|
||||
code_bytes: OnceCell::new(),
|
||||
localsplus_names,
|
||||
@@ -575,6 +628,65 @@ impl PyCode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach validated public-reader fields before publication. Superseded
|
||||
/// boxed constants are dropped before the returned code can be observed.
|
||||
pub(crate) fn from_marshaled_fields(
|
||||
mut code: CodeObject,
|
||||
fields: rustpython_compiler_core::marshal::CodeFieldObjects<PyObjectRef>,
|
||||
ctx: &Context,
|
||||
) -> Result<Self, rustpython_compiler_core::marshal::MarshalError> {
|
||||
use rustpython_compiler_core::marshal::MarshalError;
|
||||
let constants = fields
|
||||
.constants
|
||||
.downcast::<super::PyTuple>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
let names = fields
|
||||
.names
|
||||
.downcast::<super::PyTuple>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
let locals = fields
|
||||
.localsplus_names
|
||||
.downcast::<super::PyTuple>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
let kinds = fields
|
||||
.localsplus_kinds
|
||||
.downcast::<super::PyBytes>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
let lines = fields
|
||||
.linetable
|
||||
.downcast::<super::PyBytes>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
let exceptions = fields
|
||||
.exceptiontable
|
||||
.downcast::<super::PyBytes>()
|
||||
.map_err(|_| MarshalError::BadType)?;
|
||||
code.constants = bytecode::Constants::from_tuple(LiteralTupleOwner(constants));
|
||||
let mut result = Self::new(code, ctx);
|
||||
result.retain_normalized_names(names, Some(locals));
|
||||
result.localsplus_kinds_bytes = kinds;
|
||||
result.linetable_bytes = lines;
|
||||
result.exceptiontable_bytes = exceptions;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn retain_normalized_names(&mut self, names: PyTupleRef, locals: Option<PyTupleRef>) {
|
||||
let matches = |tuple: &PyTupleRef, canonical: &[&'static PyStrInterned]| {
|
||||
tuple.len() == canonical.len()
|
||||
&& tuple
|
||||
.iter()
|
||||
.zip(canonical)
|
||||
.all(|(item, name)| item.is(*name))
|
||||
};
|
||||
if matches(&names, &self.code.names) {
|
||||
self.names_tuple = names;
|
||||
}
|
||||
if let Some(locals) = locals
|
||||
&& matches(&locals, &self.localsplus_names)
|
||||
{
|
||||
self.localsplus_names_tuple = locals;
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn localsplus_name(&self, index: usize) -> &'static PyStrInterned {
|
||||
// Bytecode operands and frame-proxy indices are validated against
|
||||
@@ -596,7 +708,11 @@ impl PyCode {
|
||||
}
|
||||
|
||||
pub fn new_ref_with_bag(vm: &VirtualMachine, code: CodeObject) -> PyRef<Self> {
|
||||
PyRef::new_ref(Self::new(code), vm.ctx.types.code_type.to_owned(), None)
|
||||
PyRef::new_ref(
|
||||
Self::new(code, &vm.ctx),
|
||||
vm.ctx.types.code_type.to_owned(),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn new_ref_from_bytecode(vm: &VirtualMachine, code: bytecode::CodeObject) -> PyRef<Self> {
|
||||
@@ -900,16 +1016,7 @@ impl Constructor for PyCode {
|
||||
let instructions = CodeUnits::try_from(bytecode_bytes)
|
||||
.map_err(|e| vm.new_value_error(format!("invalid bytecode: {e}")))?;
|
||||
|
||||
// Convert constants
|
||||
let constants = args
|
||||
.consts
|
||||
.as_slice()
|
||||
.iter()
|
||||
.map(|obj| {
|
||||
// Convert PyObject to Literal constant. For now, just wrap it
|
||||
Literal(obj.clone())
|
||||
})
|
||||
.collect();
|
||||
let constants = bytecode::Constants::from_tuple(LiteralTupleOwner(args.consts));
|
||||
|
||||
// Create locations (start and end pairs)
|
||||
let row = if args.firstlineno > 0 {
|
||||
@@ -971,7 +1078,10 @@ impl Constructor for PyCode {
|
||||
exceptiontable: args.exceptiontable.as_bytes().to_vec().into_boxed_slice(),
|
||||
};
|
||||
|
||||
Ok(Self::new(code))
|
||||
let mut result = Self::new(code, &vm.ctx);
|
||||
result.linetable_bytes = args.linetable;
|
||||
result.exceptiontable_bytes = args.exceptiontable;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -989,12 +1099,12 @@ impl PyCode {
|
||||
impl Py<PyCode> {
|
||||
#[pygetset]
|
||||
pub fn co_cellvars(&self, vm: &VirtualMachine) -> PyTupleRef {
|
||||
let cellvars = self
|
||||
.cellvars
|
||||
.iter()
|
||||
.map(|name| name.to_pyobject(vm))
|
||||
.collect();
|
||||
vm.ctx.new_tuple(cellvars)
|
||||
self.cellvars_tuple
|
||||
.get_or_init(|| {
|
||||
vm.ctx
|
||||
.new_tuple(self.cellvars.iter().map(|name| name.to_object()).collect())
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
@@ -1003,27 +1113,33 @@ impl Py<PyCode> {
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
fn co_consts(&self, vm: &VirtualMachine) -> PyTupleRef {
|
||||
let consts = self.code.constants.iter().map(|x| x.0.clone()).collect();
|
||||
vm.ctx.new_tuple(consts)
|
||||
fn co_consts(&self) -> PyTupleRef {
|
||||
self.code
|
||||
.constants
|
||||
.tuple_owner()
|
||||
.expect("finalized constants")
|
||||
.0
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
fn co_names(&self, vm: &VirtualMachine) -> PyTupleRef {
|
||||
let names = self
|
||||
.code
|
||||
.names
|
||||
.deref()
|
||||
.iter()
|
||||
.map(|name| name.to_pyobject(vm))
|
||||
.collect();
|
||||
vm.ctx.new_tuple(names)
|
||||
fn co_names(&self) -> PyTupleRef {
|
||||
self.names_tuple.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
pub fn co_varnames(&self, vm: &VirtualMachine) -> PyTupleRef {
|
||||
let varnames = self.code.varnames.iter().map(|s| s.to_object()).collect();
|
||||
vm.ctx.new_tuple(varnames)
|
||||
self.varnames_tuple
|
||||
.get_or_init(|| {
|
||||
vm.ctx.new_tuple(
|
||||
self.code
|
||||
.varnames
|
||||
.iter()
|
||||
.map(|name| name.to_object())
|
||||
.collect(),
|
||||
)
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
@@ -1047,26 +1163,27 @@ impl Py<PyCode> {
|
||||
|
||||
#[pygetset]
|
||||
pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef {
|
||||
let names = self
|
||||
.code
|
||||
.freevars
|
||||
.deref()
|
||||
.iter()
|
||||
.map(|name| name.to_pyobject(vm))
|
||||
.collect();
|
||||
vm.ctx.new_tuple(names)
|
||||
self.freevars_tuple
|
||||
.get_or_init(|| {
|
||||
vm.ctx.new_tuple(
|
||||
self.code
|
||||
.freevars
|
||||
.iter()
|
||||
.map(|name| name.to_object())
|
||||
.collect(),
|
||||
)
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
pub fn co_linetable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef {
|
||||
// Return the actual linetable from the code object
|
||||
vm.ctx.new_bytes(self.code.linetable.to_vec())
|
||||
pub fn co_linetable(&self) -> PyBytesRef {
|
||||
self.linetable_bytes.clone()
|
||||
}
|
||||
|
||||
#[pygetset]
|
||||
pub fn co_exceptiontable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef {
|
||||
// Return the actual exception table from the code object
|
||||
vm.ctx.new_bytes(self.code.exceptiontable.to_vec())
|
||||
pub fn co_exceptiontable(&self) -> PyBytesRef {
|
||||
self.exceptiontable_bytes.clone()
|
||||
}
|
||||
|
||||
// spell-checker: ignore lnotab
|
||||
@@ -1398,7 +1515,13 @@ impl Py<PyCode> {
|
||||
|
||||
let constants = match co_consts {
|
||||
OptionalArg::Present(constants) => constants,
|
||||
OptionalArg::Missing => self.code.constants.iter().map(|x| x.0.clone()).collect(),
|
||||
OptionalArg::Missing => self
|
||||
.code
|
||||
.constants
|
||||
.tuple_owner()
|
||||
.expect("finalized constants")
|
||||
.0
|
||||
.clone(),
|
||||
};
|
||||
|
||||
let obj_name = match co_name {
|
||||
@@ -1408,13 +1531,7 @@ impl Py<PyCode> {
|
||||
|
||||
let names = match co_names {
|
||||
OptionalArg::Present(names) => names,
|
||||
OptionalArg::Missing => self
|
||||
.code
|
||||
.names
|
||||
.deref()
|
||||
.iter()
|
||||
.map(|name| name.to_pyobject(vm))
|
||||
.collect(),
|
||||
OptionalArg::Missing => self.names_tuple.clone(),
|
||||
};
|
||||
|
||||
let flags = CodeFlags::from_bits_truncate(match co_flags {
|
||||
@@ -1503,15 +1620,13 @@ impl Py<PyCode> {
|
||||
|
||||
// Handle linetable and exceptiontable
|
||||
let linetable = match co_linetable {
|
||||
OptionalArg::Present(linetable) => linetable.as_bytes().to_vec().into_boxed_slice(),
|
||||
OptionalArg::Missing => self.code.linetable.clone(),
|
||||
OptionalArg::Present(linetable) => linetable,
|
||||
OptionalArg::Missing => self.linetable_bytes.clone(),
|
||||
};
|
||||
|
||||
let exceptiontable = match co_exceptiontable {
|
||||
OptionalArg::Present(exceptiontable) => {
|
||||
exceptiontable.as_bytes().to_vec().into_boxed_slice()
|
||||
}
|
||||
OptionalArg::Missing => self.code.exceptiontable.clone(),
|
||||
OptionalArg::Present(exceptiontable) => exceptiontable,
|
||||
OptionalArg::Missing => self.exceptiontable_bytes.clone(),
|
||||
};
|
||||
|
||||
let new_code = CodeObject {
|
||||
@@ -1529,17 +1644,21 @@ impl Py<PyCode> {
|
||||
// FIXME: invalid locations. Actually locations is a duplication of linetable.
|
||||
// It can be removed once we move every other code to use linetable only.
|
||||
locations: self.code.locations.clone(),
|
||||
constants: constants.into_iter().map(Literal).collect(),
|
||||
names: intern_all(names, "co_names")?,
|
||||
constants: bytecode::Constants::from_tuple(LiteralTupleOwner(constants)),
|
||||
names: intern_all(names.as_slice().to_vec(), "co_names")?,
|
||||
varnames,
|
||||
cellvars,
|
||||
freevars,
|
||||
localspluskinds,
|
||||
linetable,
|
||||
exceptiontable,
|
||||
linetable: linetable.as_bytes().into(),
|
||||
exceptiontable: exceptiontable.as_bytes().into(),
|
||||
};
|
||||
|
||||
Ok(PyCode::new(new_code))
|
||||
let mut result = PyCode::new(new_code, &vm.ctx);
|
||||
result.retain_normalized_names(names, None);
|
||||
result.linetable_bytes = linetable;
|
||||
result.exceptiontable_bytes = exceptiontable;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[pymethod]
|
||||
@@ -1661,3 +1780,53 @@ impl<'a> LineTableReader<'a> {
|
||||
pub(crate) fn init(ctx: &'static Context) {
|
||||
PyCode::extend_class(ctx, ctx.types.code_type);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tuple_constants_share_owner_and_copy_on_write() {
|
||||
crate::Interpreter::without_stdlib(Default::default()).enter(|vm| {
|
||||
let child: PyObjectRef = vm.ctx.new_list(vec![]).into();
|
||||
assert_eq!(child.strong_count(), 1);
|
||||
let owner = vm.ctx.new_tuple(vec![child.clone()]);
|
||||
let mut constants = bytecode::Constants::from_tuple(LiteralTupleOwner(owner.clone()));
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
let shared = constants.clone();
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
constants[bytecode::oparg::ConstIdx::from_u32(0)] = Literal(vm.ctx.none());
|
||||
assert!(constants.tuple_owner().is_none());
|
||||
assert!(shared[bytecode::oparg::ConstIdx::from_u32(0)].0.is(&child));
|
||||
assert!(owner.as_slice()[0].is(&child));
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
drop(shared);
|
||||
drop(owner);
|
||||
assert_eq!(child.strong_count(), 1);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "compiler")]
|
||||
#[test]
|
||||
fn published_code_has_one_constant_owner_representation() {
|
||||
crate::Interpreter::without_stdlib(Default::default()).enter(|vm| {
|
||||
let compiled = vm
|
||||
.compile("pass", crate::compiler::Mode::Exec, "code_storage.py")
|
||||
.unwrap();
|
||||
let child: PyObjectRef = vm.ctx.new_list(vec![]).into();
|
||||
let mut raw = compiled.code.clone();
|
||||
raw.constants = core::iter::once(Literal(child.clone())).collect();
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
let code = PyCode::new(raw, &vm.ctx);
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
let owner = code.code.constants.tuple_owner().unwrap().0.clone();
|
||||
let cloned = code.code.clone();
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
drop(code);
|
||||
drop(cloned);
|
||||
assert_eq!(child.strong_count(), 2);
|
||||
drop(owner);
|
||||
assert_eq!(child.strong_count(), 1);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -602,6 +602,25 @@ mod decl {
|
||||
}
|
||||
Ok(crate::builtins::PyCode::new_ref_with_bag(self.vm, code).into())
|
||||
}
|
||||
fn make_code_with_fields(
|
||||
&self,
|
||||
code: CodeObject,
|
||||
constants: Vec<Self::Value>,
|
||||
_code_bytes: Vec<u8>,
|
||||
fields: marshal::CodeFieldObjects<Self::Value>,
|
||||
) -> Result<Self::Value, marshal::MarshalError> {
|
||||
if !self.allow_code {
|
||||
return Err(self.remember_python_error(
|
||||
self.vm
|
||||
.new_value_error("unmarshalling code objects is disallowed"),
|
||||
));
|
||||
}
|
||||
// The actual tuple in fields is the surviving owner. Discard the
|
||||
// compatibility channel's temporary child references first.
|
||||
drop(constants);
|
||||
let code = PyCode::from_marshaled_fields(code, fields, &self.vm.ctx)?;
|
||||
Ok(code.into_ref(&self.vm.ctx).into())
|
||||
}
|
||||
fn make_stop_iter(&self) -> Result<Self::Value, marshal::MarshalError> {
|
||||
Ok(self.vm.ctx.exceptions.stop_iteration.to_owned().into())
|
||||
}
|
||||
|
||||
@@ -56,7 +56,11 @@ impl VirtualMachine {
|
||||
|
||||
pub fn new_code(&self, code: impl IntoCodeObject) -> PyRef<PyCode> {
|
||||
let code = code.into_code_object(self);
|
||||
PyRef::new_ref(PyCode::new(code), self.ctx.types.code_type.to_owned(), None)
|
||||
PyRef::new_ref(
|
||||
PyCode::new(code, &self.ctx),
|
||||
self.ctx.types.code_type.to_owned(),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn new_tuple(&self, value: impl IntoPyTuple) -> PyTupleRef {
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Stable surviving code fields and sole constants tuple ownership."""
|
||||
|
||||
import gc
|
||||
import marshal
|
||||
import sys
|
||||
import types
|
||||
import weakref
|
||||
|
||||
SOURCE = "def sample(a):\n try:\n return a + marker\n except TypeError:\n return 0\n"
|
||||
FILENAME = "stable_code_fields.py"
|
||||
base = compile(SOURCE, FILENAME, "exec").co_consts[0]
|
||||
for name in (
|
||||
"co_consts",
|
||||
"co_names",
|
||||
"co_varnames",
|
||||
"co_freevars",
|
||||
"co_cellvars",
|
||||
"co_linetable",
|
||||
"co_exceptiontable",
|
||||
"co_code",
|
||||
):
|
||||
value = getattr(base, name)
|
||||
assert getattr(base, name) is value
|
||||
|
||||
replacement = base.replace()
|
||||
for name in ("co_consts", "co_names", "co_linetable", "co_exceptiontable"):
|
||||
assert getattr(replacement, name) is getattr(base, name)
|
||||
assert replacement.co_code == base.co_code
|
||||
assert replacement.co_code is not base.co_code
|
||||
assert replacement.co_varnames == base.co_varnames
|
||||
assert replacement.co_varnames is not base.co_varnames
|
||||
|
||||
|
||||
class Child:
|
||||
pass
|
||||
|
||||
|
||||
child = Child()
|
||||
weak = weakref.ref(child)
|
||||
values = (child, [1, 2], slice(1, 5, 2), frozenset((3, 4)))
|
||||
owned = base.replace(co_consts=values)
|
||||
assert owned.co_consts is values
|
||||
assert owned.co_consts[0] is child
|
||||
assert owned.co_consts[2] is values[2]
|
||||
assert owned.co_consts[3] is values[3]
|
||||
clone = owned.replace()
|
||||
assert clone.co_consts is values
|
||||
del child, owned, values
|
||||
gc.collect()
|
||||
assert weak() is not None
|
||||
del clone
|
||||
gc.collect()
|
||||
assert weak() is None
|
||||
|
||||
# The public constructor copies validated names, retains its constants and
|
||||
# surviving byte tables, and independently creates its exported code cache.
|
||||
input_code = bytes(bytearray(base.co_code))
|
||||
input_names = tuple(list(base.co_names))
|
||||
constructed = types.CodeType(
|
||||
base.co_argcount,
|
||||
base.co_posonlyargcount,
|
||||
base.co_kwonlyargcount,
|
||||
base.co_nlocals,
|
||||
base.co_stacksize,
|
||||
base.co_flags,
|
||||
input_code,
|
||||
base.co_consts,
|
||||
input_names,
|
||||
base.co_varnames,
|
||||
FILENAME,
|
||||
base.co_name,
|
||||
base.co_qualname,
|
||||
base.co_firstlineno,
|
||||
base.co_linetable,
|
||||
base.co_exceptiontable,
|
||||
base.co_freevars,
|
||||
base.co_cellvars,
|
||||
)
|
||||
assert constructed.co_consts is base.co_consts
|
||||
assert constructed.co_names == input_names
|
||||
assert constructed.co_names is not input_names
|
||||
assert constructed.co_linetable is base.co_linetable
|
||||
assert constructed.co_exceptiontable is base.co_exceptiontable
|
||||
assert constructed.co_code == input_code
|
||||
assert constructed.co_code is not input_code
|
||||
|
||||
# Retain the known normalization gap. Rust never mutates a published caller
|
||||
# tuple; when a member needs canonical interning it creates a normalized copy.
|
||||
raw_name = "".join(("mar", "ker"))
|
||||
assert raw_name is not base.co_names[0]
|
||||
names = (raw_name,)
|
||||
normalized = base.replace(co_names=names)
|
||||
assert normalized.co_names[0] is base.co_names[0]
|
||||
if sys.implementation.name == "rustpython":
|
||||
assert normalized.co_names is not names
|
||||
assert names[0] is raw_name
|
||||
else:
|
||||
assert normalized.co_names is names
|
||||
assert names[0] is base.co_names[0]
|
||||
|
||||
outer = compile(
|
||||
"def outer(a):\n def inner():\n return a\n return inner\n",
|
||||
FILENAME,
|
||||
"exec",
|
||||
).co_consts[0]
|
||||
inner = next(value for value in outer.co_consts if isinstance(value, types.CodeType))
|
||||
assert outer.co_cellvars == ("a",)
|
||||
assert outer.co_cellvars is outer.co_cellvars
|
||||
assert inner.co_freevars == ("a",)
|
||||
assert inner.co_freevars is inner.co_freevars
|
||||
|
||||
# Whole unchanged streams emitted by the pinned final oracle. Both field-first
|
||||
# and code-first ordering exercise reader reference identity; the current Rust
|
||||
# writer is deliberately not used to create the expected aliases.
|
||||
FIELD_FIRST = bytes.fromhex(
|
||||
"2905a901e900000000a902da066d61726b6572da09547970654572726f72f3230000008100f002030511d80f"
|
||||
"1094368d7ad00819f8dc0b14f400010511da0f10f003010511faf310000000830c100090091f039a011f039e"
|
||||
"011f03e30100000000000000000000000400000003000000f34400000080000000190055005b000000000000"
|
||||
"0000002a000000000000000000000021001e005b0200000000000000000600630500001a001d001b005d0021"
|
||||
"00680039031b006801720000000072020000002901da0161730100000026da15737461626c655f636f64655f"
|
||||
"6669656c64732e7079da0673616d706c65720b0000000100000072050000007206000000"
|
||||
)
|
||||
CODE_FIRST = bytes.fromhex(
|
||||
"2905e30100000000000000000000000400000003000000f34400000080000000190055005b00000000000000"
|
||||
"00002a000000000000000000000021001e005b0200000000000000000600630500001a001d001b005d002100"
|
||||
"680039031b006801a901e900000000a902da066d61726b6572da09547970654572726f722901da0161730100"
|
||||
"000026da15737461626c655f636f64655f6669656c64732e7079da0673616d706c65720900000001000000f3"
|
||||
"230000008100f002030511d80f1094368d7ad00819f8dc0b14f400010511da0f10f003010511faf310000000"
|
||||
"830c100090091f039a011f039e011f0372020000007204000000720a000000720b000000"
|
||||
)
|
||||
for order, payload in (("field_first", FIELD_FIRST), ("code_first", CODE_FIRST)):
|
||||
decoded = marshal.loads(payload)
|
||||
if order == "field_first":
|
||||
consts, names, lines, exceptions, code = decoded
|
||||
else:
|
||||
code, consts, names, lines, exceptions = decoded
|
||||
assert code.co_consts is consts
|
||||
assert code.co_names is names
|
||||
assert code.co_linetable is lines
|
||||
assert code.co_exceptiontable is exceptions
|
||||
assert code.co_code == base.co_code
|
||||
assert code.co_code is code.co_code
|
||||
function = types.FunctionType(code, {"marker": 10})
|
||||
assert function(2) == 12
|
||||
assert function("x") == 0
|
||||
|
||||
# Final CodeType.replace accepts tuple fields, not arbitrary iterables.
|
||||
for field in ("co_consts", "co_names"):
|
||||
try:
|
||||
base.replace(**{field: []})
|
||||
except TypeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(field + " accepted a list")
|
||||
@@ -0,0 +1,58 @@
|
||||
# Stable code fields: storage-only follow-on
|
||||
|
||||
This stage follows canonical code storage commit6746df059. It does not change the
|
||||
marshal writer, reference flags, native call ownership, permanent string interner
|
||||
or filename handling. Full serialization parity remains unresolved.
|
||||
|
||||
Runtime `Constants<Literal>` owns one immutable Python tuple. It does not keep a
|
||||
second boxed list of owning child references beside a getter cache. The compiler
|
||||
representation remains boxed. Cloning a runtime constant pool shares the tuple;
|
||||
mutable access copies the elements before replacing the owner. Published tuples
|
||||
are never mutated. The owner-bound Literal slice uses the existing transparent
|
||||
layout over PyObjectRef and never calls the slice/frozenset constant adapter.
|
||||
|
||||
Surviving names, locals-plus names/kinds and line/exception tables have stable
|
||||
Python owners. Derived variable/cell/free-name tuples are separate lazy caches.
|
||||
Constructor names are copied after validation; replace retains unchanged names.
|
||||
Public marshal decode retains actual field objects when normalized name member
|
||||
identities permit it, otherwise uses a safe normalized copy. That copy has a
|
||||
known tuple-identity difference from CPython's in-place name normalization.
|
||||
Existing constant interning behavior is preserved, not expanded.
|
||||
|
||||
Decoder input co_code bytes are not retained as the exported co_code object.
|
||||
That export remains A0's independent lazy cache of canonical normalized bytes.
|
||||
The new reader hook defaults to the existing bytes/constants/code hook chain.
|
||||
Runtime decoding drops temporary child vectors and replaced boxes before code
|
||||
publication. Reader allow_code, type, depth and incomplete-reference checks stay
|
||||
in force.
|
||||
|
||||
## Rust source API migration
|
||||
|
||||
- `PyCode::new(code, &Context)` now requires the creating runtime's context so
|
||||
final tuple/metadata objects exist before publication. Callers must still pass
|
||||
pristine instruction streams, as required by A0.
|
||||
- Implementations of compiler-core `Constant` supply `TupleOwner: Clone +
|
||||
AsRef<[Self]>`. BasicBag uses boxed constants; the runtime uses a private
|
||||
validated tuple owner. Downstream bags must provide an immutable owner-bound
|
||||
element view. No Python object type is introduced into compiler-core.
|
||||
- Public getters for co_linetable/co_exceptiontable no longer require a VM
|
||||
argument; they return retained Python bytes owners.
|
||||
- A new `MarshalBag::make_code_with_fields` is additive. Its default delegates
|
||||
`make_code_with_constants_and_bytes`, preserving earlier customization hooks.
|
||||
|
||||
The checked-in Python test retains full certified streams in both alias orders,
|
||||
checks identity and execution, and explicitly preserves the normalization gap.
|
||||
Rust tests verify actual child-owner counts, COW isolation, publication and
|
||||
reader-hook ordering. No test count is used by production serialization.
|
||||
|
||||
## Integration limits
|
||||
|
||||
The existing untracked PyCode policy and permanent-intern ownership contracts
|
||||
are unchanged. Pinned CPython only tracks code objects in Py_GIL_DISABLED
|
||||
builds; the certified GIL-enabled oracle is untracked too. This stage does not claim a general cycle-collection repair or mortal
|
||||
interning parity. Python child ownership moves to the tuple edge and ordinary
|
||||
Rust drops release it; a future traversal change must account for that real edge.
|
||||
Compilation against current main after #9080 is a separate integration gate even
|
||||
though the source audit found no direct overlap and transparent ownership remains.
|
||||
Full workspace/CAPI/release and the canonical metadata dependencies remain
|
||||
required before a general integration/CI-green claim.
|
||||
Reference in new issue
Block a user