diff --git a/crates/capi/src/descrobject.rs b/crates/capi/src/descrobject.rs index 036cbf8ba..18e18acde 100644 --- a/crates/capi/src/descrobject.rs +++ b/crates/capi/src/descrobject.rs @@ -5,8 +5,8 @@ use crate::pystate::with_vm; use crate::util::{CStrExt, FfiPtrExt}; use core::ffi::{c_char, c_int, c_void}; use rustpython_vm::builtins::{ - DescriptorMemberDef, MemberAccess, MemberKind, PY_READONLY, PY_RELATIVE_OFFSET, - PyDescriptorOwned, PyGetSet, PyMappingProxy, PyMemberDescriptor, PyType, + DescriptorMemberDef, MemberAccess, MemberKind, PY_RELATIVE_OFFSET, PyDescriptorOwned, PyGetSet, + PyMappingProxy, PyMemberDescriptor, PyType, }; use rustpython_vm::common::lock::PyRwLock; use rustpython_vm::function::PySetterValue; @@ -126,17 +126,15 @@ impl PyMemberDef { self.type_code ))); }; - if self.offset < 0 { - return Err(vm.new_system_error("PyDescr_NewMember does not support negative offsets")); - } - if self.flags & PY_RELATIVE_OFFSET != 0 { - return Err( - vm.new_system_error("PyDescr_NewMember does not support Py_RELATIVE_OFFSET") - ); + let mut offset = self.offset; + let mut flags = self.flags; + if flags & PY_RELATIVE_OFFSET != 0 { + // type creation adds tp_basicsize and clears the flag before GetOne. + offset += (rustpython_vm::object::SIZEOF_PYOBJECT_HEAD + ty.slots.basicsize) as isize; + flags &= !PY_RELATIVE_OFFSET; } let doc = unsafe { self.doc.try_as_str_opt(vm) }?.map(str::to_owned); - let readonly = self.flags & PY_READONLY != 0; let descriptor = PyMemberDescriptor { common: PyDescriptorOwned { @@ -147,11 +145,11 @@ impl PyMemberDef { member: DescriptorMemberDef { name: name.to_owned(), kind, - offset: self.offset, - flags: if readonly { PY_READONLY } else { 0 }, + offset, + flags, doc, }, - // Instance members live in the slot array. `offset` is that index. + // `offset` is a byte offset from the object to a pointer cell. access: MemberAccess::Slot, }; diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 78d10c141..1307ee211 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -355,9 +355,9 @@ pub(crate) type MemberSetterFunc = /// Where `PyMemberDef.offset` points. /// -/// Builtin payloads keep fields behind locks, so a byte offset from the -/// object header does not address them. Slot members use `offset` as an -/// index into the instance slot array. Function members ignore `offset`. +/// Slot members use a byte offset from the object to an inline pointer cell. +/// Builtin payloads keep fields behind locks, so those members use a function +/// and ignore `offset`. #[derive(Clone, Copy)] pub enum MemberAccess { Func { @@ -415,10 +415,10 @@ pub struct PyMemberDescriptor { } impl PyMemberDescriptor { - /// Slot index for an instance-slot member. `None` when the offset is not a slot. - pub(crate) fn slot_offset(&self) -> Option { + /// Byte offset of an instance-slot member. `None` when this is not a slot. + pub(crate) fn slot_offset(&self) -> Option { match self.access { - MemberAccess::Slot => Some(self.member.offset as usize), + MemberAccess::Slot => Some(self.member.offset), _ => None, } } @@ -436,9 +436,7 @@ impl PyMemberDescriptor { } match self.access { MemberAccess::Func { get, .. } => get(vm, obj), - MemberAccess::Slot => { - get_slot_from_object(&obj, self.member.offset as usize, &self.member, vm) - } + MemberAccess::Slot => get_slot_from_object(&obj, self.member.offset, &self.member, vm), MemberAccess::TupleItem => { let index = self.member.offset as usize; let tuple = obj.downcast_ref::().ok_or_else(|| { @@ -468,7 +466,7 @@ impl PyMemberDescriptor { None => Err(vm.new_attribute_error("readonly attribute")), }, MemberAccess::Slot => { - set_slot_at_object(&obj, self.member.offset as usize, &self.member, value, vm) + set_slot_at_object(&obj, self.member.offset, &self.member, value, vm) } MemberAccess::TupleItem => Err(vm.new_attribute_error("readonly attribute")), } @@ -566,7 +564,7 @@ impl PyMemberDescriptor { // PyMember_GetOne for a value stored in the instance slot array. fn get_slot_from_object( obj: &PyObject, - offset: usize, + offset: isize, member: &PyMemberDef, vm: &VirtualMachine, ) -> PyResult { @@ -589,7 +587,7 @@ fn get_slot_from_object( // PyMember_SetOne for a value stored in the instance slot array. fn set_slot_at_object( obj: &PyObject, - offset: usize, + offset: isize, member: &PyMemberDef, value: PySetterValue, vm: &VirtualMachine, diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 63bcb4afb..531fdafc3 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -2782,7 +2782,7 @@ impl Constructor for PyType { let member_def = PyMemberDef { name: mangled_name.clone(), kind: MemberKind::ObjectEx, - offset: offset as isize, + offset: crate::object::slot_member_offset(offset), flags: 0, doc: None, }; diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 7b94b38dd..bf9de22bc 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -6291,7 +6291,7 @@ impl ExecutingFrame<'_> { if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version { let slot_offset = - self.code.instructions.read_cache_u32(cache_base + 3) as usize; + self.code.instructions.read_cache_u32(cache_base + 3) as i32 as isize; if let Some(value) = owner.get_slot(slot_offset) { self.pop_stackref(); if oparg.is_method() { @@ -6381,7 +6381,7 @@ impl ExecutingFrame<'_> { if version_match { let slot_offset = - self.code.instructions.read_cache_u16(cache_base + 3) as usize; + self.code.instructions.read_cache_u16(cache_base + 3) as i16 as isize; let owner = self.pop_value(); let value = self.pop_value(); owner.set_slot(slot_offset, Some(value)); diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 3453bf1f9..32b547eef 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -30,7 +30,6 @@ use crate::{ class::StaticType, object::traverse::{MaybeTraverse, Traverse, TraverseFn}, }; -use itertools::Itertools; use alloc::fmt; @@ -244,7 +243,7 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { let old = dict.d.write().take(); drop(old); } - for slot in &ext.slots { + for slot in obj_ref.0.slot_cells() { drop(slot.store(None)); } } @@ -367,28 +366,16 @@ unsafe impl Link for GcLink { #[repr(C, align(8))] pub(super) struct ObjExt { pub(super) dict: Option, - /// Owned `PyObject*` cells. Null is empty. Each cell is one pointer so a - /// later step can place the same cells at a byte offset from the object. - pub(super) slots: Box<[PyAtomicRef]>, } impl ObjExt { - fn new( - dict: Option, - member_count: usize, - has_dict: bool, - inline_values: bool, - ) -> Self { + fn new(dict: Option, has_dict: bool, inline_values: bool) -> Self { Self { dict: if has_dict { Some(InstanceDict::from_opt(dict, inline_values)) } else { None }, - slots: core::iter::repeat_with(PyAtomicRef::::new_empty) - .take(member_count) - .collect_vec() - .into_boxed_slice(), } } } @@ -400,11 +387,46 @@ impl fmt::Debug for ObjExt { } /// Precomputed offset constants for prefix allocation. -/// All prefix components are align(8) and their sizes are multiples of 8, -/// so Layout::extend adds no inter-padding. +/// `ObjExt` and `WeakRefList` are align(8) and their sizes are multiples of 8, +/// so `Layout::extend` adds no padding between them. Slot cells are packed +/// immediately in front of `ObjExt`; any alignment padding is before the cells. const EXT_OFFSET: usize = core::mem::size_of::(); + +/// Byte offset of member cell `index` from the start of `PyInner`. +/// Cells live in front of the object, so the offset is negative. +/// +/// `ObjExt` stays flush with `PyInner`. A subclass that adds `__weakref__` +/// puts that list in front of the cells, so this offset does not move. +pub(crate) fn slot_member_offset(index: usize) -> isize { + // Cell 0 sits directly in front of ObjExt. Higher indexes extend further + // forward, so a base class offset stays valid on a subclass with more slots. + let cell = core::mem::size_of::>(); + -((EXT_OFFSET + (index + 1) * cell) as isize) +} + +fn slot_region_layout(member_count: usize) -> Option { + if member_count == 0 { + return None; + } + let cell = core::mem::size_of::>(); + let bytes = member_count * cell; + let align = core::mem::align_of::(); + // Padding goes in front of the cells so cell 0 stays flush with ObjExt. + // On wasm32 a pointer is 4 bytes and ObjExt is align 8, so an odd count + // would otherwise leave a gap where cell 0 is supposed to be. + let pad = (align - bytes % align) % align; + Some(core::alloc::Layout::from_size_align(pad + bytes, align).unwrap()) +} const WEAKREF_OFFSET: usize = core::mem::size_of::(); +/// Distance from `PyInner` back to a `WeakRefList`. +/// Layout: `[WeakRefList?][slots?][ObjExt?][PyInner]`. +fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize { + let ext = if has_ext { EXT_OFFSET } else { 0 }; + let slots = slot_region_layout(member_count).map_or(0, |layout| layout.size()); + ext + slots + WEAKREF_OFFSET +} + const _: () = assert!(core::mem::size_of::().is_multiple_of(core::mem::align_of::())); const _: () = assert!(core::mem::align_of::() >= core::mem::align_of::>()); @@ -441,7 +463,7 @@ pub(super) struct PyInner { pub(super) payload: T, } -pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::>(); +pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::>(); // ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits, // generation, owner and refs take eight bytes between them. A 64-bit header had @@ -476,8 +498,8 @@ impl PyInner { /// Access the ObjExt prefix at a negative offset from this PyInner. /// Returns None if this object was allocated without dict/slots. /// - /// Layout: [ObjExt?][WeakRefList?][PyInner] - /// ObjExt offset depends on whether WeakRefList is also present. + /// Layout: [WeakRefList?][slots?][ObjExt?][PyInner] + /// `ObjExt` is always immediately in front of `PyInner`. #[inline(always)] pub(super) fn ext_ref(&self) -> Option<&ObjExt> { let (flags, member_count) = self.read_type_flags(); @@ -485,31 +507,47 @@ impl PyInner { if !has_ext { return None; } - let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); - let offset = if has_weakref { - WEAKREF_OFFSET + EXT_OFFSET - } else { - EXT_OFFSET - }; let self_addr = (self as *const Self as *const u8).addr(); - let ext_ptr = core::ptr::with_exposed_provenance::(self_addr.wrapping_sub(offset)); + let ext_ptr = + core::ptr::with_exposed_provenance::(self_addr.wrapping_sub(EXT_OFFSET)); Some(unsafe { &*ext_ptr }) } + /// Member cells are the pointer array immediately before [`ObjExt`]. + /// Layout: `[WeakRefList?][PyAtomicRef; N][ObjExt?][PyInner]`. + pub(super) fn slot_cells(&self) -> &[PyAtomicRef] { + let Some(ext) = self.ext_ref() else { + return &[]; + }; + let (_, member_count) = self.read_type_flags(); + if member_count == 0 { + return &[]; + } + let cell = core::mem::size_of::>(); + // Index 0 is the cell adjacent to ObjExt; index i is i cells before it. + let first = (ext as *const ObjExt) + .addr() + .wrapping_sub(member_count * cell); + let ptr = core::ptr::with_exposed_provenance::>(first); + unsafe { core::slice::from_raw_parts(ptr, member_count) } + } + /// Access the WeakRefList prefix at a fixed negative offset from this PyInner. /// Returns None if the type does not support weakrefs. /// - /// Layout: [ObjExt?][WeakRefList?][PyInner] - /// WeakRefList is always immediately before PyInner (fixed WEAKREF_OFFSET). + /// Layout: [WeakRefList?][slots?][ObjExt?][PyInner] + /// The list sits in front of the slot cells so adding it on a subclass + /// does not move inherited member offsets. #[inline(always)] pub(super) fn weakref_list_ref(&self) -> Option<&WeakRefList> { - let (flags, _) = self.read_type_flags(); + let (flags, member_count) = self.read_type_flags(); if !flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF) { return None; } + let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; let self_addr = (self as *const Self as *const u8).addr(); let ptr = core::ptr::with_exposed_provenance::( - self_addr.wrapping_sub(WEAKREF_OFFSET), + self_addr.wrapping_sub(weakref_prefix_offset(has_ext, member_count)), ); Some(unsafe { &*ptr }) } @@ -1192,7 +1230,7 @@ impl PyInner { } /// Deallocate a PyInner, handling optional prefix(es). - /// Layout: [ObjExt?][WeakRefList?][PyInner] + /// Layout: `[WeakRefList?][PyAtomicRef; N][ObjExt?][PyInner]` /// /// # Safety /// `ptr` must be a valid pointer from `PyInner::new` and must not be used after this call. @@ -1210,18 +1248,21 @@ impl PyInner { // Reconstruct the same layout used in new() let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap(); - if has_ext { - layout = layout - .extend(core::alloc::Layout::new::()) - .unwrap() - .0; - } if has_weakref { layout = layout .extend(core::alloc::Layout::new::()) .unwrap() .0; } + if let Some(region) = slot_region_layout(member_count) { + layout = layout.extend(region).unwrap().0; + } + if has_ext { + layout = layout + .extend(core::alloc::Layout::new::()) + .unwrap() + .0; + } let (combined, inner_offset) = layout.extend(core::alloc::Layout::new::()).unwrap(); let combined = combined.pad_to_align(); @@ -1230,9 +1271,22 @@ impl PyInner { Self::drop_fields(ptr); - // Drop ObjExt if present (dict, slots) + // Drop member cells, then ObjExt. WeakRefList is in front of the cells. + let mut cursor = alloc_ptr; + if has_weakref { + cursor = cursor.add(core::mem::size_of::()); + } + if let Some(region) = slot_region_layout(member_count) { + let cell = core::mem::size_of::>(); + let first = cursor.add(region.size() - member_count * cell); + let cells = first.cast::>(); + for i in 0..member_count { + core::ptr::drop_in_place(cells.add(i)); + } + cursor = cursor.add(region.size()); + } if has_ext { - core::ptr::drop_in_place(alloc_ptr as *mut ObjExt); + core::ptr::drop_in_place(cursor.cast::()); } // WeakRefList has no Drop (just raw pointers), no drop_in_place needed @@ -1258,7 +1312,7 @@ impl PyInner { impl PyInner { /// Allocate a new PyInner, optionally with prefix(es). /// Returns a raw pointer to the PyInner (NOT the allocation start). - /// Layout: [ObjExt?][WeakRefList?][PyInner] + /// Layout: `[WeakRefList?][PyAtomicRef; N][ObjExt?][PyInner]` fn new(payload: T, typ: PyTypeRef, dict: Option) -> *mut Self { let member_count = typ.slots.member_count; let needs_ext = typ @@ -1277,22 +1331,30 @@ impl PyInner { ); if needs_ext || needs_weakref { - // Build layout left-to-right: [ObjExt?][WeakRefList?][PyInner] + // Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][PyInner] let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap(); - let ext_start = if needs_ext { - let (combined, offset) = - layout.extend(core::alloc::Layout::new::()).unwrap(); + let weakref_start = if needs_weakref { + let (combined, offset) = layout + .extend(core::alloc::Layout::new::()) + .unwrap(); layout = combined; Some(offset) } else { None }; - let weakref_start = if needs_weakref { - let (combined, offset) = layout - .extend(core::alloc::Layout::new::()) - .unwrap(); + let slots_start = if let Some(region) = slot_region_layout(member_count) { + let (combined, offset) = layout.extend(region).unwrap(); + layout = combined; + Some(offset) + } else { + None + }; + + let ext_start = if needs_ext { + let (combined, offset) = + layout.extend(core::alloc::Layout::new::()).unwrap(); layout = combined; Some(offset) } else { @@ -1311,12 +1373,22 @@ impl PyInner { alloc_ptr.expose_provenance(); unsafe { + if let Some(offset) = slots_start { + let region = slot_region_layout(member_count).unwrap(); + let cell = core::mem::size_of::>(); + let first = alloc_ptr.add(offset + region.size() - member_count * cell); + let cells = first.cast::>(); + for i in 0..member_count { + cells.add(i).write(PyAtomicRef::::new_empty()); + } + } + if let Some(offset) = ext_start { let ext_ptr = alloc_ptr.add(offset) as *mut ObjExt; let flags = typ.slots.flags; let has_dict = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT); let inline_values = flags.has_feature(crate::types::PyTypeFlags::INLINE_VALUES); - ext_ptr.write(ObjExt::new(dict, member_count, has_dict, inline_values)); + ext_ptr.write(ObjExt::new(dict, has_dict, inline_values)); } if let Some(offset) = weakref_start { @@ -2120,12 +2192,20 @@ impl PyObject { self.0.ref_count.is_immortal() } - pub(crate) fn get_slot(&self, offset: usize) -> Option { - self.0.ext_ref().unwrap().slots[offset].load_owned() + pub(crate) fn get_slot(&self, byte_offset: isize) -> Option { + self.slot_cell_at(byte_offset).load_owned() } - pub(crate) fn set_slot(&self, offset: usize, value: Option) { - drop(self.0.ext_ref().unwrap().slots[offset].store(value)); + pub(crate) fn set_slot(&self, byte_offset: isize, value: Option) { + drop(self.slot_cell_at(byte_offset).store(value)); + } + + fn slot_cell_at(&self, byte_offset: isize) -> &PyAtomicRef { + let addr = (self as *const Self as *const u8) + .addr() + .wrapping_add(byte_offset as usize); + let ptr = core::ptr::with_exposed_provenance::>(addr); + unsafe { &*ptr } } /// _PyObject_GC_IS_TRACKED @@ -2237,20 +2317,15 @@ impl PyObject { let (flags, member_count) = obj.0.read_type_flags(); let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; if has_ext { - let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); - let offset = if has_weakref { - WEAKREF_OFFSET + EXT_OFFSET - } else { - EXT_OFFSET - }; let self_addr = (ptr as *const u8).addr(); - let ext_ptr = - core::ptr::with_exposed_provenance_mut::(self_addr.wrapping_sub(offset)); + let ext_ptr = core::ptr::with_exposed_provenance_mut::( + self_addr.wrapping_sub(EXT_OFFSET), + ); let ext = unsafe { &mut *ext_ptr }; if let Some(dict_ref) = ext.dict.as_ref().and_then(|d| d.replace(None)) { result.push(dict_ref.into()); } - for slot in &ext.slots { + for slot in obj.0.slot_cells() { if let Some(val) = slot.store(None) { result.push(val); } @@ -2278,10 +2353,8 @@ impl PyObject { // Py_TPFLAGS_HAVE_GC types have tp_clear pub fn gc_has_clear(&self) -> bool { self.0.vtable.clear.is_some() - || self - .0 - .ext_ref() - .is_some_and(|ext| ext.dict.is_some() || !ext.slots.is_empty()) + || self.0.ext_ref().is_some_and(|ext| ext.dict.is_some()) + || self.0.read_type_flags().1 > 0 } } @@ -2916,13 +2989,15 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { static_assertions::assert_eq_align!(MaybeUninit>, PyInner); // All three core type objects are instances of `type`, which has HAS_DICT - // and HAS_WEAKREF. Their allocations therefore need both prefixes. + // and HAS_WEAKREF. Their allocations therefore need both prefixes, + // with the weakref list in front of ObjExt. let alloc_type_with_prefixes = || -> *mut PyInner { let inner_layout = core::alloc::Layout::new::>>(); let ext_layout = core::alloc::Layout::new::(); let weakref_layout = core::alloc::Layout::new::(); - let (layout, weakref_offset) = ext_layout.extend(weakref_layout).unwrap(); + // [WeakRefList][ObjExt][PyInner] — the list stays in front of ObjExt. + let (layout, ext_offset) = weakref_layout.extend(ext_layout).unwrap(); let (combined, inner_offset) = layout.extend(inner_layout).unwrap(); let combined = combined.pad_to_align(); @@ -2933,8 +3008,8 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy { alloc_ptr.expose_provenance(); unsafe { - (alloc_ptr as *mut ObjExt).write(ObjExt::new(None, 0, true, false)); - (alloc_ptr.add(weakref_offset) as *mut WeakRefList).write(WeakRefList::new()); + (alloc_ptr as *mut WeakRefList).write(WeakRefList::new()); + (alloc_ptr.add(ext_offset) as *mut ObjExt).write(ObjExt::new(None, true, false)); alloc_ptr.add(inner_offset).cast() } }; diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 75ecb3cd1..eda0d7f4b 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -472,25 +472,41 @@ impl PyAtomicRef { /// until it applies to the pointer still in the slot, and a value placed /// here is published so its memory outlives that race. pub(crate) fn load_owned(&self) -> Option { - loop { - let ptr = self.inner.load(Ordering::Acquire); - if ptr.is_null() { - return None; + let ptr = self.inner.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + // Without threading the slot's own reference keeps the object alive, + // so one incref is enough. With threading, retry when a store retires + // the pointer between the load and the incref. + #[cfg(not(feature = "threading"))] + { + unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) } + } + #[cfg(feature = "threading")] + { + let mut ptr = ptr; + loop { + if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) } + && core::ptr::eq(self.inner.load(Ordering::Acquire), ptr) + { + return Some(obj); + } + ptr = self.inner.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + core::hint::spin_loop(); } - if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) } - && core::ptr::eq(self.inner.load(Ordering::Acquire), ptr) - { - return Some(obj); - } - core::hint::spin_loop(); } } /// Replace the stored reference. Returns the previous one, still owned. + /// + /// The value placed in the slot is not marked published, so it can still + /// return to the freelist. With threading, the value that leaves the slot + /// is marked so its free waits out a reader that already loaded it. pub(crate) fn store(&self, value: Option) -> Option { - if let Some(obj) = value.as_ref() { - obj.mark_cache_published(); - } let new_ptr = match value { Some(obj) => { let ptr = obj.into_raw().as_ptr(); @@ -500,7 +516,12 @@ impl PyAtomicRef { None => null_mut(), }; let old = Radium::swap(&self.inner, new_ptr, Ordering::AcqRel); - NonNull::new(old.cast()).map(|ptr| unsafe { PyObjectRef::from_raw(ptr) }) + let old = NonNull::new(old.cast()).map(|ptr| unsafe { PyObjectRef::from_raw(ptr) }); + #[cfg(feature = "threading")] + if let Some(old) = old.as_ref() { + old.mark_cache_published(); + } + old } } diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index b6c7590a8..4f028338d 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -8,6 +8,6 @@ mod traverse_object; pub use self::core::*; pub use self::ext::*; pub use self::payload::*; -pub(crate) use core::SIZEOF_PYOBJECT_HEAD; +pub use core::SIZEOF_PYOBJECT_HEAD; pub(crate) use core::{GC_NO_OWNER, GC_PERMANENT, GC_REACHABLE, GC_UNTRACKED, GcLink, GcOwner}; pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index 575426377..c06723ce9 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -68,7 +68,9 @@ unsafe impl Traverse for PyInner { // Traverse ObjExt prefix fields (dict and slots) if present if let Some(ext) = self.ext_ref() { ext.dict.traverse(tracer_fn); - ext.slots.traverse(tracer_fn); + for slot in self.slot_cells() { + slot.traverse(tracer_fn); + } } if let Some(f) = self.vtable.trace { @@ -92,7 +94,9 @@ unsafe impl Traverse for PyInner { // Traverse ObjExt prefix fields (dict and slots) if present if let Some(ext) = self.ext_ref() { ext.dict.traverse(tracer_fn); - ext.slots.traverse(tracer_fn); + for slot in self.slot_cells() { + slot.traverse(tracer_fn); + } } T::try_traverse(&self.payload, tracer_fn); }