Address slot members by byte offset (#8811)

* Keep slot values freelist-eligible

Member cells now sit in the object allocation ahead of the extension
header, so a slot read is one pointer instead of a boxed slice. A
value stored in a slot is not marked published. With threading, only
the value that leaves the slot is, so its free can wait out a reader.
Without threading a slot read is a single incref.

Assisted-by: Grok:4.7

* Address slot members by byte offset

A slot member's offset is the byte distance from the object to its
inline pointer cell. Loads and stores, including the specialized
instructions, use that address. A relative C offset is resolved by
adding the object header and the type's basic size.

Assisted-by: Grok:4.7

* Keep base slot offsets stable on subclasses

Cell 0 sits directly in front of the extension header. Further slots
extend away from the object, so a member offset computed for a base
class still addresses the same cell on a subclass.

Assisted-by: Grok:4.7

* Pack slot cells flush against the extension header

On 32-bit targets an odd slot count is not a multiple of ObjExt's
alignment. The padding now sits in front of the cells, so cell 0
stays immediately before the header. The non-threading slot load is
also the function's return value.

Assisted-by: Grok:4.7

* Place weakref lists in front of slot cells

A subclass that adds a weakref list no longer inserts it between
ObjExt and the object, so inherited member offsets stay valid.

Assisted-by: Grok:4.7

crates/capi/src/descrobject.rs

@@ -5,8 +5,8 @@ use crate::pystate::with_vm;
 use crate::util::{CStrExt, FfiPtrExt};
 use core::ffi::{c_char, c_int, c_void};
 use rustpython_vm::builtins::{
-    DescriptorMemberDef, MemberAccess, MemberKind, PY_READONLY, PY_RELATIVE_OFFSET,
-    PyDescriptorOwned, PyGetSet, PyMappingProxy, PyMemberDescriptor, PyType,
+    DescriptorMemberDef, MemberAccess, MemberKind, PY_RELATIVE_OFFSET, PyDescriptorOwned, PyGetSet,
+    PyMappingProxy, PyMemberDescriptor, PyType,
 };
 use rustpython_vm::common::lock::PyRwLock;
 use rustpython_vm::function::PySetterValue;
@@ -126,17 +126,15 @@ impl PyMemberDef {
                 self.type_code
             )));
         };
-        if self.offset < 0 {
-            return Err(vm.new_system_error("PyDescr_NewMember does not support negative offsets"));
-        }
-        if self.flags & PY_RELATIVE_OFFSET != 0 {
-            return Err(
-                vm.new_system_error("PyDescr_NewMember does not support Py_RELATIVE_OFFSET")
-            );
+        let mut offset = self.offset;
+        let mut flags = self.flags;
+        if flags & PY_RELATIVE_OFFSET != 0 {
+            // type creation adds tp_basicsize and clears the flag before GetOne.
+            offset += (rustpython_vm::object::SIZEOF_PYOBJECT_HEAD + ty.slots.basicsize) as isize;
+            flags &= !PY_RELATIVE_OFFSET;
         }
 
         let doc = unsafe { self.doc.try_as_str_opt(vm) }?.map(str::to_owned);
-        let readonly = self.flags & PY_READONLY != 0;
 
         let descriptor = PyMemberDescriptor {
             common: PyDescriptorOwned {
@@ -147,11 +145,11 @@ impl PyMemberDef {
             member: DescriptorMemberDef {
                 name: name.to_owned(),
                 kind,
-                offset: self.offset,
-                flags: if readonly { PY_READONLY } else { 0 },
+                offset,
+                flags,
                 doc,
             },
-            // Instance members live in the slot array. `offset` is that index.
+            // `offset` is a byte offset from the object to a pointer cell.
             access: MemberAccess::Slot,
         };
 

crates/vm/src/builtins/descriptor.rs

@@ -355,9 +355,9 @@ pub(crate) type MemberSetterFunc =
 
 /// Where `PyMemberDef.offset` points.
 ///
-/// Builtin payloads keep fields behind locks, so a byte offset from the
-/// object header does not address them. Slot members use `offset` as an
-/// index into the instance slot array. Function members ignore `offset`.
+/// Slot members use a byte offset from the object to an inline pointer cell.
+/// Builtin payloads keep fields behind locks, so those members use a function
+/// and ignore `offset`.
 #[derive(Clone, Copy)]
 pub enum MemberAccess {
     Func {
@@ -415,10 +415,10 @@ pub struct PyMemberDescriptor {
 }
 
 impl PyMemberDescriptor {
-    /// Slot index for an instance-slot member. `None` when the offset is not a slot.
-    pub(crate) fn slot_offset(&self) -> Option<usize> {
+    /// Byte offset of an instance-slot member. `None` when this is not a slot.
+    pub(crate) fn slot_offset(&self) -> Option<isize> {
         match self.access {
-            MemberAccess::Slot => Some(self.member.offset as usize),
+            MemberAccess::Slot => Some(self.member.offset),
             _ => None,
         }
     }
@@ -436,9 +436,7 @@ impl PyMemberDescriptor {
         }
         match self.access {
             MemberAccess::Func { get, .. } => get(vm, obj),
-            MemberAccess::Slot => {
-                get_slot_from_object(&obj, self.member.offset as usize, &self.member, vm)
-            }
+            MemberAccess::Slot => get_slot_from_object(&obj, self.member.offset, &self.member, vm),
             MemberAccess::TupleItem => {
                 let index = self.member.offset as usize;
                 let tuple = obj.downcast_ref::<PyTuple>().ok_or_else(|| {
@@ -468,7 +466,7 @@ impl PyMemberDescriptor {
                 None => Err(vm.new_attribute_error("readonly attribute")),
             },
             MemberAccess::Slot => {
-                set_slot_at_object(&obj, self.member.offset as usize, &self.member, value, vm)
+                set_slot_at_object(&obj, self.member.offset, &self.member, value, vm)
             }
             MemberAccess::TupleItem => Err(vm.new_attribute_error("readonly attribute")),
         }
@@ -566,7 +564,7 @@ impl PyMemberDescriptor {
 // PyMember_GetOne for a value stored in the instance slot array.
 fn get_slot_from_object(
     obj: &PyObject,
-    offset: usize,
+    offset: isize,
     member: &PyMemberDef,
     vm: &VirtualMachine,
 ) -> PyResult {
@@ -589,7 +587,7 @@ fn get_slot_from_object(
 // PyMember_SetOne for a value stored in the instance slot array.
 fn set_slot_at_object(
     obj: &PyObject,
-    offset: usize,
+    offset: isize,
     member: &PyMemberDef,
     value: PySetterValue,
     vm: &VirtualMachine,

crates/vm/src/builtins/type.rs

@@ -2782,7 +2782,7 @@ impl Constructor for PyType {
                 let member_def = PyMemberDef {
                     name: mangled_name.clone(),
                     kind: MemberKind::ObjectEx,
-                    offset: offset as isize,
+                    offset: crate::object::slot_member_offset(offset),
                     flags: 0,
                     doc: None,
                 };

crates/vm/src/frame.rs

@@ -6291,7 +6291,7 @@ impl ExecutingFrame<'_> {
 
                 if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version {
                     let slot_offset =
-                        self.code.instructions.read_cache_u32(cache_base + 3) as usize;
+                        self.code.instructions.read_cache_u32(cache_base + 3) as i32 as isize;
                     if let Some(value) = owner.get_slot(slot_offset) {
                         self.pop_stackref();
                         if oparg.is_method() {
@@ -6381,7 +6381,7 @@ impl ExecutingFrame<'_> {
 
                 if version_match {
                     let slot_offset =
-                        self.code.instructions.read_cache_u16(cache_base + 3) as usize;
+                        self.code.instructions.read_cache_u16(cache_base + 3) as i16 as isize;
                     let owner = self.pop_value();
                     let value = self.pop_value();
                     owner.set_slot(slot_offset, Some(value));

crates/vm/src/object/core.rs

@@ -30,7 +30,6 @@ use crate::{
     class::StaticType,
     object::traverse::{MaybeTraverse, Traverse, TraverseFn},
 };
-use itertools::Itertools;
 
 use alloc::fmt;
 
@@ -244,7 +243,7 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
                 let old = dict.d.write().take();
                 drop(old);
             }
-            for slot in &ext.slots {
+            for slot in obj_ref.0.slot_cells() {
                 drop(slot.store(None));
             }
         }
@@ -367,28 +366,16 @@ unsafe impl Link for GcLink {
 #[repr(C, align(8))]
 pub(super) struct ObjExt {
     pub(super) dict: Option<InstanceDict>,
-    /// Owned `PyObject*` cells. Null is empty. Each cell is one pointer so a
-    /// later step can place the same cells at a byte offset from the object.
-    pub(super) slots: Box<[PyAtomicRef<PyObject>]>,
 }
 
 impl ObjExt {
-    fn new(
-        dict: Option<PyDictRef>,
-        member_count: usize,
-        has_dict: bool,
-        inline_values: bool,
-    ) -> Self {
+    fn new(dict: Option<PyDictRef>, has_dict: bool, inline_values: bool) -> Self {
         Self {
             dict: if has_dict {
                 Some(InstanceDict::from_opt(dict, inline_values))
             } else {
                 None
             },
-            slots: core::iter::repeat_with(PyAtomicRef::<PyObject>::new_empty)
-                .take(member_count)
-                .collect_vec()
-                .into_boxed_slice(),
         }
     }
 }
@@ -400,11 +387,46 @@ impl fmt::Debug for ObjExt {
 }
 
 /// Precomputed offset constants for prefix allocation.
-/// All prefix components are align(8) and their sizes are multiples of 8,
-/// so Layout::extend adds no inter-padding.
+/// `ObjExt` and `WeakRefList` are align(8) and their sizes are multiples of 8,
+/// so `Layout::extend` adds no padding between them. Slot cells are packed
+/// immediately in front of `ObjExt`; any alignment padding is before the cells.
 const EXT_OFFSET: usize = core::mem::size_of::<ObjExt>();
+
+/// Byte offset of member cell `index` from the start of `PyInner`.
+/// Cells live in front of the object, so the offset is negative.
+///
+/// `ObjExt` stays flush with `PyInner`. A subclass that adds `__weakref__`
+/// puts that list in front of the cells, so this offset does not move.
+pub(crate) fn slot_member_offset(index: usize) -> isize {
+    // Cell 0 sits directly in front of ObjExt. Higher indexes extend further
+    // forward, so a base class offset stays valid on a subclass with more slots.
+    let cell = core::mem::size_of::<PyAtomicRef<PyObject>>();
+    -((EXT_OFFSET + (index + 1) * cell) as isize)
+}
+
+fn slot_region_layout(member_count: usize) -> Option<core::alloc::Layout> {
+    if member_count == 0 {
+        return None;
+    }
+    let cell = core::mem::size_of::<PyAtomicRef<PyObject>>();
+    let bytes = member_count * cell;
+    let align = core::mem::align_of::<ObjExt>();
+    // Padding goes in front of the cells so cell 0 stays flush with ObjExt.
+    // On wasm32 a pointer is 4 bytes and ObjExt is align 8, so an odd count
+    // would otherwise leave a gap where cell 0 is supposed to be.
+    let pad = (align - bytes % align) % align;
+    Some(core::alloc::Layout::from_size_align(pad + bytes, align).unwrap())
+}
 const WEAKREF_OFFSET: usize = core::mem::size_of::<WeakRefList>();
 
+/// Distance from `PyInner` back to a `WeakRefList`.
+/// Layout: `[WeakRefList?][slots?][ObjExt?][PyInner]`.
+fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize {
+    let ext = if has_ext { EXT_OFFSET } else { 0 };
+    let slots = slot_region_layout(member_count).map_or(0, |layout| layout.size());
+    ext + slots + WEAKREF_OFFSET
+}
+
 const _: () =
     assert!(core::mem::size_of::<ObjExt>().is_multiple_of(core::mem::align_of::<ObjExt>()));
 const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<PyInner<()>>());
@@ -441,7 +463,7 @@ pub(super) struct PyInner<T> {
 
     pub(super) payload: T,
 }
-pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
+pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
 
 // ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits,
 // generation, owner and refs take eight bytes between them. A 64-bit header had
@@ -476,8 +498,8 @@ impl<T> PyInner<T> {
     /// Access the ObjExt prefix at a negative offset from this PyInner.
     /// Returns None if this object was allocated without dict/slots.
     ///
-    /// Layout: [ObjExt?][WeakRefList?][PyInner]
-    /// ObjExt offset depends on whether WeakRefList is also present.
+    /// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
+    /// `ObjExt` is always immediately in front of `PyInner`.
     #[inline(always)]
     pub(super) fn ext_ref(&self) -> Option<&ObjExt> {
         let (flags, member_count) = self.read_type_flags();
@@ -485,31 +507,47 @@ impl<T> PyInner<T> {
         if !has_ext {
             return None;
         }
-        let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF);
-        let offset = if has_weakref {
-            WEAKREF_OFFSET + EXT_OFFSET
-        } else {
-            EXT_OFFSET
-        };
         let self_addr = (self as *const Self as *const u8).addr();
-        let ext_ptr = core::ptr::with_exposed_provenance::<ObjExt>(self_addr.wrapping_sub(offset));
+        let ext_ptr =
+            core::ptr::with_exposed_provenance::<ObjExt>(self_addr.wrapping_sub(EXT_OFFSET));
         Some(unsafe { &*ext_ptr })
     }
 
+    /// Member cells are the pointer array immediately before [`ObjExt`].
+    /// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner]`.
+    pub(super) fn slot_cells(&self) -> &[PyAtomicRef<PyObject>] {
+        let Some(ext) = self.ext_ref() else {
+            return &[];
+        };
+        let (_, member_count) = self.read_type_flags();
+        if member_count == 0 {
+            return &[];
+        }
+        let cell = core::mem::size_of::<PyAtomicRef<PyObject>>();
+        // Index 0 is the cell adjacent to ObjExt; index i is i cells before it.
+        let first = (ext as *const ObjExt)
+            .addr()
+            .wrapping_sub(member_count * cell);
+        let ptr = core::ptr::with_exposed_provenance::<PyAtomicRef<PyObject>>(first);
+        unsafe { core::slice::from_raw_parts(ptr, member_count) }
+    }
+
     /// Access the WeakRefList prefix at a fixed negative offset from this PyInner.
     /// Returns None if the type does not support weakrefs.
     ///
-    /// Layout: [ObjExt?][WeakRefList?][PyInner]
-    /// WeakRefList is always immediately before PyInner (fixed WEAKREF_OFFSET).
+    /// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
+    /// The list sits in front of the slot cells so adding it on a subclass
+    /// does not move inherited member offsets.
     #[inline(always)]
     pub(super) fn weakref_list_ref(&self) -> Option<&WeakRefList> {
-        let (flags, _) = self.read_type_flags();
+        let (flags, member_count) = self.read_type_flags();
         if !flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF) {
             return None;
         }
+        let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0;
         let self_addr = (self as *const Self as *const u8).addr();
         let ptr = core::ptr::with_exposed_provenance::<WeakRefList>(
-            self_addr.wrapping_sub(WEAKREF_OFFSET),
+            self_addr.wrapping_sub(weakref_prefix_offset(has_ext, member_count)),
         );
         Some(unsafe { &*ptr })
     }
@@ -1192,7 +1230,7 @@ impl<T: PyPayload> PyInner<T> {
     }
 
     /// Deallocate a PyInner, handling optional prefix(es).
-    /// Layout: [ObjExt?][WeakRefList?][PyInner<T>]
+    /// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
     ///
     /// # Safety
     /// `ptr` must be a valid pointer from `PyInner::new` and must not be used after this call.
@@ -1210,18 +1248,21 @@ impl<T: PyPayload> PyInner<T> {
                 // Reconstruct the same layout used in new()
                 let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap();
 
-                if has_ext {
-                    layout = layout
-                        .extend(core::alloc::Layout::new::<ObjExt>())
-                        .unwrap()
-                        .0;
-                }
                 if has_weakref {
                     layout = layout
                         .extend(core::alloc::Layout::new::<WeakRefList>())
                         .unwrap()
                         .0;
                 }
+                if let Some(region) = slot_region_layout(member_count) {
+                    layout = layout.extend(region).unwrap().0;
+                }
+                if has_ext {
+                    layout = layout
+                        .extend(core::alloc::Layout::new::<ObjExt>())
+                        .unwrap()
+                        .0;
+                }
                 let (combined, inner_offset) =
                     layout.extend(core::alloc::Layout::new::<Self>()).unwrap();
                 let combined = combined.pad_to_align();
@@ -1230,9 +1271,22 @@ impl<T: PyPayload> PyInner<T> {
 
                 Self::drop_fields(ptr);
 
-                // Drop ObjExt if present (dict, slots)
+                // Drop member cells, then ObjExt. WeakRefList is in front of the cells.
+                let mut cursor = alloc_ptr;
+                if has_weakref {
+                    cursor = cursor.add(core::mem::size_of::<WeakRefList>());
+                }
+                if let Some(region) = slot_region_layout(member_count) {
+                    let cell = core::mem::size_of::<PyAtomicRef<PyObject>>();
+                    let first = cursor.add(region.size() - member_count * cell);
+                    let cells = first.cast::<PyAtomicRef<PyObject>>();
+                    for i in 0..member_count {
+                        core::ptr::drop_in_place(cells.add(i));
+                    }
+                    cursor = cursor.add(region.size());
+                }
                 if has_ext {
-                    core::ptr::drop_in_place(alloc_ptr as *mut ObjExt);
+                    core::ptr::drop_in_place(cursor.cast::<ObjExt>());
                 }
                 // WeakRefList has no Drop (just raw pointers), no drop_in_place needed
 
@@ -1258,7 +1312,7 @@ impl<T: PyPayload> PyInner<T> {
 impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
     /// Allocate a new PyInner, optionally with prefix(es).
     /// Returns a raw pointer to the PyInner (NOT the allocation start).
-    /// Layout: [ObjExt?][WeakRefList?][PyInner<T>]
+    /// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
     fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> *mut Self {
         let member_count = typ.slots.member_count;
         let needs_ext = typ
@@ -1277,22 +1331,30 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
         );
 
         if needs_ext || needs_weakref {
-            // Build layout left-to-right: [ObjExt?][WeakRefList?][PyInner]
+            // Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][PyInner]
             let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap();
 
-            let ext_start = if needs_ext {
-                let (combined, offset) =
-                    layout.extend(core::alloc::Layout::new::<ObjExt>()).unwrap();
+            let weakref_start = if needs_weakref {
+                let (combined, offset) = layout
+                    .extend(core::alloc::Layout::new::<WeakRefList>())
+                    .unwrap();
                 layout = combined;
                 Some(offset)
             } else {
                 None
             };
 
-            let weakref_start = if needs_weakref {
-                let (combined, offset) = layout
-                    .extend(core::alloc::Layout::new::<WeakRefList>())
-                    .unwrap();
+            let slots_start = if let Some(region) = slot_region_layout(member_count) {
+                let (combined, offset) = layout.extend(region).unwrap();
+                layout = combined;
+                Some(offset)
+            } else {
+                None
+            };
+
+            let ext_start = if needs_ext {
+                let (combined, offset) =
+                    layout.extend(core::alloc::Layout::new::<ObjExt>()).unwrap();
                 layout = combined;
                 Some(offset)
             } else {
@@ -1311,12 +1373,22 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
             alloc_ptr.expose_provenance();
 
             unsafe {
+                if let Some(offset) = slots_start {
+                    let region = slot_region_layout(member_count).unwrap();
+                    let cell = core::mem::size_of::<PyAtomicRef<PyObject>>();
+                    let first = alloc_ptr.add(offset + region.size() - member_count * cell);
+                    let cells = first.cast::<PyAtomicRef<PyObject>>();
+                    for i in 0..member_count {
+                        cells.add(i).write(PyAtomicRef::<PyObject>::new_empty());
+                    }
+                }
+
                 if let Some(offset) = ext_start {
                     let ext_ptr = alloc_ptr.add(offset) as *mut ObjExt;
                     let flags = typ.slots.flags;
                     let has_dict = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT);
                     let inline_values = flags.has_feature(crate::types::PyTypeFlags::INLINE_VALUES);
-                    ext_ptr.write(ObjExt::new(dict, member_count, has_dict, inline_values));
+                    ext_ptr.write(ObjExt::new(dict, has_dict, inline_values));
                 }
 
                 if let Some(offset) = weakref_start {
@@ -2120,12 +2192,20 @@ impl PyObject {
         self.0.ref_count.is_immortal()
     }
 
-    pub(crate) fn get_slot(&self, offset: usize) -> Option<PyObjectRef> {
-        self.0.ext_ref().unwrap().slots[offset].load_owned()
+    pub(crate) fn get_slot(&self, byte_offset: isize) -> Option<PyObjectRef> {
+        self.slot_cell_at(byte_offset).load_owned()
     }
 
-    pub(crate) fn set_slot(&self, offset: usize, value: Option<PyObjectRef>) {
-        drop(self.0.ext_ref().unwrap().slots[offset].store(value));
+    pub(crate) fn set_slot(&self, byte_offset: isize, value: Option<PyObjectRef>) {
+        drop(self.slot_cell_at(byte_offset).store(value));
+    }
+
+    fn slot_cell_at(&self, byte_offset: isize) -> &PyAtomicRef<Self> {
+        let addr = (self as *const Self as *const u8)
+            .addr()
+            .wrapping_add(byte_offset as usize);
+        let ptr = core::ptr::with_exposed_provenance::<PyAtomicRef<Self>>(addr);
+        unsafe { &*ptr }
     }
 
     /// _PyObject_GC_IS_TRACKED
@@ -2237,20 +2317,15 @@ impl PyObject {
         let (flags, member_count) = obj.0.read_type_flags();
         let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0;
         if has_ext {
-            let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF);
-            let offset = if has_weakref {
-                WEAKREF_OFFSET + EXT_OFFSET
-            } else {
-                EXT_OFFSET
-            };
             let self_addr = (ptr as *const u8).addr();
-            let ext_ptr =
-                core::ptr::with_exposed_provenance_mut::<ObjExt>(self_addr.wrapping_sub(offset));
+            let ext_ptr = core::ptr::with_exposed_provenance_mut::<ObjExt>(
+                self_addr.wrapping_sub(EXT_OFFSET),
+            );
             let ext = unsafe { &mut *ext_ptr };
             if let Some(dict_ref) = ext.dict.as_ref().and_then(|d| d.replace(None)) {
                 result.push(dict_ref.into());
             }
-            for slot in &ext.slots {
+            for slot in obj.0.slot_cells() {
                 if let Some(val) = slot.store(None) {
                     result.push(val);
                 }
@@ -2278,10 +2353,8 @@ impl PyObject {
     // Py_TPFLAGS_HAVE_GC types have tp_clear
     pub fn gc_has_clear(&self) -> bool {
         self.0.vtable.clear.is_some()
-            || self
-                .0
-                .ext_ref()
-                .is_some_and(|ext| ext.dict.is_some() || !ext.slots.is_empty())
+            || self.0.ext_ref().is_some_and(|ext| ext.dict.is_some())
+            || self.0.read_type_flags().1 > 0
     }
 }
 
@@ -2916,13 +2989,15 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
     static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyTuple>>, PyInner<PyTuple>);
 
     // All three core type objects are instances of `type`, which has HAS_DICT
-    // and HAS_WEAKREF. Their allocations therefore need both prefixes.
+    // and HAS_WEAKREF. Their allocations therefore need both prefixes,
+    // with the weakref list in front of ObjExt.
     let alloc_type_with_prefixes = || -> *mut PyInner<PyType> {
         let inner_layout = core::alloc::Layout::new::<MaybeUninit<PyInner<PyType>>>();
         let ext_layout = core::alloc::Layout::new::<ObjExt>();
         let weakref_layout = core::alloc::Layout::new::<WeakRefList>();
 
-        let (layout, weakref_offset) = ext_layout.extend(weakref_layout).unwrap();
+        // [WeakRefList][ObjExt][PyInner] — the list stays in front of ObjExt.
+        let (layout, ext_offset) = weakref_layout.extend(ext_layout).unwrap();
         let (combined, inner_offset) = layout.extend(inner_layout).unwrap();
         let combined = combined.pad_to_align();
 
@@ -2933,8 +3008,8 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
         alloc_ptr.expose_provenance();
 
         unsafe {
-            (alloc_ptr as *mut ObjExt).write(ObjExt::new(None, 0, true, false));
-            (alloc_ptr.add(weakref_offset) as *mut WeakRefList).write(WeakRefList::new());
+            (alloc_ptr as *mut WeakRefList).write(WeakRefList::new());
+            (alloc_ptr.add(ext_offset) as *mut ObjExt).write(ObjExt::new(None, true, false));
             alloc_ptr.add(inner_offset).cast()
         }
     };

crates/vm/src/object/ext.rs

@@ -472,25 +472,41 @@ impl PyAtomicRef<PyObject> {
     /// until it applies to the pointer still in the slot, and a value placed
     /// here is published so its memory outlives that race.
     pub(crate) fn load_owned(&self) -> Option<PyObjectRef> {
-        loop {
-            let ptr = self.inner.load(Ordering::Acquire);
-            if ptr.is_null() {
-                return None;
+        let ptr = self.inner.load(Ordering::Acquire);
+        if ptr.is_null() {
+            return None;
+        }
+        // Without threading the slot's own reference keeps the object alive,
+        // so one incref is enough. With threading, retry when a store retires
+        // the pointer between the load and the incref.
+        #[cfg(not(feature = "threading"))]
+        {
+            unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) }
+        }
+        #[cfg(feature = "threading")]
+        {
+            let mut ptr = ptr;
+            loop {
+                if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) }
+                    && core::ptr::eq(self.inner.load(Ordering::Acquire), ptr)
+                {
+                    return Some(obj);
+                }
+                ptr = self.inner.load(Ordering::Acquire);
+                if ptr.is_null() {
+                    return None;
+                }
+                core::hint::spin_loop();
             }
-            if let Some(obj) = unsafe { PyObject::try_to_owned_from_ptr(ptr.cast()) }
-                && core::ptr::eq(self.inner.load(Ordering::Acquire), ptr)
-            {
-                return Some(obj);
-            }
-            core::hint::spin_loop();
         }
     }
 
     /// Replace the stored reference. Returns the previous one, still owned.
+    ///
+    /// The value placed in the slot is not marked published, so it can still
+    /// return to the freelist. With threading, the value that leaves the slot
+    /// is marked so its free waits out a reader that already loaded it.
     pub(crate) fn store(&self, value: Option<PyObjectRef>) -> Option<PyObjectRef> {
-        if let Some(obj) = value.as_ref() {
-            obj.mark_cache_published();
-        }
         let new_ptr = match value {
             Some(obj) => {
                 let ptr = obj.into_raw().as_ptr();
@@ -500,7 +516,12 @@ impl PyAtomicRef<PyObject> {
             None => null_mut(),
         };
         let old = Radium::swap(&self.inner, new_ptr, Ordering::AcqRel);
-        NonNull::new(old.cast()).map(|ptr| unsafe { PyObjectRef::from_raw(ptr) })
+        let old = NonNull::new(old.cast()).map(|ptr| unsafe { PyObjectRef::from_raw(ptr) });
+        #[cfg(feature = "threading")]
+        if let Some(old) = old.as_ref() {
+            old.mark_cache_published();
+        }
+        old
     }
 }
 

crates/vm/src/object/mod.rs

@@ -8,6 +8,6 @@ mod traverse_object;
 pub use self::core::*;
 pub use self::ext::*;
 pub use self::payload::*;
-pub(crate) use core::SIZEOF_PYOBJECT_HEAD;
+pub use core::SIZEOF_PYOBJECT_HEAD;
 pub(crate) use core::{GC_NO_OWNER, GC_PERMANENT, GC_REACHABLE, GC_UNTRACKED, GcLink, GcOwner};
 pub use traverse::{MaybeTraverse, Traverse, TraverseFn};

crates/vm/src/object/traverse_object.rs

@@ -68,7 +68,9 @@ unsafe impl Traverse for PyInner<Erased> {
         // Traverse ObjExt prefix fields (dict and slots) if present
         if let Some(ext) = self.ext_ref() {
             ext.dict.traverse(tracer_fn);
-            ext.slots.traverse(tracer_fn);
+            for slot in self.slot_cells() {
+                slot.traverse(tracer_fn);
+            }
         }
 
         if let Some(f) = self.vtable.trace {
@@ -92,7 +94,9 @@ unsafe impl<T: MaybeTraverse> Traverse for PyInner<T> {
         // Traverse ObjExt prefix fields (dict and slots) if present
         if let Some(ext) = self.ext_ref() {
             ext.dict.traverse(tracer_fn);
-            ext.slots.traverse(tracer_fn);
+            for slot in self.slot_cells() {
+                slot.traverse(tracer_fn);
+            }
         }
         T::try_traverse(&self.payload, tracer_fn);
     }