Merge PyInner into Py (#8820)

Rename the object layout struct `PyInner<T>` to `Py<T>` and make it the
public type, replacing the `#[repr(transparent)]` newtype
`Py<T>(PyInner<T>)`. Header fields stay `pub(super)`. `PyObject` now
wraps `Py<Erased>`.

- Drop the `.0` hop on `Py<T>` accesses.
- Remove the delegating `Traverse for Py<T>` impl; the former
  `Traverse for PyInner<T>` impl now serves `Py<T>`.
- Remove `Debug for PyInner<T>`; `debug_obj` writes the
  `[PyObject ...]` form itself.
- Use `Py::from_payload_ptr` in `PyWeak::drop_inner` and drop its
  now-stale `allow(dead_code)`.

Assisted-by: Claude Code:claude-opus-5-5
This commit is contained in:
Jeong, YunWon authored and GitHub committed 2026-09-26 03:56:42 +09:00
1 parent 98d6f23fe6
commit 393689d847
6 files changed
+96 -119

No files matched your search

+1 -1
View File
@@ -101,7 +101,7 @@ unsafe impl Traverse for PyTuple {
}
thread_local! {
// A single freelist for all tuple sizes: `PyInner<PyTuple>` is a
// A single freelist for all tuple sizes: `Py<PyTuple>` is a
// fixed-size allocation (elements are a separate boxed slice that is
// dropped and replaced on reuse), so husks are interchangeable.
// freelist_push must not read the payload — it runs after tp_clear,
+1 -1
View File
@@ -1590,7 +1590,7 @@ fn cleared_frame_access() -> ! {
thread_local! {
/// Free list of dead frame objects for reuse. Entries are cleared husks
/// (`iframe == None`) whose child references were already released.
/// PyInner<FrameObject> is fixed-size (localsplus storage is out-of-line),
/// Py<FrameObject> is fixed-size (localsplus storage is out-of-line),
/// so a single bucket suffices.
static FRAME_FREELIST: core::cell::Cell<crate::object::FreeList<FrameObject>> =
const { core::cell::Cell::new(crate::object::FreeList::new()) };
+1 -1
View File
@@ -408,7 +408,7 @@ impl GcState {
///
/// Like [`Self::track_object`], but for the hot allocation path only:
/// `obj`'s `gc_bits` must still hold its freshly-initialized value of `0`
/// (true right after `PyInner::new` or a freelist pop, both of which zero
/// (true right after `Py::new` or a freelist pop, both of which zero
/// it), so the tracked bit can go in with a plain store instead of the
/// `fetch_or` `set_gc_tracked()` needs to be safe for the general case
/// (e.g. re-tracking a resurrected object, whose bits are not zero — it
+84 -110
View File
@@ -44,7 +44,7 @@ use core::{
ptr::{self, NonNull},
};
// so, PyObjectRef is basically equivalent to `PyRc<PyInner<dyn PyObjectPayload>>`, except it's
// so, PyObjectRef is basically equivalent to `PyRc<Py<dyn PyObjectPayload>>`, except it's
// only one pointer in width rather than 2. We do that by manually creating a vtable, and putting
// a &'static reference to it inside the `PyRc` rather than adjacent to it, like trait objects do.
// This can lead to faster code since there's just less data to pass around, as well as because of
@@ -61,18 +61,18 @@ use core::{
// There has to be padding in the space between the 2 fields. But, if that field is a trait object
// (like `dyn PyObjectPayload`) we don't *know* how much padding there is between the `payload`
// field and the previous field. So, Rust has to consult the vtable to know the exact offset of
// `payload` in `PyInner<dyn PyObjectPayload>`, which has a huge performance impact when *every
// `payload` in `Py<dyn PyObjectPayload>`, which has a huge performance impact when *every
// single payload access* requires a vtable lookup. Thankfully, we're able to avoid that because of
// the way we use PyObjectRef, in that whenever we want to access the payload we (almost) always
// access it from a generic function. So, rather than doing
//
// - check vtable for payload offset
// - get offset in PyInner struct
// - get offset in Py struct
// - call as_any() method of PyObjectPayload
// - call downcast_ref() method of Any
// we can just do
// - check vtable that typeid matches
// - pointer cast directly to *const PyInner<T>
// - pointer cast directly to *const Py<T>
//
// and at that point the compiler can know the offset of `payload` for us because **we've given it a
// concrete type to work with before we ever access the `payload` field**
@@ -230,7 +230,7 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
// Published objects (e.g. a tuple stored as a type attribute) must skip the
// freelist: `PyRef::new_ref` would reuse the slot and overwrite the refcount
// word with a non-atomic write, racing a reader's atomic try-incref. Route
// them through `PyInner::dealloc` instead, whose QSBR hook defers the actual
// them through `Py::dealloc` instead, whose QSBR hook defers the actual
// memory free until readers can no longer observe it.
let typ = obj_ref.class();
let pushed = if T::HAS_FREELIST
@@ -254,7 +254,7 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
if !pushed {
// Deallocate the object memory (handles ObjExt prefix if present)
unsafe { PyInner::dealloc(obj as *mut PyInner<T>) };
unsafe { Py::dealloc(obj as *mut Py<T>) };
}
// Trashcan: decrement depth and process deferred objects at outermost level
@@ -266,20 +266,20 @@ pub(super) unsafe fn debug_obj<T: PyPayload + core::fmt::Debug>(
x: &PyObject,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) };
fmt::Debug::fmt(x, f)
let x = unsafe { &*(x as *const PyObject as *const Py<T>) };
write!(f, "[PyObject {:?}]", x.payload)
}
/// Call `try_trace` on payload
pub(super) unsafe fn try_traverse_obj<T: PyPayload>(x: &PyObject, tracer_fn: &mut TraverseFn<'_>) {
let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) };
let x = unsafe { &*(x as *const PyObject as *const Py<T>) };
let payload = &x.payload;
payload.try_traverse(tracer_fn)
}
/// Call `try_clear` on payload to extract child references (tp_clear)
pub(super) unsafe fn try_clear_obj<T: PyPayload>(x: *mut PyObject, out: &mut Vec<PyObjectRef>) {
let x = unsafe { &mut *(x as *mut PyInner<T>) };
let x = unsafe { &mut *(x as *mut Py<T>) };
x.payload.try_clear(out);
}
@@ -335,8 +335,8 @@ pub(crate) const GC_REACHABLE: u32 = u32::MAX;
/// Link implementation for GC intrusive linked list tracking
pub(crate) struct GcLink;
// SAFETY: PyObject (PyInner<Erased>) is heap-allocated and pinned in memory
// once created. gc_pointers is at a fixed offset in PyInner.
// SAFETY: PyObject (Py<Erased>) is heap-allocated and pinned in memory
// once created. gc_pointers is at a fixed offset in Py.
unsafe impl Link for GcLink {
type Handle = NonNull<PyObject>;
type Target = PyObject;
@@ -350,18 +350,18 @@ unsafe impl Link for GcLink {
}
unsafe fn pointers(target: NonNull<PyObject>) -> NonNull<Pointers<PyObject>> {
let inner_ptr = target.as_ptr() as *mut PyInner<Erased>;
let inner_ptr = target.as_ptr() as *mut Py<Erased>;
unsafe { NonNull::new_unchecked(&raw mut (*inner_ptr).gc_pointers) }
}
}
/// Extension fields for objects that need dict or member slots.
/// Allocated as a prefix before PyInner when needed (prefix allocation pattern).
/// Access via `PyInner::ext_ref()` using negative offset from the object pointer.
/// Allocated as a prefix before Py when needed (prefix allocation pattern).
/// Access via `Py::ext_ref()` using negative offset from the object pointer.
///
/// align(8) ensures size_of::<ObjExt>() is always a multiple of 8,
/// so the offset from Layout::extend equals size_of::<ObjExt>() for any
/// PyInner<T> alignment (important on wasm32 where pointers are 4 bytes
/// Py<T> alignment (important on wasm32 where pointers are 4 bytes
/// but some payloads like PyWeak have align 8 due to i64 fields).
#[repr(C, align(8))]
pub(super) struct ObjExt {
@@ -392,10 +392,10 @@ impl fmt::Debug for ObjExt {
/// immediately in front of `ObjExt`; any alignment padding is before the cells.
const EXT_OFFSET: usize = core::mem::size_of::<ObjExt>();
/// Byte offset of member cell `index` from the start of `PyInner`.
/// Byte offset of member cell `index` from the start of `Py`.
/// Cells live in front of the object, so the offset is negative.
///
/// `ObjExt` stays flush with `PyInner`. A subclass that adds `__weakref__`
/// `ObjExt` stays flush with `Py`. A subclass that adds `__weakref__`
/// puts that list in front of the cells, so this offset does not move.
pub(crate) fn slot_member_offset(index: usize) -> isize {
// Cell 0 sits directly in front of ObjExt. Higher indexes extend further
@@ -419,8 +419,8 @@ fn slot_region_layout(member_count: usize) -> Option<core::alloc::Layout> {
}
const WEAKREF_OFFSET: usize = core::mem::size_of::<WeakRefList>();
/// Distance from `PyInner` back to a `WeakRefList`.
/// Layout: `[WeakRefList?][slots?][ObjExt?][PyInner]`.
/// Distance from `Py` back to a `WeakRefList`.
/// Layout: `[WeakRefList?][slots?][ObjExt?][Py]`.
fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize {
let ext = if has_ext { EXT_OFFSET } else { 0 };
let slots = slot_region_layout(member_count).map_or(0, |layout| layout.size());
@@ -429,17 +429,17 @@ fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize {
const _: () =
assert!(core::mem::size_of::<ObjExt>().is_multiple_of(core::mem::align_of::<ObjExt>()));
const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<PyInner<()>>());
const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<Py<()>>());
const _: () = assert!(
core::mem::size_of::<WeakRefList>().is_multiple_of(core::mem::align_of::<WeakRefList>())
);
const _: () = assert!(core::mem::align_of::<WeakRefList>() >= core::mem::align_of::<PyInner<()>>());
const _: () = assert!(core::mem::align_of::<WeakRefList>() >= core::mem::align_of::<Py<()>>());
/// This is an actual python object. It consists of a `typ` which is the
/// python class, and carries some rust payload optionally. This rust
/// payload can be a rust float or rust int in case of float and int objects.
#[repr(C)]
pub(super) struct PyInner<T> {
pub struct Py<T> {
pub(super) ref_count: RefCount,
pub(super) vtable: &'static PyObjVTable,
/// GC bits for free-threading (like ob_gc_bits)
@@ -463,7 +463,7 @@ pub(super) struct PyInner<T> {
pub(super) payload: T,
}
pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<Py<()>>();
// ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits,
// generation, owner and refs take eight bytes between them. A 64-bit header had
@@ -472,7 +472,7 @@ pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
// this holds.
const _: () = assert!(SIZEOF_PYOBJECT_HEAD == 5 * core::mem::size_of::<usize>() + 8);
// `PyInner::drop_fields` names `payload` and `typ`; it is only complete while
// `Py::drop_fields` names `payload` and `typ`; it is only complete while
// every other field stays trivially destructible.
const _: () = assert!(
!core::mem::needs_drop::<RefCount>()
@@ -483,23 +483,23 @@ const _: () = assert!(
&& !core::mem::needs_drop::<Pointers<PyObject>>()
);
impl<T> PyInner<T> {
impl<T> Py<T> {
/// Read type flags and member_count via raw pointers to avoid Stacked Borrows
/// violations during bootstrap, where type objects have self-referential typ pointers.
#[inline(always)]
fn read_type_flags(&self) -> (crate::types::PyTypeFlags, usize) {
let typ_ptr = self.typ.load_raw();
let slots = unsafe { core::ptr::addr_of!((*typ_ptr).0.payload.slots) };
let slots = unsafe { core::ptr::addr_of!((*typ_ptr).payload.slots) };
let flags = unsafe { core::ptr::addr_of!((*slots).flags).read() };
let member_count = unsafe { core::ptr::addr_of!((*slots).member_count).read() };
(flags, member_count)
}
/// Access the ObjExt prefix at a negative offset from this PyInner.
/// Access the ObjExt prefix at a negative offset from this Py.
/// Returns None if this object was allocated without dict/slots.
///
/// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
/// `ObjExt` is always immediately in front of `PyInner`.
/// Layout: [WeakRefList?][slots?][ObjExt?][Py]
/// `ObjExt` is always immediately in front of `Py`.
#[inline(always)]
pub(super) fn ext_ref(&self) -> Option<&ObjExt> {
let (flags, member_count) = self.read_type_flags();
@@ -514,7 +514,7 @@ impl<T> PyInner<T> {
}
/// Member cells are the pointer array immediately before [`ObjExt`].
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner]`.
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py]`.
pub(super) fn slot_cells(&self) -> &[PyAtomicRef<PyObject>] {
let Some(ext) = self.ext_ref() else {
return &[];
@@ -532,10 +532,10 @@ impl<T> PyInner<T> {
unsafe { core::slice::from_raw_parts(ptr, member_count) }
}
/// Access the WeakRefList prefix at a fixed negative offset from this PyInner.
/// Access the WeakRefList prefix at a fixed negative offset from this Py.
/// Returns None if the type does not support weakrefs.
///
/// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
/// Layout: [WeakRefList?][slots?][ObjExt?][Py]
/// The list sits in front of the slot cells so adding it on a subclass
/// does not move inherited member offsets.
#[inline(always)]
@@ -553,20 +553,6 @@ impl<T> PyInner<T> {
}
}
impl<T: fmt::Debug> fmt::Debug for PyInner<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "[PyObject {:?}]", self.payload)
}
}
unsafe impl<T: MaybeTraverse> Traverse for Py<T> {
/// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children,
/// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
self.0.traverse(tracer_fn)
}
}
unsafe impl Traverse for PyObject {
/// DO notice that call `trace` on `PyObject` means apply `tracer_fn` on `PyObject`'s children,
/// not like call `trace` on `PyObjectRef` which apply `tracer_fn` on `PyObjectRef` itself
@@ -689,8 +675,7 @@ unsafe fn try_reuse_weakref(ptr: *mut Py<PyWeak>) -> Option<PyRef<PyWeak>> {
return None;
}
let node = unsafe { &*ptr };
node.0
.ref_count
node.ref_count
.safe_inc()
.then(|| unsafe { PyRef::from_raw(ptr) })
}
@@ -824,7 +809,7 @@ impl WeakRefList {
match NonNull::new(candidate_ptr) {
Some(candidate) => {
let node = unsafe { candidate.as_ref() };
let has_callback = unsafe { (&*node.0.payload.callback.get()).is_some() };
let has_callback = unsafe { (&*node.payload.callback.get()).is_some() };
let node_cls = node.class();
// PyWeakref_CheckProxy: the basic-proxy slot is reserved for
// the canonical proxy type; subclasses and callback-less ref
@@ -860,7 +845,7 @@ impl WeakRefList {
let wr = unsafe { node.as_ref() };
// Mark weakref as dead
wr.0.payload
wr.payload
.wr_object
.store(ptr::null_mut(), Ordering::Relaxed);
@@ -872,9 +857,9 @@ impl WeakRefList {
}
// Collect callback only if we can still acquire a strong ref.
if wr.0.ref_count.safe_inc() {
if wr.ref_count.safe_inc() {
let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) };
let cb = unsafe { wr.0.payload.callback.get().replace(None) };
let cb = unsafe { wr.payload.callback.get().replace(None) };
if let Some(cb) = cb {
callbacks.push((wr_ref, cb));
}
@@ -911,7 +896,7 @@ impl WeakRefList {
let wr = unsafe { node.as_ref() };
// Mark weakref as dead
wr.0.payload
wr.payload
.wr_object
.store(ptr::null_mut(), Ordering::Relaxed);
@@ -923,9 +908,9 @@ impl WeakRefList {
}
// Collect callback without invoking only if we can keep weakref alive.
if wr.0.ref_count.safe_inc() {
if wr.ref_count.safe_inc() {
let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) };
let cb = unsafe { wr.0.payload.callback.get().replace(None) };
let cb = unsafe { wr.payload.callback.get().replace(None) };
if let Some(cb) = cb {
callbacks.push((wr_ref, cb));
}
@@ -943,7 +928,7 @@ impl WeakRefList {
let mut count = 0usize;
let mut current = NonNull::new(self.head.load(Ordering::Relaxed));
while let Some(node) = current {
if unsafe { node.as_ref() }.0.ref_count.get() > 0 {
if unsafe { node.as_ref() }.ref_count.get() > 0 {
count += 1;
}
current = unsafe { WeakLink::pointers(node).as_ref().get_next() };
@@ -957,7 +942,7 @@ impl WeakRefList {
let mut current = NonNull::new(self.head.load(Ordering::Relaxed));
while let Some(node) = current {
let wr = unsafe { node.as_ref() };
if wr.0.ref_count.safe_inc() {
if wr.ref_count.safe_inc() {
v.push(unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) });
}
current = unsafe { WeakLink::pointers(node).as_ref().get_next() };
@@ -991,7 +976,7 @@ unsafe impl Link for WeakLink {
#[inline(always)]
unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> {
// SAFETY: requirements forwarded from caller
unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers) }
unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).payload.pointers) }
}
}
@@ -1087,12 +1072,7 @@ impl PyWeak {
let wrl = obj.0.weakref_list_ref().unwrap();
// Compute our Py<PyWeak> node pointer from payload address
let offset = std::mem::offset_of!(PyInner<Self>, payload);
let py_inner = (self as *const Self)
.cast::<u8>()
.wrapping_sub(offset)
.cast::<PyInner<Self>>();
let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py<Self>) };
let node_ptr = unsafe { NonNull::new_unchecked(Py::from_payload_ptr(self).cast_mut()) };
// Unlink from list
unsafe { unlink_weakref(wrl, node_ptr) };
@@ -1214,7 +1194,7 @@ impl InstanceDict {
}
}
impl<T: PyPayload> PyInner<T> {
impl<T: PyPayload> Py<T> {
/// Run the destructors of the fields that have one, payload first.
///
/// Declaration order would drop `typ` first, and `PyAtomicRef::drop`
@@ -1229,11 +1209,11 @@ impl<T: PyPayload> PyInner<T> {
}
}
/// Deallocate a PyInner, handling optional prefix(es).
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
/// Deallocate a Py, handling optional prefix(es).
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py<T>]`
///
/// # Safety
/// `ptr` must be a valid pointer from `PyInner::new` and must not be used after this call.
/// `ptr` must be a valid pointer from `Py::new` and must not be used after this call.
unsafe fn dealloc(ptr: *mut Self) {
unsafe {
let (flags, member_count) = (*ptr).read_type_flags();
@@ -1309,10 +1289,10 @@ impl<T: PyPayload> PyInner<T> {
}
}
impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
/// Allocate a new PyInner, optionally with prefix(es).
/// Returns a raw pointer to the PyInner (NOT the allocation start).
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
impl<T: PyPayload + core::fmt::Debug> Py<T> {
/// Allocate a new Py, optionally with prefix(es).
/// Returns a raw pointer to the Py (NOT the allocation start).
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py<T>]`
fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> *mut Self {
let member_count = typ.slots.member_count;
let needs_ext = typ
@@ -1331,7 +1311,7 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
);
if needs_ext || needs_weakref {
// Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][PyInner]
// Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][Py]
let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap();
let weakref_start = if needs_weakref {
@@ -1429,7 +1409,7 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
/// Thread-local freelist storage for reusing object allocations.
///
/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw
/// `PyInner<T>` allocations without running payload destructors to avoid
/// `Py<T>` allocations without running payload destructors to avoid
/// accessing already-destroyed thread-local storage (GC state, other freelists).
pub(crate) struct FreeList<T: PyPayload> {
items: Vec<*mut PyObject>,
@@ -1461,7 +1441,7 @@ impl<T: PyPayload> Drop for FreeList<T> {
// MAX_FREELIST per type per thread.
for ptr in self.items.drain(..) {
unsafe {
alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::<PyInner<T>>());
alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::<Py<T>>());
}
}
}
@@ -1506,7 +1486,7 @@ cfg_select! {
}
#[repr(transparent)]
pub struct PyObject(PyInner<Erased>);
pub struct PyObject(Py<Erased>);
impl Deref for PyObjectRef {
type Target = PyObject;
@@ -1557,7 +1537,7 @@ impl PyObject {
/// (same guarantee as `_Py_TryIncRefShared`).
#[inline]
pub unsafe fn try_to_owned_from_ptr(ptr: *mut Self) -> Option<PyObjectRef> {
let inner = ptr.cast::<PyInner<Erased>>();
let inner = ptr.cast::<Py<Erased>>();
let ref_count = unsafe { &*core::ptr::addr_of!((*inner).ref_count) };
if ref_count.safe_inc() {
Some(PyObjectRef {
@@ -1653,7 +1633,7 @@ impl PyObjectRef {
impl PyObject {
/// Returns the WeakRefList if the type supports weakrefs (HAS_WEAKREF).
/// The WeakRefList is stored as a separate prefix before PyInner,
/// The WeakRefList is stored as a separate prefix before Py,
/// independent from ObjExt (dict/slots).
#[inline(always)]
fn weak_ref_list(&self) -> Option<&WeakRefList> {
@@ -1669,7 +1649,7 @@ impl PyObject {
None
} else {
let head = unsafe { &*head_ptr };
if head.0.ref_count.safe_inc() {
if head.ref_count.safe_inc() {
Some(unsafe { PyRef::from_raw(head_ptr) }.into())
} else {
None
@@ -1758,9 +1738,9 @@ impl PyObject {
#[deprecated(note = "use downcast_unchecked_ref instead")]
#[inline(always)]
pub const unsafe fn payload_unchecked<T: PyPayload>(&self) -> &T {
// we cast to a PyInner<T> first because we don't know T's exact offset because of
// varying alignment, but once we get a PyInner<T> the compiler can get it for us
let inner = unsafe { &*(&self.0 as *const PyInner<Erased> as *const PyInner<T>) };
// we cast to a Py<T> first because we don't know T's exact offset because of
// varying alignment, but once we get a Py<T> the compiler can get it for us
let inner = unsafe { &*(&self.0 as *const Py<Erased> as *const Py<T>) };
&inner.payload
}
@@ -2582,9 +2562,6 @@ const _: () = assert!(
const _: () =
assert!(core::mem::size_of::<Option<PyStackRef>>() == core::mem::size_of::<PyStackRef>());
#[repr(transparent)]
pub struct Py<T>(PyInner<T>);
impl<T: PyPayload> Py<T> {
pub fn downgrade(
&self,
@@ -2599,7 +2576,7 @@ impl<T: PyPayload> Py<T> {
#[inline]
pub fn payload(&self) -> &T {
&self.0.payload
&self.payload
}
/// Recover the object pointer from a pointer to its `payload` field.
@@ -2611,8 +2588,7 @@ impl<T: PyPayload> Py<T> {
#[inline]
#[cfg_attr(not(feature = "threading"), allow(dead_code))]
pub(crate) unsafe fn from_payload_ptr(payload: *const T) -> *const Self {
let offset = core::mem::offset_of!(PyInner<T>, payload);
// `Py<T>` is a newtype over `PyInner<T>`, so their addresses coincide.
let offset = core::mem::offset_of!(Self, payload);
unsafe { (payload as *const u8).sub(offset) as *const Self }
}
}
@@ -2622,7 +2598,7 @@ impl<T> ToOwned for Py<T> {
#[inline(always)]
fn to_owned(&self) -> Self::Owned {
self.0.ref_count.inc();
self.ref_count.inc();
PyRef {
ptr: NonNull::from(self),
}
@@ -2634,14 +2610,14 @@ impl<T> Deref for Py<T> {
#[inline(always)]
fn deref(&self) -> &Self::Target {
&self.0.payload
&self.payload
}
}
impl<T: PyPayload> Borrow<PyObject> for Py<T> {
#[inline(always)]
fn borrow(&self) -> &PyObject {
unsafe { &*(&self.0 as *const PyInner<T> as *const PyObject) }
unsafe { &*(self as *const Self as *const PyObject) }
}
}
@@ -2714,7 +2690,7 @@ impl<T: fmt::Debug> fmt::Debug for PyRef<T> {
impl<T> Drop for PyRef<T> {
#[inline]
fn drop(&mut self) {
if self.0.ref_count.dec() {
if self.ref_count.dec() {
unsafe { PyObject::drop_slow(self.ptr.cast::<PyObject>()) }
}
}
@@ -2782,7 +2758,7 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
};
let ptr = if let Some(cached) = cached {
let inner = cached.as_ptr() as *mut PyInner<T>;
let inner = cached.as_ptr() as *mut Py<T>;
unsafe {
core::ptr::write(&mut (*inner).ref_count, RefCount::new());
(*inner).gc_bits.store(0, Ordering::Relaxed);
@@ -2800,7 +2776,7 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
}
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
} else {
let inner = PyInner::new(payload, typ, dict);
let inner = Py::new(payload, typ, dict);
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
};
@@ -2983,20 +2959,20 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
};
use core::mem::MaybeUninit;
static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>);
static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>);
static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyTuple>>, PyInner<PyTuple>);
static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyTuple>>, PyInner<PyTuple>);
static_assertions::assert_eq_size!(MaybeUninit<Py<PyType>>, Py<PyType>);
static_assertions::assert_eq_align!(MaybeUninit<Py<PyType>>, Py<PyType>);
static_assertions::assert_eq_size!(MaybeUninit<Py<PyTuple>>, Py<PyTuple>);
static_assertions::assert_eq_align!(MaybeUninit<Py<PyTuple>>, Py<PyTuple>);
// All three core type objects are instances of `type`, which has HAS_DICT
// and HAS_WEAKREF. Their allocations therefore need both prefixes,
// with the weakref list in front of ObjExt.
let alloc_type_with_prefixes = || -> *mut PyInner<PyType> {
let inner_layout = core::alloc::Layout::new::<MaybeUninit<PyInner<PyType>>>();
let alloc_type_with_prefixes = || -> *mut Py<PyType> {
let inner_layout = core::alloc::Layout::new::<MaybeUninit<Py<PyType>>>();
let ext_layout = core::alloc::Layout::new::<ObjExt>();
let weakref_layout = core::alloc::Layout::new::<WeakRefList>();
// [WeakRefList][ObjExt][PyInner] — the list stays in front of ObjExt.
// [WeakRefList][ObjExt][Py] — the list stays in front of ObjExt.
let (layout, ext_offset) = weakref_layout.extend(ext_layout).unwrap();
let (combined, inner_offset) = layout.extend(inner_layout).unwrap();
let combined = combined.pad_to_align();
@@ -3014,20 +2990,18 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
}
};
let alloc_tuple = || {
Box::into_raw(Box::new(MaybeUninit::<PyInner<PyTuple>>::uninit()))
.cast::<PyInner<PyTuple>>()
};
let alloc_tuple =
|| Box::into_raw(Box::new(MaybeUninit::<Py<PyTuple>>::uninit())).cast::<Py<PyTuple>>();
unsafe fn init_ref_count<T>(ptr: *mut PyInner<T>) {
unsafe fn init_ref_count<T>(ptr: *mut Py<T>) {
unsafe { ptr::addr_of_mut!((*ptr).ref_count).write(RefCount::new()) };
}
unsafe fn initial_ref<T: PyPayload>(ptr: *mut PyInner<T>) -> PyRef<T> {
unsafe fn initial_ref<T: PyPayload>(ptr: *mut Py<T>) -> PyRef<T> {
unsafe { PyRef::from_raw(ptr.cast()) }
}
unsafe fn clone_raw_ref<T: PyPayload>(ptr: *mut PyInner<T>) -> PyRef<T> {
unsafe fn clone_raw_ref<T: PyPayload>(ptr: *mut Py<T>) -> PyRef<T> {
unsafe { &*ptr::addr_of!((*ptr).ref_count) }.inc();
unsafe { PyRef::from_raw(ptr.cast()) }
}
@@ -3038,7 +3012,7 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
unsafe { core::mem::transmute::<PyTupleRef, PyTypeTupleRef>(tuple) }
}
unsafe fn init_inner<T>(ptr: *mut PyInner<T>, typ: PyTypeRef, payload: T)
unsafe fn init_inner<T>(ptr: *mut Py<T>, typ: PyTypeRef, payload: T)
where
T: PyPayload + MaybeTraverse + fmt::Debug,
{
+2 -2
View File
@@ -69,7 +69,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
/// must not rely on its pre-clear contents.
///
/// # Safety
/// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0
/// `obj` must be a valid pointer to a `Py<Self>` with refcount 0
/// whose tp_clear has already run, with no outstanding borrows into the
/// payload (`PyRef::new_ref` may pop and reuse the husk immediately).
#[inline]
@@ -82,7 +82,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
/// reinitialize `ref_count`, `gc_bits`, and `payload`.
///
/// # Safety
/// The returned pointer (if Some) must point to a valid `PyInner<Self>`
/// The returned pointer (if Some) must point to a valid `Py<Self>`
/// whose payload is still initialized from a previous allocation. The caller
/// will drop and overwrite `payload` before reuse.
#[inline]
+7 -4
View File
@@ -4,7 +4,7 @@ use core::any::TypeId;
use crate::{
PyObject, PyObjectRef,
object::{
Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc,
Erased, InstanceDict, MaybeTraverse, Py, PyObjectPayload, debug_obj, default_dealloc,
try_clear_obj, try_traverse_obj,
},
};
@@ -52,8 +52,8 @@ unsafe impl Traverse for InstanceDict {
}
}
unsafe impl Traverse for PyInner<Erased> {
/// Because PyObject hold a `PyInner<Erased>`, so we need to trace it
unsafe impl Traverse for Py<Erased> {
/// Because PyObject hold a `Py<Erased>`, so we need to trace it
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
// For heap type instances, traverse the type reference.
// PyAtomicRef holds a strong reference (via PyRef::leak), so GC must
@@ -82,7 +82,10 @@ unsafe impl Traverse for PyInner<Erased> {
}
}
unsafe impl<T: MaybeTraverse> Traverse for PyInner<T> {
unsafe impl<T: MaybeTraverse> Traverse for Py<T> {
/// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children,
/// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself
///
/// Type is known, so we can call `try_trace` directly instead of using erased type vtable
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
// For heap type instances, traverse the type reference (same as erased version)