mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Merge PyInner into Py (#8820)
Rename the object layout struct `PyInner<T>` to `Py<T>` and make it the public type, replacing the `#[repr(transparent)]` newtype `Py<T>(PyInner<T>)`. Header fields stay `pub(super)`. `PyObject` now wraps `Py<Erased>`. - Drop the `.0` hop on `Py<T>` accesses. - Remove the delegating `Traverse for Py<T>` impl; the former `Traverse for PyInner<T>` impl now serves `Py<T>`. - Remove `Debug for PyInner<T>`; `debug_obj` writes the `[PyObject ...]` form itself. - Use `Py::from_payload_ptr` in `PyWeak::drop_inner` and drop its now-stale `allow(dead_code)`. Assisted-by: Claude Code:claude-opus-5-5
This commit is contained in:
1 parent
98d6f23fe6
commit
393689d847
6 files changed
+96
-119
No files matched your search
@@ -101,7 +101,7 @@ unsafe impl Traverse for PyTuple {
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
// A single freelist for all tuple sizes: `PyInner<PyTuple>` is a
|
||||
// A single freelist for all tuple sizes: `Py<PyTuple>` is a
|
||||
// fixed-size allocation (elements are a separate boxed slice that is
|
||||
// dropped and replaced on reuse), so husks are interchangeable.
|
||||
// freelist_push must not read the payload — it runs after tp_clear,
|
||||
|
||||
@@ -1590,7 +1590,7 @@ fn cleared_frame_access() -> ! {
|
||||
thread_local! {
|
||||
/// Free list of dead frame objects for reuse. Entries are cleared husks
|
||||
/// (`iframe == None`) whose child references were already released.
|
||||
/// PyInner<FrameObject> is fixed-size (localsplus storage is out-of-line),
|
||||
/// Py<FrameObject> is fixed-size (localsplus storage is out-of-line),
|
||||
/// so a single bucket suffices.
|
||||
static FRAME_FREELIST: core::cell::Cell<crate::object::FreeList<FrameObject>> =
|
||||
const { core::cell::Cell::new(crate::object::FreeList::new()) };
|
||||
|
||||
@@ -408,7 +408,7 @@ impl GcState {
|
||||
///
|
||||
/// Like [`Self::track_object`], but for the hot allocation path only:
|
||||
/// `obj`'s `gc_bits` must still hold its freshly-initialized value of `0`
|
||||
/// (true right after `PyInner::new` or a freelist pop, both of which zero
|
||||
/// (true right after `Py::new` or a freelist pop, both of which zero
|
||||
/// it), so the tracked bit can go in with a plain store instead of the
|
||||
/// `fetch_or` `set_gc_tracked()` needs to be safe for the general case
|
||||
/// (e.g. re-tracking a resurrected object, whose bits are not zero — it
|
||||
|
||||
+84
-110
@@ -44,7 +44,7 @@ use core::{
|
||||
ptr::{self, NonNull},
|
||||
};
|
||||
|
||||
// so, PyObjectRef is basically equivalent to `PyRc<PyInner<dyn PyObjectPayload>>`, except it's
|
||||
// so, PyObjectRef is basically equivalent to `PyRc<Py<dyn PyObjectPayload>>`, except it's
|
||||
// only one pointer in width rather than 2. We do that by manually creating a vtable, and putting
|
||||
// a &'static reference to it inside the `PyRc` rather than adjacent to it, like trait objects do.
|
||||
// This can lead to faster code since there's just less data to pass around, as well as because of
|
||||
@@ -61,18 +61,18 @@ use core::{
|
||||
// There has to be padding in the space between the 2 fields. But, if that field is a trait object
|
||||
// (like `dyn PyObjectPayload`) we don't *know* how much padding there is between the `payload`
|
||||
// field and the previous field. So, Rust has to consult the vtable to know the exact offset of
|
||||
// `payload` in `PyInner<dyn PyObjectPayload>`, which has a huge performance impact when *every
|
||||
// `payload` in `Py<dyn PyObjectPayload>`, which has a huge performance impact when *every
|
||||
// single payload access* requires a vtable lookup. Thankfully, we're able to avoid that because of
|
||||
// the way we use PyObjectRef, in that whenever we want to access the payload we (almost) always
|
||||
// access it from a generic function. So, rather than doing
|
||||
//
|
||||
// - check vtable for payload offset
|
||||
// - get offset in PyInner struct
|
||||
// - get offset in Py struct
|
||||
// - call as_any() method of PyObjectPayload
|
||||
// - call downcast_ref() method of Any
|
||||
// we can just do
|
||||
// - check vtable that typeid matches
|
||||
// - pointer cast directly to *const PyInner<T>
|
||||
// - pointer cast directly to *const Py<T>
|
||||
//
|
||||
// and at that point the compiler can know the offset of `payload` for us because **we've given it a
|
||||
// concrete type to work with before we ever access the `payload` field**
|
||||
@@ -230,7 +230,7 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
|
||||
// Published objects (e.g. a tuple stored as a type attribute) must skip the
|
||||
// freelist: `PyRef::new_ref` would reuse the slot and overwrite the refcount
|
||||
// word with a non-atomic write, racing a reader's atomic try-incref. Route
|
||||
// them through `PyInner::dealloc` instead, whose QSBR hook defers the actual
|
||||
// them through `Py::dealloc` instead, whose QSBR hook defers the actual
|
||||
// memory free until readers can no longer observe it.
|
||||
let typ = obj_ref.class();
|
||||
let pushed = if T::HAS_FREELIST
|
||||
@@ -254,7 +254,7 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
|
||||
|
||||
if !pushed {
|
||||
// Deallocate the object memory (handles ObjExt prefix if present)
|
||||
unsafe { PyInner::dealloc(obj as *mut PyInner<T>) };
|
||||
unsafe { Py::dealloc(obj as *mut Py<T>) };
|
||||
}
|
||||
|
||||
// Trashcan: decrement depth and process deferred objects at outermost level
|
||||
@@ -266,20 +266,20 @@ pub(super) unsafe fn debug_obj<T: PyPayload + core::fmt::Debug>(
|
||||
x: &PyObject,
|
||||
f: &mut fmt::Formatter<'_>,
|
||||
) -> fmt::Result {
|
||||
let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) };
|
||||
fmt::Debug::fmt(x, f)
|
||||
let x = unsafe { &*(x as *const PyObject as *const Py<T>) };
|
||||
write!(f, "[PyObject {:?}]", x.payload)
|
||||
}
|
||||
|
||||
/// Call `try_trace` on payload
|
||||
pub(super) unsafe fn try_traverse_obj<T: PyPayload>(x: &PyObject, tracer_fn: &mut TraverseFn<'_>) {
|
||||
let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) };
|
||||
let x = unsafe { &*(x as *const PyObject as *const Py<T>) };
|
||||
let payload = &x.payload;
|
||||
payload.try_traverse(tracer_fn)
|
||||
}
|
||||
|
||||
/// Call `try_clear` on payload to extract child references (tp_clear)
|
||||
pub(super) unsafe fn try_clear_obj<T: PyPayload>(x: *mut PyObject, out: &mut Vec<PyObjectRef>) {
|
||||
let x = unsafe { &mut *(x as *mut PyInner<T>) };
|
||||
let x = unsafe { &mut *(x as *mut Py<T>) };
|
||||
x.payload.try_clear(out);
|
||||
}
|
||||
|
||||
@@ -335,8 +335,8 @@ pub(crate) const GC_REACHABLE: u32 = u32::MAX;
|
||||
/// Link implementation for GC intrusive linked list tracking
|
||||
pub(crate) struct GcLink;
|
||||
|
||||
// SAFETY: PyObject (PyInner<Erased>) is heap-allocated and pinned in memory
|
||||
// once created. gc_pointers is at a fixed offset in PyInner.
|
||||
// SAFETY: PyObject (Py<Erased>) is heap-allocated and pinned in memory
|
||||
// once created. gc_pointers is at a fixed offset in Py.
|
||||
unsafe impl Link for GcLink {
|
||||
type Handle = NonNull<PyObject>;
|
||||
type Target = PyObject;
|
||||
@@ -350,18 +350,18 @@ unsafe impl Link for GcLink {
|
||||
}
|
||||
|
||||
unsafe fn pointers(target: NonNull<PyObject>) -> NonNull<Pointers<PyObject>> {
|
||||
let inner_ptr = target.as_ptr() as *mut PyInner<Erased>;
|
||||
let inner_ptr = target.as_ptr() as *mut Py<Erased>;
|
||||
unsafe { NonNull::new_unchecked(&raw mut (*inner_ptr).gc_pointers) }
|
||||
}
|
||||
}
|
||||
|
||||
/// Extension fields for objects that need dict or member slots.
|
||||
/// Allocated as a prefix before PyInner when needed (prefix allocation pattern).
|
||||
/// Access via `PyInner::ext_ref()` using negative offset from the object pointer.
|
||||
/// Allocated as a prefix before Py when needed (prefix allocation pattern).
|
||||
/// Access via `Py::ext_ref()` using negative offset from the object pointer.
|
||||
///
|
||||
/// align(8) ensures size_of::<ObjExt>() is always a multiple of 8,
|
||||
/// so the offset from Layout::extend equals size_of::<ObjExt>() for any
|
||||
/// PyInner<T> alignment (important on wasm32 where pointers are 4 bytes
|
||||
/// Py<T> alignment (important on wasm32 where pointers are 4 bytes
|
||||
/// but some payloads like PyWeak have align 8 due to i64 fields).
|
||||
#[repr(C, align(8))]
|
||||
pub(super) struct ObjExt {
|
||||
@@ -392,10 +392,10 @@ impl fmt::Debug for ObjExt {
|
||||
/// immediately in front of `ObjExt`; any alignment padding is before the cells.
|
||||
const EXT_OFFSET: usize = core::mem::size_of::<ObjExt>();
|
||||
|
||||
/// Byte offset of member cell `index` from the start of `PyInner`.
|
||||
/// Byte offset of member cell `index` from the start of `Py`.
|
||||
/// Cells live in front of the object, so the offset is negative.
|
||||
///
|
||||
/// `ObjExt` stays flush with `PyInner`. A subclass that adds `__weakref__`
|
||||
/// `ObjExt` stays flush with `Py`. A subclass that adds `__weakref__`
|
||||
/// puts that list in front of the cells, so this offset does not move.
|
||||
pub(crate) fn slot_member_offset(index: usize) -> isize {
|
||||
// Cell 0 sits directly in front of ObjExt. Higher indexes extend further
|
||||
@@ -419,8 +419,8 @@ fn slot_region_layout(member_count: usize) -> Option<core::alloc::Layout> {
|
||||
}
|
||||
const WEAKREF_OFFSET: usize = core::mem::size_of::<WeakRefList>();
|
||||
|
||||
/// Distance from `PyInner` back to a `WeakRefList`.
|
||||
/// Layout: `[WeakRefList?][slots?][ObjExt?][PyInner]`.
|
||||
/// Distance from `Py` back to a `WeakRefList`.
|
||||
/// Layout: `[WeakRefList?][slots?][ObjExt?][Py]`.
|
||||
fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize {
|
||||
let ext = if has_ext { EXT_OFFSET } else { 0 };
|
||||
let slots = slot_region_layout(member_count).map_or(0, |layout| layout.size());
|
||||
@@ -429,17 +429,17 @@ fn weakref_prefix_offset(has_ext: bool, member_count: usize) -> usize {
|
||||
|
||||
const _: () =
|
||||
assert!(core::mem::size_of::<ObjExt>().is_multiple_of(core::mem::align_of::<ObjExt>()));
|
||||
const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<PyInner<()>>());
|
||||
const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<Py<()>>());
|
||||
const _: () = assert!(
|
||||
core::mem::size_of::<WeakRefList>().is_multiple_of(core::mem::align_of::<WeakRefList>())
|
||||
);
|
||||
const _: () = assert!(core::mem::align_of::<WeakRefList>() >= core::mem::align_of::<PyInner<()>>());
|
||||
const _: () = assert!(core::mem::align_of::<WeakRefList>() >= core::mem::align_of::<Py<()>>());
|
||||
|
||||
/// This is an actual python object. It consists of a `typ` which is the
|
||||
/// python class, and carries some rust payload optionally. This rust
|
||||
/// payload can be a rust float or rust int in case of float and int objects.
|
||||
#[repr(C)]
|
||||
pub(super) struct PyInner<T> {
|
||||
pub struct Py<T> {
|
||||
pub(super) ref_count: RefCount,
|
||||
pub(super) vtable: &'static PyObjVTable,
|
||||
/// GC bits for free-threading (like ob_gc_bits)
|
||||
@@ -463,7 +463,7 @@ pub(super) struct PyInner<T> {
|
||||
|
||||
pub(super) payload: T,
|
||||
}
|
||||
pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
|
||||
pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<Py<()>>();
|
||||
|
||||
// ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits,
|
||||
// generation, owner and refs take eight bytes between them. A 64-bit header had
|
||||
@@ -472,7 +472,7 @@ pub const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>();
|
||||
// this holds.
|
||||
const _: () = assert!(SIZEOF_PYOBJECT_HEAD == 5 * core::mem::size_of::<usize>() + 8);
|
||||
|
||||
// `PyInner::drop_fields` names `payload` and `typ`; it is only complete while
|
||||
// `Py::drop_fields` names `payload` and `typ`; it is only complete while
|
||||
// every other field stays trivially destructible.
|
||||
const _: () = assert!(
|
||||
!core::mem::needs_drop::<RefCount>()
|
||||
@@ -483,23 +483,23 @@ const _: () = assert!(
|
||||
&& !core::mem::needs_drop::<Pointers<PyObject>>()
|
||||
);
|
||||
|
||||
impl<T> PyInner<T> {
|
||||
impl<T> Py<T> {
|
||||
/// Read type flags and member_count via raw pointers to avoid Stacked Borrows
|
||||
/// violations during bootstrap, where type objects have self-referential typ pointers.
|
||||
#[inline(always)]
|
||||
fn read_type_flags(&self) -> (crate::types::PyTypeFlags, usize) {
|
||||
let typ_ptr = self.typ.load_raw();
|
||||
let slots = unsafe { core::ptr::addr_of!((*typ_ptr).0.payload.slots) };
|
||||
let slots = unsafe { core::ptr::addr_of!((*typ_ptr).payload.slots) };
|
||||
let flags = unsafe { core::ptr::addr_of!((*slots).flags).read() };
|
||||
let member_count = unsafe { core::ptr::addr_of!((*slots).member_count).read() };
|
||||
(flags, member_count)
|
||||
}
|
||||
|
||||
/// Access the ObjExt prefix at a negative offset from this PyInner.
|
||||
/// Access the ObjExt prefix at a negative offset from this Py.
|
||||
/// Returns None if this object was allocated without dict/slots.
|
||||
///
|
||||
/// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
|
||||
/// `ObjExt` is always immediately in front of `PyInner`.
|
||||
/// Layout: [WeakRefList?][slots?][ObjExt?][Py]
|
||||
/// `ObjExt` is always immediately in front of `Py`.
|
||||
#[inline(always)]
|
||||
pub(super) fn ext_ref(&self) -> Option<&ObjExt> {
|
||||
let (flags, member_count) = self.read_type_flags();
|
||||
@@ -514,7 +514,7 @@ impl<T> PyInner<T> {
|
||||
}
|
||||
|
||||
/// Member cells are the pointer array immediately before [`ObjExt`].
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner]`.
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py]`.
|
||||
pub(super) fn slot_cells(&self) -> &[PyAtomicRef<PyObject>] {
|
||||
let Some(ext) = self.ext_ref() else {
|
||||
return &[];
|
||||
@@ -532,10 +532,10 @@ impl<T> PyInner<T> {
|
||||
unsafe { core::slice::from_raw_parts(ptr, member_count) }
|
||||
}
|
||||
|
||||
/// Access the WeakRefList prefix at a fixed negative offset from this PyInner.
|
||||
/// Access the WeakRefList prefix at a fixed negative offset from this Py.
|
||||
/// Returns None if the type does not support weakrefs.
|
||||
///
|
||||
/// Layout: [WeakRefList?][slots?][ObjExt?][PyInner]
|
||||
/// Layout: [WeakRefList?][slots?][ObjExt?][Py]
|
||||
/// The list sits in front of the slot cells so adding it on a subclass
|
||||
/// does not move inherited member offsets.
|
||||
#[inline(always)]
|
||||
@@ -553,20 +553,6 @@ impl<T> PyInner<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: fmt::Debug> fmt::Debug for PyInner<T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "[PyObject {:?}]", self.payload)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<T: MaybeTraverse> Traverse for Py<T> {
|
||||
/// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children,
|
||||
/// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself
|
||||
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
|
||||
self.0.traverse(tracer_fn)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Traverse for PyObject {
|
||||
/// DO notice that call `trace` on `PyObject` means apply `tracer_fn` on `PyObject`'s children,
|
||||
/// not like call `trace` on `PyObjectRef` which apply `tracer_fn` on `PyObjectRef` itself
|
||||
@@ -689,8 +675,7 @@ unsafe fn try_reuse_weakref(ptr: *mut Py<PyWeak>) -> Option<PyRef<PyWeak>> {
|
||||
return None;
|
||||
}
|
||||
let node = unsafe { &*ptr };
|
||||
node.0
|
||||
.ref_count
|
||||
node.ref_count
|
||||
.safe_inc()
|
||||
.then(|| unsafe { PyRef::from_raw(ptr) })
|
||||
}
|
||||
@@ -824,7 +809,7 @@ impl WeakRefList {
|
||||
match NonNull::new(candidate_ptr) {
|
||||
Some(candidate) => {
|
||||
let node = unsafe { candidate.as_ref() };
|
||||
let has_callback = unsafe { (&*node.0.payload.callback.get()).is_some() };
|
||||
let has_callback = unsafe { (&*node.payload.callback.get()).is_some() };
|
||||
let node_cls = node.class();
|
||||
// PyWeakref_CheckProxy: the basic-proxy slot is reserved for
|
||||
// the canonical proxy type; subclasses and callback-less ref
|
||||
@@ -860,7 +845,7 @@ impl WeakRefList {
|
||||
let wr = unsafe { node.as_ref() };
|
||||
|
||||
// Mark weakref as dead
|
||||
wr.0.payload
|
||||
wr.payload
|
||||
.wr_object
|
||||
.store(ptr::null_mut(), Ordering::Relaxed);
|
||||
|
||||
@@ -872,9 +857,9 @@ impl WeakRefList {
|
||||
}
|
||||
|
||||
// Collect callback only if we can still acquire a strong ref.
|
||||
if wr.0.ref_count.safe_inc() {
|
||||
if wr.ref_count.safe_inc() {
|
||||
let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) };
|
||||
let cb = unsafe { wr.0.payload.callback.get().replace(None) };
|
||||
let cb = unsafe { wr.payload.callback.get().replace(None) };
|
||||
if let Some(cb) = cb {
|
||||
callbacks.push((wr_ref, cb));
|
||||
}
|
||||
@@ -911,7 +896,7 @@ impl WeakRefList {
|
||||
let wr = unsafe { node.as_ref() };
|
||||
|
||||
// Mark weakref as dead
|
||||
wr.0.payload
|
||||
wr.payload
|
||||
.wr_object
|
||||
.store(ptr::null_mut(), Ordering::Relaxed);
|
||||
|
||||
@@ -923,9 +908,9 @@ impl WeakRefList {
|
||||
}
|
||||
|
||||
// Collect callback without invoking only if we can keep weakref alive.
|
||||
if wr.0.ref_count.safe_inc() {
|
||||
if wr.ref_count.safe_inc() {
|
||||
let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) };
|
||||
let cb = unsafe { wr.0.payload.callback.get().replace(None) };
|
||||
let cb = unsafe { wr.payload.callback.get().replace(None) };
|
||||
if let Some(cb) = cb {
|
||||
callbacks.push((wr_ref, cb));
|
||||
}
|
||||
@@ -943,7 +928,7 @@ impl WeakRefList {
|
||||
let mut count = 0usize;
|
||||
let mut current = NonNull::new(self.head.load(Ordering::Relaxed));
|
||||
while let Some(node) = current {
|
||||
if unsafe { node.as_ref() }.0.ref_count.get() > 0 {
|
||||
if unsafe { node.as_ref() }.ref_count.get() > 0 {
|
||||
count += 1;
|
||||
}
|
||||
current = unsafe { WeakLink::pointers(node).as_ref().get_next() };
|
||||
@@ -957,7 +942,7 @@ impl WeakRefList {
|
||||
let mut current = NonNull::new(self.head.load(Ordering::Relaxed));
|
||||
while let Some(node) = current {
|
||||
let wr = unsafe { node.as_ref() };
|
||||
if wr.0.ref_count.safe_inc() {
|
||||
if wr.ref_count.safe_inc() {
|
||||
v.push(unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) });
|
||||
}
|
||||
current = unsafe { WeakLink::pointers(node).as_ref().get_next() };
|
||||
@@ -991,7 +976,7 @@ unsafe impl Link for WeakLink {
|
||||
#[inline(always)]
|
||||
unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> {
|
||||
// SAFETY: requirements forwarded from caller
|
||||
unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers) }
|
||||
unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).payload.pointers) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1087,12 +1072,7 @@ impl PyWeak {
|
||||
let wrl = obj.0.weakref_list_ref().unwrap();
|
||||
|
||||
// Compute our Py<PyWeak> node pointer from payload address
|
||||
let offset = std::mem::offset_of!(PyInner<Self>, payload);
|
||||
let py_inner = (self as *const Self)
|
||||
.cast::<u8>()
|
||||
.wrapping_sub(offset)
|
||||
.cast::<PyInner<Self>>();
|
||||
let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py<Self>) };
|
||||
let node_ptr = unsafe { NonNull::new_unchecked(Py::from_payload_ptr(self).cast_mut()) };
|
||||
|
||||
// Unlink from list
|
||||
unsafe { unlink_weakref(wrl, node_ptr) };
|
||||
@@ -1214,7 +1194,7 @@ impl InstanceDict {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: PyPayload> PyInner<T> {
|
||||
impl<T: PyPayload> Py<T> {
|
||||
/// Run the destructors of the fields that have one, payload first.
|
||||
///
|
||||
/// Declaration order would drop `typ` first, and `PyAtomicRef::drop`
|
||||
@@ -1229,11 +1209,11 @@ impl<T: PyPayload> PyInner<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Deallocate a PyInner, handling optional prefix(es).
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
|
||||
/// Deallocate a Py, handling optional prefix(es).
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py<T>]`
|
||||
///
|
||||
/// # Safety
|
||||
/// `ptr` must be a valid pointer from `PyInner::new` and must not be used after this call.
|
||||
/// `ptr` must be a valid pointer from `Py::new` and must not be used after this call.
|
||||
unsafe fn dealloc(ptr: *mut Self) {
|
||||
unsafe {
|
||||
let (flags, member_count) = (*ptr).read_type_flags();
|
||||
@@ -1309,10 +1289,10 @@ impl<T: PyPayload> PyInner<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
|
||||
/// Allocate a new PyInner, optionally with prefix(es).
|
||||
/// Returns a raw pointer to the PyInner (NOT the allocation start).
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][PyInner<T>]`
|
||||
impl<T: PyPayload + core::fmt::Debug> Py<T> {
|
||||
/// Allocate a new Py, optionally with prefix(es).
|
||||
/// Returns a raw pointer to the Py (NOT the allocation start).
|
||||
/// Layout: `[WeakRefList?][PyAtomicRef<PyObject>; N][ObjExt?][Py<T>]`
|
||||
fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> *mut Self {
|
||||
let member_count = typ.slots.member_count;
|
||||
let needs_ext = typ
|
||||
@@ -1331,7 +1311,7 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
|
||||
);
|
||||
|
||||
if needs_ext || needs_weakref {
|
||||
// Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][PyInner]
|
||||
// Build layout left-to-right: [WeakRefList?][slots?][ObjExt?][Py]
|
||||
let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap();
|
||||
|
||||
let weakref_start = if needs_weakref {
|
||||
@@ -1429,7 +1409,7 @@ impl<T: PyPayload + core::fmt::Debug> PyInner<T> {
|
||||
/// Thread-local freelist storage for reusing object allocations.
|
||||
///
|
||||
/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw
|
||||
/// `PyInner<T>` allocations without running payload destructors to avoid
|
||||
/// `Py<T>` allocations without running payload destructors to avoid
|
||||
/// accessing already-destroyed thread-local storage (GC state, other freelists).
|
||||
pub(crate) struct FreeList<T: PyPayload> {
|
||||
items: Vec<*mut PyObject>,
|
||||
@@ -1461,7 +1441,7 @@ impl<T: PyPayload> Drop for FreeList<T> {
|
||||
// MAX_FREELIST per type per thread.
|
||||
for ptr in self.items.drain(..) {
|
||||
unsafe {
|
||||
alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::<PyInner<T>>());
|
||||
alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::<Py<T>>());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1506,7 +1486,7 @@ cfg_select! {
|
||||
}
|
||||
|
||||
#[repr(transparent)]
|
||||
pub struct PyObject(PyInner<Erased>);
|
||||
pub struct PyObject(Py<Erased>);
|
||||
|
||||
impl Deref for PyObjectRef {
|
||||
type Target = PyObject;
|
||||
@@ -1557,7 +1537,7 @@ impl PyObject {
|
||||
/// (same guarantee as `_Py_TryIncRefShared`).
|
||||
#[inline]
|
||||
pub unsafe fn try_to_owned_from_ptr(ptr: *mut Self) -> Option<PyObjectRef> {
|
||||
let inner = ptr.cast::<PyInner<Erased>>();
|
||||
let inner = ptr.cast::<Py<Erased>>();
|
||||
let ref_count = unsafe { &*core::ptr::addr_of!((*inner).ref_count) };
|
||||
if ref_count.safe_inc() {
|
||||
Some(PyObjectRef {
|
||||
@@ -1653,7 +1633,7 @@ impl PyObjectRef {
|
||||
|
||||
impl PyObject {
|
||||
/// Returns the WeakRefList if the type supports weakrefs (HAS_WEAKREF).
|
||||
/// The WeakRefList is stored as a separate prefix before PyInner,
|
||||
/// The WeakRefList is stored as a separate prefix before Py,
|
||||
/// independent from ObjExt (dict/slots).
|
||||
#[inline(always)]
|
||||
fn weak_ref_list(&self) -> Option<&WeakRefList> {
|
||||
@@ -1669,7 +1649,7 @@ impl PyObject {
|
||||
None
|
||||
} else {
|
||||
let head = unsafe { &*head_ptr };
|
||||
if head.0.ref_count.safe_inc() {
|
||||
if head.ref_count.safe_inc() {
|
||||
Some(unsafe { PyRef::from_raw(head_ptr) }.into())
|
||||
} else {
|
||||
None
|
||||
@@ -1758,9 +1738,9 @@ impl PyObject {
|
||||
#[deprecated(note = "use downcast_unchecked_ref instead")]
|
||||
#[inline(always)]
|
||||
pub const unsafe fn payload_unchecked<T: PyPayload>(&self) -> &T {
|
||||
// we cast to a PyInner<T> first because we don't know T's exact offset because of
|
||||
// varying alignment, but once we get a PyInner<T> the compiler can get it for us
|
||||
let inner = unsafe { &*(&self.0 as *const PyInner<Erased> as *const PyInner<T>) };
|
||||
// we cast to a Py<T> first because we don't know T's exact offset because of
|
||||
// varying alignment, but once we get a Py<T> the compiler can get it for us
|
||||
let inner = unsafe { &*(&self.0 as *const Py<Erased> as *const Py<T>) };
|
||||
&inner.payload
|
||||
}
|
||||
|
||||
@@ -2582,9 +2562,6 @@ const _: () = assert!(
|
||||
const _: () =
|
||||
assert!(core::mem::size_of::<Option<PyStackRef>>() == core::mem::size_of::<PyStackRef>());
|
||||
|
||||
#[repr(transparent)]
|
||||
pub struct Py<T>(PyInner<T>);
|
||||
|
||||
impl<T: PyPayload> Py<T> {
|
||||
pub fn downgrade(
|
||||
&self,
|
||||
@@ -2599,7 +2576,7 @@ impl<T: PyPayload> Py<T> {
|
||||
|
||||
#[inline]
|
||||
pub fn payload(&self) -> &T {
|
||||
&self.0.payload
|
||||
&self.payload
|
||||
}
|
||||
|
||||
/// Recover the object pointer from a pointer to its `payload` field.
|
||||
@@ -2611,8 +2588,7 @@ impl<T: PyPayload> Py<T> {
|
||||
#[inline]
|
||||
#[cfg_attr(not(feature = "threading"), allow(dead_code))]
|
||||
pub(crate) unsafe fn from_payload_ptr(payload: *const T) -> *const Self {
|
||||
let offset = core::mem::offset_of!(PyInner<T>, payload);
|
||||
// `Py<T>` is a newtype over `PyInner<T>`, so their addresses coincide.
|
||||
let offset = core::mem::offset_of!(Self, payload);
|
||||
unsafe { (payload as *const u8).sub(offset) as *const Self }
|
||||
}
|
||||
}
|
||||
@@ -2622,7 +2598,7 @@ impl<T> ToOwned for Py<T> {
|
||||
|
||||
#[inline(always)]
|
||||
fn to_owned(&self) -> Self::Owned {
|
||||
self.0.ref_count.inc();
|
||||
self.ref_count.inc();
|
||||
PyRef {
|
||||
ptr: NonNull::from(self),
|
||||
}
|
||||
@@ -2634,14 +2610,14 @@ impl<T> Deref for Py<T> {
|
||||
|
||||
#[inline(always)]
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0.payload
|
||||
&self.payload
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: PyPayload> Borrow<PyObject> for Py<T> {
|
||||
#[inline(always)]
|
||||
fn borrow(&self) -> &PyObject {
|
||||
unsafe { &*(&self.0 as *const PyInner<T> as *const PyObject) }
|
||||
unsafe { &*(self as *const Self as *const PyObject) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2714,7 +2690,7 @@ impl<T: fmt::Debug> fmt::Debug for PyRef<T> {
|
||||
impl<T> Drop for PyRef<T> {
|
||||
#[inline]
|
||||
fn drop(&mut self) {
|
||||
if self.0.ref_count.dec() {
|
||||
if self.ref_count.dec() {
|
||||
unsafe { PyObject::drop_slow(self.ptr.cast::<PyObject>()) }
|
||||
}
|
||||
}
|
||||
@@ -2782,7 +2758,7 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
|
||||
};
|
||||
|
||||
let ptr = if let Some(cached) = cached {
|
||||
let inner = cached.as_ptr() as *mut PyInner<T>;
|
||||
let inner = cached.as_ptr() as *mut Py<T>;
|
||||
unsafe {
|
||||
core::ptr::write(&mut (*inner).ref_count, RefCount::new());
|
||||
(*inner).gc_bits.store(0, Ordering::Relaxed);
|
||||
@@ -2800,7 +2776,7 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {
|
||||
}
|
||||
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
|
||||
} else {
|
||||
let inner = PyInner::new(payload, typ, dict);
|
||||
let inner = Py::new(payload, typ, dict);
|
||||
unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }
|
||||
};
|
||||
|
||||
@@ -2983,20 +2959,20 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
|
||||
};
|
||||
use core::mem::MaybeUninit;
|
||||
|
||||
static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>);
|
||||
static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>);
|
||||
static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyTuple>>, PyInner<PyTuple>);
|
||||
static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyTuple>>, PyInner<PyTuple>);
|
||||
static_assertions::assert_eq_size!(MaybeUninit<Py<PyType>>, Py<PyType>);
|
||||
static_assertions::assert_eq_align!(MaybeUninit<Py<PyType>>, Py<PyType>);
|
||||
static_assertions::assert_eq_size!(MaybeUninit<Py<PyTuple>>, Py<PyTuple>);
|
||||
static_assertions::assert_eq_align!(MaybeUninit<Py<PyTuple>>, Py<PyTuple>);
|
||||
|
||||
// All three core type objects are instances of `type`, which has HAS_DICT
|
||||
// and HAS_WEAKREF. Their allocations therefore need both prefixes,
|
||||
// with the weakref list in front of ObjExt.
|
||||
let alloc_type_with_prefixes = || -> *mut PyInner<PyType> {
|
||||
let inner_layout = core::alloc::Layout::new::<MaybeUninit<PyInner<PyType>>>();
|
||||
let alloc_type_with_prefixes = || -> *mut Py<PyType> {
|
||||
let inner_layout = core::alloc::Layout::new::<MaybeUninit<Py<PyType>>>();
|
||||
let ext_layout = core::alloc::Layout::new::<ObjExt>();
|
||||
let weakref_layout = core::alloc::Layout::new::<WeakRefList>();
|
||||
|
||||
// [WeakRefList][ObjExt][PyInner] — the list stays in front of ObjExt.
|
||||
// [WeakRefList][ObjExt][Py] — the list stays in front of ObjExt.
|
||||
let (layout, ext_offset) = weakref_layout.extend(ext_layout).unwrap();
|
||||
let (combined, inner_offset) = layout.extend(inner_layout).unwrap();
|
||||
let combined = combined.pad_to_align();
|
||||
@@ -3014,20 +2990,18 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
|
||||
}
|
||||
};
|
||||
|
||||
let alloc_tuple = || {
|
||||
Box::into_raw(Box::new(MaybeUninit::<PyInner<PyTuple>>::uninit()))
|
||||
.cast::<PyInner<PyTuple>>()
|
||||
};
|
||||
let alloc_tuple =
|
||||
|| Box::into_raw(Box::new(MaybeUninit::<Py<PyTuple>>::uninit())).cast::<Py<PyTuple>>();
|
||||
|
||||
unsafe fn init_ref_count<T>(ptr: *mut PyInner<T>) {
|
||||
unsafe fn init_ref_count<T>(ptr: *mut Py<T>) {
|
||||
unsafe { ptr::addr_of_mut!((*ptr).ref_count).write(RefCount::new()) };
|
||||
}
|
||||
|
||||
unsafe fn initial_ref<T: PyPayload>(ptr: *mut PyInner<T>) -> PyRef<T> {
|
||||
unsafe fn initial_ref<T: PyPayload>(ptr: *mut Py<T>) -> PyRef<T> {
|
||||
unsafe { PyRef::from_raw(ptr.cast()) }
|
||||
}
|
||||
|
||||
unsafe fn clone_raw_ref<T: PyPayload>(ptr: *mut PyInner<T>) -> PyRef<T> {
|
||||
unsafe fn clone_raw_ref<T: PyPayload>(ptr: *mut Py<T>) -> PyRef<T> {
|
||||
unsafe { &*ptr::addr_of!((*ptr).ref_count) }.inc();
|
||||
unsafe { PyRef::from_raw(ptr.cast()) }
|
||||
}
|
||||
@@ -3038,7 +3012,7 @@ pub(crate) fn init_type_hierarchy() -> BootstrapTypeHierarchy {
|
||||
unsafe { core::mem::transmute::<PyTupleRef, PyTypeTupleRef>(tuple) }
|
||||
}
|
||||
|
||||
unsafe fn init_inner<T>(ptr: *mut PyInner<T>, typ: PyTypeRef, payload: T)
|
||||
unsafe fn init_inner<T>(ptr: *mut Py<T>, typ: PyTypeRef, payload: T)
|
||||
where
|
||||
T: PyPayload + MaybeTraverse + fmt::Debug,
|
||||
{
|
||||
|
||||
@@ -69,7 +69,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
|
||||
/// must not rely on its pre-clear contents.
|
||||
///
|
||||
/// # Safety
|
||||
/// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0
|
||||
/// `obj` must be a valid pointer to a `Py<Self>` with refcount 0
|
||||
/// whose tp_clear has already run, with no outstanding borrows into the
|
||||
/// payload (`PyRef::new_ref` may pop and reuse the husk immediately).
|
||||
#[inline]
|
||||
@@ -82,7 +82,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static {
|
||||
/// reinitialize `ref_count`, `gc_bits`, and `payload`.
|
||||
///
|
||||
/// # Safety
|
||||
/// The returned pointer (if Some) must point to a valid `PyInner<Self>`
|
||||
/// The returned pointer (if Some) must point to a valid `Py<Self>`
|
||||
/// whose payload is still initialized from a previous allocation. The caller
|
||||
/// will drop and overwrite `payload` before reuse.
|
||||
#[inline]
|
||||
|
||||
@@ -4,7 +4,7 @@ use core::any::TypeId;
|
||||
use crate::{
|
||||
PyObject, PyObjectRef,
|
||||
object::{
|
||||
Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc,
|
||||
Erased, InstanceDict, MaybeTraverse, Py, PyObjectPayload, debug_obj, default_dealloc,
|
||||
try_clear_obj, try_traverse_obj,
|
||||
},
|
||||
};
|
||||
@@ -52,8 +52,8 @@ unsafe impl Traverse for InstanceDict {
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Traverse for PyInner<Erased> {
|
||||
/// Because PyObject hold a `PyInner<Erased>`, so we need to trace it
|
||||
unsafe impl Traverse for Py<Erased> {
|
||||
/// Because PyObject hold a `Py<Erased>`, so we need to trace it
|
||||
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
|
||||
// For heap type instances, traverse the type reference.
|
||||
// PyAtomicRef holds a strong reference (via PyRef::leak), so GC must
|
||||
@@ -82,7 +82,10 @@ unsafe impl Traverse for PyInner<Erased> {
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<T: MaybeTraverse> Traverse for PyInner<T> {
|
||||
unsafe impl<T: MaybeTraverse> Traverse for Py<T> {
|
||||
/// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children,
|
||||
/// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself
|
||||
///
|
||||
/// Type is known, so we can call `try_trace` directly instead of using erased type vtable
|
||||
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
|
||||
// For heap type instances, traverse the type reference (same as erased version)
|
||||
|
||||
Reference in new issue
Block a user