mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Use dict keys-version stamps and entry-index hints in LOAD_ATTR/STORE_ATTR specializations (#8350)
* Use dict keys-version stamps and entry-index hints in attr specializations Add a keys-version stamp to Dict: a globally unique u32 assigned lazily and reset on any key-set change (new key, deletion, clear). Value-only updates keep the stamp. - LoadAttrMethodWithValues / LoadAttrNondescriptorWithValues: cache the instance dict's stamp in the pointer cache and skip the shadow probe while the stamp matches. - LoadAttrWithHint: cache the entry index at specialization time; a hit is an identity check on the entry key instead of a hash probe, with self-refresh on miss. - StoreAttrWithHint / StoreAttrInstanceValue: replace values through the cached entry index via set_item_with_hint. Fixes youknowone#42 Assisted-by: Claude * Share keys-version stamps between dicts with identical layouts Derive the keys-version stamp from a shape — the ordered interned-key sequence of a hole-free dict — instead of always allocating a unique stamp. Dicts with identical layouts (instances of the same class built by the same __init__) now carry equal stamps, so a LOAD_ATTR cache entry populated by one instance skips the shadow probe for every instance sharing the layout. Shapes are held in a fixed-size lock-free table keyed by interned key addresses; dicts with holes, non-interned keys, or more than 32 keys fall back to dict-unique stamps. Assisted-by: Claude * Address clippy and review feedback for keys-version stamps - Use BuildHasher::hash_one for shape hashing (clippy manual_hash_one) - Fix set_item_with_hint doc: a refreshed hint is returned on any hint miss, not only when the hinted slot was vacant - Route both holey-dict instances through a single LOAD_ATTR cache site in the snippet test Assisted-by: Claude * Restore try_read_cached_descriptor doc comment to its function The doc block and #[inline] were left attached to store_attr_dict_hinted when it was inserted above try_read_cached_descriptor. Assisted-by: Claude * Specialize LoadAttrWithHint even when the entry index exceeds u16 hint_for_key returns None both for an absent key and for a present key whose entry index does not fit in u16, so very large instance dicts stopped specializing entirely. Use get_item_opt_refresh_hint to decide presence, degrading an unrepresentable hint to 0: the handler then keeps taking its full-probe fallback path. Assisted-by: Claude
This commit is contained in:
1 parent
dd9bce501c
commit
8d7d9d931c
5 files changed
+550
-40
No files matched your search
@@ -776,12 +776,15 @@ impl CodeUnits {
|
||||
/// Store a pointer-sized value atomically in the pointer cache at `index`.
|
||||
///
|
||||
/// Uses a single `AtomicUsize` store to prevent torn writes when
|
||||
/// multiple threads specialize the same instruction concurrently.
|
||||
/// multiple threads specialize the same instruction concurrently. The
|
||||
/// tear-free width also makes this the right slot for non-pointer guard
|
||||
/// values (e.g. dict keys-version stamps) that must never be observed
|
||||
/// half-written.
|
||||
///
|
||||
/// # Safety
|
||||
/// - `index` must be in bounds.
|
||||
/// - `value` must be `0` or a valid `*const PyObject` encoded as `usize`.
|
||||
/// - Callers must follow the cache invalidation/upgrade protocol:
|
||||
/// - When the slot holds a `*const PyObject` encoded as `usize` (or `0`),
|
||||
/// callers must follow the cache invalidation/upgrade protocol:
|
||||
/// invalidate the version guard before writing and publish the new
|
||||
/// version after writing.
|
||||
pub unsafe fn write_cache_ptr(&self, index: usize, value: usize) {
|
||||
|
||||
@@ -753,6 +753,63 @@ impl Py<PyDict> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup trying a cached entry index hint first.
|
||||
///
|
||||
/// When the hint misses but the key is present, also returns a refreshed
|
||||
/// hint (`None` when the hint hit or no hint is representable).
|
||||
pub(crate) fn get_item_opt_refresh_hint<K: DictKey + ?Sized>(
|
||||
&self,
|
||||
key: &K,
|
||||
hint: u16,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Option<(PyObjectRef, Option<u16>)>> {
|
||||
if self.exact_dict(vm) {
|
||||
if let Some(value) = self.entries.get_hint(vm, key, usize::from(hint))? {
|
||||
return Ok(Some((value, None)));
|
||||
}
|
||||
self.entries.get_with_hint(vm, key)
|
||||
} else {
|
||||
Ok(self.get_item_opt(key, vm)?.map(|value| (value, None)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Store using a cached entry index hint for the value-replace fast path.
|
||||
///
|
||||
/// On a hint miss, returns a refreshed hint for the key (`None` when the
|
||||
/// hint hit or no hint is representable).
|
||||
pub(crate) fn set_item_with_hint<K: DictKey + ?Sized>(
|
||||
&self,
|
||||
key: &K,
|
||||
hint: u16,
|
||||
value: PyObjectRef,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Option<u16>> {
|
||||
if self.exact_dict(vm) {
|
||||
self.entries
|
||||
.insert_with_hint(vm, key, usize::from(hint), value)
|
||||
} else {
|
||||
self.as_object().set_item(key, value, vm)?;
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Current keys-version stamp of the underlying storage (0 if unset).
|
||||
pub(crate) fn keys_version(&self) -> u32 {
|
||||
self.entries.keys_version()
|
||||
}
|
||||
|
||||
/// Current keys-version stamp, assigning one if none is set.
|
||||
///
|
||||
/// Returns 0 for dict subclasses: their lookup can be overridden, so a
|
||||
/// key-set attestation on the raw storage must never be cached for them.
|
||||
pub(crate) fn assign_keys_version(&self, vm: &VirtualMachine) -> u32 {
|
||||
if self.exact_dict(vm) {
|
||||
self.entries.assign_keys_version()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_item<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> PyResult {
|
||||
if self.exact_dict(vm) {
|
||||
self.inner_getitem(key, vm)
|
||||
|
||||
+243
-2
@@ -20,8 +20,8 @@ use alloc::fmt;
|
||||
use core::mem::size_of;
|
||||
use core::ops::ControlFlow;
|
||||
use core::sync::atomic::{
|
||||
AtomicU64,
|
||||
Ordering::{Acquire, Release},
|
||||
AtomicU32, AtomicU64,
|
||||
Ordering::{AcqRel, Acquire, Relaxed, Release},
|
||||
};
|
||||
use num_traits::ToPrimitive;
|
||||
|
||||
@@ -40,6 +40,104 @@ type EntryIndex = usize;
|
||||
pub(crate) struct Dict<T = PyObjectRef> {
|
||||
inner: PyRwLock<DictInner<T>>,
|
||||
version: AtomicU64,
|
||||
/// Keys-version stamp, assigned lazily by `assign_keys_version` and
|
||||
/// reset to 0 whenever the key set changes. Value-only updates keep it.
|
||||
///
|
||||
/// A nonzero stamp identifies either a *shape* — an exact hole-free
|
||||
/// entry sequence of interned string keys, shared by every dict with
|
||||
/// that layout — or, when no shape is derivable, this dict's key set
|
||||
/// frozen at assignment time. Either way a stamp match guarantees the
|
||||
/// entry layout is exactly the one the stamp was issued for, so entry
|
||||
/// indexes cached against a stamp stay valid wherever the stamp matches.
|
||||
keys_version: AtomicU32,
|
||||
}
|
||||
|
||||
/// Source of keys-version stamps. Allocated globally so a shape stamp and a
|
||||
/// dict-unique stamp can never collide.
|
||||
static KEYS_VERSION: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
/// Allocate a new keys-version stamp. Returns 0 once the stamp space is
|
||||
/// exhausted; stamps are only allocated on specialization, so exhaustion is
|
||||
/// unrealistic in practice.
|
||||
fn next_keys_version() -> u32 {
|
||||
KEYS_VERSION
|
||||
.fetch_update(Relaxed, Relaxed, |v| v.checked_add(1))
|
||||
.map_or(0, |v| v + 1)
|
||||
}
|
||||
|
||||
/// Largest key count eligible for a shared shape stamp.
|
||||
const SHAPE_MAX_KEYS: usize = 32;
|
||||
/// Shape registry slot count (power of two).
|
||||
const SHAPE_TABLE_SIZE: usize = 1 << 12;
|
||||
/// Linear-probe limit before giving up on registering a shape.
|
||||
const SHAPE_MAX_PROBE: usize = 8;
|
||||
|
||||
/// A registered shape: the ordered interned-key-pointer sequence of a
|
||||
/// hole-free dict, plus the stamp shared by every dict with that layout.
|
||||
/// Interned strings are never freed, so the addresses are stable identities.
|
||||
struct ShapeData {
|
||||
keys: Box<[usize]>,
|
||||
stamp: u32,
|
||||
}
|
||||
|
||||
/// Lock-free registry mapping shapes to shared stamps. Fixed-size open
|
||||
/// addressing; slots are installed with CAS and never removed, so a stamp
|
||||
/// permanently means "exactly this entry sequence". Registered `ShapeData`
|
||||
/// is intentionally leaked (bounded by the table size). Lock-free makes the
|
||||
/// registry safe across fork() without reinitialization.
|
||||
static SHAPE_TABLE: std::sync::LazyLock<Box<[core::sync::atomic::AtomicPtr<ShapeData>]>> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
(0..SHAPE_TABLE_SIZE)
|
||||
.map(|_| core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()))
|
||||
.collect()
|
||||
});
|
||||
|
||||
fn shape_stamp(shape: &[usize]) -> Option<u32> {
|
||||
use core::hash::BuildHasher;
|
||||
use core::sync::atomic::AtomicPtr;
|
||||
// The hasher seed must be process-stable so equal shapes always probe
|
||||
// the same slots.
|
||||
static SHAPE_HASHER: std::sync::LazyLock<rapidhash::quality::RandomState> =
|
||||
std::sync::LazyLock::new(Default::default);
|
||||
let hash = SHAPE_HASHER.hash_one(shape) as usize;
|
||||
let mut candidate: *mut ShapeData = core::ptr::null_mut();
|
||||
let mut result = None;
|
||||
for probe in 0..SHAPE_MAX_PROBE {
|
||||
let slot: &AtomicPtr<ShapeData> = &SHAPE_TABLE[(hash + probe) & (SHAPE_TABLE_SIZE - 1)];
|
||||
let mut installed = slot.load(Acquire);
|
||||
if installed.is_null() {
|
||||
if candidate.is_null() {
|
||||
let stamp = next_keys_version();
|
||||
if stamp == 0 {
|
||||
break;
|
||||
}
|
||||
candidate = Box::into_raw(Box::new(ShapeData {
|
||||
keys: shape.into(),
|
||||
stamp,
|
||||
}));
|
||||
}
|
||||
match slot.compare_exchange(core::ptr::null_mut(), candidate, AcqRel, Acquire) {
|
||||
Ok(_) => {
|
||||
// SAFETY: candidate was just leaked into the table.
|
||||
result = Some(unsafe { (*candidate).stamp });
|
||||
candidate = core::ptr::null_mut();
|
||||
break;
|
||||
}
|
||||
Err(current) => installed = current,
|
||||
}
|
||||
}
|
||||
// SAFETY: non-null slots reference leaked ShapeData, never freed.
|
||||
let data = unsafe { &*installed };
|
||||
if *data.keys == *shape {
|
||||
result = Some(data.stamp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !candidate.is_null() {
|
||||
// SAFETY: the candidate lost the race and was never shared.
|
||||
drop(unsafe { Box::from_raw(candidate) });
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
unsafe impl<T: Traverse> Traverse for Dict<T> {
|
||||
@@ -105,6 +203,7 @@ impl<T: Clone> Clone for Dict<T> {
|
||||
Self {
|
||||
inner: PyRwLock::new(self.inner.read().clone()),
|
||||
version: AtomicU64::new(0),
|
||||
keys_version: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -119,6 +218,7 @@ impl<T> Default for Dict<T> {
|
||||
entries: Vec::new(),
|
||||
}),
|
||||
version: AtomicU64::new(0),
|
||||
keys_version: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -272,6 +372,83 @@ impl<T: Clone> Dict<T> {
|
||||
self.version.fetch_add(1, Release);
|
||||
}
|
||||
|
||||
/// Current keys-version stamp, or 0 if none has been assigned since the
|
||||
/// last key-set change. Equal nonzero stamps guarantee an unchanged key
|
||||
/// set (values may differ).
|
||||
pub(crate) fn keys_version(&self) -> u32 {
|
||||
self.keys_version.load(Acquire)
|
||||
}
|
||||
|
||||
/// Return the current keys-version stamp, assigning one if none is set.
|
||||
/// Returns 0 only if no stamp could be allocated.
|
||||
///
|
||||
/// When the dict is hole-free and all keys are interned strings, the
|
||||
/// stamp is the *shared shape stamp* for that exact key sequence, so
|
||||
/// dicts with identical layouts (e.g. instances of the same class built
|
||||
/// by the same `__init__`) carry equal stamps and one cached stamp or
|
||||
/// entry index serves them all. Otherwise a dict-unique stamp is used.
|
||||
///
|
||||
/// The shape inspection and the stamp install happen under the inner
|
||||
/// read lock. Key-set changes reset the stamp under the write lock, so
|
||||
/// an installed stamp always attests the layout it was derived from.
|
||||
pub(crate) fn assign_keys_version(&self) -> u32 {
|
||||
let version = self.keys_version.load(Acquire);
|
||||
if version != 0 {
|
||||
return version;
|
||||
}
|
||||
let inner = self.read();
|
||||
// Re-check under the lock: a concurrent assign may have won.
|
||||
let version = self.keys_version.load(Acquire);
|
||||
if version != 0 {
|
||||
return version;
|
||||
}
|
||||
let new_version = Self::derive_shape_stamp(&inner).unwrap_or_else(next_keys_version);
|
||||
if new_version == 0 {
|
||||
return 0;
|
||||
}
|
||||
// Only install over 0 so an already-valid stamp is never replaced.
|
||||
match self
|
||||
.keys_version
|
||||
.compare_exchange(0, new_version, AcqRel, Acquire)
|
||||
{
|
||||
Ok(_) => new_version,
|
||||
Err(current) => current,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the shared shape stamp for the current layout, if it
|
||||
/// qualifies: hole-free entries, bounded size, all keys interned strings.
|
||||
fn derive_shape_stamp(inner: &DictInner<T>) -> Option<u32> {
|
||||
if inner.entries.len() != inner.used || inner.used > SHAPE_MAX_KEYS {
|
||||
return None;
|
||||
}
|
||||
let shape = inner
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let key = &entry.as_ref()?.key;
|
||||
key.is_interned()
|
||||
.then(|| key.as_ref() as *const PyObject as usize)
|
||||
})
|
||||
.collect::<Option<Vec<usize>>>()?;
|
||||
shape_stamp(&shape)
|
||||
}
|
||||
|
||||
/// Reset the keys-version stamp on a key-set change (insert of a new
|
||||
/// key, deletion, or clear). Value-only updates keep the stamp.
|
||||
///
|
||||
/// Must be called while holding the write lock, *before* the key set is
|
||||
/// modified: a lock-free stamp reader that still observes the old stamp
|
||||
/// then provably ran before the change became visible, so acting on the
|
||||
/// old key set is linearizable. A stamp assigned concurrently (between
|
||||
/// this reset and the mutation) can only be trusted by a caller whose
|
||||
/// subsequent probe serializes after the mutation through the inner
|
||||
/// lock, which then reflects the new key set. Since stamps are never
|
||||
/// reused, a cached stamp can never spuriously match again.
|
||||
fn invalidate_keys_version(&self) {
|
||||
self.keys_version.store(0, Release);
|
||||
}
|
||||
|
||||
fn read(&self) -> PyRwLockReadGuard<'_, DictInner<T>> {
|
||||
self.inner.read()
|
||||
}
|
||||
@@ -320,6 +497,7 @@ impl<T: Clone> Dict<T> {
|
||||
// Dict was resized since lookup, retry
|
||||
continue;
|
||||
}
|
||||
self.invalidate_keys_version();
|
||||
inner.unchecked_push(index_index, hash, key.to_pyobject(vm), value, entry_index);
|
||||
self.bump_version();
|
||||
break None;
|
||||
@@ -366,6 +544,62 @@ impl<T: Clone> Dict<T> {
|
||||
Ok(u16::try_from(index).ok())
|
||||
}
|
||||
|
||||
/// Retrieve a key along with its entry index, for hint caching.
|
||||
///
|
||||
/// Same as [`Self::get`], but on a hit also returns the entry index
|
||||
/// usable as a `hint` for [`Self::get_hint`] (`None` if it doesn't fit).
|
||||
pub(crate) fn get_with_hint<K: DictKey + ?Sized>(
|
||||
&self,
|
||||
vm: &VirtualMachine,
|
||||
key: &K,
|
||||
) -> PyResult<Option<(T, Option<u16>)>> {
|
||||
let hash = key.key_hash(vm)?;
|
||||
let ret = loop {
|
||||
let (entry, index_index) = self.lookup(vm, key, hash, None)?;
|
||||
if let Some(index) = entry.index() {
|
||||
let inner = self.read();
|
||||
if let Some(entry) = inner.get_entry_checked(index, index_index) {
|
||||
// The dict was not changed since we did lookup
|
||||
break Some((entry.value.clone(), u16::try_from(index).ok()));
|
||||
}
|
||||
// The dict was changed since we did lookup. Let's try again.
|
||||
} else {
|
||||
break None;
|
||||
}
|
||||
};
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
/// Replace the value at entry index `hint` if that entry's key is
|
||||
/// identical to `key`, otherwise fall back to a full probing store.
|
||||
///
|
||||
/// On a hint miss, returns a refreshed hint for the key (`None` when the
|
||||
/// hint hit or no hint is representable).
|
||||
pub(crate) fn insert_with_hint<K: DictKey + ?Sized>(
|
||||
&self,
|
||||
vm: &VirtualMachine,
|
||||
key: &K,
|
||||
hint: usize,
|
||||
value: T,
|
||||
) -> PyResult<Option<u16>> {
|
||||
let value = {
|
||||
let mut inner = self.write();
|
||||
match inner.entries.get_mut(hint) {
|
||||
Some(Some(entry)) if key.key_is(&entry.key) => {
|
||||
let removed = core::mem::replace(&mut entry.value, value);
|
||||
self.bump_version();
|
||||
drop(inner);
|
||||
// defer dec RC until after the lock is released
|
||||
drop(removed);
|
||||
return Ok(None);
|
||||
}
|
||||
_ => value,
|
||||
}
|
||||
};
|
||||
self.insert(vm, key, value)?;
|
||||
self.hint_for_key(vm, key)
|
||||
}
|
||||
|
||||
/// Fast path lookup using a cached entry index (`hint`).
|
||||
///
|
||||
/// Returns `None` if the hint is stale or the key no longer matches.
|
||||
@@ -433,6 +667,7 @@ impl<T: Clone> Dict<T> {
|
||||
pub(crate) fn clear(&self) {
|
||||
let _removed = {
|
||||
let mut inner = self.write();
|
||||
self.invalidate_keys_version();
|
||||
inner.indices.clear();
|
||||
inner.indices.resize(8, IndexEntry::FREE);
|
||||
inner.used = 0;
|
||||
@@ -521,6 +756,7 @@ impl<T: Clone> Dict<T> {
|
||||
if inner.indices.get(index_index) != Some(&entry) {
|
||||
continue;
|
||||
}
|
||||
self.invalidate_keys_version();
|
||||
inner.unchecked_push(index_index, hash, key.to_owned(), value, entry);
|
||||
self.bump_version();
|
||||
break None;
|
||||
@@ -551,6 +787,7 @@ impl<T: Clone> Dict<T> {
|
||||
let value = default
|
||||
.take()
|
||||
.expect("default must only be computed on insertion")();
|
||||
self.invalidate_keys_version();
|
||||
inner.unchecked_push(
|
||||
index_index,
|
||||
hash,
|
||||
@@ -594,6 +831,7 @@ impl<T: Clone> Dict<T> {
|
||||
.expect("default must only be computed on insertion")();
|
||||
let key_obj = key.to_pyobject(vm);
|
||||
let ret = (key_obj.clone(), value.clone());
|
||||
self.invalidate_keys_version();
|
||||
inner.unchecked_push(index_index, hash, key_obj, value, index_entry);
|
||||
self.bump_version();
|
||||
return Ok(ret);
|
||||
@@ -787,6 +1025,7 @@ impl<T: Clone> Dict<T> {
|
||||
// The dict was changed since we did lookup. Let's try again.
|
||||
_ => return Ok(ControlFlow::Continue(())),
|
||||
}
|
||||
self.invalidate_keys_version();
|
||||
*unsafe {
|
||||
// index_index is result of lookup
|
||||
inner.indices.get_unchecked_mut(index_index)
|
||||
@@ -822,6 +1061,7 @@ impl<T: Clone> Dict<T> {
|
||||
break entry;
|
||||
}
|
||||
};
|
||||
self.invalidate_keys_version();
|
||||
inner.used -= 1;
|
||||
*unsafe {
|
||||
// entry.index always refers valid index
|
||||
@@ -843,6 +1083,7 @@ impl<T: Clone> Dict<T> {
|
||||
/// This is used for circular reference resolution in GC.
|
||||
/// Requires &mut self to avoid lock contention.
|
||||
pub(crate) fn drain_entries(&mut self) -> impl Iterator<Item = (PyObjectRef, T)> + '_ {
|
||||
self.keys_version.store(0, Release);
|
||||
let inner = self.inner.get_mut();
|
||||
inner.used = 0;
|
||||
inner.filled = 0;
|
||||
|
||||
+102
-35
@@ -4430,18 +4430,11 @@ impl ExecutingFrame<'_> {
|
||||
let type_version = self.code.instructions.read_cache_u32(cache_base + 1);
|
||||
|
||||
if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version {
|
||||
// Check instance dict doesn't shadow the method
|
||||
let shadowed = if let Some(dict) = owner.dict() {
|
||||
match dict.get_item_opt(attr_name, vm) {
|
||||
Ok(Some(_)) => true,
|
||||
Ok(None) => false,
|
||||
Err(_) => {
|
||||
// Dict lookup error -> use safe path.
|
||||
return self.load_attr_slow(vm, oparg);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
// Check instance dict doesn't shadow the method.
|
||||
let shadowed = match self.shadowing_instance_attr(cache_base, attr_name, vm) {
|
||||
Ok(shadowed) => shadowed.is_some(),
|
||||
// Dict lookup error -> use safe path.
|
||||
Err(_) => return self.load_attr_slow(vm, oparg),
|
||||
};
|
||||
|
||||
if !shadowed
|
||||
@@ -4489,16 +4482,29 @@ impl ExecutingFrame<'_> {
|
||||
if type_version != 0
|
||||
&& owner.class().tp_version_tag.load(Acquire) == type_version
|
||||
&& let Some(dict) = owner.dict()
|
||||
&& let Some(value) = dict.get_item_opt(attr_name, vm)?
|
||||
{
|
||||
self.pop_value();
|
||||
if oparg.is_method() {
|
||||
self.push_value(value);
|
||||
self.push_value_opt(None);
|
||||
} else {
|
||||
self.push_value(value);
|
||||
// Try the cached entry index first; a hit is an identity
|
||||
// check on the entry key instead of a hash probe.
|
||||
let hint = self.code.instructions.read_cache_u16(cache_base + 3);
|
||||
if let Some((value, refreshed)) =
|
||||
dict.get_item_opt_refresh_hint(attr_name, hint, vm)?
|
||||
{
|
||||
if let Some(new_hint) = refreshed {
|
||||
unsafe {
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_u16(cache_base + 3, new_hint);
|
||||
}
|
||||
}
|
||||
self.pop_value();
|
||||
if oparg.is_method() {
|
||||
self.push_value(value);
|
||||
self.push_value_opt(None);
|
||||
} else {
|
||||
self.push_value(value);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
self.load_attr_slow(vm, oparg)
|
||||
@@ -4559,9 +4565,7 @@ impl ExecutingFrame<'_> {
|
||||
|
||||
if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version {
|
||||
// Instance dict has priority — check if attr is shadowed
|
||||
if let Some(dict) = owner.dict()
|
||||
&& let Some(value) = dict.get_item_opt(attr_name, vm)?
|
||||
{
|
||||
if let Some(value) = self.shadowing_instance_attr(cache_base, attr_name, vm)? {
|
||||
self.pop_value();
|
||||
if oparg.is_method() {
|
||||
self.push_value(value);
|
||||
@@ -4725,7 +4729,10 @@ impl ExecutingFrame<'_> {
|
||||
{
|
||||
self.pop_value(); // owner
|
||||
let value = self.pop_value();
|
||||
dict.set_item(attr_name, value, vm)?;
|
||||
// The key was absent at specialization time, but this
|
||||
// very store inserts it; hint learning makes later
|
||||
// executions replace by entry index.
|
||||
self.store_attr_dict_hinted(&dict, attr_name, value, cache_base, vm)?;
|
||||
return Ok(None);
|
||||
}
|
||||
self.store_attr(vm, attr_idx)
|
||||
@@ -4744,7 +4751,7 @@ impl ExecutingFrame<'_> {
|
||||
{
|
||||
self.pop_value(); // owner
|
||||
let value = self.pop_value();
|
||||
dict.set_item(attr_name, value, vm)?;
|
||||
self.store_attr_dict_hinted(&dict, attr_name, value, cache_base, vm)?;
|
||||
return Ok(None);
|
||||
}
|
||||
self.store_attr(vm, attr_idx)
|
||||
@@ -8055,6 +8062,61 @@ impl ExecutingFrame<'_> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Store an instance attribute through the cached entry index at
|
||||
/// `cache_base + 3`, refreshing the cache when the hint missed.
|
||||
fn store_attr_dict_hinted(
|
||||
&mut self,
|
||||
dict: &Py<PyDict>,
|
||||
attr_name: &'static PyStrInterned,
|
||||
value: PyObjectRef,
|
||||
cache_base: usize,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<()> {
|
||||
let hint = self.code.instructions.read_cache_u16(cache_base + 3);
|
||||
if let Some(new_hint) = dict.set_item_with_hint(attr_name, hint, value, vm)? {
|
||||
unsafe {
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_u16(cache_base + 3, new_hint);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shadow check for method/nondescriptor loads: return the instance
|
||||
/// attribute shadowing the cached class attr, or `None` if not shadowed.
|
||||
///
|
||||
/// A keys-version stamp of the instance dict is kept in the pointer cache
|
||||
/// at `cache_base + 3`. While the dict reports the same stamp, its key
|
||||
/// set is unchanged since the name was last verified absent, so the probe
|
||||
/// is skipped. On a verified-absent probe the current stamp is recorded
|
||||
/// for the next execution.
|
||||
fn shadowing_instance_attr(
|
||||
&self,
|
||||
cache_base: usize,
|
||||
attr_name: &'static PyStrInterned,
|
||||
vm: &VirtualMachine,
|
||||
) -> PyResult<Option<PyObjectRef>> {
|
||||
let Some(dict) = self.top_value().dict() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let stamp = self.code.instructions.read_cache_ptr(cache_base + 3);
|
||||
if stamp != 0 && stamp == dict.keys_version() as usize {
|
||||
return Ok(None);
|
||||
}
|
||||
// Take the stamp before probing so it attests the probed key set.
|
||||
let stamp = dict.assign_keys_version(vm);
|
||||
if let Some(value) = dict.get_item_opt(attr_name, vm)? {
|
||||
return Ok(Some(value));
|
||||
}
|
||||
unsafe {
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_ptr(cache_base + 3, stamp as usize);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Read a cached descriptor pointer and validate it against the expected
|
||||
/// type version, using a lock-free double-check pattern:
|
||||
/// 1. read pointer → incref (try_to_owned)
|
||||
@@ -8408,10 +8470,12 @@ impl ExecutingFrame<'_> {
|
||||
// attribute is missing on both the class and the current
|
||||
// instance, keep the generic opcode and just enter
|
||||
// cooldown instead of specializing a repeated miss path.
|
||||
let has_instance_attr = if let Some(dict) = obj.dict() {
|
||||
match dict.get_item_opt(attr_name, _vm) {
|
||||
Ok(Some(_)) => true,
|
||||
Ok(None) => false,
|
||||
// A present attribute always specializes; when no entry
|
||||
// index is representable the hint degrades to 0 and the
|
||||
// handler simply keeps taking its full-probe fallback.
|
||||
let instance_attr_hint = if let Some(dict) = obj.dict() {
|
||||
match dict.get_item_opt_refresh_hint(attr_name, 0, _vm) {
|
||||
Ok(present) => present.map(|(_, refreshed)| refreshed.unwrap_or(0)),
|
||||
Err(_) => {
|
||||
unsafe {
|
||||
self.code.instructions.write_adaptive_counter(
|
||||
@@ -8427,13 +8491,14 @@ impl ExecutingFrame<'_> {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
None
|
||||
};
|
||||
if has_instance_attr {
|
||||
if let Some(hint) = instance_attr_hint {
|
||||
unsafe {
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_u32(cache_base + 1, type_version);
|
||||
self.code.instructions.write_cache_u16(cache_base + 3, hint);
|
||||
}
|
||||
self.specialize_at(instr_idx, cache_base, Instruction::LoadAttrWithHint);
|
||||
} else {
|
||||
@@ -9951,9 +10016,8 @@ impl ExecutingFrame<'_> {
|
||||
}
|
||||
}
|
||||
} else if let Some(dict) = owner.dict() {
|
||||
let use_hint = match dict.get_item_opt(attr_name, vm) {
|
||||
Ok(Some(_)) => true,
|
||||
Ok(None) => false,
|
||||
let hint = match dict.hint_for_key(attr_name, vm) {
|
||||
Ok(hint) => hint,
|
||||
Err(_) => {
|
||||
unsafe {
|
||||
self.code.instructions.write_adaptive_counter(
|
||||
@@ -9970,11 +10034,14 @@ impl ExecutingFrame<'_> {
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_u32(cache_base + 1, type_version);
|
||||
self.code
|
||||
.instructions
|
||||
.write_cache_u16(cache_base + 3, hint.unwrap_or(0));
|
||||
}
|
||||
self.specialize_at(
|
||||
instr_idx,
|
||||
cache_base,
|
||||
if use_hint {
|
||||
if hint.is_some() {
|
||||
Instruction::StoreAttrWithHint
|
||||
} else {
|
||||
Instruction::StoreAttrInstanceValue
|
||||
|
||||
@@ -69,3 +69,145 @@ def check_latin1_subscr_singleton_after_warmup():
|
||||
|
||||
|
||||
check_latin1_subscr_singleton_after_warmup()
|
||||
|
||||
|
||||
## LOAD_ATTR_METHOD_WITH_VALUES: keys-version shadow check
|
||||
|
||||
|
||||
class MethodHolder:
|
||||
def m(self):
|
||||
return "method"
|
||||
|
||||
|
||||
def method_shadowed_after_specialization():
|
||||
obj = MethodHolder()
|
||||
obj.pad = 1
|
||||
for _ in range(300):
|
||||
assert obj.m() == "method"
|
||||
# Shadowing after warmup must deopt the stamp-based shadow skip.
|
||||
obj.m = lambda: "instance"
|
||||
assert obj.m() == "instance"
|
||||
del obj.m
|
||||
assert obj.m() == "method"
|
||||
obj.__dict__["m"] = lambda: "dict"
|
||||
assert obj.m() == "dict"
|
||||
del obj.__dict__["m"]
|
||||
assert obj.m() == "method"
|
||||
|
||||
|
||||
method_shadowed_after_specialization()
|
||||
|
||||
|
||||
def method_with_value_only_updates():
|
||||
obj = MethodHolder()
|
||||
obj.pad = 0
|
||||
for i in range(500):
|
||||
obj.pad = i # value-only update keeps the keys-version stamp
|
||||
assert obj.m() == "method"
|
||||
|
||||
|
||||
method_with_value_only_updates()
|
||||
|
||||
|
||||
## LOAD_ATTR_WITH_HINT / STORE_ATTR: entry-index hint invalidation
|
||||
|
||||
|
||||
class Plain:
|
||||
pass
|
||||
|
||||
|
||||
def load_hint_survives_key_churn():
|
||||
obj = Plain()
|
||||
obj.a = 1
|
||||
obj.b = 2
|
||||
obj.x = "first"
|
||||
for _ in range(300):
|
||||
assert obj.x == "first"
|
||||
del obj.a
|
||||
del obj.b
|
||||
assert obj.x == "first"
|
||||
del obj.x
|
||||
try:
|
||||
obj.x
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("expected AttributeError")
|
||||
obj.x = "second"
|
||||
assert obj.x == "second"
|
||||
|
||||
|
||||
load_hint_survives_key_churn()
|
||||
|
||||
|
||||
def store_hint_survives_dict_replacement():
|
||||
obj = Plain()
|
||||
obj.v = 0
|
||||
for i in range(500):
|
||||
obj.v = i
|
||||
assert obj.v == i
|
||||
obj.__dict__ = {"v": "fresh"}
|
||||
for i in range(300):
|
||||
obj.v = i
|
||||
assert obj.v == i
|
||||
obj.__dict__.clear()
|
||||
obj.v = "back"
|
||||
assert obj.v == "back"
|
||||
|
||||
|
||||
store_hint_survives_dict_replacement()
|
||||
|
||||
|
||||
## Shared shape stamps: same-layout instances share the shadow-check stamp
|
||||
|
||||
|
||||
class ShapedCounter:
|
||||
def __init__(self):
|
||||
self.a = 1
|
||||
self.b = 2
|
||||
|
||||
def m(self):
|
||||
return "method"
|
||||
|
||||
|
||||
def shape_sharing_shadow_one_instance():
|
||||
objs = [ShapedCounter() for _ in range(30)]
|
||||
for _ in range(300):
|
||||
for o in objs:
|
||||
assert o.m() == "method"
|
||||
objs[13].m = lambda: "thirteen"
|
||||
for i, o in enumerate(objs):
|
||||
expected = "thirteen" if i == 13 else "method"
|
||||
assert o.m() == expected
|
||||
del objs[13].m
|
||||
for o in objs:
|
||||
assert o.m() == "method"
|
||||
|
||||
|
||||
shape_sharing_shadow_one_instance()
|
||||
|
||||
|
||||
def holey_dict_falls_back():
|
||||
class Holey:
|
||||
def m(self):
|
||||
return "g"
|
||||
|
||||
def call_m(obj):
|
||||
# single LOAD_ATTR cache site shared by both instances
|
||||
return obj.m()
|
||||
|
||||
g1, g2 = Holey(), Holey()
|
||||
for o in (g1, g2):
|
||||
o.x = 1
|
||||
o.y = 2
|
||||
o.z = 3
|
||||
del o.y # leaves a hole in the entries
|
||||
for _ in range(300):
|
||||
assert call_m(g1) == "g"
|
||||
assert call_m(g2) == "g"
|
||||
g2.m = lambda: "g2"
|
||||
assert call_m(g1) == "g"
|
||||
assert call_m(g2) == "g2"
|
||||
|
||||
|
||||
holey_dict_falls_back()
|
||||
Reference in new issue
Block a user