Separate constructor and instance vectorcall dispatch (#9320)

Keep constructor fast paths separate from per-instance vectorcall support.
Track automatic eligibility without discarding raw support required by
PyVectorcall_Call, and recheck eligibility in retained callable wrappers.

Implementation and source review were AI-assisted. The model version
identifier is unavailable; automated checks are not human verification.

Assisted-by: OpenAI Codex:model version unavailable

Co-authored-by: youknowdot <336763669+youknowdot@users.noreply.github.com>
This commit is contained in:
youknowdotandyouknowdot authored and GitHub committed 2026-10-23 17:59:19 +09:00
1 parent 44b96328ed
commit afe5d9030a
19 files changed
+214 -102

No files matched your search

+1 -1
View File
@@ -203,7 +203,7 @@ pub unsafe extern "C" fn PyVectorcall_Call(
Some(kwnames.as_slice())
};
callable.vectorcall(args, num_positional_args, kwnames, vm)
callable.vectorcall_call(args, num_positional_args, kwnames, vm)
})
}
+5
View File
@@ -1414,6 +1414,11 @@ where
slots.#slot_ident.store(Some(Self::#ident as _));
slots.has_release_buffer.store(Self::RELEASE_BUFFER);
}
} else if slot_name == "vectorcall" {
quote_spanned! { span =>
slots.#slot_ident.store(Some(Self::#ident as _));
slots.flags.insert(::rustpython_vm::types::PyTypeFlags::HAVE_VECTORCALL);
}
} else if POINTER_SLOTS.contains(&slot_name.as_str()) {
quote_spanned! { span =>
slots.#slot_ident.store(Some(PointerSlot::from(Self::#ident())));
+1 -1
View File
@@ -222,7 +222,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.bool_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_bool));
}
+6
View File
@@ -331,6 +331,12 @@ pub(crate) fn init(context: &'static Context) {
.slots
.vectorcall
.store(Some(vectorcall_native_function));
context
.types
.builtin_function_or_method_type
.slots
.flags
.insert(crate::types::PyTypeFlags::HAVE_VECTORCALL);
PyNativeMethod::extend_class(context, context.types.builtin_method_type);
}
+10
View File
@@ -1658,6 +1658,11 @@ pub(crate) fn init(ctx: &'static Context) {
.slots
.vectorcall
.store(Some(vectorcall_method_descriptor));
ctx.types
.method_descriptor_type
.slots
.flags
.insert(crate::types::PyTypeFlags::HAVE_VECTORCALL);
PyClassMethodDescriptor::extend_class(ctx, ctx.types.classmethod_descriptor_type);
PyWrapper::extend_class(ctx, ctx.types.wrapper_descriptor_type);
ctx.types
@@ -1665,6 +1670,11 @@ pub(crate) fn init(ctx: &'static Context) {
.slots
.vectorcall
.store(Some(vectorcall_wrapper));
ctx.types
.wrapper_descriptor_type
.slots
.flags
.insert(crate::types::PyTypeFlags::HAVE_VECTORCALL);
PyMethodWrapper::extend_class(ctx, ctx.types.method_wrapper_type);
}
+2 -2
View File
@@ -2599,7 +2599,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.dict_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_dict));
PyDictKeys::extend_class(context, context.types.dict_keys_type);
PyDictKeyIterator::extend_class(context, context.types.dict_keyiterator_type);
@@ -2615,7 +2615,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.frozendict_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_frozendict));
}
+1 -1
View File
@@ -585,5 +585,5 @@ fn vectorcall_float(
#[rustfmt::skip] // to avoid line splitting
pub(crate) fn init(context: &'static Context) {
PyFloat::extend_class(context, context.types.float_type);
context.types.float_type.slots().vectorcall.store(Some(vectorcall_float));
context.types.float_type.slots().constructor_vectorcall.store(Some(vectorcall_float));
}
+12
View File
@@ -1876,6 +1876,12 @@ pub(crate) fn init(context: &'static Context) {
.slots
.vectorcall
.store(Some(vectorcall_function));
context
.types
.function_type
.slots
.flags
.insert(crate::types::PyTypeFlags::HAVE_VECTORCALL);
PyBoundMethod::extend_class(context, context.types.bound_method_type);
context
@@ -1884,6 +1890,12 @@ pub(crate) fn init(context: &'static Context) {
.slots
.vectorcall
.store(Some(vectorcall_bound_method));
context
.types
.bound_method_type
.slots
.flags
.insert(crate::types::PyTypeFlags::HAVE_VECTORCALL);
PyCell::extend_class(context, context.types.cell_type);
}
+1 -1
View File
@@ -946,6 +946,6 @@ pub(crate) fn init(context: &'static Context) {
.types
.int_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_int));
}
+4 -1
View File
@@ -1202,7 +1202,10 @@ fn vectorcall_list(
pub(crate) fn init(context: &'static Context) {
let list_type = &context.types.list_type;
PyList::extend_class(context, list_type);
list_type.slots.vectorcall.store(Some(vectorcall_list));
list_type
.slots
.constructor_vectorcall
.store(Some(vectorcall_list));
PyListIterator::extend_class(context, context.types.list_iterator_type);
PyListReverseIterator::extend_class(context, context.types.list_reverseiterator_type);
+2 -2
View File
@@ -1885,7 +1885,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.set_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_set));
PyFrozenSet::extend_class(context, context.types.frozenset_type);
@@ -1893,7 +1893,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.frozenset_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_frozenset));
PySetIterator::extend_class(context, context.types.set_iterator_type);
+1 -1
View File
@@ -1959,7 +1959,7 @@ pub(crate) fn init(ctx: &'static Context) {
ctx.types
.str_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_str));
PyStrIterator::extend_class(ctx, ctx.types.str_iterator_type);
+1 -1
View File
@@ -852,7 +852,7 @@ pub(crate) fn init(context: &'static Context) {
.types
.tuple_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_tuple));
}
+22 -29
View File
@@ -27,8 +27,8 @@ use crate::{
protocol::{PyIterReturn, PyNumberMethods},
types::{
AsNumber, Callable, Constructor, GetAttr, Initializer, NewFunc, PyTypeFlags, PyTypeSlots,
Representable, SLOT_DEFS, SetAttr, TypeDataRef, TypeDataRefMut, TypeDataSlot, fn_addr,
new_wrapper,
Representable, SLOT_DEFS, SetAttr, TypeDataRef, TypeDataRefMut, TypeDataSlot,
VectorCallFunc, fn_addr, new_wrapper,
},
};
use core::{
@@ -3433,26 +3433,15 @@ fn subtype_get_weakref(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
* The magical type type
*/
/// Vectorcall for PyType (PEP 590).
/// Fast path: type(x) returns x.__class__ without constructing FuncArgs.
///
/// # Implementation note: `slots.vectorcall` dual use
///
/// CPython has three distinct fields on PyTypeObject:
/// - `tp_vectorcall`: constructor fast path (e.g. `list_vectorcall`)
/// - `tp_vectorcall_offset`: per-instance vectorcall for callables (e.g. functions)
/// - `tp_call`: standard call slot
///
/// RustPython collapses the first two into a single `slots.vectorcall`. The
/// `call.is_none()` guard below distinguishes the two uses: callable types have
/// `slots.call` set, so their `slots.vectorcall` is for calling instances,
/// not for construction.
///
/// This heuristic is correct for all current builtins but is not a general
/// solution — `type` itself is both callable and has a constructor vectorcall,
/// handled by the explicit `zelf.is(type_type)` check above the guard.
/// If more such types arise, consider splitting into a dedicated
/// `constructor_vectorcall` slot.
// Types can be visible to custom mro/__classcell__ code during initialization.
// A missing new slot or constructor implementation leaves ordinary call as fallback.
fn resolve_type_vectorcall(obj: &PyObject) -> Option<VectorCallFunc> {
let typ = obj.downcast_ref::<PyType>()?;
typ.slots.new.load()?;
typ.slots.constructor_vectorcall.load()
}
/// Vectorcall for type's own constructor (PEP 590).
fn vectorcall_type(
zelf_obj: &PyObject,
args: Vec<PyObjectRef>,
@@ -3468,12 +3457,6 @@ fn vectorcall_type(
if nargs == 1 && no_kwargs {
return Ok(args[0].obj_type());
}
} else if zelf.slots().call.load().is_none() && zelf.slots().new.load().is_some() {
// Per-type constructor vectorcall for non-callable types (dict, list, int, etc.)
// Also guard on slots.new to avoid dispatching for DISALLOW_INSTANTIATION types.
if let Some(type_vc) = zelf.slots().vectorcall.load() {
return type_vc(zelf_obj, args, nargs, kwnames, vm);
}
}
// Fallback: construct FuncArgs and use standard call
@@ -3486,8 +3469,18 @@ pub(crate) fn init(ctx: &'static Context) {
ctx.types
.type_type
.slots
.vectorcall
.constructor_vectorcall
.store(Some(vectorcall_type));
ctx.types
.type_type
.slots
.vectorcall_resolver
.store(Some(resolve_type_vectorcall));
ctx.types
.type_type
.slots
.flags
.insert(PyTypeFlags::HAVE_VECTORCALL);
}
pub(crate) fn call_slot_new(
+5 -2
View File
@@ -7410,7 +7410,8 @@ impl ExecutingFrame<'_> {
let nargs: u32 = arg.into();
let callable = self.nth_value(nargs + 1);
if let Some(cls) = callable.downcast_ref::<PyType>()
&& cls.slots().vectorcall.load().is_some()
&& cls.slots().constructor_vectorcall.load().is_some()
&& callable.vectorcall_func().is_some()
{
let (callable, args_vec) = self.take_call_args(nargs as usize);
let effective_nargs = args_vec.len();
@@ -11396,7 +11397,9 @@ impl ExecutingFrame<'_> {
return;
}
}
if cls.slots().vectorcall.load().is_some() {
if cls.slots().constructor_vectorcall.load().is_some()
&& callable.vectorcall_func().is_some()
{
self.specialize_at(instr_idx, cache_base, Instruction::CallBuiltinClass);
return;
}
+101 -24
View File
@@ -1,7 +1,7 @@
use crate::{
builtins::{PyBoundMethod, PyFunction},
function::{FuncArgs, IntoFuncArgs},
types::{GenericMethod, VectorCallFunc},
types::{GenericMethod, PyTypeFlags, VectorCallFunc},
{PyObject, PyObjectRef, PyResult, VirtualMachine},
};
@@ -55,6 +55,78 @@ impl PyObject {
};
callable.invoke_vectorcall(args, nargs, kwnames, vm)
}
/// PyVectorcall_Call: invoke retained raw support without automatic eligibility
/// or a tp_call fallback.
pub fn vectorcall_call(
&self,
args: Vec<PyObjectRef>,
nargs: usize,
kwnames: Option<&[PyObjectRef]>,
vm: &VirtualMachine,
) -> PyResult {
let vc = self.raw_vectorcall_func().ok_or_else(|| {
vm.new_type_error(format!(
"'{}' object does not support vectorcall",
self.class().slot_name()
))
})?;
self.invoke_vectorcall_func(vc, args, nargs, kwnames, vm)
}
#[inline]
pub(crate) fn vectorcall_func(&self) -> Option<VectorCallFunc> {
if self
.class()
.slots()
.flags
.has_feature(PyTypeFlags::HAVE_VECTORCALL)
{
self.raw_vectorcall_func()
} else {
None
}
}
#[inline]
fn raw_vectorcall_func(&self) -> Option<VectorCallFunc> {
let slots = self.class().slots();
slots.vectorcall.load().or_else(|| {
slots
.vectorcall_resolver
.load()
.and_then(|resolve| resolve(self))
})
}
#[inline]
fn invoke_vectorcall_func(
&self,
vc: VectorCallFunc,
args: Vec<PyObjectRef>,
nargs: usize,
kwnames: Option<&[PyObjectRef]>,
vm: &VirtualMachine,
) -> PyResult {
if !vm.use_tracing.get() {
return vc(self, args, nargs, kwnames, vm);
}
let is_python_callable = self.downcast_ref::<PyFunction>().is_some()
|| self.downcast_ref::<PyBoundMethod>().is_some();
if is_python_callable {
vc(self, args, nargs, kwnames, vm)
} else {
let callable = self.to_owned();
vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?;
let result = vc(self, args, nargs, kwnames, vm);
if result.is_ok() {
vm.trace_event(TraceEvent::CReturn, Some(callable))?;
} else {
let _ = vm.trace_event(TraceEvent::CException, Some(callable));
}
result
}
}
}
#[derive(Debug)]
@@ -68,7 +140,7 @@ impl<'a> PyCallable<'a> {
pub fn new(obj: &'a PyObject) -> Option<Self> {
let slots = obj.class().slots();
let call = slots.call.load()?;
let vectorcall = slots.vectorcall.load();
let vectorcall = obj.vectorcall_func();
Some(PyCallable {
obj,
call,
@@ -78,10 +150,17 @@ impl<'a> PyCallable<'a> {
pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult {
let args = args.into_args(vm);
// _PyVectorcall_Call
if self.vectorcall.is_some() && args.kwargs.is_empty() {
if self.vectorcall_revoked() {
// A retained wrapper must use current tp_call, not its cached snapshot.
return self.obj.call_with_args(args, vm);
}
if let Some(vc) = self.vectorcall
&& args.kwargs.is_empty()
{
let nargs = args.args.len();
return self.invoke_vectorcall(args.args, nargs, None, vm);
return self
.obj
.invoke_vectorcall_func(vc, args.args, nargs, None, vm);
}
if !vm.use_tracing.get() {
return (self.call)(self.obj, args, vm);
@@ -106,7 +185,7 @@ impl<'a> PyCallable<'a> {
}
}
/// Vectorcall dispatch: use vectorcall slot if available, else fall back to FuncArgs.
/// Use automatic vectorcall if available, else fall back to FuncArgs.
#[inline]
pub fn invoke_vectorcall(
&self,
@@ -115,31 +194,29 @@ impl<'a> PyCallable<'a> {
kwnames: Option<&[PyObjectRef]>,
vm: &VirtualMachine,
) -> PyResult {
if self.vectorcall_revoked() {
return self.obj.vectorcall(args, nargs, kwnames, vm);
}
if let Some(vc) = self.vectorcall {
if !vm.use_tracing.get() {
return vc(self.obj, args, nargs, kwnames, vm);
}
let is_python_callable = self.obj.downcast_ref::<PyFunction>().is_some()
|| self.obj.downcast_ref::<PyBoundMethod>().is_some();
if is_python_callable {
vc(self.obj, args, nargs, kwnames, vm)
} else {
let callable = self.obj.to_owned();
vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?;
let result = vc(self.obj, args, nargs, kwnames, vm);
if result.is_ok() {
vm.trace_event(TraceEvent::CReturn, Some(callable))?;
} else {
let _ = vm.trace_event(TraceEvent::CException, Some(callable));
}
result
}
self.obj
.invoke_vectorcall_func(vc, args, nargs, kwnames, vm)
} else {
// Fallback: convert owned Vec to FuncArgs (move, no clone)
let func_args = FuncArgs::from_vectorcall_owned(args, nargs, kwnames);
self.invoke(func_args, vm)
}
}
#[inline]
fn vectorcall_revoked(&self) -> bool {
self.vectorcall.is_some()
&& !self
.obj
.class()
.slots()
.flags
.has_feature(PyTypeFlags::HAVE_VECTORCALL)
}
}
/// Trace events for sys.settrace and sys.setprofile.
+1 -1
View File
@@ -321,7 +321,7 @@ mod _functools {
let combined_nargs = combined_args.len();
let kwnames = kwnames.unwrap_or_default();
if func.class().slots.vectorcall.load().is_none() {
if func.vectorcall_func().is_none() {
// FuncArgs callables retain their existing keyword binding path.
// In particular, do not introduce vectorcall's hash callbacks
// when the wrapped callable does not support vectorcall.
+14 -11
View File
@@ -149,7 +149,12 @@ pub struct PyTypeSlots {
// More standard operations (here for binary compatibility)
pub hash: AtomicCell<Option<HashFunc>>,
pub call: AtomicCell<Option<GenericMethod>>,
/// Retained instance-call implementation, independent of automatic eligibility.
pub vectorcall: AtomicCell<Option<VectorCallFunc>>,
/// Nullable per-object lookup used by type's constructor dispatch.
pub(crate) vectorcall_resolver: AtomicCell<Option<VectorcallResolver>>,
/// Constructor implementation for this type object; never inherited.
pub constructor_vectorcall: AtomicCell<Option<VectorCallFunc>>,
pub str: AtomicCell<Option<StringifyFunc>>,
pub repr: AtomicCell<Option<StringifyFunc>>,
pub getattro: AtomicCell<Option<GetattroFunc>>,
@@ -253,6 +258,7 @@ bitflagset::bitflagset! {
const IMMUTABLETYPE = 8;
const HEAPTYPE = 9;
const BASETYPE = 10;
const HAVE_VECTORCALL = 11;
const HAVE_GC = 14;
const METHOD_DESCRIPTOR = 17;
const IS_ABSTRACT = 20;
@@ -369,6 +375,7 @@ pub(crate) type VectorCallFunc = fn(
Option<&[PyObjectRef]>, // kwnames (keyword argument names)
&VirtualMachine,
) -> PyResult;
type VectorcallResolver = fn(&PyObject) -> Option<VectorCallFunc>;
pub(crate) type HashFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyHash>;
// CallFunc = GenericMethod
pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyRef<PyStr>>;
@@ -1066,20 +1073,17 @@ impl PyType {
}
SlotAccessor::TpCall => {
update_main_slot!(call, call_wrapper, Call);
// When __call__ is overridden in Python, clear vectorcall
// so the slow path through call_wrapper is used.
if ADD {
self.slots.vectorcall.store(None);
// Generic __call__ disables automatic dispatch, not retained raw support.
// A missing call slot alone does not revoke the CPython eligibility flag.
if self.slots.call.load().map(fn_addr)
== Some(fn_addr(call_wrapper as GenericMethod))
{
self.slots.flags.remove(PyTypeFlags::HAVE_VECTORCALL);
}
}
SlotAccessor::TpIter => update_main_slot!(iter, iter_wrapper, Iter),
SlotAccessor::TpIternext => update_main_slot!(iternext, iternext_wrapper, IterNext),
SlotAccessor::TpInit => {
update_main_slot!(init, init_wrapper, Init);
if ADD {
self.slots.vectorcall.store(None);
}
}
SlotAccessor::TpInit => update_main_slot!(init, init_wrapper, Init),
SlotAccessor::TpNew => {
// __new__ is a staticmethod, not a PyWrapper descriptor, so
// lookup_slot_in_mro cannot classify it. Resolve __new__
@@ -1102,7 +1106,6 @@ impl PyType {
};
if needs_wrapper {
self.slots.new.store(Some(new_wrapper));
self.slots.vectorcall.store(None);
} else {
let inherited = self.base.deref().and_then(|base| base.slots.new.load());
self.slots.new.store(inherited);
+24 -24
View File
@@ -2,7 +2,7 @@
//!
//! This module provides a centralized array of all slot definitions,
use super::{PyComparisonOp, PyTypeSlots, fn_addr};
use super::{PyComparisonOp, PyTypeFlags, PyTypeSlots, fn_addr};
use crate::builtins::descriptor::SlotFunc;
use num_enum::TryFromPrimitive;
@@ -465,23 +465,7 @@ impl SlotAccessor {
Self::TpHash => inherit_main!(hash),
Self::TpRepr => inherit_main!(repr),
Self::TpStr => inherit_main!(str),
Self::TpCall => {
inherit_main!(call);
// CPython does not inherit tp_vectorcall to heap types at all; it only
// inherits tp_vectorcall_offset (the per-instance callable fast path).
// RustPython approximates this by inheriting vectorcall only from types
// with a call slot (instance-call vectorcall), not from types that use
// vectorcall as a constructor fast path (call=None).
// See vectorcall_type() in type.rs for the dual-use design rationale.
let inherited_vc = mro.iter().find_map(|cls| {
if cls.payload().slots.call.load().is_some() {
cls.payload().slots.vectorcall.load()
} else {
None
}
});
typ.slots.vectorcall.store(inherited_vc);
}
Self::TpCall => inherit_main!(call),
Self::TpIter => inherit_main!(iter),
Self::TpIternext => inherit_main!(iternext),
Self::TpInit => inherit_main!(init),
@@ -617,14 +601,21 @@ impl SlotAccessor {
Self::TpRepr => copy_main!(repr),
Self::TpStr => copy_main!(str),
Self::TpCall => {
copy_main!(call);
// See inherit_from_mro TpCall for rationale.
// Initial/idempotent role inheritance, not concurrent role replacement.
if typ.slots.vectorcall.load().is_none()
&& base.slots.call.load().is_some()
&& let Some(base_val) = base.slots.vectorcall.load()
&& typ.slots.vectorcall_resolver.load().is_none()
{
typ.slots.vectorcall.store(Some(base_val));
typ.slots.vectorcall.store(base.slots.vectorcall.load());
typ.slots
.vectorcall_resolver
.store(base.slots.vectorcall_resolver.load());
}
if typ.slots.call.load().is_none()
&& base.slots.flags.has_feature(PyTypeFlags::HAVE_VECTORCALL)
{
typ.slots.flags.insert(PyTypeFlags::HAVE_VECTORCALL);
}
copy_main!(call);
}
Self::TpIter => copy_main!(iter),
Self::TpIternext => copy_main!(iternext),
@@ -646,7 +637,16 @@ impl SlotAccessor {
Self::TpDel => copy_main!(del),
Self::TpGetattro => copy_main!(getattro),
Self::TpSetattro => copy_main!(setattro),
Self::TpDescrGet => copy_main!(descr_get),
Self::TpDescrGet => {
copy_main!(descr_get);
if typ.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE)
&& base.slots.flags.has_feature(PyTypeFlags::METHOD_DESCRIPTOR)
&& let Some(descr_get) = typ.slots.descr_get.load()
&& base.slots.descr_get.load().map(fn_addr) == Some(fn_addr(descr_get))
{
typ.slots.flags.insert(PyTypeFlags::METHOD_DESCRIPTOR);
}
}
Self::TpDescrSet => copy_main!(descr_set),
Self::TpRichcompare => copy_main!(richcompare),