Implement __slots__ with member_descriptor (#4106)

This commit is contained in:
Lee Dogeon authored and GitHub committed 2022-08-25 02:11:23 +09:00
1 parent 05d73dd718
commit 4ccdc19208
11 files changed
+391 -25

No files matched your search

-2
View File
@@ -93,8 +93,6 @@ class TestRlcompleter(unittest.TestCase):
self.assertEqual(completer.complete('f.b', 0), 'f.bar')
self.assertEqual(f.calls, 1)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_uncreated_attr(self):
# Attributes like properties and slots should be completed even when
# they haven't been created on an instance
+75
View File
@@ -24,6 +24,7 @@ enum AttrName {
Slot,
Attr,
ExtendClass,
Member,
}
impl std::fmt::Display for AttrName {
@@ -36,6 +37,7 @@ impl std::fmt::Display for AttrName {
Self::Slot => "pyslot",
Self::Attr => "pyattr",
Self::ExtendClass => "extend_class",
Self::Member => "pymember",
};
s.fmt(f)
}
@@ -52,6 +54,7 @@ impl FromStr for AttrName {
"pyslot" => Self::Slot,
"pyattr" => Self::Attr,
"extend_class" => Self::ExtendClass,
"pymember" => Self::Member,
s => {
return Err(s.to_owned());
}
@@ -423,6 +426,11 @@ struct ExtendClassItem {
inner: ContentItemInner<AttrName>,
}
/// #[pymember]
struct MemberItem {
inner: ContentItemInner<AttrName>,
}
impl ContentItem for MethodItem {
type AttrName = AttrName;
fn inner(&self) -> &ContentItemInner<AttrName> {
@@ -453,6 +461,12 @@ impl ContentItem for ExtendClassItem {
&self.inner
}
}
impl ContentItem for MemberItem {
type AttrName = AttrName;
fn inner(&self) -> &ContentItemInner<AttrName> {
&self.inner
}
}
struct ImplItemArgs<'a, Item: ItemLike> {
item: &'a Item,
@@ -661,6 +675,42 @@ where
}
}
impl<Item> ImplItem<Item> for MemberItem
where
Item: ItemLike + ToTokens + GetIdent,
{
fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> {
let func = args
.item
.function_or_method()
.map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?;
let ident = &func.sig().ident;
let item_attr = args.attrs.remove(self.index());
let item_meta = MemberItemMeta::from_attr(ident.clone(), &item_attr)?;
let py_name = item_meta.member_name()?;
let tokens = {
quote_spanned! { ident.span() =>
class.set_str_attr(
#py_name,
ctx.new_member(#py_name, Self::#ident, class),
ctx,
);
}
};
args.context.impl_extend_items.add_item(
ident.clone(),
vec![py_name],
args.cfgs.to_vec(),
tokens,
5,
)?;
Ok(())
}
}
#[derive(Default)]
#[allow(clippy::type_complexity)]
struct GetSetNursery {
@@ -930,6 +980,28 @@ impl SlotItemMeta {
}
}
struct MemberItemMeta(ItemMetaInner);
impl ItemMeta for MemberItemMeta {
const ALLOWED_NAMES: &'static [&'static str] = &["magic"];
fn from_inner(inner: ItemMetaInner) -> Self {
Self(inner)
}
fn inner(&self) -> &ItemMetaInner {
&self.0
}
}
impl MemberItemMeta {
fn member_name(&self) -> Result<String> {
let inner = self.inner();
let magic = inner._bool("magic")?;
let name = inner.item_name();
Ok(if magic { format!("__{}__", name) } else { name })
}
}
struct ExtractedImplAttrs {
with_impl: TokenStream,
with_slots: TokenStream,
@@ -1049,6 +1121,9 @@ where
ExtendClass => Box::new(ExtendClassItem {
inner: ContentItemInner { index, attr_name },
}),
Member => Box::new(MemberItem {
inner: ContentItemInner { index, attr_name },
}),
})
}
+1
View File
@@ -23,6 +23,7 @@ pub(crate) const ALL_ALLOWED_NAMES: &[&str] = &[
"pyattr",
"pyslot",
"extend_class",
"pymember",
];
#[derive(Clone)]
+149
View File
@@ -0,0 +1,149 @@
use super::{PyStr, PyType, PyTypeRef};
use crate::{
class::PyClassImpl,
types::{Constructor, GetDescriptor, Unconstructible},
AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
};
use rustpython_common::lock::PyRwLock;
#[derive(Debug)]
pub struct DescrObject {
pub typ: PyTypeRef,
pub name: String,
pub qualname: PyRwLock<Option<String>>,
}
#[derive(Debug)]
pub enum MemberKind {
ObjectEx = 16,
}
pub enum MemberGetter {
Getter(fn(&VirtualMachine, PyObjectRef) -> PyResult),
Offset(usize),
}
pub struct MemberDef {
pub name: String,
pub kind: MemberKind,
pub getter: MemberGetter,
pub doc: Option<String>,
}
impl MemberDef {
fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
match self.getter {
MemberGetter::Getter(getter) => (getter)(vm, obj),
MemberGetter::Offset(offset) => get_slot_from_object(obj, offset, self, vm),
}
}
}
impl std::fmt::Debug for MemberDef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MemberDef")
.field("name", &self.name)
.field("kind", &self.kind)
.field("doc", &self.doc)
.finish()
}
}
#[pyclass(name = "member_descriptor", module = false)]
#[derive(Debug)]
pub struct MemberDescrObject {
pub common: DescrObject,
pub member: MemberDef,
}
impl PyPayload for MemberDescrObject {
fn class(vm: &VirtualMachine) -> &'static Py<PyType> {
vm.ctx.types.member_descriptor_type
}
}
fn calculate_qualname(descr: &DescrObject, vm: &VirtualMachine) -> PyResult<Option<String>> {
if let Some(qualname) = vm.get_attribute_opt(descr.typ.to_owned().into(), "__qualname__")? {
let str = qualname.downcast::<PyStr>().map_err(|_| {
vm.new_type_error(
"<descriptor>.__objclass__.__qualname__ is not a unicode object".to_owned(),
)
})?;
Ok(Some(format!("{}.{}", str, descr.name)))
} else {
Ok(None)
}
}
#[pyclass(with(GetDescriptor, Constructor), flags(BASETYPE))]
impl MemberDescrObject {
#[pymethod(magic)]
fn repr(zelf: PyRef<Self>) -> String {
format!(
"<member '{}' of '{}' objects>",
zelf.common.name,
zelf.common.typ.name(),
)
}
#[pyproperty(magic)]
fn doc(zelf: PyRef<Self>) -> Option<String> {
zelf.member.doc.to_owned()
}
#[pyproperty(magic)]
fn qualname(&self, vm: &VirtualMachine) -> PyResult<Option<String>> {
let qualname = self.common.qualname.read();
Ok(if qualname.is_none() {
drop(qualname);
let calculated = calculate_qualname(&self.common, vm)?;
*self.common.qualname.write() = calculated.to_owned();
calculated
} else {
qualname.to_owned()
})
}
}
// PyMember_GetOne
fn get_slot_from_object(
obj: PyObjectRef,
offset: usize,
member: &MemberDef,
vm: &VirtualMachine,
) -> PyResult {
let slot = match member.kind {
MemberKind::ObjectEx => obj.get_slot(offset).ok_or_else(|| {
vm.new_attribute_error(format!(
"'{}' object has no attribute '{}'",
obj.class().name(),
member.name
))
})?,
};
Ok(slot)
}
impl Unconstructible for MemberDescrObject {}
impl GetDescriptor for MemberDescrObject {
fn descr_get(
zelf: PyObjectRef,
obj: Option<PyObjectRef>,
_cls: Option<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult {
match obj {
Some(x) => {
let zelf = Self::_zelf(zelf, vm)?;
zelf.member.get(x, vm)
}
None => Ok(zelf),
}
}
}
pub fn init(context: &Context) {
let member_descriptor_type = &context.types.member_descriptor_type;
MemberDescrObject::extend_class(context, member_descriptor_type);
}
+16 -8
View File
@@ -5,7 +5,10 @@ use super::{
tuple::PyTupleTyped, PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStr, PyStrRef,
PyTupleRef, PyType, PyTypeRef,
};
#[cfg(feature = "jit")]
use crate::common::lock::OnceCell;
use crate::common::lock::PyMutex;
use crate::convert::ToPyObject;
use crate::function::ArgMapping;
use crate::{
bytecode,
@@ -16,8 +19,6 @@ use crate::{
types::{Callable, Comparable, Constructor, GetAttr, GetDescriptor, PyComparisonOp},
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
};
#[cfg(feature = "jit")]
use crate::{common::lock::OnceCell, convert::ToPyObject};
use itertools::Itertools;
#[cfg(feature = "jit")]
use rustpython_jit::CompiledCode;
@@ -352,14 +353,21 @@ impl PyFunction {
self.defaults_and_kwdefaults.lock().1 = kwdefaults
}
#[pyproperty(magic)]
fn globals(&self) -> PyDictRef {
self.globals.clone()
// {"__closure__", T_OBJECT, OFF(func_closure), READONLY},
// {"__doc__", T_OBJECT, OFF(func_doc), 0},
// {"__globals__", T_OBJECT, OFF(func_globals), READONLY},
// {"__module__", T_OBJECT, OFF(func_module), 0},
// {"__builtins__", T_OBJECT, OFF(func_builtins), READONLY},
#[pymember(magic)]
fn globals(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult {
let zelf = Self::_zelf(zelf, vm)?;
Ok(zelf.globals.clone().into())
}
#[pyproperty(magic)]
fn closure(&self) -> Option<PyTupleTyped<PyCellRef>> {
self.closure.clone()
#[pymember(magic)]
fn closure(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult {
let zelf = Self::_zelf(zelf, vm)?;
Ok(vm.unwrap_or_none(zelf.closure.clone().map(|x| x.to_pyobject(vm))))
}
#[pyproperty(magic)]
+1
View File
@@ -89,6 +89,7 @@ pub use zip::PyZip;
#[path = "union.rs"]
pub(crate) mod union_;
pub use union_::PyUnion;
pub(crate) mod descriptor;
pub use float::try_to_bigint as try_f64_to_bigint;
pub use int::try_to_float as try_bigint_to_f64;
+97 -12
View File
@@ -2,14 +2,9 @@ use super::{
mappingproxy::PyMappingProxy, object, union_, PyClassMethod, PyDictRef, PyList, PyStaticMethod,
PyStr, PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyWeak,
};
use crate::common::{
ascii,
borrow::BorrowedValue,
lock::{PyRwLock, PyRwLockReadGuard},
};
use crate::{
builtins::function::PyCellRef,
builtins::PyBaseExceptionRef,
builtins::{descriptor::MemberGetter, function::PyCellRef, tuple::PyTupleTyped},
class::{PyClassImpl, StaticType},
convert::ToPyObject,
function::{FuncArgs, KwArgs, OptionalArg, PySetterValue},
@@ -18,6 +13,18 @@ use crate::{
types::{Callable, GetAttr, PyTypeFlags, PyTypeSlots, SetAttr},
AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
};
use crate::{
builtins::{
descriptor::{DescrObject, MemberDef, MemberDescrObject, MemberKind},
tuple::IntoPyTuple,
},
common::{
ascii,
borrow::BorrowedValue,
lock::{PyRwLock, PyRwLockReadGuard},
},
protocol::PyIterReturn,
};
use indexmap::{map::Entry, IndexMap};
use itertools::Itertools;
use std::{borrow::Borrow, collections::HashSet, fmt, ops::Deref, pin::Pin, ptr::NonNull};
@@ -39,6 +46,7 @@ pub struct PyType {
#[derive(Default)]
pub struct HeapTypeExt {
pub number_methods: PyNumberMethods,
pub slots: Option<PyTupleTyped<PyStrRef>>,
}
pub struct PointerSlot<T>(NonNull<T>);
@@ -129,7 +137,15 @@ impl PyType {
slots: PyTypeSlots,
metaclass: PyRef<Self>,
) -> Result<PyRef<Self>, String> {
Self::new_verbose_ref(name, bases[0].clone(), bases, attrs, slots, metaclass)
Self::new_verbose_ref(
name,
bases[0].clone(),
bases,
attrs,
slots,
HeapTypeExt::default(),
metaclass,
)
}
fn new_verbose_ref(
name: &str,
@@ -137,6 +153,7 @@ impl PyType {
bases: Vec<PyRef<Self>>,
attrs: PyAttributes,
mut slots: PyTypeSlots,
heaptype_ext: HeapTypeExt,
metaclass: PyRef<Self>,
) -> Result<PyRef<Self>, String> {
// Check for duplicates in bases.
@@ -167,7 +184,7 @@ impl PyType {
subclasses: PyRwLock::default(),
attributes: PyRwLock::new(attrs),
slots,
heaptype_ext: Some(Pin::new(Box::new(HeapTypeExt::default()))),
heaptype_ext: Some(Pin::new(Box::new(heaptype_ext))),
},
metaclass,
None,
@@ -618,11 +635,79 @@ impl PyType {
// TODO: Flags is currently initialized with HAS_DICT. Should be
// updated when __slots__ are supported (toggling the flag off if
// a class has __slots__ defined).
let flags = PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT;
let slots = PyTypeSlots::from_flags(flags);
let heaptype_slots: Option<PyTupleTyped<PyStrRef>> =
if let Some(x) = attributes.get(identifier!(vm, __slots__)) {
Some(if x.to_owned().class().is(vm.ctx.types.str_type) {
PyTupleTyped::<PyStrRef>::try_from_object(
vm,
vec![x.to_owned()].into_pytuple(vm).into(),
)?
} else {
let iter = x.to_owned().get_iter(vm)?;
let elements = {
let mut elements = Vec::new();
while let PyIterReturn::Return(element) = iter.next(vm)? {
elements.push(element);
}
elements
};
PyTupleTyped::<PyStrRef>::try_from_object(vm, elements.into_pytuple(vm).into())?
})
} else {
None
};
let typ = Self::new_verbose_ref(name.as_str(), base, bases, attributes, slots, metatype)
.map_err(|e| vm.new_type_error(e))?;
let base_member_count = base.slots.member_count;
let member_count: usize =
base.slots.member_count + heaptype_slots.as_ref().map(|x| x.len()).unwrap_or(0);
let flags = PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT;
let heaptype_ext = HeapTypeExt {
slots: heaptype_slots.to_owned(),
..HeapTypeExt::default()
};
let slots = PyTypeSlots {
member_count,
..PyTypeSlots::from_flags(flags)
};
let typ = Self::new_verbose_ref(
name.as_str(),
base,
bases,
attributes,
slots,
heaptype_ext,
metatype,
)
.map_err(|e| vm.new_type_error(e))?;
if let Some(ref slots) = heaptype_slots {
let mut offset = base_member_count;
for member in slots.as_slice() {
let member_def = MemberDef {
name: member.to_string(),
kind: MemberKind::ObjectEx,
getter: MemberGetter::Offset(offset),
doc: None,
};
let member_descriptor: PyRef<MemberDescrObject> = vm.new_pyref(MemberDescrObject {
common: DescrObject {
typ: typ.to_owned(),
name: member.to_string(),
qualname: PyRwLock::new(None),
},
member: member_def,
});
let attr_name = vm.ctx.intern_str(member.to_string());
if !typ.has_attr(attr_name) {
typ.set_attr(attr_name, member_descriptor.into());
}
offset += 1;
}
}
if let Some(cell) = typ.attributes.write().get(identifier!(vm, __classcell__)) {
let cell = PyCellRef::try_from_object(vm, cell.clone()).map_err(|_| {
+9
View File
@@ -115,6 +115,7 @@ struct PyInner<T> {
weak_list: WeakRefList,
payload: T,
slots: Vec<Option<PyObjectRef>>,
}
impl<T: fmt::Debug> fmt::Debug for PyInner<T> {
@@ -426,6 +427,7 @@ impl InstanceDict {
impl<T: PyObjectPayload> PyInner<T> {
fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> Box<Self> {
let member_count = typ.slots.member_count;
Box::new(PyInner {
ref_count: RefCount::new(),
typeid: TypeId::of::<T>(),
@@ -434,6 +436,7 @@ impl<T: PyObjectPayload> PyInner<T> {
dict: dict.map(InstanceDict::new),
weak_list: WeakRefList::new(),
payload,
slots: vec![None; member_count],
})
}
}
@@ -794,6 +797,10 @@ impl PyObject {
pub(crate) fn is_interned(&self) -> bool {
self.0.ref_count.is_leaked()
}
pub(crate) fn get_slot(&self, offset: usize) -> Option<PyObjectRef> {
self.0.slots[offset].clone()
}
}
impl Borrow<PyObject> for PyObjectRef {
@@ -1124,6 +1131,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) {
dict: None,
weak_list: WeakRefList::new(),
payload: type_payload,
slots: Vec::new(),
},
Uninit { typ }
)));
@@ -1135,6 +1143,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) {
dict: None,
weak_list: WeakRefList::new(),
payload: object_payload,
slots: Vec::new(),
},
Uninit { typ },
)));
+3
View File
@@ -81,6 +81,9 @@ pub struct PyTypeSlots {
// tp_subclasses
// tp_weaklist
pub del: AtomicCell<Option<DelFunc>>,
// The count of tp_members.
pub member_count: usize,
}
impl PyTypeSlots {
+7 -3
View File
@@ -1,9 +1,10 @@
use crate::{
builtins::{
asyncgenerator, bool_, builtinfunc, bytearray, bytes, classmethod, code, complex,
coroutine, dict, enumerate, filter, float, frame, function, generator, genericalias,
getset, int, iter, list, map, mappingproxy, memory, module, namespace, object, property,
pystr, range, set, singletons, slice, staticmethod, super_, traceback, tuple,
coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator,
genericalias, getset, int, iter, list, map, mappingproxy, memory, module, namespace,
object, property, pystr, range, set, singletons, slice, staticmethod, super_, traceback,
tuple,
type_::{self, PyType},
union_, weakproxy, weakref, zip,
},
@@ -88,6 +89,7 @@ pub struct TypeZoo {
pub not_implemented_type: &'static Py<PyType>,
pub generic_alias_type: &'static Py<PyType>,
pub union_type: &'static Py<PyType>,
pub member_descriptor_type: &'static Py<PyType>,
}
impl TypeZoo {
@@ -172,6 +174,7 @@ impl TypeZoo {
not_implemented_type: singletons::PyNotImplemented::init_bare_type(),
generic_alias_type: genericalias::PyGenericAlias::init_bare_type(),
union_type: union_::PyUnion::init_bare_type(),
member_descriptor_type: descriptor::MemberDescrObject::init_bare_type(),
}
}
@@ -220,5 +223,6 @@ impl TypeZoo {
traceback::init(context);
genericalias::init(context);
union_::init(context);
descriptor::init(context);
}
}
+33
View File
@@ -3,6 +3,7 @@ use crate::{
builtinfunc::{PyBuiltinFunction, PyBuiltinMethod, PyNativeFuncDef},
bytes,
code::{self, PyCode},
descriptor::{DescrObject, MemberDef, MemberDescrObject, MemberGetter, MemberKind},
getset::PyGetSet,
object, pystr,
type_::PyAttributes,
@@ -17,10 +18,12 @@ use crate::{
intern::{Internable, MaybeInterned, StringPool},
object::{Py, PyObjectPayload, PyObjectRef, PyPayload, PyRef},
types::{PyTypeFlags, PyTypeSlots, TypeZoo},
PyResult, VirtualMachine,
};
use num_bigint::BigInt;
use num_complex::Complex64;
use num_traits::ToPrimitive;
use rustpython_common::lock::PyRwLock;
#[derive(Debug)]
pub struct Context {
@@ -191,6 +194,7 @@ declare_const_name! {
__set_name__,
__setattr__,
__setitem__,
__slots__,
__str__,
__sub__,
__subclasscheck__,
@@ -449,6 +453,35 @@ impl Context {
PyNativeFuncDef::new(f.into_func(), PyStr::new_ref(name, self))
}
#[inline]
pub fn new_member(
&self,
name: &str,
getter: fn(&VirtualMachine, PyObjectRef) -> PyResult,
class: &'static Py<PyType>,
) -> PyRef<MemberDescrObject> {
let member_def = MemberDef {
name: name.to_owned(),
kind: MemberKind::ObjectEx,
getter: MemberGetter::Getter(getter),
doc: None,
};
let member_descriptor = MemberDescrObject {
common: DescrObject {
typ: class.to_owned(),
name: name.to_owned(),
qualname: PyRwLock::new(None),
},
member: member_def,
};
PyRef::new_ref(
member_descriptor,
self.types.member_descriptor_type.to_owned(),
None,
)
}
// #[deprecated]
pub fn new_function<F, FKind>(&self, name: impl Into<PyStr>, f: F) -> PyRef<PyBuiltinFunction>
where