From d422de6f00be78172fd81dad49642d50b13fb7d6 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 16 Aug 2020 01:02:39 +0900 Subject: [PATCH 01/13] Rework derive and rewrite pymodule.rs --- Cargo.lock | 10 + derive/Cargo.toml | 1 + derive/src/pyclass.rs | 182 +++++--- derive/src/pymodule.rs | 799 +++++++++++++++++------------------ derive/src/util.rs | 487 ++++++++++++--------- vm/src/stdlib/collections.rs | 2 + vm/src/stdlib/itertools.rs | 19 + vm/src/stdlib/os.rs | 9 +- vm/src/stdlib/pystruct.rs | 2 + vm/src/stdlib/random.rs | 1 + vm/src/stdlib/symtable.rs | 2 + vm/src/stdlib/zlib.rs | 2 + 12 files changed, 852 insertions(+), 664 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78f26b6d5..95d9ea975 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1340,6 +1340,7 @@ dependencies = [ "rustpython-bytecode", "rustpython-compiler", "syn", + "syn-ext", ] [[package]] @@ -1688,6 +1689,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syn-ext" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba44d62b41442cbed900f2f3a39b3d8240b56bd34fcc574bc684bfd43a1a8c9" +dependencies = [ + "syn", +] + [[package]] name = "term" version = "0.5.2" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 4d7c6d496..8bd5ba717 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -12,6 +12,7 @@ proc-macro = true [dependencies] syn = { version = "1.0", features = ["full"] } +syn-ext = { version = "0.1", features = ["full"] } quote = "1.0" proc-macro2 = "1.0" rustpython-compiler = { path = "../compiler", version = "0.1.1" } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 52ac439e6..c7f44b2da 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,7 +1,7 @@ use super::Diagnostic; use crate::util::{ - attribute_arg, def_to_name, meta_into_nesteds, optional_attribute_arg, path_eq, ItemIdent, - ItemMeta, ItemType, + attribute_arg, def_to_name, optional_attribute_arg, path_eq, AttributeExt, ContentItem, + ContentItemInner, ItemIdent, ItemMeta, ItemMetaInner, }; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, quote_spanned, ToTokens}; @@ -10,6 +10,25 @@ use syn::{ parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Index, Item, Lit, Meta, NestedMeta, }; +use syn_ext::types::*; + +struct MethodItem { + pub inner: ContentItemInner, +} +struct PropertyItem { + pub inner: ContentItemInner, +} + +impl ContentItem for MethodItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} +impl ContentItem for PropertyItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} #[derive(Default)] struct Class { @@ -70,9 +89,11 @@ impl Class { } } - fn extract_method(ident: &Ident, nesteds: Vec) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pymethod", &ident, &nesteds, ItemMeta::ATTRIBUTE_NAMES)?; + fn extract_method( + ident: &Ident, + nested: PunctuatedNestedMeta, + ) -> Result { + let item_meta = MethodItemMeta::from_nested("pymethod", &ident, nested.into_iter())?; Ok(ClassItem::Method { item_ident: ident.clone(), py_name: item_meta.method_name()?, @@ -81,23 +102,20 @@ impl Class { fn extract_classmethod( ident: &Ident, - nesteds: Vec, + nested: PunctuatedNestedMeta, ) -> Result { - let item_meta = ItemMeta::from_nested_meta( - "pyclassmethod", - &ident, - &nesteds, - ItemMeta::ATTRIBUTE_NAMES, - )?; + let item_meta = MethodItemMeta::from_nested("pyclassmethod", &ident, nested.into_iter())?; Ok(ClassItem::ClassMethod { item_ident: ident.clone(), py_name: item_meta.method_name()?, }) } - fn extract_property(ident: &Ident, nesteds: Vec) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyproperty", &ident, &nesteds, ItemMeta::PROPERTY_NAMES)?; + fn extract_property( + ident: &Ident, + nested: PunctuatedNestedMeta, + ) -> Result { + let item_meta = PropertyItemMeta::from_nested("pyproperty", &ident, nested.into_iter())?; Ok(ClassItem::Property { py_name: item_meta.property_name()?, item_ident: ident.clone(), @@ -105,13 +123,12 @@ impl Class { }) } - fn extract_slot(ident: &Ident, meta: Meta) -> Result { + fn extract_slot(ident: &Ident, nested: PunctuatedNestedMeta) -> Result { let pyslot_err = "#[pyslot] must be of the form #[pyslot] or #[pyslot(slotname)]"; - let nesteds = meta_into_nesteds(meta).map_err(|meta| err_span!(meta, "{}", pyslot_err))?; - if nesteds.len() > 1 { - return Err(Diagnostic::spanned_error("e!(#(#nesteds)*), pyslot_err)); + if nested.len() > 1 { + return Err(Diagnostic::spanned_error("e!(#nested), pyslot_err)); } - let slot_ident = if nesteds.is_empty() { + let slot_ident = if nested.is_empty() { let ident_str = ident.to_string(); if let Some(stripped) = ident_str.strip_prefix("tp_") { proc_macro2::Ident::new(stripped, ident.span()) @@ -119,7 +136,7 @@ impl Class { ident.clone() } } else { - match nesteds.into_iter().next().unwrap() { + match nested.into_iter().next().unwrap() { NestedMeta::Meta(Meta::Path(path)) => path .get_ident() .cloned() @@ -135,39 +152,22 @@ impl Class { fn extract_item_from_syn(&mut self, item: &mut ItemIdent) -> Result<(), Diagnostic> { let mut attr_idxs = Vec::new(); - for (i, meta) in item - .attrs - .iter() - .filter_map(|attr| attr.parse_meta().ok()) - .enumerate() - { - let meta_span = meta.span(); - let name = match meta.path().get_ident() { + for (i, attr) in item.attrs.iter_mut().enumerate() { + let name = match attr.path.get_ident() { Some(name) => name, None => continue, }; - assert!(item.typ == ItemType::Method); - let into_nested = || { - meta_into_nesteds(meta.clone()).map_err(|meta| { - err_span!( - meta, - "#[{name} = \"...\"] cannot be a name/value, you probably meant \ - #[{name}(name = \"...\")]", - name = name.to_string(), - ) - }) - }; let item = match name.to_string().as_str() { - "pymethod" => Self::extract_method(item.ident, into_nested()?)?, - "pyclassmethod" => Self::extract_classmethod(item.ident, into_nested()?)?, - "pyproperty" => Self::extract_property(item.ident, into_nested()?)?, - "pyslot" => Self::extract_slot(item.ident, meta)?, + "pymethod" => Self::extract_method(item.ident, attr.promoted_nested()?)?, + "pyclassmethod" => Self::extract_classmethod(item.ident, attr.promoted_nested()?)?, + "pyproperty" => Self::extract_property(item.ident, attr.promoted_nested()?)?, + "pyslot" => Self::extract_slot(item.ident, attr.promoted_nested()?)?, _ => { continue; } }; - self.add_item(item, meta_span)?; + self.add_item(item, attr.span())?; attr_idxs.push(i); } let mut i = 0; @@ -187,6 +187,96 @@ impl Class { } } +struct MethodItemMeta(ItemMetaInner); + +impl ItemMeta for MethodItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl MethodItemMeta { + fn method_name(&self) -> Result { + let inner = self.inner(); + let name = inner._optional_str("name")?; + let magic = inner._bool("magic")?; + Ok(if let Some(name) = name { + name + } else { + let name = inner.ident.to_string(); + if magic { + format!("__{}__", name) + } else { + name + } + }) + } +} + +struct PropertyItemMeta(ItemMetaInner); + +impl ItemMeta for PropertyItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic", "setter"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl PropertyItemMeta { + fn property_name(&self) -> Result { + let inner = self.inner(); + let magic = inner._bool("magic")?; + let setter = inner._bool("setter")?; + let name = inner._optional_str("name")?; + + Ok(if let Some(name) = name { + name + } else { + let sig_name = inner.ident.to_string(); + let name = if setter { + if let Some(name) = sig_name.strip_prefix("set_") { + if name.is_empty() { + bail_span!( + &inner.ident, + "A #[{}(setter)] fn with a set_* name must \ + have something after \"set_\"", + inner.parent_type + ) + } + name.to_string() + } else { + bail_span!( + &inner.ident, + "A #[{}(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of \"set_*\"", + inner.parent_type + ) + } + } else { + sig_name + }; + if magic { + format!("__{}__", name) + } else { + name + } + }) + } + + fn setter(&self) -> syn::Result { + self.inner()._bool("setter") + } +} + fn extract_impl_items(mut items: Vec) -> Result { let mut diagnostics: Vec = Vec::new(); @@ -365,7 +455,6 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> Result { Some(ItemIdent { - typ: ItemType::Method, attrs, ident: &sig.ident, }) @@ -399,7 +488,6 @@ pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> Result { Some(ItemIdent { - typ: ItemType::Method, attrs, ident: &sig.ident, }) diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index aed918c57..ddfe35b5b 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -1,443 +1,63 @@ -use super::Diagnostic; +use crate::error::Diagnostic; use crate::util::{ - def_to_name, meta_into_nesteds, optional_attribute_arg, ItemIdent, ItemMeta, ItemType, + AttributeExt, ContentItem, ContentItemInner, ItemMeta, ItemMetaInner, ItemNursery, + SimpleItemMeta, ALL_ALLOWED_NAMES, }; -use proc_macro2::{Span, TokenStream as TokenStream2}; +use proc_macro2::TokenStream; use quote::{quote, quote_spanned, ToTokens}; -use std::collections::HashMap; -use syn::{parse_quote, spanned::Spanned, AttributeArgs, Ident, Item, Meta, NestedMeta, UseTree}; +use syn::{parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Result, UseTree}; +use syn_ext::ext::*; struct Module { name: String, - items: HashMap<(String, Vec), ModuleItem>, + items: ItemNursery, } -#[derive(PartialEq, Eq, Hash)] -enum ModuleItem { - Function { - item_ident: Ident, - py_name: String, - }, - EvaluatedAttr { - item_ident: Ident, - py_name: String, - }, - ConstAttr { - item_ident: Ident, - py_name: String, - }, - ClassAttr { - item_ident: Ident, - py_name: Option, - }, - Class { - item_ident: Ident, - py_name: String, - }, -} - -impl ModuleItem { - fn name(&self) -> String { - use ModuleItem::*; - match self { - Function { py_name, .. } => py_name.clone(), - EvaluatedAttr { py_name, .. } => py_name.clone(), - ConstAttr { py_name, .. } => py_name.clone(), - ClassAttr { py_name, .. } => py_name.clone().unwrap_or_else(String::new), - Class { py_name, .. } => py_name.clone(), - } - } -} - -impl Module { - fn add_item( - &mut self, - item: ModuleItem, - cfgs: Vec, - span: Span, - ) -> Result<(), Diagnostic> { - if let Some(existing) = self.items.insert((item.name(), cfgs), item) { - Err(Diagnostic::span_error( - span, - format!( - "Duplicate #[py*] attribute on pymodule: {}", - existing.name() - ), - )) - } else { - Ok(()) - } - } - - fn extract_function(ident: &Ident, nesteds: Vec) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyfunction", &ident, &nesteds, ItemMeta::SIMPLE_NAMES)?; - Ok(ModuleItem::Function { - item_ident: ident.clone(), - py_name: item_meta.simple_name()?, - }) - } - - fn extract_class(ident: &Ident, nesteds: Vec) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyclass", &ident, &nesteds, ItemMeta::SIMPLE_NAMES)?; - Ok(ModuleItem::Class { - item_ident: ident.clone(), - py_name: item_meta.simple_name()?, - }) - } - - fn extract_struct_sequence( - ident: &Ident, - nesteds: Vec, - ) -> Result { - let item_meta = ItemMeta::from_nested_meta( - "pystruct_sequence", - &ident, - &nesteds, - ItemMeta::STRUCT_SEQUENCE_NAMES, - )?; - Ok(ModuleItem::Class { - item_ident: ident.clone(), - py_name: item_meta.simple_name()?, - }) - } - - fn extract_evaluated_attr( - ident: &Ident, - nesteds: Vec, - ) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyattr", &ident, &nesteds, ItemMeta::SIMPLE_NAMES)?; - Ok(ModuleItem::EvaluatedAttr { - item_ident: ident.clone(), - py_name: item_meta.simple_name()?, - }) - } - - fn extract_const_attr( - ident: &Ident, - nesteds: Vec, - ) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyattr", &ident, &nesteds, ItemMeta::SIMPLE_NAMES)?; - Ok(ModuleItem::ConstAttr { - item_ident: ident.clone(), - py_name: item_meta.simple_name()?, - }) - } - - fn extract_class_attr( - ident: &Ident, - nesteds: Vec, - ) -> Result { - let item_meta = - ItemMeta::from_nested_meta("pyattr", &ident, &nesteds, ItemMeta::SIMPLE_NAMES)?; - Ok(ModuleItem::ClassAttr { - item_ident: ident.clone(), - py_name: item_meta.optional_name(), - }) - } - - fn extract_item_from_syn(&mut self, item: &mut ItemIdent) -> Result<(), Diagnostic> { - let mut attr_idxs = Vec::new(); - let mut items = Vec::new(); - let mut cfgs = Vec::new(); - let mut has_class = false; - for (i, attr) in item.attrs.iter_mut().enumerate() { - let meta = attr.parse_meta()?; - let meta_span = meta.span(); - let name = match meta.path().get_ident() { - Some(name) => name.clone(), - None => continue, - }; - - let into_nested = || { - meta_into_nesteds(meta.clone()).map_err(|meta| { - err_span!( - meta, - "#[{name} = \"...\"] cannot be a name/value, you probably meant \ - #[{name}(name = \"...\")]", - name = name.to_string(), - ) - }) - }; - match name.to_string().as_str() { - "pyfunction" => { - assert!(item.typ == ItemType::Fn); - attr_idxs.push(i); - items.push(( - Self::extract_function(item.ident, into_nested()?)?, - meta_span, - )); - } - "pyattr" => { - attr_idxs.push(i); - let item = match item.typ { - ItemType::Fn => Self::extract_evaluated_attr(item.ident, into_nested()?)?, - ItemType::Const => Self::extract_const_attr(item.ident, into_nested()?)?, - ItemType::Struct | ItemType::Enum => { - if has_class { - return Err(err_span!( - meta, - "#[pyattr] must be placed on top of #[pyclass] or #[pystruct_sequence]", - )); - } - Self::extract_class_attr(item.ident, into_nested()?)? - } - ItemType::Use => Self::extract_const_attr(item.ident, into_nested()?)?, - ItemType::Method => unreachable!(), - }; - items.push((item, meta_span)); - } - attr_name @ "pyclass" | attr_name @ "pystruct_sequence" => { - assert!(item.typ == ItemType::Struct); - - let nested_or = into_nested(); - let mut metalist = match &meta { - Meta::Path(path) => parse_quote!(#path()), - Meta::List(metalist) => metalist.clone(), - _ => unreachable!(), - }; - - if let Ok(nested) = &nested_or { - if optional_attribute_arg("py..", "module", &nested)?.is_none() { - let module_name = &self.name; - metalist.nested.push(parse_quote! {module = #module_name}); - *attr = parse_quote!(#[#metalist]); - } - } - - if has_class { - Err(err_span!( - meta, - "#[{}] in module cannot be duplicated", - attr_name, - )) - } else if !items - .iter() - .all(|(item, _span)| matches!(item, ModuleItem::ClassAttr{..})) - { - Err(err_span!( - meta, - "#[{}] is allowed to have only #[pyattr] as siblings", - attr_name, - )) - } else { - has_class = true; - Ok(()) - }?; - - let class = match attr_name { - "pyclass" => Self::extract_class(item.ident, nested_or?)?, - "pystruct_sequence" => { - // TODO: validate pystruct_sequence doesn't have module - Self::extract_struct_sequence(item.ident, nested_or?)? - } - _ => unreachable!(), - }; - - if items.is_empty() { - items.push((class, meta_span)); - } else { - for (attr, _) in items.iter_mut() { - match attr { - ModuleItem::ClassAttr { py_name, .. } => { - if py_name.is_none() { - *py_name = Some(class.name()); - } - } - _ => unreachable!(), - } - } - } - } - "cfg" => { - cfgs.push(meta); - continue; - } - _ => { - continue; - } - }; - } - for (item, meta) in items { - self.add_item(item, cfgs.clone(), meta)?; - } - let mut i = 0; - let mut attr_idxs = &*attr_idxs; - item.attrs.retain(|_| { - let drop = attr_idxs.first().copied() == Some(i); - if drop { - attr_idxs = &attr_idxs[1..]; - } - i += 1; - !drop - }); - for (i, idx) in attr_idxs.iter().enumerate() { - item.attrs.remove(idx - i); - } - // TODO: Add module to pystruct_sequence - Ok(()) - } -} - -fn extract_module_items( - mut items: Vec, - module_name: &str, -) -> Result { - let mut diagnostics: Vec = Vec::new(); - - let mut module = Module { - name: module_name.to_owned(), - items: Default::default(), - }; - - for item in items.iter_mut() { - push_diag_result!(diagnostics, module.extract_item_from_syn(item),); - } - - let items = module - .items - .into_iter() - .map(|((_name, cfgs), item)| match item { - ModuleItem::Function { - item_ident, - py_name, - } => { - let new_func = quote_spanned!( - item_ident.span() => - vm.ctx.new_function_named(#item_ident, #module_name.to_owned(), #py_name.to_owned())); - quote! { - #( #[ #cfgs ])* - vm.__module_set_attr(&module, #py_name, #new_func).unwrap(); - } - } - ModuleItem::EvaluatedAttr { - item_ident, - py_name, - } => { - let new_attr = quote_spanned!( - item_ident.span() => - vm.new_pyobj(#item_ident(vm))); - quote! { - #( #[ #cfgs ])* - vm.__module_set_attr(&module, #py_name, #new_attr).unwrap(); - } - } - ModuleItem::ConstAttr { - item_ident, - py_name, - } => { - let new_attr = quote_spanned!( - item_ident.span() => - vm.new_pyobj(#item_ident)); - quote! { - #( #[ #cfgs ])* - vm.__module_set_attr(&module, #py_name, #new_attr).unwrap(); - } - } - class => { - let (item_ident, py_name) = match class { - ModuleItem::ClassAttr { - item_ident, - py_name: Some(py_name), - } => (item_ident, py_name), - ModuleItem::Class { - item_ident, - py_name, - } => (item_ident, py_name), - _ => unreachable!() - }; - let new_class = quote_spanned!( - item_ident.span() => - #item_ident::make_class(&vm.ctx)); - quote! { - #( #[ #cfgs ])* - vm.__module_set_attr(&module, #py_name, #new_class).unwrap(); - } - } - - }); - - Diagnostic::from_vec(diagnostics)?; - - Ok(quote! { - #(#items)* - }) -} - -pub fn impl_pymodule(attr: AttributeArgs, item: Item) -> Result { - let mut module = match item { +pub fn impl_pymodule( + attr: AttributeArgs, + module_item: Item, +) -> std::result::Result { + let mut module_item = match module_item { Item::Mod(m) => m, other => bail_span!(other, "#[pymodule] can only be on a module declaration"), }; - let module_name = def_to_name("pymodule", &module.ident, &attr)?; - - let (_, content) = match module.content.as_mut() { - Some(c) => c, - None => bail_span!( - module, - "#[pymodule] can only be on a module declaration with body" - ), + let module_meta = + SimpleItemMeta::from_nested("pymodule", &module_item.ident, attr.into_iter())?; + let mut module_context = Module { + name: module_meta.simple_name()?, + items: ItemNursery::default(), }; + let content = module_item.unbraced_content_mut()?; - let items = content - .iter_mut() - .filter_map(|item| { - let item = match item { - Item::Fn(syn::ItemFn { attrs, sig, .. }) => ItemIdent { - typ: ItemType::Fn, - attrs, - ident: &sig.ident, - }, - Item::Struct(syn::ItemStruct { attrs, ident, .. }) => ItemIdent { - typ: ItemType::Struct, - attrs, - ident, - }, - Item::Enum(syn::ItemEnum { attrs, ident, .. }) => ItemIdent { - typ: ItemType::Enum, - attrs, - ident, - }, - Item::Const(syn::ItemConst { attrs, ident, .. }) => ItemIdent { - typ: ItemType::Const, - attrs, - ident, - }, - Item::Use(syn::ItemUse { attrs, tree, .. }) => { - let ident = match tree { - UseTree::Path(path) => match &*path.tree { - UseTree::Name(name) => &name.ident, - UseTree::Rename(rename) => &rename.rename, - _ => return None, - }, - _ => return None, - }; - ItemIdent { - typ: ItemType::Use, - attrs, - ident, - } - } - _ => return None, - }; - Some(item) - }) - .collect(); + for item in content.iter_mut() { + let (attrs, ident) = if let Some(v) = item_declaration(item) { + v + } else { + continue; + }; + let (pyitems, cfgs) = attrs_to_items(attrs, new_item)?; + for pyitem in pyitems.iter().rev() { + pyitem.gen_module_item(ModuleItemArgs { + ident: ident.clone(), + item, + module: &mut module_context, + cfgs: cfgs.as_slice(), + })?; + } + } - let extend_mod = extract_module_items(items, &module_name)?; + let module_name = module_context.name.as_str(); + let items = module_context.items; content.extend(vec![ parse_quote! { - pub(crate) const MODULE_NAME: &str = #module_name; + pub(crate) const MODULE_NAME: &'static str = #module_name; }, parse_quote! { pub(crate) fn extend_module( vm: &::rustpython_vm::vm::VirtualMachine, module: &::rustpython_vm::pyobject::PyObjectRef, ) { - #extend_mod + #items } }, parse_quote! { @@ -452,5 +72,346 @@ pub fn impl_pymodule(attr: AttributeArgs, item: Item) -> Result Option<(&[syn::Attribute], Ident)> { + let (attrs, ident) = match &item { + Item::Fn(i @ syn::ItemFn { .. }) => (i.attrs.as_slice(), &i.sig.ident), + Item::Struct(syn::ItemStruct { attrs, ident, .. }) => (attrs.as_slice(), ident), + Item::Enum(syn::ItemEnum { attrs, ident, .. }) => (attrs.as_slice(), ident), + Item::Const(syn::ItemConst { attrs, ident, .. }) => (attrs.as_slice(), ident), + Item::Use(syn::ItemUse { attrs, tree, .. }) => { + let ident = match tree { + UseTree::Path(path) => match &*path.tree { + UseTree::Name(name) => &name.ident, + UseTree::Rename(rename) => &rename.rename, + _ => return None, + }, + _ => return None, + }; + (attrs.as_slice(), ident) + } + _ => return None, + }; + Some((attrs, ident.clone())) +} + +fn new_item(index: usize, attr_name: String, pyattrs: Option>) -> Box { + assert!(ALL_ALLOWED_NAMES.contains(&attr_name.as_str())); + match attr_name.as_str() { + "pyfunction" => Box::new(FunctionItem { + inner: ContentItemInner { index, attr_name }, + }), + "pyattr" => Box::new(AttributeItem { + inner: ContentItemInner { index, attr_name }, + }), + "pystruct_sequence" | "pyclass" => Box::new(ClassItem { + inner: ContentItemInner { index, attr_name }, + pyattrs: pyattrs.unwrap_or_else(Vec::new), + }), + other => unreachable!("#[pymodule] doesn't accept #[{}]", other), + } +} + +fn attrs_to_items(attrs: &[Attribute], new_item: F) -> Result<(Vec, Vec)> +where + F: Fn(usize, String, Option>) -> R, +{ + let mut cfgs: Vec = Vec::new(); + let mut result = Vec::new(); + + let mut iter = attrs.iter().enumerate().peekable(); + while let Some((_, attr)) = iter.peek() { + // take all cfgs but no py items + let attr = *attr; + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + cfgs.push(attr.clone()); + } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + break; + } + iter.next(); + } + + let mut closed = false; + let mut pyattrs = Vec::new(); + for (i, attr) in iter { + // take py items but no cfgs + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + return Err(syn::Error::new_spanned( + attr, + "#[py*] items must be placed under `cfgs`", + )); + } + if !ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + continue; + } else if closed { + return Err(syn::Error::new_spanned( + attr, + "Only one #[pyattr] annotated #[py*] item can exist", + )); + } + + if attr_name == "pyattr" { + if !result.is_empty() { + return Err(syn::Error::new_spanned( + attr, + "#[pyattr] must be placed on top of other #[py*] items", + )); + } + pyattrs.push((i, attr_name)); + continue; + } + + if pyattrs.is_empty() { + result.push(new_item(i, attr_name, None)); + } else { + if !["pyclass", "pystruct_sequence"].contains(&attr_name.as_str()) { + return Err(syn::Error::new_spanned( + attr, + "#[pyattr] #[pyclass] is the only supported composition", + )); + } + let pyattr_indexes = pyattrs.iter().map(|(i, _)| i).copied().collect(); + result.push(new_item(i, attr_name, Some(pyattr_indexes))); + pyattrs = Vec::new(); + closed = true; + } + } + for (index, attr_name) in pyattrs { + assert!(!closed); + result.push(new_item(index, attr_name, None)); + } + Ok((result, cfgs)) +} + +struct ClassItemMeta(ItemMetaInner); + +impl ItemMeta for ClassItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["module", "name"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +/// #[pyfunction] +struct FunctionItem { + inner: ContentItemInner, +} + +/// #[pyclass] or #[pystruct_sequence] +struct ClassItem { + inner: ContentItemInner, + pyattrs: Vec, +} + +/// #[pyattr] +struct AttributeItem { + inner: ContentItemInner, +} + +impl ContentItem for FunctionItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} + +impl ContentItem for ClassItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} + +impl ContentItem for AttributeItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} + +struct ModuleItemArgs<'a> { + ident: Ident, + item: &'a mut Item, + module: &'a mut Module, + cfgs: &'a [Attribute], +} + +impl<'a> ModuleItemArgs<'a> { + fn module_name(&'a self) -> &'a str { + self.module.name.as_str() + } + fn with_quote_args(&self, f: F) -> TokenStream + where + F: Fn(&Ident, &str) -> TokenStream, + { + f(&self.ident, self.module_name()) + } +} + +trait ModuleItem: ContentItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()>; +} + +impl ModuleItem for FunctionItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let attrs = match args.item { + Item::Fn(syn::ItemFn { ref mut attrs, .. }) => attrs, + other => return Err(self.new_syn_error(other.span(), "can only be on a function")), + }; + let item_attr = attrs.remove(self.index()); + let item_meta = SimpleItemMeta::from_nested( + "pyfunction", + &args.ident, + item_attr.promoted_nested()?.into_iter(), + )?; + + let py_name = item_meta.simple_name()?; + let item = args.with_quote_args(|ident, module| { + let new_func = quote_spanned!( + args.ident.span() => vm.ctx.new_function_named(#ident, #module.to_owned(), #py_name.to_owned()) + ); + quote! { + vm.__module_set_attr(&module, #py_name, #new_func).unwrap(); + } + }); + + args.module + .items + .add_item(py_name, args.cfgs.to_vec(), item)?; + Ok(()) + } +} + +impl ModuleItem for ClassItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let attrs = match args.item { + Item::Struct(syn::ItemStruct { ref mut attrs, .. }) => attrs, + Item::Enum(syn::ItemEnum { ref mut attrs, .. }) => attrs, + other => { + return Err( + self.new_syn_error(other.span(), "can only be on a struct or enum declaration") + ); + } + }; + let (module_name, class_name) = { + let class_attr = &mut attrs[self.inner.index]; + if self.pyattrs.is_empty() { + // check noattr before ClassItemMeta::from_nested + let noattr = class_attr.try_remove_name("noattr")?; + if noattr.is_none() { + return Err(syn::Error::new_spanned( + class_attr, + format!( + "#[{name}] requires #[pyattr] to be a module attribute. \ + To keep it free type, try #[{name}(noattr)]", + name = self.attr_name() + ), + )); + } + } + let class_meta = ClassItemMeta::from_nested( + "pyclass/pystruct_sequence", + &args.ident, + class_attr.promoted_nested()?.into_iter(), + )?; + let module_name = args.module.name.clone(); + class_attr.fill_nested_meta("module", || { + parse_quote! {module = #module_name} + })?; + let class_name = class_meta.simple_name()?; + (module_name, class_name) + }; + for attr_index in self.pyattrs.iter().rev() { + let attr_attr = attrs.remove(*attr_index); + let nested = attr_attr.promoted_nested()?; + let item_meta = SimpleItemMeta::from_nested("pyattr", &args.ident, nested.into_iter())?; + + let py_name = item_meta + .optional_name() + .unwrap_or_else(|| class_name.clone()); + let ident = &args.ident; + let new_class = quote_spanned!(ident.span() => + #ident::make_class(&vm.ctx); + ); + let item = quote! { + let new_class = #new_class; + new_class.set_str_attr("__module__", vm.ctx.new_str(#module_name)); + vm.__module_set_attr(&module, #py_name, new_class).unwrap(); + }; + + args.module + .items + .add_item(py_name.clone(), args.cfgs.to_vec(), item)?; + } + Ok(()) + } +} + +impl ModuleItem for AttributeItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let ident = &args.ident; + let get_py_name = |attrs: &mut Vec| -> Result<_> { + let nested = attrs[self.inner.index].promoted_nested()?; + let item_meta = SimpleItemMeta::from_nested("pyattr", &ident, nested.into_iter())?; + let py_name = item_meta.simple_name()?; + Ok(py_name) + }; + let (attrs, py_name, tokens) = match args.item { + Item::Fn(syn::ItemFn { ref mut attrs, .. }) => { + let py_name = get_py_name(attrs)?; + ( + attrs, + py_name.clone(), + quote! { + vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident(vm))).unwrap(); + }, + ) + } + Item::Const(syn::ItemConst { ref mut attrs, .. }) => { + let py_name = get_py_name(attrs)?; + ( + attrs, + py_name.clone(), + quote! { + vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident)).unwrap(); + }, + ) + } + Item::Use(syn::ItemUse { ref mut attrs, .. }) => { + let py_name = get_py_name(attrs)?; + ( + attrs, + py_name.clone(), + quote! { + vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident)).unwrap(); + }, + ) + } + other => { + return Err( + self.new_syn_error(other.span(), "can only be on a function, const and use") + ) + } + }; + attrs.remove(self.index()); + + args.module + .items + .add_item(py_name, args.cfgs.to_vec(), tokens)?; + + Ok(()) + } } diff --git a/derive/src/util.rs b/derive/src/util.rs index 0bf95c9b9..d1343cf5c 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -1,6 +1,209 @@ use super::Diagnostic; +use proc_macro2::{Span, TokenStream}; +use quote::{quote, ToTokens}; use std::collections::HashMap; -use syn::{Attribute, Ident, Lit, Meta, NestedMeta, Path}; +use syn::{spanned::Spanned, Attribute, Ident, Lit, Meta, MetaList, NestedMeta, Path, Result}; +use syn_ext::ext::{AttributeExt as SynAttributeExt, *}; +use syn_ext::types::PunctuatedNestedMeta; + +pub(crate) const ALL_ALLOWED_NAMES: &[&str] = &[ + "pymethod", + "pyproperty", + "pyfunction", + "pyclass", + "pystruct_sequence", + "pyattr", +]; + +#[derive(Default)] +pub(crate) struct ItemNursery(HashMap<(String, Vec), TokenStream>); + +impl ItemNursery { + pub fn add_item( + &mut self, + name: String, + cfgs: Vec, + tokens: TokenStream, + ) -> Result<()> { + if let Some(existing) = self.0.insert((name, cfgs), tokens) { + Err(syn::Error::new_spanned( + existing, + "Duplicated #[py*] attribute found", + )) + } else { + Ok(()) + } + } +} + +impl ToTokens for ItemNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(self.0.iter().map(|((_, cfgs), item)| { + quote! { + #( #cfgs )* + { + #item + } + } + })) + } +} + +#[derive(Clone)] +pub(crate) struct ContentItemInner { + pub index: usize, + pub attr_name: String, +} + +pub(crate) trait ContentItem { + fn inner(&self) -> &ContentItemInner; + fn index(&self) -> usize { + self.inner().index + } + fn attr_name(&self) -> &str { + self.inner().attr_name.as_str() + } + fn new_syn_error(&self, span: Span, message: &str) -> syn::Error { + syn::Error::new(span, format!("#[{}] {}", self.attr_name(), message)) + } +} + +pub(crate) struct ItemMetaInner { + pub ident: Ident, + pub parent_type: &'static str, + pub meta: HashMap, +} + +impl ItemMetaInner { + pub fn from_nested( + parent_type: &'static str, + ident: &Ident, + nested: I, + allowed_names: &[&'static str], + ) -> Result + where + I: std::iter::Iterator, + { + let (named_map, lits) = nested.into_unique_map_and_lits(|path| { + if let Some(ident) = path.get_ident() { + let name = ident.to_string(); + if allowed_names.contains(&name.as_str()) { + Ok(Some(name)) + } else { + Err(syn::Error::new_spanned( + ident, + format!( + "#[{}({})] is not one of allowed attributes {}", + parent_type, + name, + allowed_names.join(", ") + ), + )) + } + } else { + Ok(None) + } + })?; + if !lits.is_empty() { + return Err(syn::Error::new_spanned( + ident, + format!("#[{}(..)] cannot contain literal", parent_type), + )); + } + + Ok(Self { + ident: ident.clone(), + parent_type, + meta: named_map, + }) + } + + pub fn _optional_str(&self, key: &str) -> Result> { + let value = if let Some((_, meta)) = self.meta.get(key) { + match meta { + Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Str(lit), + .. + }) => Some(lit.value()), + other => { + return Err(syn::Error::new_spanned( + other, + format!( + "#[{}({} = ...)] must exist as a string", + self.parent_type, key + ), + )); + } + } + } else { + None + }; + Ok(value) + } + + pub fn _bool(&self, key: &str) -> Result { + let value = if let Some((_, meta)) = self.meta.get(key) { + match meta { + Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Bool(lit), + .. + }) => lit.value, + Meta::Path(_) => true, + other => { + return Err(syn::Error::new_spanned( + other, + format!("#[{}({})] is expected", self.parent_type, key), + )) + } + } + } else { + false + }; + Ok(value) + } +} + +pub(crate) trait ItemMeta: Sized { + const ALLOWED_NAMES: &'static [&'static str]; + + fn from_nested(parent_type: &'static str, ident: &Ident, nested: I) -> Result + where + I: std::iter::Iterator, + { + Ok(Self::from_inner(ItemMetaInner::from_nested( + parent_type, + ident, + nested, + Self::ALLOWED_NAMES, + )?)) + } + + fn from_inner(inner: ItemMetaInner) -> Self; + fn inner(&self) -> &ItemMetaInner; + + fn simple_name(&self) -> Result { + let inner = self.inner(); + Ok(inner + ._optional_str("name")? + .unwrap_or_else(|| inner.ident.to_string())) + } + + fn optional_name(&self) -> Option { + self.inner()._optional_str("name").ok().flatten() + } +} +pub(crate) struct SimpleItemMeta(pub ItemMetaInner); + +impl ItemMeta for SimpleItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} pub(crate) fn path_eq(path: &Path, s: &str) -> bool { path.get_ident().map_or(false, |id| id == s) @@ -10,7 +213,7 @@ pub(crate) fn def_to_name( attr_name: &'static str, ident: &Ident, attrs: &[NestedMeta], -) -> Result { +) -> std::result::Result { optional_attribute_arg(attr_name, "name", attrs) .transpose() .unwrap_or_else(|| Ok(ident.to_string())) @@ -20,7 +223,7 @@ pub(crate) fn attribute_arg( attr_name: &'static str, arg_name: &'static str, attrs: &[NestedMeta], -) -> Result { +) -> std::result::Result { if let Some(r) = optional_attribute_arg(attr_name, arg_name, attrs).transpose() { r } else { @@ -33,11 +236,14 @@ pub(crate) fn attribute_arg( } } -pub(crate) fn optional_attribute_arg( +pub(crate) fn optional_attribute_arg<'a, I>( attr_name: &'static str, arg_name: &'static str, - attrs: &[NestedMeta], -) -> Result, Diagnostic> { + attrs: I, +) -> std::result::Result, Diagnostic> +where + I: std::iter::IntoIterator, +{ let mut arg_value = None; for attr in attrs { match attr { @@ -69,201 +275,90 @@ pub(crate) fn optional_attribute_arg( Ok(arg_value) } -pub(crate) fn meta_into_nesteds(meta: Meta) -> Result, Meta> { - match meta { - Meta::Path(_) => Ok(Vec::new()), - Meta::List(list) => Ok(list.nested.into_iter().collect()), - Meta::NameValue(_) => Err(meta), - } +pub(crate) trait AttributeExt: SynAttributeExt { + fn promoted_nested(&self) -> Result; + fn try_remove_name(&mut self, name: &str) -> Result>; + fn fill_nested_meta(&mut self, name: &str, new_item: F) -> Result<()> + where + F: Fn() -> NestedMeta; } -#[derive(PartialEq)] -pub(crate) enum ItemType { - Fn, - Method, - Struct, - Enum, - Const, - Use, +impl AttributeExt for Attribute { + fn promoted_nested(&self) -> Result { + let list = self.promoted_list().map_err(|mut e| { + let name = self.get_ident().unwrap().to_string(); + e.combine(syn::Error::new_spanned( + self, + format!( + "#[{name} = \"...\"] cannot be a name/value, you probably meant \ + #[{name}(name = \"...\")]", + name = name, + ), + )); + e + })?; + Ok(list.nested) + } + + fn try_remove_name(&mut self, item_name: &str) -> Result> { + self.try_meta_mut(|meta| { + let nested = match meta { + Meta::List(MetaList { ref mut nested, .. }) => Ok(nested), + other => Err(syn::Error::new( + other.span(), + format!( + "#[{name}(...)] doesn't contain '{item}' to remove", + name = other.get_ident().unwrap().to_string(), + item = item_name + ), + )), + }?; + + let mut found = None; + for (i, item) in nested.iter().enumerate() { + let ident = if let Some(ident) = item.get_ident() { + ident + } else { + continue; + }; + if *ident != item_name { + continue; + } + if found.is_some() { + return Err(syn::Error::new( + item.span(), + format!( + "#[py..({}...)] must be unique but found multiple times", + item_name, + ), + )); + } + found = Some(i); + } + + Ok(found.map(|idx| nested.remove(idx).into_value())) + }) + } + + fn fill_nested_meta(&mut self, name: &str, new_item: F) -> Result<()> + where + F: Fn() -> NestedMeta, + { + self.try_meta_mut(|meta| { + let list = meta.promote_to_list(Default::default())?; + let has_name = list + .nested + .iter() + .any(|nmeta| nmeta.get_path().map_or(false, |p| path_eq(p, name))); + if !has_name { + list.nested.push(new_item()) + } + Ok(()) + }) + } } pub(crate) struct ItemIdent<'a> { - pub typ: ItemType, pub attrs: &'a mut Vec, pub ident: &'a Ident, } - -pub(crate) struct ItemMeta<'a> { - ident: &'a Ident, - parent_type: &'static str, - meta: HashMap>, -} - -impl<'a> ItemMeta<'a> { - pub const SIMPLE_NAMES: &'static [&'static str] = &["name"]; - pub const STRUCT_SEQUENCE_NAMES: &'static [&'static str] = &["module", "name"]; - pub const ATTRIBUTE_NAMES: &'static [&'static str] = &["name", "magic"]; - pub const PROPERTY_NAMES: &'static [&'static str] = &["name", "magic", "setter"]; - - pub fn from_nested_meta( - parent_type: &'static str, - ident: &'a Ident, - nested_meta: &[NestedMeta], - names: &[&'static str], - ) -> Result { - let mut extracted = Self { - ident, - parent_type, - meta: HashMap::new(), - }; - - let validate_name = |name: &str, extracted: &Self| -> Result<(), Diagnostic> { - if names.contains(&name) { - if extracted.meta.contains_key(name) { - bail_span!(ident, "#[{}] must have only one '{}'", parent_type, name); - } else { - Ok(()) - } - } else { - bail_span!( - ident, - "#[{}({})] is not one of allowed attributes {}", - parent_type, - name, - names.join(", ") - ); - } - }; - - for meta in nested_meta { - let meta = match meta { - NestedMeta::Meta(meta) => meta, - NestedMeta::Lit(_) => continue, - }; - - match meta { - Meta::NameValue(name_value) => { - if let Some(ident) = name_value.path.get_ident() { - let name = ident.to_string(); - validate_name(&name, &extracted)?; - extracted.meta.insert(name, Some(name_value.lit.clone())); - } - } - Meta::Path(path) => { - if let Some(ident) = path.get_ident() { - let name = ident.to_string(); - validate_name(&name, &extracted)?; - extracted.meta.insert(name, None); - } else { - continue; - } - } - _ => (), - } - } - - Ok(extracted) - } - - fn _str(&self, key: &str) -> Result, Diagnostic> { - Ok(match self.meta.get(key) { - Some(Some(lit)) => { - if let Lit::Str(s) = lit { - Some(s.value()) - } else { - bail_span!( - &self.ident, - "#[{}({} = ...)] must be a string", - self.parent_type, - key - ); - } - } - Some(None) => { - bail_span!( - &self.ident, - "#[{}({} = ...)] is expected", - self.parent_type, - key, - ); - } - None => None, - }) - } - - fn _bool(&self, key: &str) -> Result { - Ok(match self.meta.get(key) { - Some(Some(_)) => { - bail_span!(&self.ident, "#[{}({})] is expected", self.parent_type, key,); - } - Some(None) => true, - None => false, - }) - } - - pub fn simple_name(&self) -> Result { - Ok(self._str("name")?.unwrap_or_else(|| self.ident.to_string())) - } - - pub fn optional_name(&self) -> Option { - self.simple_name().ok() - } - - pub fn method_name(&self) -> Result { - let name = self._str("name")?; - let magic = self._bool("magic")?; - Ok(if let Some(name) = name { - name - } else { - let name = self.ident.to_string(); - if magic { - format!("__{}__", name) - } else { - name - } - }) - } - - pub fn property_name(&self) -> Result { - let magic = self._bool("magic")?; - let setter = self._bool("setter")?; - let name = self._str("name")?; - - Ok(if let Some(name) = name { - name - } else { - let sig_name = self.ident.to_string(); - let name = if setter { - if let Some(name) = sig_name.strip_prefix("set_") { - if name.is_empty() { - bail_span!( - &self.ident, - "A #[{}(setter)] fn with a set_* name must \ - have something after \"set_\"", - self.parent_type - ) - } - name.to_string() - } else { - bail_span!( - &self.ident, - "A #[{}(setter)] fn must either have a `name` \ - parameter or a fn name along the lines of \"set_*\"", - self.parent_type - ) - } - } else { - sig_name - }; - if magic { - format!("__{}__", name) - } else { - name - } - }) - } - - pub fn setter(&self) -> Result { - self._bool("setter") - } -} diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs index 989bebe87..83605c49a 100644 --- a/vm/src/stdlib/collections.rs +++ b/vm/src/stdlib/collections.rs @@ -17,6 +17,7 @@ mod _collections { use crossbeam_utils::atomic::AtomicCell; + #[pyattr] #[pyclass(name = "deque")] #[derive(Debug)] struct PyDeque { @@ -402,6 +403,7 @@ mod _collections { } } + #[pyattr] #[pyclass(name = "_deque_iterator")] #[derive(Debug)] struct PyDequeIterator { diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 1a520e605..4c45a006c 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -22,6 +22,7 @@ mod decl { }; use crate::vm::VirtualMachine; + #[pyattr] #[pyclass(name = "chain")] #[derive(Debug)] struct PyItertoolsChain { @@ -107,6 +108,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "compress")] #[derive(Debug)] struct PyItertoolsCompress { @@ -158,6 +160,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "count")] #[derive(Debug)] struct PyItertoolsCount { @@ -210,6 +213,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "cycle")] #[derive(Debug)] struct PyItertoolsCycle { @@ -271,6 +275,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "repeat")] #[derive(Debug)] struct PyItertoolsRepeat { @@ -328,6 +333,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "starmap")] #[derive(Debug)] struct PyItertoolsStarmap { @@ -369,6 +375,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "takewhile")] #[derive(Debug)] struct PyItertoolsTakewhile { @@ -428,6 +435,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "dropwhile")] #[derive(Debug)] struct PyItertoolsDropwhile { @@ -512,6 +520,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "groupby")] struct PyItertoolsGroupBy { iterable: PyObjectRef, @@ -620,6 +629,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "_grouper")] #[derive(Debug)] struct PyItertoolsGrouper { @@ -671,6 +681,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "islice")] #[derive(Debug)] struct PyItertoolsIslice { @@ -797,6 +808,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "filterfalse")] #[derive(Debug)] struct PyItertoolsFilterFalse { @@ -853,6 +865,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "accumulate")] #[derive(Debug)] struct PyItertoolsAccumulate { @@ -937,6 +950,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "tee")] #[derive(Debug)] struct PyItertoolsTee { @@ -1015,6 +1029,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "product")] #[derive(Debug)] struct PyItertoolsProduct { @@ -1132,6 +1147,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "combinations")] #[derive(Debug)] struct PyItertoolsCombinations { @@ -1231,6 +1247,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "combinations_with_replacement")] #[derive(Debug)] struct PyItertoolsCombinationsWithReplacement { @@ -1325,6 +1342,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "permutations")] #[derive(Debug)] struct PyItertoolsPermutations { @@ -1455,6 +1473,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "zip_longest")] #[derive(Debug)] struct PyItertoolsZipLongest { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 2b96c8ad1..4690d1876 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -293,8 +293,8 @@ mod _os { rust_file(fileno); } - #[pyfunction] #[cfg(any(unix, windows, target_os = "wasi"))] + #[pyfunction] pub(crate) fn open( name: PyPathLike, flags: OpenFlags, @@ -350,8 +350,8 @@ mod _os { Ok(raw_file_number(handle)) } - #[pyfunction] #[cfg(not(any(unix, windows, target_os = "wasi")))] + #[pyfunction] pub(crate) fn open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error("os.open not implemented on this platform".to_owned())) } @@ -474,6 +474,7 @@ mod _os { mode.process_path(path, vm) } + #[pyattr] #[pyclass] #[derive(Debug)] struct DirEntry { @@ -557,6 +558,7 @@ mod _os { } } + #[pyattr] #[pyclass(name = "ScandirIter")] #[derive(Debug)] struct ScandirIterator { @@ -634,6 +636,7 @@ mod _os { .into_object()) } + #[pyattr] #[pystruct_sequence(module = "os", name = "stat_result")] #[derive(Debug)] pub(super) struct StatResult { @@ -830,6 +833,7 @@ mod _os { .into_owned() } + #[pyattr] #[pystruct_sequence(module = "os", name = "terminal_size")] #[allow(dead_code)] pub(super) struct PyTerminalSize { @@ -1739,6 +1743,7 @@ mod posix { Ok(ret_mask) } + #[pyattr] #[pystruct_sequence(module = "os", name = "uname_result")] #[derive(Debug)] struct UnameResult { diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index d18d59161..5dcf4d27f 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -786,6 +786,7 @@ mod _struct { .with_ref(|buf| format_spec.unpack(&buf[offset..offset + size], vm)) } + #[pyattr] #[pyclass(name = "unpack_iterator")] #[derive(Debug)] struct UnpackIterator { @@ -870,6 +871,7 @@ mod _struct { Ok(format_spec.size()) } + #[pyattr] #[pyclass(name = "Struct")] #[derive(Debug)] struct PyStruct { diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index f8e3ab4bd..aab9e6909 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -53,6 +53,7 @@ mod _random { } } + #[pyattr] #[pyclass(name = "Random")] #[derive(Debug)] struct PyRandom { diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 95d616af0..fe5fa1316 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -60,6 +60,7 @@ mod decl { type PySymbolTableRef = PyRef; type PySymbolRef = PyRef; + #[pyattr] #[pyclass(name = "SymbolTable")] struct PySymbolTable { symtable: symboltable::SymbolTable, @@ -176,6 +177,7 @@ mod decl { } } + #[pyattr] #[pyclass(name = "Symbol")] struct PySymbol { symbol: symboltable::Symbol, diff --git a/vm/src/stdlib/zlib.rs b/vm/src/stdlib/zlib.rs index 3e5d81f38..eb9a1ed1e 100644 --- a/vm/src/stdlib/zlib.rs +++ b/vm/src/stdlib/zlib.rs @@ -213,6 +213,7 @@ mod decl { unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(vm)), } } + #[pyattr] #[pyclass(name = "Decompress")] #[derive(Debug)] struct PyDecompress { @@ -373,6 +374,7 @@ mod decl { unconsumed: Vec, } + #[pyattr] #[pyclass(name = "Compress")] #[derive(Debug)] struct PyCompress { From 5e9df32b18bf0a2671e36781c3d07ba2e0a89402 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 06:06:37 +0900 Subject: [PATCH 02/13] `#[pyclass]` must contains `name` --- derive/src/pyclass.rs | 21 ++++--- derive/src/pymodule.rs | 24 +++---- derive/src/util.rs | 112 +++++++++++++-------------------- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/os.rs | 2 +- wasm/lib/src/browser_module.rs | 4 +- 6 files changed, 68 insertions(+), 97 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index c7f44b2da..9df92d5df 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,7 +1,7 @@ use super::Diagnostic; use crate::util::{ - attribute_arg, def_to_name, optional_attribute_arg, path_eq, AttributeExt, ContentItem, - ContentItemInner, ItemIdent, ItemMeta, ItemMetaInner, + path_eq, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ItemIdent, ItemMeta, + ItemMetaInner, }; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, quote_spanned, ToTokens}; @@ -563,17 +563,18 @@ fn generate_class_def( } pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result { - let (item, ident, attrs) = match item { - Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), - Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), + let (ident, attrs) = match &item { + Item::Struct(syn::ItemStruct { ident, attrs, .. }) => (ident, attrs), + Item::Enum(syn::ItemEnum { ident, attrs, .. }) => (ident, attrs), other => bail_span!( other, "#[pyclass] can only be on a struct or enum declaration" ), }; - let class_name = def_to_name("pyclass", &ident, &attr)?; - let module_name = optional_attribute_arg("pystruct_sequence", "module", &attr)?; + let class_meta = ClassItemMeta::from_nested("pyclass", ident, attr.into_iter())?; + let class_name = class_meta.class_name()?; + let module_name = class_meta.module()?; let class_def = generate_class_def(&ident, &class_name, module_name.as_deref(), &attrs)?; let ret = quote! { @@ -592,8 +593,10 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - /// #[pyfunction] struct FunctionItem { inner: ContentItemInner, @@ -322,8 +309,13 @@ impl ModuleItem for ClassItem { )); } } + let static_name = match self.attr_name() { + "pyclass" => "pyclass", + "pystruct_sequence" => "pystruct_sequence", + _ => unreachable!(), + }; let class_meta = ClassItemMeta::from_nested( - "pyclass/pystruct_sequence", + static_name, &args.ident, class_attr.promoted_nested()?.into_iter(), )?; @@ -331,7 +323,7 @@ impl ModuleItem for ClassItem { class_attr.fill_nested_meta("module", || { parse_quote! {module = #module_name} })?; - let class_name = class_meta.simple_name()?; + let class_name = class_meta.class_name()?; (module_name, class_name) }; for attr_index in self.pyattrs.iter().rev() { diff --git a/derive/src/util.rs b/derive/src/util.rs index d1343cf5c..01faca656 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -1,8 +1,7 @@ -use super::Diagnostic; use proc_macro2::{Span, TokenStream}; use quote::{quote, ToTokens}; use std::collections::HashMap; -use syn::{spanned::Spanned, Attribute, Ident, Lit, Meta, MetaList, NestedMeta, Path, Result}; +use syn::{spanned::Spanned, Attribute, Ident, Meta, MetaList, NestedMeta, Path, Result}; use syn_ext::ext::{AttributeExt as SynAttributeExt, *}; use syn_ext::types::PunctuatedNestedMeta; @@ -205,76 +204,53 @@ impl ItemMeta for SimpleItemMeta { } } +pub(crate) struct ClassItemMeta(ItemMetaInner); + +impl ItemMeta for ClassItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["module", "name"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl ClassItemMeta { + pub fn class_name(&self) -> Result { + const KEY: &str = "name"; + let inner = self.inner(); + let value = if let Some((_, meta)) = inner.meta.get(KEY) { + match meta { + Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Str(lit), + .. + }) => Some(lit.value()), + Meta::Path(_) => Some(inner.ident.to_string()), + _ => None, + } + } else { + None + }.ok_or_else(|| syn::Error::new_spanned( + &inner.ident, + format!( + "#[{attr_name}(name = ...)] must exist as a string. Try #[{attr_name}(name)] to use rust type name.", + attr_name=inner.parent_type + ), + ))?; + Ok(value) + } + + pub fn module(&self) -> Result> { + self.inner()._optional_str("module") + } +} + pub(crate) fn path_eq(path: &Path, s: &str) -> bool { path.get_ident().map_or(false, |id| id == s) } -pub(crate) fn def_to_name( - attr_name: &'static str, - ident: &Ident, - attrs: &[NestedMeta], -) -> std::result::Result { - optional_attribute_arg(attr_name, "name", attrs) - .transpose() - .unwrap_or_else(|| Ok(ident.to_string())) -} - -pub(crate) fn attribute_arg( - attr_name: &'static str, - arg_name: &'static str, - attrs: &[NestedMeta], -) -> std::result::Result { - if let Some(r) = optional_attribute_arg(attr_name, arg_name, attrs).transpose() { - r - } else { - bail_span!( - attrs[0], - "#[{}({} = ...)] must exist but not found", - attr_name, - arg_name - ) - } -} - -pub(crate) fn optional_attribute_arg<'a, I>( - attr_name: &'static str, - arg_name: &'static str, - attrs: I, -) -> std::result::Result, Diagnostic> -where - I: std::iter::IntoIterator, -{ - let mut arg_value = None; - for attr in attrs { - match attr { - NestedMeta::Meta(Meta::NameValue(name_value)) - if path_eq(&name_value.path, arg_name) => - { - if let Lit::Str(lit) = &name_value.lit { - if arg_value.is_some() { - bail_span!( - name_value.lit, - "#[{}({} = ...)] must be unique but found multiple times", - attr_name, - arg_name - ); - } - arg_value = Some(lit.value()); - } else { - bail_span!( - name_value.lit, - "#[{}({} = ...)] must be a string", - attr_name, - arg_name - ); - } - } - _ => continue, - } - } - Ok(arg_value) -} - pub(crate) trait AttributeExt: SynAttributeExt { fn promoted_nested(&self) -> Result; fn try_remove_name(&mut self, name: &str) -> Result>; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index dcb0761c3..5b3355bf0 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -647,7 +647,7 @@ mod fileio { flag as u32 } - #[pyclass] + #[pyclass(name)] #[derive(Debug)] struct FileIO { fd: AtomicCell, diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 4690d1876..df7cdbc7e 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -475,7 +475,7 @@ mod _os { } #[pyattr] - #[pyclass] + #[pyclass(name)] #[derive(Debug)] struct DirEntry { entry: fs::DirEntry, diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 4ebe8e0d2..1e72a7acd 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -248,7 +248,7 @@ impl PyPromise { } } -#[pyclass] +#[pyclass(name)] #[derive(Debug)] struct Document { doc: web_sys::Document, @@ -276,7 +276,7 @@ impl Document { } } -#[pyclass] +#[pyclass(name)] #[derive(Debug)] struct Element { elem: web_sys::Element, From c9c1ac9532057126ee917d7447207f4c4f525a30 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 14:46:15 +0900 Subject: [PATCH 03/13] Give module names for classes --- vm/src/stdlib/array.rs | 4 ++-- vm/src/stdlib/csv.rs | 2 +- vm/src/stdlib/hashlib.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/re.rs | 4 ++-- vm/src/stdlib/socket.rs | 2 +- vm/src/stdlib/ssl.rs | 4 ++-- vm/src/stdlib/thread.rs | 6 +++--- vm/src/stdlib/unicodedata.rs | 2 +- vm/src/stdlib/winreg.rs | 2 +- wasm/lib/src/browser_module.rs | 6 +++--- wasm/lib/src/js_module.rs | 2 +- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 630ec6445..4e2359dc7 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -246,7 +246,7 @@ def_array_enum!( (Double, f64, 'd'), ); -#[pyclass(name = "array")] +#[pyclass(module = "array", name = "array")] #[derive(Debug)] pub struct PyArray { array: PyRwLock, @@ -530,7 +530,7 @@ impl PyArray { } } -#[pyclass(name = "array_iterator")] +#[pyclass(module = "array", name = "array_iterator")] #[derive(Debug)] pub struct PyArrayIter { position: AtomicCell, diff --git a/vm/src/stdlib/csv.rs b/vm/src/stdlib/csv.rs index 3752ea318..862a3f8d8 100644 --- a/vm/src/stdlib/csv.rs +++ b/vm/src/stdlib/csv.rs @@ -124,7 +124,7 @@ impl ReadState { } } -#[pyclass(name = "Reader")] +#[pyclass(module = "csv", name = "Reader")] struct Reader { state: PyRwLock, } diff --git a/vm/src/stdlib/hashlib.rs b/vm/src/stdlib/hashlib.rs index 6c465c95f..913895d26 100644 --- a/vm/src/stdlib/hashlib.rs +++ b/vm/src/stdlib/hashlib.rs @@ -14,7 +14,7 @@ use sha1::Sha1; use sha2::{Sha224, Sha256, Sha384, Sha512}; use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; // TODO: , Shake128, Shake256}; -#[pyclass(name = "hasher")] +#[pyclass(module = "hashlib", name = "hasher")] struct PyHasher { name: String, buffer: PyRwLock, diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 5b3355bf0..5001859db 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -647,7 +647,7 @@ mod fileio { flag as u32 } - #[pyclass(name)] + #[pyclass(module = "io", name)] #[derive(Debug)] struct FileIO { fd: AtomicCell, diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index 2bac83ef6..10107467e 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -16,7 +16,7 @@ use crate::obj::objtype::PyClassRef; use crate::pyobject::{BorrowValue, PyClassImpl, PyObjectRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; -#[pyclass(name = "Pattern")] +#[pyclass(module = "re", name = "Pattern")] #[derive(Debug)] struct PyPattern { regex: Regex, @@ -69,7 +69,7 @@ impl PyValue for PyPattern { } /// Inner data for a match object. -#[pyclass(name = "Match")] +#[pyclass(module = "re", name = "Match")] struct PyMatch { haystack: PyStringRef, captures: Vec>>, diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 1cddeedbc..2e44a8a96 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -54,7 +54,7 @@ mod c { }; } -#[pyclass(name = "socket")] +#[pyclass(module = "socket", name = "socket")] #[derive(Debug)] pub struct PySocket { kind: AtomicCell, diff --git a/vm/src/stdlib/ssl.rs b/vm/src/stdlib/ssl.rs index b28992be0..81bc71b28 100644 --- a/vm/src/stdlib/ssl.rs +++ b/vm/src/stdlib/ssl.rs @@ -228,7 +228,7 @@ fn ssl_rand_pseudo_bytes(n: i32, vm: &VirtualMachine) -> PyResult<(Vec, bool } } -#[pyclass(name = "_SSLContext")] +#[pyclass(module = "ssl", name = "_SSLContext")] struct PySslContext { ctx: PyRwLock, check_hostname: bool, @@ -504,7 +504,7 @@ struct LoadVerifyLocationsArgs { cadata: Option>, } -#[pyclass(name = "_SSLSocket")] +#[pyclass(module = "ssl", name = "_SSLSocket")] struct PySslSocket { ctx: PyRef, stream: PyRwLock>>, diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs index 57dad7a15..8b0ff4f28 100644 --- a/vm/src/stdlib/thread.rs +++ b/vm/src/stdlib/thread.rs @@ -92,7 +92,7 @@ macro_rules! repr_lock_impl { }}; } -#[pyclass(name = "lock")] +#[pyclass(module = "thread", name = "lock")] struct PyLock { mu: RawMutex, } @@ -146,7 +146,7 @@ impl PyLock { } pub type RawRMutex = RawReentrantMutex; -#[pyclass(name = "RLock")] +#[pyclass(module = "thread", name = "RLock")] struct PyRLock { mu: RawRMutex, } @@ -277,7 +277,7 @@ fn thread_count(vm: &VirtualMachine) -> usize { vm.state.thread_count.load() } -#[pyclass(name = "_local")] +#[pyclass(module = "thread", name = "_local")] #[derive(Debug)] struct PyLocal { data: ThreadLocal, diff --git a/vm/src/stdlib/unicodedata.rs b/vm/src/stdlib/unicodedata.rs index 441f7e75f..961eabe14 100644 --- a/vm/src/stdlib/unicodedata.rs +++ b/vm/src/stdlib/unicodedata.rs @@ -54,7 +54,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { module } -#[pyclass(name = "UCD")] +#[pyclass(module = "unicodedata", name = "UCD")] #[derive(Debug)] struct PyUCD { unic_version: UnicodeVersion, diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs index c6bfabefa..326a2b0f1 100644 --- a/vm/src/stdlib/winreg.rs +++ b/vm/src/stdlib/winreg.rs @@ -14,7 +14,7 @@ use std::io; use winapi::shared::winerror; use winreg::{enums::RegType, RegKey, RegValue}; -#[pyclass(name = "HKEYType")] +#[pyclass(module = "winreg", name = "HKEYType")] #[derive(Debug)] struct PyHKEY { key: PyRwLock, diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 1e72a7acd..b8416606a 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -162,7 +162,7 @@ fn browser_cancel_animation_frame(id: i32, vm: &VirtualMachine) -> PyResult { Ok(vm.get_none()) } -#[pyclass(name = "Promise")] +#[pyclass(module = "browser", name = "Promise")] #[derive(Debug)] pub struct PyPromise { value: Promise, @@ -248,7 +248,7 @@ impl PyPromise { } } -#[pyclass(name)] +#[pyclass(module = "browser", name)] #[derive(Debug)] struct Document { doc: web_sys::Document, @@ -276,7 +276,7 @@ impl Document { } } -#[pyclass(name)] +#[pyclass(module = "browser", name)] #[derive(Debug)] struct Element { elem: web_sys::Element, diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index 9651296f3..a52d12cf1 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -30,7 +30,7 @@ extern "C" { fn instance_of(lhs: &JsValue, rhs: &JsValue) -> Result; } -#[pyclass(name = "JsValue")] +#[pyclass(module = "_js", name = "JsValue")] #[derive(Debug)] pub struct PyJsValue { value: JsValue, From 9d025b0e4e39b465995865ca8d8d709d9fb620bb Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 14:46:49 +0900 Subject: [PATCH 04/13] `#[pyclass]` now requires `module` field --- derive/src/pyclass.rs | 2 +- derive/src/util.rs | 42 ++++++++++++++++++++++++++++++++- vm/src/exceptions.rs | 2 +- vm/src/frame.rs | 2 +- vm/src/obj/objasyncgenerator.rs | 8 +++---- vm/src/obj/objbool.rs | 2 +- vm/src/obj/objbuiltinfunc.rs | 4 ++-- vm/src/obj/objbytearray.rs | 4 ++-- vm/src/obj/objbytes.rs | 4 ++-- vm/src/obj/objclassmethod.rs | 2 +- vm/src/obj/objcode.rs | 2 +- vm/src/obj/objcomplex.rs | 2 +- vm/src/obj/objcoroutine.rs | 4 ++-- vm/src/obj/objdict.rs | 6 ++--- vm/src/obj/objellipsis.rs | 2 +- vm/src/obj/objenumerate.rs | 2 +- vm/src/obj/objfilter.rs | 2 +- vm/src/obj/objfloat.rs | 2 +- vm/src/obj/objfunction.rs | 4 ++-- vm/src/obj/objgenerator.rs | 2 +- vm/src/obj/objgetset.rs | 2 +- vm/src/obj/objint.rs | 2 +- vm/src/obj/objiter.rs | 4 ++-- vm/src/obj/objlist.rs | 6 ++--- vm/src/obj/objmap.rs | 2 +- vm/src/obj/objmappingproxy.rs | 2 +- vm/src/obj/objmemory.rs | 2 +- vm/src/obj/objmodule.rs | 2 +- vm/src/obj/objnamespace.rs | 2 +- vm/src/obj/objnone.rs | 2 +- vm/src/obj/objobject.rs | 2 +- vm/src/obj/objproperty.rs | 2 +- vm/src/obj/objrange.rs | 4 ++-- vm/src/obj/objset.rs | 6 ++--- vm/src/obj/objslice.rs | 2 +- vm/src/obj/objstaticmethod.rs | 2 +- vm/src/obj/objstr.rs | 6 ++--- vm/src/obj/objsuper.rs | 2 +- vm/src/obj/objtraceback.rs | 2 +- vm/src/obj/objtuple.rs | 4 ++-- vm/src/obj/objtype.rs | 2 +- vm/src/obj/objweakproxy.rs | 2 +- vm/src/obj/objweakref.rs | 2 +- vm/src/obj/objzip.rs | 2 +- 44 files changed, 103 insertions(+), 63 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 9df92d5df..430260819 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -596,7 +596,7 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result Result> { - self.inner()._optional_str("module") + const KEY: &str = "module"; + let inner = self.inner(); + let value = if let Some((_, meta)) = inner.meta.get(KEY) { + match meta { + Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Str(lit), + .. + }) => Ok(Some(lit.value())), + Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Bool(lit), + .. + }) => if lit.value { + Err(lit.span()) + } else { + Ok(None) + } + other => Err(other.span()), + } + } else { + Err(inner.ident.span()) + }.map_err(|span| syn::Error::new( + span, + format!( + "#[{attr_name}(module = ...)] must exist as a string or false. Try #[{attr_name}(module=false)] for built-in types.", + attr_name=inner.parent_type + ), + ))?; + Ok(value) + } + + pub fn mandatory_module(&self) -> Result { + let inner = self.inner(); + let value = self.module().ok().flatten(). + ok_or_else(|| syn::Error::new_spanned( + &inner.ident, + format!( + "#[{attr_name}(module = ...)] must exist as a string. Built-in module is not allowed here.", + attr_name=inner.parent_type + ), + ))?; + Ok(value) } } diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 6cf1d8f29..f1bebec9d 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -21,7 +21,7 @@ use std::io::{self, BufRead, BufReader}; use crossbeam_utils::atomic::AtomicCell; -#[pyclass(name = "BaseException")] +#[pyclass(module = false, name = "BaseException")] pub struct PyBaseException { traceback: PyRwLock>, cause: PyRwLock>, diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 263096ca3..0fcc8fd95 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -87,7 +87,7 @@ struct FrameState { blocks: Vec, } -#[pyclass(name = "frame")] +#[pyclass(module = false, name = "frame")] pub struct Frame { pub code: PyCodeRef, pub scope: Scope, diff --git a/vm/src/obj/objasyncgenerator.rs b/vm/src/obj/objasyncgenerator.rs index 740c17d2f..2dd744ade 100644 --- a/vm/src/obj/objasyncgenerator.rs +++ b/vm/src/obj/objasyncgenerator.rs @@ -9,7 +9,7 @@ use crate::vm::VirtualMachine; use crossbeam_utils::atomic::AtomicCell; -#[pyclass(name = "async_generator")] +#[pyclass(name = "async_generator", module = false)] #[derive(Debug)] pub struct PyAsyncGen { inner: Coro, @@ -114,7 +114,7 @@ impl PyAsyncGen { } } -#[pyclass(name = "async_generator_wrapped_value")] +#[pyclass(module = false, name = "async_generator_wrapped_value")] #[derive(Debug)] pub(crate) struct PyAsyncGenWrappedValue(pub PyObjectRef); impl PyValue for PyAsyncGenWrappedValue { @@ -158,7 +158,7 @@ enum AwaitableState { Closed, } -#[pyclass(name = "async_generator_asend")] +#[pyclass(module = false, name = "async_generator_asend")] #[derive(Debug)] pub(crate) struct PyAsyncGenASend { ag: PyAsyncGenRef, @@ -253,7 +253,7 @@ impl PyAsyncGenASend { } } -#[pyclass(name = "async_generator_athrow")] +#[pyclass(module = false, name = "async_generator_athrow")] #[derive(Debug)] pub(crate) struct PyAsyncGenAThrow { ag: PyAsyncGenRef, diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index 3fac7ae6a..fd9e29c84 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -81,7 +81,7 @@ pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { /// Returns True when the argument x is true, False otherwise. /// The builtins True and False are the only two instances of the class bool. /// The class bool is a subclass of the class int, and cannot be subclassed. -#[pyclass(name = "bool")] +#[pyclass(name = "bool", module = false)] pub(crate) struct PyBool; #[pyimpl] diff --git a/vm/src/obj/objbuiltinfunc.rs b/vm/src/obj/objbuiltinfunc.rs index a2dfe57ef..8bb4cf457 100644 --- a/vm/src/obj/objbuiltinfunc.rs +++ b/vm/src/obj/objbuiltinfunc.rs @@ -9,7 +9,7 @@ use crate::pyobject::{ use crate::slots::{SlotCall, SlotDescriptor}; use crate::vm::VirtualMachine; -#[pyclass(name = "builtin_function_or_method")] +#[pyclass(name = "builtin_function_or_method", module = false)] pub struct PyBuiltinFunction { value: PyNativeFunc, module: Option, @@ -70,7 +70,7 @@ impl PyBuiltinFunction { } } -#[pyclass(name = "method_descriptor")] +#[pyclass(module = false, name = "method_descriptor")] pub struct PyBuiltinMethod { function: PyBuiltinFunction, } diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 4f2776618..0d6cd8f2e 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -33,7 +33,7 @@ use crate::vm::VirtualMachine; /// - a bytes or a buffer object\n \ /// - any object implementing the buffer API.\n \ /// - an integer"; -#[pyclass(name = "bytearray")] +#[pyclass(module = false, name = "bytearray")] #[derive(Debug)] pub struct PyByteArray { inner: PyRwLock, @@ -606,7 +606,7 @@ impl PyByteArray { // obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; // } -#[pyclass(name = "bytearray_iterator")] +#[pyclass(module = false, name = "bytearray_iterator")] #[derive(Debug)] pub struct PyByteArrayIterator { position: AtomicCell, diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 2519b2991..5b44da6b2 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -32,7 +32,7 @@ use rustpython_common::hash::PyHash; /// - a text string encoded using the specified encoding\n \ /// - any object implementing the buffer API.\n \ /// - an integer"; -#[pyclass(name = "bytes")] +#[pyclass(module = false, name = "bytes")] #[derive(Clone, Debug)] pub struct PyBytes { inner: PyBytesInner, @@ -488,7 +488,7 @@ impl PyBytes { } } -#[pyclass(name = "bytes_iterator")] +#[pyclass(module = false, name = "bytes_iterator")] #[derive(Debug)] pub struct PyBytesIterator { position: AtomicCell, diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index 440cf63eb..edffe5987 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -26,7 +26,7 @@ use crate::vm::VirtualMachine; /// /// Class methods are different than C++ or Java static methods. /// If you want those, see the staticmethod builtin. -#[pyclass(name = "classmethod")] +#[pyclass(module = false, name = "classmethod")] #[derive(Clone, Debug)] pub struct PyClassMethod { callable: PyObjectRef, diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index f3196b7ad..14772e07f 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -12,7 +12,7 @@ use crate::vm::VirtualMachine; pub type PyCodeRef = PyRef; -#[pyclass(name = "code")] +#[pyclass(module = false, name = "code")] pub struct PyCode { pub code: bytecode::CodeObject, } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 6ce41459e..93e7b5963 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -14,7 +14,7 @@ use rustpython_common::{float_ops, hash}; /// Create a complex number from a real part and an optional imaginary part. /// /// This is equivalent to (real + imag*1j) where imag defaults to 0. -#[pyclass(name = "complex")] +#[pyclass(module = false, name = "complex")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyComplex { value: Complex64, diff --git a/vm/src/obj/objcoroutine.rs b/vm/src/obj/objcoroutine.rs index d1faacc2b..35cc43a61 100644 --- a/vm/src/obj/objcoroutine.rs +++ b/vm/src/obj/objcoroutine.rs @@ -9,7 +9,7 @@ use crate::vm::VirtualMachine; pub type PyCoroutineRef = PyRef; -#[pyclass(name = "coroutine")] +#[pyclass(module = false, name = "coroutine")] #[derive(Debug)] pub struct PyCoroutine { inner: Coro, @@ -95,7 +95,7 @@ impl PyCoroutine { } } -#[pyclass(name = "coroutine_wrapper")] +#[pyclass(module = false, name = "coroutine_wrapper")] #[derive(Debug)] pub struct PyCoroutineWrapper { coro: PyCoroutineRef, diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 7b46aec79..5e034bddb 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -18,7 +18,7 @@ use std::mem::size_of; pub type DictContentType = dictdatatype::Dict; -#[pyclass(name = "dict")] +#[pyclass(module = false, name = "dict")] #[derive(Default)] pub struct PyDict { entries: DictContentType, @@ -553,7 +553,7 @@ impl Iterator for DictIter { macro_rules! dict_iterator { ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { - #[pyclass(name = $class_name)] + #[pyclass(module=false,name = $class_name)] #[derive(Debug)] pub(crate) struct $name { pub dict: PyDictRef, @@ -598,7 +598,7 @@ macro_rules! dict_iterator { } } - #[pyclass(name = $iter_class_name)] + #[pyclass(module=false,name = $iter_class_name)] #[derive(Debug)] pub(crate) struct $iter_name { pub dict: PyDictRef, diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index ea61b933c..7c97b9248 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -6,7 +6,7 @@ pub(crate) fn init(context: &PyContext) { PyEllipsis::extend_class(context, &context.ellipsis_type()); } -#[pyclass(name = "EllipsisType")] +#[pyclass(module = false, name = "EllipsisType")] #[derive(Debug)] pub struct PyEllipsis; diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 3589ea501..d699ce8f4 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -11,7 +11,7 @@ use crate::function::OptionalArg; use crate::pyobject::{BorrowValue, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -#[pyclass(name = "enumerate")] +#[pyclass(module = false, name = "enumerate")] #[derive(Debug)] pub struct PyEnumerate { counter: PyRwLock, diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index 07774d841..f8cf51e47 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -10,7 +10,7 @@ pub type PyFilterRef = PyRef; /// /// Return an iterator yielding those items of iterable for which function(item) /// is true. If function is None, return the items that are true. -#[pyclass(name = "filter")] +#[pyclass(module = false, name = "filter")] #[derive(Debug)] pub struct PyFilter { predicate: PyObjectRef, diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index eca839910..b61eb6930 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -19,7 +19,7 @@ use crate::vm::VirtualMachine; use rustpython_common::{float_ops, hash}; /// Convert a string or number to a floating point number, if possible. -#[pyclass(name = "float")] +#[pyclass(module = false, name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyFloat { value: f64, diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index c07267b6b..bdf845cdb 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -20,7 +20,7 @@ use itertools::Itertools; pub type PyFunctionRef = PyRef; -#[pyclass(name = "function")] +#[pyclass(module = false, name = "function")] #[derive(Debug)] pub struct PyFunction { code: PyCodeRef, @@ -287,7 +287,7 @@ impl PyFunction { } } -#[pyclass(name = "method")] +#[pyclass(module = false, name = "method")] #[derive(Debug)] pub struct PyBoundMethod { // TODO: these shouldn't be public diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index e91e24406..1578a9ad1 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -12,7 +12,7 @@ use crate::vm::VirtualMachine; pub type PyGeneratorRef = PyRef; -#[pyclass(name = "generator")] +#[pyclass(module = false, name = "generator")] #[derive(Debug)] pub struct PyGenerator { inner: Coro, diff --git a/vm/src/obj/objgetset.rs b/vm/src/obj/objgetset.rs index b7c43358e..af307d1ee 100644 --- a/vm/src/obj/objgetset.rs +++ b/vm/src/obj/objgetset.rs @@ -144,7 +144,7 @@ where } } -#[pyclass(name = "getset_descriptor")] +#[pyclass(module = false, name = "getset_descriptor")] pub struct PyGetSet { name: String, getter: Option, diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 563263539..9c2030025 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -40,7 +40,7 @@ use rustpython_common::hash; /// Base 0 means to interpret the base from the string as an integer literal. /// >>> int('0b100', base=0) /// 4 -#[pyclass(name = "int")] +#[pyclass(module = false, name = "int")] #[derive(Debug)] pub struct PyInt { value: BigInt, diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 080f5606f..2f13b6a2c 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -149,7 +149,7 @@ pub fn length_hint(vm: &VirtualMachine, iter: PyObjectRef) -> PyResult, @@ -222,7 +222,7 @@ pub fn seq_iter_method(obj: PyObjectRef) -> PySequenceIterator { PySequenceIterator::new_forward(obj) } -#[pyclass(name = "callable_iterator")] +#[pyclass(module = false, name = "callable_iterator")] #[derive(Debug)] pub struct PyCallableIterator { callable: PyCallable, diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 577f8b372..e6b5d1119 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -27,7 +27,7 @@ use crate::vm::{ReprGuard, VirtualMachine}; /// /// If no argument is given, the constructor creates a new empty list. /// The argument must be an iterable if specified. -#[pyclass(name = "list")] +#[pyclass(module = false, name = "list")] #[derive(Default)] pub struct PyList { elements: PyRwLock>, @@ -825,7 +825,7 @@ fn do_sort( Ok(()) } -#[pyclass(name = "list_iterator")] +#[pyclass(module = false, name = "list_iterator")] #[derive(Debug)] pub struct PyListIterator { pub position: AtomicCell, @@ -864,7 +864,7 @@ impl PyListIterator { } } -#[pyclass(name = "list_reverseiterator")] +#[pyclass(module = false, name = "list_reverseiterator")] #[derive(Debug)] pub struct PyListReverseIterator { pub position: AtomicCell, diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index d1b0aa6cc..4df54f4d6 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -8,7 +8,7 @@ use crate::vm::VirtualMachine; /// /// Make an iterator that computes the function using arguments from /// each of the iterables. Stops when the shortest iterable is exhausted. -#[pyclass(name = "map")] +#[pyclass(module = false, name = "map")] #[derive(Debug)] pub struct PyMap { mapper: PyObjectRef, diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs index dfc5799a2..2661ddd09 100644 --- a/vm/src/obj/objmappingproxy.rs +++ b/vm/src/obj/objmappingproxy.rs @@ -9,7 +9,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -#[pyclass(name = "mappingproxy")] +#[pyclass(module = false, name = "mappingproxy")] #[derive(Debug)] pub struct PyMappingProxy { mapping: MappingProxyInner, diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 75b9aff13..b58b6b717 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -6,7 +6,7 @@ use crate::pyobject::{ use crate::stdlib::array::PyArray; use crate::vm::VirtualMachine; -#[pyclass(name = "memoryview")] +#[pyclass(module = false, name = "memoryview")] #[derive(Debug)] pub struct PyMemoryView { obj_ref: PyObjectRef, diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index 2cf988311..0a1c7b32b 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -7,7 +7,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -#[pyclass(name = "module")] +#[pyclass(module = false, name = "module")] #[derive(Debug)] pub struct PyModule {} pub type PyModuleRef = PyRef; diff --git a/vm/src/obj/objnamespace.rs b/vm/src/obj/objnamespace.rs index ddf1f9feb..3850f5a77 100644 --- a/vm/src/obj/objnamespace.rs +++ b/vm/src/obj/objnamespace.rs @@ -6,7 +6,7 @@ use crate::vm::VirtualMachine; /// A simple attribute-based namespace. /// /// SimpleNamespace(**kwargs) -#[pyclass(name = "SimpleNamespace")] +#[pyclass(module = false, name = "SimpleNamespace")] #[derive(Debug)] pub struct PyNamespace; diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index 09ea8ce64..450ca9fd6 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -4,7 +4,7 @@ use crate::pyobject::{ }; use crate::vm::VirtualMachine; -#[pyclass(name = "NoneType")] +#[pyclass(module = false, name = "NoneType")] #[derive(Debug)] pub struct PyNone; pub type PyNoneRef = PyRef; diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 085e3b8d5..f1adbb149 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -13,7 +13,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; /// The most base type -#[pyclass(name = "object")] +#[pyclass(module = false, name = "object")] #[derive(Debug)] pub struct PyBaseObject; diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index dc3da6e6d..0f5f12dc8 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -43,7 +43,7 @@ use crate::vm::VirtualMachine; /// @x.deleter /// def x(self): /// del self._x -#[pyclass(name = "property")] +#[pyclass(module = false, name = "property")] #[derive(Debug)] pub struct PyProperty { getter: Option, diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 4ee30d738..3e4ae757b 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -25,7 +25,7 @@ use rustpython_common::hash::PyHash; /// start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3. /// These are exactly the valid indices for a list of 4 elements. /// When step is given, it specifies the increment (or decrement). -#[pyclass(name = "range")] +#[pyclass(module = false, name = "range")] #[derive(Debug, Clone)] pub struct PyRange { pub start: PyIntRef, @@ -400,7 +400,7 @@ impl PyRange { } } -#[pyclass(name = "range_iterator")] +#[pyclass(module = false, name = "range_iterator")] #[derive(Debug)] pub struct PyRangeIterator { position: AtomicCell, diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b721550bd..9dfb0c60c 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -21,7 +21,7 @@ pub type SetContentType = dictdatatype::Dict<()>; /// set(iterable) -> new set object /// /// Build an unordered collection of unique elements. -#[pyclass(name = "set")] +#[pyclass(module = false, name = "set")] #[derive(Default)] pub struct PySet { inner: PySetInner, @@ -32,7 +32,7 @@ pub type PySetRef = PyRef; /// frozenset(iterable) -> frozenset object /// /// Build an immutable unordered collection of unique elements. -#[pyclass(name = "frozenset")] +#[pyclass(module = false, name = "frozenset")] #[derive(Default)] pub struct PyFrozenSet { inner: PySetInner, @@ -824,7 +824,7 @@ struct SetSizeInfo { position: usize, } -#[pyclass(name = "set_iterator")] +#[pyclass(module = false, name = "set_iterator")] pub(crate) struct PySetIterator { dict: PyRc, size_info: crossbeam_utils::atomic::AtomicCell, diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index a48ddc40a..78516cfa5 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -9,7 +9,7 @@ use crate::vm::VirtualMachine; use num_bigint::{BigInt, ToBigInt}; use num_traits::{One, Signed, Zero}; -#[pyclass(name = "slice")] +#[pyclass(module = false, name = "slice")] #[derive(Debug)] pub struct PySlice { pub start: Option, diff --git a/vm/src/obj/objstaticmethod.rs b/vm/src/obj/objstaticmethod.rs index 1a6b8329b..ec031286a 100644 --- a/vm/src/obj/objstaticmethod.rs +++ b/vm/src/obj/objstaticmethod.rs @@ -4,7 +4,7 @@ use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyVa use crate::slots::SlotDescriptor; use crate::vm::VirtualMachine; -#[pyclass(name = "staticmethod")] +#[pyclass(module = false, name = "staticmethod")] #[derive(Clone, Debug)] pub struct PyStaticMethod { pub callable: PyObjectRef, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 85c74e631..0f78dcab4 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -41,7 +41,7 @@ use rustpython_common::hash; /// or repr(object). /// encoding defaults to sys.getdefaultencoding(). /// errors defaults to 'strict'." -#[pyclass(name = "str")] +#[pyclass(module = false, name = "str")] #[derive(Debug)] pub struct PyString { value: String, @@ -102,7 +102,7 @@ impl TryIntoRef for &str { } } -#[pyclass(name = "str_iterator")] +#[pyclass(module = false, name = "str_iterator")] #[derive(Debug)] pub struct PyStringIterator { pub string: PyStringRef, @@ -135,7 +135,7 @@ impl PyStringIterator { } } -#[pyclass(name = "str_reverseiterator")] +#[pyclass(module = false, name = "str_reverseiterator")] #[derive(Debug)] pub struct PyStringReverseIterator { pub position: AtomicCell, diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 70148cb85..06171900e 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -21,7 +21,7 @@ use itertools::Itertools; pub type PySuperRef = PyRef; -#[pyclass(name = "super")] +#[pyclass(module = false, name = "super")] #[derive(Debug)] pub struct PySuper { typ: PyClassRef, diff --git a/vm/src/obj/objtraceback.rs b/vm/src/obj/objtraceback.rs index b8c13f332..18c062a4c 100644 --- a/vm/src/obj/objtraceback.rs +++ b/vm/src/obj/objtraceback.rs @@ -3,7 +3,7 @@ use crate::obj::objtype::PyClassRef; use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyValue}; use crate::vm::VirtualMachine; -#[pyclass(name = "traceback")] +#[pyclass(module = false, name = "traceback")] #[derive(Debug)] pub struct PyTraceback { pub next: Option, // TODO: Make mutable diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 533b3661e..cfaec4e2e 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -18,7 +18,7 @@ use rustpython_common::hash::PyHash; /// tuple(iterable) -> tuple initialized from iterable's items /// /// If the argument is a tuple, the return value is the same object. -#[pyclass(name = "tuple")] +#[pyclass(module = false, name = "tuple")] pub struct PyTuple { elements: Vec, } @@ -255,7 +255,7 @@ impl PyTuple { } } -#[pyclass(name = "tuple_iterator")] +#[pyclass(module = false, name = "tuple_iterator")] #[derive(Debug)] pub struct PyTupleIterator { position: AtomicCell, diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index d2003b9ad..ade7ffcc9 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -24,7 +24,7 @@ use std::ops::Deref; /// type(object_or_name, bases, dict) /// type(object) -> the object's type /// type(name, bases, dict) -> a new type -#[pyclass(name = "type")] +#[pyclass(module = false, name = "type")] pub struct PyClass { pub name: String, pub base: Option, diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs index 44f0fee64..d0c382f9a 100644 --- a/vm/src/obj/objweakproxy.rs +++ b/vm/src/obj/objweakproxy.rs @@ -4,7 +4,7 @@ use crate::function::OptionalArg; use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -#[pyclass(name = "weakproxy")] +#[pyclass(module = false, name = "weakproxy")] #[derive(Debug)] pub struct PyWeakProxy { weak: PyWeak, diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index f43ba2831..ab3740fca 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -11,7 +11,7 @@ use rustpython_common::rc::{PyRc, PyWeak as Weak}; use crossbeam_utils::atomic::AtomicCell; -#[pyclass(name = "ref")] +#[pyclass(module = false, name = "ref")] #[derive(Debug)] pub struct PyWeak { referent: Weak>, diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index f107b2aba..6ff877ea7 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -6,7 +6,7 @@ use crate::vm::VirtualMachine; pub type PyZipRef = PyRef; -#[pyclass(name = "zip")] +#[pyclass(module = false, name = "zip")] #[derive(Debug)] pub struct PyZip { iterators: Vec, From 8d46a96068a7f5df96112ec5b2678b5929cfddfa Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 15:14:11 +0900 Subject: [PATCH 05/13] use meta item rather than parent name --- derive/src/pyclass.rs | 56 +++++++++++++++++++-------------- derive/src/pymodule.rs | 30 +++++++++--------- derive/src/util.rs | 71 +++++++++++++++++++++++++----------------- 3 files changed, 90 insertions(+), 67 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 430260819..089e1ab12 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -90,35 +90,38 @@ impl Class { } fn extract_method( - ident: &Ident, - nested: PunctuatedNestedMeta, + item: &Ident, + (meta, nested): (&Ident, PunctuatedNestedMeta), ) -> Result { - let item_meta = MethodItemMeta::from_nested("pymethod", &ident, nested.into_iter())?; + let item_meta = + MethodItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; Ok(ClassItem::Method { - item_ident: ident.clone(), + item_ident: item.clone(), py_name: item_meta.method_name()?, }) } fn extract_classmethod( - ident: &Ident, - nested: PunctuatedNestedMeta, + item: &Ident, + (meta, nested): (&Ident, PunctuatedNestedMeta), ) -> Result { - let item_meta = MethodItemMeta::from_nested("pyclassmethod", &ident, nested.into_iter())?; + let item_meta = + MethodItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; Ok(ClassItem::ClassMethod { - item_ident: ident.clone(), + item_ident: item.clone(), py_name: item_meta.method_name()?, }) } fn extract_property( - ident: &Ident, - nested: PunctuatedNestedMeta, + item: &Ident, + (meta, nested): (&Ident, PunctuatedNestedMeta), ) -> Result { - let item_meta = PropertyItemMeta::from_nested("pyproperty", &ident, nested.into_iter())?; + let item_meta = + PropertyItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; Ok(ClassItem::Property { + item_ident: item.clone(), py_name: item_meta.property_name()?, - item_ident: ident.clone(), setter: item_meta.setter()?, }) } @@ -159,9 +162,13 @@ impl Class { }; let item = match name.to_string().as_str() { - "pymethod" => Self::extract_method(item.ident, attr.promoted_nested()?)?, - "pyclassmethod" => Self::extract_classmethod(item.ident, attr.promoted_nested()?)?, - "pyproperty" => Self::extract_property(item.ident, attr.promoted_nested()?)?, + "pymethod" => Self::extract_method(item.ident, attr.ident_and_promoted_nested()?)?, + "pyclassmethod" => { + Self::extract_classmethod(item.ident, attr.ident_and_promoted_nested()?)? + } + "pyproperty" => { + Self::extract_property(item.ident, attr.ident_and_promoted_nested()?)? + } "pyslot" => Self::extract_slot(item.ident, attr.promoted_nested()?)?, _ => { continue; @@ -208,7 +215,7 @@ impl MethodItemMeta { Ok(if let Some(name) = name { name } else { - let name = inner.ident.to_string(); + let name = inner.item_name(); if magic { format!("__{}__", name) } else { @@ -241,24 +248,24 @@ impl PropertyItemMeta { Ok(if let Some(name) = name { name } else { - let sig_name = inner.ident.to_string(); + let sig_name = inner.item_name(); let name = if setter { if let Some(name) = sig_name.strip_prefix("set_") { if name.is_empty() { bail_span!( - &inner.ident, + &inner.meta_ident, "A #[{}(setter)] fn with a set_* name must \ have something after \"set_\"", - inner.parent_type + inner.meta_name() ) } name.to_string() } else { bail_span!( - &inner.ident, + &inner.meta_ident, "A #[{}(setter)] fn must either have a `name` \ parameter or a fn name along the lines of \"set_*\"", - inner.parent_type + inner.meta_name() ) } } else { @@ -572,7 +579,8 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result m, other => bail_span!(other, "#[pymodule] can only be on a module declaration"), }; + let fake_ident = Ident::new("pymodule", module_item.span()); let module_meta = - SimpleItemMeta::from_nested("pymodule", &module_item.ident, attr.into_iter())?; + SimpleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; let mut module_context = Module { name: module_meta.simple_name()?, items: ItemNursery::default(), @@ -260,8 +261,8 @@ impl ModuleItem for FunctionItem { }; let item_attr = attrs.remove(self.index()); let item_meta = SimpleItemMeta::from_nested( - "pyfunction", - &args.ident, + args.ident.clone(), + item_attr.get_ident().unwrap().clone(), item_attr.promoted_nested()?.into_iter(), )?; @@ -309,14 +310,10 @@ impl ModuleItem for ClassItem { )); } } - let static_name = match self.attr_name() { - "pyclass" => "pyclass", - "pystruct_sequence" => "pystruct_sequence", - _ => unreachable!(), - }; + let class_meta = ClassItemMeta::from_nested( - static_name, - &args.ident, + args.ident.clone(), + class_attr.get_ident().unwrap().clone(), class_attr.promoted_nested()?.into_iter(), )?; let module_name = args.module.name.clone(); @@ -328,8 +325,12 @@ impl ModuleItem for ClassItem { }; for attr_index in self.pyattrs.iter().rev() { let attr_attr = attrs.remove(*attr_index); - let nested = attr_attr.promoted_nested()?; - let item_meta = SimpleItemMeta::from_nested("pyattr", &args.ident, nested.into_iter())?; + let (meta_ident, nested) = attr_attr.ident_and_promoted_nested()?; + let item_meta = SimpleItemMeta::from_nested( + args.ident.clone(), + meta_ident.clone(), + nested.into_iter(), + )?; let py_name = item_meta .optional_name() @@ -356,8 +357,9 @@ impl ModuleItem for AttributeItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { let ident = &args.ident; let get_py_name = |attrs: &mut Vec| -> Result<_> { - let nested = attrs[self.inner.index].promoted_nested()?; - let item_meta = SimpleItemMeta::from_nested("pyattr", &ident, nested.into_iter())?; + let (meta_ident, nested) = attrs[self.inner.index].ident_and_promoted_nested()?; + let item_meta = + SimpleItemMeta::from_nested(ident.clone(), meta_ident.clone(), nested.into_iter())?; let py_name = item_meta.simple_name()?; Ok(py_name) }; diff --git a/derive/src/util.rs b/derive/src/util.rs index b9803d28b..7dff1113b 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -68,22 +68,22 @@ pub(crate) trait ContentItem { } pub(crate) struct ItemMetaInner { - pub ident: Ident, - pub parent_type: &'static str, - pub meta: HashMap, + pub item_ident: Ident, + pub meta_ident: Ident, + pub meta_map: HashMap, } impl ItemMetaInner { pub fn from_nested( - parent_type: &'static str, - ident: &Ident, + item_ident: Ident, + meta_ident: Ident, nested: I, allowed_names: &[&'static str], ) -> Result where I: std::iter::Iterator, { - let (named_map, lits) = nested.into_unique_map_and_lits(|path| { + let (meta_map, lits) = nested.into_unique_map_and_lits(|path| { if let Some(ident) = path.get_ident() { let name = ident.to_string(); if allowed_names.contains(&name.as_str()) { @@ -93,7 +93,7 @@ impl ItemMetaInner { ident, format!( "#[{}({})] is not one of allowed attributes {}", - parent_type, + meta_ident.to_string(), name, allowed_names.join(", ") ), @@ -105,20 +105,28 @@ impl ItemMetaInner { })?; if !lits.is_empty() { return Err(syn::Error::new_spanned( - ident, - format!("#[{}(..)] cannot contain literal", parent_type), + &meta_ident, + format!("#[{}(..)] cannot contain literal", meta_ident.to_string()), )); } Ok(Self { - ident: ident.clone(), - parent_type, - meta: named_map, + item_ident, + meta_ident, + meta_map, }) } + pub fn item_name(&self) -> String { + self.item_ident.to_string() + } + + pub fn meta_name(&self) -> String { + self.meta_ident.to_string() + } + pub fn _optional_str(&self, key: &str) -> Result> { - let value = if let Some((_, meta)) = self.meta.get(key) { + let value = if let Some((_, meta)) = self.meta_map.get(key) { match meta { Meta::NameValue(syn::MetaNameValue { lit: syn::Lit::Str(lit), @@ -129,7 +137,8 @@ impl ItemMetaInner { other, format!( "#[{}({} = ...)] must exist as a string", - self.parent_type, key + self.meta_name(), + key ), )); } @@ -141,7 +150,7 @@ impl ItemMetaInner { } pub fn _bool(&self, key: &str) -> Result { - let value = if let Some((_, meta)) = self.meta.get(key) { + let value = if let Some((_, meta)) = self.meta_map.get(key) { match meta { Meta::NameValue(syn::MetaNameValue { lit: syn::Lit::Bool(lit), @@ -151,7 +160,7 @@ impl ItemMetaInner { other => { return Err(syn::Error::new_spanned( other, - format!("#[{}({})] is expected", self.parent_type, key), + format!("#[{}({})] is expected", self.meta_name(), key), )) } } @@ -165,13 +174,13 @@ impl ItemMetaInner { pub(crate) trait ItemMeta: Sized { const ALLOWED_NAMES: &'static [&'static str]; - fn from_nested(parent_type: &'static str, ident: &Ident, nested: I) -> Result + fn from_nested(item_ident: Ident, meta_ident: Ident, nested: I) -> Result where I: std::iter::Iterator, { Ok(Self::from_inner(ItemMetaInner::from_nested( - parent_type, - ident, + item_ident, + meta_ident, nested, Self::ALLOWED_NAMES, )?)) @@ -184,7 +193,7 @@ pub(crate) trait ItemMeta: Sized { let inner = self.inner(); Ok(inner ._optional_str("name")? - .unwrap_or_else(|| inner.ident.to_string())) + .unwrap_or_else(|| inner.item_name())) } fn optional_name(&self) -> Option { @@ -221,22 +230,22 @@ impl ClassItemMeta { pub fn class_name(&self) -> Result { const KEY: &str = "name"; let inner = self.inner(); - let value = if let Some((_, meta)) = inner.meta.get(KEY) { + let value = if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { lit: syn::Lit::Str(lit), .. }) => Some(lit.value()), - Meta::Path(_) => Some(inner.ident.to_string()), + Meta::Path(_) => Some(inner.item_name()), _ => None, } } else { None }.ok_or_else(|| syn::Error::new_spanned( - &inner.ident, + &inner.meta_ident, format!( "#[{attr_name}(name = ...)] must exist as a string. Try #[{attr_name}(name)] to use rust type name.", - attr_name=inner.parent_type + attr_name=inner.meta_name() ), ))?; Ok(value) @@ -245,7 +254,7 @@ impl ClassItemMeta { pub fn module(&self) -> Result> { const KEY: &str = "module"; let inner = self.inner(); - let value = if let Some((_, meta)) = inner.meta.get(KEY) { + let value = if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { lit: syn::Lit::Str(lit), @@ -262,12 +271,12 @@ impl ClassItemMeta { other => Err(other.span()), } } else { - Err(inner.ident.span()) + Err(inner.item_ident.span()) }.map_err(|span| syn::Error::new( span, format!( "#[{attr_name}(module = ...)] must exist as a string or false. Try #[{attr_name}(module=false)] for built-in types.", - attr_name=inner.parent_type + attr_name=inner.meta_name() ), ))?; Ok(value) @@ -277,10 +286,10 @@ impl ClassItemMeta { let inner = self.inner(); let value = self.module().ok().flatten(). ok_or_else(|| syn::Error::new_spanned( - &inner.ident, + &inner.meta_ident, format!( "#[{attr_name}(module = ...)] must exist as a string. Built-in module is not allowed here.", - attr_name=inner.parent_type + attr_name=inner.meta_name() ), ))?; Ok(value) @@ -293,6 +302,7 @@ pub(crate) fn path_eq(path: &Path, s: &str) -> bool { pub(crate) trait AttributeExt: SynAttributeExt { fn promoted_nested(&self) -> Result; + fn ident_and_promoted_nested(&self) -> Result<(&Ident, PunctuatedNestedMeta)>; fn try_remove_name(&mut self, name: &str) -> Result>; fn fill_nested_meta(&mut self, name: &str, new_item: F) -> Result<()> where @@ -315,6 +325,9 @@ impl AttributeExt for Attribute { })?; Ok(list.nested) } + fn ident_and_promoted_nested(&self) -> Result<(&Ident, PunctuatedNestedMeta)> { + Ok((self.get_ident().unwrap(), self.promoted_nested()?)) + } fn try_remove_name(&mut self, item_name: &str) -> Result> { self.try_meta_mut(|meta| { From e4fa05ae70f54ee51ab4968f6e2897d88ab4634d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 16:13:44 +0900 Subject: [PATCH 06/13] attrs replacement technique --- derive/src/pyclass.rs | 4 +- derive/src/pymodule.rs | 159 ++++++++++++++++++----------------------- 2 files changed, 72 insertions(+), 91 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 089e1ab12..65b6e96b8 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -7,8 +7,8 @@ use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::{quote, quote_spanned, ToTokens}; use std::collections::HashMap; use syn::{ - parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Index, Item, Lit, Meta, - NestedMeta, + parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, + Meta, NestedMeta, TraitItem, }; use syn_ext::types::*; diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index e0f5ba883..6c6272b5b 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -10,7 +10,7 @@ use syn_ext::ext::*; struct Module { name: String, - items: ItemNursery, + module_extend_items: ItemNursery, } pub fn impl_pymodule( @@ -26,30 +26,37 @@ pub fn impl_pymodule( SimpleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; let mut module_context = Module { name: module_meta.simple_name()?, - items: ItemNursery::default(), + module_extend_items: ItemNursery::default(), }; - let content = module_item.unbraced_content_mut()?; + let items = module_item.unbraced_content_mut()?; - for item in content.iter_mut() { - let (attrs, ident) = if let Some(v) = item_declaration(item) { - v + let debug_attrs: Vec = vec![parse_quote!(#[RustPython derive bug!])]; + for item in items.iter_mut() { + let mut attrs = if let Ok(attrs) = item.attrs_mut() { + std::mem::replace(attrs, debug_attrs.clone()) } else { continue; }; - let (pyitems, cfgs) = attrs_to_items(attrs, new_item)?; - for pyitem in pyitems.iter().rev() { - pyitem.gen_module_item(ModuleItemArgs { - ident: ident.clone(), - item, - module: &mut module_context, - cfgs: cfgs.as_slice(), - })?; - } + let mut gen_module_item = || -> Result<()> { + let (pyitems, cfgs) = attrs_to_pyitems(&attrs, new_item)?; + for pyitem in pyitems.iter().rev() { + pyitem.gen_module_item(ModuleItemArgs { + item, + attrs: &mut attrs, + module: &mut module_context, + cfgs: cfgs.as_slice(), + })?; + } + Ok(()) + }; + let result = gen_module_item(); + let _ = std::mem::replace(item.attrs_mut().unwrap(), attrs); + result? } let module_name = module_context.name.as_str(); - let items = module_context.items; - content.extend(vec![ + let module_extend_items = module_context.module_extend_items; + items.extend(vec![ parse_quote! { pub(crate) const MODULE_NAME: &'static str = #module_name; }, @@ -58,7 +65,7 @@ pub fn impl_pymodule( vm: &::rustpython_vm::vm::VirtualMachine, module: &::rustpython_vm::pyobject::PyObjectRef, ) { - #items + #module_extend_items } }, parse_quote! { @@ -76,28 +83,6 @@ pub fn impl_pymodule( Ok(module_item.into_token_stream()) } -fn item_declaration(item: &syn::Item) -> Option<(&[syn::Attribute], Ident)> { - let (attrs, ident) = match &item { - Item::Fn(i @ syn::ItemFn { .. }) => (i.attrs.as_slice(), &i.sig.ident), - Item::Struct(syn::ItemStruct { attrs, ident, .. }) => (attrs.as_slice(), ident), - Item::Enum(syn::ItemEnum { attrs, ident, .. }) => (attrs.as_slice(), ident), - Item::Const(syn::ItemConst { attrs, ident, .. }) => (attrs.as_slice(), ident), - Item::Use(syn::ItemUse { attrs, tree, .. }) => { - let ident = match tree { - UseTree::Path(path) => match &*path.tree { - UseTree::Name(name) => &name.ident, - UseTree::Rename(rename) => &rename.rename, - _ => return None, - }, - _ => return None, - }; - (attrs.as_slice(), ident) - } - _ => return None, - }; - Some((attrs, ident.clone())) -} - fn new_item(index: usize, attr_name: String, pyattrs: Option>) -> Box { assert!(ALL_ALLOWED_NAMES.contains(&attr_name.as_str())); match attr_name.as_str() { @@ -115,7 +100,7 @@ fn new_item(index: usize, attr_name: String, pyattrs: Option>) -> Box } } -fn attrs_to_items(attrs: &[Attribute], new_item: F) -> Result<(Vec, Vec)> +fn attrs_to_pyitems(attrs: &[Attribute], new_item: F) -> Result<(Vec, Vec)> where F: Fn(usize, String, Option>) -> R, { @@ -231,8 +216,8 @@ impl ContentItem for AttributeItem { } struct ModuleItemArgs<'a> { - ident: Ident, - item: &'a mut Item, + item: &'a Item, + attrs: &'a mut Vec, module: &'a mut Module, cfgs: &'a [Attribute], } @@ -241,12 +226,6 @@ impl<'a> ModuleItemArgs<'a> { fn module_name(&'a self) -> &'a str { self.module.name.as_str() } - fn with_quote_args(&self, f: F) -> TokenStream - where - F: Fn(&Ident, &str) -> TokenStream, - { - f(&self.ident, self.module_name()) - } } trait ModuleItem: ContentItem { @@ -255,29 +234,31 @@ trait ModuleItem: ContentItem { impl ModuleItem for FunctionItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let attrs = match args.item { - Item::Fn(syn::ItemFn { ref mut attrs, .. }) => attrs, + let ident = match args.item { + Item::Fn(syn::ItemFn { sig, .. }) => sig.ident.clone(), other => return Err(self.new_syn_error(other.span(), "can only be on a function")), }; - let item_attr = attrs.remove(self.index()); + + let item_attr = args.attrs.remove(self.index()); let item_meta = SimpleItemMeta::from_nested( - args.ident.clone(), + ident.clone(), item_attr.get_ident().unwrap().clone(), item_attr.promoted_nested()?.into_iter(), )?; let py_name = item_meta.simple_name()?; - let item = args.with_quote_args(|ident, module| { + let item = { + let module = args.module_name(); let new_func = quote_spanned!( - args.ident.span() => vm.ctx.new_function_named(#ident, #module.to_owned(), #py_name.to_owned()) + ident.span() => vm.ctx.new_function_named(#ident, #module.to_owned(), #py_name.to_owned()) ); quote! { vm.__module_set_attr(&module, #py_name, #new_func).unwrap(); } - }); + }; args.module - .items + .module_extend_items .add_item(py_name, args.cfgs.to_vec(), item)?; Ok(()) } @@ -285,17 +266,13 @@ impl ModuleItem for FunctionItem { impl ModuleItem for ClassItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let attrs = match args.item { - Item::Struct(syn::ItemStruct { ref mut attrs, .. }) => attrs, - Item::Enum(syn::ItemEnum { ref mut attrs, .. }) => attrs, - other => { - return Err( - self.new_syn_error(other.span(), "can only be on a struct or enum declaration") - ); - } + let ident = match args.item { + Item::Struct(syn::ItemStruct { ident, .. }) => ident.clone(), + Item::Enum(syn::ItemEnum { ident, .. }) => ident.clone(), + other => return Err(self.new_syn_error(other.span(), "can only be on a function")), }; let (module_name, class_name) = { - let class_attr = &mut attrs[self.inner.index]; + let class_attr = &mut args.attrs[self.inner.index]; if self.pyattrs.is_empty() { // check noattr before ClassItemMeta::from_nested let noattr = class_attr.try_remove_name("noattr")?; @@ -312,7 +289,7 @@ impl ModuleItem for ClassItem { } let class_meta = ClassItemMeta::from_nested( - args.ident.clone(), + ident.clone(), class_attr.get_ident().unwrap().clone(), class_attr.promoted_nested()?.into_iter(), )?; @@ -324,18 +301,14 @@ impl ModuleItem for ClassItem { (module_name, class_name) }; for attr_index in self.pyattrs.iter().rev() { - let attr_attr = attrs.remove(*attr_index); + let attr_attr = args.attrs.remove(*attr_index); let (meta_ident, nested) = attr_attr.ident_and_promoted_nested()?; - let item_meta = SimpleItemMeta::from_nested( - args.ident.clone(), - meta_ident.clone(), - nested.into_iter(), - )?; + let item_meta = + SimpleItemMeta::from_nested(ident.clone(), meta_ident.clone(), nested.into_iter())?; let py_name = item_meta .optional_name() .unwrap_or_else(|| class_name.clone()); - let ident = &args.ident; let new_class = quote_spanned!(ident.span() => #ident::make_class(&vm.ctx); ); @@ -346,7 +319,7 @@ impl ModuleItem for ClassItem { }; args.module - .items + .module_extend_items .add_item(py_name.clone(), args.cfgs.to_vec(), item)?; } Ok(()) @@ -355,39 +328,47 @@ impl ModuleItem for ClassItem { impl ModuleItem for AttributeItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let ident = &args.ident; - let get_py_name = |attrs: &mut Vec| -> Result<_> { + let get_py_name = |attrs: &mut Vec, ident: &Ident| -> Result<_> { let (meta_ident, nested) = attrs[self.inner.index].ident_and_promoted_nested()?; let item_meta = SimpleItemMeta::from_nested(ident.clone(), meta_ident.clone(), nested.into_iter())?; let py_name = item_meta.simple_name()?; Ok(py_name) }; - let (attrs, py_name, tokens) = match args.item { - Item::Fn(syn::ItemFn { ref mut attrs, .. }) => { - let py_name = get_py_name(attrs)?; + let (py_name, tokens) = match args.item { + Item::Fn(syn::ItemFn { sig, .. }) => { + let ident = &sig.ident; + let py_name = get_py_name(args.attrs, &ident)?; ( - attrs, py_name.clone(), quote! { vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident(vm))).unwrap(); }, ) } - Item::Const(syn::ItemConst { ref mut attrs, .. }) => { - let py_name = get_py_name(attrs)?; + Item::Const(syn::ItemConst { ident, .. }) => { + let py_name = get_py_name(args.attrs, &ident)?; ( - attrs, py_name.clone(), quote! { vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident)).unwrap(); }, ) } - Item::Use(syn::ItemUse { ref mut attrs, .. }) => { - let py_name = get_py_name(attrs)?; + Item::Use(syn::ItemUse { + tree: UseTree::Path(path), + .. + }) => { + let ident = match &*path.tree { + UseTree::Name(name) => &name.ident, + UseTree::Rename(rename) => &rename.rename, + other => { + return Err(self.new_syn_error(other.span(), "can only be a simple use")); + } + }; + + let py_name = get_py_name(args.attrs, &ident)?; ( - attrs, py_name.clone(), quote! { vm.__module_set_attr(&module, #py_name, vm.new_pyobj(#ident)).unwrap(); @@ -400,10 +381,10 @@ impl ModuleItem for AttributeItem { ) } }; - attrs.remove(self.index()); + args.attrs.remove(self.index()); args.module - .items + .module_extend_items .add_item(py_name, args.cfgs.to_vec(), tokens)?; Ok(()) From 2458307dc3a04116915928d68912437346270214 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sat, 22 Aug 2020 18:06:40 +0900 Subject: [PATCH 07/13] rewrite pyclass --- Cargo.lock | 4 +- derive/Cargo.toml | 2 +- derive/src/error.rs | 24 +- derive/src/pyclass.rs | 1119 ++++++++++++++++++++++------------------ derive/src/pymodule.rs | 43 +- derive/src/util.rs | 36 +- 6 files changed, 691 insertions(+), 537 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95d9ea975..b3830d05b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1691,9 +1691,9 @@ dependencies = [ [[package]] name = "syn-ext" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba44d62b41442cbed900f2f3a39b3d8240b56bd34fcc574bc684bfd43a1a8c9" +checksum = "48de58fdf4f8294294c09653384dcb02e8e90b10d61cc0cf554bfbc2e9eaad9a" dependencies = [ "syn", ] diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 8bd5ba717..c3b0e4306 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -12,7 +12,7 @@ proc-macro = true [dependencies] syn = { version = "1.0", features = ["full"] } -syn-ext = { version = "0.1", features = ["full"] } +syn-ext = { version = "0.2", features = ["full"] } quote = "1.0" proc-macro2 = "1.0" rustpython-compiler = { path = "../compiler", version = "0.1.1" } diff --git a/derive/src/error.rs b/derive/src/error.rs index 34b0fc44a..784f033e1 100644 --- a/derive/src/error.rs +++ b/derive/src/error.rs @@ -43,19 +43,19 @@ macro_rules! bail_span { ) } -macro_rules! push_err_span { - ($diagnostics:expr, $($t:tt)*) => { - $diagnostics.push(err_span!($($t)*)) - }; -} +// macro_rules! push_err_span { +// ($diagnostics:expr, $($t:tt)*) => { +// $diagnostics.push(err_span!($($t)*)) +// }; +// } -macro_rules! push_diag_result { - ($diags:expr, $x:expr $(,)?) => { - if let Err(e) = $x { - $diags.push(e); - } - }; -} +// macro_rules! push_diag_result { +// ($diags:expr, $x:expr $(,)?) => { +// if let Err(e) = $x { +// $diags.push(e); +// } +// }; +// } #[derive(Debug)] pub struct Diagnostic { diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 65b6e96b8..1fb6c8854 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,518 +1,104 @@ use super::Diagnostic; use crate::util::{ - path_eq, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ItemIdent, ItemMeta, - ItemMetaInner, + path_eq, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, + ItemMetaInner, ItemNursery, ALL_ALLOWED_NAMES, }; -use proc_macro2::{Span, TokenStream as TokenStream2}; +use proc_macro2::TokenStream; use quote::{quote, quote_spanned, ToTokens}; use std::collections::HashMap; use syn::{ - parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, - Meta, NestedMeta, TraitItem, + parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Index, Item, Lit, Meta, + NestedMeta, Result, }; -use syn_ext::types::*; - -struct MethodItem { - pub inner: ContentItemInner, -} -struct PropertyItem { - pub inner: ContentItemInner, -} - -impl ContentItem for MethodItem { - fn inner(&self) -> &ContentItemInner { - &self.inner - } -} -impl ContentItem for PropertyItem { - fn inner(&self) -> &ContentItemInner { - &self.inner - } -} +use syn_ext::ext::*; #[derive(Default)] -struct Class { - // Unlike pymodule, meta variant is not supported - items: HashMap, +struct ImplContext { + impl_extend_items: ItemNursery, + getset_items: GetSetNursery, } -#[derive(PartialEq, Eq, Hash)] -enum ClassItem { - Method { - item_ident: Ident, - py_name: String, - }, - ClassMethod { - item_ident: Ident, - py_name: String, - }, - Property { - item_ident: Ident, - py_name: String, - setter: bool, - }, - Slot { - slot_ident: Ident, - item_ident: Ident, - }, -} - -impl ClassItem { - fn name(&self) -> String { - use ClassItem::*; - match self { - Method { py_name, .. } => py_name.clone(), - ClassMethod { py_name, .. } => py_name.clone(), - Property { - py_name, setter, .. - } => { - if *setter { - format!("{}.setter", py_name) - } else { - py_name.clone() - } - } - Slot { slot_ident, .. } => format!("#slot({})", slot_ident), - } - } -} - -impl Class { - fn add_item(&mut self, item: ClassItem, span: Span) -> Result<(), Diagnostic> { - if let Some(existing) = self.items.insert(item.name(), item) { - Err(Diagnostic::span_error( - span, - format!("Duplicate #[py*] attribute on pyimpl: {}", existing.name()), - )) - } else { - Ok(()) - } - } - - fn extract_method( - item: &Ident, - (meta, nested): (&Ident, PunctuatedNestedMeta), - ) -> Result { - let item_meta = - MethodItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; - Ok(ClassItem::Method { - item_ident: item.clone(), - py_name: item_meta.method_name()?, - }) - } - - fn extract_classmethod( - item: &Ident, - (meta, nested): (&Ident, PunctuatedNestedMeta), - ) -> Result { - let item_meta = - MethodItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; - Ok(ClassItem::ClassMethod { - item_ident: item.clone(), - py_name: item_meta.method_name()?, - }) - } - - fn extract_property( - item: &Ident, - (meta, nested): (&Ident, PunctuatedNestedMeta), - ) -> Result { - let item_meta = - PropertyItemMeta::from_nested(item.clone(), meta.clone(), nested.into_iter())?; - Ok(ClassItem::Property { - item_ident: item.clone(), - py_name: item_meta.property_name()?, - setter: item_meta.setter()?, - }) - } - - fn extract_slot(ident: &Ident, nested: PunctuatedNestedMeta) -> Result { - let pyslot_err = "#[pyslot] must be of the form #[pyslot] or #[pyslot(slotname)]"; - if nested.len() > 1 { - return Err(Diagnostic::spanned_error("e!(#nested), pyslot_err)); - } - let slot_ident = if nested.is_empty() { - let ident_str = ident.to_string(); - if let Some(stripped) = ident_str.strip_prefix("tp_") { - proc_macro2::Ident::new(stripped, ident.span()) - } else { - ident.clone() - } - } else { - match nested.into_iter().next().unwrap() { - NestedMeta::Meta(Meta::Path(path)) => path - .get_ident() - .cloned() - .ok_or_else(|| err_span!(path, "{}", pyslot_err))?, - bad => bail_span!(bad, "{}", pyslot_err), - } - }; - Ok(ClassItem::Slot { - slot_ident, - item_ident: ident.clone(), - }) - } - - fn extract_item_from_syn(&mut self, item: &mut ItemIdent) -> Result<(), Diagnostic> { - let mut attr_idxs = Vec::new(); - for (i, attr) in item.attrs.iter_mut().enumerate() { - let name = match attr.path.get_ident() { - Some(name) => name, - None => continue, - }; - - let item = match name.to_string().as_str() { - "pymethod" => Self::extract_method(item.ident, attr.ident_and_promoted_nested()?)?, - "pyclassmethod" => { - Self::extract_classmethod(item.ident, attr.ident_and_promoted_nested()?)? - } - "pyproperty" => { - Self::extract_property(item.ident, attr.ident_and_promoted_nested()?)? - } - "pyslot" => Self::extract_slot(item.ident, attr.promoted_nested()?)?, - _ => { - continue; - } - }; - self.add_item(item, attr.span())?; - attr_idxs.push(i); - } - let mut i = 0; - let mut attr_idxs = &*attr_idxs; - item.attrs.retain(|_| { - let drop = attr_idxs.first().copied() == Some(i); - if drop { - attr_idxs = &attr_idxs[1..]; - } - i += 1; - !drop - }); - for (i, idx) in attr_idxs.iter().enumerate() { - item.attrs.remove(idx - i); - } - Ok(()) - } -} - -struct MethodItemMeta(ItemMetaInner); - -impl ItemMeta for MethodItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic"]; - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl MethodItemMeta { - fn method_name(&self) -> Result { - let inner = self.inner(); - let name = inner._optional_str("name")?; - let magic = inner._bool("magic")?; - Ok(if let Some(name) = name { - name - } else { - let name = inner.item_name(); - if magic { - format!("__{}__", name) - } else { - name - } - }) - } -} - -struct PropertyItemMeta(ItemMetaInner); - -impl ItemMeta for PropertyItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic", "setter"]; - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl PropertyItemMeta { - fn property_name(&self) -> Result { - let inner = self.inner(); - let magic = inner._bool("magic")?; - let setter = inner._bool("setter")?; - let name = inner._optional_str("name")?; - - Ok(if let Some(name) = name { - name - } else { - let sig_name = inner.item_name(); - let name = if setter { - if let Some(name) = sig_name.strip_prefix("set_") { - if name.is_empty() { - bail_span!( - &inner.meta_ident, - "A #[{}(setter)] fn with a set_* name must \ - have something after \"set_\"", - inner.meta_name() - ) - } - name.to_string() - } else { - bail_span!( - &inner.meta_ident, - "A #[{}(setter)] fn must either have a `name` \ - parameter or a fn name along the lines of \"set_*\"", - inner.meta_name() - ) - } - } else { - sig_name - }; - if magic { - format!("__{}__", name) - } else { - name - } - }) - } - - fn setter(&self) -> syn::Result { - self.inner()._bool("setter") - } -} - -fn extract_impl_items(mut items: Vec) -> Result { - let mut diagnostics: Vec = Vec::new(); - - let mut class = Class::default(); - - for item in items.iter_mut() { - push_diag_result!(diagnostics, class.extract_item_from_syn(item),); - } - - let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); - for item in class.items.values() { - if let ClassItem::Property { - ref item_ident, - ref py_name, - setter, - } = item - { - let entry = properties.entry(py_name).or_default(); - let func = if *setter { &mut entry.1 } else { &mut entry.0 }; - if func.is_some() { - bail_span!( - item_ident, - "Multiple property accessors with name {:?}", - py_name - ) - } - *func = Some(item_ident); - } - } - let properties = properties - .into_iter() - .map(|(name, prop)| { - let getter_func = match prop.0 { - Some(func) => func, - None => { - push_err_span!( - diagnostics, - prop.1.unwrap(), - "Property {:?} is missing a getter", - name - ); - return TokenStream2::new(); - } - }; - let (new, setter) = match prop.1 { - Some(func) => (quote! { with_get_set }, quote! { , &Self::#func }), - None => (quote! { with_get }, quote! { }), - }; - let str_name = name.to_string(); - quote! { - class.set_str_attr( - #name, - ::rustpython_vm::pyobject::PyObject::new( - ::rustpython_vm::obj::objgetset::PyGetSet::#new(#str_name.into(), &Self::#getter_func #setter), - ctx.getset_type(), None) - ); - } - }) - .collect::>(); - let methods = class.items.values().filter_map(|item| match item { - ClassItem::Method { - item_ident, - py_name, - } => { - let new_meth = quote_spanned!(item_ident.span()=> .new_method(Self::#item_ident)); - Some(quote! { - class.set_str_attr(#py_name, ctx#new_meth); - }) - } - ClassItem::ClassMethod { - item_ident, - py_name, - } => { - let new_meth = quote_spanned!(item_ident.span()=> .new_classmethod(Self::#item_ident)); - Some(quote! { - class.set_str_attr(#py_name, ctx#new_meth); - }) - } - ClassItem::Slot { - slot_ident, - item_ident, - } => { - let transform = if vec!["new", "call"].contains(&slot_ident.to_string().as_str()) { - quote! { ::rustpython_vm::function::IntoPyNativeFunc::into_func } - } else { - quote! { ::std::boxed::Box::new } - }; - let into_func = quote_spanned! {item_ident.span()=> - #transform(Self::#item_ident) - }; - Some(quote! { - (*class.slots.write()).#slot_ident = Some(#into_func); - }) - } - _ => None, - }); - - Diagnostic::from_vec(diagnostics)?; - - Ok(quote! { - #(#methods)* - #(#properties)* - }) -} - -fn extract_impl_attrs(attr: AttributeArgs) -> Result<(TokenStream2, TokenStream2), Diagnostic> { - let mut withs = Vec::new(); - let mut flags = vec![quote! { ::rustpython_vm::slots::PyTpFlags::DEFAULT.bits() }]; - #[cfg(debug_assertions)] - { - flags.push(quote! { - | ::rustpython_vm::slots::PyTpFlags::_CREATED_WITH_FLAGS.bits() - }); - } - - for attr in attr { - match attr { - NestedMeta::Meta(Meta::List(syn::MetaList { path, nested, .. })) => { - if path_eq(&path, "with") { - for meta in nested { - match meta { - NestedMeta::Meta(Meta::Path(path)) => { - withs.push(quote! { - ::__extend_py_class(ctx, class); - }); - } - meta => { - bail_span!(meta, "#[pyimpl(with(...))] arguments should be paths") - } - } - } - } else if path_eq(&path, "flags") { - for meta in nested { - match meta { - NestedMeta::Meta(Meta::Path(path)) => { - if let Some(ident) = path.get_ident() { - flags.push(quote! { - | ::rustpython_vm::slots::PyTpFlags::#ident.bits() - }); - } else { - bail_span!( - path, - "#[pyimpl(flags(...))] arguments should be ident" - ) - } - } - meta => { - bail_span!(meta, "#[pyimpl(flags(...))] arguments should be ident") - } - } - } - } else { - bail_span!(path, "Unknown pyimpl attribute") - } - } - attr => bail_span!(attr, "Unknown pyimpl attribute"), - } - } - - Ok(( - quote! { - #(#withs)* - }, - quote! { - #(#flags)* - }, - )) -} - -pub fn impl_pyimpl(attr: AttributeArgs, item: Item) -> Result { +pub(crate) fn impl_pyimpl( + attr: AttributeArgs, + item: Item, +) -> std::result::Result { match item { Item::Impl(mut imp) => { - let items = imp - .items - .iter_mut() - .filter_map(|item| match item { - syn::ImplItem::Method(syn::ImplItemMethod { attrs, sig, .. }) => { - Some(ItemIdent { + let mut context = ImplContext::default(); + for item in imp.items.iter_mut() { + item.try_split_attr_mut(|attrs, item| { + let (pyitems, cfgs) = + attrs_to_content_items(&attrs, new_impl_item::)?; + for pyitem in pyitems.iter().rev() { + pyitem.gen_impl_item(ImplItemArgs { + item, attrs, - ident: &sig.ident, - }) + context: &mut context, + cfgs: cfgs.as_slice(), + })?; } - _ => None, - }) - .collect(); - let extend_impl = extract_impl_items(items)?; - let (with_impl, flags) = extract_impl_attrs(attr)?; - let ty = &imp.self_ty; - let ret = quote! { - #imp - impl ::rustpython_vm::pyobject::PyClassImpl for #ty { - const TP_FLAGS: ::rustpython_vm::slots::PyTpFlags = ::rustpython_vm::slots::PyTpFlags::from_bits_truncate(#flags); + Ok(()) + })?; + } + context.getset_items.validate()?; - fn impl_extend_class( + let tokens = { + let (with_impl, flags) = extract_impl_attrs(attr)?; + let ty = &imp.self_ty; + let getset_impl = &context.getset_items; + let extend_impl = &context.impl_extend_items; + quote! { + #imp + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + const TP_FLAGS: ::rustpython_vm::slots::PyTpFlags = ::rustpython_vm::slots::PyTpFlags::from_bits_truncate(#flags); + + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #getset_impl + #extend_impl + #with_impl + } + } + } + }; + Ok(tokens) + } + Item::Trait(mut trai) => { + let mut context = ImplContext::default(); + for item in trai.items.iter_mut() { + item.try_split_attr_mut(|attrs, item| { + let (pyitems, cfgs) = + attrs_to_content_items(&attrs, new_impl_item::)?; + for pyitem in pyitems.iter().rev() { + pyitem.gen_impl_item(ImplItemArgs { + item, + attrs, + context: &mut context, + cfgs: cfgs.as_slice(), + })?; + } + Ok(()) + })?; + } + context.getset_items.validate()?; + + let tokens = { + let getset_impl = &context.getset_items; + let extend_impl = &context.impl_extend_items; + trai.items.push(parse_quote! { + fn __extend_py_class( ctx: &::rustpython_vm::pyobject::PyContext, class: &::rustpython_vm::obj::objtype::PyClassRef, ) { + #getset_impl #extend_impl - #with_impl } - } + }); + trai.into_token_stream() }; - Ok(ret) - } - Item::Trait(mut trai) => { - let items = trai - .items - .iter_mut() - .filter_map(|item| match item { - syn::TraitItem::Method(syn::TraitItemMethod { attrs, sig, .. }) => { - Some(ItemIdent { - attrs, - ident: &sig.ident, - }) - } - _ => None, - }) - .collect(); - let extend_impl = extract_impl_items(items)?; - let item = parse_quote! { - fn __extend_py_class( - ctx: &::rustpython_vm::pyobject::PyContext, - class: &::rustpython_vm::obj::objtype::PyClassRef, - ) { - #extend_impl - } - }; - trai.items.push(item); - Ok(trai.into_token_stream()) + Ok(tokens) } item => Ok(quote!(#item)), } @@ -523,7 +109,7 @@ fn generate_class_def( name: &str, module_name: Option<&str>, attrs: &[Attribute], -) -> Result { +) -> std::result::Result { let mut doc: Option> = None; for attr in attrs.iter() { if attr.path.is_ident("doc") { @@ -552,13 +138,12 @@ fn generate_class_def( } else { name.to_owned() }; - let module_name = match module_name { Some(v) => quote!(Some(#v) ), None => quote!(None), }; - let ret = quote! { + let tokens = quote! { impl ::rustpython_vm::pyobject::PyClassDef for #ident { const NAME: &'static str = #name; const MODULE_NAME: Option<&'static str> = #module_name; @@ -566,10 +151,13 @@ fn generate_class_def( const DOC: Option<&'static str> = #doc; } }; - Ok(ret) + Ok(tokens) } -pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result { +pub(crate) fn impl_pyclass( + attr: AttributeArgs, + item: Item, +) -> std::result::Result { let (ident, attrs) = match &item { Item::Struct(syn::ItemStruct { ident, attrs, .. }) => (ident, attrs), Item::Enum(syn::ItemEnum { ident, attrs, .. }) => (ident, attrs), @@ -592,7 +180,10 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result { +pub(crate) fn impl_pystruct_sequence( + attr: AttributeArgs, + item: Item, +) -> std::result::Result { let struc = if let Item::Struct(struc) = item { struc } else { @@ -683,3 +274,533 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result &ContentItemInner { + &self.inner + } +} +impl ContentItem for PropertyItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} +impl ContentItem for SlotItem { + fn inner(&self) -> &ContentItemInner { + &self.inner + } +} + +struct ImplItemArgs<'a, Item: ItemLike> { + item: &'a Item, + attrs: &'a mut Vec, + context: &'a mut ImplContext, + cfgs: &'a [Attribute], +} + +trait ImplItem: ContentItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()>; +} + +impl ImplItem for MethodItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let ident = if args.item.is_function_or_method() { + Ok(args.item.get_ident().unwrap()) + } else { + Err(self.new_syn_error(args.item.span(), "can only be on a method")) + }?; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = MethodItemMeta::from_nested( + ident.clone(), + item_attr.get_ident().unwrap().clone(), + item_attr.promoted_nested()?.into_iter(), + )?; + + let py_name = item_meta.method_name()?; + let new_func = Ident::new(&format!("new_{}", &self.method_type), args.item.span()); + let tokens = { + let new_func = quote_spanned!( + ident.span() => .#new_func(Self::#ident) + ); + quote! { + class.set_str_attr(#py_name, ctx#new_func); + } + }; + + args.context + .impl_extend_items + .add_item(py_name, args.cfgs.to_vec(), tokens)?; + Ok(()) + } +} + +impl ImplItem for PropertyItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> + where + Item: ItemLike + ToTokens + GetIdent, + { + let ident = if args.item.is_function_or_method() { + Ok(args.item.get_ident().unwrap()) + } else { + Err(self.new_syn_error(args.item.span(), "can only be on a method")) + }?; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = PropertyItemMeta::from_nested( + ident.clone(), + item_attr.get_ident().unwrap().clone(), + item_attr.promoted_nested()?.into_iter(), + )?; + + let py_name = item_meta.property_name()?; + let setter = item_meta.setter()?; + args.context + .getset_items + .add_item(py_name, args.cfgs.to_vec(), setter, ident.clone())?; + Ok(()) + } +} + +impl ImplItem for SlotItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> + where + Item: ItemLike + ToTokens + GetIdent, + { + let ident = if args.item.is_function_or_method() { + Ok(args.item.get_ident().unwrap()) + } else { + Err(self.new_syn_error(args.item.span(), "can only be on a method")) + }?; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = SlotItemMeta::from_nested( + ident.clone(), + item_attr.get_ident().unwrap().clone(), + item_attr.promoted_nested()?.into_iter(), + )?; + + let slot_ident = item_meta.slot_name()?; + let slot_name = slot_ident.to_string(); + let tokens = { + let transform = if vec!["new", "call"].contains(&slot_name.as_str()) { + quote! { ::rustpython_vm::function::IntoPyNativeFunc::into_func } + } else { + quote! { ::std::boxed::Box::new } + }; + let into_func = quote_spanned! {ident.span() => + #transform(Self::#ident) + }; + quote! { + (*class.slots.write()).#slot_ident = Some(#into_func); + } + }; + + args.context.impl_extend_items.add_item( + format!("(slot {})", slot_name), + args.cfgs.to_vec(), + tokens, + )?; + + Ok(()) + } +} + +#[derive(Default)] +#[allow(clippy::type_complexity)] +struct GetSetNursery { + map: HashMap<(String, Vec), (Option, Option)>, + validated: bool, +} + +impl GetSetNursery { + fn add_item( + &mut self, + name: String, + cfgs: Vec, + setter: bool, + item_ident: Ident, + ) -> Result<()> { + assert!(!self.validated, "new item is not allowed after validation"); + if setter && !cfgs.is_empty() { + return Err(syn::Error::new_spanned( + item_ident, + "Property setter does not allow #[cfg]", + )); + } + let entry = self.map.entry((name.clone(), cfgs)).or_default(); + let func = if setter { &mut entry.1 } else { &mut entry.0 }; + if func.is_some() { + return Err(syn::Error::new_spanned( + item_ident, + format!("Multiple property accessors with name '{}'", name), + )); + } + *func = Some(item_ident); + Ok(()) + } + + fn validate(&mut self) -> Result<()> { + let mut errors = Vec::new(); + for ((name, _cfgs), (getter, setter)) in self.map.iter() { + if getter.is_none() { + errors.push(syn::Error::new_spanned( + setter.as_ref().unwrap(), + format!("Property '{}' is missing a getter", name), + )); + }; + } + if !errors.is_empty() { + return Err(errors.into_error()); + } + self.validated = true; + Ok(()) + } +} + +impl ToTokens for GetSetNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + assert!(self.validated, "Call `validate()` before token generation"); + tokens.extend(self.map.iter().map(|((name, cfgs), (getter, setter))| { + let (constructor, setter) = match setter { + Some(setter) => (quote! { with_get_set }, quote_spanned! { setter.span() => , &Self::#setter }), + None => (quote! { with_get }, quote! { }), + }; + quote! { + #( #cfgs )* + class.set_str_attr( + #name, + ::rustpython_vm::pyobject::PyObject::new( + ::rustpython_vm::obj::objgetset::PyGetSet::#constructor(#name.into(), &Self::#getter #setter), + ctx.getset_type(), None) + ); + } + })); + } +} + +struct MethodItemMeta(ItemMetaInner); + +impl ItemMeta for MethodItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl MethodItemMeta { + fn method_name(&self) -> Result { + let inner = self.inner(); + let name = inner._optional_str("name")?; + let magic = inner._bool("magic")?; + Ok(if let Some(name) = name { + name + } else { + let name = inner.item_name(); + if magic { + format!("__{}__", name) + } else { + name + } + }) + } +} + +struct PropertyItemMeta(ItemMetaInner); + +impl ItemMeta for PropertyItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic", "setter"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl PropertyItemMeta { + fn property_name(&self) -> Result { + let inner = self.inner(); + let magic = inner._bool("magic")?; + let setter = inner._bool("setter")?; + let name = inner._optional_str("name")?; + let py_name = if let Some(name) = name { + name + } else { + let sig_name = inner.item_name(); + let name = if setter { + if let Some(name) = sig_name.strip_prefix("set_") { + if name.is_empty() { + return Err(syn::Error::new_spanned( + &inner.meta_ident, + format!( + "A #[{}(setter)] fn with a set_* name must \ + have something after \"set_\"", + inner.meta_name() + ), + )); + } + name.to_string() + } else { + return Err(syn::Error::new_spanned( + &inner.meta_ident, + format!( + "A #[{}(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of \"set_*\"", + inner.meta_name() + ), + )); + } + } else { + sig_name + }; + if magic { + format!("__{}__", name) + } else { + name + } + }; + Ok(py_name) + } + + fn setter(&self) -> Result { + self.inner()._bool("setter") + } +} + +struct SlotItemMeta(ItemMetaInner); + +impl ItemMeta for SlotItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &[]; // not used + + fn from_nested(item_ident: Ident, meta_ident: Ident, mut nested: I) -> Result + where + I: std::iter::Iterator, + { + let meta_map = if let Some(nmeta) = nested.next() { + match nmeta { + NestedMeta::Meta(meta) => { + Some([("name".to_owned(), (0, meta))].iter().cloned().collect()) + } + _ => None, + } + } else { + Some(HashMap::default()) + }; + match (meta_map, nested.next()) { + (Some(meta_map), None) => Ok(Self::from_inner(ItemMetaInner { + item_ident, + meta_ident, + meta_map, + })), + _ => Err(syn::Error::new_spanned( + meta_ident, + "#[pyslot] must be of the form #[pyslot] or #[pyslot(slotname)]", + )), + } + } + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl SlotItemMeta { + fn slot_name(&self) -> Result { + let inner = self.inner(); + let slot_name = if let Some((_, meta)) = inner.meta_map.get("name") { + match meta { + Meta::Path(path) => path.get_ident().cloned(), + _ => None, + } + } else { + let ident_str = self.inner().item_name(); + let name = if let Some(stripped) = ident_str.strip_prefix("tp_") { + proc_macro2::Ident::new(stripped, inner.item_ident.span()) + } else { + inner.item_ident.clone() + }; + Some(name) + }; + slot_name.ok_or_else(|| { + syn::Error::new_spanned( + &inner.meta_ident, + "#[pyslot] must be of the form #[pyslot] or #[pyslot(slotname)]", + ) + }) + } +} + +fn extract_impl_attrs( + attr: AttributeArgs, +) -> std::result::Result<(TokenStream, TokenStream), Diagnostic> { + let mut withs = Vec::new(); + let mut flags = vec![quote! { ::rustpython_vm::slots::PyTpFlags::DEFAULT.bits() }]; + #[cfg(debug_assertions)] + { + flags.push(quote! { + | ::rustpython_vm::slots::PyTpFlags::_CREATED_WITH_FLAGS.bits() + }); + } + + for attr in attr { + match attr { + NestedMeta::Meta(Meta::List(syn::MetaList { path, nested, .. })) => { + if path_eq(&path, "with") { + for meta in nested { + match meta { + NestedMeta::Meta(Meta::Path(path)) => { + withs.push(quote! { + ::__extend_py_class(ctx, class); + }); + } + meta => { + bail_span!(meta, "#[pyimpl(with(...))] arguments should be paths") + } + } + } + } else if path_eq(&path, "flags") { + for meta in nested { + match meta { + NestedMeta::Meta(Meta::Path(path)) => { + if let Some(ident) = path.get_ident() { + flags.push(quote! { + | ::rustpython_vm::slots::PyTpFlags::#ident.bits() + }); + } else { + bail_span!( + path, + "#[pyimpl(flags(...))] arguments should be ident" + ) + } + } + meta => { + bail_span!(meta, "#[pyimpl(flags(...))] arguments should be ident") + } + } + } + } else { + bail_span!(path, "Unknown pyimpl attribute") + } + } + attr => bail_span!(attr, "Unknown pyimpl attribute"), + } + } + + Ok(( + quote! { + #(#withs)* + }, + quote! { + #(#flags)* + }, + )) +} + +fn new_impl_item(index: usize, attr_name: String) -> Box> +where + Item: ItemLike + ToTokens + GetIdent, +{ + assert!(ALL_ALLOWED_NAMES.contains(&attr_name.as_str())); + match attr_name.as_str() { + attr_name @ "pymethod" | attr_name @ "pyclassmethod" => Box::new(MethodItem { + inner: ContentItemInner { + index, + attr_name: attr_name.to_owned(), + }, + method_type: attr_name.strip_prefix("py").unwrap().to_owned(), + }), + "pyproperty" => Box::new(PropertyItem { + inner: ContentItemInner { index, attr_name }, + }), + "pyslot" => Box::new(SlotItem { + inner: ContentItemInner { index, attr_name }, + }), + other => unreachable!("#[pyimpl] doesn't accept #[{}]", other), + } +} + +fn attrs_to_content_items( + attrs: &[Attribute], + new_item: F, +) -> Result<(Vec, Vec)> +where + F: Fn(usize, String) -> R, +{ + let mut cfgs: Vec = Vec::new(); + let mut result = Vec::new(); + + let mut iter = attrs.iter().enumerate().peekable(); + while let Some((_, attr)) = iter.peek() { + // take all cfgs but no py items + let attr = *attr; + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + cfgs.push(attr.clone()); + } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + break; + } + iter.next(); + } + + for (i, attr) in iter { + // take py items but no cfgs + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + return Err(syn::Error::new_spanned( + attr, + "#[py*] items must be placed under `cfgs`", + )); + } + if !ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + continue; + } + + result.push(new_item(i, attr_name)); + } + Ok((result, cfgs)) +} diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index 6c6272b5b..c0187fa84 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -8,7 +8,7 @@ use quote::{quote, quote_spanned, ToTokens}; use syn::{parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Result, UseTree}; use syn_ext::ext::*; -struct Module { +struct ModuleContext { name: String, module_extend_items: ItemNursery, } @@ -24,36 +24,33 @@ pub fn impl_pymodule( let fake_ident = Ident::new("pymodule", module_item.span()); let module_meta = SimpleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; - let mut module_context = Module { + + // generation resources + let mut module_context = ModuleContext { name: module_meta.simple_name()?, module_extend_items: ItemNursery::default(), }; - let items = module_item.unbraced_content_mut()?; + let items = module_item.items_mut().ok_or_else(|| { + module_meta.new_meta_error("requires actual module, not a module declaration") + })?; - let debug_attrs: Vec = vec![parse_quote!(#[RustPython derive bug!])]; + // collect to context for item in items.iter_mut() { - let mut attrs = if let Ok(attrs) = item.attrs_mut() { - std::mem::replace(attrs, debug_attrs.clone()) - } else { - continue; - }; - let mut gen_module_item = || -> Result<()> { - let (pyitems, cfgs) = attrs_to_pyitems(&attrs, new_item)?; + item.try_split_attr_mut(|attrs, item| { + let (pyitems, cfgs) = attrs_to_module_items(&attrs, new_module_item)?; for pyitem in pyitems.iter().rev() { pyitem.gen_module_item(ModuleItemArgs { item, - attrs: &mut attrs, + attrs, module: &mut module_context, cfgs: cfgs.as_slice(), })?; } Ok(()) - }; - let result = gen_module_item(); - let _ = std::mem::replace(item.attrs_mut().unwrap(), attrs); - result? + })?; } + // append additional items let module_name = module_context.name.as_str(); let module_extend_items = module_context.module_extend_items; items.extend(vec![ @@ -83,7 +80,11 @@ pub fn impl_pymodule( Ok(module_item.into_token_stream()) } -fn new_item(index: usize, attr_name: String, pyattrs: Option>) -> Box { +fn new_module_item( + index: usize, + attr_name: String, + pyattrs: Option>, +) -> Box { assert!(ALL_ALLOWED_NAMES.contains(&attr_name.as_str())); match attr_name.as_str() { "pyfunction" => Box::new(FunctionItem { @@ -100,7 +101,7 @@ fn new_item(index: usize, attr_name: String, pyattrs: Option>) -> Box } } -fn attrs_to_pyitems(attrs: &[Attribute], new_item: F) -> Result<(Vec, Vec)> +fn attrs_to_module_items(attrs: &[Attribute], new_item: F) -> Result<(Vec, Vec)> where F: Fn(usize, String, Option>) -> R, { @@ -218,7 +219,7 @@ impl ContentItem for AttributeItem { struct ModuleItemArgs<'a> { item: &'a Item, attrs: &'a mut Vec, - module: &'a mut Module, + module: &'a mut ModuleContext, cfgs: &'a [Attribute], } @@ -269,7 +270,9 @@ impl ModuleItem for ClassItem { let ident = match args.item { Item::Struct(syn::ItemStruct { ident, .. }) => ident.clone(), Item::Enum(syn::ItemEnum { ident, .. }) => ident.clone(), - other => return Err(self.new_syn_error(other.span(), "can only be on a function")), + other => { + return Err(self.new_syn_error(other.span(), "can only be on a struct or enum")) + } }; let (module_name, class_name) = { let class_attr = &mut args.attrs[self.inner.index]; diff --git a/derive/src/util.rs b/derive/src/util.rs index 7dff1113b..250870b20 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -7,11 +7,13 @@ use syn_ext::types::PunctuatedNestedMeta; pub(crate) const ALL_ALLOWED_NAMES: &[&str] = &[ "pymethod", + "pyclassmethod", "pyproperty", "pyfunction", "pyclass", "pystruct_sequence", "pyattr", + "pyslot", ]; #[derive(Default)] @@ -199,6 +201,14 @@ pub(crate) trait ItemMeta: Sized { fn optional_name(&self) -> Option { self.inner()._optional_str("name").ok().flatten() } + + fn new_meta_error(&self, msg: &str) -> syn::Error { + let inner = self.inner(); + syn::Error::new_spanned( + &inner.meta_ident, + format!("#[{}] {}", inner.meta_name(), msg), + ) + } } pub(crate) struct SimpleItemMeta(pub ItemMetaInner); @@ -387,7 +397,27 @@ impl AttributeExt for Attribute { } } -pub(crate) struct ItemIdent<'a> { - pub attrs: &'a mut Vec, - pub ident: &'a Ident, +pub(crate) trait ErrorVec { + fn into_error(self) -> syn::Error; + fn ok_or_push(&mut self, r: Result) -> Option; +} + +impl ErrorVec for Vec { + fn into_error(self) -> syn::Error { + let mut iter = self.into_iter(); + let mut first = iter.next().expect("Cannot convert empty vec to syn::Error"); + for err in iter { + first.combine(err); + } + first + } + fn ok_or_push(&mut self, r: Result) -> Option { + match r { + Ok(v) => Some(v), + Err(e) => { + self.push(e); + None + } + } + } } From 29dc4dabbbcdb9a5a03a403b91eb78908c473bc2 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 05:25:17 +0900 Subject: [PATCH 08/13] multiple error tracking --- derive/src/pyclass.rs | 125 +++++++++++++++++++++-------------------- derive/src/pymodule.rs | 92 ++++++++++++++++++------------ derive/src/util.rs | 24 +++++--- 3 files changed, 138 insertions(+), 103 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 1fb6c8854..96c055e54 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -16,92 +16,99 @@ use syn_ext::ext::*; struct ImplContext { impl_extend_items: ItemNursery, getset_items: GetSetNursery, + errors: Vec, } pub(crate) fn impl_pyimpl( attr: AttributeArgs, item: Item, ) -> std::result::Result { - match item { + let mut context = ImplContext::default(); + let mut tokens = match item { Item::Impl(mut imp) => { - let mut context = ImplContext::default(); for item in imp.items.iter_mut() { - item.try_split_attr_mut(|attrs, item| { + let r = item.try_split_attr_mut(|attrs, item| { let (pyitems, cfgs) = attrs_to_content_items(&attrs, new_impl_item::)?; for pyitem in pyitems.iter().rev() { - pyitem.gen_impl_item(ImplItemArgs { + let r = pyitem.gen_impl_item(ImplItemArgs { item, attrs, context: &mut context, cfgs: cfgs.as_slice(), - })?; + }); + context.errors.ok_or_push(r); } Ok(()) - })?; + }); + context.errors.ok_or_push(r); } - context.getset_items.validate()?; + context.errors.ok_or_push(context.getset_items.validate()); - let tokens = { - let (with_impl, flags) = extract_impl_attrs(attr)?; - let ty = &imp.self_ty; - let getset_impl = &context.getset_items; - let extend_impl = &context.impl_extend_items; - quote! { - #imp - impl ::rustpython_vm::pyobject::PyClassImpl for #ty { - const TP_FLAGS: ::rustpython_vm::slots::PyTpFlags = ::rustpython_vm::slots::PyTpFlags::from_bits_truncate(#flags); + let (with_impl, flags) = extract_impl_attrs(attr)?; + let ty = &imp.self_ty; + let getset_impl = &context.getset_items; + let extend_impl = &context.impl_extend_items; + quote! { + #imp + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + const TP_FLAGS: ::rustpython_vm::slots::PyTpFlags = ::rustpython_vm::slots::PyTpFlags::from_bits_truncate(#flags); - fn impl_extend_class( - ctx: &::rustpython_vm::pyobject::PyContext, - class: &::rustpython_vm::obj::objtype::PyClassRef, - ) { - #getset_impl - #extend_impl - #with_impl - } - } - } - }; - Ok(tokens) - } - Item::Trait(mut trai) => { - let mut context = ImplContext::default(); - for item in trai.items.iter_mut() { - item.try_split_attr_mut(|attrs, item| { - let (pyitems, cfgs) = - attrs_to_content_items(&attrs, new_impl_item::)?; - for pyitem in pyitems.iter().rev() { - pyitem.gen_impl_item(ImplItemArgs { - item, - attrs, - context: &mut context, - cfgs: cfgs.as_slice(), - })?; - } - Ok(()) - })?; - } - context.getset_items.validate()?; - - let tokens = { - let getset_impl = &context.getset_items; - let extend_impl = &context.impl_extend_items; - trai.items.push(parse_quote! { - fn __extend_py_class( + fn impl_extend_class( ctx: &::rustpython_vm::pyobject::PyContext, class: &::rustpython_vm::obj::objtype::PyClassRef, ) { #getset_impl #extend_impl + #with_impl } + } + } + } + Item::Trait(mut trai) => { + let mut context = ImplContext::default(); + for item in trai.items.iter_mut() { + let r = item.try_split_attr_mut(|attrs, item| { + let (pyitems, cfgs) = + attrs_to_content_items(&attrs, new_impl_item::)?; + for pyitem in pyitems.iter().rev() { + let r = pyitem.gen_impl_item(ImplItemArgs { + item, + attrs, + context: &mut context, + cfgs: cfgs.as_slice(), + }); + context.errors.ok_or_push(r); + } + Ok(()) }); - trai.into_token_stream() - }; - Ok(tokens) + context.errors.ok_or_push(r); + } + context.errors.ok_or_push(context.getset_items.validate()); + + let getset_impl = &context.getset_items; + let extend_impl = &context.impl_extend_items; + trai.items.push(parse_quote! { + fn __extend_py_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #getset_impl + #extend_impl + } + }); + trai.into_token_stream() + } + item => quote!(#item), + }; + if let Some(error) = context.errors.into_error() { + let error = Diagnostic::from(error); + tokens = quote! { + #tokens + #error } - item => Ok(quote!(#item)), } + Ok(tokens) } fn generate_class_def( @@ -478,9 +485,7 @@ impl GetSetNursery { )); }; } - if !errors.is_empty() { - return Err(errors.into_error()); - } + errors.into_result()?; self.validated = true; Ok(()) } diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index c0187fa84..d85ae3553 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -1,6 +1,6 @@ use crate::error::Diagnostic; use crate::util::{ - AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ItemMeta, ItemNursery, + AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES, }; use proc_macro2::TokenStream; @@ -8,9 +8,11 @@ use quote::{quote, quote_spanned, ToTokens}; use syn::{parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Result, UseTree}; use syn_ext::ext::*; +#[derive(Default)] struct ModuleContext { name: String, module_extend_items: ItemNursery, + errors: Vec, } pub fn impl_pymodule( @@ -19,16 +21,16 @@ pub fn impl_pymodule( ) -> std::result::Result { let mut module_item = match module_item { Item::Mod(m) => m, - other => bail_span!(other, "#[pymodule] can only be on a module declaration"), + other => bail_span!(other, "#[pymodule] can only be on a full module"), }; let fake_ident = Ident::new("pymodule", module_item.span()); let module_meta = SimpleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; // generation resources - let mut module_context = ModuleContext { + let mut context = ModuleContext { name: module_meta.simple_name()?, - module_extend_items: ItemNursery::default(), + ..Default::default() }; let items = module_item.items_mut().ok_or_else(|| { module_meta.new_meta_error("requires actual module, not a module declaration") @@ -36,23 +38,25 @@ pub fn impl_pymodule( // collect to context for item in items.iter_mut() { - item.try_split_attr_mut(|attrs, item| { + let r = item.try_split_attr_mut(|attrs, item| { let (pyitems, cfgs) = attrs_to_module_items(&attrs, new_module_item)?; for pyitem in pyitems.iter().rev() { - pyitem.gen_module_item(ModuleItemArgs { + let r = pyitem.gen_module_item(ModuleItemArgs { item, attrs, - module: &mut module_context, + context: &mut context, cfgs: cfgs.as_slice(), - })?; + }); + context.errors.ok_or_push(r); } Ok(()) - })?; + }); + context.errors.ok_or_push(r); } // append additional items - let module_name = module_context.name.as_str(); - let module_extend_items = module_context.module_extend_items; + let module_name = context.name.as_str(); + let module_extend_items = context.module_extend_items; items.extend(vec![ parse_quote! { pub(crate) const MODULE_NAME: &'static str = #module_name; @@ -77,7 +81,15 @@ pub fn impl_pymodule( }, ]); - Ok(module_item.into_token_stream()) + Ok(if let Some(error) = context.errors.into_error() { + let error = Diagnostic::from(error); + quote! { + #module_item + #error + } + } else { + module_item.into_token_stream() + }) } fn new_module_item( @@ -219,13 +231,13 @@ impl ContentItem for AttributeItem { struct ModuleItemArgs<'a> { item: &'a Item, attrs: &'a mut Vec, - module: &'a mut ModuleContext, + context: &'a mut ModuleContext, cfgs: &'a [Attribute], } impl<'a> ModuleItemArgs<'a> { fn module_name(&'a self) -> &'a str { - self.module.name.as_str() + self.context.name.as_str() } } @@ -258,7 +270,7 @@ impl ModuleItem for FunctionItem { } }; - args.module + args.context .module_extend_items .add_item(py_name, args.cfgs.to_vec(), item)?; Ok(()) @@ -284,7 +296,7 @@ impl ModuleItem for ClassItem { class_attr, format!( "#[{name}] requires #[pyattr] to be a module attribute. \ - To keep it free type, try #[{name}(noattr)]", + To keep it free type, try #[{name}(noattr)]", name = self.attr_name() ), )); @@ -296,7 +308,7 @@ impl ModuleItem for ClassItem { class_attr.get_ident().unwrap().clone(), class_attr.promoted_nested()?.into_iter(), )?; - let module_name = args.module.name.clone(); + let module_name = args.context.name.clone(); class_attr.fill_nested_meta("module", || { parse_quote! {module = #module_name} })?; @@ -304,26 +316,34 @@ impl ModuleItem for ClassItem { (module_name, class_name) }; for attr_index in self.pyattrs.iter().rev() { - let attr_attr = args.attrs.remove(*attr_index); - let (meta_ident, nested) = attr_attr.ident_and_promoted_nested()?; - let item_meta = - SimpleItemMeta::from_nested(ident.clone(), meta_ident.clone(), nested.into_iter())?; + let mut loop_unit = || { + let attr_attr = args.attrs.remove(*attr_index); + let (meta_ident, nested) = attr_attr.ident_and_promoted_nested()?; + let item_meta = SimpleItemMeta::from_nested( + ident.clone(), + meta_ident.clone(), + nested.into_iter(), + )?; - let py_name = item_meta - .optional_name() - .unwrap_or_else(|| class_name.clone()); - let new_class = quote_spanned!(ident.span() => - #ident::make_class(&vm.ctx); - ); - let item = quote! { - let new_class = #new_class; - new_class.set_str_attr("__module__", vm.ctx.new_str(#module_name)); - vm.__module_set_attr(&module, #py_name, new_class).unwrap(); + let py_name = item_meta + .optional_name() + .unwrap_or_else(|| class_name.clone()); + let new_class = quote_spanned!(ident.span() => + #ident::make_class(&vm.ctx); + ); + let item = quote! { + let new_class = #new_class; + new_class.set_str_attr("__module__", vm.ctx.new_str(#module_name)); + vm.__module_set_attr(&module, #py_name, new_class).unwrap(); + }; + + args.context + .module_extend_items + .add_item(py_name, args.cfgs.to_vec(), item)?; + Ok(()) }; - - args.module - .module_extend_items - .add_item(py_name.clone(), args.cfgs.to_vec(), item)?; + let r = loop_unit(); + args.context.errors.ok_or_push(r); } Ok(()) } @@ -386,7 +406,7 @@ impl ModuleItem for AttributeItem { }; args.attrs.remove(self.index()); - args.module + args.context .module_extend_items .add_item(py_name, args.cfgs.to_vec(), tokens)?; diff --git a/derive/src/util.rs b/derive/src/util.rs index 250870b20..2fa3e40ef 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -397,19 +397,29 @@ impl AttributeExt for Attribute { } } -pub(crate) trait ErrorVec { - fn into_error(self) -> syn::Error; +pub(crate) trait ErrorVec: Sized { + fn into_error(self) -> Option; + fn into_result(self) -> Result<()> { + if let Some(error) = self.into_error() { + Err(error) + } else { + Ok(()) + } + } fn ok_or_push(&mut self, r: Result) -> Option; } impl ErrorVec for Vec { - fn into_error(self) -> syn::Error { + fn into_error(self) -> Option { let mut iter = self.into_iter(); - let mut first = iter.next().expect("Cannot convert empty vec to syn::Error"); - for err in iter { - first.combine(err); + if let Some(mut first) = iter.next() { + for err in iter { + first.combine(err); + } + Some(first) + } else { + None } - first } fn ok_or_push(&mut self, r: Result) -> Option { match r { From 5edaf12d2a9e2c20f15b2bf22c20ca4fc03dc445 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 18:37:25 +0900 Subject: [PATCH 09/13] pyclass_ident_and_attrs --- derive/src/pyclass.rs | 14 +++----------- derive/src/pymodule.rs | 12 +++--------- derive/src/util.rs | 12 ++++++++++++ 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index 96c055e54..aa77b4e63 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -1,7 +1,7 @@ use super::Diagnostic; use crate::util::{ - path_eq, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, - ItemMetaInner, ItemNursery, ALL_ALLOWED_NAMES, + path_eq, pyclass_ident_and_attrs, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, + ErrorVec, ItemMeta, ItemMetaInner, ItemNursery, ALL_ALLOWED_NAMES, }; use proc_macro2::TokenStream; use quote::{quote, quote_spanned, ToTokens}; @@ -165,15 +165,7 @@ pub(crate) fn impl_pyclass( attr: AttributeArgs, item: Item, ) -> std::result::Result { - let (ident, attrs) = match &item { - Item::Struct(syn::ItemStruct { ident, attrs, .. }) => (ident, attrs), - Item::Enum(syn::ItemEnum { ident, attrs, .. }) => (ident, attrs), - other => bail_span!( - other, - "#[pyclass] can only be on a struct or enum declaration" - ), - }; - + let (ident, attrs) = pyclass_ident_and_attrs(&item)?; let fake_ident = Ident::new("pyclass", item.span()); let class_meta = ClassItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; let class_name = class_meta.class_name()?; diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs index d85ae3553..152236fd3 100644 --- a/derive/src/pymodule.rs +++ b/derive/src/pymodule.rs @@ -1,7 +1,7 @@ use crate::error::Diagnostic; use crate::util::{ - AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery, - SimpleItemMeta, ALL_ALLOWED_NAMES, + pyclass_ident_and_attrs, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, + ItemMeta, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES, }; use proc_macro2::TokenStream; use quote::{quote, quote_spanned, ToTokens}; @@ -279,13 +279,7 @@ impl ModuleItem for FunctionItem { impl ModuleItem for ClassItem { fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let ident = match args.item { - Item::Struct(syn::ItemStruct { ident, .. }) => ident.clone(), - Item::Enum(syn::ItemEnum { ident, .. }) => ident.clone(), - other => { - return Err(self.new_syn_error(other.span(), "can only be on a struct or enum")) - } - }; + let (ident, _) = pyclass_ident_and_attrs(&args.item)?; let (module_name, class_name) = { let class_attr = &mut args.attrs[self.inner.index]; if self.pyattrs.is_empty() { diff --git a/derive/src/util.rs b/derive/src/util.rs index 2fa3e40ef..0759ce91a 100644 --- a/derive/src/util.rs +++ b/derive/src/util.rs @@ -397,6 +397,18 @@ impl AttributeExt for Attribute { } } +pub(crate) fn pyclass_ident_and_attrs(item: &syn::Item) -> Result<(&Ident, &[Attribute])> { + use syn::Item::*; + match item { + Struct(syn::ItemStruct { ident, attrs, .. }) => Ok((ident, attrs)), + Enum(syn::ItemEnum { ident, attrs, .. }) => Ok((ident, attrs)), + other => Err(syn::Error::new_spanned( + other, + "#[pyclass] can only be on a struct or enum declaration", + )), + } +} + pub(crate) trait ErrorVec: Sized { fn into_error(self) -> Option; fn into_result(self) -> Result<()> { From df5948e2dbca8fce94cce48cf6ff08008b4b494a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 18:47:26 +0900 Subject: [PATCH 10/13] fixup pyclass --- derive/src/pyclass.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index aa77b4e63..d8f28df41 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -360,10 +360,7 @@ impl ImplItem for PropertyItem where Item: ItemLike + ToTokens + GetIdent, { - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> - where - Item: ItemLike + ToTokens + GetIdent, - { + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { let ident = if args.item.is_function_or_method() { Ok(args.item.get_ident().unwrap()) } else { @@ -390,10 +387,7 @@ impl ImplItem for SlotItem where Item: ItemLike + ToTokens + GetIdent, { - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> - where - Item: ItemLike + ToTokens + GetIdent, - { + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { let ident = if args.item.is_function_or_method() { Ok(args.item.get_ident().unwrap()) } else { From 0b59f20d967ba3d7b79e177175e8075e077c18be Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 18:47:35 +0900 Subject: [PATCH 11/13] pystruct_sequence uses make_class_with_base --- derive/src/pyclass.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index d8f28df41..b5fab17b3 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -265,9 +265,8 @@ pub(crate) fn impl_pystruct_sequence( fn make_class( ctx: &::rustpython_vm::pyobject::PyContext, ) -> ::rustpython_vm::obj::objtype::PyClassRef { - let py_class = Self::create_bare_type(&ctx.type_type(), &ctx.tuple_type()); - Self::extend_class(ctx, &py_class); - py_class + use crate::pyobject::PyClassDef; + Self::make_class_with_base(ctx, Self::NAME, &ctx.tuple_type()) } } }; From dded3ffc53005104161f413e72e16f152cf71929 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 19:10:08 +0900 Subject: [PATCH 12/13] Remove redundant clone from new_class --- vm/src/pyobject.rs | 8 +-- vm/src/stdlib/ast.rs | 128 +++++++++++++++++----------------- vm/src/stdlib/io.rs | 20 +++--- vm/src/stdlib/pystruct.rs | 2 +- vm/src/stdlib/socket.rs | 4 +- wasm/lib/src/wasm_builtins.rs | 2 +- 6 files changed, 82 insertions(+), 82 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index a5dda94c4..f78e18250 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -464,8 +464,8 @@ impl PyContext { .unwrap() } - pub fn new_class(&self, name: &str, base: PyClassRef, flags: PyTpFlags) -> PyClassRef { - create_type_with_flags(name, &self.type_type(), &base, flags) + pub fn new_class(&self, name: &str, base: &PyClassRef, flags: PyTpFlags) -> PyClassRef { + create_type_with_flags(name, &self.type_type(), base, flags) } pub fn new_namespace(&self) -> PyObjectRef { @@ -1481,10 +1481,10 @@ pub trait PyClassImpl: PyClassDef { } fn make_class(ctx: &PyContext) -> PyClassRef { - Self::make_class_with_base(ctx, Self::NAME, ctx.object()) + Self::make_class_with_base(ctx, Self::NAME, &ctx.object()) } - fn make_class_with_base(ctx: &PyContext, name: &str, base: PyClassRef) -> PyClassRef { + fn make_class_with_base(ctx: &PyContext, name: &str, base: &PyClassRef) -> PyClassRef { let py_class = ctx.new_class(name, base, Self::TP_FLAGS); Self::extend_class(ctx, &py_class); py_class diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 81ba3a05f..05c0325e8 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -644,72 +644,72 @@ pub(crate) fn parse(vm: &VirtualMachine, source: &str, mode: Mode) -> PyResult { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; - let ast_base = py_class!(ctx, "AST", ctx.object(), PyTpFlags::HAS_DICT, {}); + let ast_base = py_class!(ctx, "AST", &ctx.object(), PyTpFlags::HAS_DICT, {}); py_module!(vm, MODULE_NAME, { // TODO: There's got to be a better way! - "alias" => py_class!(ctx, "alias", ast_base.clone(), {}), - "arg" => py_class!(ctx, "arg", ast_base.clone(), {}), - "arguments" => py_class!(ctx, "arguments", ast_base.clone(), {}), - "AnnAssign" => py_class!(ctx, "AnnAssign", ast_base.clone(), {}), - "Assign" => py_class!(ctx, "Assign", ast_base.clone(), {}), - "AugAssign" => py_class!(ctx, "AugAssign", ast_base.clone(), {}), - "AsyncFor" => py_class!(ctx, "AsyncFor", ast_base.clone(), {}), - "AsyncFunctionDef" => py_class!(ctx, "AsyncFunctionDef", ast_base.clone(), {}), - "AsyncWith" => py_class!(ctx, "AsyncWith", ast_base.clone(), {}), - "Assert" => py_class!(ctx, "Assert", ast_base.clone(), {}), - "Attribute" => py_class!(ctx, "Attribute", ast_base.clone(), {}), - "Await" => py_class!(ctx, "Await", ast_base.clone(), {}), - "BinOp" => py_class!(ctx, "BinOp", ast_base.clone(), {}), - "BoolOp" => py_class!(ctx, "BoolOp", ast_base.clone(), {}), - "Break" => py_class!(ctx, "Break", ast_base.clone(), {}), - "Bytes" => py_class!(ctx, "Bytes", ast_base.clone(), {}), - "Call" => py_class!(ctx, "Call", ast_base.clone(), {}), - "ClassDef" => py_class!(ctx, "ClassDef", ast_base.clone(), {}), - "Compare" => py_class!(ctx, "Compare", ast_base.clone(), {}), - "comprehension" => py_class!(ctx, "comprehension", ast_base.clone(), {}), - "Continue" => py_class!(ctx, "Continue", ast_base.clone(), {}), - "Delete" => py_class!(ctx, "Delete", ast_base.clone(), {}), - "Dict" => py_class!(ctx, "Dict", ast_base.clone(), {}), - "DictComp" => py_class!(ctx, "DictComp", ast_base.clone(), {}), - "Ellipsis" => py_class!(ctx, "Ellipsis", ast_base.clone(), {}), - "Expr" => py_class!(ctx, "Expr", ast_base.clone(), {}), - "ExceptHandler" => py_class!(ctx, "ExceptHandler", ast_base.clone(), {}), - "For" => py_class!(ctx, "For", ast_base.clone(), {}), - "FormattedValue" => py_class!(ctx, "FormattedValue", ast_base.clone(), {}), - "FunctionDef" => py_class!(ctx, "FunctionDef", ast_base.clone(), {}), - "GeneratorExp" => py_class!(ctx, "GeneratorExp", ast_base.clone(), {}), - "Global" => py_class!(ctx, "Global", ast_base.clone(), {}), - "If" => py_class!(ctx, "If", ast_base.clone(), {}), - "IfExp" => py_class!(ctx, "IfExp", ast_base.clone(), {}), - "Import" => py_class!(ctx, "Import", ast_base.clone(), {}), - "ImportFrom" => py_class!(ctx, "ImportFrom", ast_base.clone(), {}), - "JoinedStr" => py_class!(ctx, "JoinedStr", ast_base.clone(), {}), - "keyword" => py_class!(ctx, "keyword", ast_base.clone(), {}), - "Lambda" => py_class!(ctx, "Lambda", ast_base.clone(), {}), - "List" => py_class!(ctx, "List", ast_base.clone(), {}), - "ListComp" => py_class!(ctx, "ListComp", ast_base.clone(), {}), - "Module" => py_class!(ctx, "Module", ast_base.clone(), {}), - "Name" => py_class!(ctx, "Name", ast_base.clone(), {}), - "NameConstant" => py_class!(ctx, "NameConstant", ast_base.clone(), {}), - "Nonlocal" => py_class!(ctx, "Nonlocal", ast_base.clone(), {}), - "Num" => py_class!(ctx, "Num", ast_base.clone(), {}), - "Pass" => py_class!(ctx, "Pass", ast_base.clone(), {}), - "Raise" => py_class!(ctx, "Raise", ast_base.clone(), {}), - "Return" => py_class!(ctx, "Return", ast_base.clone(), {}), - "Set" => py_class!(ctx, "Set", ast_base.clone(), {}), - "SetComp" => py_class!(ctx, "SetComp", ast_base.clone(), {}), - "Starred" => py_class!(ctx, "Starred", ast_base.clone(), {}), - "Starred" => py_class!(ctx, "Starred", ast_base.clone(), {}), - "Str" => py_class!(ctx, "Str", ast_base.clone(), {}), - "Subscript" => py_class!(ctx, "Subscript", ast_base.clone(), {}), - "Try" => py_class!(ctx, "Try", ast_base.clone(), {}), - "Tuple" => py_class!(ctx, "Tuple", ast_base.clone(), {}), - "UnaryOp" => py_class!(ctx, "UnaryOp", ast_base.clone(), {}), - "While" => py_class!(ctx, "While", ast_base.clone(), {}), - "With" => py_class!(ctx, "With", ast_base.clone(), {}), - "withitem" => py_class!(ctx, "withitem", ast_base.clone(), {}), - "Yield" => py_class!(ctx, "Yield", ast_base.clone(), {}), - "YieldFrom" => py_class!(ctx, "YieldFrom", ast_base.clone(), {}), + "alias" => py_class!(ctx, "alias", &ast_base, {}), + "arg" => py_class!(ctx, "arg", &ast_base, {}), + "arguments" => py_class!(ctx, "arguments", &ast_base, {}), + "AnnAssign" => py_class!(ctx, "AnnAssign", &ast_base, {}), + "Assign" => py_class!(ctx, "Assign", &ast_base, {}), + "AugAssign" => py_class!(ctx, "AugAssign", &ast_base, {}), + "AsyncFor" => py_class!(ctx, "AsyncFor", &ast_base, {}), + "AsyncFunctionDef" => py_class!(ctx, "AsyncFunctionDef", &ast_base, {}), + "AsyncWith" => py_class!(ctx, "AsyncWith", &ast_base, {}), + "Assert" => py_class!(ctx, "Assert", &ast_base, {}), + "Attribute" => py_class!(ctx, "Attribute", &ast_base, {}), + "Await" => py_class!(ctx, "Await", &ast_base, {}), + "BinOp" => py_class!(ctx, "BinOp", &ast_base, {}), + "BoolOp" => py_class!(ctx, "BoolOp", &ast_base, {}), + "Break" => py_class!(ctx, "Break", &ast_base, {}), + "Bytes" => py_class!(ctx, "Bytes", &ast_base, {}), + "Call" => py_class!(ctx, "Call", &ast_base, {}), + "ClassDef" => py_class!(ctx, "ClassDef", &ast_base, {}), + "Compare" => py_class!(ctx, "Compare", &ast_base, {}), + "comprehension" => py_class!(ctx, "comprehension", &ast_base, {}), + "Continue" => py_class!(ctx, "Continue", &ast_base, {}), + "Delete" => py_class!(ctx, "Delete", &ast_base, {}), + "Dict" => py_class!(ctx, "Dict", &ast_base, {}), + "DictComp" => py_class!(ctx, "DictComp", &ast_base, {}), + "Ellipsis" => py_class!(ctx, "Ellipsis", &ast_base, {}), + "Expr" => py_class!(ctx, "Expr", &ast_base, {}), + "ExceptHandler" => py_class!(ctx, "ExceptHandler", &ast_base, {}), + "For" => py_class!(ctx, "For", &ast_base, {}), + "FormattedValue" => py_class!(ctx, "FormattedValue", &ast_base, {}), + "FunctionDef" => py_class!(ctx, "FunctionDef", &ast_base, {}), + "GeneratorExp" => py_class!(ctx, "GeneratorExp", &ast_base, {}), + "Global" => py_class!(ctx, "Global", &ast_base, {}), + "If" => py_class!(ctx, "If", &ast_base, {}), + "IfExp" => py_class!(ctx, "IfExp", &ast_base, {}), + "Import" => py_class!(ctx, "Import", &ast_base, {}), + "ImportFrom" => py_class!(ctx, "ImportFrom", &ast_base, {}), + "JoinedStr" => py_class!(ctx, "JoinedStr", &ast_base, {}), + "keyword" => py_class!(ctx, "keyword", &ast_base, {}), + "Lambda" => py_class!(ctx, "Lambda", &ast_base, {}), + "List" => py_class!(ctx, "List", &ast_base, {}), + "ListComp" => py_class!(ctx, "ListComp", &ast_base, {}), + "Module" => py_class!(ctx, "Module", &ast_base, {}), + "Name" => py_class!(ctx, "Name", &ast_base, {}), + "NameConstant" => py_class!(ctx, "NameConstant", &ast_base, {}), + "Nonlocal" => py_class!(ctx, "Nonlocal", &ast_base, {}), + "Num" => py_class!(ctx, "Num", &ast_base, {}), + "Pass" => py_class!(ctx, "Pass", &ast_base, {}), + "Raise" => py_class!(ctx, "Raise", &ast_base, {}), + "Return" => py_class!(ctx, "Return", &ast_base, {}), + "Set" => py_class!(ctx, "Set", &ast_base, {}), + "SetComp" => py_class!(ctx, "SetComp", &ast_base, {}), + "Starred" => py_class!(ctx, "Starred", &ast_base, {}), + "Starred" => py_class!(ctx, "Starred", &ast_base, {}), + "Str" => py_class!(ctx, "Str", &ast_base, {}), + "Subscript" => py_class!(ctx, "Subscript", &ast_base, {}), + "Try" => py_class!(ctx, "Try", &ast_base, {}), + "Tuple" => py_class!(ctx, "Tuple", &ast_base, {}), + "UnaryOp" => py_class!(ctx, "UnaryOp", &ast_base, {}), + "While" => py_class!(ctx, "While", &ast_base, {}), + "With" => py_class!(ctx, "With", &ast_base, {}), + "withitem" => py_class!(ctx, "withitem", &ast_base, {}), + "Yield" => py_class!(ctx, "Yield", &ast_base, {}), + "YieldFrom" => py_class!(ctx, "YieldFrom", &ast_base, {}), "AST" => ast_base, "PyCF_ONLY_AST" => ctx.new_int(PY_COMPILE_FLAG_AST_ONLY), }) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 5001859db..078dc8028 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -877,7 +877,7 @@ mod fileio { } pub fn make_fileio(ctx: &crate::pyobject::PyContext, raw_io_base: PyClassRef) -> PyClassRef { - FileIO::make_class_with_base(ctx, FileIO::NAME, raw_io_base) + FileIO::make_class_with_base(ctx, FileIO::NAME, &raw_io_base) } } @@ -1268,7 +1268,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; // IOBase the abstract base class of the IO Module - let io_base = py_class!(ctx, "_IOBase", ctx.object(), { + let io_base = py_class!(ctx, "_IOBase", &ctx.object(), { "__enter__" => ctx.new_method(io_base_cm_enter), "__exit__" => ctx.new_method(io_base_cm_exit), "seekable" => ctx.new_method(io_base_seekable), @@ -1289,17 +1289,17 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); // IOBase Subclasses - let raw_io_base = py_class!(ctx, "_RawIOBase", io_base.clone(), { + let raw_io_base = py_class!(ctx, "_RawIOBase", &io_base, { "read" => ctx.new_method(raw_io_base_read), }); - let buffered_io_base = py_class!(ctx, "_BufferedIOBase", io_base.clone(), {}); + let buffered_io_base = py_class!(ctx, "_BufferedIOBase", &io_base, {}); //TextIO Base has no public constructor - let text_io_base = py_class!(ctx, "_TextIOBase", io_base.clone(), {}); + let text_io_base = py_class!(ctx, "_TextIOBase", &io_base, {}); // BufferedIOBase Subclasses - let buffered_reader = py_class!(ctx, "BufferedReader", buffered_io_base.clone(), { + let buffered_reader = py_class!(ctx, "BufferedReader", &buffered_io_base, { //workaround till the buffered classes can be fixed up to be more //consistent with the python model //For more info see: https://github.com/RustPython/RustPython/issues/547 @@ -1314,7 +1314,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "mode" => ctx.new_readonly_getset("mode", buffered_io_base_mode), }); - let buffered_writer = py_class!(ctx, "BufferedWriter", buffered_io_base.clone(), { + let buffered_writer = py_class!(ctx, "BufferedWriter", &buffered_io_base, { //workaround till the buffered classes can be fixed up to be more //consistent with the python model //For more info see: https://github.com/RustPython/RustPython/issues/547 @@ -1330,7 +1330,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); //TextIOBase Subclass - let text_io_wrapper = py_class!(ctx, "TextIOWrapper", text_io_base.clone(), { + let text_io_wrapper = py_class!(ctx, "TextIOWrapper", &text_io_base, { "__init__" => ctx.new_method(text_io_wrapper_init), "seekable" => ctx.new_method(text_io_wrapper_seekable), "seek" => ctx.new_method(text_io_wrapper_seek), @@ -1344,7 +1344,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); //StringIO: in-memory text - let string_io = py_class!(ctx, "StringIO", text_io_base.clone(), { + let string_io = py_class!(ctx, "StringIO", &text_io_base, { "__module__" => ctx.new_str("_io"), (slot new) => string_io_new, "seek" => ctx.new_method(PyStringIORef::seek), @@ -1360,7 +1360,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); //BytesIO: in-memory bytes - let bytes_io = py_class!(ctx, "BytesIO", buffered_io_base.clone(), { + let bytes_io = py_class!(ctx, "BytesIO", &buffered_io_base, { (slot new) => bytes_io_new, "read" => ctx.new_method(PyBytesIORef::read), "read1" => ctx.new_method(PyBytesIORef::read), diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 5dcf4d27f..64629ac90 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -972,7 +972,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { let struct_error = ctx.new_class( "struct.error", - ctx.exceptions.exception_type.clone(), + &ctx.exceptions.exception_type, Default::default(), ); diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 2e44a8a96..c8d2b381d 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -654,12 +654,12 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let socket_timeout = ctx.new_class( "socket.timeout", - vm.ctx.exceptions.os_error.clone(), + &vm.ctx.exceptions.os_error, Default::default(), ); let socket_gaierror = ctx.new_class( "socket.gaierror", - vm.ctx.exceptions.os_error.clone(), + &vm.ctx.exceptions.os_error, Default::default(), ); diff --git a/wasm/lib/src/wasm_builtins.rs b/wasm/lib/src/wasm_builtins.rs index 2366ff903..6f6f9ec04 100644 --- a/wasm/lib/src/wasm_builtins.rs +++ b/wasm/lib/src/wasm_builtins.rs @@ -32,7 +32,7 @@ pub fn make_stdout_object( let flush_method = ctx.new_method(|_self: PyObjectRef| {}); // there's not really any point to storing this class so that there's a consistent type object, // we just want a half-decent repr() output - let cls = py_class!(ctx, "JSStdout", vm.ctx.object(), { + let cls = py_class!(ctx, "JSStdout", &vm.ctx.object(), { "write" => write_method, "flush" => flush_method, }); From fd3aba5033ed6f145fe569b8aea739ce12e1845f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Aug 2020 18:23:46 +0900 Subject: [PATCH 13/13] doc using syn-ext --- Cargo.lock | 4 ++-- derive/Cargo.toml | 2 +- derive/src/pyclass.rs | 30 ++++++------------------------ 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3830d05b..96410a0b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1691,9 +1691,9 @@ dependencies = [ [[package]] name = "syn-ext" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de58fdf4f8294294c09653384dcb02e8e90b10d61cc0cf554bfbc2e9eaad9a" +checksum = "da910e9ec6f88d3f591893025a3040fe2659b7f7ea4eacce5469b7ebe5287339" dependencies = [ "syn", ] diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c3b0e4306..89fd2c03e 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -12,7 +12,7 @@ proc-macro = true [dependencies] syn = { version = "1.0", features = ["full"] } -syn-ext = { version = "0.2", features = ["full"] } +syn-ext = { version = "0.2.1", features = ["full"] } quote = "1.0" proc-macro2 = "1.0" rustpython-compiler = { path = "../compiler", version = "0.1.1" } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index b5fab17b3..bab55ffb0 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -7,8 +7,8 @@ use proc_macro2::TokenStream; use quote::{quote, quote_spanned, ToTokens}; use std::collections::HashMap; use syn::{ - parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Index, Item, Lit, Meta, - NestedMeta, Result, + parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Index, Item, Meta, NestedMeta, + Result, }; use syn_ext::ext::*; @@ -117,29 +117,11 @@ fn generate_class_def( module_name: Option<&str>, attrs: &[Attribute], ) -> std::result::Result { - let mut doc: Option> = None; - for attr in attrs.iter() { - if attr.path.is_ident("doc") { - let meta = attr.parse_meta().expect("expected doc attr to be a meta"); - if let Meta::NameValue(name_value) = meta { - if let Lit::Str(s) = name_value.lit { - let val = s.value().trim().to_owned(); - match doc { - Some(ref mut doc) => doc.push(val), - None => doc = Some(vec![val]), - } - } - } - } - } - let doc = match doc { - Some(doc) => { - let doc = doc.join("\n"); - quote!(Some(#doc)) - } - None => quote!(None), + let doc = if let Some(doc) = attrs.doc() { + quote!(Some(#doc)) + } else { + quote!(None) }; - let module_class_name = if let Some(module_name) = module_name { format!("{}.{}", module_name, name) } else {