diff --git a/Cargo.lock b/Cargo.lock index 623a723ff..cac8cb546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1804,7 +1804,7 @@ dependencies = [ [[package]] name = "ruff_source_location" version = "0.0.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "memchr", "once_cell", @@ -1814,7 +1814,7 @@ dependencies = [ [[package]] name = "ruff_text_size" version = "0.0.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "serde", ] @@ -1875,7 +1875,7 @@ dependencies = [ [[package]] name = "rustpython-ast" version = "0.2.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "num-bigint", "rustpython-literal", @@ -1999,7 +1999,7 @@ dependencies = [ [[package]] name = "rustpython-literal" version = "0.2.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "hexf-parse", "lexical-parse-float", @@ -2010,7 +2010,7 @@ dependencies = [ [[package]] name = "rustpython-parser" version = "0.2.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "anyhow", "itertools", @@ -2032,7 +2032,7 @@ dependencies = [ [[package]] name = "rustpython-parser-core" version = "0.2.0" -source = "git+https://github.com/youknowone/RustPython-parser.git?rev=6fa3d0f90a5115680ded1ed1606918b4b26251de#6fa3d0f90a5115680ded1ed1606918b4b26251de" +source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df" dependencies = [ "itertools", "lz4_flex", diff --git a/Cargo.toml b/Cargo.toml index 492b8e368..60674bd0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,10 @@ rustpython-pylib = { path = "pylib" } rustpython-stdlib = { path = "stdlib" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" } -rustpython-literal = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "6fa3d0f90a5115680ded1ed1606918b4b26251de" } -rustpython-parser-core = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "6fa3d0f90a5115680ded1ed1606918b4b26251de" } -rustpython-parser = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "6fa3d0f90a5115680ded1ed1606918b4b26251de" } -rustpython-ast = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "6fa3d0f90a5115680ded1ed1606918b4b26251de" } +rustpython-literal = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" } +rustpython-parser-core = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" } +rustpython-parser = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" } +rustpython-ast = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" } # rustpython-literal = { path = "../RustPython-parser/literal" } # rustpython-parser-core = { path = "../RustPython-parser/core" } # rustpython-parser = { path = "../RustPython-parser/parser" } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index e64054708..148b996eb 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -565,7 +565,7 @@ impl Compiler { // we do this here because `from __future__` still executes that `from` statement at runtime, // we still need to compile the ImportFrom down below ImportFrom(ast::StmtImportFrom { module, names, .. }) - if module.as_deref() == Some("__future__") => + if module.as_ref().map(|id| id.as_str()) == Some("__future__") => { self.compile_future_features(names)? } @@ -582,16 +582,16 @@ impl Compiler { value: num_traits::Zero::zero(), }); self.emit_constant(ConstantData::None); - let idx = self.name(&name.name); + let idx = self.name(name.name.as_str()); emit!(self, Instruction::ImportName { idx }); if let Some(alias) = &name.asname { - for part in name.name.split('.').skip(1) { + for part in name.name.as_str().split('.').skip(1) { let idx = self.name(part); emit!(self, Instruction::LoadAttr { idx }); } - self.store_name(alias)? + self.store_name(alias.as_str())? } else { - self.store_name(name.name.split('.').next().unwrap())? + self.store_name(name.name.as_str().split('.').next().unwrap())? } } } @@ -600,7 +600,7 @@ impl Compiler { module, names, }) => { - let import_star = names.iter().any(|n| n.node.name == "*"); + let import_star = names.iter().any(|n| n.node.name.as_str() == "*"); let from_list = if import_star { if self.ctx.in_func() { @@ -616,16 +616,16 @@ impl Compiler { names .iter() .map(|n| ConstantData::Str { - value: n.node.name.to_owned(), + value: n.node.name.to_string(), }) .collect() }; - let module_idx = module.as_ref().map(|s| self.name(s)); + let module_idx = module.as_ref().map(|s| self.name(s.as_str())); // from .... import (*fromlist) self.emit_constant(ConstantData::Integer { - value: (*level).unwrap_or(0).into(), + value: level.as_ref().map_or(0, |level| level.to_u32()).into(), }); self.emit_constant(ConstantData::Tuple { elements: from_list, @@ -644,15 +644,15 @@ impl Compiler { for name in names { let name = &name.node; - let idx = self.name(&name.name); + let idx = self.name(name.name.as_str()); // import symbol from module: emit!(self, Instruction::ImportFrom { idx }); // Store module under proper name: if let Some(alias) = &name.asname { - self.store_name(alias)? + self.store_name(alias.as_str())? } else { - self.store_name(&name.name)? + self.store_name(name.name.as_str())? } } @@ -747,7 +747,7 @@ impl Compiler { returns, .. }) => self.compile_function_def( - name, + name.as_str(), args, body, decorator_list, @@ -762,7 +762,7 @@ impl Compiler { returns, .. }) => self.compile_function_def( - name, + name.as_str(), args, body, decorator_list, @@ -775,7 +775,7 @@ impl Compiler { bases, keywords, decorator_list, - }) => self.compile_class_def(name, body, bases, keywords, decorator_list)?, + }) => self.compile_class_def(name.as_str(), body, bases, keywords, decorator_list)?, Assert(StmtAssert { test, msg }) => { // if some flag, ignore all assert statements! if self.opts.optimize == 0 { @@ -885,12 +885,12 @@ impl Compiler { fn compile_delete(&mut self, expression: &ast::located::Expr) -> CompileResult<()> { match &expression.node { ast::ExprKind::Name(ast::ExprName { id, .. }) => { - self.compile_name(id, NameUsage::Delete)? + self.compile_name(id.as_str(), NameUsage::Delete)? } ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr, NameUsage::Delete)?; + self.check_forbidden_name(attr.as_str(), NameUsage::Delete)?; self.compile_expression(value)?; - let idx = self.name(attr); + let idx = self.name(attr.as_str()); emit!(self, Instruction::DeleteAttr { idx }); } ast::ExprKind::Subscript(ast::ExprSubscript { value, slice, .. }) => { @@ -935,7 +935,7 @@ impl Compiler { .zip(&args.kw_defaults) { self.emit_constant(ConstantData::Str { - value: kw.node.arg.clone(), + value: kw.node.arg.to_string(), }); self.compile_expression(default)?; } @@ -968,16 +968,16 @@ impl Compiler { .chain(&args.args) .chain(&args.kwonlyargs); for name in args_iter { - self.varname(&name.node.arg)?; + self.varname(name.node.arg.as_str())?; } if let Some(name) = args.vararg.as_deref() { self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS; - self.varname(&name.node.arg)?; + self.varname(name.node.arg.as_str())?; } if let Some(name) = args.kwarg.as_deref() { self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS; - self.varname(&name.node.arg)?; + self.varname(name.node.arg.as_str())?; } Ok(func_flags) @@ -1066,7 +1066,7 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = name { - self.store_name(alias)? + self.store_name(alias.as_str())? } else { // Drop exception from top of stack: emit!(self, Instruction::Pop); @@ -1224,7 +1224,7 @@ impl Compiler { for arg in args_iter { if let Some(annotation) = &arg.node.annotation { self.emit_constant(ConstantData::Str { - value: self.mangle(&arg.node.arg).into_owned(), + value: self.mangle(arg.node.arg.as_str()).into_owned(), }); self.compile_annotation(annotation)?; num_annotations += 1; @@ -1750,7 +1750,7 @@ impl Compiler { let annotations = self.name("__annotations__"); emit!(self, Instruction::LoadNameAny(annotations)); self.emit_constant(ConstantData::Str { - value: self.mangle(id).into_owned(), + value: self.mangle(id.as_str()).into_owned(), }); emit!(self, Instruction::StoreSubscript); } else { @@ -1763,16 +1763,16 @@ impl Compiler { fn compile_store(&mut self, target: &ast::located::Expr) -> CompileResult<()> { match &target.node { - ast::ExprKind::Name(ast::ExprName { id, .. }) => self.store_name(id)?, + ast::ExprKind::Name(ast::ExprName { id, .. }) => self.store_name(id.as_str())?, ast::ExprKind::Subscript(ast::ExprSubscript { value, slice, .. }) => { self.compile_expression(value)?; self.compile_expression(slice)?; emit!(self, Instruction::StoreSubscript); } ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr, NameUsage::Store)?; + self.check_forbidden_name(attr.as_str(), NameUsage::Store)?; self.compile_expression(value)?; - let idx = self.name(attr); + let idx = self.name(attr.as_str()); emit!(self, Instruction::StoreAttr { idx }); } ast::ExprKind::List(ast::ExprList { elts, .. }) @@ -1845,6 +1845,7 @@ impl Compiler { let kind = match &target.node { ast::ExprKind::Name(ast::ExprName { id, .. }) => { + let id = id.as_str(); self.compile_name(id, NameUsage::Load)?; AugAssignKind::Name { id } } @@ -1856,6 +1857,7 @@ impl Compiler { AugAssignKind::Subscript } ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => { + let attr = attr.as_str(); self.check_forbidden_name(attr, NameUsage::Store)?; self.compile_expression(value)?; emit!(self, Instruction::Duplicate); @@ -2108,7 +2110,7 @@ impl Compiler { } Attribute(ast::ExprAttribute { value, attr, .. }) => { self.compile_expression(value)?; - let idx = self.name(attr); + let idx = self.name(attr.as_str()); emit!(self, Instruction::LoadAttr { idx }); } Compare(ast::ExprCompare { @@ -2230,12 +2232,12 @@ impl Compiler { emit!( self, Instruction::FormatValue { - conversion: ConversionFlag::from_op_arg(*conversion) + conversion: ConversionFlag::from_op_arg(conversion.to_u32()) .expect("invalid conversion flag"), }, ); } - Name(ast::ExprName { id, .. }) => self.load_name(id)?, + Name(ast::ExprName { id, .. }) => self.load_name(id.as_str())?, Lambda(ast::ExprLambda { args, body }) => { let prev_ctx = self.ctx; @@ -2390,7 +2392,7 @@ impl Compiler { for keyword in sub_keywords { if let Some(name) = &keyword.node.arg { self.emit_constant(ConstantData::Str { - value: name.to_owned(), + value: name.to_string(), }); self.compile_expression(&keyword.node.value)?; sub_size += 1; @@ -2415,7 +2417,7 @@ impl Compiler { let method = if let ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) = &func.node { self.compile_expression(value)?; - let idx = self.name(attr); + let idx = self.name(attr.as_str()); emit!(self, Instruction::LoadMethod { idx }); true } else { @@ -2464,7 +2466,7 @@ impl Compiler { for keyword in keywords { if let Some(name) = &keyword.node.arg { - self.check_forbidden_name(name, NameUsage::Store)?; + self.check_forbidden_name(name.as_str(), NameUsage::Store)?; } } @@ -2487,7 +2489,7 @@ impl Compiler { for keyword in keywords { if let Some(name) = &keyword.node.arg { kwarg_names.push(ConstantData::Str { - value: name.to_owned(), + value: name.to_string(), }); } else { // This means **kwargs! @@ -2611,7 +2613,7 @@ impl Compiler { let mut loop_labels = vec![]; for generator in generators { - if generator.is_async > 0 { + if generator.is_async { unimplemented!("async for comprehensions"); } @@ -2706,7 +2708,7 @@ impl Compiler { return Err(self.error(CodegenErrorType::InvalidFuturePlacement)); } for feature in features { - match &*feature.node.name { + match feature.node.name.as_str() { // Python 3 features; we've already implemented them by default "nested_scopes" | "generators" | "division" | "absolute_import" | "with_statement" | "print_function" | "unicode_literals" => {} diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index bf63cc5df..d5243d09c 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -612,7 +612,7 @@ impl SymbolTableBuilder { } else { SymbolUsage::Parameter }; - self.register_name(¶meter.node.arg, usage, parameter.location()) + self.register_name(parameter.node.arg.as_str(), usage, parameter.location()) } fn scan_parameters_annotations( @@ -644,9 +644,9 @@ impl SymbolTableBuilder { use ast::{StmtKind::*, *}; let location = statement.location(); if let ImportFrom(StmtImportFrom { module, names, .. }) = &statement.node { - if module.as_deref() == Some("__future__") { + if module.as_ref().map(|id| id.as_str()) == Some("__future__") { for feature in names { - if feature.node.name == "annotations" { + if feature.node.name.as_str() == "annotations" { self.future_annotations = true; } } @@ -655,12 +655,12 @@ impl SymbolTableBuilder { match &statement.node { Global(StmtGlobal { names }) => { for name in names { - self.register_name(name, SymbolUsage::Global, location)?; + self.register_name(name.as_str(), SymbolUsage::Global, location)?; } } Nonlocal(StmtNonlocal { names }) => { for name in names { - self.register_name(name, SymbolUsage::Nonlocal, location)?; + self.register_name(name.as_str(), SymbolUsage::Nonlocal, location)?; } } FunctionDef(StmtFunctionDef { @@ -680,11 +680,11 @@ impl SymbolTableBuilder { .. }) => { self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name, SymbolUsage::Assigned, location)?; + self.register_name(name.as_str(), SymbolUsage::Assigned, location)?; if let Some(expression) = returns { self.scan_annotation(expression)?; } - self.enter_function(name, args, location.row)?; + self.enter_function(name.as_str(), args, location.row)?; self.scan_statements(body)?; self.leave_scope(); } @@ -695,8 +695,8 @@ impl SymbolTableBuilder { keywords, decorator_list, }) => { - self.enter_scope(name, SymbolTableType::Class, location.row.get()); - let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_owned())); + self.enter_scope(name.as_str(), SymbolTableType::Class, location.row.get()); + let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string())); self.register_name("__module__", SymbolUsage::Assigned, location)?; self.register_name("__qualname__", SymbolUsage::Assigned, location)?; self.register_name("__doc__", SymbolUsage::Assigned, location)?; @@ -709,7 +709,7 @@ impl SymbolTableBuilder { self.scan_expression(&keyword.node.value, ExpressionContext::Load)?; } self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name, SymbolUsage::Assigned, location)?; + self.register_name(name.as_str(), SymbolUsage::Assigned, location)?; } Expr(StmtExpr { value }) => self.scan_expression(value, ExpressionContext::Load)?, If(StmtIf { test, body, orelse }) => { @@ -748,11 +748,11 @@ impl SymbolTableBuilder { for name in names { if let Some(alias) = &name.node.asname { // `import my_module as my_alias` - self.register_name(alias, SymbolUsage::Imported, location)?; + self.register_name(alias.as_str(), SymbolUsage::Imported, location)?; } else { // `import module` self.register_name( - name.node.name.split('.').next().unwrap(), + name.node.name.as_str().split('.').next().unwrap(), SymbolUsage::Imported, location, )?; @@ -789,8 +789,8 @@ impl SymbolTableBuilder { }) => { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &target.node { - ast::ExprKind::Name(ast::ExprName { id, .. }) if *simple > 0 => { - self.register_name(id, SymbolUsage::AnnotationAssigned, location)?; + ast::ExprKind::Name(ast::ExprName { id, .. }) if *simple => { + self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, location)?; } _ => { self.scan_expression(target, ExpressionContext::Store)?; @@ -833,7 +833,7 @@ impl SymbolTableBuilder { self.scan_expression(expression, ExpressionContext::Load)?; } if let Some(name) = name { - self.register_name(name, SymbolUsage::Assigned, location)?; + self.register_name(name.as_str(), SymbolUsage::Assigned, location)?; } self.scan_statements(body)?; } @@ -988,6 +988,7 @@ impl SymbolTableBuilder { } } Name(ExprName { id, .. }) => { + let id = id.as_str(); // Determine the contextual usage of this symbol: match context { ExpressionContext::Delete => { @@ -1048,6 +1049,7 @@ impl SymbolTableBuilder { // propagate the scope of the named assigned named and not to // propagate inner names. if let Name(ExprName { id, .. }) = &target.node { + let id = id.as_str(); let table = self.tables.last().unwrap(); if table.typ == SymbolTableType::Comprehension { self.register_name( diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 650c6f827..46f570c9c 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -165,18 +165,18 @@ impl Node for ast::located::Located { column: OneIndexed::from_zero_indexed(column), }) } - let row = Node::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?; + let row = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?; let column = - Node::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?; - let location = make_location(row, column); + ast::Int::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?; + let location = make_location(row.to_u32(), column.to_u32()); let end_row = get_node_field_opt(vm, &object, "end_lineno")? - .map(|obj| Node::ast_from_object(vm, obj)) + .map(|obj| ast::Int::ast_from_object(vm, obj)) .transpose()?; let end_column = get_node_field_opt(vm, &object, "end_col_offset")? - .map(|obj| Node::ast_from_object(vm, obj)) + .map(|obj| ast::Int::ast_from_object(vm, obj)) .transpose()?; let end_location = if let (Some(row), Some(column)) = (end_row, end_column) { - make_location(row, column) + make_location(row.to_u32(), column.to_u32()) } else { None }; @@ -223,23 +223,37 @@ fn node_add_location( }; } -impl Node for String { +impl Node for ast::String { fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx.new_str(self).into() } fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - PyStrRef::try_from_object(vm, object).map(|s| s.as_str().to_owned()) + let py_str = PyStrRef::try_from_object(vm, object)?; + Ok(py_str.as_str().to_owned()) } } -impl Node for u32 { +impl Node for ast::Identifier { fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self).into() + let id: String = self.into(); + vm.ctx.new_str(id).into() } fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { - object.try_into_value(vm) + let py_str = PyStrRef::try_from_object(vm, object)?; + Ok(ast::Identifier::new(py_str.as_str())) + } +} + +impl Node for ast::Int { + fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(self.to_u32()).into() + } + + fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult { + let value = object.try_into_value(vm)?; + Ok(ast::Int::new(value)) } }