mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 18:59:04 +08:00
Code Activity

Replace Python parser with ruff parser (#5494)

Browse Source 
* stage1

* compiler pass build

* introduce rustpython-compiler-source

* stage2

* fixup

* pass compile

* Fix hello world compiler test

* Fix code generation for if-elif-else statement

* Fix code generation for lambda expression

* Fix code generation for integers

* Fix code generation for fstrings

* Fix code generation for if statement

* Fix code generation for if statement

* Fix code generation for if statement

* Fix code generation for fstring

* Fix code generation for class definition

* Replace feature flags

* Initialize frozen core modules

* Allow __future__ import after module doc comment

* Disable ast module

* Commit remaining fixes for compile errors in examples

* Fix some warnings

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Split ast stdlib module into files

* Fix codegen for positional arguments with defaults

* Update ast stdlib module

* Update ast stdlib module

* Extract string and constant handling from expression.rs

* Always add required fields to AST nodes

* Compile doc strings correctly again

* Enable "ast" Cargo feature by default

* Refactor compilation of big integer literal

* Update ast stdlib module

* Update ast stdlib module

* Update ast stdlib module

* Reset barebones example

* Fix some left-over warnings

* Undo accidential change

* Adapt shell to ruff parser

* Pin parser to v0.4.10

* fix clippy

* Add TODO about interactive mode

* Fix compilation of complex number expression

* Remove moved code

* Update test case to ruff v0.4.10

* Apply suggestion

Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>

* Apply suggestion

Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>

* Fix compilation of fstring expression

* Fix compilation of fstring expression

* Fix wasm compile errors

* Attach correct source locations to ast objects

* Fix some more wasm compile errors

* Consider compile mode and enable AST stdlib module again

* Fix incorrect AST source location end column

* Fix compile error if "compiler" feature is not enabled

* Fix regrtests

* Fix some test_ast tests

* Add source range to type ignore

* Fix incompatibility with Rust 2024 edition

* Fix todos by implementing missing ast conversions and deleting unused code

* Appease clippy

* Fix remaining ast tests

* Fix remaining ast tests

* Mark/fix remaining tests

* Fix more

* Hacky windows fix

---------

Co-authored-by: Kangzhi Shi <shikangzhi@gmail.com>
Co-authored-by: Jeong YunWon <jeong@youknowone.org>
Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>
Co-authored-by: Noa <coolreader18@gmail.com>
This commit is contained in:
Stefan LukasJeong, YunWonKangzhi ShiJeong YunWonNoa
 authored and GitHub committed 2025-03-19 21:06:03 -05:00
1 parent 45c0fa0e77
commit a6b4ef7f5d
70 files changed
+9990 -6867
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+125 -91
View File
@@ -1007,6 +1007,15 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.14.0"
@@ -1051,12 +1060,6 @@ dependencies = [
"cpufeatures",
]
[[package]]
name = "lalrpop-util"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553"
[[package]]
name = "lambert_w"
version = "1.0.17"
@@ -1958,6 +1961,90 @@ dependencies = [
"syn 2.0.98",
]
[[package]]
name = "ruff_python_ast"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"aho-corasick",
"bitflags 2.8.0",
"is-macro",
"itertools 0.13.0",
"once_cell",
"ruff_python_trivia",
"ruff_source_file",
"ruff_text_size",
"rustc-hash 1.1.0",
]
[[package]]
name = "ruff_python_codegen"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"once_cell",
"ruff_python_ast",
"ruff_python_literal",
"ruff_python_parser",
"ruff_source_file",
"ruff_text_size",
]
[[package]]
name = "ruff_python_literal"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"bitflags 2.8.0",
"itertools 0.13.0",
"ruff_python_ast",
"unic-ucd-category",
]
[[package]]
name = "ruff_python_parser"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"bitflags 2.8.0",
"bstr",
"memchr",
"ruff_python_ast",
"ruff_python_trivia",
"ruff_text_size",
"rustc-hash 1.1.0",
"static_assertions",
"unicode-ident",
"unicode-normalization",
"unicode_names2",
]
[[package]]
name = "ruff_python_trivia"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"itertools 0.13.0",
"ruff_source_file",
"ruff_text_size",
"unicode-ident",
]
[[package]]
name = "ruff_source_file"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
dependencies = [
"memchr",
"once_cell",
"ruff_text_size",
]
[[package]]
name = "ruff_text_size"
version = "0.0.0"
source = "git+https://github.com/astral-sh/ruff.git?tag=v0.4.10#b54922fd7394c36cdc390fd21aaee99206ebc361"
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -2006,26 +2093,14 @@ dependencies = [
"libc",
"log",
"pyo3",
"ruff_python_parser",
"rustpython-compiler",
"rustpython-parser",
"rustpython-pylib",
"rustpython-stdlib",
"rustpython-vm",
"rustyline",
]
[[package]]
name = "rustpython-ast"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"is-macro",
"malachite-bigint",
"rustpython-literal",
"rustpython-parser-core",
"static_assertions",
]
[[package]]
name = "rustpython-codegen"
version = "0.4.0"
@@ -2036,13 +2111,18 @@ dependencies = [
"insta",
"itertools 0.14.0",
"log",
"malachite-bigint",
"num-complex",
"num-traits",
"rustpython-ast",
"ruff_python_ast",
"ruff_python_codegen",
"ruff_python_parser",
"ruff_source_file",
"ruff_text_size",
"rustpython-common",
"rustpython-compiler-core",
"rustpython-parser",
"rustpython-parser-core",
"rustpython-compiler-source",
"thiserror 2.0.11",
]
[[package]]
@@ -2075,9 +2155,14 @@ dependencies = [
name = "rustpython-compiler"
version = "0.4.0"
dependencies = [
"ruff_python_ast",
"ruff_python_parser",
"ruff_source_file",
"ruff_text_size",
"rustpython-codegen",
"rustpython-compiler-core",
"rustpython-parser",
"rustpython-compiler-source",
"thiserror 2.0.11",
]
[[package]]
@@ -2089,10 +2174,20 @@ dependencies = [
"lz4_flex",
"malachite-bigint",
"num-complex",
"rustpython-parser-core",
"ruff_python_ast",
"ruff_python_parser",
"ruff_source_file",
"serde",
]
[[package]]
name = "rustpython-compiler-source"
version = "0.4.0"
dependencies = [
"ruff_source_file",
"ruff_text_size",
]
[[package]]
name = "rustpython-derive"
version = "0.4.0"
@@ -2113,7 +2208,6 @@ dependencies = [
"quote",
"rustpython-compiler-core",
"rustpython-doc",
"rustpython-parser-core",
"syn 2.0.98",
"syn-ext",
"textwrap 0.16.1",
@@ -2166,48 +2260,6 @@ dependencies = [
"unic-ucd-category",
]
[[package]]
name = "rustpython-parser"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"anyhow",
"is-macro",
"itertools 0.11.0",
"lalrpop-util",
"log",
"malachite-bigint",
"num-traits",
"phf",
"phf_codegen",
"rustc-hash 1.1.0",
"rustpython-ast",
"rustpython-parser-core",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustpython-parser-core"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"is-macro",
"memchr",
"rustpython-parser-vendored",
]
[[package]]
name = "rustpython-parser-vendored"
version = "0.4.0"
source = "git+https://github.com/RustPython/Parser.git?rev=d2f137b372ec08ce4a243564a80f8f9153c45a23#d2f137b372ec08ce4a243564a80f8f9153c45a23"
dependencies = [
"memchr",
"once_cell",
]
[[package]]
name = "rustpython-pylib"
version = "0.4.0"
@@ -2344,19 +2396,21 @@ dependencies = [
"paste",
"rand 0.9.0",
"result-like",
"ruff_python_ast",
"ruff_python_parser",
"ruff_source_file",
"ruff_text_size",
"rustc_version",
"rustix",
"rustpython-ast",
"rustpython-codegen",
"rustpython-common",
"rustpython-compiler",
"rustpython-compiler-core",
"rustpython-compiler-source",
"rustpython-derive",
"rustpython-format",
"rustpython-jit",
"rustpython-literal",
"rustpython-parser",
"rustpython-parser-core",
"rustpython-sre_engine",
"rustyline",
"schannel",
@@ -2388,8 +2442,8 @@ dependencies = [
"console_error_panic_hook",
"getrandom 0.2.15",
"js-sys",
"ruff_python_parser",
"rustpython-common",
"rustpython-parser",
"rustpython-pylib",
"rustpython-stdlib",
"rustpython-vm",
@@ -2793,15 +2847,6 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00"
[[package]]
name = "tiny-keccak"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
dependencies = [
"crunchy",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
@@ -2879,17 +2924,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
[[package]]
name = "unic-emoji-char"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-normal"
version = "0.9.0"
Cargo.toml
+11 -5
View File
@@ -28,7 +28,7 @@ rustpython-compiler = { workspace = true }
rustpython-pylib = { workspace = true, optional = true }
rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
rustpython-vm = { workspace = true, features = ["compiler"] }
rustpython-parser = { workspace = true }
ruff_python_parser = { workspace = true }
cfg-if = { workspace = true }
log = { workspace = true }
@@ -94,7 +94,7 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies =
[workspace]
resolver = "2"
members = [
"compiler", "compiler/core", "compiler/codegen",
"compiler", "compiler/core", "compiler/codegen", "compiler/source",
".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl",
"wasm/lib",
]
@@ -108,6 +108,7 @@ repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[workspace.dependencies]
rustpython-compiler-source = { path = "compiler/source" }
rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
rustpython-compiler = { path = "compiler", version = "0.4.0" }
rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }
@@ -121,15 +122,20 @@ rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.
rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
# rustpython-literal = { version = "0.4.0" }
# rustpython-parser-core = { version = "0.4.0" }
# rustpython-parser = { version = "0.4.0" }
# rustpython-ast = { version = "0.4.0" }
# rustpython-format= { version = "0.4.0" }
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-parser = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-ast = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
rustpython-format = { git = "https://github.com/RustPython/Parser.git", version = "0.4.0", rev = "d2f137b372ec08ce4a243564a80f8f9153c45a23" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-parser-core = { path = "../RustPython-parser/core" }
Lib/test/test_ast.py
Vendored
+8 -6
View File
@@ -388,6 +388,8 @@ class AST_Tests(unittest.TestCase):
with self.assertRaises(ValueError):
compile(tree, '<string>', 'exec')
# XXX RUSTPYTHON: we always require that end ranges be present
@unittest.expectedFailure
def test_compilation_of_ast_nodes_with_default_end_position_values(self):
tree = ast.Module(body=[
ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0),
@@ -1531,6 +1533,8 @@ Module(
malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])
self.assertRaises(ValueError, ast.literal_eval, malformed)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_literal_eval_trailing_ws(self):
self.assertEqual(ast.literal_eval(" -1"), -1)
self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1549,6 +1553,8 @@ Module(
with self.assertRaisesRegex(ValueError, msg):
ast.literal_eval(node)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_literal_eval_syntax_errors(self):
with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
ast.literal_eval(r'''
@@ -1569,6 +1575,8 @@ Module(
compile(mod, 'test', 'exec')
self.assertIn("invalid integer value: None", str(cm.exception))
# XXX RUSTPYTHON: we always require that end ranges be present
@unittest.expectedFailure
def test_level_as_none(self):
body = [ast.ImportFrom(module='time',
names=[ast.alias(name='sleep',
@@ -2034,8 +2042,6 @@ class ASTValidatorTests(unittest.TestCase):
call = ast.Call(func, args, bad_keywords)
self.expr(call, "must have Load context")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_num(self):
with warnings.catch_warnings(record=True) as wlog:
warnings.filterwarnings('ignore', '', DeprecationWarning)
@@ -2733,8 +2739,6 @@ class EndPositionTests(unittest.TestCase):
binop = self._parse_value(s_orig)
self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_source_segment_padded(self):
s_orig = dedent('''
class C:
@@ -2756,8 +2760,6 @@ class EndPositionTests(unittest.TestCase):
self._check_content(s, y, 'y = 1')
self._check_content(s, z, 'z = 1')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_source_segment_tabs(self):
s = dedent('''
class C:
Lib/test/test_codeop.py
Vendored
+2
View File
@@ -323,6 +323,8 @@ class CodeopTests(unittest.TestCase):
with self.assertRaisesRegex(SyntaxError, message):
compile_command(code, symbol='exec')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_syntax_errors(self):
self.assertSyntaxErrorMatches(
dedent("""\
Lib/test/test_eof.py
Vendored
+10 -2
View File
@@ -7,9 +7,9 @@ from test.support import script_helper
from test.support import warnings_helper
import unittest
# TODO: RUSTPYTHON
@unittest.expectedFailure
class EOFTestCase(unittest.TestCase):
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_EOF_single_quote(self):
expect = "unterminated string literal (detected at line 1) (<string>, line 1)"
for quote in ("'", "\""):
@@ -22,6 +22,8 @@ class EOFTestCase(unittest.TestCase):
else:
raise support.TestFailed
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_EOFS(self):
expect = ("unterminated triple-quoted string literal (detected at line 1) (<string>, line 1)")
try:
@@ -32,6 +34,8 @@ class EOFTestCase(unittest.TestCase):
else:
raise support.TestFailed
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_EOFS_with_file(self):
expect = ("(<string>, line 1)")
with os_helper.temp_dir() as temp_dir:
@@ -39,6 +43,8 @@ class EOFTestCase(unittest.TestCase):
rc, out, err = script_helper.assert_python_failure(file_name)
self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err)
# TODO: RUSTPYTHON
@unittest.expectedFailure
@warnings_helper.ignore_warnings(category=SyntaxWarning)
def test_eof_with_line_continuation(self):
expect = "unexpected EOF while parsing (<string>, line 1)"
@@ -49,6 +55,8 @@ class EOFTestCase(unittest.TestCase):
else:
raise support.TestFailed
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_line_continuation_EOF(self):
"""A continuation at the end of input must be an error; bpo2180."""
expect = 'unexpected EOF while parsing (<string>, line 1)'
Lib/test/test_exceptions.py
Vendored
+2
View File
@@ -50,6 +50,8 @@ class ExceptionTests(unittest.TestCase):
self.assertEqual(buf1, buf2)
self.assertEqual(exc.__name__, excname)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def testRaising(self):
self.raise_catch(AttributeError, "AttributeError")
self.assertRaises(AttributeError, getattr, sys, "undefined_attribute")
Lib/test/test_fstring.py
Vendored
+10 -4
View File
@@ -305,8 +305,6 @@ f'{a * x()} {a * x()} {a * x()}'
self.assertEqual(name.col_offset, 22)
self.assertEqual(name.end_col_offset, 25)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_ast_line_numbers_multiline_fstring(self):
# See bpo-30465 for details.
expr = """
@@ -509,8 +507,6 @@ x = (
self.assertEqual(type(format_spec), ast.JoinedStr)
self.assertEqual(len(format_spec.values), 0)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_docstring(self):
def f():
f"""Not a docstring"""
@@ -535,6 +531,8 @@ x = (
exec(c)
self.assertEqual(x[0], "foo3")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_compile_time_concat_errors(self):
self.assertAllRaise(
SyntaxError,
@@ -550,6 +548,8 @@ x = (
self.assertEqual(f"a", "a")
self.assertEqual(f" ", " ")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_unterminated_string(self):
self.assertAllRaise(
SyntaxError,
@@ -562,6 +562,8 @@ x = (
],
)
# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
def test_mismatched_parens(self):
self.assertAllRaise(
@@ -666,6 +668,8 @@ x = (
],
)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_compile_time_concat(self):
x = "def"
self.assertEqual("abc" f"## {x}ghi", "abc## defghi")
@@ -1516,6 +1520,8 @@ x = (
],
)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_del(self):
self.assertAllRaise(
SyntaxError,
Lib/test/test_future_stmt/test_future.py
Vendored
+8 -2
View File
@@ -67,6 +67,8 @@ class FutureTest(unittest.TestCase):
from test.test_future_stmt import badsyntax_future5
self.check_syntax_error(cm.exception, "badsyntax_future5", 4)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_badfuture6(self):
with self.assertRaises(SyntaxError) as cm:
from test.test_future_stmt import badsyntax_future6
@@ -132,8 +134,6 @@ class FutureTest(unittest.TestCase):
exec("from __future__ import unicode_literals; x = ''", {}, scope)
self.assertIsInstance(scope["x"], str)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_syntactical_future_repl(self):
p = spawn_python('-i')
p.stdin.write(b"from __future__ import barry_as_FLUFL\n")
@@ -198,6 +198,8 @@ class AnnotationsFutureTestCase(unittest.TestCase):
)
return scope
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_annotations(self):
eq = self.assertAnnotationEqual
eq('...')
@@ -362,6 +364,8 @@ class AnnotationsFutureTestCase(unittest.TestCase):
eq('(((a, b)))', '(a, b)')
eq("1 + 2 + 3")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_fstring_debug_annotations(self):
# f-strings with '=' don't round trip very well, so set the expected
# result explicitly.
@@ -372,6 +376,8 @@ class AnnotationsFutureTestCase(unittest.TestCase):
self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'")
self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_infinity_numbers(self):
inf = "1e" + repr(sys.float_info.max_10_exp + 1)
infj = f"{inf}j"
Lib/test/test_global.py
Vendored
+6
View File
@@ -12,6 +12,8 @@ class GlobalTests(unittest.TestCase):
self.enterContext(check_warnings())
warnings.filterwarnings("error", module="<test string>")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test1(self):
prog_text_1 = """\
def wrong1():
@@ -22,6 +24,8 @@ def wrong1():
"""
check_syntax_error(self, prog_text_1, lineno=4, offset=5)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test2(self):
prog_text_2 = """\
def wrong2():
@@ -30,6 +34,8 @@ def wrong2():
"""
check_syntax_error(self, prog_text_2, lineno=3, offset=5)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test3(self):
prog_text_3 = """\
def wrong3():
Lib/test/test_grammar.py
Vendored
+2
View File
@@ -1062,6 +1062,8 @@ class GrammarTests(unittest.TestCase):
self.assertEqual(g2(False), 0)
self.assertEqual(g2(True), ('end', 1))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_yield(self):
# Allowed as standalone statement
def g(): yield 1
Lib/test/test_keywordonlyarg.py
Vendored
-2
View File
@@ -40,8 +40,6 @@ class KeywordOnlyArgTestCase(unittest.TestCase):
compile(s, "<test>", "single")
self.assertRaises(SyntaxError, shouldRaiseSyntaxError, codestr)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def testSyntaxErrorForFunctionDefinition(self):
self.assertRaisesSyntaxError("def f(p, *):\n pass\n")
self.assertRaisesSyntaxError("def f(p1, *, p1=100):\n pass\n")
Lib/test/test_named_expressions.py
Vendored
+30 -4
View File
@@ -4,62 +4,80 @@ GLOBAL_VAR = None
class NamedExpressionInvalidTest(unittest.TestCase):
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_01(self):
code = """x := 0"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_02(self):
code = """x = y := 0"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_03(self):
code = """y := f(x)"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_04(self):
code = """y0 = y1 := f(x)"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON
@unittest.expectedFailure # wrong error message
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_06(self):
code = """((a, b) := (1, 2))"""
with self.assertRaisesRegex(SyntaxError, "cannot use assignment expressions with tuple"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_07(self):
code = """def spam(a = b := 42): pass"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_08(self):
code = """def spam(a: b := 42 = 5): pass"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_09(self):
code = """spam(a=b := 'c')"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_10(self):
code = """spam(x = y := f(x))"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_11(self):
code = """spam(a=1, b := 2)"""
@@ -67,6 +85,8 @@ class NamedExpressionInvalidTest(unittest.TestCase):
"positional argument follows keyword argument"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_12(self):
code = """spam(a=1, (b := 2))"""
@@ -74,6 +94,8 @@ class NamedExpressionInvalidTest(unittest.TestCase):
"positional argument follows keyword argument"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_13(self):
code = """spam(a=1, (b := 2))"""
@@ -81,14 +103,16 @@ class NamedExpressionInvalidTest(unittest.TestCase):
"positional argument follows keyword argument"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_14(self):
code = """(x := lambda: y := 1)"""
with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
exec(code, {}, {})
# TODO: RUSTPYTHON
@unittest.expectedFailure # wrong error message
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_15(self):
code = """(lambda: x := 1)"""
@@ -96,6 +120,8 @@ class NamedExpressionInvalidTest(unittest.TestCase):
"cannot use assignment expressions with lambda"):
exec(code, {}, {})
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_named_expression_invalid_16(self):
code = "[i + 1 for i in i := [1,2]]"
Lib/test/test_positional_only_arg.py
Vendored
+6
View File
@@ -22,6 +22,8 @@ class PositionalOnlyTestCase(unittest.TestCase):
with self.assertRaisesRegex(SyntaxError, regex):
compile(codestr + "\n", "<test>", "single")
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_invalid_syntax_errors(self):
check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
@@ -43,6 +45,8 @@ class PositionalOnlyTestCase(unittest.TestCase):
check_syntax_error(self, "def f(a, /, c, /, d, *, e): pass")
check_syntax_error(self, "def f(a, *, c, /, d, e): pass")
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_invalid_syntax_errors_async(self):
check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
@@ -236,6 +240,8 @@ class PositionalOnlyTestCase(unittest.TestCase):
x = lambda a, b, /, : a + b
self.assertEqual(x(1, 2), 3)
# TODO: RUSTPYTHON: wrong error message
@unittest.expectedFailure
def test_invalid_syntax_lambda(self):
check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument")
check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument")
Lib/test/test_syntax.py
Vendored
+18 -4
View File
@@ -742,7 +742,8 @@ SyntaxError: expression cannot contain assignment, perhaps you meant "=="?
>>> f(x.y=1)
Traceback (most recent call last):
SyntaxError: expression cannot contain assignment, perhaps you meant "=="?
>>> f((x)=2)
# TODO: RUSTPYTHON
>>> f((x)=2) # doctest: +SKIP
Traceback (most recent call last):
SyntaxError: expression cannot contain assignment, perhaps you meant "=="?
>>> f(True=1)
@@ -1649,7 +1650,8 @@ Corner-cases that used to crash:
Invalid pattern matching constructs:
>>> match ...:
# TODO: RUSTPYTHON nothing raised.
>>> match ...: # doctest: +SKIP
... case 42 as _:
... ...
Traceback (most recent call last):
@@ -1876,12 +1878,16 @@ class SyntaxTestCase(unittest.TestCase):
offset=7
)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_curly_brace_after_primary_raises_immediately(self):
self._check_error("f{}", "invalid syntax", mode="single")
def test_assign_call(self):
self._check_error("f() = 1", "assign")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_assign_del(self):
self._check_error("del (,)", "invalid syntax")
self._check_error("del 1", "cannot delete literal")
@@ -1994,6 +2000,8 @@ class SyntaxTestCase(unittest.TestCase):
"unindent does not match .* level",
subclass=IndentationError)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_kwargs_last(self):
self._check_error("int(base=10, '2')",
"positional argument follows keyword argument")
@@ -2005,6 +2013,8 @@ class SyntaxTestCase(unittest.TestCase):
"positional argument follows "
"keyword argument unpacking")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_kwargs_last3(self):
self._check_error("int(**{'base': 10}, *['2'])",
"iterable argument unpacking follows "
@@ -2073,8 +2083,6 @@ if x:
code += f"{' '*4*12}pass"
self._check_error(code, "too many statically nested blocks")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_barry_as_flufl_with_syntax_errors(self):
# The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if
# is reading the wrong token in the presence of syntax errors later
@@ -2092,6 +2100,8 @@ def func2():
"""
self._check_error(code, "expected ':'")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_invalid_line_continuation_error_position(self):
self._check_error(r"a = 3 \ 4",
"unexpected character after line continuation character",
@@ -2103,6 +2113,8 @@ def func2():
"unexpected character after line continuation character",
lineno=3, offset=4)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_invalid_line_continuation_left_recursive(self):
# Check bpo-42218: SyntaxErrors following left-recursive rules
# (t_primary_raw in this case) need to be tested explicitly
@@ -2146,6 +2158,8 @@ case(34)
"""
compile(code, "<string>", "exec")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_multiline_compiler_error_points_to_the_end(self):
self._check_error(
"call(\na=1,\na=1\n)",
Lib/test/test_timeit.py
Vendored
-2
View File
@@ -91,8 +91,6 @@ class TestTimeit(unittest.TestCase):
self.assertRaises(SyntaxError, timeit.Timer, setup='from timeit import *')
self.assertRaises(SyntaxError, timeit.Timer, setup=' pass')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_timer_empty_stmt(self):
timeit.Timer(stmt='')
timeit.Timer(stmt=' \n\t\f')
Lib/test/test_unicode_identifiers.py
Vendored
-4
View File
@@ -2,8 +2,6 @@ import unittest
class PEP3131Test(unittest.TestCase):
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_valid(self):
class T:
ä = 1
@@ -15,8 +13,6 @@ class PEP3131Test(unittest.TestCase):
self.assertEqual(getattr(T, '\u87d2'), 3)
self.assertEqual(getattr(T, 'x\U000E0100'), 4)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_non_bmp_normalized(self):
𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1
self.assertIn("Unicode", dir())
compiler/Cargo.toml
+7 -1
View File
@@ -11,7 +11,13 @@ license.workspace = true
[dependencies]
rustpython-codegen = { workspace = true }
rustpython-compiler-core = { workspace = true }
rustpython-parser = { workspace = true }
rustpython-compiler-source = { workspace = true }
# rustpython-parser = { workspace = true }
ruff_python_parser = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_source_file = { workspace = true }
ruff_text_size = { workspace = true }
thiserror = { workspace = true }
[lints]
workspace = true
compiler/codegen/Cargo.toml
+12 -4
View File
@@ -10,10 +10,16 @@ license.workspace = true
[dependencies]
rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] }
# rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] }
rustpython-common = { workspace = true }
rustpython-parser-core = { workspace = true }
# rustpython-parser-core = { workspace = true }
rustpython-compiler-core = { workspace = true }
rustpython-compiler-source = {workspace = true }
ruff_python_parser = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_text_size = { workspace = true }
ruff_source_file = { workspace = true }
ruff_python_codegen = { workspace = true }
ahash = { workspace = true }
bitflags = { workspace = true }
@@ -22,11 +28,13 @@ itertools = { workspace = true }
log = { workspace = true }
num-complex = { workspace = true }
num-traits = { workspace = true }
thiserror = { workspace = true }
malachite-bigint = { workspace = true }
[dev-dependencies]
rustpython-parser = { workspace = true }
# rustpython-parser = { workspace = true }
insta = { workspace = true }
[lints]
workspace = true
workspace = true
compiler/codegen/src/compile.rs
+822 -601
View File
File diff suppressed because it is too large. Load diff
compiler/codegen/src/error.rs
+18 -1
View File
@@ -1,6 +1,23 @@
use ruff_source_file::SourceLocation;
use std::fmt;
use thiserror::Error;
pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>;
// pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>;
#[derive(Error, Debug)]
pub struct CodegenError {
pub location: Option<SourceLocation>,
#[source]
pub error: CodegenErrorType,
pub source_path: String,
}
impl fmt::Display for CodegenError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// TODO:
self.error.fmt(f)
}
}
#[derive(Debug)]
#[non_exhaustive]
compiler/codegen/src/ir.rs
+7 -4
View File
@@ -1,10 +1,11 @@
use std::ops;
use crate::IndexSet;
use ruff_source_file::{OneIndexed, SourceLocation};
use rustpython_compiler_core::bytecode::{
CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg,
};
use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
// use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct BlockIdx(pub u32);
@@ -37,11 +38,12 @@ impl ops::IndexMut<BlockIdx> for Vec<Block> {
}
}
#[derive(Debug, Copy, Clone)]
#[derive(Debug, Clone)]
pub struct InstructionInfo {
pub instr: Instruction,
pub arg: OpArg,
pub target: BlockIdx,
// pub range: TextRange,
pub location: SourceLocation,
}
@@ -68,7 +70,7 @@ pub struct CodeInfo {
pub arg_count: u32,
pub kwonlyarg_count: u32,
pub source_path: String,
pub first_line_number: LineNumber,
pub first_line_number: OneIndexed,
pub obj_name: String, // Name of the object that created this code object
pub blocks: Vec<Block>,
@@ -134,7 +136,8 @@ impl CodeInfo {
*arg = new_arg;
}
let (extras, lo_arg) = arg.split();
locations.extend(std::iter::repeat(info.location).take(arg.instr_size()));
locations
.extend(std::iter::repeat(info.location.clone()).take(arg.instr_size()));
instructions.extend(
extras
.map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte))
compiler/codegen/src/lib.rs
+46
View File
@@ -14,3 +14,49 @@ pub mod ir;
pub mod symboltable;
pub use compile::CompileOpts;
use ruff_python_ast::Expr;
pub trait ToPythonName {
/// Returns a short name for the node suitable for use in error messages.
fn python_name(&self) -> &'static str;
}
impl ToPythonName for Expr {
fn python_name(&self) -> &'static str {
match self {
Expr::BoolOp { .. } | Expr::BinOp { .. } | Expr::UnaryOp { .. } => "operator",
Expr::Subscript { .. } => "subscript",
Expr::Await { .. } => "await expression",
Expr::Yield { .. } | Expr::YieldFrom { .. } => "yield expression",
Expr::Compare { .. } => "comparison",
Expr::Attribute { .. } => "attribute",
Expr::Call { .. } => "function call",
Expr::BooleanLiteral(b) => {
if b.value {
"True"
} else {
"False"
}
}
Expr::EllipsisLiteral(_) => "ellipsis",
Expr::NoneLiteral(_) => "None",
Expr::NumberLiteral(_) | Expr::BytesLiteral(_) | Expr::StringLiteral(_) => "literal",
Expr::Tuple(_) => "tuple",
Expr::List { .. } => "list",
Expr::Dict { .. } => "dict display",
Expr::Set { .. } => "set display",
Expr::ListComp { .. } => "list comprehension",
Expr::DictComp { .. } => "dict comprehension",
Expr::SetComp { .. } => "set comprehension",
Expr::Generator { .. } => "generator expression",
Expr::Starred { .. } => "starred",
Expr::Slice { .. } => "slice",
Expr::FString { .. } => "f-string expression",
Expr::Name { .. } => "name",
Expr::Lambda { .. } => "lambda",
Expr::If { .. } => "conditional expression",
Expr::Named { .. } => "named expression",
Expr::IpyEscapeCommand(_) => todo!(),
}
}
}
compiler/codegen/src/symboltable.rs
+275 -188
View File
@@ -12,8 +12,16 @@ use crate::{
error::{CodegenError, CodegenErrorType},
};
use bitflags::bitflags;
use rustpython_ast::{self as ast, located::Located};
use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
use ruff_python_ast::{
self as ast, Comprehension, Decorator, Expr, Identifier, ModExpression, ModModule, Parameter,
ParameterWithDefault, Parameters, Pattern, PatternMatchAs, PatternMatchClass,
PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchStar, PatternMatchValue,
Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams,
};
use ruff_text_size::{Ranged, TextRange};
use rustpython_compiler_source::{SourceCode, SourceLocation};
// use rustpython_ast::{self as ast, located::Located};
// use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
use std::{borrow::Cow, fmt};
/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope.
@@ -51,15 +59,18 @@ impl SymbolTable {
}
}
pub fn scan_program(program: &[ast::located::Stmt]) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_statements(program)?;
pub fn scan_program(
program: &ModModule,
source_code: SourceCode<'_>,
) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new(source_code);
builder.scan_statements(program.body.as_ref())?;
builder.finish()
}
pub fn scan_expr(expr: &ast::located::Expr) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_expression(expr, ExpressionContext::Load)?;
pub fn scan_expr(expr: &ModExpression, source_code: SourceCode<'_>) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new(source_code);
builder.scan_expression(expr.body.as_ref(), ExpressionContext::Load)?;
builder.finish()
}
}
@@ -176,11 +187,8 @@ pub struct SymbolTableError {
impl SymbolTableError {
pub fn into_codegen_error(self, source_path: String) -> CodegenError {
CodegenError {
location: self.location,
error: CodegenErrorType::SyntaxError(self.error),
location: self.location.map(|l| SourceLocation {
row: l.row,
column: l.column,
}),
source_path,
}
}
@@ -551,11 +559,12 @@ enum SymbolUsage {
Iter,
}
struct SymbolTableBuilder {
struct SymbolTableBuilder<'src> {
class_name: Option<String>,
// Scope stack.
tables: Vec<SymbolTable>,
future_annotations: bool,
source_code: SourceCode<'src>,
}
/// Enum to indicate in what mode an expression
@@ -571,19 +580,20 @@ enum ExpressionContext {
IterDefinitionExp,
}
impl SymbolTableBuilder {
fn new() -> Self {
impl<'src> SymbolTableBuilder<'src> {
fn new(source_code: SourceCode<'src>) -> Self {
let mut this = Self {
class_name: None,
tables: vec![],
future_annotations: false,
source_code,
};
this.enter_scope("top", SymbolTableType::Module, 0);
this
}
}
impl SymbolTableBuilder {
impl SymbolTableBuilder<'_> {
fn finish(mut self) -> Result<SymbolTable, SymbolTableError> {
assert_eq!(self.tables.len(), 1);
let mut symbol_table = self.tables.pop().unwrap();
@@ -607,38 +617,34 @@ impl SymbolTableBuilder {
self.tables.last_mut().unwrap().sub_tables.push(table);
}
fn scan_statements(&mut self, statements: &[ast::located::Stmt]) -> SymbolTableResult {
fn line_index_start(&self, range: TextRange) -> u32 {
self.source_code.line_index(range.start()).get() as _
}
fn scan_statements(&mut self, statements: &[Stmt]) -> SymbolTableResult {
for statement in statements {
self.scan_statement(statement)?;
}
Ok(())
}
fn scan_parameters(
&mut self,
parameters: &[ast::located::ArgWithDefault],
) -> SymbolTableResult {
fn scan_parameters(&mut self, parameters: &[ParameterWithDefault]) -> SymbolTableResult {
for parameter in parameters {
let usage = if parameter.def.annotation.is_some() {
SymbolUsage::AnnotationParameter
} else {
SymbolUsage::Parameter
};
self.register_name(parameter.def.arg.as_str(), usage, parameter.def.location())?;
self.scan_parameter(&parameter.parameter)?;
}
Ok(())
}
fn scan_parameter(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
fn scan_parameter(&mut self, parameter: &Parameter) -> SymbolTableResult {
let usage = if parameter.annotation.is_some() {
SymbolUsage::AnnotationParameter
} else {
SymbolUsage::Parameter
};
self.register_name(parameter.arg.as_str(), usage, parameter.location())
self.register_ident(&parameter.name, usage)
}
fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult {
fn scan_annotation(&mut self, annotation: &Expr) -> SymbolTableResult {
if self.future_annotations {
Ok(())
} else {
@@ -646,8 +652,8 @@ impl SymbolTableBuilder {
}
}
fn scan_statement(&mut self, statement: &ast::located::Stmt) -> SymbolTableResult {
use ast::located::*;
fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult {
use ruff_python_ast::*;
if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement {
if module.as_ref().map(|id| id.as_str()) == Some("__future__") {
for feature in names {
@@ -658,101 +664,109 @@ impl SymbolTableBuilder {
}
}
match &statement {
Stmt::Global(StmtGlobal { names, range }) => {
Stmt::Global(StmtGlobal { names, .. }) => {
for name in names {
self.register_name(name.as_str(), SymbolUsage::Global, range.start)?;
self.register_ident(name, SymbolUsage::Global)?;
}
}
Stmt::Nonlocal(StmtNonlocal { names, range }) => {
Stmt::Nonlocal(StmtNonlocal { names, .. }) => {
for name in names {
self.register_name(name.as_str(), SymbolUsage::Nonlocal, range.start)?;
self.register_ident(name, SymbolUsage::Nonlocal)?;
}
}
Stmt::FunctionDef(StmtFunctionDef {
name,
body,
args,
decorator_list,
type_params,
returns,
range,
..
})
| Stmt::AsyncFunctionDef(StmtAsyncFunctionDef {
name,
body,
args,
parameters,
decorator_list,
type_params,
returns,
range,
..
}) => {
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?;
self.scan_decorators(decorator_list, ExpressionContext::Load)?;
self.register_ident(name, SymbolUsage::Assigned)?;
if let Some(expression) = returns {
self.scan_annotation(expression)?;
}
if !type_params.is_empty() {
if let Some(type_params) = type_params {
self.enter_scope(
&format!("<generic parameters of {}>", name.as_str()),
SymbolTableType::TypeParams,
range.start.row.get(),
// FIXME: line no
self.line_index_start(*range),
);
self.scan_type_params(type_params)?;
}
self.enter_function(name.as_str(), args, range.start.row)?;
self.enter_scope_with_parameters(
name.as_str(),
parameters,
self.line_index_start(*range),
)?;
self.scan_statements(body)?;
self.leave_scope();
if !type_params.is_empty() {
if type_params.is_some() {
self.leave_scope();
}
}
Stmt::ClassDef(StmtClassDef {
name,
body,
bases,
keywords,
arguments,
decorator_list,
type_params,
range,
}) => {
if !type_params.is_empty() {
if let Some(type_params) = type_params {
self.enter_scope(
&format!("<generic parameters of {}>", name.as_str()),
SymbolTableType::TypeParams,
range.start.row.get(),
self.line_index_start(type_params.range),
);
self.scan_type_params(type_params)?;
}
self.enter_scope(name.as_str(), SymbolTableType::Class, range.start.row.get());
self.enter_scope(
name.as_str(),
SymbolTableType::Class,
self.line_index_start(*range),
);
let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string()));
self.register_name("__module__", SymbolUsage::Assigned, range.start)?;
self.register_name("__qualname__", SymbolUsage::Assigned, range.start)?;
self.register_name("__doc__", SymbolUsage::Assigned, range.start)?;
self.register_name("__class__", SymbolUsage::Assigned, range.start)?;
self.register_name("__module__", SymbolUsage::Assigned, *range)?;
self.register_name("__qualname__", SymbolUsage::Assigned, *range)?;
self.register_name("__doc__", SymbolUsage::Assigned, *range)?;
self.register_name("__class__", SymbolUsage::Assigned, *range)?;
self.scan_statements(body)?;
self.leave_scope();
self.class_name = prev_class;
self.scan_expressions(bases, ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expression(&keyword.value, ExpressionContext::Load)?;
if let Some(arguments) = arguments {
self.scan_expressions(&arguments.args, ExpressionContext::Load)?;
for keyword in &arguments.keywords {
self.scan_expression(&keyword.value, ExpressionContext::Load)?;
}
}
if !type_params.is_empty() {
if type_params.is_some() {
self.leave_scope();
}
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?;
self.scan_decorators(decorator_list, ExpressionContext::Load)?;
self.register_ident(name, SymbolUsage::Assigned)?;
}
Stmt::Expr(StmtExpr { value, .. }) => {
self.scan_expression(value, ExpressionContext::Load)?
}
Stmt::If(StmtIf {
test, body, orelse, ..
test,
body,
elif_else_clauses,
..
}) => {
self.scan_expression(test, ExpressionContext::Load)?;
self.scan_statements(body)?;
self.scan_statements(orelse)?;
for elif in elif_else_clauses {
if let Some(test) = &elif.test {
self.scan_expression(test, ExpressionContext::Load)?;
}
self.scan_statements(&elif.body)?;
}
}
Stmt::For(StmtFor {
target,
@@ -760,13 +774,6 @@ impl SymbolTableBuilder {
body,
orelse,
..
})
| Stmt::AsyncFor(StmtAsyncFor {
target,
iter,
body,
orelse,
..
}) => {
self.scan_expression(target, ExpressionContext::Store)?;
self.scan_expression(iter, ExpressionContext::Load)?;
@@ -783,18 +790,18 @@ impl SymbolTableBuilder {
Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => {
// No symbols here.
}
Stmt::Import(StmtImport { names, range })
| Stmt::ImportFrom(StmtImportFrom { names, range, .. }) => {
Stmt::Import(StmtImport { names, .. })
| Stmt::ImportFrom(StmtImportFrom { names, .. }) => {
for name in names {
if let Some(alias) = &name.asname {
// `import my_module as my_alias`
self.register_name(alias.as_str(), SymbolUsage::Imported, range.start)?;
self.register_ident(alias, SymbolUsage::Imported)?;
} else {
// `import module`
self.register_name(
name.name.split('.').next().unwrap(),
SymbolUsage::Imported,
range.start,
name.name.range,
)?;
}
}
@@ -831,11 +838,7 @@ impl SymbolTableBuilder {
// https://github.com/python/cpython/blob/main/Python/symtable.c#L1233
match &**target {
Expr::Name(ast::ExprName { id, .. }) if *simple => {
self.register_name(
id.as_str(),
SymbolUsage::AnnotationAssigned,
range.start,
)?;
self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, *range)?;
}
_ => {
self.scan_expression(target, ExpressionContext::Store)?;
@@ -846,8 +849,7 @@ impl SymbolTableBuilder {
self.scan_expression(value, ExpressionContext::Load)?;
}
}
Stmt::With(StmtWith { items, body, .. })
| Stmt::AsyncWith(StmtAsyncWith { items, body, .. }) => {
Stmt::With(StmtWith { items, body, .. }) => {
for item in items {
self.scan_expression(&item.context_expr, ExpressionContext::Load)?;
if let Some(expression) = &item.optional_vars {
@@ -861,14 +863,7 @@ impl SymbolTableBuilder {
handlers,
orelse,
finalbody,
range,
})
| Stmt::TryStar(StmtTryStar {
body,
handlers,
orelse,
finalbody,
range,
..
}) => {
self.scan_statements(body)?;
for handler in handlers {
@@ -882,7 +877,7 @@ impl SymbolTableBuilder {
self.scan_expression(expression, ExpressionContext::Load)?;
}
if let Some(name) = name {
self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?;
self.register_ident(name, SymbolUsage::Assigned)?;
}
self.scan_statements(body)?;
}
@@ -892,8 +887,10 @@ impl SymbolTableBuilder {
Stmt::Match(StmtMatch { subject, cases, .. }) => {
self.scan_expression(subject, ExpressionContext::Load)?;
for case in cases {
// TODO: below
// self.scan_pattern(&case.pattern, ExpressionContext::Load)?;
self.scan_pattern(&case.pattern)?;
if let Some(guard) = &case.guard {
self.scan_expression(guard, ExpressionContext::Load)?;
}
self.scan_statements(&case.body)?;
}
}
@@ -909,13 +906,14 @@ impl SymbolTableBuilder {
name,
value,
type_params,
range,
..
}) => {
if !type_params.is_empty() {
if let Some(type_params) = type_params {
self.enter_scope(
&name.to_string(),
// &name.to_string(),
"TypeAlias",
SymbolTableType::TypeParams,
range.start.row.get(),
self.line_index_start(type_params.range),
);
self.scan_type_params(type_params)?;
self.scan_expression(value, ExpressionContext::Load)?;
@@ -925,13 +923,25 @@ impl SymbolTableBuilder {
}
self.scan_expression(name, ExpressionContext::Store)?;
}
Stmt::IpyEscapeCommand(_) => todo!(),
}
Ok(())
}
fn scan_decorators(
&mut self,
decorators: &[Decorator],
context: ExpressionContext,
) -> SymbolTableResult {
for decorator in decorators {
self.scan_expression(&decorator.expression, context)?;
}
Ok(())
}
fn scan_expressions(
&mut self,
expressions: &[ast::located::Expr],
expressions: &[Expr],
context: ExpressionContext,
) -> SymbolTableResult {
for expression in expressions {
@@ -942,10 +952,10 @@ impl SymbolTableBuilder {
fn scan_expression(
&mut self,
expression: &ast::located::Expr,
expression: &Expr,
context: ExpressionContext,
) -> SymbolTableResult {
use ast::located::*;
use ruff_python_ast::*;
match expression {
Expr::BinOp(ExprBinOp {
left,
@@ -984,16 +994,12 @@ impl SymbolTableBuilder {
}) => {
self.scan_expression(value, ExpressionContext::Load)?;
}
Expr::Dict(ExprDict {
keys,
values,
range: _,
}) => {
for (key, value) in keys.iter().zip(values.iter()) {
if let Some(key) = key {
Expr::Dict(ExprDict { items, range: _ }) => {
for item in items {
if let Some(key) = &item.key {
self.scan_expression(key, context)?;
}
self.scan_expression(value, context)?;
self.scan_expression(&item.value, context)?;
}
}
Expr::Await(ExprAwait { value, range: _ }) => {
@@ -1012,7 +1018,6 @@ impl SymbolTableBuilder {
}) => {
self.scan_expression(operand, context)?;
}
Expr::Constant(ExprConstant { range: _, .. }) => {}
Expr::Starred(ExprStarred {
value, range: _, ..
}) => {
@@ -1039,26 +1044,27 @@ impl SymbolTableBuilder {
self.scan_expression(step, context)?;
}
}
Expr::GeneratorExp(ExprGeneratorExp {
Expr::Generator(ExprGenerator {
elt,
generators,
range,
..
}) => {
self.scan_comprehension("genexpr", elt, None, generators, range.start)?;
self.scan_comprehension("genexpr", elt, None, generators, *range)?;
}
Expr::ListComp(ExprListComp {
elt,
generators,
range,
}) => {
self.scan_comprehension("genexpr", elt, None, generators, range.start)?;
self.scan_comprehension("genexpr", elt, None, generators, *range)?;
}
Expr::SetComp(ExprSetComp {
elt,
generators,
range,
}) => {
self.scan_comprehension("genexpr", elt, None, generators, range.start)?;
self.scan_comprehension("genexpr", elt, None, generators, *range)?;
}
Expr::DictComp(ExprDictComp {
key,
@@ -1066,12 +1072,11 @@ impl SymbolTableBuilder {
generators,
range,
}) => {
self.scan_comprehension("genexpr", key, Some(value), generators, range.start)?;
self.scan_comprehension("genexpr", key, Some(value), generators, *range)?;
}
Expr::Call(ExprCall {
func,
args,
keywords,
arguments,
range: _,
}) => {
match context {
@@ -1083,43 +1088,27 @@ impl SymbolTableBuilder {
}
}
self.scan_expressions(args, ExpressionContext::Load)?;
for keyword in keywords {
self.scan_expressions(&arguments.args, ExpressionContext::Load)?;
for keyword in &arguments.keywords {
self.scan_expression(&keyword.value, ExpressionContext::Load)?;
}
}
Expr::FormattedValue(ExprFormattedValue {
value,
format_spec,
range: _,
..
}) => {
self.scan_expression(value, ExpressionContext::Load)?;
if let Some(spec) = format_spec {
self.scan_expression(spec, ExpressionContext::Load)?;
}
}
Expr::JoinedStr(ExprJoinedStr { values, range: _ }) => {
for value in values {
self.scan_expression(value, ExpressionContext::Load)?;
}
}
Expr::Name(ExprName { id, range, .. }) => {
let id = id.as_str();
// Determine the contextual usage of this symbol:
match context {
ExpressionContext::Delete => {
self.register_name(id, SymbolUsage::Assigned, range.start)?;
self.register_name(id, SymbolUsage::Used, range.start)?;
self.register_name(id, SymbolUsage::Assigned, *range)?;
self.register_name(id, SymbolUsage::Used, *range)?;
}
ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
self.register_name(id, SymbolUsage::Used, range.start)?;
self.register_name(id, SymbolUsage::Used, *range)?;
}
ExpressionContext::Store => {
self.register_name(id, SymbolUsage::Assigned, range.start)?;
self.register_name(id, SymbolUsage::Assigned, *range)?;
}
ExpressionContext::Iter => {
self.register_name(id, SymbolUsage::Iter, range.start)?;
self.register_name(id, SymbolUsage::Iter, *range)?;
}
}
// Interesting stuff about the __class__ variable:
@@ -1128,15 +1117,27 @@ impl SymbolTableBuilder {
&& self.tables.last().unwrap().typ == SymbolTableType::Function
&& id == "super"
{
self.register_name("__class__", SymbolUsage::Used, range.start)?;
self.register_name("__class__", SymbolUsage::Used, *range)?;
}
}
Expr::Lambda(ExprLambda {
args,
body,
parameters,
range: _,
}) => {
self.enter_function("lambda", args, expression.location().row)?;
if let Some(parameters) = parameters {
self.enter_scope_with_parameters(
"lambda",
parameters,
self.line_index_start(expression.range()),
)?;
} else {
self.enter_scope(
"lambda",
SymbolTableType::Function,
self.line_index_start(expression.range()),
);
}
match context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
@@ -1147,7 +1148,25 @@ impl SymbolTableBuilder {
}
self.leave_scope();
}
Expr::IfExp(ExprIfExp {
Expr::FString(ExprFString { value, .. }) => {
for expr in value.elements().filter_map(|x| x.as_expression()) {
self.scan_expression(&expr.expression, ExpressionContext::Load)?;
if let Some(format_spec) = &expr.format_spec {
for element in format_spec.elements.expressions() {
self.scan_expression(&element.expression, ExpressionContext::Load)?
}
}
}
}
// Constants
Expr::StringLiteral(_)
| Expr::BytesLiteral(_)
| Expr::NumberLiteral(_)
| Expr::BooleanLiteral(_)
| Expr::NoneLiteral(_)
| Expr::EllipsisLiteral(_) => {}
Expr::IpyEscapeCommand(_) => todo!(),
Expr::If(ExprIf {
test,
body,
orelse,
@@ -1158,7 +1177,7 @@ impl SymbolTableBuilder {
self.scan_expression(orelse, ExpressionContext::Load)?;
}
Expr::NamedExpr(ExprNamedExpr {
Expr::Named(ExprNamed {
target,
value,
range,
@@ -1167,9 +1186,9 @@ impl SymbolTableBuilder {
// comprehension iterator definitions
if let ExpressionContext::IterDefinitionExp = context {
return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
location: Some(target.location()),
});
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
location: Some(self.source_code.source_location(target.range().start())),
});
}
self.scan_expression(value, ExpressionContext::Load)?;
@@ -1185,13 +1204,13 @@ impl SymbolTableBuilder {
self.register_name(
id,
SymbolUsage::AssignedNamedExprInComprehension,
range.start,
*range,
)?;
} else {
// omit one recursion. When the handling of an store changes for
// Identifiers this needs adapted - more forward safe would be
// calling scan_expression directly.
self.register_name(id, SymbolUsage::Assigned, range.start)?;
self.register_name(id, SymbolUsage::Assigned, *range)?;
}
} else {
self.scan_expression(target, ExpressionContext::Store)?;
@@ -1204,20 +1223,20 @@ impl SymbolTableBuilder {
fn scan_comprehension(
&mut self,
scope_name: &str,
elt1: &ast::located::Expr,
elt2: Option<&ast::located::Expr>,
generators: &[ast::located::Comprehension],
location: SourceLocation,
elt1: &Expr,
elt2: Option<&Expr>,
generators: &[Comprehension],
range: TextRange,
) -> SymbolTableResult {
// Comprehensions are compiled as functions, so create a scope for them:
self.enter_scope(
scope_name,
SymbolTableType::Comprehension,
location.row.get(),
self.line_index_start(range),
);
// Register the passed argument to the generator function as the name ".0"
self.register_name(".0", SymbolUsage::Parameter, location)?;
self.register_name(".0", SymbolUsage::Parameter, range)?;
self.scan_expression(elt1, ExpressionContext::Load)?;
if let Some(elt2) = elt2 {
@@ -1247,91 +1266,159 @@ impl SymbolTableBuilder {
Ok(())
}
fn scan_type_params(&mut self, type_params: &[ast::located::TypeParam]) -> SymbolTableResult {
for type_param in type_params {
fn scan_type_params(&mut self, type_params: &TypeParams) -> SymbolTableResult {
for type_param in &type_params.type_params {
match type_param {
ast::located::TypeParam::TypeVar(ast::TypeParamTypeVar {
TypeParam::TypeVar(TypeParamTypeVar {
name,
bound,
range: type_var_range,
..
}) => {
self.register_name(name.as_str(), SymbolUsage::Assigned, type_var_range.start)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, *type_var_range)?;
if let Some(binding) = bound {
self.scan_expression(binding, ExpressionContext::Load)?;
}
}
ast::located::TypeParam::ParamSpec(ast::TypeParamParamSpec {
TypeParam::ParamSpec(TypeParamParamSpec {
name,
range: param_spec_range,
..
}) => {
self.register_name(name, SymbolUsage::Assigned, param_spec_range.start)?;
self.register_name(name, SymbolUsage::Assigned, *param_spec_range)?;
}
ast::located::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple {
TypeParam::TypeVarTuple(TypeParamTypeVarTuple {
name,
range: type_var_tuple_range,
..
}) => {
self.register_name(name, SymbolUsage::Assigned, type_var_tuple_range.start)?;
self.register_name(name, SymbolUsage::Assigned, *type_var_tuple_range)?;
}
}
}
Ok(())
}
fn enter_function(
fn scan_patterns(&mut self, patterns: &[Pattern]) -> SymbolTableResult {
for pattern in patterns {
self.scan_pattern(pattern)?;
}
Ok(())
}
fn scan_pattern(&mut self, pattern: &Pattern) -> SymbolTableResult {
use Pattern::*;
match pattern {
MatchValue(PatternMatchValue { value, .. }) => {
self.scan_expression(value, ExpressionContext::Load)?
}
MatchSingleton(_) => {}
MatchSequence(PatternMatchSequence { patterns, .. }) => self.scan_patterns(patterns)?,
MatchMapping(PatternMatchMapping {
keys,
patterns,
rest,
..
}) => {
self.scan_expressions(keys, ExpressionContext::Load)?;
self.scan_patterns(patterns)?;
if let Some(rest) = rest {
self.register_ident(rest, SymbolUsage::Assigned)?;
}
}
MatchClass(PatternMatchClass { cls, arguments, .. }) => {
self.scan_expression(cls, ExpressionContext::Load)?;
self.scan_patterns(&arguments.patterns)?;
for kw in &arguments.keywords {
self.scan_pattern(&kw.pattern)?;
}
}
MatchStar(PatternMatchStar { name, .. }) => {
if let Some(name) = name {
self.register_ident(name, SymbolUsage::Assigned)?;
}
}
MatchAs(PatternMatchAs { pattern, name, .. }) => {
if let Some(pattern) = pattern {
self.scan_pattern(pattern)?;
}
if let Some(name) = name {
self.register_ident(name, SymbolUsage::Assigned)?;
}
}
MatchOr(PatternMatchOr { patterns, .. }) => self.scan_patterns(patterns)?,
}
Ok(())
}
fn enter_scope_with_parameters(
&mut self,
name: &str,
args: &ast::located::Arguments,
line_number: LineNumber,
parameters: &Parameters,
line_number: u32,
) -> SymbolTableResult {
// Evaluate eventual default parameters:
for default in args
for default in parameters
.posonlyargs
.iter()
.chain(args.args.iter())
.chain(args.kwonlyargs.iter())
.chain(parameters.args.iter())
.chain(parameters.kwonlyargs.iter())
.filter_map(|arg| arg.default.as_ref())
{
self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext?
}
// Annotations are scanned in outer scope:
for annotation in args
for annotation in parameters
.posonlyargs
.iter()
.chain(args.args.iter())
.chain(args.kwonlyargs.iter())
.filter_map(|arg| arg.def.annotation.as_ref())
.chain(parameters.args.iter())
.chain(parameters.kwonlyargs.iter())
.filter_map(|arg| arg.parameter.annotation.as_ref())
{
self.scan_annotation(annotation)?;
}
if let Some(annotation) = args.vararg.as_ref().and_then(|arg| arg.annotation.as_ref()) {
if let Some(annotation) = parameters
.vararg
.as_ref()
.and_then(|arg| arg.annotation.as_ref())
{
self.scan_annotation(annotation)?;
}
if let Some(annotation) = args.kwarg.as_ref().and_then(|arg| arg.annotation.as_ref()) {
if let Some(annotation) = parameters
.kwarg
.as_ref()
.and_then(|arg| arg.annotation.as_ref())
{
self.scan_annotation(annotation)?;
}
self.enter_scope(name, SymbolTableType::Function, line_number.get());
self.enter_scope(name, SymbolTableType::Function, line_number);
// Fill scope with parameter names:
self.scan_parameters(&args.posonlyargs)?;
self.scan_parameters(&args.args)?;
self.scan_parameters(&args.kwonlyargs)?;
if let Some(name) = &args.vararg {
self.scan_parameters(&parameters.posonlyargs)?;
self.scan_parameters(&parameters.args)?;
self.scan_parameters(&parameters.kwonlyargs)?;
if let Some(name) = &parameters.vararg {
self.scan_parameter(name)?;
}
if let Some(name) = &args.kwarg {
if let Some(name) = &parameters.kwarg {
self.scan_parameter(name)?;
}
Ok(())
}
fn register_ident(&mut self, ident: &Identifier, role: SymbolUsage) -> SymbolTableResult {
self.register_name(ident.as_str(), role, ident.range)
}
fn register_name(
&mut self,
name: &str,
role: SymbolUsage,
location: SourceLocation,
range: TextRange,
) -> SymbolTableResult {
let location = self.source_code.source_location(range.start());
let location = Some(location);
let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap();
compiler/core/Cargo.toml
+4 -1
View File
@@ -9,7 +9,10 @@ repository.workspace = true
license.workspace = true
[dependencies]
rustpython-parser-core = { workspace = true, features=["location"] }
# rustpython-parser-core = { workspace = true, features=["location"] }
ruff_python_ast = { workspace = true }
ruff_python_parser = { workspace = true }
ruff_source_file = { workspace = true }
bitflags = { workspace = true }
itertools = { workspace = true }
compiler/core/src/bytecode.rs
+4 -2
View File
@@ -5,11 +5,13 @@ use bitflags::bitflags;
use itertools::Itertools;
use malachite_bigint::BigInt;
use num_complex::Complex64;
use rustpython_parser_core::source_code::{OneIndexed, SourceLocation};
pub use ruff_python_ast::ConversionFlag;
// use rustpython_parser_core::source_code::{OneIndexed, SourceLocation};
use ruff_source_file::{OneIndexed, SourceLocation};
use std::marker::PhantomData;
use std::{collections::BTreeSet, fmt, hash, mem};
pub use rustpython_parser_core::ConversionFlag;
// pub use rustpython_parser_core::ConversionFlag;
pub trait Constant: Sized {
type Name: AsRef<str>;
compiler/core/src/marshal.rs
+7 -7
View File
@@ -1,7 +1,7 @@
use crate::bytecode::*;
use malachite_bigint::{BigInt, Sign};
use num_complex::Complex64;
use rustpython_parser_core::source_code::{OneIndexed, SourceLocation};
use ruff_source_file::{OneIndexed, SourceLocation};
use std::convert::Infallible;
pub const FORMAT_VERSION: u32 = 4;
@@ -185,8 +185,8 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>(
let locations = (0..len)
.map(|_| {
Ok(SourceLocation {
row: OneIndexed::new(rdr.read_u32()?).ok_or(MarshalError::InvalidLocation)?,
column: OneIndexed::from_zero_indexed(rdr.read_u32()?),
row: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?,
column: OneIndexed::from_zero_indexed(rdr.read_u32()? as _),
})
})
.collect::<Result<Box<[SourceLocation]>>>()?;
@@ -200,7 +200,7 @@ pub fn deserialize_code<R: Read, Bag: ConstantBag>(
let len = rdr.read_u32()?;
let source_path = bag.make_name(rdr.read_str(len)?);
let first_line_number = OneIndexed::new(rdr.read_u32()?);
let first_line_number = OneIndexed::new(rdr.read_u32()? as _);
let max_stackdepth = rdr.read_u32()?;
let len = rdr.read_u32()?;
@@ -589,8 +589,8 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
write_len(buf, code.locations.len());
for loc in &*code.locations {
buf.write_u32(loc.row.get());
buf.write_u32(loc.column.to_zero_indexed());
buf.write_u32(loc.row.get() as _);
buf.write_u32(loc.column.to_zero_indexed() as _);
}
buf.write_u16(code.flags.bits());
@@ -601,7 +601,7 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
write_vec(buf, code.source_path.as_ref().as_bytes());
buf.write_u32(code.first_line_number.map_or(0, |x| x.get()));
buf.write_u32(code.first_line_number.map_or(0, |x| x.get() as _));
buf.write_u32(code.max_stackdepth);
write_vec(buf, code.obj_name.as_ref().as_bytes());
compiler/core/src/mode.rs
+5 -3
View File
@@ -1,4 +1,4 @@
pub use rustpython_parser_core::mode::ModeParseError;
pub use ruff_python_parser::ModeParseError;
#[derive(Clone, Copy)]
pub enum Mode {
@@ -22,12 +22,14 @@ impl std::str::FromStr for Mode {
}
}
impl From<Mode> for rustpython_parser_core::Mode {
impl From<Mode> for ruff_python_parser::Mode {
fn from(mode: Mode) -> Self {
match mode {
Mode::Exec => Self::Module,
Mode::Eval => Self::Expression,
Mode::Single | Mode::BlockExpr => Self::Interactive,
// TODO: Improve ruff API
// ruff does not have an interactive mode
Mode::Single | Mode::BlockExpr => Self::Ipython,
}
}
}
compiler/source/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "rustpython-compiler-source"
description = "RustPython Source and Index"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
repository.workspace = true
license.workspace = true
[dependencies]
ruff_source_file = { workspace = true }
ruff_text_size = { workspace = true }
[lints]
workspace = true
compiler/source/src/lib.rs
+42
View File
@@ -0,0 +1,42 @@
pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, SourceLocation};
use ruff_text_size::TextRange;
pub use ruff_text_size::TextSize;
#[derive(Clone)]
pub struct SourceCode<'src> {
pub path: &'src str,
pub text: &'src str,
pub index: LineIndex,
}
impl<'src> SourceCode<'src> {
pub fn new(path: &'src str, text: &'src str) -> Self {
let index = LineIndex::from_source_text(text);
Self { path, text, index }
}
pub fn line_index(&self, offset: TextSize) -> LineNumber {
self.index.line_index(offset)
}
pub fn source_location(&self, offset: TextSize) -> SourceLocation {
self.index.source_location(offset, self.text)
}
pub fn get_range(&'src self, range: TextRange) -> &'src str {
&self.text[range.start().to_usize()..range.end().to_usize()]
}
}
pub struct SourceCodeOwned {
pub path: String,
pub text: String,
pub index: LineIndex,
}
impl SourceCodeOwned {
pub fn new(path: String, text: String) -> Self {
let index = LineIndex::from_source_text(&text);
Self { path, text, index }
}
}
compiler/src/lib.rs
+261 -52
View File
@@ -1,69 +1,128 @@
use ruff_source_file::SourceLocation;
use rustpython_codegen::{compile, symboltable};
use rustpython_parser::ast::{self as ast, ConstantOptimizer, fold::Fold};
pub use rustpython_codegen::compile::CompileOpts;
pub use rustpython_compiler_core::{Mode, bytecode::CodeObject};
pub use rustpython_parser::{Parse, source_code::LinearLocator};
use rustpython_compiler_source::SourceCode;
// these modules are out of repository. re-exporting them here for convenience.
pub use ruff_python_ast as ast;
pub use ruff_python_parser as parser;
pub use rustpython_codegen as codegen;
pub use rustpython_compiler_core as core;
pub use rustpython_parser as parser;
pub use rustpython_compiler_source as source;
use thiserror::Error;
#[derive(Debug)]
#[derive(Error, Debug)]
pub enum CompileErrorType {
Codegen(rustpython_codegen::error::CodegenErrorType),
Parse(parser::ParseErrorType),
#[error(transparent)]
Codegen(#[from] codegen::error::CodegenErrorType),
#[error(transparent)]
Parse(#[from] parser::ParseErrorType),
}
impl std::error::Error for CompileErrorType {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
CompileErrorType::Codegen(e) => e.source(),
CompileErrorType::Parse(e) => e.source(),
}
}
#[derive(Error, Debug)]
pub struct ParseError {
#[source]
pub error: parser::ParseErrorType,
pub location: SourceLocation,
pub source_path: String,
}
impl std::fmt::Display for CompileErrorType {
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CompileErrorType::Codegen(e) => e.fmt(f),
CompileErrorType::Parse(e) => e.fmt(f),
}
}
}
impl From<rustpython_codegen::error::CodegenErrorType> for CompileErrorType {
fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self {
CompileErrorType::Codegen(source)
}
}
impl From<parser::ParseErrorType> for CompileErrorType {
fn from(source: parser::ParseErrorType) -> Self {
CompileErrorType::Parse(source)
self.error.fmt(f)
}
}
pub type CompileError = rustpython_parser::source_code::LocatedError<CompileErrorType>;
#[derive(Error, Debug)]
pub enum CompileError {
#[error(transparent)]
Codegen(#[from] codegen::error::CodegenError),
#[error(transparent)]
Parse(#[from] ParseError),
}
impl CompileError {
pub fn from_ruff_parse_error(error: parser::ParseError, source_code: &SourceCode<'_>) -> Self {
let location = source_code.source_location(error.location.start());
Self::Parse(ParseError {
error: error.error,
location,
source_path: source_code.path.to_owned(),
})
}
pub fn location(&self) -> Option<SourceLocation> {
match self {
CompileError::Codegen(codegen_error) => codegen_error.location.clone(),
CompileError::Parse(parse_error) => Some(parse_error.location.clone()),
}
}
pub fn python_location(&self) -> (usize, usize) {
match self {
CompileError::Codegen(codegen_error) => {
if let Some(location) = &codegen_error.location {
(location.row.get(), location.column.get())
} else {
(0, 0)
}
}
CompileError::Parse(parse_error) => (
parse_error.location.row.get(),
parse_error.location.column.get(),
),
}
}
pub fn source_path(&self) -> &str {
match self {
CompileError::Codegen(codegen_error) => &codegen_error.source_path,
CompileError::Parse(parse_error) => &parse_error.source_path,
}
}
}
/// Compile a given source code into a bytecode object.
pub fn compile(
source: &str,
mode: Mode,
source_path: String,
source_path: &str,
opts: CompileOpts,
) -> Result<CodeObject, CompileError> {
let mut locator = LinearLocator::new(source);
let mut ast = match parser::parse(source, mode.into(), &source_path) {
Ok(x) => x,
Err(e) => return Err(locator.locate_error(e)),
};
if opts.optimize > 0 {
ast = ConstantOptimizer::new()
.fold_mod(ast)
.unwrap_or_else(|e| match e {});
}
let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {});
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into())
// TODO: do this less hackily; ruff's parser should translate a CRLF line
// break in a multiline string into just an LF in the parsed value
#[cfg(windows)]
let source = &source.replace("\r\n", "\n");
let source_code = SourceCode::new(source_path, source);
_compile(source_code, mode, opts)
// let index = LineIndex::from_source_text(source);
// let source_code = SourceCode::new(source, &index);
// let mut locator = LinearLocator::new(source);
// let mut ast = match parser::parse(source, mode.into(), &source_path) {
// Ok(x) => x,
// Err(e) => return Err(locator.locate_error(e)),
// };
// TODO:
// if opts.optimize > 0 {
// ast = ConstantOptimizer::new()
// .fold_mod(ast)
// .unwrap_or_else(|e| match e {});
// }
// let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {});
}
fn _compile(
source_code: SourceCode<'_>,
mode: Mode,
opts: CompileOpts,
) -> Result<CodeObject, CompileError> {
let parsed = parser::parse(source_code.text, mode.into())
.map_err(|err| CompileError::from_ruff_parse_error(err, &source_code))?;
let ast = parsed.into_syntax();
compile::compile_top(ast, source_code, mode, opts).map_err(|e| e.into())
}
pub fn compile_symtable(
@@ -71,20 +130,170 @@ pub fn compile_symtable(
mode: Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let mut locator = LinearLocator::new(source);
let source_code = SourceCode::new(source_path, source);
_compile_symtable(source_code, mode)
}
pub fn _compile_symtable(
source_code: SourceCode<'_>,
mode: Mode,
) -> Result<symboltable::SymbolTable, CompileError> {
let res = match mode {
Mode::Exec | Mode::Single | Mode::BlockExpr => {
let ast =
ast::Suite::parse(source, source_path).map_err(|e| locator.locate_error(e))?;
let ast = locator.fold(ast).unwrap();
symboltable::SymbolTable::scan_program(&ast)
let ast = ruff_python_parser::parse_module(source_code.text)
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_code.clone())
}
Mode::Eval => {
let expr =
ast::Expr::parse(source, source_path).map_err(|e| locator.locate_error(e))?;
let expr = locator.fold(expr).unwrap();
symboltable::SymbolTable::scan_expr(&expr)
let ast =
ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Expression)
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_code))?;
symboltable::SymbolTable::scan_expr(
&ast.into_syntax().expect_expression(),
source_code.clone(),
)
}
};
res.map_err(|e| e.into_codegen_error(source_path.to_owned()).into())
res.map_err(|e| e.into_codegen_error(source_code.path.to_owned()).into())
}
#[test]
fn test_compile() {
let code = "x = 'abc'";
let compiled = compile(&code, Mode::Single, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_phello() {
let code = r#"
initialized = True
def main():
print("Hello world!")
if __name__ == '__main__':
main()
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_if_elif_else() {
let code = r#"
if False:
pass
elif False:
pass
elif False:
pass
else:
pass
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_lambda() {
let code = r#"
lambda: 'a'
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_lambda2() {
let code = r#"
(lambda x: f'hello, {x}')('world}')
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_lambda3() {
let code = r#"
def g():
pass
def f():
if False:
return lambda x: g(x)
elif False:
return g
else:
return g
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_int() {
let code = r#"
a = 0xFF
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_bigint() {
let code = r#"
a = 0xFFFFFFFFFFFFFFFFFFFFFFFF
"#;
let compiled = compile(&code, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_compile_fstring() {
let code1 = r#"
assert f"1" == '1'
"#;
let compiled = compile(&code1, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
let code2 = r#"
assert f"{1}" == '1'
"#;
let compiled = compile(&code2, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
let code3 = r#"
assert f"{1+1}" == '2'
"#;
let compiled = compile(&code3, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
let code4 = r#"
assert f"{{{(lambda: f'{1}')}" == '{1'
"#;
let compiled = compile(&code4, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
let code5 = r#"
assert f"a{1}" == 'a1'
"#;
let compiled = compile(&code5, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
let code6 = r#"
assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}'
"#;
let compiled = compile(&code6, Mode::Exec, "<>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
#[test]
fn test_simple_enum() {
let code = r#"
import enum
@enum._simple_enum(enum.IntFlag, boundary=enum.KEEP)
class RegexFlag:
NOFLAG = 0
DEBUG = 1
print(RegexFlag.NOFLAG & RegexFlag.DEBUG)
"#;
let compiled = compile(&code, Mode::Exec, "<string>", CompileOpts::default());
dbg!(compiled.expect("compile error"));
}
derive-impl/Cargo.toml
+1 -1
View File
@@ -10,7 +10,7 @@ license.workspace = true
[dependencies]
rustpython-compiler-core = { workspace = true }
rustpython-parser-core = { workspace = true }
# rustpython-parser-core = { workspace = true }
rustpython-doc = { workspace = true }
itertools = { workspace = true }
derive/src/lib.rs
+1 -1
View File
@@ -246,7 +246,7 @@ impl derive_impl::Compiler for Compiler {
module_name: String,
) -> Result<rustpython_compiler::CodeObject, Box<dyn std::error::Error>> {
use rustpython_compiler::{CompileOpts, compile};
Ok(compile(source, mode, module_name, CompileOpts::default())?)
Ok(compile(source, mode, &module_name, CompileOpts::default())?)
}
}
examples/dis.rs
+1 -1
View File
@@ -79,7 +79,7 @@ fn display_script(
expand_code_objects: bool,
) -> Result<(), Box<dyn Error>> {
let source = fs::read_to_string(path)?;
let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
let code = compiler::compile(&source, mode, &path.to_string_lossy(), opts)?;
println!("{}:", path.display());
if expand_code_objects {
println!("{}", code.display_expand_code_objects());
examples/parse_folder.rs
+11 -9
View File
@@ -8,7 +8,8 @@ extern crate env_logger;
#[macro_use]
extern crate log;
use rustpython_parser::{Parse, ast};
use ruff_python_parser::parse_module;
use rustpython_compiler::ast;
use std::{
path::{Path, PathBuf},
time::{Duration, Instant},
@@ -68,18 +69,19 @@ fn parse_python_file(filename: &Path) -> ParsedFile {
info!("Parsing file {:?}", filename);
match std::fs::read_to_string(filename) {
Err(e) => ParsedFile {
// filename: Box::new(filename.to_path_buf()),
// code: "".to_owned(),
filename: Box::new(filename.to_path_buf()),
code: "".to_owned(),
num_lines: 0,
result: Err(e.to_string()),
},
Ok(source) => {
let num_lines = source.lines().count();
let result =
ast::Suite::parse(&source, &filename.to_string_lossy()).map_err(|e| e.to_string());
let result = parse_module(&source)
.map(|x| x.into_suite())
.map_err(|e| e.to_string());
ParsedFile {
// filename: Box::new(filename.to_path_buf()),
// code: source.to_string(),
filename: Box::new(filename.to_path_buf()),
code: source.to_string(),
num_lines,
result,
}
@@ -132,8 +134,8 @@ struct ScanResult {
}
struct ParsedFile {
// filename: Box<PathBuf>,
// code: String,
filename: Box<PathBuf>,
code: String,
num_lines: usize,
result: ParseResult,
}
src/shell.rs
+12 -14
View File
@@ -1,10 +1,12 @@
mod helper;
use rustpython_parser::{ParseErrorType, Tok, lexer::LexicalErrorType};
use rustpython_compiler::{
CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType,
};
use rustpython_vm::{
AsObject, PyResult, VirtualMachine,
builtins::PyBaseExceptionRef,
compiler::{self, CompileError, CompileErrorType},
compiler::{self},
readline::{Readline, ReadlineResult},
scope::Scope,
};
@@ -35,29 +37,25 @@ fn shell_exec(
ShellExecResult::Ok
}
}
Err(CompileError {
error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)),
Err(CompileError::Parse(ParseError {
error: ParseErrorType::Lexical(LexicalErrorType::Eof),
..
})
| Err(CompileError {
error: CompileErrorType::Parse(ParseErrorType::Eof),
..
}) => ShellExecResult::Continue,
})) => ShellExecResult::Continue,
Err(err) => {
// bad_error == true if we are handling an error that should be thrown even if we are continuing
// if its an indentation error, set to true if we are continuing and the error is on column 0,
// since indentations errors on columns other than 0 should be ignored.
// if its an unrecognized token for dedent, set to false
let bad_error = match err.error {
CompileErrorType::Parse(ref p) => {
let bad_error = match err {
CompileError::Parse(ref p) => {
if matches!(
p,
p.error,
ParseErrorType::Lexical(LexicalErrorType::IndentationError)
) {
continuing && err.location.is_some()
continuing // && p.location.is_some()
} else {
!matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
true // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
}
}
_ => true, // It is a bad error for everything else
stdlib/src/faulthandler.rs
+1 -1
View File
@@ -10,7 +10,7 @@ mod decl {
stderr,
" File \"{}\", line {} in {}",
frame.code.source_path,
frame.current_location().row.to_usize(),
frame.current_location().row,
frame.code.obj_name
)
}
vm/Cargo.toml
+13 -8
View File
@@ -19,9 +19,9 @@ freeze-stdlib = ["encodings"]
jit = ["rustpython-jit"]
threading = ["rustpython-common/threading"]
compiler = ["parser", "codegen", "rustpython-compiler"]
ast = ["rustpython-ast"]
ast = ["ruff_python_ast", "ruff_text_size", "ruff_source_file"]
codegen = ["rustpython-codegen", "ast"]
parser = ["rustpython-parser", "ast"]
parser = ["ast"]
serde = ["dep:serde"]
wasmbind = ["chrono/wasmbind", "getrandom/wasm_js", "wasm-bindgen"]
@@ -32,10 +32,15 @@ rustpython-common = { workspace = true }
rustpython-derive = { workspace = true }
rustpython-jit = { workspace = true, optional = true }
rustpython-ast = { workspace = true, optional = true }
rustpython-parser = { workspace = true, optional = true }
ruff_python_ast = { workspace = true, optional = true }
ruff_python_parser = { workspace = true }
ruff_text_size = { workspace = true, optional = true }
ruff_source_file = { workspace = true, optional = true }
# rustpython-ast = { workspace = true, optional = true }
# rustpython-parser = { workspace = true, optional = true }
rustpython-compiler-core = { workspace = true }
rustpython-parser-core = { workspace = true }
rustpython-compiler-source = { workspace = true }
# rustpython-parser-core = { workspace = true }
rustpython-literal = { workspace = true }
rustpython-format = { workspace = true }
rustpython-sre_engine = { workspace = true }
@@ -87,9 +92,9 @@ unicode_names2 = { workspace = true }
# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
unicode-casing = "0.1.0"
# update version all at the same time
unic-ucd-bidi = "0.9.0"
unic-ucd-category = "0.9.0"
unic-ucd-ident = "0.9.0"
unic-ucd-bidi = "0.9.0"
unic-ucd-category = "0.9.0"
unic-ucd-ident = "0.9.0"
[target.'cfg(unix)'.dependencies]
rustix = { workspace = true }
vm/src/builtins/code.rs
+3 -3
View File
@@ -11,7 +11,7 @@ use crate::{
convert::ToPyObject,
frozen,
function::{FuncArgs, OptionalArg},
source_code::OneIndexed,
source::LineNumber,
types::Representable,
};
use malachite_bigint::BigInt;
@@ -279,7 +279,7 @@ impl PyCode {
#[pygetset]
fn co_firstlineno(&self) -> u32 {
self.code.first_line_number.map_or(0, |n| n.get())
self.code.first_line_number.map_or(0, |n| n.get() as _)
}
#[pygetset]
@@ -351,7 +351,7 @@ impl PyCode {
};
let first_line_number = match args.co_firstlineno {
OptionalArg::Present(first_line_number) => OneIndexed::new(first_line_number),
OptionalArg::Present(first_line_number) => LineNumber::new(first_line_number as _),
OptionalArg::Missing => self.code.first_line_number,
};
vm/src/builtins/frame.rs
+1 -1
View File
@@ -60,7 +60,7 @@ impl Frame {
#[pygetset]
pub fn f_lineno(&self) -> usize {
self.current_location().row.to_usize()
self.current_location().row.get()
}
#[pygetset]
vm/src/builtins/traceback.rs
+2 -2
View File
@@ -2,7 +2,7 @@ use rustpython_common::lock::PyMutex;
use super::PyType;
use crate::{
Context, Py, PyPayload, PyRef, class::PyClassImpl, frame::FrameRef, source_code::LineNumber,
Context, Py, PyPayload, PyRef, class::PyClassImpl, frame::FrameRef, source::LineNumber,
};
#[pyclass(module = false, name = "traceback", traverse)]
@@ -47,7 +47,7 @@ impl PyTraceback {
#[pygetset]
fn tb_lineno(&self) -> usize {
self.lineno.to_usize()
self.lineno.get()
}
#[pygetset]
vm/src/compiler.rs
+27 -15
View File
@@ -1,37 +1,49 @@
#[cfg(feature = "rustpython-codegen")]
#[cfg(feature = "codegen")]
pub use rustpython_codegen::CompileOpts;
#[cfg(feature = "rustpython-compiler")]
#[cfg(feature = "compiler")]
pub use rustpython_compiler::*;
#[cfg(not(feature = "rustpython-compiler"))]
#[cfg(not(feature = "compiler"))]
pub use rustpython_compiler_source as source;
#[cfg(not(feature = "compiler"))]
pub use rustpython_compiler_core::Mode;
#[cfg(not(feature = "rustpython-compiler"))]
#[cfg(not(feature = "compiler"))]
pub use rustpython_compiler_core as core;
#[cfg(not(feature = "rustpython-compiler"))]
pub use rustpython_parser_core as parser;
#[cfg(not(feature = "compiler"))]
pub use ruff_python_parser as parser;
#[cfg(not(feature = "rustpython-compiler"))]
#[cfg(not(feature = "compiler"))]
mod error {
#[cfg(all(feature = "rustpython-parser", feature = "rustpython-codegen"))]
#[cfg(all(feature = "parser", feature = "codegen"))]
panic!("Use --features=compiler to enable both parser and codegen");
#[derive(Debug, thiserror::Error)]
pub enum CompileErrorType {
#[cfg(feature = "rustpython-codegen")]
#[cfg(feature = "codegen")]
#[error(transparent)]
Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
#[cfg(feature = "rustpython-parser")]
Codegen(#[from] super::codegen::error::CodegenErrorType),
#[cfg(feature = "parser")]
#[error(transparent)]
Parse(#[from] rustpython_parser::error::ParseErrorType),
Parse(#[from] super::parser::ParseErrorType),
}
pub type CompileError = rustpython_parser_core::source_code::LocatedError<CompileErrorType>;
#[derive(Debug, thiserror::Error)]
pub enum CompileError {
#[cfg(feature = "codegen")]
#[error(transparent)]
Codegen(#[from] super::codegen::error::CodegenError),
#[cfg(feature = "parser")]
#[error(transparent)]
Parse(#[from] super::parser::ParseError),
}
}
#[cfg(not(feature = "rustpython-compiler"))]
#[cfg(not(feature = "compiler"))]
pub use error::{CompileError, CompileErrorType};
#[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
#[cfg(any(feature = "parser", feature = "codegen"))]
impl crate::convert::ToPyException for (CompileError, Option<&str>) {
fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef {
vm.new_syntax_error(&self.0, self.1)
vm/src/exceptions.rs
+1 -1
View File
@@ -378,7 +378,7 @@ fn write_traceback_entry<W: Write>(
tb_entry.lineno,
tb_entry.frame.code.obj_name
)?;
print_source_line(output, filename, tb_entry.lineno.to_usize())?;
print_source_line(output, filename, tb_entry.lineno.get())?;
Ok(())
}
vm/src/frame.rs
+3 -3
View File
@@ -15,7 +15,7 @@ use crate::{
function::{ArgMapping, Either, FuncArgs},
protocol::{PyIter, PyIterReturn},
scope::Scope,
source_code::SourceLocation,
source::SourceLocation,
stdlib::{builtins, typing::_typing},
vm::{Context, PyMethod},
};
@@ -166,7 +166,7 @@ impl Frame {
}
pub fn current_location(&self) -> SourceLocation {
self.code.locations[self.lasti() as usize - 1]
self.code.locations[self.lasti() as usize - 1].clone()
}
pub fn lasti(&self) -> u32 {
@@ -380,7 +380,7 @@ impl ExecutingFrame<'_> {
// 2. Add new entry with current execution position (filename, lineno, code_object) to traceback.
// 3. Unwind block stack till appropriate handler is found.
let loc = frame.code.locations[idx];
let loc = frame.code.locations[idx].clone();
let next = exception.traceback();
let new_traceback =
PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row);
vm/src/lib.rs
+1 -1
View File
@@ -84,7 +84,7 @@ pub mod warn;
#[cfg(windows)]
pub mod windows;
pub use self::compiler::parser::source_code;
pub use self::compiler::source;
pub use self::convert::{TryFromBorrowedObject, TryFromObject};
pub use self::object::{
AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact,
vm/src/stdlib/ast.rs
+211 -278
View File
@@ -3,81 +3,55 @@
//! This module makes use of the parser logic, and translates all ast nodes
//! into python ast.AST objects.
mod r#gen;
mod pyast;
use crate::builtins::{PyInt, PyStr};
use crate::stdlib::ast::module::{Mod, ModInteractive};
use crate::stdlib::ast::node::BoxedSlice;
use crate::stdlib::ast::python::_ast;
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
VirtualMachine,
builtins::{self, PyDict, PyModule, PyStrRef, PyType},
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult,
TryFromObject, VirtualMachine,
builtins::PyIntRef,
builtins::{PyDict, PyModule, PyStrRef, PyType},
class::{PyClassImpl, StaticType},
compiler::CompileError,
compiler::core::bytecode::OpArgType,
convert::ToPyException,
source_code::{LinearLocator, OneIndexed, SourceLocation, SourceRange},
compiler::{CompileError, ParseError},
convert::ToPyObject,
source::SourceCode,
source::SourceLocation,
};
use num_complex::Complex64;
use num_traits::{ToPrimitive, Zero};
use rustpython_ast::{self as ast, fold::Fold};
#[cfg(feature = "rustpython-codegen")]
use node::Node;
use ruff_python_ast as ruff;
use ruff_source_file::OneIndexed;
use ruff_text_size::{Ranged, TextRange, TextSize};
use rustpython_compiler_source::SourceCodeOwned;
#[cfg(feature = "parser")]
use ruff_python_parser as parser;
#[cfg(feature = "codegen")]
use rustpython_codegen as codegen;
#[cfg(feature = "rustpython-parser")]
use rustpython_parser as parser;
#[pymodule]
mod _ast {
use crate::{
AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyStrRef, PyTupleRef},
function::FuncArgs,
};
#[pyattr]
#[pyclass(module = "_ast", name = "AST")]
#[derive(Debug, PyPayload)]
pub(crate) struct NodeAst;
pub(crate) use python::_ast::NodeAst;
#[pyclass(flags(BASETYPE, HAS_DICT))]
impl NodeAst {
#[pyslot]
#[pymethod(magic)]
fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> {
let fields = zelf.get_attr("_fields", vm)?;
let fields: Vec<PyStrRef> = fields.try_to_value(vm)?;
let numargs = args.args.len();
if numargs > fields.len() {
return Err(vm.new_type_error(format!(
"{} constructor takes at most {} positional argument{}",
zelf.class().name(),
fields.len(),
if fields.len() == 1 { "" } else { "s" },
)));
}
for (name, arg) in fields.iter().zip(args.args) {
zelf.set_attr(name, arg, vm)?;
}
for (key, value) in args.kwargs {
if let Some(pos) = fields.iter().position(|f| f.as_str() == key) {
if pos < numargs {
return Err(vm.new_type_error(format!(
"{} got multiple values for argument '{}'",
zelf.class().name(),
key
)));
}
}
zelf.set_attr(vm.ctx.intern_str(key), value, vm)?;
}
Ok(())
}
mod python;
#[pyattr(name = "_fields")]
fn fields(ctx: &Context) -> PyTupleRef {
ctx.empty_tuple.clone()
}
}
#[pyattr(name = "PyCF_ONLY_AST")]
use super::PY_COMPILE_FLAG_AST_ONLY;
}
mod argument;
mod basic;
mod constant;
mod elif_else_clause;
mod exception;
mod expression;
mod module;
mod node;
mod operator;
mod other;
mod parameter;
mod pattern;
mod statement;
mod string;
mod type_ignore;
mod type_parameters;
fn get_node_field(vm: &VirtualMachine, obj: &PyObject, field: &'static str, typ: &str) -> PyResult {
vm.get_attribute_opt(obj.to_owned(), field)?
@@ -94,248 +68,197 @@ fn get_node_field_opt(
.filter(|obj| !vm.is_none(obj)))
}
trait Node: Sized {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef;
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self>;
fn get_int_field(
vm: &VirtualMachine,
obj: &PyObject,
field: &'static str,
typ: &str,
) -> PyResult<PyRefExact<PyInt>> {
get_node_field(vm, obj, field, typ)?
.downcast_exact(vm)
.map_err(|_| vm.new_type_error(format!("field \"{field}\" must have integer type")))
}
impl<T: Node> Node for Vec<T> {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx
.new_list(
self.into_iter()
.map(|node| node.ast_to_object(vm))
.collect(),
)
.into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, obj))
}
struct PySourceRange {
start: PySourceLocation,
end: PySourceLocation,
}
impl<T: Node> Node for Box<T> {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
(*self).ast_to_object(vm)
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
T::ast_from_object(vm, object).map(Box::new)
}
pub struct PySourceLocation {
row: Row,
column: Column,
}
impl<T: Node> Node for Option<T> {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
match self {
Some(node) => node.ast_to_object(vm),
None => vm.ctx.none(),
impl PySourceLocation {
fn to_source_location(&self) -> SourceLocation {
SourceLocation {
row: self.row.get_one_indexed(),
column: self.column.get_one_indexed(),
}
}
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
if vm.is_none(&object) {
Ok(None)
} else {
Ok(Some(T::ast_from_object(vm, object)?))
}
/// A one-based index into the lines.
#[derive(Clone, Copy)]
struct Row(OneIndexed);
impl Row {
fn get(self) -> usize {
self.0.get()
}
fn get_one_indexed(self) -> OneIndexed {
self.0
}
}
/// An UTF-8 index into the line.
#[derive(Clone, Copy)]
struct Column(TextSize);
impl Column {
fn get(self) -> usize {
self.0.to_usize()
}
fn get_one_indexed(self) -> OneIndexed {
OneIndexed::from_zero_indexed(self.get())
}
}
fn text_range_to_source_range(
source_code: &SourceCodeOwned,
text_range: TextRange,
) -> PySourceRange {
let index = &source_code.index;
let source = &source_code.text;
if source.is_empty() {
return PySourceRange {
start: PySourceLocation {
row: Row(OneIndexed::from_zero_indexed(0)),
column: Column(TextSize::new(0)),
},
end: PySourceLocation {
row: Row(OneIndexed::from_zero_indexed(0)),
column: Column(TextSize::new(0)),
},
};
}
let start_row = index.line_index(text_range.start());
let end_row = index.line_index(text_range.end());
let start_col = text_range.start() - index.line_start(start_row, source);
let end_col = text_range.end() - index.line_start(end_row, source);
PySourceRange {
start: PySourceLocation {
row: Row(start_row),
column: Column(start_col),
},
end: PySourceLocation {
row: Row(end_row),
column: Column(end_col),
},
}
}
fn range_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
name: &str,
) -> PyResult<SourceRange> {
fn make_location(row: u32, column: u32) -> Option<SourceLocation> {
Some(SourceLocation {
row: OneIndexed::new(row)?,
column: OneIndexed::from_zero_indexed(column),
})
}
let row = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "lineno", name)?)?;
let column = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "col_offset", name)?)?;
let location = make_location(row.to_u32(), column.to_u32());
let end_row = get_node_field_opt(vm, &object, "end_lineno")?
.map(|obj| ast::Int::ast_from_object(vm, obj))
.transpose()?;
let end_column = get_node_field_opt(vm, &object, "end_col_offset")?
.map(|obj| ast::Int::ast_from_object(vm, obj))
.transpose()?;
let end_location = if let (Some(row), Some(column)) = (end_row, end_column) {
make_location(row.to_u32(), column.to_u32())
} else {
None
) -> PyResult<TextRange> {
let start_row = get_int_field(vm, &object, "lineno", name)?;
let start_column = get_int_field(vm, &object, "col_offset", name)?;
let end_row = get_int_field(vm, &object, "end_lineno", name)?;
let end_column = get_int_field(vm, &object, "end_col_offset", name)?;
let location = PySourceRange {
start: PySourceLocation {
row: Row(OneIndexed::new(start_row.try_to_primitive(vm)?).unwrap()),
column: Column(TextSize::new(start_column.try_to_primitive(vm)?)),
},
end: PySourceLocation {
row: Row(OneIndexed::new(end_row.try_to_primitive(vm)?).unwrap()),
column: Column(TextSize::new(end_column.try_to_primitive(vm)?)),
},
};
let range = SourceRange {
start: location.unwrap_or_default(),
end: end_location,
};
Ok(range)
Ok(source_range_to_text_range(source_code, location))
}
fn node_add_location(dict: &Py<PyDict>, range: SourceRange, vm: &VirtualMachine) {
fn source_range_to_text_range(source_code: &SourceCodeOwned, location: PySourceRange) -> TextRange {
let source = &source_code.text;
let index = &source_code.index;
if source.is_empty() {
return TextRange::new(TextSize::new(0), TextSize::new(0));
}
let start = index.offset(
location.start.row.get_one_indexed(),
location.start.column.get_one_indexed(),
source,
);
let end = index.offset(
location.end.row.get_one_indexed(),
location.end.column.get_one_indexed(),
source,
);
TextRange::new(start, end)
}
fn node_add_location(
dict: &Py<PyDict>,
range: TextRange,
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
) {
let range = text_range_to_source_range(source_code, range);
dict.set_item("lineno", vm.ctx.new_int(range.start.row.get()).into(), vm)
.unwrap();
dict.set_item(
"col_offset",
vm.ctx.new_int(range.start.column.to_zero_indexed()).into(),
vm.ctx.new_int(range.start.column.get()).into(),
vm,
)
.unwrap();
if let Some(end_location) = range.end {
dict.set_item(
"end_lineno",
vm.ctx.new_int(end_location.row.get()).into(),
vm,
)
dict.set_item("end_lineno", vm.ctx.new_int(range.end.row.get()).into(), vm)
.unwrap();
dict.set_item(
"end_col_offset",
vm.ctx.new_int(end_location.column.to_zero_indexed()).into(),
vm,
)
.unwrap();
};
dict.set_item(
"end_col_offset",
vm.ctx.new_int(range.end.column.get()).into(),
vm,
)
.unwrap();
}
impl Node for ast::String {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_str(self).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let py_str = PyStrRef::try_from_object(vm, object)?;
Ok(py_str.as_str().to_owned())
}
}
impl Node for ast::Identifier {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
let id: String = self.into();
vm.ctx.new_str(id).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let py_str = PyStrRef::try_from_object(vm, object)?;
Ok(ast::Identifier::new(py_str.as_str()))
}
}
impl Node for ast::Int {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_int(self.to_u32()).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let value = object.try_into_value(vm)?;
Ok(ast::Int::new(value))
}
}
impl Node for bool {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_int(self as u8).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
i32::try_from_object(vm, object).map(|i| i != 0)
}
}
impl Node for ast::Constant {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
match self {
ast::Constant::None => vm.ctx.none(),
ast::Constant::Bool(b) => vm.ctx.new_bool(b).into(),
ast::Constant::Str(s) => vm.ctx.new_str(s).into(),
ast::Constant::Bytes(b) => vm.ctx.new_bytes(b).into(),
ast::Constant::Int(i) => vm.ctx.new_int(i).into(),
ast::Constant::Tuple(t) => vm
.ctx
.new_tuple(t.into_iter().map(|c| c.ast_to_object(vm)).collect())
.into(),
ast::Constant::Float(f) => vm.ctx.new_float(f).into(),
ast::Constant::Complex { real, imag } => vm.new_pyobj(Complex64::new(real, imag)),
ast::Constant::Ellipsis => vm.ctx.ellipsis(),
}
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let constant = match_class!(match object {
ref i @ builtins::int::PyInt => {
let value = i.as_bigint();
if object.class().is(vm.ctx.types.bool_type) {
ast::Constant::Bool(!value.is_zero())
} else {
ast::Constant::Int(value.clone())
}
}
ref f @ builtins::float::PyFloat => ast::Constant::Float(f.to_f64()),
ref c @ builtins::complex::PyComplex => {
let c = c.to_complex();
ast::Constant::Complex {
real: c.re,
imag: c.im,
}
}
ref s @ builtins::pystr::PyStr => ast::Constant::Str(s.as_str().to_owned()),
ref b @ builtins::bytes::PyBytes => ast::Constant::Bytes(b.as_bytes().to_owned()),
ref t @ builtins::tuple::PyTuple => {
ast::Constant::Tuple(
t.iter()
.map(|elt| Self::ast_from_object(vm, elt.clone()))
.collect::<Result<_, _>>()?,
)
}
builtins::singletons::PyNone => ast::Constant::None,
builtins::slice::PyEllipsis => ast::Constant::Ellipsis,
obj =>
return Err(vm.new_type_error(format!(
"invalid type in Constant: type '{}'",
obj.class().name()
))),
});
Ok(constant)
}
}
impl Node for ast::ConversionFlag {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_int(self as u8).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
i32::try_from_object(vm, object)?
.to_u32()
.and_then(ast::ConversionFlag::from_op_arg)
.ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned()))
}
}
impl Node for ast::located::Arguments {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
self.into_python_arguments().ast_to_object(vm)
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
ast::located::PythonArguments::ast_from_object(vm, object)
.map(ast::located::PythonArguments::into_arguments)
}
}
#[cfg(feature = "rustpython-parser")]
#[cfg(feature = "parser")]
pub(crate) fn parse(
vm: &VirtualMachine,
source: &str,
mode: parser::Mode,
) -> Result<PyObjectRef, CompileError> {
let mut locator = LinearLocator::new(source);
let top = parser::parse(source, mode, "<unknown>").map_err(|e| locator.locate_error(e))?;
let top = locator.fold_mod(top).unwrap();
Ok(top.ast_to_object(vm))
let source_code = SourceCodeOwned::new("".to_owned(), source.to_owned());
let top = parser::parse(source, mode)
.map_err(|parse_error| ParseError {
error: parse_error.error,
location: text_range_to_source_range(&source_code, parse_error.location)
.start
.to_source_location(),
source_path: "<unknown>".to_string(),
})?
.into_syntax();
let top = match top {
ruff::Mod::Module(m) => Mod::Module(m),
ruff::Mod::Expression(e) => Mod::Expression(e),
};
Ok(top.ast_to_object(vm, &source_code))
}
#[cfg(feature = "rustpython-codegen")]
#[cfg(feature = "codegen")]
pub(crate) fn compile(
vm: &VirtualMachine,
object: PyObjectRef,
@@ -348,14 +271,24 @@ pub(crate) fn compile(
opts.optimize = optimize;
}
let ast = Node::ast_from_object(vm, object)?;
let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts)
.map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source
let source_code = SourceCodeOwned::new(filename.to_owned(), "".to_owned());
let ast: Mod = Node::ast_from_object(vm, &source_code, object)?;
let ast = match ast {
Mod::Module(m) => ruff::Mod::Module(m),
Mod::Interactive(ModInteractive { range, body }) => {
ruff::Mod::Module(ruff::ModModule { range, body })
}
Mod::Expression(e) => ruff::Mod::Expression(e),
Mod::FunctionType(_) => todo!(),
};
// TODO: create a textual representation of the ast
let text = "";
let source_code = SourceCode::new(filename, text);
let code = codegen::compile::compile_top(ast, source_code, mode, opts)
.map_err(|err| vm.new_syntax_error(&err.into(), None))?; // FIXME source
Ok(vm.ctx.new_code(code).into())
}
// Required crate visibility for inclusion by gen.rs
pub(crate) use _ast::NodeAst;
// Used by builtins::compile()
pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400;
@@ -398,6 +331,6 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _ast::make_module(vm);
r#gen::extend_module_nodes(vm, &module);
pyast::extend_module_nodes(vm, &module);
module
}
vm/src/stdlib/ast/argument.rs
+161
View File
@@ -0,0 +1,161 @@
use super::*;
pub(super) struct PositionalArguments {
pub range: TextRange,
pub args: Box<[ruff::Expr]>,
}
impl Node for PositionalArguments {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { args, range: _ } = self;
BoxedSlice(args).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let args: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?;
Ok(Self {
args: args.0,
range: TextRange::default(), // TODO
})
}
}
pub(super) struct KeywordArguments {
pub range: TextRange,
pub keywords: Box<[ruff::Keyword]>,
}
impl Node for KeywordArguments {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { keywords, range: _ } = self;
// TODO: use range
BoxedSlice(keywords).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?;
Ok(Self {
keywords: keywords.0,
range: TextRange::default(), // TODO
})
}
}
pub(super) fn merge_function_call_arguments(
pos_args: PositionalArguments,
key_args: KeywordArguments,
) -> ruff::Arguments {
let range = pos_args.range.cover(key_args.range);
ruff::Arguments {
range,
args: pos_args.args,
keywords: key_args.keywords,
}
}
pub(super) fn split_function_call_arguments(
args: ruff::Arguments,
) -> (PositionalArguments, KeywordArguments) {
let ruff::Arguments {
range: _,
args,
keywords,
} = args;
let positional_arguments_range = args
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default();
// debug_assert!(range.contains_range(positional_arguments_range));
let positional_arguments = PositionalArguments {
range: positional_arguments_range,
args,
};
let keyword_arguments_range = keywords
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default();
// debug_assert!(range.contains_range(keyword_arguments_range));
let keyword_arguments = KeywordArguments {
range: keyword_arguments_range,
keywords,
};
(positional_arguments, keyword_arguments)
}
pub(super) fn split_class_def_args(
args: Option<Box<ruff::Arguments>>,
) -> (Option<PositionalArguments>, Option<KeywordArguments>) {
let args = match args {
None => return (None, None),
Some(args) => *args,
};
let ruff::Arguments {
range: _,
args,
keywords,
} = args;
let positional_arguments_range = args
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default();
// debug_assert!(range.contains_range(positional_arguments_range));
let positional_arguments = PositionalArguments {
range: positional_arguments_range,
args,
};
let keyword_arguments_range = keywords
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default();
// debug_assert!(range.contains_range(keyword_arguments_range));
let keyword_arguments = KeywordArguments {
range: keyword_arguments_range,
keywords,
};
(Some(positional_arguments), Some(keyword_arguments))
}
pub(super) fn merge_class_def_args(
positional_arguments: Option<PositionalArguments>,
keyword_arguments: Option<KeywordArguments>,
) -> Option<Box<ruff::Arguments>> {
if positional_arguments.is_none() && keyword_arguments.is_none() {
return None;
}
let args = if let Some(positional_arguments) = positional_arguments {
positional_arguments.args
} else {
vec![].into_boxed_slice()
};
let keywords = if let Some(keyword_arguments) = keyword_arguments {
keyword_arguments.keywords
} else {
vec![].into_boxed_slice()
};
Some(Box::new(ruff::Arguments {
range: Default::default(), // TODO
args,
keywords,
}))
}
vm/src/stdlib/ast/basic.rs
+50
View File
@@ -0,0 +1,50 @@
use rustpython_codegen::compile::ruff_int_to_bigint;
use super::*;
impl Node for ruff::Identifier {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
let id = self.as_str();
vm.ctx.new_str(id).into()
}
fn ast_from_object(
vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let py_str = PyStrRef::try_from_object(vm, object)?;
Ok(ruff::Identifier::new(py_str.as_str(), TextRange::default()))
}
}
impl Node for ruff::Int {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
vm.ctx.new_int(ruff_int_to_bigint(&self).unwrap()).into()
}
fn ast_from_object(
vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
// FIXME: performance
let value: PyIntRef = object.try_into_value(vm)?;
let value = value.as_bigint().to_string();
Ok(value.parse().unwrap())
}
}
impl Node for bool {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
vm.ctx.new_int(self as u8).into()
}
fn ast_from_object(
vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
i32::try_from_object(vm, object).map(|i| i != 0)
}
}
vm/src/stdlib/ast/constant.rs
+396
View File
@@ -0,0 +1,396 @@
use super::*;
use crate::builtins::{PyComplex, PyFrozenSet, PyTuple};
use ruff::str_prefix::StringLiteralPrefix;
#[derive(Debug)]
pub(super) struct Constant {
pub(super) range: TextRange,
pub(super) value: ConstantLiteral,
}
impl Constant {
pub(super) fn new_str(
value: impl Into<Box<str>>,
prefix: StringLiteralPrefix,
range: TextRange,
) -> Self {
let value = value.into();
Self {
range,
value: ConstantLiteral::Str { value, prefix },
}
}
pub(super) fn new_int(value: ruff::Int, range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Int(value),
}
}
pub(super) fn new_float(value: f64, range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Float(value),
}
}
pub(super) fn new_complex(real: f64, imag: f64, range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Complex { real, imag },
}
}
pub(super) fn new_bytes(value: Box<[u8]>, range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Bytes(value),
}
}
pub(super) fn new_bool(value: bool, range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Bool(value),
}
}
pub(super) fn new_none(range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::None,
}
}
pub(super) fn new_ellipsis(range: TextRange) -> Self {
Self {
range,
value: ConstantLiteral::Ellipsis,
}
}
pub(crate) fn into_expr(self) -> ruff::Expr {
constant_to_ruff_expr(self)
}
}
#[derive(Debug)]
pub(crate) enum ConstantLiteral {
None,
Bool(bool),
Str {
value: Box<str>,
prefix: StringLiteralPrefix,
},
Bytes(Box<[u8]>),
Int(ruff::Int),
Tuple(Vec<ConstantLiteral>),
FrozenSet(Vec<ConstantLiteral>),
Float(f64),
Complex {
real: f64,
imag: f64,
},
Ellipsis,
}
// constructor
impl Node for Constant {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { range, value } = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeExprConstant::static_type().to_owned())
.unwrap();
let kind = match &value {
ConstantLiteral::Str {
prefix: StringLiteralPrefix::Unicode,
..
} => vm.ctx.new_str("u").into(),
_ => vm.ctx.none(),
};
let value = value.ast_to_object(vm, source_code);
let dict = node.as_object().dict().unwrap();
dict.set_item("value", value, vm).unwrap();
dict.set_item("kind", kind, vm).unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let value_object = get_node_field(vm, &object, "value", "Constant")?;
let value = Node::ast_from_object(vm, source_code, value_object)?;
Ok(Self {
value,
// kind: get_node_field_opt(_vm, &_object, "kind")?
// .map(|obj| Node::ast_from_object(_vm, obj))
// .transpose()?,
range: range_from_object(vm, source_code, object, "Constant")?,
})
}
}
impl Node for ConstantLiteral {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
ConstantLiteral::None => vm.ctx.none(),
ConstantLiteral::Bool(value) => vm.ctx.new_bool(value).to_pyobject(vm),
ConstantLiteral::Str { value, .. } => vm.ctx.new_str(value).to_pyobject(vm),
ConstantLiteral::Bytes(value) => vm.ctx.new_bytes(value.into()).to_pyobject(vm),
ConstantLiteral::Int(value) => value.ast_to_object(vm, source_code),
ConstantLiteral::Tuple(value) => {
let value = value
.into_iter()
.map(|c| c.ast_to_object(vm, source_code))
.collect();
vm.ctx.new_tuple(value).to_pyobject(vm)
}
ConstantLiteral::FrozenSet(value) => PyFrozenSet::from_iter(
vm,
value.into_iter().map(|c| c.ast_to_object(vm, source_code)),
)
.unwrap()
.into_pyobject(vm),
ConstantLiteral::Float(value) => vm.ctx.new_float(value).into_pyobject(vm),
ConstantLiteral::Complex { real, imag } => vm
.ctx
.new_complex(num_complex::Complex::new(real, imag))
.into_pyobject(vm),
ConstantLiteral::Ellipsis => vm.ctx.ellipsis(),
}
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
value_object: PyObjectRef,
) -> PyResult<Self> {
let cls = value_object.class();
let value = if cls.is(vm.ctx.types.none_type) {
ConstantLiteral::None
} else if cls.is(vm.ctx.types.bool_type) {
ConstantLiteral::Bool(if value_object.is(&vm.ctx.true_value) {
true
} else if value_object.is(&vm.ctx.false_value) {
false
} else {
value_object.try_to_value(vm)?
})
} else if cls.is(vm.ctx.types.str_type) {
ConstantLiteral::Str {
value: value_object.try_to_value::<String>(vm)?.into(),
prefix: StringLiteralPrefix::Empty,
}
} else if cls.is(vm.ctx.types.bytes_type) {
ConstantLiteral::Bytes(value_object.try_to_value::<Vec<u8>>(vm)?.into())
} else if cls.is(vm.ctx.types.int_type) {
ConstantLiteral::Int(Node::ast_from_object(vm, source_code, value_object)?)
} else if cls.is(vm.ctx.types.tuple_type) {
let tuple = value_object.downcast::<PyTuple>().map_err(|obj| {
vm.new_type_error(format!(
"Expected type {}, not {}",
PyTuple::static_type().name(),
obj.class().name()
))
})?;
let tuple = tuple
.into_iter()
.cloned()
.map(|object| Node::ast_from_object(vm, source_code, object))
.collect::<PyResult<_>>()?;
ConstantLiteral::Tuple(tuple)
} else if cls.is(vm.ctx.types.frozenset_type) {
let set = value_object.downcast::<PyFrozenSet>().unwrap();
let elements = set
.elements()
.into_iter()
.map(|object| Node::ast_from_object(vm, source_code, object))
.collect::<PyResult<_>>()?;
ConstantLiteral::FrozenSet(elements)
} else if cls.is(vm.ctx.types.float_type) {
let float = value_object.try_into_value(vm)?;
ConstantLiteral::Float(float)
} else if cls.is(vm.ctx.types.complex_type) {
let complex = value_object.try_complex(vm)?;
let complex = match complex {
None => {
return Err(vm.new_type_error(format!(
"Expected type {}, not {}",
PyComplex::static_type().name(),
value_object.class().name()
)));
}
Some((value, _was_coerced)) => value,
};
ConstantLiteral::Complex {
real: complex.re,
imag: complex.im,
}
} else if cls.is(vm.ctx.types.ellipsis_type) {
ConstantLiteral::Ellipsis
} else {
return Err(vm.new_type_error(format!(
"invalid type in Constant: {}",
value_object.class().name()
)));
};
Ok(value)
}
}
fn constant_to_ruff_expr(value: Constant) -> ruff::Expr {
let Constant { value, range } = value;
match value {
ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { range }),
ConstantLiteral::Bool(value) => {
ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { range, value })
}
ConstantLiteral::Str { value, prefix } => {
ruff::Expr::StringLiteral(ruff::ExprStringLiteral {
range,
value: ruff::StringLiteralValue::single(ruff::StringLiteral {
range,
value,
flags: ruff::StringLiteralFlags::default().with_prefix(prefix),
}),
})
}
ConstantLiteral::Bytes(value) => {
ruff::Expr::BytesLiteral(ruff::ExprBytesLiteral {
range,
value: ruff::BytesLiteralValue::single(ruff::BytesLiteral {
range,
value,
flags: Default::default(), // TODO
}),
})
}
ConstantLiteral::Int(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral {
range,
value: ruff::Number::Int(value),
}),
ConstantLiteral::Tuple(value) => ruff::Expr::Tuple(ruff::ExprTuple {
range,
elts: value
.into_iter()
.map(|value| {
constant_to_ruff_expr(Constant {
range: TextRange::default(),
value,
})
})
.collect(),
ctx: ruff::ExprContext::Load,
// TODO: Does this matter?
parenthesized: true,
}),
ConstantLiteral::FrozenSet(value) => ruff::Expr::Call(ruff::ExprCall {
range,
// idk lol
func: Box::new(ruff::Expr::Name(ruff::ExprName {
range: TextRange::default(),
id: "frozenset".to_owned(),
ctx: ruff::ExprContext::Load,
})),
arguments: ruff::Arguments {
range,
args: value
.into_iter()
.map(|value| {
constant_to_ruff_expr(Constant {
range: TextRange::default(),
value,
})
})
.collect(),
keywords: Box::default(),
},
}),
ConstantLiteral::Float(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral {
range,
value: ruff::Number::Float(value),
}),
ConstantLiteral::Complex { real, imag } => {
ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral {
range,
value: ruff::Number::Complex { real, imag },
})
}
ConstantLiteral::Ellipsis => {
ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { range })
}
}
}
pub(super) fn number_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprNumberLiteral,
) -> PyObjectRef {
let ruff::ExprNumberLiteral { range, value } = constant;
let c = match value {
ruff::Number::Int(n) => Constant::new_int(n, range),
ruff::Number::Float(n) => Constant::new_float(n, range),
ruff::Number::Complex { real, imag } => Constant::new_complex(real, imag, range),
};
c.ast_to_object(vm, source_code)
}
pub(super) fn string_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprStringLiteral,
) -> PyObjectRef {
let ruff::ExprStringLiteral { range, value } = constant;
let prefix = value
.iter()
.next()
.map_or(StringLiteralPrefix::Empty, |part| part.flags.prefix());
let c = Constant::new_str(value.to_str(), prefix, range);
c.ast_to_object(vm, source_code)
}
pub(super) fn bytes_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprBytesLiteral,
) -> PyObjectRef {
let ruff::ExprBytesLiteral { range, value } = constant;
let bytes = value.as_slice().iter().flat_map(|b| b.value.iter());
let c = Constant::new_bytes(bytes.copied().collect(), range);
c.ast_to_object(vm, source_code)
}
pub(super) fn boolean_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprBooleanLiteral,
) -> PyObjectRef {
let ruff::ExprBooleanLiteral { range, value } = constant;
let c = Constant::new_bool(value, range);
c.ast_to_object(vm, source_code)
}
pub(super) fn none_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprNoneLiteral,
) -> PyObjectRef {
let ruff::ExprNoneLiteral { range } = constant;
let c = Constant::new_none(range);
c.ast_to_object(vm, source_code)
}
pub(super) fn ellipsis_literal_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
constant: ruff::ExprEllipsisLiteral,
) -> PyObjectRef {
let ruff::ExprEllipsisLiteral { range } = constant;
let c = Constant::new_ellipsis(range);
c.ast_to_object(vm, source_code)
}
vm/src/stdlib/ast/elif_else_clause.rs
+88
View File
@@ -0,0 +1,88 @@
use super::*;
pub(super) fn ast_to_object(
clause: ruff::ElifElseClause,
mut rest: std::vec::IntoIter<ruff::ElifElseClause>,
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
) -> PyObjectRef {
let ruff::ElifElseClause { range, test, body } = clause;
let Some(test) = test else {
assert!(rest.len() == 0);
return body.ast_to_object(vm, source_code);
};
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeStmtIf::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("test", test.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("body", body.ast_to_object(vm, source_code), vm)
.unwrap();
let orelse = if let Some(next) = rest.next() {
if next.test.is_some() {
vm.ctx
.new_list(vec![ast_to_object(next, rest, vm, source_code)])
.into()
} else {
next.body.ast_to_object(vm, source_code)
}
} else {
vm.ctx.new_list(vec![]).into()
};
dict.set_item("orelse", orelse, vm).unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
pub(super) fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<ruff::StmtIf> {
let test = Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "test", "If")?)?;
let body = Node::ast_from_object(vm, source_code, get_node_field(vm, &object, "body", "If")?)?;
let orelse: Vec<ruff::Stmt> = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "orelse", "If")?,
)?;
let range = range_from_object(vm, source_code, object, "If")?;
let elif_else_clauses = if let [ruff::Stmt::If(_)] = &*orelse {
let Some(ruff::Stmt::If(ruff::StmtIf {
range,
test,
body,
mut elif_else_clauses,
})) = orelse.into_iter().next()
else {
unreachable!()
};
elif_else_clauses.insert(
0,
ruff::ElifElseClause {
range,
test: Some(*test),
body,
},
);
elif_else_clauses
} else {
vec![ruff::ElifElseClause {
range,
test: None,
body: orelse,
}]
};
Ok(ruff::StmtIf {
test,
body,
elif_else_clauses,
range,
})
}
vm/src/stdlib/ast/exception.rs
+75
View File
@@ -0,0 +1,75 @@
use super::*;
// sum
impl Node for ruff::ExceptHandler {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
ruff::ExceptHandler::ExceptHandler(cons) => cons.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(
if _cls.is(pyast::NodeExceptHandlerExceptHandler::static_type()) {
ruff::ExceptHandler::ExceptHandler(
ruff::ExceptHandlerExceptHandler::ast_from_object(_vm, source_code, _object)?,
)
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of excepthandler, but got {}",
_object.repr(_vm)?
)));
},
)
}
}
// constructor
impl Node for ruff::ExceptHandlerExceptHandler {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let ruff::ExceptHandlerExceptHandler {
type_,
name,
body,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(
_vm,
pyast::NodeExceptHandlerExceptHandler::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("type", type_.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("body", body.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(ruff::ExceptHandlerExceptHandler {
type_: get_node_field_opt(_vm, &_object, "type")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
name: get_node_field_opt(_vm, &_object, "name")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
body: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "body", "ExceptHandler")?,
)?,
range: range_from_object(_vm, source_code, _object, "ExceptHandler")?,
})
}
}
vm/src/stdlib/ast/expression.rs
+1197
View File
File diff suppressed because it is too large. Load diff
vm/src/stdlib/ast/gen.rs
Generated
-5496
View File
File diff suppressed because it is too large. Load diff
vm/src/stdlib/ast/module.rs
+221
View File
@@ -0,0 +1,221 @@
use super::*;
use crate::stdlib::ast::type_ignore::TypeIgnore;
/// Represents the different types of Python module structures.
///
/// This enum is used to represent the various possible forms of a Python module
/// in an Abstract Syntax Tree (AST). It can correspond to:
///
/// - `Module`: A standard Python script, containing a sequence of statements
/// (e.g., assignments, function calls), possibly with type ignores.
/// - `Interactive`: A representation of code executed in an interactive
/// Python session (e.g., the REPL or Jupyter notebooks), where statements
/// are evaluated one at a time.
/// - `Expression`: A single expression without any surrounding statements.
/// This is typically used in scenarios like `eval()` or in expression-only
/// contexts.
/// - `FunctionType`: A function signature with argument and return type
/// annotations, representing the type hints of a function (e.g., `def add(x: int, y: int) -> int`).
pub(super) enum Mod {
Module(ruff::ModModule),
Interactive(ModInteractive),
Expression(ruff::ModExpression),
FunctionType(ModFunctionType),
}
// sum
impl Node for Mod {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
Self::Module(cons) => cons.ast_to_object(vm, source_code),
Self::Interactive(cons) => cons.ast_to_object(vm, source_code),
Self::Expression(cons) => cons.ast_to_object(vm, source_code),
Self::FunctionType(cons) => cons.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let cls = object.class();
Ok(if cls.is(pyast::NodeModModule::static_type()) {
Self::Module(ruff::ModModule::ast_from_object(vm, source_code, object)?)
} else if cls.is(pyast::NodeModInteractive::static_type()) {
Self::Interactive(ModInteractive::ast_from_object(vm, source_code, object)?)
} else if cls.is(pyast::NodeModExpression::static_type()) {
Self::Expression(ruff::ModExpression::ast_from_object(
vm,
source_code,
object,
)?)
} else if cls.is(pyast::NodeModFunctionType::static_type()) {
Self::FunctionType(ModFunctionType::ast_from_object(vm, source_code, object)?)
} else {
return Err(vm.new_type_error(format!(
"expected some sort of mod, but got {}",
object.repr(vm)?
)));
})
}
}
// constructor
impl Node for ruff::ModModule {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let ruff::ModModule {
body,
// type_ignores,
range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeModModule::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("body", body.ast_to_object(vm, source_code), vm)
.unwrap();
// TODO: Improve ruff API
// ruff ignores type_ignore comments currently.
let type_ignores: Vec<TypeIgnore> = vec![];
dict.set_item(
"type_ignores",
type_ignores.ast_to_object(vm, source_code),
vm,
)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(ruff::ModModule {
body: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "body", "Module")?,
)?,
// type_ignores: Node::ast_from_object(
// _vm,
// get_node_field(_vm, &_object, "type_ignores", "Module")?,
// )?,
range: Default::default(),
})
}
}
pub(super) struct ModInteractive {
pub(crate) range: TextRange,
pub(crate) body: Vec<ruff::Stmt>,
}
// constructor
impl Node for ModInteractive {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { body, range } = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("body", body.ast_to_object(vm, source_code), vm)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
body: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "body", "Interactive")?,
)?,
range: Default::default(),
})
}
}
// constructor
impl Node for ruff::ModExpression {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { body, range } = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("body", body.ast_to_object(vm, source_code), vm)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
body: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "body", "Expression")?,
)?,
range: Default::default(),
})
}
}
pub(super) struct ModFunctionType {
pub(crate) argtypes: Box<[ruff::Expr]>,
pub(crate) returns: ruff::Expr,
pub(crate) range: TextRange,
}
// constructor
impl Node for ModFunctionType {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let ModFunctionType {
argtypes,
returns,
range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeModFunctionType::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item(
"argtypes",
BoxedSlice(argtypes).ast_to_object(vm, source_code),
vm,
)
.unwrap();
dict.set_item("returns", returns.ast_to_object(vm, source_code), vm)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(ModFunctionType {
argtypes: {
let argtypes: BoxedSlice<_> = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "argtypes", "FunctionType")?,
)?;
argtypes.0
},
returns: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "returns", "FunctionType")?,
)?,
range: Default::default(),
})
}
}
vm/src/stdlib/ast/node.rs
+93
View File
@@ -0,0 +1,93 @@
use crate::{PyObjectRef, PyResult, VirtualMachine};
use rustpython_compiler_source::SourceCodeOwned;
pub(crate) trait Node: Sized {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef;
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self>;
/// Used in `Option::ast_from_object`; if `true`, that impl will return None.
fn is_none(&self) -> bool {
false
}
}
impl<T: Node> Node for Vec<T> {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
vm.ctx
.new_list(
self.into_iter()
.map(|node| node.ast_to_object(vm, source_code))
.collect(),
)
.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, source_code, obj))
}
}
impl<T: Node> Node for Box<T> {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
(*self).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
T::ast_from_object(vm, source_code, object).map(Box::new)
}
fn is_none(&self) -> bool {
(**self).is_none()
}
}
impl<T: Node> Node for Option<T> {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
Some(node) => node.ast_to_object(vm, source_code),
None => vm.ctx.none(),
}
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
if vm.is_none(&object) {
Ok(None)
} else {
let x = T::ast_from_object(vm, source_code, object)?;
Ok((!x.is_none()).then_some(x))
}
}
}
pub(super) struct BoxedSlice<T>(pub(super) Box<[T]>);
impl<T: Node> Node for BoxedSlice<T> {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
self.0.into_vec().ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self(
<Vec<T> as Node>::ast_from_object(vm, source_code, object)?.into_boxed_slice(),
))
}
}
vm/src/stdlib/ast/operator.rs
+185
View File
@@ -0,0 +1,185 @@
use super::*;
// sum
impl Node for ruff::BoolOp {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
let node_type = match self {
ruff::BoolOp::And => pyast::NodeBoolOpAnd::static_type(),
ruff::BoolOp::Or => pyast::NodeBoolOpOr::static_type(),
};
NodeAst
.into_ref_with_type(vm, node_type.to_owned())
.unwrap()
.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeBoolOpAnd::static_type()) {
ruff::BoolOp::And
} else if _cls.is(pyast::NodeBoolOpOr::static_type()) {
ruff::BoolOp::Or
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of boolop, but got {}",
_object.repr(_vm)?
)));
})
}
}
// sum
impl Node for ruff::Operator {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
let node_type = match self {
ruff::Operator::Add => pyast::NodeOperatorAdd::static_type(),
ruff::Operator::Sub => pyast::NodeOperatorSub::static_type(),
ruff::Operator::Mult => pyast::NodeOperatorMult::static_type(),
ruff::Operator::MatMult => pyast::NodeOperatorMatMult::static_type(),
ruff::Operator::Div => pyast::NodeOperatorDiv::static_type(),
ruff::Operator::Mod => pyast::NodeOperatorMod::static_type(),
ruff::Operator::Pow => pyast::NodeOperatorPow::static_type(),
ruff::Operator::LShift => pyast::NodeOperatorLShift::static_type(),
ruff::Operator::RShift => pyast::NodeOperatorRShift::static_type(),
ruff::Operator::BitOr => pyast::NodeOperatorBitOr::static_type(),
ruff::Operator::BitXor => pyast::NodeOperatorBitXor::static_type(),
ruff::Operator::BitAnd => pyast::NodeOperatorBitAnd::static_type(),
ruff::Operator::FloorDiv => pyast::NodeOperatorFloorDiv::static_type(),
};
NodeAst
.into_ref_with_type(vm, node_type.to_owned())
.unwrap()
.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeOperatorAdd::static_type()) {
ruff::Operator::Add
} else if _cls.is(pyast::NodeOperatorSub::static_type()) {
ruff::Operator::Sub
} else if _cls.is(pyast::NodeOperatorMult::static_type()) {
ruff::Operator::Mult
} else if _cls.is(pyast::NodeOperatorMatMult::static_type()) {
ruff::Operator::MatMult
} else if _cls.is(pyast::NodeOperatorDiv::static_type()) {
ruff::Operator::Div
} else if _cls.is(pyast::NodeOperatorMod::static_type()) {
ruff::Operator::Mod
} else if _cls.is(pyast::NodeOperatorPow::static_type()) {
ruff::Operator::Pow
} else if _cls.is(pyast::NodeOperatorLShift::static_type()) {
ruff::Operator::LShift
} else if _cls.is(pyast::NodeOperatorRShift::static_type()) {
ruff::Operator::RShift
} else if _cls.is(pyast::NodeOperatorBitOr::static_type()) {
ruff::Operator::BitOr
} else if _cls.is(pyast::NodeOperatorBitXor::static_type()) {
ruff::Operator::BitXor
} else if _cls.is(pyast::NodeOperatorBitAnd::static_type()) {
ruff::Operator::BitAnd
} else if _cls.is(pyast::NodeOperatorFloorDiv::static_type()) {
ruff::Operator::FloorDiv
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of operator, but got {}",
_object.repr(_vm)?
)));
})
}
}
// sum
impl Node for ruff::UnaryOp {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
let node_type = match self {
ruff::UnaryOp::Invert => pyast::NodeUnaryOpInvert::static_type(),
ruff::UnaryOp::Not => pyast::NodeUnaryOpNot::static_type(),
ruff::UnaryOp::UAdd => pyast::NodeUnaryOpUAdd::static_type(),
ruff::UnaryOp::USub => pyast::NodeUnaryOpUSub::static_type(),
};
NodeAst
.into_ref_with_type(vm, node_type.to_owned())
.unwrap()
.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeUnaryOpInvert::static_type()) {
ruff::UnaryOp::Invert
} else if _cls.is(pyast::NodeUnaryOpNot::static_type()) {
ruff::UnaryOp::Not
} else if _cls.is(pyast::NodeUnaryOpUAdd::static_type()) {
ruff::UnaryOp::UAdd
} else if _cls.is(pyast::NodeUnaryOpUSub::static_type()) {
ruff::UnaryOp::USub
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of unaryop, but got {}",
_object.repr(_vm)?
)));
})
}
}
// sum
impl Node for ruff::CmpOp {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
let node_type = match self {
ruff::CmpOp::Eq => pyast::NodeCmpOpEq::static_type(),
ruff::CmpOp::NotEq => pyast::NodeCmpOpNotEq::static_type(),
ruff::CmpOp::Lt => pyast::NodeCmpOpLt::static_type(),
ruff::CmpOp::LtE => pyast::NodeCmpOpLtE::static_type(),
ruff::CmpOp::Gt => pyast::NodeCmpOpGt::static_type(),
ruff::CmpOp::GtE => pyast::NodeCmpOpGtE::static_type(),
ruff::CmpOp::Is => pyast::NodeCmpOpIs::static_type(),
ruff::CmpOp::IsNot => pyast::NodeCmpOpIsNot::static_type(),
ruff::CmpOp::In => pyast::NodeCmpOpIn::static_type(),
ruff::CmpOp::NotIn => pyast::NodeCmpOpNotIn::static_type(),
};
NodeAst
.into_ref_with_type(vm, node_type.to_owned())
.unwrap()
.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeCmpOpEq::static_type()) {
ruff::CmpOp::Eq
} else if _cls.is(pyast::NodeCmpOpNotEq::static_type()) {
ruff::CmpOp::NotEq
} else if _cls.is(pyast::NodeCmpOpLt::static_type()) {
ruff::CmpOp::Lt
} else if _cls.is(pyast::NodeCmpOpLtE::static_type()) {
ruff::CmpOp::LtE
} else if _cls.is(pyast::NodeCmpOpGt::static_type()) {
ruff::CmpOp::Gt
} else if _cls.is(pyast::NodeCmpOpGtE::static_type()) {
ruff::CmpOp::GtE
} else if _cls.is(pyast::NodeCmpOpIs::static_type()) {
ruff::CmpOp::Is
} else if _cls.is(pyast::NodeCmpOpIsNot::static_type()) {
ruff::CmpOp::IsNot
} else if _cls.is(pyast::NodeCmpOpIn::static_type()) {
ruff::CmpOp::In
} else if _cls.is(pyast::NodeCmpOpNotIn::static_type()) {
ruff::CmpOp::NotIn
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of cmpop, but got {}",
_object.repr(_vm)?
)));
})
}
}
vm/src/stdlib/ast/other.rs
+131
View File
@@ -0,0 +1,131 @@
use super::*;
use num_traits::ToPrimitive;
impl Node for ruff::ConversionFlag {
fn ast_to_object(self, vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
vm.ctx.new_int(self as u8).into()
}
fn ast_from_object(
vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
i32::try_from_object(vm, object)?
.to_u32()
.and_then(ruff::ConversionFlag::from_op_arg)
.ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned()))
}
}
// /// This is just a string, not strictly an AST node. But it makes AST conversions easier.
// impl Node for ruff::name::Name {
// fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
// vm.ctx.new_str(self.as_str()).to_pyobject(vm)
// }
// fn ast_from_object(vm: &VirtualMachine, source_code: &SourceCodeOwned, object: PyObjectRef) -> PyResult<Self> {
// match object.downcast::<PyStr>() {
// Ok(name) => Ok(Self::new(name)),
// Err(_) => Err(vm.new_value_error("expected str for name".to_owned())),
// }
// }
// }
impl Node for ruff::Decorator {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
ruff::Expr::ast_to_object(self.expression, vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let expression = ruff::Expr::ast_from_object(vm, source_code, object)?;
let range = expression.range();
Ok(ruff::Decorator { expression, range })
}
}
// product
impl Node for ruff::Alias {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
asname,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeAlias::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("name", name.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("asname", asname.ast_to_object(vm, source_code), vm)
.unwrap();
node_add_location(&dict, _range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "name", "alias")?,
)?,
asname: get_node_field_opt(vm, &object, "asname")?
.map(|obj| Node::ast_from_object(vm, source_code, obj))
.transpose()?,
range: range_from_object(vm, source_code, object, "alias")?,
})
}
}
// product
impl Node for ruff::WithItem {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
context_expr,
optional_vars,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeWithItem::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item(
"context_expr",
context_expr.ast_to_object(vm, source_code),
vm,
)
.unwrap();
dict.set_item(
"optional_vars",
optional_vars.ast_to_object(vm, source_code),
vm,
)
.unwrap();
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
context_expr: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "context_expr", "withitem")?,
)?,
optional_vars: get_node_field_opt(vm, &object, "optional_vars")?
.map(|obj| Node::ast_from_object(vm, source_code, obj))
.transpose()?,
range: Default::default(),
})
}
}
vm/src/stdlib/ast/parameter.rs
+401
View File
@@ -0,0 +1,401 @@
use super::*;
// product
impl Node for ruff::Parameters {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
posonlyargs,
args,
vararg,
kwonlyargs,
kwarg,
range,
} = self;
let (posonlyargs, args, defaults) =
extract_positional_parameter_defaults(posonlyargs, args);
let (kwonlyargs, kw_defaults) = extract_keyword_parameter_defaults(kwonlyargs);
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item(
"posonlyargs",
posonlyargs.ast_to_object(vm, source_code),
vm,
)
.unwrap();
dict.set_item("args", args.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("vararg", vararg.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("kwonlyargs", kwonlyargs.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item(
"kw_defaults",
kw_defaults.ast_to_object(vm, source_code),
vm,
)
.unwrap();
dict.set_item("kwarg", kwarg.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("defaults", defaults.ast_to_object(vm, source_code), vm)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let kwonlyargs = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "kwonlyargs", "arguments")?,
)?;
let kw_defaults = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "kw_defaults", "arguments")?,
)?;
let kwonlyargs = merge_keyword_parameter_defaults(kwonlyargs, kw_defaults);
let posonlyargs = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "posonlyargs", "arguments")?,
)?;
let args = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "args", "arguments")?,
)?;
let defaults = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "defaults", "arguments")?,
)?;
let (posonlyargs, args) = merge_positional_parameter_defaults(posonlyargs, args, defaults);
Ok(Self {
posonlyargs,
args,
vararg: get_node_field_opt(vm, &object, "vararg")?
.map(|obj| Node::ast_from_object(vm, source_code, obj))
.transpose()?,
kwonlyargs,
kwarg: get_node_field_opt(vm, &object, "kwarg")?
.map(|obj| Node::ast_from_object(vm, source_code, obj))
.transpose()?,
range: Default::default(),
})
}
fn is_none(&self) -> bool {
self.is_empty()
}
}
// product
impl Node for ruff::Parameter {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
annotation,
// type_comment,
range,
} = self;
// ruff covers the ** in range but python expects it to start at the ident
let range = TextRange::new(name.start(), range.end());
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodeArg::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("arg", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item(
"annotation",
annotation.ast_to_object(_vm, source_code),
_vm,
)
.unwrap();
// dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm)
// .unwrap();
node_add_location(&dict, range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "arg", "arg")?,
)?,
annotation: get_node_field_opt(_vm, &_object, "annotation")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
// type_comment: get_node_field_opt(_vm, &_object, "type_comment")?
// .map(|obj| Node::ast_from_object(_vm, obj))
// .transpose()?,
range: range_from_object(_vm, source_code, _object, "arg")?,
})
}
}
// product
impl Node for ruff::Keyword {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
arg,
value,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodeKeyword::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("arg", arg.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("value", value.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
arg: get_node_field_opt(_vm, &_object, "arg")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
value: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "value", "keyword")?,
)?,
range: range_from_object(_vm, source_code, _object, "keyword")?,
})
}
}
struct PositionalParameters {
pub _range: TextRange, // TODO: Use this
pub args: Box<[ruff::Parameter]>,
}
impl Node for PositionalParameters {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
BoxedSlice(self.args).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let args: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?;
Ok(Self {
args: args.0,
_range: TextRange::default(), // TODO
})
}
}
struct KeywordParameters {
pub _range: TextRange, // TODO: Use this
pub keywords: Box<[ruff::Parameter]>,
}
impl Node for KeywordParameters {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
BoxedSlice(self.keywords).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?;
Ok(Self {
keywords: keywords.0,
_range: TextRange::default(), // TODO
})
}
}
struct ParameterDefaults {
pub _range: TextRange, // TODO: Use this
defaults: Box<[Option<Box<ruff::Expr>>]>,
}
impl Node for ParameterDefaults {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
BoxedSlice(self.defaults).ast_to_object(vm, source_code)
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let defaults: BoxedSlice<_> = Node::ast_from_object(vm, source_code, object)?;
Ok(Self {
defaults: defaults.0,
_range: TextRange::default(), // TODO
})
}
}
fn extract_positional_parameter_defaults(
pos_only_args: Vec<ruff::ParameterWithDefault>,
args: Vec<ruff::ParameterWithDefault>,
) -> (
PositionalParameters,
PositionalParameters,
ParameterDefaults,
) {
let mut defaults = vec![];
defaults.extend(pos_only_args.iter().map(|item| item.default.clone()));
defaults.extend(args.iter().map(|item| item.default.clone()));
// If some positional parameters have no default value,
// the "defaults" list contains only the defaults of the last "n" parameters.
// Remove all positional parameters without a default value.
defaults.retain(Option::is_some);
let defaults = ParameterDefaults {
_range: defaults
.iter()
.flatten()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default(),
defaults: defaults.into_boxed_slice(),
};
let pos_only_args = PositionalParameters {
_range: pos_only_args
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default(),
args: {
let pos_only_args: Vec<_> = pos_only_args
.iter()
.map(|item| item.parameter.clone())
.collect();
pos_only_args.into_boxed_slice()
},
};
let args = PositionalParameters {
_range: args
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default(),
args: {
let args: Vec<_> = args.iter().map(|item| item.parameter.clone()).collect();
args.into_boxed_slice()
},
};
(pos_only_args, args, defaults)
}
/// Merges the keyword parameters with their default values, opposite of [`extract_positional_parameter_defaults`].
fn merge_positional_parameter_defaults(
posonlyargs: PositionalParameters,
args: PositionalParameters,
defaults: ParameterDefaults,
) -> (
Vec<ruff::ParameterWithDefault>,
Vec<ruff::ParameterWithDefault>,
) {
let posonlyargs = posonlyargs.args;
let args = args.args;
let defaults = defaults.defaults;
let mut posonlyargs: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(posonlyargs)
.map(|parameter| ruff::ParameterWithDefault {
range: Default::default(),
parameter,
default: None,
})
.collect();
let mut args: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(args)
.map(|parameter| ruff::ParameterWithDefault {
range: Default::default(),
parameter,
default: None,
})
.collect();
// If an argument has a default value, insert it
// Note that "defaults" will only contain default values for the last "n" parameters
// so we need to skip the first "total_argument_count - n" arguments.
let default_argument_count = posonlyargs.len() + args.len() - defaults.len();
for (arg, default) in posonlyargs
.iter_mut()
.chain(args.iter_mut())
.skip(default_argument_count)
.zip(defaults)
{
arg.default = default;
}
(posonlyargs, args)
}
fn extract_keyword_parameter_defaults(
kw_only_args: Vec<ruff::ParameterWithDefault>,
) -> (KeywordParameters, ParameterDefaults) {
let mut defaults = vec![];
defaults.extend(kw_only_args.iter().map(|item| item.default.clone()));
let defaults = ParameterDefaults {
_range: defaults
.iter()
.flatten()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default(),
defaults: defaults.into_boxed_slice(),
};
let kw_only_args = KeywordParameters {
_range: kw_only_args
.iter()
.map(|item| item.range())
.reduce(|acc, next| acc.cover(next))
.unwrap_or_default(),
keywords: {
let kw_only_args: Vec<_> = kw_only_args
.iter()
.map(|item| item.parameter.clone())
.collect();
kw_only_args.into_boxed_slice()
},
};
(kw_only_args, defaults)
}
/// Merges the keyword parameters with their default values, opposite of [`extract_keyword_parameter_defaults`].
fn merge_keyword_parameter_defaults(
kw_only_args: KeywordParameters,
defaults: ParameterDefaults,
) -> Vec<ruff::ParameterWithDefault> {
std::iter::zip(kw_only_args.keywords, defaults.defaults)
.map(|(parameter, default)| ruff::ParameterWithDefault {
parameter,
default,
range: Default::default(),
})
.collect()
}
vm/src/stdlib/ast/pattern.rs
+515
View File
@@ -0,0 +1,515 @@
use super::*;
// product
impl Node for ruff::MatchCase {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
pattern,
guard,
body,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodeMatchCase::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("pattern", pattern.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("guard", guard.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("body", body.ast_to_object(_vm, source_code), _vm)
.unwrap();
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
pattern: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "pattern", "match_case")?,
)?,
guard: get_node_field_opt(_vm, &_object, "guard")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
body: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "body", "match_case")?,
)?,
range: Default::default(),
})
}
}
// sum
impl Node for ruff::Pattern {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
ruff::Pattern::MatchValue(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchSingleton(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchSequence(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchMapping(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchClass(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchStar(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchAs(cons) => cons.ast_to_object(vm, source_code),
ruff::Pattern::MatchOr(cons) => cons.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodePatternMatchValue::static_type()) {
ruff::Pattern::MatchValue(ruff::PatternMatchValue::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchSingleton::static_type()) {
ruff::Pattern::MatchSingleton(ruff::PatternMatchSingleton::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchSequence::static_type()) {
ruff::Pattern::MatchSequence(ruff::PatternMatchSequence::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchMapping::static_type()) {
ruff::Pattern::MatchMapping(ruff::PatternMatchMapping::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchClass::static_type()) {
ruff::Pattern::MatchClass(ruff::PatternMatchClass::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchStar::static_type()) {
ruff::Pattern::MatchStar(ruff::PatternMatchStar::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchAs::static_type()) {
ruff::Pattern::MatchAs(ruff::PatternMatchAs::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodePatternMatchOr::static_type()) {
ruff::Pattern::MatchOr(ruff::PatternMatchOr::ast_from_object(
_vm,
source_code,
_object,
)?)
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of pattern, but got {}",
_object.repr(_vm)?
)));
})
}
}
// constructor
impl Node for ruff::PatternMatchValue {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
value,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodePatternMatchValue::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("value", value.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
value: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "value", "MatchValue")?,
)?,
range: range_from_object(_vm, source_code, _object, "MatchValue")?,
})
}
}
// constructor
impl Node for ruff::PatternMatchSingleton {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
value,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(
_vm,
pyast::NodePatternMatchSingleton::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("value", value.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
value: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "value", "MatchSingleton")?,
)?,
range: range_from_object(_vm, source_code, _object, "MatchSingleton")?,
})
}
}
impl Node for ruff::Singleton {
fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
todo!()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
todo!()
}
}
// constructor
impl Node for ruff::PatternMatchSequence {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
patterns,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(
_vm,
pyast::NodePatternMatchSequence::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
patterns: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "patterns", "MatchSequence")?,
)?,
range: range_from_object(_vm, source_code, _object, "MatchSequence")?,
})
}
}
// constructor
impl Node for ruff::PatternMatchMapping {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
keys,
patterns,
rest,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(
_vm,
pyast::NodePatternMatchMapping::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("keys", keys.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("rest", rest.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
keys: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "keys", "MatchMapping")?,
)?,
patterns: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "patterns", "MatchMapping")?,
)?,
rest: get_node_field_opt(_vm, &_object, "rest")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
range: range_from_object(_vm, source_code, _object, "MatchMapping")?,
})
}
}
// constructor
impl Node for ruff::PatternMatchClass {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
cls,
arguments,
range: _range,
} = self;
let (patterns, kwd_attrs, kwd_patterns) = split_pattern_match_class(arguments);
let node = NodeAst
.into_ref_with_type(vm, pyast::NodePatternMatchClass::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("cls", cls.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("patterns", patterns.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("kwd_attrs", kwd_attrs.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item(
"kwd_patterns",
kwd_patterns.ast_to_object(vm, source_code),
vm,
)
.unwrap();
node_add_location(&dict, _range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let patterns = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "patterns", "MatchClass")?,
)?;
let kwd_attrs = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "kwd_attrs", "MatchClass")?,
)?;
let kwd_patterns = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "kwd_patterns", "MatchClass")?,
)?;
let (patterns, keywords) = merge_pattern_match_class(patterns, kwd_attrs, kwd_patterns);
Ok(Self {
cls: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "cls", "MatchClass")?,
)?,
range: range_from_object(vm, source_code, object, "MatchClass")?,
arguments: ruff::PatternArguments {
range: Default::default(),
patterns,
keywords,
},
})
}
}
struct PatternMatchClassPatterns {
pub _range: TextRange, // TODO: Use this
}
impl Node for PatternMatchClassPatterns {
fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
todo!()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
todo!()
}
}
struct PatternMatchClassKeywordAttributes {
pub _range: TextRange, // TODO: Use this
}
impl Node for PatternMatchClassKeywordAttributes {
fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
todo!()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
todo!()
}
}
struct PatternMatchClassKeywordPatterns {
pub _range: TextRange, // TODO: Use this
}
impl Node for PatternMatchClassKeywordPatterns {
fn ast_to_object(self, _vm: &VirtualMachine, _source_code: &SourceCodeOwned) -> PyObjectRef {
todo!()
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
todo!()
}
}
// constructor
impl Node for ruff::PatternMatchStar {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodePatternMatchStar::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: get_node_field_opt(_vm, &_object, "name")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
range: range_from_object(_vm, source_code, _object, "MatchStar")?,
})
}
}
// constructor
impl Node for ruff::PatternMatchAs {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
pattern,
name,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodePatternMatchAs::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("pattern", pattern.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
pattern: get_node_field_opt(_vm, &_object, "pattern")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
name: get_node_field_opt(_vm, &_object, "name")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
range: range_from_object(_vm, source_code, _object, "MatchAs")?,
})
}
}
// constructor
impl Node for ruff::PatternMatchOr {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
patterns,
range: _range,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodePatternMatchOr::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("patterns", patterns.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
patterns: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "patterns", "MatchOr")?,
)?,
range: range_from_object(_vm, source_code, _object, "MatchOr")?,
})
}
}
fn split_pattern_match_class(
_arguments: ruff::PatternArguments,
) -> (
PatternMatchClassPatterns,
PatternMatchClassKeywordAttributes,
PatternMatchClassKeywordPatterns,
) {
todo!()
}
/// Merges the pattern match class attributes and patterns, opposite of [`split_pattern_match_class`].
fn merge_pattern_match_class(
_patterns: PatternMatchClassPatterns,
_kwd_attrs: PatternMatchClassKeywordAttributes,
_kwd_patterns: PatternMatchClassKeywordPatterns,
) -> (Vec<ruff::Pattern>, Vec<ruff::PatternKeyword>) {
todo!()
}
vm/src/stdlib/ast/pyast.rs
+2448
View File
File diff suppressed because it is too large. Load diff
vm/src/stdlib/ast/python.rs
+57
View File
@@ -0,0 +1,57 @@
use super::PY_COMPILE_FLAG_AST_ONLY;
#[pymodule]
pub(crate) mod _ast {
use crate::{
AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyStrRef, PyTupleRef},
function::FuncArgs,
};
#[pyattr]
#[pyclass(module = "_ast", name = "AST")]
#[derive(Debug, PyPayload)]
pub(crate) struct NodeAst;
#[pyclass(flags(BASETYPE, HAS_DICT))]
impl NodeAst {
#[pyslot]
#[pymethod(magic)]
fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> {
let fields = zelf.get_attr("_fields", vm)?;
let fields: Vec<PyStrRef> = fields.try_to_value(vm)?;
let numargs = args.args.len();
if numargs > fields.len() {
return Err(vm.new_type_error(format!(
"{} constructor takes at most {} positional argument{}",
zelf.class().name(),
fields.len(),
if fields.len() == 1 { "" } else { "s" },
)));
}
for (name, arg) in fields.iter().zip(args.args) {
zelf.set_attr(name, arg, vm)?;
}
for (key, value) in args.kwargs {
if let Some(pos) = fields.iter().position(|f| f.as_str() == key) {
if pos < numargs {
return Err(vm.new_type_error(format!(
"{} got multiple values for argument '{}'",
zelf.class().name(),
key
)));
}
}
zelf.set_attr(vm.ctx.intern_str(key), value, vm)?;
}
Ok(())
}
#[pyattr(name = "_fields")]
fn fields(ctx: &Context) -> PyTupleRef {
ctx.empty_tuple.clone()
}
}
#[pyattr(name = "PyCF_ONLY_AST")]
use super::PY_COMPILE_FLAG_AST_ONLY;
}
vm/src/stdlib/ast/statement.rs
+1222
View File
File diff suppressed because it is too large. Load diff
vm/src/stdlib/ast/string.rs
+354
View File
@@ -0,0 +1,354 @@
use super::constant::{Constant, ConstantLiteral};
use super::*;
fn ruff_fstring_value_into_iter(
mut fstring_value: ruff::FStringValue,
) -> impl Iterator<Item = ruff::FStringPart> + 'static {
let default = ruff::FStringPart::FString(ruff::FString {
range: Default::default(),
elements: Default::default(),
flags: Default::default(),
});
(0..fstring_value.as_slice().len()).map(move |i| {
let fstring_value = &mut fstring_value;
let tmp = fstring_value.into_iter().nth(i).unwrap();
std::mem::replace(tmp, default.clone())
})
}
fn ruff_fstring_element_into_iter(
mut fstring_element: ruff::FStringElements,
) -> impl Iterator<Item = ruff::FStringElement> + 'static {
let default = ruff::FStringElement::Literal(ruff::FStringLiteralElement {
range: Default::default(),
value: Default::default(),
});
(0..fstring_element.into_iter().len()).map(move |i| {
let fstring_element = &mut fstring_element;
let tmp = fstring_element.into_iter().nth(i).unwrap();
std::mem::replace(tmp, default.clone())
})
}
fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec<JoinedStrPart> {
match fstring_part {
ruff::FStringPart::Literal(ruff::StringLiteral {
range,
value,
flags,
}) => {
vec![JoinedStrPart::Constant(Constant::new_str(
value,
flags.prefix(),
range,
))]
}
ruff::FStringPart::FString(ruff::FString {
range: _,
elements,
flags: _, // TODO
}) => ruff_fstring_element_into_iter(elements)
.map(ruff_fstring_element_to_joined_str_part)
.collect(),
}
}
fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> JoinedStrPart {
match element {
ruff::FStringElement::Literal(ruff::FStringLiteralElement { range, value }) => {
JoinedStrPart::Constant(Constant::new_str(
value,
ruff::str_prefix::StringLiteralPrefix::Empty,
range,
))
}
ruff::FStringElement::Expression(ruff::FStringExpressionElement {
range,
expression,
debug_text: _, // TODO: What is this?
conversion,
format_spec,
}) => JoinedStrPart::FormattedValue(FormattedValue {
value: expression,
conversion,
format_spec: ruff_format_spec_to_joined_str(format_spec),
range,
}),
}
}
fn ruff_format_spec_to_joined_str(
format_spec: Option<Box<ruff::FStringFormatSpec>>,
) -> Option<Box<JoinedStr>> {
match format_spec {
None => None,
Some(format_spec) => {
let ruff::FStringFormatSpec { range, elements } = *format_spec;
let values: Vec<_> = ruff_fstring_element_into_iter(elements)
.map(ruff_fstring_element_to_joined_str_part)
.collect();
let values = values.into_boxed_slice();
Some(Box::new(JoinedStr { values, range }))
}
}
}
fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> ruff::FStringPart {
match element {
ruff::FStringElement::Literal(value) => {
let ruff::FStringLiteralElement { range, value } = value;
ruff::FStringPart::Literal(ruff::StringLiteral {
range,
value,
flags: Default::default(),
})
}
ruff::FStringElement::Expression(value) => {
let ruff::FStringExpressionElement {
range,
expression,
debug_text,
conversion,
format_spec,
} = value;
ruff::FStringPart::FString(ruff::FString {
range,
elements: vec![ruff::FStringElement::Expression(
ruff::FStringExpressionElement {
range,
expression,
debug_text,
conversion,
format_spec,
},
)]
.into(),
flags: Default::default(),
})
}
}
}
fn joined_str_to_ruff_format_spec(
joined_str: Option<Box<JoinedStr>>,
) -> Option<Box<ruff::FStringFormatSpec>> {
match joined_str {
None => None,
Some(joined_str) => {
let JoinedStr { range, values } = *joined_str;
let elements: Vec<_> = Box::into_iter(values)
.map(joined_str_part_to_ruff_fstring_element)
.collect();
let format_spec = ruff::FStringFormatSpec {
range,
elements: elements.into(),
};
Some(Box::new(format_spec))
}
}
}
#[derive(Debug)]
pub(super) struct JoinedStr {
pub(super) range: TextRange,
pub(super) values: Box<[JoinedStrPart]>,
}
impl JoinedStr {
pub(super) fn into_expr(self) -> ruff::Expr {
let Self { range, values } = self;
ruff::Expr::FString(ruff::ExprFString {
range: Default::default(),
value: match values.len() {
// ruff represents an empty fstring like this:
0 => ruff::FStringValue::single(ruff::FString {
range,
elements: vec![].into(),
flags: Default::default(),
}),
1 => ruff::FStringValue::single(
Box::<[_]>::into_iter(values)
.map(joined_str_part_to_ruff_fstring_element)
.map(|element| ruff::FString {
range,
elements: vec![element].into(),
flags: Default::default(),
})
.next()
.expect("FString has exactly one part"),
),
_ => ruff::FStringValue::concatenated(
Box::<[_]>::into_iter(values)
.map(joined_str_part_to_ruff_fstring_element)
.map(ruff_fstring_element_to_ruff_fstring_part)
.collect(),
),
},
})
}
}
fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::FStringElement {
match part {
JoinedStrPart::FormattedValue(value) => {
ruff::FStringElement::Expression(ruff::FStringExpressionElement {
range: value.range,
expression: value.value.clone(),
debug_text: None, // TODO: What is this?
conversion: value.conversion,
format_spec: joined_str_to_ruff_format_spec(value.format_spec),
})
}
JoinedStrPart::Constant(value) => {
ruff::FStringElement::Literal(ruff::FStringLiteralElement {
range: value.range,
value: match value.value {
ConstantLiteral::Str { value, .. } => value,
_ => todo!(),
},
})
}
}
}
// constructor
impl Node for JoinedStr {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { values, range } = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeExprJoinedStr::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item(
"values",
BoxedSlice(values).ast_to_object(vm, source_code),
vm,
)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let values: BoxedSlice<_> = Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "values", "JoinedStr")?,
)?;
Ok(Self {
values: values.0,
range: range_from_object(vm, source_code, object, "JoinedStr")?,
})
}
}
#[derive(Debug)]
pub(super) enum JoinedStrPart {
FormattedValue(FormattedValue),
Constant(Constant),
}
// constructor
impl Node for JoinedStrPart {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
JoinedStrPart::FormattedValue(value) => value.ast_to_object(vm, source_code),
JoinedStrPart::Constant(value) => value.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
let cls = object.class();
if cls.is(pyast::NodeExprFormattedValue::static_type()) {
Ok(Self::FormattedValue(Node::ast_from_object(
vm,
source_code,
object,
)?))
} else {
Ok(Self::Constant(Node::ast_from_object(
vm,
source_code,
object,
)?))
}
}
}
#[derive(Debug)]
pub(super) struct FormattedValue {
value: Box<ruff::Expr>,
conversion: ruff::ConversionFlag,
format_spec: Option<Box<JoinedStr>>,
range: TextRange,
}
// constructor
impl Node for FormattedValue {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
value,
conversion,
format_spec,
range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeExprFormattedValue::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("value", value.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item("conversion", conversion.ast_to_object(vm, source_code), vm)
.unwrap();
dict.set_item(
"format_spec",
format_spec.ast_to_object(vm, source_code),
vm,
)
.unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
value: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "value", "FormattedValue")?,
)?,
conversion: Node::ast_from_object(
vm,
source_code,
get_node_field(vm, &object, "conversion", "FormattedValue")?,
)?,
format_spec: get_node_field_opt(vm, &object, "format_spec")?
.map(|obj| Node::ast_from_object(vm, source_code, obj))
.transpose()?,
range: range_from_object(vm, source_code, object, "FormattedValue")?,
})
}
}
pub(super) fn fstring_to_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
expression: ruff::ExprFString,
) -> PyObjectRef {
let ruff::ExprFString { range, value } = expression;
let values: Vec<_> = ruff_fstring_value_into_iter(value)
.flat_map(fstring_part_to_joined_str_part)
.collect();
let values = values.into_boxed_slice();
let c = JoinedStr { range, values };
c.ast_to_object(vm, source_code)
}
vm/src/stdlib/ast/type_ignore.rs
+73
View File
@@ -0,0 +1,73 @@
use super::*;
pub(super) enum TypeIgnore {
TypeIgnore(TypeIgnoreTypeIgnore),
}
// sum
impl Node for TypeIgnore {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
TypeIgnore::TypeIgnore(cons) => cons.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeTypeIgnoreTypeIgnore::static_type()) {
TypeIgnore::TypeIgnore(TypeIgnoreTypeIgnore::ast_from_object(
_vm,
source_code,
_object,
)?)
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of type_ignore, but got {}",
_object.repr(_vm)?
)));
})
}
}
pub(super) struct TypeIgnoreTypeIgnore {
range: TextRange,
lineno: PyRefExact<PyInt>,
tag: PyRefExact<PyStr>,
}
// constructor
impl Node for TypeIgnoreTypeIgnore {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self { lineno, tag, range } = self;
let node = NodeAst
.into_ref_with_type(
vm,
pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("lineno", lineno.to_pyobject(vm), vm).unwrap();
dict.set_item("tag", tag.to_pyobject(vm), vm).unwrap();
node_add_location(&dict, range, vm, source_code);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_code: &SourceCodeOwned,
object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
lineno: get_node_field(vm, &object, "lineno", "TypeIgnore")?
.downcast_exact(vm)
.unwrap(),
tag: get_node_field(vm, &object, "tag", "TypeIgnore")?
.downcast_exact(vm)
.unwrap(),
range: range_from_object(vm, source_code, object, "TypeIgnore")?,
})
}
}
vm/src/stdlib/ast/type_parameters.rs
+197
View File
@@ -0,0 +1,197 @@
use super::*;
impl Node for ruff::TypeParams {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
self.type_params.ast_to_object(vm, source_code)
}
fn ast_from_object(
_vm: &VirtualMachine,
_source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let type_params: Vec<ruff::TypeParam> = Node::ast_from_object(_vm, _source_code, _object)?;
let range = Option::zip(type_params.first(), type_params.last())
.map(|(first, last)| first.range().cover(last.range()))
.unwrap_or_default();
Ok(Self { type_params, range })
}
fn is_none(&self) -> bool {
self.type_params.is_empty()
}
}
// sum
impl Node for ruff::TypeParam {
fn ast_to_object(self, vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
match self {
Self::TypeVar(cons) => cons.ast_to_object(vm, source_code),
Self::ParamSpec(cons) => cons.ast_to_object(vm, source_code),
Self::TypeVarTuple(cons) => cons.ast_to_object(vm, source_code),
}
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
let _cls = _object.class();
Ok(if _cls.is(pyast::NodeTypeParamTypeVar::static_type()) {
ruff::TypeParam::TypeVar(ruff::TypeParamTypeVar::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodeTypeParamParamSpec::static_type()) {
ruff::TypeParam::ParamSpec(ruff::TypeParamParamSpec::ast_from_object(
_vm,
source_code,
_object,
)?)
} else if _cls.is(pyast::NodeTypeParamTypeVarTuple::static_type()) {
ruff::TypeParam::TypeVarTuple(ruff::TypeParamTypeVarTuple::ast_from_object(
_vm,
source_code,
_object,
)?)
} else {
return Err(_vm.new_type_error(format!(
"expected some sort of type_param, but got {}",
_object.repr(_vm)?
)));
})
}
}
// constructor
impl Node for ruff::TypeParamTypeVar {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
bound,
range: _range,
default: _,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodeTypeParamTypeVar::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item("bound", bound.ast_to_object(_vm, source_code), _vm)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "name", "TypeVar")?,
)?,
bound: get_node_field_opt(_vm, &_object, "bound")?
.map(|obj| Node::ast_from_object(_vm, source_code, obj))
.transpose()?,
default: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "default_value", "TypeVar")?,
)?,
range: range_from_object(_vm, source_code, _object, "TypeVar")?,
})
}
}
// constructor
impl Node for ruff::TypeParamParamSpec {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
range: _range,
default,
} = self;
let node = NodeAst
.into_ref_with_type(_vm, pyast::NodeTypeParamParamSpec::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item(
"default_value",
default.ast_to_object(_vm, source_code),
_vm,
)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "name", "ParamSpec")?,
)?,
default: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "default_value", "ParamSpec")?,
)?,
range: range_from_object(_vm, source_code, _object, "ParamSpec")?,
})
}
}
// constructor
impl Node for ruff::TypeParamTypeVarTuple {
fn ast_to_object(self, _vm: &VirtualMachine, source_code: &SourceCodeOwned) -> PyObjectRef {
let Self {
name,
range: _range,
default,
} = self;
let node = NodeAst
.into_ref_with_type(
_vm,
pyast::NodeTypeParamTypeVarTuple::static_type().to_owned(),
)
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("name", name.ast_to_object(_vm, source_code), _vm)
.unwrap();
dict.set_item(
"default_value",
default.ast_to_object(_vm, source_code),
_vm,
)
.unwrap();
node_add_location(&dict, _range, _vm, source_code);
node.into()
}
fn ast_from_object(
_vm: &VirtualMachine,
source_code: &SourceCodeOwned,
_object: PyObjectRef,
) -> PyResult<Self> {
Ok(Self {
name: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "name", "TypeVarTuple")?,
)?,
default: Node::ast_from_object(
_vm,
source_code,
get_node_field(_vm, &_object, "default_value", "TypeVarTuple")?,
)?,
range: range_from_object(_vm, source_code, _object, "TypeVarTuple")?,
})
}
}
vm/src/stdlib/builtins.rs
+12 -7
View File
@@ -110,10 +110,15 @@ mod builtins {
_feature_version: OptionalArg<i32>,
}
#[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
#[cfg(any(feature = "parser", feature = "compiler"))]
#[pyfunction]
fn compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult {
#[cfg(feature = "rustpython-ast")]
#[cfg(not(feature = "ast"))]
{
_ = args; // to disable unused warning
return Err(vm.new_type_error("AST Not Supported".to_owned()));
}
#[cfg(feature = "ast")]
{
use crate::{class::PyClassImpl, stdlib::ast};
@@ -155,16 +160,16 @@ mod builtins {
}
}
#[cfg(not(feature = "rustpython-parser"))]
#[cfg(not(feature = "parser"))]
{
const PARSER_NOT_SUPPORTED: &str = "can't compile() source code when the `parser` feature of rustpython is disabled";
Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned()))
}
#[cfg(feature = "rustpython-parser")]
#[cfg(feature = "parser")]
{
use crate::convert::ToPyException;
use num_traits::Zero;
use rustpython_parser as parser;
use ruff_python_parser as parser;
let source = ArgStrOrBytesLike::try_from_object(vm, args.source)?;
let source = source.borrow_bytes();
@@ -180,11 +185,11 @@ mod builtins {
}
if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() {
#[cfg(not(feature = "rustpython-compiler"))]
#[cfg(not(feature = "compiler"))]
{
Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned()))
}
#[cfg(feature = "rustpython-compiler")]
#[cfg(feature = "compiler")]
{
let mode = mode_str
.parse::<crate::compiler::Mode>()
vm/src/stdlib/mod.rs
+4 -4
View File
@@ -1,4 +1,4 @@
#[cfg(feature = "rustpython-ast")]
#[cfg(feature = "ast")]
pub(crate) mod ast;
pub mod atexit;
pub mod builtins;
@@ -15,7 +15,7 @@ mod operator;
// mod re;
mod sre;
mod string;
#[cfg(feature = "rustpython-compiler")]
#[cfg(feature = "compiler")]
mod symtable;
mod sysconfigdata;
#[cfg(feature = "threading")]
@@ -98,12 +98,12 @@ pub fn get_module_inits() -> StdlibMap {
sys::sysconfigdata_name() => sysconfigdata::make_module,
}
// parser related modules:
#[cfg(feature = "rustpython-ast")]
#[cfg(feature = "ast")]
{
"_ast" => ast::make_module,
}
// compiler related modules:
#[cfg(feature = "rustpython-compiler")]
#[cfg(feature = "compiler")]
{
"symtable" => symtable::make_module,
}
vm/src/vm/compile.rs
+1 -1
View File
@@ -23,7 +23,7 @@ impl VirtualMachine {
source_path: String,
opts: CompileOpts,
) -> Result<PyRef<PyCode>, CompileError> {
compiler::compile(source, mode, source_path, opts).map(|code| self.ctx.new_code(code))
compiler::compile(source, mode, &source_path, opts).map(|code| self.ctx.new_code(code))
}
pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> {
vm/src/vm/vm_new.rs
+45 -18
View File
@@ -256,22 +256,39 @@ impl VirtualMachine {
self.new_exception_msg(overflow_error, msg)
}
#[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
#[cfg(any(feature = "parser", feature = "compiler"))]
pub fn new_syntax_error(
&self,
error: &crate::compiler::CompileError,
source: Option<&str>,
) -> PyBaseExceptionRef {
use crate::source_code::SourceLocation;
use crate::source::SourceLocation;
let syntax_error_type = match &error.error {
#[cfg(feature = "rustpython-parser")]
crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => {
self.ctx.exceptions.indentation_error
}
#[cfg(feature = "rustpython-parser")]
crate::compiler::CompileErrorType::Parse(p) if p.is_tab_error() => {
self.ctx.exceptions.tab_error
let syntax_error_type = match &error {
#[cfg(feature = "parser")]
// FIXME: this condition will cause TabError even when the matching actual error is IndentationError
crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
error:
ruff_python_parser::ParseErrorType::Lexical(
ruff_python_parser::lexer::LexicalErrorType::IndentationError,
),
..
}) => self.ctx.exceptions.tab_error,
#[cfg(feature = "parser")]
crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
error: ruff_python_parser::ParseErrorType::UnexpectedIndentation,
..
}) => self.ctx.exceptions.indentation_error,
#[cfg(feature = "parser")]
crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
error: ruff_python_parser::ParseErrorType::OtherError(s),
..
}) => {
if s.starts_with("Expected an indented block after") {
self.ctx.exceptions.indentation_error
} else {
self.ctx.exceptions.syntax_error
}
}
_ => self.ctx.exceptions.syntax_error,
}
@@ -281,18 +298,32 @@ impl VirtualMachine {
fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> {
let line = source
.split('\n')
.nth(loc?.row.to_zero_indexed_usize())?
.nth(loc?.row.to_zero_indexed())?
.to_owned();
Some(line + "\n")
}
let statement = if let Some(source) = source {
get_statement(source, error.location)
get_statement(source, error.location())
} else {
None
};
let syntax_error = self.new_exception_msg(syntax_error_type, error.error.to_string());
let mut msg = error.to_string();
if let Some(msg) = msg.get_mut(..1) {
msg.make_ascii_lowercase();
}
match error {
#[cfg(feature = "parser")]
crate::compiler::CompileError::Parse(rustpython_compiler::ParseError {
error:
ruff_python_parser::ParseErrorType::FStringError(_)
| ruff_python_parser::ParseErrorType::UnexpectedExpressionToken,
..
}) => msg.insert_str(0, "invalid syntax: "),
_ => {}
}
let syntax_error = self.new_exception_msg(syntax_error_type, msg);
let (lineno, offset) = error.python_location();
let lineno = self.ctx.new_int(lineno);
let offset = self.ctx.new_int(offset);
@@ -311,11 +342,7 @@ impl VirtualMachine {
.unwrap();
syntax_error
.as_object()
.set_attr(
"filename",
self.ctx.new_str(error.source_path.clone()),
self,
)
.set_attr("filename", self.ctx.new_str(error.source_path()), self)
.unwrap();
syntax_error
}
wasm/lib/Cargo.toml
+1 -1
View File
@@ -23,7 +23,7 @@ rustpython-stdlib = { workspace = true, default-features = false, optional = tru
# make sure no threading! otherwise wasm build will fail
rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde", "wasmbind"] }
rustpython-parser = { workspace = true }
ruff_python_parser = { workspace = true }
serde = { workspace = true }
wasm-bindgen = { workspace = true }
wasm/lib/src/convert.rs
+10 -11
View File
@@ -3,11 +3,10 @@
use crate::js_module;
use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm};
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
use rustpython_parser::{ParseErrorType, lexer::LexicalErrorType};
use rustpython_vm::{
AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine,
builtins::PyBaseExceptionRef,
compiler::{CompileError, CompileErrorType},
compiler::{CompileError, ParseError, parser::ParseErrorType, parser::lexer::LexicalErrorType},
exceptions,
function::{ArgBytesLike, FuncArgs},
py_serde,
@@ -252,21 +251,21 @@ pub fn syntax_err(err: CompileError) -> SyntaxError {
let _ = Reflect::set(
&js_err,
&"row".into(),
&(err.location.unwrap().row.get()).into(),
&(err.location().unwrap().row.get()).into(),
);
let _ = Reflect::set(
&js_err,
&"col".into(),
&(err.location.unwrap().column.get()).into(),
&(err.location().unwrap().column.get()).into(),
);
// | ParseErrorType::UnrecognizedToken(Token::Dedent, _)
let can_continue = matches!(
&err.error,
CompileErrorType::Parse(
ParseErrorType::Eof
| ParseErrorType::Lexical(LexicalErrorType::Eof)
| ParseErrorType::Lexical(LexicalErrorType::IndentationError)
| ParseErrorType::UnrecognizedToken(rustpython_parser::Tok::Dedent, _)
)
&err,
CompileError::Parse(ParseError {
error: ParseErrorType::Lexical(LexicalErrorType::Eof)
| ParseErrorType::Lexical(LexicalErrorType::IndentationError),
..
})
);
let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into());
js_err