From cb394ebdc4b91b65d0f19d7e96004b78fcc79481 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Fri, 1 Jun 2018 21:36:19 +0200 Subject: [PATCH] Initial import from rspython repo --- .travis.yml | 6 + Cargo.lock | 740 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 17 + build.rs | 5 + src/compiler/ast.rs | 123 ++++++ src/compiler/builtins.rs | 28 ++ src/compiler/bytecode.rs | 92 +++++ src/compiler/compile.rs | 273 +++++++++++++ src/compiler/lexer.rs | 600 +++++++++++++++++++++++++++++ src/compiler/mod.rs | 22 ++ src/compiler/parser.rs | 104 +++++ src/compiler/pyobject.rs | 241 ++++++++++++ src/compiler/python.lalrpop | 261 +++++++++++++ src/compiler/token.rs | 94 +++++ src/compiler/vm.rs | 227 +++++++++++ src/main.rs | 43 +++ 16 files changed, 2876 insertions(+) create mode 100644 .travis.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 build.rs create mode 100644 src/compiler/ast.rs create mode 100644 src/compiler/builtins.rs create mode 100644 src/compiler/bytecode.rs create mode 100644 src/compiler/compile.rs create mode 100644 src/compiler/lexer.rs create mode 100644 src/compiler/mod.rs create mode 100644 src/compiler/parser.rs create mode 100644 src/compiler/pyobject.rs create mode 100644 src/compiler/python.lalrpop create mode 100644 src/compiler/token.rs create mode 100644 src/compiler/vm.rs create mode 100644 src/main.rs diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..0cb9b894e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ + +language: rust + +rust: + - stable + - beta diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..b4de83100 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,740 @@ +[[package]] +name = "RustPython" +version = "0.0.1" +dependencies = [ + "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayref" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ascii-canvas" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-set" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit-vec" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "block-buffer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byte-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "debug_unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "diff" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "digest" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "docopt" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "either" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ena" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "env_logger" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fixedbitset" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lalrpop" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "petgraph 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lalrpop-snap" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "petgraph 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lalrpop-util" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.41" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ordermap" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "petgraph" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_shared" +version = "0.7.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex-syntax" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "siphasher" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "string_cache" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "string_cache_codegen" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "string_cache_shared" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strsim" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strsim" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "term" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wincolor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" +"checksum ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b385d69402821a1c254533a011a312531cbcc0e3e24f19bbb4747a5a2daf37e2" +"checksum atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0fd4c0631f06448cc45a6bbb3b710ebb7ff8ccb96a0800c994afe23a70d5df2" +"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" +"checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" +"checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" +"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" +"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" +"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" +"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3" +"checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" +"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" +"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" +"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" +"checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5" +"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" +"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" +"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf" +"checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f" +"checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c" +"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)" = "ac8ebf8343a981e2fa97042b14768f02ed3e1d602eac06cae6166df3c8ced206" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" +"checksum petgraph 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "8b30dc85588cd02b9b76f5e386535db546d21dc68506cff2abebee0b6445e8e4" +"checksum phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "05a079dd052e7b674d21cb31cbb6c05efd56a2cd2827db7692e2f1a507ebd998" +"checksum phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2261d544c2bb6aa3b10022b0be371b9c7c64f762ef28c6f5d4f1ef6d97b5930" +"checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +"checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" +"checksum proc-macro2 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1fa93823f53cfd0f5ac117b189aed6cfdfb2cfc0a9d82e956dd7927595ed7d46" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +"checksum quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e44651a0dc4cdd99f71c83b561e221f714912d11af1a4dff0631f923d53af035" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" +"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" +"checksum regex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75ecf88252dce580404a22444fc7d626c01815debba56a7f4f536772a5ff19d3" +"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" +"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" +"checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b" +"checksum serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)" = "fba5be06346c5200249c8c8ca4ccba4a09e8747c71c16e420bd359a0db4d8f91" +"checksum serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)" = "79e4620ba6fbe051fc7506fab6f84205823564d55da18d55b695160fb3479cd8" +"checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" +"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" +"checksum string_cache 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2bc8ae5fbdcafd8573e2bfc14acb54720e27bbf102376ed05d6052f7eb238b27" +"checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191" +"checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" +"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" +"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6dfd71b2be5a58ee30a6f8ea355ba8290d397131c00dfa55c3d34e6e13db5101" +"checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" +"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..e647fa7af --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ + +[package] +name = "RustPython" +version = "0.0.1" +authors = [ "Shing Lyu", "Windel Bouwman" ] +build = "build.rs" + +[build-dependencies] +lalrpop="0.15.1" + +[dependencies] +clap="2.20.0" +lalrpop-util="0.15.1" +regex="0.2.2" +log="0.4.1" +env_logger="0.5.10" + diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..23c7d3f80 --- /dev/null +++ b/build.rs @@ -0,0 +1,5 @@ +extern crate lalrpop; + +fn main() { + lalrpop::process_root().unwrap(); +} diff --git a/src/compiler/ast.rs b/src/compiler/ast.rs new file mode 100644 index 000000000..22999ad3e --- /dev/null +++ b/src/compiler/ast.rs @@ -0,0 +1,123 @@ +/* + * Implement abstract syntax tree nodes for the python language. + */ + +/* +#[derive(Debug)] +pub struct Location { + pub row: i32, + pub column: i32, +} + +#[derive(Debug)] +pub struct Node { + pub location: Location, +} +*/ + +#[derive(Debug, PartialEq)] +pub struct Program { + pub statements: Vec, +} + +#[derive(Debug, PartialEq)] +pub enum Statement { + Break, + Continue, + Return { + value: Option>, + }, + Import { + name: String, + }, + Pass, + Assert { + test: Expression, + msg: Option, + }, + Delete { + targets: Vec, + }, + Assign { + targets: Vec, + value: Expression, + }, + Expression { + expression: Expression, + }, + If { + test: Expression, + body: Vec, + }, + While { + test: Expression, + body: Vec, + }, + With { + items: Expression, + body: Vec, + }, + For { + target: Vec, + iter: Vec, + body: Vec, + or_else: Option>, + }, + ClassDef { + name: String, + // TODO: docstring: String, + }, + FunctionDef { + name: String, + // docstring: String, + body: Vec, + }, +} + +#[derive(Debug, PartialEq)] +pub enum Expression { + Binop { + a: Box, + op: Operator, + b: Box, + }, + Call { + function: Box, + args: Vec, + }, + Number { + value: i32, + }, + List { + elements: Vec, + }, + Tuple { + elements: Vec, + }, + String { + value: String, + }, + Identifier { + name: String, + }, + True, + False, + None, +} + +#[derive(Debug, PartialEq)] +pub enum Operator { + Add, + Sub, + Mult, + MatMult, + Div, + Mod, + Pow, + LShift, + RShift, + BitOr, + BitXor, + BitAnd, + FloorDiv, +} diff --git a/src/compiler/builtins.rs b/src/compiler/builtins.rs new file mode 100644 index 000000000..7230ffaf8 --- /dev/null +++ b/src/compiler/builtins.rs @@ -0,0 +1,28 @@ +/* + * Some functions are built into the interpreter, for example the print + * function is such a builtin function. + * + * Inspiration can be found here: + * https://github.com/python/cpython/blob/master/Python/bltinmodule.c + */ + +use super::pyobject::PyObjectRef; +use std::io::{self, Write}; + +pub fn fill_scope() { + // scope[String::from("print")] = print; +} + +pub fn print(args: Vec) { + // println!("Woot: {:?}", args); + trace!("print called with {:?}", args); + for a in args { + print!("{} ", a.borrow_mut().str()); + } + println!(); + io::stdout().flush().unwrap(); +} + +fn any() {} + +fn all() {} diff --git a/src/compiler/bytecode.rs b/src/compiler/bytecode.rs new file mode 100644 index 000000000..eb6b16141 --- /dev/null +++ b/src/compiler/bytecode.rs @@ -0,0 +1,92 @@ +/* + * Implement python as a virtual machine with bytecodes. + */ + +/* +let load_const_string = 0x16; +let call_function = 0x64; +*/ + +/* + * Primitive instruction type, which can be encoded and decoded. + */ +use std::collections::HashMap; + +#[derive(Debug)] +pub struct CodeObject { + pub instructions: Vec, + pub label_map: HashMap, +} + +impl CodeObject { + pub fn new() -> CodeObject { + CodeObject { + instructions: Vec::new(), + label_map: HashMap::new(), + } + } +} + +pub type Label = usize; + +#[derive(Debug)] +pub enum Instruction { + LoadName { name: String }, + StoreName { name: String }, + LoadConst { value: Constant }, + UnaryOperation { op: UnaryOperator }, + BinaryOperation { op: BinaryOperator }, + Pop, + GetIter, + Pass, + Continue, + Break, + Jump { target: Label }, + JumpIf { target: Label }, + CallFunction { count: usize }, + ForIter, + ReturnValue, + PushBlock { start: Label, end: Label }, + PopBlock, + BuildTuple { size: usize }, + BuildList { size: usize }, + BuildMap { size: usize }, +} + +#[derive(Debug)] +pub enum Constant { + Integer { value: i32 }, + // TODO: Float { value: f64 }, + String { value: String }, +} + +#[derive(Debug)] +pub enum BinaryOperator { + Power, + Multiply, + MatrixMultiply, + Divide, + FloorDivide, + Modulo, + Add, + Subtract, + Lshift, + Rshift, + And, + Xor, + Or, +} + +#[derive(Debug)] +pub enum UnaryOperator { + Not, + Minus, +} + +/* +Maintain a stack of blocks on the VM. +pub enum BlockType { + Loop, + Except, +} +*/ diff --git a/src/compiler/compile.rs b/src/compiler/compile.rs new file mode 100644 index 000000000..d02013c69 --- /dev/null +++ b/src/compiler/compile.rs @@ -0,0 +1,273 @@ +/* + * Take an AST and transform it into bytecode + */ + +use super::ast; +use super::bytecode::{self, CodeObject, Instruction}; + +struct Compiler { + code_object: CodeObject, + nxt_label: usize, +} + +pub fn compile(p: ast::Program) -> CodeObject { + let mut compiler = Compiler::new(); + compiler.compile_program(p); + compiler.code_object +} + +type Label = usize; + +impl Compiler { + fn new() -> Self { + Compiler { + code_object: CodeObject::new(), + nxt_label: 0, + } + } + + fn compile_program(&mut self, program: ast::Program) { + self.compile_statements(program.statements); + } + + fn compile_statements(&mut self, statements: Vec) { + for statement in statements { + self.compile_statement(statement) + } + } + + // Generate a new label + fn new_label(&mut self) -> Label { + let l = self.nxt_label; + self.nxt_label += 1; + l + } + + // Assign current position the given label + fn set_label(&mut self, label: Label) { + let position = self.code_object.instructions.len(); + // assert!(label not in self.label_map) + self.code_object.label_map.insert(label, position); + } + + fn compile_statement(&mut self, statement: ast::Statement) { + trace!("Compiling {:?}", statement); + match statement { + ast::Statement::Import { name } => {} + ast::Statement::Expression { expression } => { + self.compile_expression(expression); + + // Pop result of stack, since we not use it: + self.emit(Instruction::Pop); + } + ast::Statement::If { test, body } => { + self.compile_expression(test); + let else_label = self.new_label(); + self.emit(Instruction::JumpIf { target: else_label }); + self.compile_statements(body); + self.set_label(else_label); + } + ast::Statement::While { test, body } => { + let start_label = self.new_label(); + let end_label = self.new_label(); + self.set_label(start_label); + + self.compile_expression(test); + self.emit(Instruction::UnaryOperation { op: bytecode::UnaryOperator::Not }); + self.emit(Instruction::JumpIf { target: end_label }); + self.compile_statements(body); + self.emit(Instruction::Jump { target: start_label }); + self.set_label(end_label); + } + ast::Statement::With { items, body } => { + // TODO + } + ast::Statement::For { + target, + iter, + body, + or_else, + } => { + // The thing iterated: + for i in iter { + self.compile_expression(i); + } + + // Retrieve iterator + self.emit(Instruction::GetIter); + + // Start loop + let start_label = self.new_label(); + let end_label = self.new_label(); + self.emit(Instruction::PushBlock { + start: start_label, + end: end_label, + }); + self.set_label(start_label); + self.emit(Instruction::ForIter); + + // Start of loop iteration, set targets: + for t in target { + match t { + ast::Expression::Identifier { name } => { + self.emit(Instruction::StoreName { name: name }); + } + _ => panic!("Not impl"), + } + } + + // Body of loop: + self.compile_statements(body); + self.set_label(end_label); + self.emit(Instruction::PopBlock); + } + ast::Statement::FunctionDef { name, body } => { + self.compile_statements(body); + } + ast::Statement::ClassDef { name } => { + // TODO? + } + ast::Statement::Assert { test, msg } => { + // TODO: if some flag, ignore all assert statements! + + self.compile_expression(test); + + // if true, jump over raise: + + self.emit(Instruction::LoadName { + name: String::from("AssertionError"), + }); + match msg { + Some(e) => { + self.compile_expression(e); + self.emit(Instruction::CallFunction { count: 1 }); + } + None => { + self.emit(Instruction::CallFunction { count: 0 }); + } + } + // TODO? + } + ast::Statement::Break => { + self.emit(Instruction::Break); + } + ast::Statement::Continue => { + self.emit(Instruction::Continue); + } + ast::Statement::Return { value } => { + match value { + Some(e) => { + let size = e.len(); + for v in e { + self.compile_expression(v); + } + + // If we have more than 1 return value, make it a tuple: + if size > 1 { + self.emit(Instruction::BuildTuple { size }); + } + } + None => { + // TODO: Put none on stack + } + } + + self.emit(Instruction::ReturnValue); + } + ast::Statement::Assign { targets, value } => { + self.compile_expression(value); + + for target in targets { + match target { + ast::Expression::Identifier { name } => { + self.emit(Instruction::StoreName { name: name }); + } + _ => { + panic!("WTF"); + } + } + } + } + ast::Statement::Delete { targets } => { + // Remove the given names from the scope + // self.emit(Instruction::DeleteName); + } + ast::Statement::Pass => { + self.emit(Instruction::Pass); + } + } + } + + fn compile_expression(&mut self, expression: ast::Expression) { + trace!("Compiling {:?}", expression); + match expression { + ast::Expression::Call { function, args } => { + self.compile_expression(*function); + let count = args.len(); + for arg in args { + self.compile_expression(arg) + } + self.emit(Instruction::CallFunction { count: count }); + } + ast::Expression::Binop { a, op, b } => { + self.compile_expression(*a); + self.compile_expression(*b); + + // Perform operation: + let i = match op { + ast::Operator::Add => bytecode::BinaryOperator::Add, + ast::Operator::Sub => bytecode::BinaryOperator::Subtract, + ast::Operator::Mult => bytecode::BinaryOperator::Multiply, + ast::Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply, + ast::Operator::Div => bytecode::BinaryOperator::Divide, + ast::Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide, + ast::Operator::Mod => bytecode::BinaryOperator::Modulo, + ast::Operator::Pow => bytecode::BinaryOperator::Power, + ast::Operator::LShift => bytecode::BinaryOperator::Lshift, + ast::Operator::RShift => bytecode::BinaryOperator::Rshift, + ast::Operator::BitOr => bytecode::BinaryOperator::Or, + ast::Operator::BitXor => bytecode::BinaryOperator::Xor, + ast::Operator::BitAnd => bytecode::BinaryOperator::And, + }; + let i = Instruction::BinaryOperation { op: i }; + self.emit(i); + } + ast::Expression::Number { value } => { + self.emit(Instruction::LoadConst { value: bytecode::Constant::Integer { value: value } }); + } + ast::Expression::List { elements } => { + let size = elements.len(); + for element in elements { + self.compile_expression(element); + } + self.emit(Instruction::BuildList { size: size }); + } + ast::Expression::Tuple { elements } => { + let size = elements.len(); + for element in elements { + self.compile_expression(element); + } + self.emit(Instruction::BuildTuple { size: size }); + } + ast::Expression::True => { + self.emit(Instruction::LoadConst { value: bytecode::Constant::Integer { value: 1 } }); + } + ast::Expression::False => { + self.emit(Instruction::LoadConst { value: bytecode::Constant::Integer { value: 0 } }); + } + ast::Expression::None => { + self.emit(Instruction::LoadConst { value: bytecode::Constant::Integer { value: 0 } }); + } + ast::Expression::String { value } => { + self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: value } }); + } + ast::Expression::Identifier { name } => { + self.emit(Instruction::LoadName { name }); + } + } + } + + fn emit(&mut self, instruction: Instruction) { + self.code_object.instructions.push(instruction); + } +} diff --git a/src/compiler/lexer.rs b/src/compiler/lexer.rs new file mode 100644 index 000000000..4522aead5 --- /dev/null +++ b/src/compiler/lexer.rs @@ -0,0 +1,600 @@ +use std::str::FromStr; +use std::str::CharIndices; +pub use super::token::Tok; +use std::iter::FromIterator; +use std::collections::HashMap; + +pub struct Lexer<'input> { + chars: CharIndices<'input>, + at_begin_of_line: bool, + nesting: usize, // Amount of parenthesis + indentation_stack: Vec, + chr0: Option, + chr1: Option, + location: usize, +} + +#[derive(Debug)] +pub enum LexicalError { + StringError, +} + +pub type Spanned = Result<(usize, Tok, usize), LexicalError>; + +pub fn lex_source(source: &String) -> Vec { + let lexer = Lexer::new(source); + Vec::from_iter(lexer.map(|x| x.unwrap().1)) +} + +impl<'input> Lexer<'input> { + pub fn new(input: &'input str) -> Self { + let mut lxr = Lexer { + chars: input.char_indices(), + at_begin_of_line: true, + nesting: 0, + indentation_stack: vec![0], + chr0: None, + location: 0, + chr1: None, + }; + lxr.next_char(); + lxr.next_char(); + lxr + } + + // Lexer helper functions: + fn lex_identifier(&mut self) -> Spanned { + let mut name = String::new(); + let start_pos = self.location; + while self.is_char() { + name.push(self.next_char().unwrap()); + } + let end_pos = self.location; + + let mut keywords: HashMap = HashMap::new(); + + // Alphabetical keywords: + keywords.insert(String::from("False"), Tok::False); + keywords.insert(String::from("None"), Tok::None); + keywords.insert(String::from("True"), Tok::True); + + keywords.insert(String::from("and"), Tok::And); + keywords.insert(String::from("as"), Tok::As); + keywords.insert(String::from("assert"), Tok::Assert); + keywords.insert(String::from("break"), Tok::Break); + keywords.insert(String::from("class"), Tok::Class); + keywords.insert(String::from("continue"), Tok::Continue); + keywords.insert(String::from("def"), Tok::Def); + keywords.insert(String::from("del"), Tok::Del); + keywords.insert(String::from("elif"), Tok::Elif); + keywords.insert(String::from("else"), Tok::Else); + keywords.insert(String::from("except"), Tok::Except); + keywords.insert(String::from("finally"), Tok::Finally); + keywords.insert(String::from("for"), Tok::For); + keywords.insert(String::from("from"), Tok::From); + keywords.insert(String::from("global"), Tok::Global); + keywords.insert(String::from("if"), Tok::If); + keywords.insert(String::from("import"), Tok::Import); + keywords.insert(String::from("in"), Tok::In); + keywords.insert(String::from("is"), Tok::Is); + keywords.insert(String::from("lambda"), Tok::Lambda); + keywords.insert(String::from("nonlocal"), Tok::Nonlocal); + keywords.insert(String::from("not"), Tok::Not); + keywords.insert(String::from("or"), Tok::Or); + keywords.insert(String::from("pass"), Tok::Pass); + keywords.insert(String::from("raise"), Tok::Raise); + keywords.insert(String::from("return"), Tok::Return); + keywords.insert(String::from("try"), Tok::Try); + keywords.insert(String::from("while"), Tok::While); + keywords.insert(String::from("with"), Tok::With); + keywords.insert(String::from("yield"), Tok::Yield); + + if keywords.contains_key(&name) { + Ok((start_pos, keywords.remove(&name).unwrap(), end_pos)) + } else { + Ok((start_pos, Tok::Name { name: name }, end_pos)) + } + } + + fn lex_number(&mut self) -> Spanned { + let mut value_text = String::new(); + + let start_pos = self.location; + while self.is_number() { + value_text.push(self.next_char().unwrap()); + } + let end_pos = self.location; + + let value = i32::from_str(&value_text).unwrap(); + + return Ok((start_pos, Tok::Number { value: value }, end_pos)); + } + + fn lex_comment(&mut self) { + // Skip everything until end of line + self.next_char(); + loop { + match self.next_char() { + Some(c) => { + if c == '\n' { + return; + } + } + None => return, + } + } + } + + fn lex_string(&mut self) -> Spanned { + let quote_char = self.next_char().unwrap(); + let mut string_content = String::new(); + let start_pos = self.location; + + loop { + match self.next_char() { + Some(c) => { + if c == quote_char { + break; + } else { + string_content.push(c); + } + } + None => { + return Err(LexicalError::StringError); + } + } + } + let end_pos = self.location; + + return Ok(( + start_pos, + Tok::String { + value: string_content, + }, + end_pos, + )); + } + + fn is_char(&self) -> bool { + match self.chr0 { + Some('a'...'z') => return true, + _ => return false, + } + } + + fn is_number(&self) -> bool { + match self.chr0 { + Some('0'...'9') => return true, + _ => return false, + } + } + + fn next_char(&mut self) -> Option { + let c = self.chr0; + let nxt = self.chars.next(); + self.chr0 = self.chr1; + self.chr1 = nxt.map(|x| x.1); + self.location = match nxt { + Some(p) => p.0, + None => 99999, + }; + c + } +} + +/* Implement iterator pattern for the get_tok function. + +Calling the next element in the iterator will yield the next lexical +token. +*/ +impl<'input> Iterator for Lexer<'input> { + type Item = Spanned; + + fn next(&mut self) -> Option { + // Idea: create some sort of hash map for single char tokens: + // let mut X = HashMap::new(); + // X.insert('=', Tok::Equal); + + // Detect indentation levels + loop { + if self.at_begin_of_line { + self.at_begin_of_line = false; + + // Determine indentation: + let mut col: usize = 0; + loop { + match self.chr0 { + Some(' ') => { + self.next_char(); + col += 1; + } + _ => { + break; + } + } + } + + if self.nesting == 0 { + // Determine indent or dedent: + let current_indentation = *self.indentation_stack.last().unwrap(); + if col == current_indentation { + // Same same + } else if col > current_indentation { + // New indentation level: + self.indentation_stack.push(col); + return Some(Ok((0, Tok::Indent, 0))); + } else if col < current_indentation { + // One or more dedentations + return Some(Ok((0, Tok::Dedent, 0))); + } + } + } + + match self.chr0 { + Some('0'...'9') => return Some(self.lex_number()), + // TODO: 'A'...'Z' + Some('a'...'z') => return Some(self.lex_identifier()), + Some('#') => { + self.lex_comment(); + continue; + } + Some('"') => { + return Some(self.lex_string()); + } + Some('\'') => { + return Some(self.lex_string()); + } + Some('=') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((self.location, Tok::EqEqual, self.location + 1))); + } + _ => return Some(Ok((self.location, Tok::Equal, self.location + 1))), + } + } + Some('+') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((self.location, Tok::PlusEqual, self.location + 1))); + } + _ => return Some(Ok((self.location, Tok::Plus, self.location + 1))), + } + } + Some('*') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::StarEqual, self.location + 1))); + } + Some('*') => { + self.next_char(); + return Some(Ok((tok_start, Tok::DoubleStar, self.location + 1))); + } + _ => return Some(Ok((tok_start, Tok::Star, self.location + 1))), + } + } + Some('/') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::SlashEqual, self.location + 1))); + } + Some('/') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok(( + tok_start, + Tok::DoubleSlashEqual, + self.location + 1, + ))); + } + _ => { + return Some(Ok(( + tok_start, + Tok::DoubleSlash, + self.location + 1, + ))) + } + } + } + _ => return Some(Ok((tok_start, Tok::Slash, self.location + 1))), + } + } + Some('%') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((self.location, Tok::PercentEqual, self.location + 1))); + } + _ => return Some(Ok((self.location, Tok::Percent, self.location + 1))), + } + } + Some('|') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((self.location, Tok::VbarEqual, self.location + 1))); + } + _ => return Some(Ok((self.location, Tok::Vbar, self.location + 1))), + } + } + Some('^') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok(( + self.location, + Tok::CircumflexEqual, + self.location + 1, + ))); + } + _ => return Some(Ok((self.location, Tok::CircumFlex, self.location + 1))), + } + } + Some('&') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((self.location, Tok::AmperEqual, self.location + 1))); + } + _ => return Some(Ok((self.location, Tok::Amper, self.location + 1))), + } + } + Some('-') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::MinusEqual, self.location + 1))); + } + _ => return Some(Ok((tok_start, Tok::Minus, self.location + 1))), + } + } + Some('@') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::AtEqual, self.location + 1))); + } + _ => return Some(Ok((tok_start, Tok::At, self.location + 1))), + } + } + Some('~') => { + self.next_char(); + return Some(Ok((0, Tok::Tilde, 0))); + } + Some('(') => { + self.next_char(); + self.nesting += 1; + return Some(Ok((0, Tok::Lpar, 0))); + } + Some(')') => { + self.next_char(); + self.nesting -= 1; + return Some(Ok((0, Tok::Rpar, 0))); + } + Some('[') => { + self.next_char(); + self.nesting += 1; + return Some(Ok((0, Tok::Lsqb, 0))); + } + Some(']') => { + self.next_char(); + self.nesting -= 1; + return Some(Ok((self.location, Tok::Rsqb, self.location + 1))); + } + Some('{') => { + self.next_char(); + self.nesting += 1; + return Some(Ok((0, Tok::Lbrace, 0))); + } + Some('}') => { + self.next_char(); + self.nesting -= 1; + return Some(Ok((self.location, Tok::Rbrace, self.location + 1))); + } + Some(':') => { + self.next_char(); + return Some(Ok((self.location, Tok::Colon, self.location + 1))); + } + Some(';') => { + self.next_char(); + return Some(Ok((self.location, Tok::Semi, self.location + 1))); + } + Some('<') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('<') => { + self.next_char(); + match self.chr0 { + Some('=') => { + return Some(Ok(( + tok_start, + Tok::LeftShiftEqual, + self.location + 1, + ))) + } + _ => { + return Some(Ok((tok_start, Tok::LeftShift, self.location + 1))) + } + } + } + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::LessEqual, self.location + 1))); + } + _ => return Some(Ok((tok_start, Tok::Less, self.location + 1))), + } + } + Some('>') => { + let tok_start = self.location; + self.next_char(); + match self.chr0 { + Some('>') => { + self.next_char(); + match self.chr0 { + Some('=') => { + return Some(Ok(( + tok_start, + Tok::RightShiftEqual, + self.location + 1, + ))) + } + _ => { + return Some(Ok((tok_start, Tok::RightShift, self.location + 1))) + } + } + } + Some('=') => { + self.next_char(); + return Some(Ok((tok_start, Tok::GreaterEqual, self.location + 1))); + } + _ => return Some(Ok((tok_start, Tok::Greater, self.location + 1))), + } + } + Some(',') => { + self.next_char(); + return Some(Ok((self.location, Tok::Comma, self.location + 1))); + } + Some('.') => { + self.next_char(); + return Some(Ok((self.location, Tok::Dot, self.location + 1))); + } + Some('\n') => { + self.next_char(); + + // Depending on the nesting level, we emit newline or not: + if self.nesting == 0 { + self.at_begin_of_line = true; + return Some(Ok((self.location, Tok::Newline, self.location + 1))); + } else { + continue; + } + } + Some(' ') => { + // Skip whitespaces + self.next_char(); + continue; + } + None => return None, + _ => { + let c = self.next_char(); + panic!("Not impl {:?}", c) + } // Ignore all the rest.. + } + } + } +} + +#[cfg(test)] +mod tests { + use super::Tok; + use super::lex_source; + + #[test] + fn test_line_comment() { + let source = String::from(r"99232 # Bladibla"); + let tokens = lex_source(&source); + assert_eq!(tokens, vec![Tok::Number { value: 99232 }]); + } + + #[test] + fn test_assignment() { + let source = String::from(r"avariable = 99 + 2-0"); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::Name { + name: String::from("avariable"), + }, + Tok::Equal, + Tok::Number { value: 99 }, + Tok::Plus, + Tok::Number { value: 2 }, + Tok::Minus, + Tok::Number { value: 0 }, + ] + ); + } + + #[test] + fn test_indentation() { + let source = String::from("def foo():\n return 99\n"); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::Def, + Tok::Name { + name: String::from("foo"), + }, + Tok::Lpar, + Tok::Rpar, + Tok::Colon, + Tok::Newline, + Tok::Indent, + Tok::Return, + Tok::Number { value: 99 }, + Tok::Newline, + Tok::Dedent, + ] + ); + } + + #[test] + fn test_newline_in_brackets() { + let source = String::from("x = [\n 1,2\n]\n"); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::Name { + name: String::from("x"), + }, + Tok::Equal, + Tok::Lsqb, + Tok::Number { value: 1 }, + Tok::Comma, + Tok::Number { value: 2 }, + Tok::Rsqb, + Tok::Newline, + ] + ); + } + + #[test] + fn test_operators() { + let source = String::from("//////=/ /"); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::DoubleSlash, + Tok::DoubleSlash, + Tok::DoubleSlashEqual, + Tok::Slash, + Tok::Slash, + ] + ); + } + +} diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs new file mode 100644 index 000000000..f14365bf1 --- /dev/null +++ b/src/compiler/mod.rs @@ -0,0 +1,22 @@ +// This file makes this directory a submodule. + +mod parser; +mod python; +mod ast; +mod token; +mod lexer; +mod compile; +mod bytecode; +mod builtins; +mod pyobject; +mod vm; + +pub use self::parser::parse; +pub use self::compile::compile; +pub use self::vm::evaluate; + + +// Mimic eval code objects: +//pub fn eval_code() -> pyobject::PyObjectRef { + +//} \ No newline at end of file diff --git a/src/compiler/parser.rs b/src/compiler/parser.rs new file mode 100644 index 000000000..e4a1f1f9e --- /dev/null +++ b/src/compiler/parser.rs @@ -0,0 +1,104 @@ +use std::error::Error; +use std::path::Path; +use std::fs::File; +use std::io::Read; + +use super::python; +use super::ast; +use super::lexer; + +fn read_file(filename: &Path) -> Result { + match File::open(&filename) { + Ok(mut file) => { + let mut s = String::new(); + + match file.read_to_string(&mut s) { + Err(why) => Err(String::from("Reading file failed: ") + why.description()), + Ok(_) => Ok(s), + } + } + Err(why) => Err(String::from("Opening file failed: ") + why.description()), + } +} + +/* + * Parse python code. + * Grammar may be inspired by antlr grammar for python: + * https://github.com/antlr/grammars-v4/tree/master/python3 + */ + +pub fn parse(filename: &Path) -> Result { + info!("Parsing: {}", filename.display()); + match read_file(filename) { + Ok(txt) => { + debug!("Read contents of file: {}", txt); + parse_source(&txt) + } + Err(msg) => Err(msg), + } +} + +pub fn parse_source(source: &String) -> Result { + let lxr = lexer::Lexer::new(&source); + match python::ProgramParser::new().parse(lxr) { + Err(why) => Err(String::from(format!("{:?}", why))), + Ok(p) => Ok(p), + } +} + +#[cfg(test)] +mod tests { + use super::parse_source; + use super::ast; + + #[test] + fn test_parse_print_hello() { + let source = String::from("print('Hello world')\n"); + let parse_ast = parse_source(&source).unwrap(); + assert_eq!( + parse_ast, + ast::Program { + statements: vec![ + ast::Statement::Expression { + expression: ast::Expression::Call { + function: Box::new(ast::Expression::Identifier { + name: String::from("print"), + }), + args: vec![ + ast::Expression::String { + value: String::from("Hello world"), + }, + ], + }, + }, + ], + } + ); + } + + #[test] + fn test_parse_print_2() { + let source = String::from("print('Hello world', 2)\n"); + let parse_ast = parse_source(&source).unwrap(); + assert_eq!( + parse_ast, + ast::Program { + statements: vec![ + ast::Statement::Expression { + expression: ast::Expression::Call { + function: Box::new(ast::Expression::Identifier { + name: String::from("print"), + }), + args: vec![ + ast::Expression::String { + value: String::from("Hello world"), + }, + ast::Expression::Number { value: 2 }, + ], + }, + }, + ], + } + ); + } +} diff --git a/src/compiler/pyobject.rs b/src/compiler/pyobject.rs new file mode 100644 index 000000000..ce491b899 --- /dev/null +++ b/src/compiler/pyobject.rs @@ -0,0 +1,241 @@ +use std::rc::Rc; +use std::cell::RefCell; +use std::ops::{Add, Mul, Sub}; + +/* Python objects and references. + +Okay, so each python object itself is an class itself (PyObject). Each +python object can have several references to it (PyObjectRef). These +references are Rc (reference counting) rust smart pointers. So when +all references are destroyed, the object itself also can be cleaned up. +Basically reference counting, but then done by rust. + +*/ + +/* +The PyRef type implements +https://doc.rust-lang.org/std/cell/index.html#introducing-mutability-inside-of-something-immutable +*/ +pub type PyRef = Rc>; +pub type PyObjectRef = PyRef; + +#[derive(Debug)] +pub enum PyObject { + String { + value: String, + }, + Integer { + value: i32, + }, + List { + elements: Vec, + }, + Tuple { + elements: Vec, + }, + Dict, + Iterator { + position: usize, + iterated_obj: PyObjectRef, + }, + None, + RustFunction { + function: fn(Vec), + }, +} + +/* +impl PyObjectRef { + pub fn steal(&self) -> &mut PyObject { + self.borrow_mut() + } +}*/ + +impl PyObject { + pub fn call(&self, args: Vec) { + match *self { + PyObject::RustFunction { ref function } => { + function(args); + } + _ => { + println!("Not impl {:?}", self); + panic!("Not impl"); + } + } + } + + pub fn str(&self) -> String { + match *self { + PyObject::String { ref value } => value.clone(), + PyObject::Integer { ref value } => format!("{:?}", value), + PyObject::List { ref elements } => format!("{:?}", elements), + PyObject::Tuple { ref elements } => format!("{:?}", elements), + PyObject::None => String::from("None"), + _ => { + println!("Not impl {:?}", self); + panic!("Not impl"); + } + } + } + + // Implement iterator protocol: + pub fn nxt(&mut self) -> Option { + match *self { + PyObject::Iterator { + ref mut position, + iterated_obj: ref iterated_obj_ref, + } => { + let iterated_obj = &*iterated_obj_ref.borrow_mut(); + match iterated_obj { + &PyObject::List { ref elements } => { + if *position < elements.len() { + let obj_ref = elements[*position].clone(); + *position += 1; + Some(obj_ref) + } else { + None + } + } + _ => { + panic!("NOT IMPL"); + } + } + } + _ => { + panic!("NOT IMPL"); + } + } + } + + // Move this object into a reference object, transferring ownership. + pub fn into_ref(self) -> PyObjectRef { + Rc::new(RefCell::new(self)) + } +} + +impl<'a> Add<&'a PyObject> for &'a PyObject { + type Output = PyObject; + + fn add(self, rhs: &'a PyObject) -> Self::Output { + match self { + &PyObject::Integer { ref value } => { + let value1 = value; + match rhs { + &PyObject::Integer { ref value } => { + let value2 = value; + PyObject::Integer { + value: value1 + value2, + } + } + _ => { + panic!("NOT IMPL"); + } + } + } + _ => { + // TODO: Lookup __add__ method in dictionary? + panic!("NOT IMPL"); + } + } + } +} + +impl<'a> Sub<&'a PyObject> for &'a PyObject { + type Output = PyObject; + + fn sub(self, rhs: &'a PyObject) -> Self::Output { + match self { + &PyObject::Integer { value } => { + let value1 = value; + match rhs { + &PyObject::Integer { value } => { + let value2 = value; + PyObject::Integer { + value: value1 - value2, + } + } + _ => { + panic!("NOT IMPL"); + } + } + } + _ => { + panic!("NOT IMPL"); + } + } + } +} + +impl<'a> Mul<&'a PyObject> for &'a PyObject { + type Output = PyObject; + + fn mul(self, rhs: &'a PyObject) -> Self::Output { + match self { + &PyObject::Integer { value } => { + let value1 = value; + match rhs { + &PyObject::Integer { value } => { + let value2 = value; + PyObject::Integer { + value: value1 * value2, + } + } + _ => { + panic!("NOT IMPL"); + } + } + } + &PyObject::String { ref value } => { + let value1 = value; + match rhs { + &PyObject::Integer { value } => { + let value2 = value; + let mut result = String::new(); + for _x in 0..value2 { + result.push_str(value1.as_str()); + } + PyObject::String { value: result } + } + _ => { + panic!("NOT IMPL"); + } + } + } + _ => { + panic!("NOT IMPL"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::PyObject; + + #[test] + fn test_add_py_integers() { + let a = PyObject::Integer { value: 33 }; + let b = PyObject::Integer { value: 12 }; + let c = &a + &b; + match c { + PyObject::Integer { value } => assert_eq!(value, 45), + _ => assert!(false), + } + } + + #[test] + fn test_multiply_str() { + let a = PyObject::String { + value: String::from("Hello "), + }; + let b = PyObject::Integer { value: 4 }; + let c = &a * &b; + match c { + PyObject::String { value } => { + assert_eq!(value, String::from("Hello Hello Hello Hello ")) + } + _ => assert!(false), + } + } + +} diff --git a/src/compiler/python.lalrpop b/src/compiler/python.lalrpop new file mode 100644 index 000000000..00f03b1b9 --- /dev/null +++ b/src/compiler/python.lalrpop @@ -0,0 +1,261 @@ +// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar +use super::ast; +use super::lexer; +use std::iter::FromIterator; + +grammar; + +pub Program: ast::Program = { + => ast::Program { statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) }, +}; + +// A file line either has a declaration, or an empty newline: +FileLine: Option = { + => Some(s), + "\n" => None, +}; + +Suite: Vec = { + => vec![s], + "\n" indent dedent => s, +}; + +Statement: ast::Statement = { + SimpleStatement, + CompoundStatement, +}; + +SimpleStatement: ast::Statement = { + "\n" => s, +}; + +SmallStatement: ast::Statement = { + // => ast::Statement::Expression { expression: e }, + ExpressionStatement, + "pass" => ast::Statement::Pass, + FlowStatement, + ImportStatement, + AssertStatement, +}; + +ExpressionStatement: ast::Statement = { + => { + //match e2 { + // None => ast::Statement::Expression { expression: e }, + // Some(e3) => ast::Statement::Expression { expression: e }, + //} + if e2.len() > 0 { + // Dealing with assignment here + // TODO: for rhs in e2 { + let rhs = e2.into_iter().next().unwrap(); + // ast::Expression::Tuple { elements: e2.into_iter().next().unwrap() + let v = rhs.into_iter().next().unwrap(); + let lhs = ast::Statement::Assign { targets: e, value: v }; + lhs + } else { + if e.len() > 1 { + panic!("Not good?"); + // ast::Statement::Expression { expression: e[0] } + } else { + ast::Statement::Expression { expression: e.into_iter().next().unwrap() } + } + } + }, +}; + +AssignSuffix: Vec = { + "=" => e, +}; + +FlowStatement: ast::Statement = { + "break" => ast::Statement::Break, + "continue" => ast::Statement::Continue, + "return" => ast::Statement::Return { value: t}, + // raise + // yield +}; + +ImportStatement: ast::Statement = { + "import" => ast::Statement::Import { name: n }, +}; + +DottedName: String = { + => n, +}; + +AssertStatement: ast::Statement = { + "assert" => ast::Statement::Assert { + test: t, + msg: match m { + Some(e) => Some(e.1), + None => None, + } + }, +}; + +CompoundStatement: ast::Statement = { + "if" ":" => ast::Statement::If { test: e, body: s }, + "while" ":" => ast::Statement::While { test: e, body: s }, + "for" "in" ":" => ast::Statement::For { target: e, iter: t, body: s, or_else: None }, + "def" "(" ")" ":" => ast::Statement::FunctionDef { name: i, body: s }, + WithStatement, +}; + +WithStatement: ast::Statement = { + "with" "as" ":" => ast::Statement::With { items: t, body: s }, +}; + +Test: ast::Expression = { + => e, +}; + +OrTest: ast::Expression = { + => e, +}; + +AndTest: ast::Expression = { + => e, +}; + +NotTest: ast::Expression = { + => e, +}; + +Expression: ast::Expression = { + "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, + => e, +}; + +XorExpression: ast::Expression = { + "^" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitXor, b: Box::new(e2) }, + => e, +}; + +AndExpression: ast::Expression = { + "&" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitAnd, b: Box::new(e2) }, + => e, +}; + +ArithmaticExpression: ast::Expression = { + => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) }, + Term, +}; + +AddOp: ast::Operator = { + "+" => ast::Operator::Add, + "-" => ast::Operator::Sub, +}; + +Term: ast::Expression = { + => ast::Expression::Binop { a: Box::new(a), op: op, b: Box::new(b) }, + Factor, +}; + +MulOp: ast::Operator = { + "*" => ast::Operator::Mult, + "/" => ast::Operator::Div, + "//" => ast::Operator::FloorDiv, + "%" => ast::Operator::Mod, + "@" => ast::Operator::MatMult, +}; + +Factor: ast::Expression = { + => e, +}; + +Power: ast::Expression = { + => e, +}; + +Atom: ast::Expression = { + "(" ")" => ast::Expression::Call { function: Box::new(f), args: a }, + => ast::Expression::String { value: s }, + => ast::Expression::Number { value: n }, + => ast::Expression::Identifier { name: i }, + "[" "]" => ast::Expression::List { elements: e }, + "(" ")" => e, + "True" => ast::Expression::True, + "False" => ast::Expression::False, + "None" => ast::Expression::None, +}; + +ExpressionList: Vec = { + > => e, +}; + +TestList: Vec = { + => { + let mut l = vec![e1]; + l.extend(e2.into_iter().map(|x| x.1)); + l + } +}; + +FunctionArguments: Vec = { + > => e, +}; + +Comma: Vec = { + ",")*> => { + let mut items = items; + items.extend(last); + items + } +}; + +Number: i32 = => s; +String: String = { + => s, +}; +Identifier: String = => s; + +// Hook external lexer: +extern { + type Location = usize; + type Error = lexer::LexicalError; + + enum lexer::Tok { + indent => lexer::Tok::Indent, + dedent => lexer::Tok::Dedent, + "+" => lexer::Tok::Plus, + "-" => lexer::Tok::Minus, + ":" => lexer::Tok::Colon, + "," => lexer::Tok::Comma, + "*" => lexer::Tok::Star, + "&" => lexer::Tok::Amper, + "@" => lexer::Tok::At, + "%" => lexer::Tok::Percent, + "//" => lexer::Tok::DoubleSlash, + "^" => lexer::Tok::CircumFlex, + "|" => lexer::Tok::Vbar, + "/" => lexer::Tok::Slash, + "(" => lexer::Tok::Lpar, + ")" => lexer::Tok::Rpar, + "[" => lexer::Tok::Lsqb, + "]" => lexer::Tok::Rsqb, + "=" => lexer::Tok::Equal, + "+=" => lexer::Tok::PlusEqual, + "-=" => lexer::Tok::MinusEqual, + "assert" => lexer::Tok::Assert, + "import" => lexer::Tok::Import, + "break" => lexer::Tok::Break, + "continue" => lexer::Tok::Break, + "return" => lexer::Tok::Return, + "pass" => lexer::Tok::Pass, + "if" => lexer::Tok::If, + "while" => lexer::Tok::While, + "for" => lexer::Tok::For, + "in" => lexer::Tok::In, + "with" => lexer::Tok::With, + "as" => lexer::Tok::As, + "def" => lexer::Tok::Def, + "class" => lexer::Tok::Class, + "True" => lexer::Tok::True, + "False" => lexer::Tok::False, + "None" => lexer::Tok::None, + number => lexer::Tok::Number { value: }, + string => lexer::Tok::String { value: }, + name => lexer::Tok::Name { name: }, + "\n" => lexer::Tok::Newline, + } +} diff --git a/src/compiler/token.rs b/src/compiler/token.rs new file mode 100644 index 000000000..85c70aeed --- /dev/null +++ b/src/compiler/token.rs @@ -0,0 +1,94 @@ + +// Loosely based on token.h from CPython source: +#[derive(Debug)] +#[derive(PartialEq)] +pub enum Tok { + Name { name: String }, + Number { value: i32 }, + String { value: String }, + Newline, + Indent, + Dedent, + Lpar, + Rpar, + Lsqb, + Rsqb, + Colon, + Comma, + Semi, + Plus, + Minus, + Star, + Slash, + Vbar, // '|' + Amper, // '&' + Less, + Greater, + Equal, + Dot, + Percent, + Lbrace, + Rbrace, + EqEqual, + NotEqual, + LessEqual, + GreaterEqual, + Tilde, + CircumFlex, + LeftShift, + RightShift, + DoubleStar, + PlusEqual, + MinusEqual, + StarEqual, + SlashEqual, + PercentEqual, + AmperEqual, // '&=' + VbarEqual, + CircumflexEqual, // '^=' + LeftShiftEqual, + RightShiftEqual, + DoubleSlash, // '//' + DoubleSlashEqual, + At, + AtEqual, + Rarrow, + Ellipses, + + // Keywords (alphabetically): + + False, + None, + True, + + And, + As, + Assert, + Break, + Class, + Continue, + Def, + Del, + Elif, + Else, + Except, + Finally, + For, + From, + Global, + If, + Import, + In, + Is, + Lambda, + Nonlocal, + Not, + Or, + Pass, + Raise, + Return, + Try, + While, + With, + Yield, +} diff --git a/src/compiler/vm.rs b/src/compiler/vm.rs new file mode 100644 index 000000000..c6c33e974 --- /dev/null +++ b/src/compiler/vm.rs @@ -0,0 +1,227 @@ + +/* + * Implement virtual machine to run instructions. + */ + +use std::collections::HashMap; +use std::cell::RefMut; +use std::ops::Deref; + +use super::bytecode; +use super::builtins; +use super::pyobject::{PyObject, PyObjectRef}; + +// use objects::objects; + +// Container of the virtual machine state: +pub fn evaluate(code: bytecode::CodeObject) { + let mut vm = VirtualMachine::new(); + + // Register built in function: + vm.scope.insert(String::from("print"), PyObject::RustFunction { function: builtins::print }.into_ref()); + + // { stack: Vec::new() }; + vm.run(code); +} + +// Objects are live when they are on stack, or referenced by a name (for now) + +struct VirtualMachine { + call_stack: Vec, + stack: Vec, + // A stack of block to keep track of continue break locations. + block_stack: Vec<(bytecode::Label, bytecode::Label)>, + scope: HashMap, + program_counter: usize, +} + +impl VirtualMachine { + fn new() -> VirtualMachine { + VirtualMachine { + call_stack: Vec::new(), + stack: Vec::new(), + block_stack: Vec::new(), + scope: HashMap::new(), + program_counter: 0, + } + } + + fn run(&mut self, code: bytecode::CodeObject) { + // Process instruction at the program counter until exception or finished. + let num_instructions = code.instructions.len(); + // self.call_stack.push(code); + + // let current_frame = self.call_stack.last().unwrap(); + let current_frame = code; + + loop { + // for instruction in code.instructions { + if self.program_counter > current_frame.instructions.len() { + break; + } + + let ref instruction = current_frame.instructions[self.program_counter]; + trace!("Executing instruction: {:?} (stacksize={:?}, pc={:?}/{:?}", instruction, self.stack.len(), self.program_counter, num_instructions); + self.program_counter += 1; + self.execute_instruction(&instruction); + } + + // TODO: Pop call stack? + } + + // Execute a single instruction: + fn execute_instruction(&mut self, instruction: &bytecode::Instruction) { + // trace!("Executing instruction: {:?} (stacksize={:?}", instruction, self.stack.len()); + match instruction { + &bytecode::Instruction::LoadConst { ref value } => { + let obj = match value { + &bytecode::Constant::Integer { ref value } => { PyObject::Integer { value: *value } }, + // &bytecode::Constant::Float + &bytecode::Constant::String { ref value } => { PyObject::String { value: value.clone() } }, + }; + self.stack.push(obj.into_ref().clone()); + }, + &bytecode::Instruction::LoadName { ref name } => { + // Lookup name in scope and put it onto the stack! + let obj = &self.scope[name]; + self.stack.push(obj.clone()); + }, + &bytecode::Instruction::StoreName { ref name } => { + // take top of stack and assign in scope: + let obj = self.stack.pop().unwrap(); + self.scope.insert(name.clone(), obj); + }, + &bytecode::Instruction::Pop => { + // Pop value from stack and ignore. + self.stack.pop(); + }, + &bytecode::Instruction::BuildList { size } => { + let elements = self.pop_multiple(size); + let list_obj = PyObject::List { elements: elements }.into_ref(); + self.stack.push(list_obj); + }, + &bytecode::Instruction::BuildTuple { size } => { + let elements = self.pop_multiple(size); + let list_obj = PyObject::Tuple { elements: elements }.into_ref(); + self.stack.push(list_obj); + }, + &bytecode::Instruction::BuildMap { size } => { + let mut elements = Vec::new(); + for _x in 0..size { + let key = self.stack.pop().unwrap(); + let obj = self.stack.pop().unwrap(); + elements.push((key,obj)); + } + panic!("To be implemented!") + //let list_obj = PyObject::Tuple { elements: elements }.into_ref(); + //self.stack.push(list_obj); + }, + &bytecode::Instruction::BinaryOperation { ref op } => { + self.execute_binop(op); + }, + &bytecode::Instruction::UnaryOperation { ref op } => { + panic!("TODO"); + // self.execute_binop(op); + }, + &bytecode::Instruction::ReturnValue => { + self.stack.pop(); + }, + &bytecode::Instruction::PushBlock { start, end } => { + self.block_stack.push((start, end)); + }, + &bytecode::Instruction::PopBlock => { + self.block_stack.pop(); + } + &bytecode::Instruction::GetIter => { + let iterated_obj = self.stack.pop().unwrap(); + let iter_obj = PyObject::Iterator { + position: 0, iterated_obj: iterated_obj + }.into_ref(); + self.stack.push(iter_obj); + }, + &bytecode::Instruction::ForIter => { + // The top of stack contains the iterator, lets push it forward: + let next_obj: Option = { + let top_of_stack = self.stack.last().unwrap(); + let mut ref_mut: RefMut = top_of_stack.deref().borrow_mut(); + // We require a mutable pyobject here to update the iterator: + let mut iterator = ref_mut; // &mut PyObject = ref_mut.; + // let () = iterator; + iterator.nxt() + }; + + // Check the next object: + match next_obj { + Some(v) => { + self.stack.push(v); + }, + None => { + // End of for loop + let end_label = self.block_stack.last().unwrap().1; + self.jump(end_label); + } + } + }, + &bytecode::Instruction::CallFunction { count } => { + let mut args: Vec = self.pop_multiple(count); + let func_ref = self.stack.pop().unwrap(); + let f=func_ref.borrow();// = &*func_ref.borrow(); + f.call(args); + // call_stack.push(); + // If a builtin function, then call directly, otherwise, execute it? + // execute(function.code); + }, + &bytecode::Instruction::Jump { target } => { + self.jump(target); + } + &bytecode::Instruction::JumpIf { target } => { + let obj = self.stack.pop().unwrap(); + // TODO: determine if this value is True-ish: + let result: bool = true; + if result { + self.jump(target); + } + } + &bytecode::Instruction::Break => { + let end_label = self.block_stack.last().unwrap().1; + self.jump(end_label); + }, + &bytecode::Instruction::Pass => { + // Ah, this is nice, just relax! + }, + &bytecode::Instruction::Continue => { + let start_label = self.block_stack.last().unwrap().0; + self.jump(start_label); + }, + } + } + + fn pop_multiple(&mut self, count: usize) -> Vec { + let mut objs: Vec = Vec::new(); + for _x in 0..count { + objs.push(self.stack.pop().unwrap()); + } + objs.reverse(); + objs + } + + fn jump(&mut self, label: bytecode::Label) { + // let current_frame = self.call_stack.last().unwrap(); + // self.program_counter = current_frame.label_map[&label]; + } + + fn execute_binop(&mut self, op: &bytecode::BinaryOperator) { + let b_ref = self.stack.pop().unwrap(); + let a_ref = self.stack.pop().unwrap(); + let b = &*b_ref.borrow(); + let a = &*a_ref.borrow(); + let result = match op { + &bytecode::BinaryOperator::Subtract => a - b, + &bytecode::BinaryOperator::Add => a + b, + &bytecode::BinaryOperator::Multiply => a * b, + // &bytecode::BinaryOperator::Div => a / b, + _ => panic!("NOT IMPL"), + }; + self.stack.push(result.into_ref()); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 000000000..9afac3f50 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,43 @@ + +extern crate clap; +#[macro_use] +extern crate log; +extern crate env_logger; +use clap::{Arg, App}; +use std::path::Path; +mod compiler; + + +fn main() { + env_logger::init(); + let matches = App::new("RsPython") + .version("0.0.1") + .author("Windel Bouwman") + .about("Rust implementation of the Python language") + .arg(Arg::with_name("script") + .required(true) + .index(1)) + .arg(Arg::with_name("v") + .short("v") + .multiple(true) + .help("Give the verbosity")) + .get_matches(); + + // Figure out the filename: + let script_file = matches.value_of("script").unwrap_or("foo"); + info!("Running file {}", script_file); + + // Parse an ast from it: + let filepath = Path::new(script_file); + match compiler::parse(filepath) { + Ok(program) => { + debug!("Got ast: {:?}", program); + let bytecode = compiler::compile(program); + debug!("Code object: {:?}", bytecode); + compiler::evaluate(bytecode); + info!("Great succes!!"); + }, + Err(msg) => error!("Parsing went horribly wrong: {}", msg), + } +} +