diff --git a/.github/ISSUE_TEMPLATE/report-incompatibility.md b/.github/ISSUE_TEMPLATE/report-incompatibility.md new file mode 100644 index 000000000..e917e9432 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/report-incompatibility.md @@ -0,0 +1,16 @@ +--- +name: Report incompatibility +about: Report an incompatibility between RustPython and CPython +title: '' +labels: feat +assignees: '' + +--- + +## Feature + + + +## Python Documentation + + diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md new file mode 100644 index 000000000..84a7b19ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -0,0 +1,24 @@ +--- +name: RFC +about: Make a suggestion in a Request for Comments format to RustPython +title: "[RFC] " +labels: RFC +assignees: '' + +--- + +## Summary + + + +## Detailed Explanation + + + +## Drawbacks, Rationale, and Alternatives + + + +## Unresolved Questions + + diff --git a/Cargo.lock b/Cargo.lock index 0303e06e9..9430a9f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,6 +78,16 @@ dependencies = [ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bincode" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bit-set" version = "0.5.1" @@ -124,6 +134,11 @@ dependencies = [ "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bumpalo" version = "2.3.0" @@ -195,6 +210,14 @@ dependencies = [ "python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "diff" version = "0.1.11" @@ -323,6 +346,31 @@ dependencies = [ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hexf" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hexf-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hexf-parse" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "humantime" version = "1.2.0" @@ -331,6 +379,11 @@ dependencies = [ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indexmap" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "itertools" version = "0.8.0" @@ -448,7 +501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -479,6 +532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -487,6 +541,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -553,6 +608,19 @@ name = "precomputed-hash" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro-hack" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.4.27" @@ -561,6 +629,15 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pwd" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "python3-sys" version = "0.2.1" @@ -575,6 +652,11 @@ name = "quick-error" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "quote" version = "0.6.11" @@ -795,7 +877,7 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", - "rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -818,6 +900,7 @@ dependencies = [ "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -826,9 +909,13 @@ dependencies = [ name = "rustpython_vm" version = "0.1.0" dependencies = [ + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -838,13 +925,13 @@ dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustpython_derive 0.1.0", "rustpython_parser 0.0.1", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", @@ -869,14 +956,14 @@ dependencies = [ [[package]] name = "rustyline" -version = "2.1.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -910,6 +997,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "serde" version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" @@ -1014,6 +1104,16 @@ name = "strsim" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "0.15.29" @@ -1024,6 +1124,14 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synstructure" version = "0.10.1" @@ -1147,6 +1255,11 @@ name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -1340,12 +1453,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49665c62e0e700857531fa5d3763e91b539ff1abeebd56808d378b495870d60d" "checksum block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d75255892aeb580d3c566f213a2b6fdc1c66667839f45719ee1d30ebf2aea591" +"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum bumpalo 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f6a6bc2ba935b1e2f810787ceba8dfe86cbc164cee55925cae715541186673c4" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" @@ -1356,6 +1471,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b489034e723e7f5109fecd19b719e664f89ef925be785885252469e9822fa940" +"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" "checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" @@ -1372,7 +1488,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e54653cc32d838771a36532647afad59c4bf7155745eeeec406f71fd5d7e7538" +"checksum hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "22eadcfadba76a730b2764eaa577d045f35e0ef5174b9c5b46adf1ee42b85e12" +"checksum hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79296f72d53a89096cbc9a88c9547ee8dfe793388674620e2207593d370550ac" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum js-sys 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "d4bda84b98977341bb6ba1086ecbd9eab8bc929de0f2923c118baf76c21dd0c8" @@ -1387,7 +1507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" "checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" "checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30" -"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" +"checksum nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46f0f3210768d796e8fa79ec70ee6af172dacbe7147f5e69be5240a47778302b" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" "checksum num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "57450397855d951f1a41305e54851b1a7b8f5d2e349543a02a2effe25459f718" @@ -1401,9 +1521,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" "checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +"checksum proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b773f824ff2a495833f85fcdddcf85e096949971decada2e93249fa2c6c3d32f" +"checksum proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f674ccc446da486175527473ec8aa064f980b0966bbf767ee743a5dff6244a7" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" +"checksum pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5dd32d8bece608e144ca20251e714ed107cdecdabb20c2d383cfc687825106a5" "checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" "checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" @@ -1427,7 +1551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rustc-demangle 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "adacaae16d02b6ec37fdc7acfcddf365978de76d1983d3ee22afc260e1ca9619" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6de8ecd7fad7731f306f69b6e10ec5a3178c61e464dcc06979427aa4cc891145" -"checksum rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6010155119d53aac4f5b987cb8f6ea913d0d64d9b237da36f8f96a90cb3f5385" +"checksum rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b" "checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" @@ -1446,7 +1570,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eea1eee654ef80933142157fdad9dd8bc43cf7c74e999e369263496f04ff4da" "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" @@ -1464,6 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" diff --git a/Cargo.toml b/Cargo.toml index ef2838a1c..71db1d839 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ env_logger="0.5.10" clap = "2.31.2" rustpython_parser = {path = "parser"} rustpython_vm = {path = "vm"} -rustyline = "2.1.0" +rustyline = "4.1.0" xdg = "2.2.0" [dev-dependencies.cpython] diff --git a/Lib/heapq.py b/Lib/heapq.py new file mode 100644 index 000000000..0e3555cf9 --- /dev/null +++ b/Lib/heapq.py @@ -0,0 +1,601 @@ +"""Heap queue algorithm (a.k.a. priority queue). + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +Usage: + +heap = [] # creates an empty heap +heappush(heap, item) # pushes a new item on the heap +item = heappop(heap) # pops the smallest item from the heap +item = heap[0] # smallest item on the heap without popping it +heapify(x) # transforms list into a heap, in-place, in linear time +item = heapreplace(heap, item) # pops and returns smallest item, and adds + # new item; the heap size is unchanged + +Our API differs from textbook heap algorithms as follows: + +- We use 0-based indexing. This makes the relationship between the + index for a node and the indexes for its children slightly less + obvious, but is more suitable since Python uses 0-based indexing. + +- Our heappop() method returns the smallest item, not the largest. + +These two make it possible to view the heap as a regular Python list +without surprises: heap[0] is the smallest item, and heap.sort() +maintains the heap invariant! +""" + +# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger + +__about__ = """Heap queues + +[explanation by François Pinard] + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +The strange invariant above is meant to be an efficient memory +representation for a tournament. The numbers below are `k', not a[k]: + + 0 + + 1 2 + + 3 4 5 6 + + 7 8 9 10 11 12 13 14 + + 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + +In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +a usual binary tournament we see in sports, each cell is the winner +over the two cells it tops, and we can trace the winner down the tree +to see all opponents s/he had. However, in many computer applications +of such tournaments, we do not need to trace the history of a winner. +To be more memory efficient, when a winner is promoted, we try to +replace it by something else at a lower level, and the rule becomes +that a cell and the two cells it tops contain three different items, +but the top cell "wins" over the two topped cells. + +If this heap invariant is protected at all time, index 0 is clearly +the overall winner. The simplest algorithmic way to remove it and +find the "next" winner is to move some loser (let's say cell 30 in the +diagram above) into the 0 position, and then percolate this new 0 down +the tree, exchanging values, until the invariant is re-established. +This is clearly logarithmic on the total number of items in the tree. +By iterating over all items, you get an O(n ln n) sort. + +A nice feature of this sort is that you can efficiently insert new +items while the sort is going on, provided that the inserted items are +not "better" than the last 0'th element you extracted. This is +especially useful in simulation contexts, where the tree holds all +incoming events, and the "win" condition means the smallest scheduled +time. When an event schedule other events for execution, they are +scheduled into the future, so they can easily go into the heap. So, a +heap is a good structure for implementing schedulers (this is what I +used for my MIDI sequencer :-). + +Various structures for implementing schedulers have been extensively +studied, and heaps are good for this, as they are reasonably speedy, +the speed is almost constant, and the worst case is not much different +than the average case. However, there are other representations which +are more efficient overall, yet the worst cases might be terrible. + +Heaps are also very useful in big disk sorts. You most probably all +know that a big sort implies producing "runs" (which are pre-sorted +sequences, which size is usually related to the amount of CPU memory), +followed by a merging passes for these runs, which merging is often +very cleverly organised[1]. It is very important that the initial +sort produces the longest runs possible. Tournaments are a good way +to that. If, using all the memory available to hold a tournament, you +replace and percolate items that happen to fit the current run, you'll +produce runs which are twice the size of the memory for random input, +and much better for input fuzzily ordered. + +Moreover, if you output the 0'th item on disk and get an input which +may not fit in the current tournament (because the value "wins" over +the last output value), it cannot fit in the heap, so the size of the +heap decreases. The freed memory could be cleverly reused immediately +for progressively building a second heap, which grows at exactly the +same rate the first heap is melting. When the first heap completely +vanishes, you switch heaps and start a new run. Clever and quite +effective! + +In a word, heaps are useful memory structures to know. I use them in +a few applications, and I think it is good to keep a `heap' module +around. :-) + +-------------------- +[1] The disk balancing algorithms which are current, nowadays, are +more annoying than clever, and this is a consequence of the seeking +capabilities of the disks. On devices which cannot seek, like big +tape drives, the story was quite different, and one had to be very +clever to ensure (far in advance) that each tape movement will be the +most effective possible (that is, will best participate at +"progressing" the merge). Some tapes were even able to read +backwards, and this was also used to avoid the rewinding time. +Believe me, real good tape sorts were quite spectacular to watch! +From all times, sorting has always been a Great Art! :-) +""" + +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', + 'nlargest', 'nsmallest', 'heappushpop'] + +def heappush(heap, item): + """Push item onto heap, maintaining the heap invariant.""" + heap.append(item) + _siftdown(heap, 0, len(heap)-1) + +def heappop(heap): + """Pop the smallest item off the heap, maintaining the heap invariant.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup(heap, 0) + return returnitem + return lastelt + +def heapreplace(heap, item): + """Pop and return the current smallest value, and add the new item. + + This is more efficient than heappop() followed by heappush(), and can be + more appropriate when using a fixed-size heap. Note that the value + returned may be larger than item! That constrains reasonable uses of + this routine unless written as part of a conditional replacement: + + if item > heap[0]: + item = heapreplace(heap, item) + """ + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup(heap, 0) + return returnitem + +def heappushpop(heap, item): + """Fast version of a heappush followed by a heappop.""" + if heap and heap[0] < item: + item, heap[0] = heap[0], item + _siftup(heap, 0) + return item + +def heapify(x): + """Transform list into a heap, in-place, in O(len(x)) time.""" + n = len(x) + # Transform bottom-up. The largest index there's any point to looking at + # is the largest with a child index in-range, so must have 2*i + 1 < n, + # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so + # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is + # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. + for i in reversed(range(n//2)): + _siftup(x, i) + +def _heappop_max(heap): + """Maxheap version of a heappop.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup_max(heap, 0) + return returnitem + return lastelt + +def _heapreplace_max(heap, item): + """Maxheap version of a heappop followed by a heappush.""" + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup_max(heap, 0) + return returnitem + +def _heapify_max(x): + """Transform list into a maxheap, in-place, in O(len(x)) time.""" + n = len(x) + for i in reversed(range(n//2)): + _siftup_max(x, i) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +def _siftdown(heap, startpos, pos): + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if newitem < parent: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +# The child indices of heap index pos are already heaps, and we want to make +# a heap at index pos too. We do this by bubbling the smaller child of +# pos up (and so on with that child's children, etc) until hitting a leaf, +# then using _siftdown to move the oddball originally at index pos into place. +# +# We *could* break out of the loop as soon as we find a pos where newitem <= +# both its children, but turns out that's not a good idea, and despite that +# many books write the algorithm that way. During a heap pop, the last array +# element is sifted in, and that tends to be large, so that comparing it +# against values starting from the root usually doesn't pay (= usually doesn't +# get us out of the loop early). See Knuth, Volume 3, where this is +# explained and quantified in an exercise. +# +# Cutting the # of comparisons is important, since these routines have no +# way to extract "the priority" from an array element, so that intelligence +# is likely to be hiding in custom comparison methods, or in array elements +# storing (priority, record) tuples. Comparisons are thus potentially +# expensive. +# +# On random arrays of length 1000, making this change cut the number of +# comparisons made by heapify() a little, and those made by exhaustive +# heappop() a lot, in accord with theory. Here are typical results from 3 +# runs (3 just to demonstrate how small the variance is): +# +# Compares needed by heapify Compares needed by 1000 heappops +# -------------------------- -------------------------------- +# 1837 cut to 1663 14996 cut to 8680 +# 1855 cut to 1659 14966 cut to 8678 +# 1847 cut to 1660 15024 cut to 8703 +# +# Building the heap by using heappush() 1000 times instead required +# 2198, 2148, and 2219 compares: heapify() is more efficient, when +# you can use it. +# +# The total compares needed by list.sort() on the same lists were 8627, +# 8627, and 8632 (this should be compared to the sum of heapify() and +# heappop() compares): list.sort() is (unsurprisingly!) more efficient +# for sorting. + +def _siftup(heap, pos): + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[childpos] < heap[rightpos]: + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown(heap, startpos, pos) + +def _siftdown_max(heap, startpos, pos): + 'Maxheap variant of _siftdown' + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if parent < newitem: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +def _siftup_max(heap, pos): + 'Maxheap variant of _siftup' + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the larger child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of larger child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[rightpos] < heap[childpos]: + childpos = rightpos + # Move the larger child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown_max(heap, startpos, pos) + +def merge(*iterables, key=None, reverse=False): + '''Merge multiple sorted inputs into a single sorted output. + + Similar to sorted(itertools.chain(*iterables)) but returns a generator, + does not pull the data into memory all at once, and assumes that each of + the input streams is already sorted (smallest to largest). + + >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) + [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] + + If *key* is not None, applies a key function to each element to determine + its sort order. + + >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) + ['dog', 'cat', 'fish', 'horse', 'kangaroo'] + + ''' + + h = [] + h_append = h.append + + if reverse: + _heapify = _heapify_max + _heappop = _heappop_max + _heapreplace = _heapreplace_max + direction = -1 + else: + _heapify = heapify + _heappop = heappop + _heapreplace = heapreplace + direction = 1 + + if key is None: + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + h_append([next(), order * direction, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + value, order, next = s = h[0] + yield value + s[0] = next() # raises StopIteration when exhausted + _heapreplace(h, s) # restore heap condition + except StopIteration: + _heappop(h) # remove empty iterator + if h: + # fast case when only a single iterator remains + value, order, next = h[0] + yield value + yield from next.__self__ + return + + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + value = next() + h_append([key(value), order * direction, value, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + key_value, order, value, next = s = h[0] + yield value + value = next() + s[0] = key(value) + s[2] = value + _heapreplace(h, s) + except StopIteration: + _heappop(h) + if h: + key_value, order, value, next = h[0] + yield value + yield from next.__self__ + + +# Algorithm notes for nlargest() and nsmallest() +# ============================================== +# +# Make a single pass over the data while keeping the k most extreme values +# in a heap. Memory consumption is limited to keeping k values in a list. +# +# Measured performance for random inputs: +# +# number of comparisons +# n inputs k-extreme values (average of 5 trials) % more than min() +# ------------- ---------------- --------------------- ----------------- +# 1,000 100 3,317 231.7% +# 10,000 100 14,046 40.5% +# 100,000 100 105,749 5.7% +# 1,000,000 100 1,007,751 0.8% +# 10,000,000 100 10,009,401 0.1% +# +# Theoretical number of comparisons for k smallest of n random inputs: +# +# Step Comparisons Action +# ---- -------------------------- --------------------------- +# 1 1.66 * k heapify the first k-inputs +# 2 n - k compare remaining elements to top of heap +# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap +# 4 k * lg2(k) - (k/2) final sort of the k most extreme values +# +# Combining and simplifying for a rough estimate gives: +# +# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) +# +# Computing the number of comparisons for step 3: +# ----------------------------------------------- +# * For the i-th new value from the iterable, the probability of being in the +# k most extreme values is k/i. For example, the probability of the 101st +# value seen being in the 100 most extreme values is 100/101. +# * If the value is a new extreme value, the cost of inserting it into the +# heap is 1 + log(k, 2). +# * The probability times the cost gives: +# (k/i) * (1 + log(k, 2)) +# * Summing across the remaining n-k elements gives: +# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) +# * This reduces to: +# (H(n) - H(k)) * k * (1 + log(k, 2)) +# * Where H(n) is the n-th harmonic number estimated by: +# gamma = 0.5772156649 +# H(n) = log(n, e) + gamma + 1 / (2 * n) +# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence +# * Substituting the H(n) formula: +# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) +# +# Worst-case for step 3: +# ---------------------- +# In the worst case, the input data is reversed sorted so that every new element +# must be inserted in the heap: +# +# comparisons = 1.66 * k + log(k, 2) * (n - k) +# +# Alternative Algorithms +# ---------------------- +# Other algorithms were not used because they: +# 1) Took much more auxiliary memory, +# 2) Made multiple passes over the data. +# 3) Made more comparisons in common cases (small k, large n, semi-random input). +# See the more detailed comparison of approach at: +# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest + +def nsmallest(n, iterable, key=None): + """Find the n smallest elements in a dataset. + + Equivalent to: sorted(iterable, key=key)[:n] + """ + + # Short-cut for n==1 is to use min() + if n == 1: + it = iter(iterable) + sentinel = object() + result = min(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + # put the range(n) first so that zip() doesn't + # consume one too many elements from the iterator + result = [(elem, i) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + if elem < top: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order += 1 + result.sort() + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + k = key(elem) + if k < top: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order += 1 + result.sort() + return [elem for (k, order, elem) in result] + +def nlargest(n, iterable, key=None): + """Find the n largest elements in a dataset. + + Equivalent to: sorted(iterable, key=key, reverse=True)[:n] + """ + + # Short-cut for n==1 is to use max() + if n == 1: + it = iter(iterable) + sentinel = object() + result = max(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key, reverse=True)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + if top < elem: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + k = key(elem) + if top < k: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (k, order, elem) in result] + +# If available, use C implementation +try: + from _heapq import * +except ImportError: + pass +try: + from _heapq import _heapreplace_max +except ImportError: + pass +try: + from _heapq import _heapify_max +except ImportError: + pass +try: + from _heapq import _heappop_max +except ImportError: + pass + + +if __name__ == "__main__": + + import doctest + print(doctest.testmod()) diff --git a/Lib/operator.py b/Lib/operator.py index f21920064..0e2e53efc 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -1,15 +1,464 @@ +""" +Operator Interface + +This module exports a set of functions corresponding to the intrinsic +operators of Python. For example, operator.add(x, y) is equivalent +to the expression x+y. The function names are those used for special +methods; variants without leading and trailing '__' are also provided +for convenience. + +This is the pure Python implementation of the module. +""" + +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', + 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', + 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', + 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', + 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', + 'setitem', 'sub', 'truediv', 'truth', 'xor'] + +from builtins import abs as _abs -# Comparison operations: +# Comparison Operations *******************************************************# + +def lt(a, b): + "Same as a < b." + return a < b + +def le(a, b): + "Same as a <= b." + return a <= b def eq(a, b): + "Same as a == b." return a == b - def ne(a, b): + "Same as a != b." return a != b - def ge(a, b): + "Same as a >= b." return a >= b +def gt(a, b): + "Same as a > b." + return a > b + +# Logical Operations **********************************************************# + +def not_(a): + "Same as not a." + return not a + +def truth(a): + "Return True if a is true, False otherwise." + return True if a else False + +def is_(a, b): + "Same as a is b." + return a is b + +def is_not(a, b): + "Same as a is not b." + return a is not b + +# Mathematical/Bitwise Operations *********************************************# + +def abs(a): + "Same as abs(a)." + return _abs(a) + +def add(a, b): + "Same as a + b." + return a + b + +def and_(a, b): + "Same as a & b." + return a & b + +def floordiv(a, b): + "Same as a // b." + return a // b + +def index(a): + "Same as a.__index__()." + return a.__index__() + +def inv(a): + "Same as ~a." + return ~a +invert = inv + +def lshift(a, b): + "Same as a << b." + return a << b + +def mod(a, b): + "Same as a % b." + return a % b + +def mul(a, b): + "Same as a * b." + return a * b + +def matmul(a, b): + "Same as a @ b." + return a @ b + +def neg(a): + "Same as -a." + return -a + +def or_(a, b): + "Same as a | b." + return a | b + +def pos(a): + "Same as +a." + return +a + +def pow(a, b): + "Same as a ** b." + return a ** b + +def rshift(a, b): + "Same as a >> b." + return a >> b + +def sub(a, b): + "Same as a - b." + return a - b + +def truediv(a, b): + "Same as a / b." + return a / b + +def xor(a, b): + "Same as a ^ b." + return a ^ b + +# Sequence Operations *********************************************************# + +def concat(a, b): + "Same as a + b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + return a + b + +def contains(a, b): + "Same as b in a (note reversed operands)." + return b in a + +def countOf(a, b): + "Return the number of times b occurs in a." + count = 0 + for i in a: + if i == b: + count += 1 + return count + +def delitem(a, b): + "Same as del a[b]." + del a[b] + +def getitem(a, b): + "Same as a[b]." + return a[b] + +def indexOf(a, b): + "Return the first index of b in a." + for i, j in enumerate(a): + if j == b: + return i + else: + raise ValueError('sequence.index(x): x not in sequence') + +def setitem(a, b, c): + "Same as a[b] = c." + a[b] = c + +def length_hint(obj, default=0): + """ + Return an estimate of the number of items in obj. + This is useful for presizing containers when building from an iterable. + + If the object supports len(), the result will be exact. Otherwise, it may + over- or under-estimate by an arbitrary amount. The result will be an + integer >= 0. + """ + if not isinstance(default, int): + msg = ("'%s' object cannot be interpreted as an integer" % + type(default).__name__) + raise TypeError(msg) + + try: + return len(obj) + except TypeError: + pass + + try: + hint = type(obj).__length_hint__ + except AttributeError: + return default + + try: + val = hint(obj) + except TypeError: + return default + if val is NotImplemented: + return default + if not isinstance(val, int): + msg = ('__length_hint__ must be integer, not %s' % + type(val).__name__) + raise TypeError(msg) + if val < 0: + msg = '__length_hint__() should return >= 0' + raise ValueError(msg) + return val + +# Generalized Lookup Objects **************************************************# + +class attrgetter: + """ + Return a callable object that fetches the given attribute(s) from its operand. + After f = attrgetter('name'), the call f(r) returns r.name. + After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). + After h = attrgetter('name.first', 'name.last'), the call h(r) returns + (r.name.first, r.name.last). + """ + __slots__ = ('_attrs', '_call') + + def __init__(self, attr, *attrs): + if not attrs: + if not isinstance(attr, str): + raise TypeError('attribute name must be a string') + self._attrs = (attr,) + names = attr.split('.') + def func(obj): + for name in names: + obj = getattr(obj, name) + return obj + self._call = func + else: + self._attrs = (attr,) + attrs + getters = tuple(map(attrgetter, self._attrs)) + def func(obj): + return tuple(getter(obj) for getter in getters) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__qualname__, + ', '.join(map(repr, self._attrs))) + + def __reduce__(self): + return self.__class__, self._attrs + +class itemgetter: + """ + Return a callable object that fetches the given item(s) from its operand. + After f = itemgetter(2), the call f(r) returns r[2]. + After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) + """ + __slots__ = ('_items', '_call') + + def __init__(self, item, *items): + if not items: + self._items = (item,) + def func(obj): + return obj[item] + self._call = func + else: + self._items = items = (item,) + items + def func(obj): + return tuple(obj[i] for i in items) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(map(repr, self._items))) + + def __reduce__(self): + return self.__class__, self._items + +class methodcaller: + """ + Return a callable object that calls the given method on its operand. + After f = methodcaller('name'), the call f(r) returns r.name(). + After g = methodcaller('name', 'date', foo=1), the call g(r) returns + r.name('date', foo=1). + """ + __slots__ = ('_name', '_args', '_kwargs') + + def __init__(*args, **kwargs): + if len(args) < 2: + msg = "methodcaller needs at least one argument, the method name" + raise TypeError(msg) + self = args[0] + self._name = args[1] + if not isinstance(self._name, str): + raise TypeError('method name must be a string') + self._args = args[2:] + self._kwargs = kwargs + + def __call__(self, obj): + return getattr(obj, self._name)(*self._args, **self._kwargs) + + def __repr__(self): + args = [repr(self._name)] + args.extend(map(repr, self._args)) + args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(args)) + + def __reduce__(self): + if not self._kwargs: + return self.__class__, (self._name,) + self._args + else: + from functools import partial + return partial(self.__class__, self._name, **self._kwargs), self._args + + +# In-place Operations *********************************************************# + +def iadd(a, b): + "Same as a += b." + a += b + return a + +def iand(a, b): + "Same as a &= b." + a &= b + return a + +def iconcat(a, b): + "Same as a += b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + a += b + return a + +def ifloordiv(a, b): + "Same as a //= b." + a //= b + return a + +def ilshift(a, b): + "Same as a <<= b." + a <<= b + return a + +def imod(a, b): + "Same as a %= b." + a %= b + return a + +def imul(a, b): + "Same as a *= b." + a *= b + return a + +def imatmul(a, b): + "Same as a @= b." + a @= b + return a + +def ior(a, b): + "Same as a |= b." + a |= b + return a + +def ipow(a, b): + "Same as a **= b." + a **=b + return a + +def irshift(a, b): + "Same as a >>= b." + a >>= b + return a + +def isub(a, b): + "Same as a -= b." + a -= b + return a + +def itruediv(a, b): + "Same as a /= b." + a /= b + return a + +def ixor(a, b): + "Same as a ^= b." + a ^= b + return a + + +try: + from _operator import * +except ImportError: + pass +else: + from _operator import __doc__ + +# All of these "__func__ = func" assignments have to happen after importing +# from _operator to make sure they're set to the right function +__lt__ = lt +__le__ = le +__eq__ = eq +__ne__ = ne +__ge__ = ge +__gt__ = gt +__not__ = not_ +__abs__ = abs +__add__ = add +__and__ = and_ +__floordiv__ = floordiv +__index__ = index +__inv__ = inv +__invert__ = invert +__lshift__ = lshift +__mod__ = mod +__mul__ = mul +__matmul__ = matmul +__neg__ = neg +__or__ = or_ +__pos__ = pos +__pow__ = pow +__rshift__ = rshift +__sub__ = sub +__truediv__ = truediv +__xor__ = xor +__concat__ = concat +__contains__ = contains +__delitem__ = delitem +__getitem__ = getitem +__setitem__ = setitem +__iadd__ = iadd +__iand__ = iand +__iconcat__ = iconcat +__ifloordiv__ = ifloordiv +__ilshift__ = ilshift +__imod__ = imod +__imul__ = imul +__imatmul__ = imatmul +__ior__ = ior +__ipow__ = ipow +__irshift__ = irshift +__isub__ = isub +__itruediv__ = itruediv +__ixor__ = ixor diff --git a/Lib/os.py b/Lib/os.py index c768946c5..cb49481df 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -133,34 +133,3 @@ def getenv(key, default=None): The optional second argument can specify an alternate default. key, default and the result are str.""" return environ.get(key, default) - - -def fspath(path): - """Return the path representation of a path-like object. - - If str or bytes is passed in, it is returned unchanged. Otherwise the - os.PathLike interface is used to get the path representation. If the - path representation is not str or bytes, TypeError is raised. If the - provided path is not str, bytes, or os.PathLike, TypeError is raised. - """ - if isinstance(path, (str, bytes)): - return path - - # Work from the object's type to match method resolution of other magic - # methods. - path_type = type(path) - try: - path_repr = path_type.__fspath__(path) - except AttributeError: - if hasattr(path_type, '__fspath__'): - raise - else: - raise TypeError("expected str, bytes or os.PathLike object, " - "not " + path_type.__name__) - if isinstance(path_repr, (str, bytes)): - return path_repr - else: - raise TypeError("expected {}.__fspath__() to return str or bytes, " - "not {}".format(path_type.__name__, - type(path_repr).__name__)) - diff --git a/Lib/reprlib.py b/Lib/reprlib.py new file mode 100644 index 000000000..616b3439b --- /dev/null +++ b/Lib/reprlib.py @@ -0,0 +1,161 @@ +"""Redo the builtin repr() (representation) but with limits on most sizes.""" + +__all__ = ["Repr", "repr", "recursive_repr"] + +import builtins +from itertools import islice +from _thread import get_ident + +def recursive_repr(fillvalue='...'): + 'Decorator to make a repr function return fillvalue for a recursive call' + + def decorating_function(user_function): + repr_running = set() + + def wrapper(self): + key = id(self), get_ident() + if key in repr_running: + return fillvalue + repr_running.add(key) + try: + result = user_function(self) + finally: + repr_running.discard(key) + return result + + # Can't use functools.wraps() here because of bootstrap issues + wrapper.__module__ = getattr(user_function, '__module__') + wrapper.__doc__ = getattr(user_function, '__doc__') + wrapper.__name__ = getattr(user_function, '__name__') + wrapper.__qualname__ = getattr(user_function, '__qualname__') + wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + return wrapper + + return decorating_function + +class Repr: + + def __init__(self): + self.maxlevel = 6 + self.maxtuple = 6 + self.maxlist = 6 + self.maxarray = 5 + self.maxdict = 4 + self.maxset = 6 + self.maxfrozenset = 6 + self.maxdeque = 6 + self.maxstring = 30 + self.maxlong = 40 + self.maxother = 30 + + def repr(self, x): + return self.repr1(x, self.maxlevel) + + def repr1(self, x, level): + typename = type(x).__name__ + if ' ' in typename: + parts = typename.split() + typename = '_'.join(parts) + if hasattr(self, 'repr_' + typename): + return getattr(self, 'repr_' + typename)(x, level) + else: + return self.repr_instance(x, level) + + def _repr_iterable(self, x, level, left, right, maxiter, trail=''): + n = len(x) + if level <= 0 and n: + s = '...' + else: + newlevel = level - 1 + repr1 = self.repr1 + pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] + if n > maxiter: pieces.append('...') + s = ', '.join(pieces) + if n == 1 and trail: right = trail + right + return '%s%s%s' % (left, s, right) + + def repr_tuple(self, x, level): + return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') + + def repr_list(self, x, level): + return self._repr_iterable(x, level, '[', ']', self.maxlist) + + def repr_array(self, x, level): + if not x: + return "array('%s')" % x.typecode + header = "array('%s', [" % x.typecode + return self._repr_iterable(x, level, header, '])', self.maxarray) + + def repr_set(self, x, level): + if not x: + return 'set()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, '{', '}', self.maxset) + + def repr_frozenset(self, x, level): + if not x: + return 'frozenset()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, 'frozenset({', '})', + self.maxfrozenset) + + def repr_deque(self, x, level): + return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) + + def repr_dict(self, x, level): + n = len(x) + if n == 0: return '{}' + if level <= 0: return '{...}' + newlevel = level - 1 + repr1 = self.repr1 + pieces = [] + for key in islice(_possibly_sorted(x), self.maxdict): + keyrepr = repr1(key, newlevel) + valrepr = repr1(x[key], newlevel) + pieces.append('%s: %s' % (keyrepr, valrepr)) + if n > self.maxdict: pieces.append('...') + s = ', '.join(pieces) + return '{%s}' % (s,) + + def repr_str(self, x, level): + s = builtins.repr(x[:self.maxstring]) + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = builtins.repr(x[:i] + x[len(x)-j:]) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_int(self, x, level): + s = builtins.repr(x) # XXX Hope this isn't too slow... + if len(s) > self.maxlong: + i = max(0, (self.maxlong-3)//2) + j = max(0, self.maxlong-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_instance(self, x, level): + try: + s = builtins.repr(x) + # Bugs in x.__repr__() can cause arbitrary + # exceptions -- then make up something + except Exception: + return '<%s instance at %#x>' % (x.__class__.__name__, id(x)) + if len(s) > self.maxother: + i = max(0, (self.maxother-3)//2) + j = max(0, self.maxother-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + +def _possibly_sorted(x): + # Since not all sequences of items can be sorted and comparison + # functions may raise arbitrary exceptions, return an unsorted + # sequence in that case. + try: + return sorted(x) + except Exception: + return list(x) + +aRepr = Repr() +repr = aRepr.repr diff --git a/Lib/types.py b/Lib/types.py new file mode 100644 index 000000000..a7034901c --- /dev/null +++ b/Lib/types.py @@ -0,0 +1,295 @@ +""" +Define names for built-in types that aren't directly accessible as a builtin. +""" +import sys + +# Iterators in Python aren't a matter of type but of protocol. A large +# and changing number of builtin types implement *some* flavor of +# iterator. Don't check the type! Use hasattr to check for both +# "__iter__" and "__next__" attributes instead. + +def _f(): pass +FunctionType = type(_f) +LambdaType = type(lambda: None) # Same as FunctionType +CodeType = type(_f.__code__) +MappingProxyType = type(type.__dict__) +SimpleNamespace = type(sys.implementation) + +def _g(): + yield 1 +GeneratorType = type(_g()) + +# async def _c(): pass +# _c = _c() +# CoroutineType = type(_c) +# _c.close() # Prevent ResourceWarning + +# async def _ag(): +# yield +# _ag = _ag() +# AsyncGeneratorType = type(_ag) + +class _C: + def _m(self): pass +MethodType = type(_C()._m) + +BuiltinFunctionType = type(len) +BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + +WrapperDescriptorType = type(object.__init__) +MethodWrapperType = type(object().__str__) +MethodDescriptorType = type(str.join) +ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + +ModuleType = type(sys) + +# try: +# raise TypeError +# except TypeError: +# tb = sys.exc_info()[2] +# TracebackType = type(tb) +# FrameType = type(tb.tb_frame) +# tb = None; del tb + +# For Jython, the following two types are identical +GetSetDescriptorType = type(FunctionType.__code__) +# MemberDescriptorType = type(FunctionType.__globals__) + +del sys, _f, _g, _C # Not for export + + +# Provide a PEP 3115 compliant mechanism for class creation +def new_class(name, bases=(), kwds=None, exec_body=None): + """Create a class object dynamically using the appropriate metaclass.""" + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) + if exec_body is not None: + exec_body(ns) + if resolved_bases is not bases: + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + shift = 0 + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entries__"): + continue + new_base = base.__mro_entries__(bases) + updated = True + if not isinstance(new_base, tuple): + raise TypeError("__mro_entries__ must return a tuple") + else: + new_bases[i+shift:i+shift+1] = new_base + shift += len(new_base) - 1 + if not updated: + return bases + return tuple(new_bases) + +def prepare_class(name, bases=(), kwds=None): + """Call the __prepare__ method of the appropriate metaclass. + + Returns (metaclass, namespace, kwds) as a 3-tuple + + *metaclass* is the appropriate metaclass + *namespace* is the prepared class namespace + *kwds* is an updated copy of the passed in kwds argument with any + 'metaclass' entry removed. If no kwds argument is passed in, this will + be an empty dict. + """ + if kwds is None: + kwds = {} + else: + kwds = dict(kwds) # Don't alter the provided mapping + if 'metaclass' in kwds: + meta = kwds.pop('metaclass') + else: + if bases: + meta = type(bases[0]) + else: + meta = type + if isinstance(meta, type): + # when meta is a type, we first determine the most-derived metaclass + # instead of invoking the initial candidate directly + meta = _calculate_meta(meta, bases) + if hasattr(meta, '__prepare__'): + ns = meta.__prepare__(name, bases, **kwds) + else: + ns = {} + return meta, ns, kwds + +def _calculate_meta(meta, bases): + """Calculate the most derived metaclass.""" + winner = meta + for base in bases: + base_meta = type(base) + if issubclass(winner, base_meta): + continue + if issubclass(base_meta, winner): + winner = base_meta + continue + # else: + raise TypeError("metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases") + return winner + +class DynamicClassAttribute: + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + This allows one to have properties active on an instance, and have virtual + attributes on the class with the same name (see Enum for an example). + + """ + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + # next two lines make DynamicClassAttribute act the same as property + self.__doc__ = doc or fget.__doc__ + self.overwrite_doc = doc is None + # support for abstract methods + self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False)) + + def __get__(self, instance, ownerclass=None): + if instance is None: + if self.__isabstractmethod__: + return self + raise AttributeError() + elif self.fget is None: + raise AttributeError("unreadable attribute") + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError("can't set attribute") + self.fset(instance, value) + + def __delete__(self, instance): + if self.fdel is None: + raise AttributeError("can't delete attribute") + self.fdel(instance) + + def getter(self, fget): + fdoc = fget.__doc__ if self.overwrite_doc else None + result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def setter(self, fset): + result = type(self)(self.fget, fset, self.fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def deleter(self, fdel): + result = type(self)(self.fget, self.fset, fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + +class _GeneratorWrapper: + # TODO: Implement this in C. + def __init__(self, gen): + self.__wrapped = gen + self.__isgen = gen.__class__ is GeneratorType + self.__name__ = getattr(gen, '__name__', None) + self.__qualname__ = getattr(gen, '__qualname__', None) + def send(self, val): + return self.__wrapped.send(val) + def throw(self, tp, *rest): + return self.__wrapped.throw(tp, *rest) + def close(self): + return self.__wrapped.close() + @property + def gi_code(self): + return self.__wrapped.gi_code + @property + def gi_frame(self): + return self.__wrapped.gi_frame + @property + def gi_running(self): + return self.__wrapped.gi_running + @property + def gi_yieldfrom(self): + return self.__wrapped.gi_yieldfrom + cr_code = gi_code + cr_frame = gi_frame + cr_running = gi_running + cr_await = gi_yieldfrom + def __next__(self): + return next(self.__wrapped) + def __iter__(self): + if self.__isgen: + return self.__wrapped + return self + __await__ = __iter__ + +def coroutine(func): + """Convert regular generator function to a coroutine.""" + + if not callable(func): + raise TypeError('types.coroutine() expects a callable') + + if (func.__class__ is FunctionType and + getattr(func, '__code__', None).__class__ is CodeType): + + co_flags = func.__code__.co_flags + + # Check if 'func' is a coroutine function. + # (0x180 == CO_COROUTINE | CO_ITERABLE_COROUTINE) + if co_flags & 0x180: + return func + + # Check if 'func' is a generator function. + # (0x20 == CO_GENERATOR) + if co_flags & 0x20: + # TODO: Implement this in C. + co = func.__code__ + func.__code__ = CodeType( + co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, + co.co_stacksize, + co.co_flags | 0x100, # 0x100 == CO_ITERABLE_COROUTINE + co.co_code, + co.co_consts, co.co_names, co.co_varnames, co.co_filename, + co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, + co.co_cellvars) + return func + + # The following code is primarily to support functions that + # return generator-like objects (for instance generators + # compiled with Cython). + + # Delay functools and _collections_abc import for speeding up types import. + import functools + import _collections_abc + @functools.wraps(func) + def wrapped(*args, **kwargs): + coro = func(*args, **kwargs) + if (coro.__class__ is CoroutineType or + coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100): + # 'coro' is a native coroutine object or an iterable coroutine + return coro + if (isinstance(coro, _collections_abc.Generator) and + not isinstance(coro, _collections_abc.Coroutine)): + # 'coro' is either a pure Python generator iterator, or it + # implements collections.abc.Generator (and does not implement + # collections.abc.Coroutine). + return _GeneratorWrapper(coro) + # 'coro' is either an instance of collections.abc.Coroutine or + # some other object -- pass it through. + return coro + + return wrapped + + +__all__ = [n for n in globals() if n[:1] != '_'] diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 72ab44d4c..c88ca6769 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "128"] + extern crate proc_macro; #[macro_use] @@ -38,3 +40,10 @@ pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { let item = parse_macro_input!(item as Item); result_to_tokens(pyclass::impl_pyimpl(attr, item)) } + +#[proc_macro_attribute] +pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + result_to_tokens(pyclass::impl_pystruct_sequence(attr, item)) +} diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs index a2cf57b4a..d216b9357 100644 --- a/derive/src/pyclass.rs +++ b/derive/src/pyclass.rs @@ -2,7 +2,9 @@ use super::Diagnostic; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use std::collections::HashMap; -use syn::{Attribute, AttributeArgs, Ident, ImplItem, Item, Lit, Meta, MethodSig, NestedMeta}; +use syn::{ + Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, Meta, MethodSig, NestedMeta, +}; enum ClassItem { Method { @@ -266,16 +268,12 @@ pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result Result { - let (item, ident, attrs) = match item { - Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), - Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), - other => bail_span!( - other, - "#[pyclass] can only be on a struct or enum declaration" - ), - }; - +fn generate_class_def( + ident: &Ident, + attr_name: &'static str, + attr: AttributeArgs, + attrs: &Vec, +) -> Result { let mut class_name = None; for attr in attr { if let NestedMeta::Meta(meta) = attr { @@ -284,7 +282,11 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result = #doc; @@ -324,3 +325,92 @@ pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result Result { + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), + other => bail_span!( + other, + "#[pyclass] can only be on a struct or enum declaration" + ), + }; + + let class_def = generate_class_def(&ident, "pyclass", attr, &attrs)?; + + let ret = quote! { + #item + #class_def + }; + Ok(ret) +} + +pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result { + let struc = if let Item::Struct(struc) = item { + struc + } else { + bail_span!( + item, + "#[pystruct_sequence] can only be on a struct declaration" + ) + }; + let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?; + let mut properties = Vec::new(); + let mut field_names = Vec::new(); + for (i, field) in struc.fields.iter().enumerate() { + let idx = Index::from(i); + if let Some(ref field_name) = field.ident { + let field_name_str = field_name.to_string(); + // TODO add doc to the generated property + let property = quote! { + class.set_str_attr( + #field_name_str, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(|zelf: &::rustpython_vm::obj::objtuple::PyTuple, + _vm: &::rustpython_vm::vm::VirtualMachine| + zelf.fast_getitem(#idx)) + .create(), + ); + }; + properties.push(property); + field_names.push(quote!(#field_name)); + } else { + field_names.push(quote!(#idx)); + } + } + + let ty = &struc.ident; + let ret = quote! { + #struc + #class_def + impl #ty { + fn into_struct_sequence(&self, + vm: &::rustpython_vm::vm::VirtualMachine, + cls: ::rustpython_vm::obj::objtype::PyClassRef, + ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { + let tuple: ::rustpython_vm::obj::objtuple::PyTuple = + vec![#(::rustpython_vm::pyobject::IntoPyObject + ::into_pyobject(self.#field_names, vm)? + ),*].into(); + ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) + } + } + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#properties)* + } + + fn make_class( + ctx: &::rustpython_vm::pyobject::PyContext + ) -> ::rustpython_vm::obj::objtype::PyClassRef { + let py_class = ctx.new_class(::NAME, ctx.tuple_type()); + Self::extend_class(ctx, &py_class); + py_class + } + } + }; + Ok(ret) +} diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e3c4e36bd..6cc7805c6 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -16,3 +16,4 @@ num-bigint = "0.2" num-traits = "0.2" unicode-xid = "0.1.0" unic-emoji-char = "0.9.0" +serde = { version = "1.0.66", features = ["derive"] } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index ec0677fe8..941e04131 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -4,6 +4,8 @@ pub use super::lexer::Location; use num_bigint::BigInt; +use serde::{Deserialize, Serialize}; + /* #[derive(Debug)] @@ -42,6 +44,7 @@ pub struct Located { pub type LocatedStatement = Located; /// Abstract syntax tree nodes for python statements. +#[allow(clippy::large_enum_variant)] #[derive(Debug, PartialEq)] pub enum Statement { Break, @@ -381,7 +384,7 @@ pub enum Number { } /// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum ConversionFlag { /// Converts by calling `str()`. Str, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 1c20df038..3efdf6fbc 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -7,6 +7,7 @@ extern crate unicode_xid; pub use super::token::Tok; use num_bigint::BigInt; use num_traits::Num; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; @@ -71,7 +72,7 @@ pub enum LexicalErrorType { OtherError(String), } -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Location { row: usize, column: usize, diff --git a/src/main.rs b/src/main.rs index 2100226c2..cd20462ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -159,7 +159,7 @@ fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), Com Ok(value) => { // Save non-None values as "_" - use rustpython_vm::pyobject::{IdProtocol, IntoPyObject, ItemProtocol}; + use rustpython_vm::pyobject::{IdProtocol, IntoPyObject}; if !value.is(&vm.get_none()) { let key = objstr::PyString::from("_").into_pyobject(vm); diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index 8d883a9dd..06d46a980 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -86,6 +86,14 @@ assert bool(complex(0, 0)) is False assert bool(complex(0, 1)) is True assert bool(complex(1, 0)) is True +# __hash__ + +assert hash(complex(1)) == hash(float(1)) == hash(int(1)) +assert hash(complex(-1)) == hash(float(-1)) == hash(int(-1)) +assert hash(complex(3.14)) == hash(float(3.14)) +assert hash(complex(-float('inf'))) == hash(-float('inf')) +assert hash(1j) != hash(1) + # numbers.Complex a = complex(3, 4) diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index b5fbf8d0b..3e808597c 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -1,3 +1,5 @@ +assert isinstance(dir(), list) +assert '__builtins__' in dir() class A: def test(): @@ -21,6 +23,16 @@ assert ['h', 'q'] == dir(B()) # This calls type.__dir__ so isn't changed (but inheritance works)! assert 'test' in dir(A) +# eval() takes any mapping-like type, so dir() must support them +# TODO: eval() should take any mapping as locals, not just dict-derived types +class A(dict): + def __getitem__(self, x): + return dir + def keys(self): + yield 6 + yield 5 +assert eval("dir()", {}, A()) == [5, 6] + import socket assert "AF_INET" in dir(socket) diff --git a/tests/snippets/builtin_open.py b/tests/snippets/builtin_open.py index 41bcf2b80..d7c310a92 100644 --- a/tests/snippets/builtin_open.py +++ b/tests/snippets/builtin_open.py @@ -6,5 +6,14 @@ assert 'RustPython' in fd.read() assert_raises(FileNotFoundError, lambda: open('DoesNotExist')) # Use open as a context manager -with open('README.md') as fp: - fp.read() +with open('README.md', 'rt') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'r') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'rb') as fp: + contents = fp.read() + assert type(contents) == bytes, "type is " + str(type(contents)) diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index dff0852c4..f69edeac1 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -691,3 +691,18 @@ assert a == bytearray(b'abdabcd') a = bytearray(b'hello, world') a.reverse() assert a == bytearray(b'dlrow ,olleh') + +# __setitem__ +a = bytearray(b'test') +a[0] = 1 +assert a == bytearray(b'\x01est') +with assertRaises(TypeError): + a[0] = b'a' +with assertRaises(TypeError): + a[0] = memoryview(b'a') +a[:2] = [0, 9] +assert a == bytearray(b'\x00\x09st') +a[1:3] = b'test' +assert a == bytearray(b'\x00testt') +a[:6] = memoryview(b'test') +assert a == bytearray(b'test') diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 270d2c07e..4545f7f58 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -198,3 +198,22 @@ z = {'c': 3, 'd': 3, 'e': 3} w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} + +assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) + +class A: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, A) +class B: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, B) + +s = {1: 0, A(): 1, B(): 2} +assert len(s) == 3 +assert s[1] == 0 +assert s[A()] == 1 +assert s[B()] == 2 diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 82f2b86e3..101d58cc9 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -98,6 +98,21 @@ assert float(b'2.99e-23') == 2.99e-23 assert_raises(ValueError, lambda: float('foo')) assert_raises(OverflowError, lambda: float(2**10000)) +# check eq and hash for small numbers + +assert 1.0 == 1 +assert 1.0 == True +assert 0.0 == 0 +assert 0.0 == False +assert hash(1.0) == hash(1) +assert hash(1.0) == hash(True) +assert hash(0.0) == hash(0) +assert hash(0.0) == hash(False) +assert hash(1.0) != hash(1.0000000001) + +assert 5.0 in {3, 4, 5} +assert {-1: 2}[-1.0] == 2 + # check that magic methods are implemented for ints and floats assert 1.0.__add__(1.0) == 2.0 @@ -181,3 +196,21 @@ assert str(1.123456789) == '1.123456789' a = .5 assert a == 0.5 +assert float.fromhex('0x0.0p+0') == 0.0 +assert float.fromhex('-0x0.0p+0') == -0.0 +assert float.fromhex('0x1.000000p+0') == 1.0 +assert float.fromhex('-0x1.800000p+0') == -1.5 +assert float.fromhex('inf') == float('inf') +assert math.isnan(float.fromhex('nan')) + +assert (0.0).hex() == '0x0.0p+0' +assert (-0.0).hex() == '-0x0.0p+0' +assert (1.0).hex() == '0x1.0000000000000p+0' +assert (-1.5).hex() == '-0x1.8000000000000p+0' +assert float('inf').hex() == 'inf' +assert float('-inf').hex() == '-inf' +assert float('nan').hex() == 'nan' + +#for _ in range(10000): +# f = random.random() * random.randint(0, 0x10000000000000000) +# assert f == float.fromhex(f.hex()) diff --git a/tests/snippets/frozen.py b/tests/snippets/frozen.py new file mode 100644 index 000000000..d03658c19 --- /dev/null +++ b/tests/snippets/frozen.py @@ -0,0 +1,2 @@ +import __hello__ +assert __hello__.initialized == True diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index b9c5a1626..86c92617a 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -81,3 +81,17 @@ def fubar(x, y, obj=None): rest = [4, 5] fubar(obj=6, *rest) + + +# https://www.python.org/dev/peps/pep-0468/ +def func(**kwargs): + return list(kwargs.items()) + +empty_kwargs = func() +assert empty_kwargs == [] + +kwargs = func(a=1, b=2) +assert kwargs == [('a', 1), ('b', 2)] + +kwargs = func(a=1, b=2, c=3) +assert kwargs == [('a', 1), ('b', 2), ('c', 3)] diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py new file mode 100644 index 000000000..44e7e1042 --- /dev/null +++ b/tests/snippets/imp.py @@ -0,0 +1,28 @@ +import _imp +import time as import_time + +assert _imp.is_builtin("time") == True +assert _imp.is_builtin("os") == False +assert _imp.is_builtin("not existing module") == False + +assert _imp.is_frozen("__hello__") == True +assert _imp.is_frozen("os") == False + +class FakeSpec: + def __init__(self, name): + self.name = name + +A = FakeSpec("time") + +imp_time = _imp.create_builtin(A) +assert imp_time.sleep == import_time.sleep + +B = FakeSpec("not existing module") +assert _imp.create_builtin(B) == None + +_imp.exec_builtin(imp_time) == 0 + +_imp.get_frozen_object("__hello__") + +hello = _imp.init_frozen("__hello__") +assert hello.initialized == True diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index be41c3257..b947797ac 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -79,6 +79,13 @@ assert int.from_bytes(b'\x00\x10', 'little') == 4096 assert int.from_bytes(b'\xfc\x00', 'big', signed=True) == -1024 assert int.from_bytes(b'\xfc\x00', 'big', signed=False) == 64512 +assert (1024).to_bytes(4, 'big') == b'\x00\x00\x04\x00' +assert (1024).to_bytes(2, 'little', signed=True) == b'\x00\x04' +assert (-1024).to_bytes(4, 'big', signed=True) == b'\xff\xff\xfc\x00' +assert (-1024).to_bytes(4, 'little', signed=True) == b'\x00\xfc\xff\xff' +assert (2147483647).to_bytes(8, 'big', signed=False) == b'\x00\x00\x00\x00\x7f\xff\xff\xff' +assert (-2147483648).to_bytes(8, 'little', signed=True) == b'\x00\x00\x00\x80\xff\xff\xff\xff' + with assertRaises(TypeError): int(base=2) diff --git a/tests/snippets/math_module.py b/tests/snippets/math_module.py index 1064610f1..dc7fc0706 100644 --- a/tests/snippets/math_module.py +++ b/tests/snippets/math_module.py @@ -1,5 +1,5 @@ import math -from testutils import assertRaises +from testutils import assertRaises, assert_raises # assert(math.exp(2) == math.exp(2.0)) # assert(math.exp(True) == math.exp(1.0)) @@ -78,3 +78,12 @@ with assertRaises(TypeError): math.ceil(object()) with assertRaises(TypeError): math.floor(object()) + +assert str(math.frexp(0.0)) == str((+0.0, 0)) +assert str(math.frexp(-0.0)) == str((-0.0, 0)) +assert math.frexp(1) == (0.5, 1) +assert math.frexp(1.5) == (0.75, 1) + +assert math.frexp(float('inf')) == (float('inf'), 0) +assert str(math.frexp(float('nan'))) == str((float('nan'), 0)) +assert_raises(TypeError, lambda: math.frexp(None)) diff --git a/tests/snippets/set.py b/tests/snippets/set.py index ba20fbae6..50b8f97d8 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -27,6 +27,26 @@ assert set([1,2]) < set([1,2,3]) assert not set([1,2]) < set([1,2]) assert not set([1,3]) < set([1,2]) +assert (set() == []) is False +assert set().__eq__([]) == NotImplemented +assert_raises(TypeError, lambda: set() < [], "'<' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() <= [], "'<=' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() > [], "'>' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() >= [], "'>=' not supported between instances of 'set' and 'list'") +assert set().issuperset([]) +assert set().issubset([]) +assert not set().issuperset([1, 2, 3]) +assert set().issubset([1, 2]) + +assert (set() == 3) is False +assert set().__eq__(3) == NotImplemented +assert_raises(TypeError, lambda: set() < 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() <= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() > 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() >= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issuperset(3), "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issubset(3), "'int' object is not iterable") + class Hashable(object): def __init__(self, obj): self.obj = obj @@ -83,6 +103,12 @@ assert set([1,2,3]).isdisjoint(set([5,6])) == True assert set([1,2,3]).isdisjoint(set([2,5,6])) == False assert set([1,2,3]).isdisjoint([5,6]) == True +assert_raises(TypeError, lambda: set() & []) +assert_raises(TypeError, lambda: set() | []) +assert_raises(TypeError, lambda: set() ^ []) +assert_raises(TypeError, lambda: set() + []) +assert_raises(TypeError, lambda: set() - []) + assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -103,7 +129,7 @@ a = set([1,2]) b = a.pop() assert b in [1,2] c = a.pop() -assert (c in [1,2] and c != b) +assert (c in [1,2] and c != b) assert_raises(KeyError, lambda: a.pop()) a = set([1,2,3]) @@ -268,3 +294,36 @@ assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) + +class A: + def __hash__(self): + return 1 +class B: + def __hash__(self): + return 1 + +s = {1, A(), B()} +assert len(s) == 3 + +s = {True} +s.add(1.0) +assert str(s) == '{True}' + +class EqObject: + def __init__(self, eq): + self.eq = eq + def __eq__(self, other): + return self.eq + def __hash__(self): + return bool(self.eq) + +assert 'x' == (EqObject('x') == EqObject('x')) +s = {EqObject('x')} +assert EqObject('x') in s +assert '[]' == (EqObject('[]') == EqObject('[]')) +s = {EqObject([])} +assert EqObject([]) not in s +x = object() +assert x == (EqObject(x) == EqObject(x)) +s = {EqObject(x)} +assert EqObject(x) in s diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py index e73e162bd..e56b3e47d 100644 --- a/tests/snippets/stdlib_binascii.py +++ b/tests/snippets/stdlib_binascii.py @@ -33,3 +33,7 @@ with assertRaises(ValueError): with assertRaises(ValueError): uh(b"nn") # Non-hexadecimal digit found + +assert binascii.crc32(b"hello world") == 222957957 +assert binascii.crc32(b"hello world", 555555) == 1216827162 +assert binascii.crc32(b"goodbye interesting world",777777) == 1885538403 diff --git a/tests/snippets/stdlib_io.py b/tests/snippets/stdlib_io.py index 503759df7..e6da760d2 100644 --- a/tests/snippets/stdlib_io.py +++ b/tests/snippets/stdlib_io.py @@ -8,3 +8,9 @@ result = bb.read() assert len(result) <= 8*1024 assert len(result) >= 0 assert isinstance(result, bytes) + +with FileIO('README.md') as fio: + res = fio.read() + assert len(result) <= 8*1024 + assert len(result) >= 0 + assert isinstance(result, bytes) diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py index 61c031dd7..e8285fac9 100644 --- a/tests/snippets/stdlib_itertools.py +++ b/tests/snippets/stdlib_itertools.py @@ -3,6 +3,24 @@ import itertools from testutils import assertRaises +# itertools.chain tests +chain = itertools.chain + +# empty +assert list(chain()) == [] +assert list(chain([], "", b"", ())) == [] + +assert list(chain([1, 2, 3, 4])) == [1, 2, 3, 4] +assert list(chain("ab", "cd", (), 'e')) == ['a', 'b', 'c', 'd', 'e'] +with assertRaises(TypeError): + list(chain(1)) + +x = chain("ab", 1) +assert next(x) == 'a' +assert next(x) == 'b' +with assertRaises(TypeError): + next(x) + # itertools.count tests # default arguments @@ -82,3 +100,86 @@ with assertRaises(StopIteration): r = itertools.repeat(1, -1) with assertRaises(StopIteration): next(r) + + +# itertools.starmap tests +starmap = itertools.starmap + +assert list(starmap(pow, zip(range(3), range(1,7)))) == [0**1, 1**2, 2**3] +assert list(starmap(pow, [])) == [] +assert list(starmap(pow, [iter([4,5])])) == [4**5] +with assertRaises(TypeError): + starmap(pow) + + +# itertools.takewhile tests + +from itertools import takewhile as tw + +t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# not iterable +with assertRaises(TypeError): + tw(lambda n: n < 1, 1) + +# not callable +t = tw(5, [1, 2]) +with assertRaises(TypeError): + next(t) + +# non-bool predicate +t = tw(lambda n: n, [1, 2, 0]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# bad predicate prototype +t = tw(lambda: True, [1]) +with assertRaises(TypeError): + next(t) + +# StopIteration before attempting to call (bad) predicate +t = tw(lambda: True, []) +with assertRaises(StopIteration): + next(t) + +# doesn't try again after the first predicate failure +t = tw(lambda n: n < 1, [1, 0]) +with assertRaises(StopIteration): + next(t) +with assertRaises(StopIteration): + next(t) + + +# itertools.islice tests + +def assert_matches_seq(it, seq): + assert list(it) == list(seq) + +i = itertools.islice + +it = i([1, 2, 3, 4, 5], 3) +assert_matches_seq(it, [1, 2, 3]) + +it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2], None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, None) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2, 3], 1, None, None) +assert_matches_seq(it, [2, 3]) + +it = i([1, 2, 3], None, 2, None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, 3) +assert_matches_seq(it, [1]) diff --git a/tests/snippets/stdlib_marshal.py b/tests/snippets/stdlib_marshal.py new file mode 100644 index 000000000..118bb1bdd --- /dev/null +++ b/tests/snippets/stdlib_marshal.py @@ -0,0 +1,7 @@ +import marshal +orig = compile("1 + 1", "", 'eval') + +dumped = marshal.dumps(orig) +loaded = marshal.loads(dumped) + +assert eval(loaded) == eval(orig) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index 61fa42855..fe0b75721 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -49,6 +49,10 @@ else: assert os.altsep == None assert os.pathsep == ":" +assert os.fspath("Testing") == "Testing" +assert os.fspath(b"Testing") == b"Testing" +assert_raises(TypeError, lambda: os.fspath([1,2,3])) + class TestWithTempDir(): def __enter__(self): if os.name == "nt": @@ -217,3 +221,8 @@ with TestWithTempDir() as tmpdir: os.chdir(tmpdir) assert os.getcwd() == tmpdir os.path.exists(FILE_NAME) + +# supports +assert isinstance(os.supports_fd, set) +assert isinstance(os.supports_dir_fd, set) +assert isinstance(os.supports_follow_symlinks, set) diff --git a/tests/snippets/stdlib_types.py b/tests/snippets/stdlib_types.py new file mode 100644 index 000000000..b8011bf40 --- /dev/null +++ b/tests/snippets/stdlib_types.py @@ -0,0 +1,10 @@ +import types + +from testutils import assertRaises + +ns = types.SimpleNamespace(a=2, b='Rust') + +assert ns.a == 2 +assert ns.b == "Rust" +with assertRaises(AttributeError): + _ = ns.c diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 521b56353..aaeaed5f3 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -37,10 +37,11 @@ assert 3 * "xy" == "xyxyxy" assert 0 * "x" == "" assert -1 * "x" == "" +assert_raises(OverflowError, lambda: 'xy' * 234234234234234234234234234234) + a = 'Hallo' assert a.lower() == 'hallo' assert a.upper() == 'HALLO' -assert a.split('al') == ['H', 'lo'] assert a.startswith('H') assert a.startswith(('H', 1)) assert a.startswith(('A', 'H')) @@ -59,7 +60,9 @@ assert not a.isnumeric() assert a.istitle() assert a.isalpha() - +s = '1 2 3' +assert s.split(' ', 1) == ['1', '2 3'] +assert s.rsplit(' ', 1) == ['1 2', '3'] b = ' hallo ' assert b.strip() == 'hallo' @@ -203,3 +206,13 @@ def try_mutate_str(): word[0] = 'x' assert_raises(TypeError, try_mutate_str) + +ss = ['Hello', '안녕', '👋'] +bs = [b'Hello', b'\xec\x95\x88\xeb\x85\x95', b'\xf0\x9f\x91\x8b'] + +for s, b in zip(ss, bs): + assert s.encode() == b + +for s, b, e in zip(ss, bs, ['u8', 'U8', 'utf-8', 'UTF-8', 'utf_8']): + assert s.encode(e) == b + # assert s.encode(encoding=e) == b diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index 423ecf08d..23dd70f9b 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -7,3 +7,14 @@ assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "w assert isinstance(sys.builtin_module_names, tuple) assert 'sys' in sys.builtin_module_names + +assert isinstance(sys.implementation.name, str) +assert isinstance(sys.implementation.cache_tag, str) + +assert sys.getfilesystemencoding() == 'utf-8' +assert sys.getfilesystemencodeerrors().startswith('surrogate') + +assert isinstance(sys.flags, tuple) +assert type(sys.flags).__name__ == "flags" +assert type(sys.flags.optimize) is int +assert sys.flags[3] == sys.flags.optimize diff --git a/tests/snippets/test_re.py b/tests/snippets/test_re.py index 2463f380b..2bc7308e6 100644 --- a/tests/snippets/test_re.py +++ b/tests/snippets/test_re.py @@ -11,3 +11,5 @@ print(mo) # assert isinstance(mo, re.Match) assert mo.start() == 1 assert mo.end() == 5 + +assert re.escape('python.exe') == 'python\\.exe' diff --git a/tests/snippets/warnings.py b/tests/snippets/warnings.py new file mode 100644 index 000000000..95092e506 --- /dev/null +++ b/tests/snippets/warnings.py @@ -0,0 +1,3 @@ +import _warnings + +_warnings.warn("Test") diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 8ac5ebabb..922163d01 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,8 +6,8 @@ edition = "2018" [dependencies] bitflags = "1.0.4" -num-complex = "0.2" -num-bigint = "0.2.1" +num-complex = { version = "0.2", features = ["serde"] } +num-bigint = { version = "0.2.1", features = ["serde"] } num-traits = "0.2" num-integer = "0.1.39" num-rational = "0.2.1" @@ -15,8 +15,7 @@ rand = "0.5" log = "0.3" rustpython_derive = {path = "../derive"} rustpython_parser = {path = "../parser"} -serde = "1.0.66" -serde_derive = "1.0.66" +serde = { version = "1.0.66", features = ["derive"] } serde_json = "1.0.26" byteorder = "1.2.6" regex = "1" @@ -28,8 +27,16 @@ unicode-xid = "0.1.0" lazy_static = "^1.0.1" lexical = "2.0.0" itertools = "^0.8.0" +hexf = "0.1.0" +indexmap = "1.0.2" +crc = "^1.0.0" +bincode = "1.1.4" + # TODO: release and publish to crates.io [dependencies.unicode-casing] git = "https://github.com/OddCoincidence/unicode-casing" rev = "90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" + +[target.'cfg(all(unix, not(target_os = "android")))'.dependencies] +pwd = "1" diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 53d760cc8..9afcd0e2c 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -111,15 +111,13 @@ fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> vm.del_attr(&obj, attr.into_object()) } -fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - Ok(vm.get_locals().into_object()) - } else { - let obj = args.args.into_iter().next().unwrap(); - let seq = vm.call_method(&obj, "__dir__", vec![])?; - let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; - Ok(sorted) - } +fn builtin_dir(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { + let seq = match obj { + OptionalArg::Present(obj) => vm.call_method(&obj, "__dir__", vec![])?, + OptionalArg::Missing => vm.call_method(&vm.get_locals().into_object(), "keys", vec![])?, + }; + let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; + Ok(sorted) } fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -136,6 +134,7 @@ fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { /// Implements `eval`. /// See also: https://docs.python.org/3/library/functions.html#eval fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: support any mapping for `locals` arg_check!( vm, args, @@ -303,8 +302,7 @@ fn builtin_hasattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> fn builtin_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - - vm.call_method(obj, "__hash__", vec![]) + vm._hash(obj).and_then(|v| Ok(vm.new_int(v))) } // builtin_help @@ -614,6 +612,16 @@ impl Printer for std::io::StdoutLock<'_> { } } +pub fn builtin_exit(exit_code_arg: OptionalArg, vm: &VirtualMachine) -> PyResult<()> { + if let OptionalArg::Present(exit_code_obj) = exit_code_arg { + match i32::try_from_object(&vm, exit_code_obj.clone()) { + Ok(code) => std::process::exit(code), + _ => println!("{}", vm.to_str(&exit_code_obj)?.as_str()), + } + } + std::process::exit(0); +} + pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { let stdout = io::stdout(); @@ -824,6 +832,8 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "tuple" => ctx.tuple_type(), "type" => ctx.type_type(), "zip" => ctx.zip_type(), + "exit" => ctx.new_rustfunc(builtin_exit), + "quit" => ctx.new_rustfunc(builtin_exit), "__import__" => ctx.new_rustfunc(builtin_import), // Constants diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 09bb7e70a..5df445d74 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -8,14 +8,16 @@ use num_bigint::BigInt; use num_complex::Complex64; use rustpython_parser::ast; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt; /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct CodeObject { pub instructions: Vec, + /// Jump targets. pub label_map: HashMap, pub locations: Vec, pub arg_names: Vec, // Names of positional arguments @@ -29,6 +31,7 @@ pub struct CodeObject { } bitflags! { + #[derive(Serialize, Deserialize)] pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; const HAS_KW_ONLY_DEFAULTS = 0x02; @@ -38,7 +41,7 @@ bitflags! { pub type Label = usize; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum NameScope { Local, NonLocal, @@ -46,7 +49,7 @@ pub enum NameScope { } /// A Single bytecode instruction. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Instruction { Import { name: String, @@ -187,14 +190,14 @@ pub enum Instruction { use self::Instruction::*; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum CallType { Positional(usize), Keyword(usize), Ex(bool), } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Constant { Integer { value: BigInt }, Float { value: f64 }, @@ -208,7 +211,7 @@ pub enum Constant { Ellipsis, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum ComparisonOperator { Greater, GreaterOrEqual, @@ -222,7 +225,7 @@ pub enum ComparisonOperator { IsNot, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum BinaryOperator { Power, Multiply, @@ -240,7 +243,7 @@ pub enum BinaryOperator { Or, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum UnaryOperator { Not, Invert, @@ -248,7 +251,7 @@ pub enum UnaryOperator { Plus, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Varargs { None, Unnamed, diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index b0bd4894f..ca20f2012 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,18 +1,25 @@ use crate::obj::objbool; -use crate::obj::objint; +use crate::pyhash; use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -use num_traits::ToPrimitive; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 /// And: http://code.activestate.com/recipes/578375/ -use std::collections::HashMap; +use std::collections::{hash_map::DefaultHasher, HashMap}; +use std::hash::{Hash, Hasher}; + +/// hash value of an object returned by __hash__ +type HashValue = pyhash::PyHash; +/// index calculated by resolving collision +type HashIndex = pyhash::PyHash; +/// entry index mapped in indices +type EntryIndex = usize; #[derive(Clone)] pub struct Dict { size: usize, - indices: HashMap, + indices: HashMap, entries: Vec>>, } @@ -28,7 +35,7 @@ impl Default for Dict { #[derive(Clone)] struct DictEntry { - hash: usize, + hash: HashValue, key: PyObjectRef, value: T, } @@ -40,12 +47,27 @@ pub struct DictSize { } impl Dict { - pub fn new() -> Self { - Dict { - size: 0, - indices: HashMap::new(), - entries: Vec::new(), - } + fn unchecked_push( + &mut self, + hash_index: HashIndex, + hash_value: HashValue, + key: &PyObjectRef, + value: T, + ) { + let entry = DictEntry { + hash: hash_value, + key: key.clone(), + value, + }; + let entry_index = self.entries.len(); + self.entries.push(Some(entry)); + self.indices.insert(hash_index, entry_index); + self.size += 1; + } + + fn unchecked_delete(&mut self, entry_index: EntryIndex) { + self.entries[entry_index] = None; + self.size -= 1; } /// Store a key @@ -65,29 +87,21 @@ impl Dict { hash_value, } => { // New key: - let entry = DictEntry { - hash: hash_value, - key: key.clone(), - value, - }; - let index = self.entries.len(); - self.entries.push(Some(entry)); - self.indices.insert(hash_index, index); - self.size += 1; + self.unchecked_push(hash_index, hash_value, key, value); Ok(()) } } } pub fn contains(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - if let LookupResult::Existing(_index) = self.lookup(vm, key)? { + if let LookupResult::Existing(_) = self.lookup(vm, key)? { Ok(true) } else { Ok(false) } } - fn unchecked_get(&self, index: usize) -> T { + fn unchecked_get(&self, index: EntryIndex) -> T { if let Some(entry) = &self.entries[index] { entry.value.clone() } else { @@ -112,9 +126,7 @@ impl Dict { /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { - if let LookupResult::Existing(index) = self.lookup(vm, key)? { - self.entries[index] = None; - self.size -= 1; + if self.delete_if_exists(vm, key)? { Ok(()) } else { let key_repr = vm.to_pystr(key)?; @@ -122,6 +134,31 @@ impl Dict { } } + pub fn delete_if_exists(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + if let LookupResult::Existing(entry_index) = self.lookup(vm, key)? { + self.unchecked_delete(entry_index); + Ok(true) + } else { + Ok(false) + } + } + + pub fn delete_or_insert( + &mut self, + vm: &VirtualMachine, + key: &PyObjectRef, + value: T, + ) -> PyResult<()> { + match self.lookup(vm, &key)? { + LookupResult::Existing(entry_index) => self.unchecked_delete(entry_index), + LookupResult::NewIndex { + hash_value, + hash_index, + } => self.unchecked_push(hash_index, hash_value, &key, value), + }; + Ok(()) + } + pub fn len(&self) -> usize { self.size } @@ -137,7 +174,7 @@ impl Dict { } } - pub fn next_entry(&self, position: &mut usize) -> Option<(&PyObjectRef, &T)> { + pub fn next_entry(&self, position: &mut EntryIndex) -> Option<(&PyObjectRef, &T)> { while *position < self.entries.len() { if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { *position += 1; @@ -152,11 +189,19 @@ impl Dict { position.size != self.size || self.entries.len() != position.entries_size } + pub fn keys<'a>(&'a self) -> Box + 'a> { + Box::new( + self.entries + .iter() + .filter_map(|v| v.as_ref().and_then(|v| Some(v.key.clone()))), + ) + } + /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash_value = calc_hash(vm, key)?; + let hash_value = collection_hash(vm, key)?; let perturb = hash_value; - let mut hash_index: usize = hash_value; + let mut hash_index: HashIndex = hash_value; loop { if self.indices.contains_key(&hash_index) { // Now we have an index, lets check the key. @@ -199,33 +244,46 @@ impl Dict { pub fn pop(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { if let LookupResult::Existing(index) = self.lookup(vm, key)? { let value = self.unchecked_get(index); - self.entries[index] = None; - self.size -= 1; + self.unchecked_delete(index); Ok(value) } else { let key_repr = vm.to_pystr(key)?; Err(vm.new_key_error(format!("Key not found: {}", key_repr))) } } + + pub fn pop_front(&mut self) -> Option<(PyObjectRef, T)> { + let mut entry_index = 0; + match self.next_entry(&mut entry_index) { + Some((key, value)) => { + let item = (key.clone(), value.clone()); + self.unchecked_delete(entry_index - 1); + Some(item) + } + None => None, + } + } } enum LookupResult { NewIndex { - hash_value: usize, - hash_index: usize, + hash_value: HashValue, + hash_index: HashIndex, }, // return not found, index into indices - Existing(usize), // Existing record, index into entries + Existing(EntryIndex), // Existing record, index into entries } -fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash = vm.call_method(key, "__hash__", vec![])?; - Ok(objint::get_value(&hash).to_usize().unwrap()) +fn collection_hash(vm: &VirtualMachine, object: &PyObjectRef) -> PyResult { + let raw_hash = vm._hash(object)?; + let mut hasher = DefaultHasher::new(); + raw_hash.hash(&mut hasher); + Ok(hasher.finish() as HashValue) } /// Invoke __eq__ on two keys fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { let result = vm._eq(key1.clone(), key2.clone())?; - Ok(objbool::get_value(&result)) + objbool::boolval(vm, result) } #[cfg(test)] @@ -235,7 +293,7 @@ mod tests { #[test] fn test_insert() { let mut vm = VirtualMachine::new(); - let mut dict = Dict::new(); + let mut dict = Dict::default(); assert_eq!(0, dict.len()); let key1 = vm.new_bool(true); diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index a8b753e1a..4f1f94744 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -4,13 +4,16 @@ use crate::obj::objtype; use crate::obj::objtype::PyClassRef; use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use std::fs::File; +use std::io::{BufRead, BufReader}; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let zelf = args.args[0].clone(); let msg = if args.args.len() > 1 { args.args[1].clone() } else { - vm.new_str("No msg".to_string()) + let empty_string = String::default(); + vm.new_str(empty_string) }; let traceback = vm.ctx.new_list(Vec::new()); vm.set_attr(&zelf, "msg", msg)?; @@ -18,7 +21,7 @@ fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } -// print excption chain +/// Print exception chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { let mut had_cause = false; if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { @@ -39,41 +42,71 @@ pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { print_exception_inner(vm, exc) } -// Print exception including traceback: +fn print_source_line(filename: String, lineno: usize) { + // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/master/Python/traceback.c#L393 + // TODO: support different encodings + let file = match File::open(filename) { + Ok(file) => file, + Err(_) => { + return; + } + }; + let file = BufReader::new(file); + + for (i, line) in file.lines().enumerate() { + if i + 1 == lineno { + if let Ok(line) = line { + // Indented with 4 spaces + println!(" {}", line.trim_start()); + } + return; + } + } +} + +/// Print exception occurrence location from traceback element +fn print_traceback_entry(vm: &VirtualMachine, tb_entry: &PyObjectRef) { + if objtype::isinstance(&tb_entry, &vm.ctx.tuple_type()) { + let location_attrs = objsequence::get_elements_tuple(&tb_entry); + let filename = if let Ok(x) = vm.to_str(&location_attrs[0]) { + x.value.clone() + } else { + "".to_string() + }; + + let lineno = if let Ok(x) = vm.to_str(&location_attrs[1]) { + x.value.clone() + } else { + "".to_string() + }; + + let obj_name = if let Ok(x) = vm.to_str(&location_attrs[2]) { + x.value.clone() + } else { + "".to_string() + }; + + println!( + r##" File "{}", line {}, in {}"##, + filename, lineno, obj_name + ); + print_source_line(filename, lineno.parse().unwrap()); + } else { + println!(" File ??"); + return; + } +} + +/// Print exception with traceback pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let mut elements = objsequence::get_elements(&tb).to_vec(); - elements.reverse(); - for element in elements.iter() { - if objtype::isinstance(&element, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&element); - let filename = if let Ok(x) = vm.to_str(&element[0]) { - x.value.clone() - } else { - "".to_string() - }; + let mut tb_entries = objsequence::get_elements_list(&tb).to_vec(); + tb_entries.reverse(); - let lineno = if let Ok(x) = vm.to_str(&element[1]) { - x.value.clone() - } else { - "".to_string() - }; - - let obj_name = if let Ok(x) = vm.to_str(&element[2]) { - x.value.clone() - } else { - "".to_string() - }; - - println!( - r##" File "{}", line {}, in {}"##, - filename, lineno, obj_name - ); - } else { - println!(" File ??"); - } + for exc_location in tb_entries.iter() { + print_traceback_entry(vm, exc_location); } } } else { @@ -100,8 +133,11 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { panic!("Error message must be set"); }; - let s = format!("{}: {}", exc.class().name, msg); - Ok(vm.new_str(s)) + let mut exc_repr = exc.class().name.clone(); + if !msg.is_empty() { + &exc_repr.push_str(&format!(": {}", msg)); + } + Ok(vm.new_str(exc_repr)) } #[derive(Debug)] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 5955660b6..88a1b31ad 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -23,6 +23,7 @@ use crate::pyobject::{ TypeProtocol, }; use crate::vm::VirtualMachine; +use indexmap::IndexMap; use itertools::Itertools; /* @@ -123,15 +124,15 @@ impl Scope { self.locals.iter().next().cloned() } - pub fn child_scope_with_locals(&self, locals: PyDictRef) -> Scope { + pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope { Scope { locals: self.locals.clone().insert(locals), globals: self.globals.clone(), } } - pub fn child_scope(&self, ctx: &PyContext) -> Scope { - self.child_scope_with_locals(ctx.new_dict()) + pub fn new_child_scope(&self, ctx: &PyContext) -> Scope { + self.new_child_scope_with_locals(ctx.new_dict()) } } @@ -172,8 +173,7 @@ impl NameProtocol for Scope { fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { self.locals .iter() - .skip(1) - .next() + .nth(1) .expect("no outer scope for non-local") .set_item(name, value, vm) .unwrap(); @@ -243,7 +243,7 @@ pub enum ExecutionResult { Yield(PyObjectRef), } -// A valid execution result, or an exception +/// A valid execution result, or an exception pub type FrameResult = Result, PyObjectRef>; impl Frame { @@ -285,9 +285,11 @@ impl Frame { Ok(Some(value)) => { break Ok(value); } + // Instruction raised an exception Err(exception) => { - // unwind block stack on exception and find any handlers. - // Add an entry in the traceback: + // 1. Extract traceback from exception's '__traceback__' attr. + // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. + // 3. Unwind block stack till appropriate handler is found. assert!(objtype::isinstance( &exception, &vm.ctx.exceptions.base_exception_type @@ -296,13 +298,12 @@ impl Frame { .get_attribute(exception.clone(), "__traceback__") .unwrap(); trace!("Adding to traceback: {:?} {:?}", traceback, lineno); - let pos = vm.ctx.new_tuple(vec![ + let raise_location = vm.ctx.new_tuple(vec![ vm.ctx.new_str(filename.clone()), vm.ctx.new_int(lineno.get_row()), vm.ctx.new_str(run_obj_name.clone()), ]); - objlist::PyListRef::try_from_object(vm, traceback)?.append(pos, vm); - // exception.__trace + objlist::PyListRef::try_from_object(vm, traceback)?.append(raise_location, vm); match self.unwind_exception(vm, exception) { None => {} Some(exception) => { @@ -333,7 +334,7 @@ impl Frame { ins2 } - // Execute a single instruction: + /// Execute a single instruction. fn execute_instruction(&self, vm: &VirtualMachine) -> FrameResult { let instruction = self.fetch_instruction(); { @@ -565,9 +566,7 @@ impl Frame { } = &block.typ { debug_assert!(end1 == end2); - - // call exit now with no exception: - self.with_exit(vm, &context_manager, None)?; + self.call_context_manager_exit_no_exception(vm, &context_manager)?; } else { unreachable!("Block stack is incorrect, expected a with block"); } @@ -650,22 +649,22 @@ impl Frame { // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm + let func_obj = vm .ctx .new_function(code_obj, scope, defaults, kw_only_defaults); let name = qualified_name.value.split('.').next_back().unwrap(); - vm.set_attr(&obj, "__name__", vm.new_str(name.to_string()))?; - vm.set_attr(&obj, "__qualname__", qualified_name)?; + vm.set_attr(&func_obj, "__name__", vm.new_str(name.to_string()))?; + vm.set_attr(&func_obj, "__qualname__", qualified_name)?; let module = self .scope .globals .get_item_option("__name__", vm)? .unwrap_or_else(|| vm.get_none()); - vm.set_attr(&obj, "__module__", module)?; - vm.set_attr(&obj, "__annotations__", annotations)?; + vm.set_attr(&func_obj, "__module__", module)?; + vm.set_attr(&func_obj, "__annotations__", annotations)?; - self.push_value(obj); + self.push_value(func_obj); Ok(None) } bytecode::Instruction::CallFunction { typ } => { @@ -674,7 +673,7 @@ impl Frame { let args: Vec = self.pop_multiple(*count); PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } bytecode::CallType::Keyword(count) => { @@ -697,7 +696,7 @@ impl Frame { .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { - vec![] + IndexMap::new() }; let args = self.pop_value(); let args = vm.extract_elements(&args)?; @@ -949,7 +948,7 @@ impl Frame { BlockType::With { context_manager, .. } => { - match self.with_exit(vm, &context_manager, None) { + match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { // __exit__ went wrong, @@ -976,7 +975,7 @@ impl Frame { } BlockType::With { context_manager, .. - } => match self.with_exit(vm, &context_manager, None) { + } => match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { panic!("Exception in with __exit__ {:?}", exc); @@ -1006,12 +1005,12 @@ impl Frame { end, context_manager, } => { - match self.with_exit(vm, &context_manager, Some(exc.clone())) { - Ok(exit_action) => { - match objbool::boolval(vm, exit_action) { - Ok(handle_exception) => { - if handle_exception { - // We handle the exception, so return! + match self.call_context_manager_exit(vm, &context_manager, exc.clone()) { + Ok(exit_result_obj) => { + match objbool::boolval(vm, exit_result_obj) { + // If __exit__ method returned True, suppress the exception and continue execution. + Ok(suppress_exception) => { + if suppress_exception { self.jump(end); return None; } else { @@ -1022,7 +1021,6 @@ impl Frame { return Some(exit_exc); } } - // if objtype::isinstance } Err(exit_exc) => { // TODO: what about original exception? @@ -1031,36 +1029,39 @@ impl Frame { } } BlockType::Loop { .. } => {} - // Exception was already poped on Raised. + // Exception was already popped on Raised. BlockType::ExceptHandler => {} } } Some(exc) } - fn with_exit( + fn call_context_manager_exit_no_exception( &self, vm: &VirtualMachine, context_manager: &PyObjectRef, - exc: Option, ) -> PyResult { - // Assume top of stack is __exit__ method: // TODO: do we want to put the exit call on the stack? - // let exit_method = self.pop_value(); - // let args = PyFuncArgs::default(); - // TODO: what happens when we got an error during handling exception? - let args = if let Some(exc) = exc { - let exc_type = exc.class().into_object(); - let exc_val = exc.clone(); - let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? - vec![exc_type, exc_val, exc_tb] - } else { - let exc_type = vm.ctx.none(); - let exc_val = vm.ctx.none(); - let exc_tb = vm.ctx.none(); - vec![exc_type, exc_val, exc_tb] - }; - vm.call_method(context_manager, "__exit__", args) + // TODO: what happens when we got an error during execution of __exit__? + vm.call_method( + context_manager, + "__exit__", + vec![vm.ctx.none(), vm.ctx.none(), vm.ctx.none()], + ) + } + + fn call_context_manager_exit( + &self, + vm: &VirtualMachine, + context_manager: &PyObjectRef, + exc: PyObjectRef, + ) -> PyResult { + // TODO: do we want to put the exit call on the stack? + // TODO: what happens when we got an error during execution of __exit__? + let exc_type = exc.class().into_object(); + let exc_val = exc.clone(); + let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? + vm.call_method(context_manager, "__exit__", vec![exc_type, exc_val, exc_tb]) } fn store_name( @@ -1131,7 +1132,7 @@ impl Frame { fn jump(&self, label: bytecode::Label) { let target_pc = self.code.label_map[&label]; - trace!("program counter from {:?} to {:?}", self.lasti, target_pc); + trace!("jump from {:?} to {:?}", self.lasti, target_pc); *self.lasti.borrow_mut() = target_pc; } @@ -1317,14 +1318,14 @@ impl Frame { || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { Ok(exception) - } else if let Ok(exception) = PyClassRef::try_from_object(vm, exception) { - if objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) { - let exception = vm.new_empty_exception(exception)?; + } else if let Ok(exc_type) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exc_type, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exc_type)?; Ok(exception) } else { let msg = format!( "Can only raise BaseException derived types, not {}", - exception + exc_type ); Err(vm.new_type_error(msg)) } diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs new file mode 100644 index 000000000..cc6eb85fe --- /dev/null +++ b/vm/src/frozen.rs @@ -0,0 +1,11 @@ +use std::collections::hash_map::HashMap; + +const HELLO: &str = "initialized = True +print(\"Hello world!\") +"; + +pub fn get_module_inits() -> HashMap { + let mut modules = HashMap::new(); + modules.insert("__hello__".to_string(), HELLO); + modules +} diff --git a/vm/src/function.rs b/vm/src/function.rs index 170c395f6..22802ac1f 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; use std::mem; use std::ops::RangeInclusive; +use indexmap::IndexMap; + use crate::obj::objtuple::PyTuple; use crate::obj::objtype::{isinstance, PyClassRef}; use crate::pyobject::{ @@ -17,7 +19,8 @@ use self::OptionalArg::*; #[derive(Debug, Default, Clone)] pub struct PyFuncArgs { pub args: Vec, - pub kwargs: Vec<(String, PyObjectRef)>, + // sorted map, according to https://www.python.org/dev/peps/pep-0468/ + pub kwargs: IndexMap, } /// Conversion from vector of python objects to function arguments. @@ -25,7 +28,7 @@ impl From> for PyFuncArgs { fn from(args: Vec) -> Self { PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } } @@ -34,7 +37,7 @@ impl From for PyFuncArgs { fn from(arg: PyObjectRef) -> Self { PyFuncArgs { args: vec![arg], - kwargs: vec![], + kwargs: IndexMap::new(), } } } @@ -58,9 +61,12 @@ impl FromArgs for PyFuncArgs { impl PyFuncArgs { pub fn new(mut args: Vec, kwarg_names: Vec) -> PyFuncArgs { - let mut kwargs = vec![]; - for name in kwarg_names.iter().rev() { - kwargs.push((name.clone(), args.pop().unwrap())); + // last `kwarg_names.len()` elements of args in order of appearance in the call signature + let kwarg_values = args.drain((args.len() - kwarg_names.len())..); + + let mut kwargs = IndexMap::new(); + for (name, value) in kwarg_names.iter().zip(kwarg_values) { + kwargs.insert(name.clone(), value); } PyFuncArgs { args, kwargs } } @@ -79,21 +85,14 @@ impl PyFuncArgs { } pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return arg_value.clone(); - } - } - default.clone() + self.kwargs + .get(key) + .cloned() + .unwrap_or_else(|| default.clone()) } pub fn get_optional_kwarg(&self, key: &str) -> Option { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return Some(arg_value.clone()); - } - } - None + self.kwargs.get(key).cloned() } pub fn get_optional_kwarg_with_type( @@ -132,19 +131,12 @@ impl PyFuncArgs { } pub fn take_keyword(&mut self, name: &str) -> Option { - // TODO: change kwarg representation so this scan isn't necessary - if let Some(index) = self - .kwargs - .iter() - .position(|(arg_name, _)| arg_name == name) - { - Some(self.kwargs.remove(index).1) - } else { - None - } + self.kwargs.remove(name) } - pub fn remaining_keyword<'a>(&'a mut self) -> impl Iterator + 'a { + pub fn remaining_keywords<'a>( + &'a mut self, + ) -> impl Iterator + 'a { self.kwargs.drain(..) } @@ -192,7 +184,10 @@ impl PyFuncArgs { given_args, ))) } else if !self.kwargs.is_empty() { - Err(vm.new_type_error(format!("Unexpected keyword argument {}", self.kwargs[0].0))) + Err(vm.new_type_error(format!( + "Unexpected keyword argument {}", + self.kwargs.keys().next().unwrap() + ))) } else { Ok(bound) } @@ -264,7 +259,7 @@ where { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { let mut kwargs = HashMap::new(); - for (name, value) in args.remaining_keyword() { + for (name, value) in args.remaining_keywords() { kwargs.insert(name, T::try_from_object(vm, value)?); } Ok(KwArgs(kwargs)) @@ -379,6 +374,17 @@ impl OptionalArg { Missing => f(), } } + + pub fn map_or_else(self, default: D, f: F) -> U + where + D: FnOnce() -> U, + F: FnOnce(T) -> U, + { + match self { + Present(value) => f(value), + Missing => default(), + } + } } impl FromArgs for OptionalArg @@ -538,7 +544,7 @@ pub fn single_or_tuple_any) -> PyResult>( match_class!(obj, obj @ T => predicate(obj), tuple @ PyTuple => { - for obj in tuple.elements.borrow().iter() { + for obj in tuple.elements.iter() { let inner_val = PyRef::::try_from_object(vm, obj.clone())?; if predicate(inner_val)? { return Ok(true); diff --git a/vm/src/import.rs b/vm/src/import.rs index 8445e65f9..0e8b7ba53 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -18,6 +18,8 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s // First, see if we already loaded the module: if let Ok(module) = sys_modules.get_item(module_name.to_string(), vm) { Ok(module) + } else if let Some(frozen) = vm.frozen.borrow().get(module_name) { + import_file(vm, module_name, "frozen".to_string(), frozen.to_string()) } else if let Some(make_module_func) = vm.stdlib_inits.borrow().get(module_name) { let module = make_module_func(vm); sys_modules.set_item(module_name, module.clone(), vm)?; @@ -67,7 +69,7 @@ pub fn import_file( fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result { let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap(); - let mut paths: Vec = objsequence::get_elements(&sys_path) + let mut paths: Vec = objsequence::get_elements_list(&sys_path) .iter() .map(|item| PathBuf::from(objstr::get_value(item))) .collect(); diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 8b14454fe..24f28709e 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -20,13 +20,6 @@ extern crate lexical; #[macro_use] extern crate log; // extern crate env_logger; -extern crate num_bigint; -extern crate num_complex; -extern crate num_integer; -extern crate num_traits; -extern crate serde; -extern crate serde_json; -extern crate statrs; extern crate rustpython_parser; #[macro_use] @@ -52,9 +45,11 @@ pub mod eval; mod exceptions; pub mod format; pub mod frame; +mod frozen; pub mod function; pub mod import; pub mod obj; +mod pyhash; pub mod pyobject; pub mod stdlib; mod symboltable; diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 333673985..c3267ff15 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -157,6 +157,19 @@ macro_rules! extend_class { } } +#[macro_export] +macro_rules! py_namespace { + ( $vm:expr, { $($name:expr => $value:expr),* $(,)* }) => { + { + let namespace = $vm.ctx.new_namespace(); + $( + $vm.set_attr(&namespace, $name, $value).unwrap(); + )* + namespace + } + } +} + /// Macro to match on the built-in class of a Python object. /// /// Like `match`, `match_class!` must be exhaustive, so a default arm with diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index c9401d8c4..ad7af74d5 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -23,6 +23,7 @@ pub mod objmap; pub mod objmappingproxy; pub mod objmemory; pub mod objmodule; +pub mod objnamespace; pub mod objnone; pub mod objobject; pub mod objproperty; diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index c1a999e45..8276a46f7 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -19,6 +19,7 @@ impl TryFromObject for bool { } } +/// Convert Python bool into Rust bool. pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { Ok(if let Ok(f) = vm.get_method(obj.clone(), "__bool__") { let bool_res = vm.invoke(f, PyFuncArgs::default())?; diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 8b09a53e9..51d608b09 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -135,7 +135,7 @@ impl PyByteArrayRef { } #[pymethod(name = "__hash__")] - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type: bytearray".to_string())) } @@ -166,6 +166,16 @@ impl PyByteArrayRef { self.inner.borrow().getitem(needle, vm) } + #[pymethod(name = "__setitem__")] + fn setitem( + self, + needle: Either, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.borrow_mut().setitem(needle, value, vm) + } + #[pymethod(name = "isalnum")] fn isalnum(self, vm: &VirtualMachine) -> PyResult { self.inner.borrow().isalnum(vm) @@ -330,7 +340,7 @@ impl PyByteArrayRef { let pos = bytes .iter() .position(|b| *b == x) - .ok_or(vm.new_value_error("value not found in bytearray".to_string()))?; + .ok_or_else(|| vm.new_value_error("value not found in bytearray".to_string()))?; bytes.remove(pos); diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs index ae318e628..32a58255f 100644 --- a/vm/src/obj/objbyteinner.rs +++ b/vm/src/obj/objbyteinner.rs @@ -2,6 +2,7 @@ use crate::obj::objint::PyIntRef; use crate::obj::objnone::PyNoneRef; use crate::obj::objslice::PySliceRef; use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; use crate::pyobject::Either; use crate::pyobject::PyRef; use crate::pyobject::PyValue; @@ -12,17 +13,12 @@ use core::ops::Range; use num_bigint::BigInt; use crate::function::OptionalArg; - -use crate::vm::VirtualMachine; - use crate::pyobject::{PyResult, TypeProtocol}; - -use crate::obj::objstr::{PyString, PyStringRef}; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; +use crate::vm::VirtualMachine; use super::objint; use super::objsequence::{is_valid_slice_arg, PySliceableSequence}; +use super::objstr::{PyString, PyStringRef}; use crate::obj::objint::PyInt; use num_integer::Integer; @@ -78,24 +74,33 @@ pub struct ByteInnerNewOptions { encoding: OptionalArg, } +//same algorithm as cpython +pub fn normalize_encoding(encoding: &str) -> String { + let mut res = String::new(); + let mut punct = false; + + for c in encoding.chars() { + if c.is_alphanumeric() || c == '.' { + if punct && !res.is_empty() { + res.push('_') + } + res.push(c.to_ascii_lowercase()); + punct = false; + } else { + punct = true; + } + } + res +} + impl ByteInnerNewOptions { pub fn get_value(self, vm: &VirtualMachine) -> PyResult { // First handle bytes(string, encoding[, errors]) if let OptionalArg::Present(enc) = self.encoding { if let OptionalArg::Present(eval) = self.val_option { if let Ok(input) = eval.downcast::() { - let encoding = enc.as_str(); - if encoding.to_lowercase() == "utf8" || encoding.to_lowercase() == "utf-8" - // TODO: different encoding - { - return Ok(PyByteInner { - elements: input.value.as_bytes().to_vec(), - }); - } else { - return Err( - vm.new_value_error(format!("unknown encoding: {}", encoding)), //should be lookup error - ); - } + let inner = PyByteInner::from_string(&input.value, enc.as_str(), vm)?; + return Ok(inner); } else { return Err(vm.new_type_error("encoding without a string argument".to_string())); } @@ -315,6 +320,20 @@ impl ByteInnerSplitlinesOptions { } impl PyByteInner { + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + let normalized = normalize_encoding(encoding); + if normalized == "utf_8" || normalized == "utf8" || normalized == "u8" { + Ok(PyByteInner { + elements: value.as_bytes().to_vec(), + }) + } else { + // TODO: different encoding + Err( + vm.new_value_error(format!("unknown encoding: {}", encoding)), // should be lookup error + ) + } + } + pub fn repr(&self) -> PyResult { let mut res = String::with_capacity(self.elements.len()); for i in self.elements.iter() { @@ -379,10 +398,8 @@ impl PyByteInner { } } - pub fn hash(&self) -> usize { - let mut hasher = DefaultHasher::new(); - self.elements.hash(&mut hasher); - hasher.finish() as usize + pub fn hash(&self) -> pyhash::PyHash { + pyhash::hash_value(&self.elements) } pub fn add(&self, other: PyByteInner) -> Vec { @@ -432,6 +449,70 @@ impl PyByteInner { } } + fn setindex(&mut self, int: PyIntRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + let result = match_class!(object, + i @ PyInt => { + if let Some(value) = i.as_bigint().to_u8() { + Ok(value) + }else{ + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } + }, + _ => {Err(vm.new_type_error("an integer is required".to_string()))} + ); + let value = result?; + self.elements[idx] = value; + Ok(vm.new_int(value)) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + fn setslice( + &mut self, + slice: PySliceRef, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let sec = match PyIterable::try_from_object(vm, object.clone()) { + Ok(sec) => { + let items: Result, _> = sec.iter(vm)?.collect(); + Ok(items? + .into_iter() + .map(|obj| u8::try_from_object(vm, obj)) + .collect::>>()?) + } + _ => match_class!(object, + i @ PyMemoryView => { + Ok(i.get_obj_value().unwrap()) + }, + _ => Err(vm.new_index_error( + "can assign only bytes, buffers, or iterables of ints in range(0, 256)" + .to_string()))), + }; + let items = sec?; + let range = self + .elements + .get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); + self.elements.splice(range, items); + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) + } + + pub fn setitem( + &mut self, + needle: Either, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match needle { + Either::A(int) => self.setindex(int, object, vm), + Either::B(slice) => self.setslice(slice, object, vm), + } + } + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { Ok(vm.new_bool( !self.elements.is_empty() @@ -692,7 +773,7 @@ impl PyByteInner { Either::A(byte) => byte.elements, Either::B(tuple) => { let mut flatten = vec![]; - for v in objsequence::get_elements(tuple.as_object()).to_vec() { + for v in objsequence::get_elements_tuple(tuple.as_object()).to_vec() { flatten.extend(PyByteInner::try_from_object(vm, v)?.elements) } flatten diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 102c70d4f..33016e511 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,8 +1,8 @@ use crate::obj::objint::PyIntRef; - use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStringRef; use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; use crate::pyobject::Either; use crate::vm::VirtualMachine; @@ -45,6 +45,13 @@ impl PyBytes { inner: PyByteInner { elements }, } } + + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + Ok(PyBytes { + inner: PyByteInner::from_string(value, encoding, vm)?, + }) + } + pub fn get_value(&self) -> &[u8] { &self.inner.elements } @@ -125,7 +132,7 @@ impl PyBytesRef { } #[pymethod(name = "__hash__")] - fn hash(self, _vm: &VirtualMachine) -> usize { + fn hash(self, _vm: &VirtualMachine) -> pyhash::PyHash { self.inner.hash() } diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index 60549cd3a..51a73d614 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -34,6 +34,7 @@ impl PyValue for PyCode { } impl PyCodeRef { + #[allow(clippy::new_ret_no_self)] fn new(_cls: PyClassRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("Cannot directly create code object".to_string())) } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 677763213..7a6df8705 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -2,6 +2,7 @@ use num_complex::Complex64; use num_traits::Zero; use crate::function::OptionalArg; +use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, }; @@ -125,12 +126,12 @@ impl PyComplex { #[pymethod(name = "__float__")] fn float(&self, vm: &VirtualMachine) -> PyResult { - return Err(vm.new_type_error(String::from("Can't convert complex to float"))); + Err(vm.new_type_error(String::from("Can't convert complex to float"))) } #[pymethod(name = "__int__")] fn int(&self, vm: &VirtualMachine) -> PyResult { - return Err(vm.new_type_error(String::from("Can't convert complex to int"))); + Err(vm.new_type_error(String::from("Can't convert complex to int"))) } #[pymethod(name = "__mul__")] @@ -248,4 +249,12 @@ impl PyComplex { let value = Complex64::new(real, imag); PyComplex { value }.into_ref_with_type(vm, cls) } + + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + let re_hash = pyhash::hash_float(self.value.re); + let im_hash = pyhash::hash_float(self.value.im); + + re_hash + im_hash * pyhash::IMAG + } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 87cf3010b..87e4b1919 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -264,15 +264,11 @@ impl PyDictRef { fn popitem(self, vm: &VirtualMachine) -> PyResult { let mut entries = self.entries.borrow_mut(); - let (key, value) = match entries.next_entry(&mut 0) { - Some((key, value)) => (key.clone(), value.clone()), - None => { - return Err(vm.new_key_error("popitem(): dictionary is empty".to_string())); - } - }; - - entries.delete(vm, &key)?; - Ok(vm.ctx.new_tuple(vec![key, value])) + if let Some((key, value)) = entries.pop_front() { + Ok(vm.ctx.new_tuple(vec![key, value])) + } else { + Err(vm.new_key_error("popitem(): dictionary is empty".to_string())) + } } /// Take a python dictionary and convert it to attributes. @@ -298,7 +294,7 @@ impl PyDictRef { Ok(PyDict { entries }.into_ref(vm)) } - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index add61a684..2d6287d7b 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -3,15 +3,18 @@ use super::objint; use super::objstr; use super::objtype; use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; +use hexf; use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; -use num_traits::{ToPrimitive, Zero}; +use num_traits::{float::Float, ToPrimitive, Zero}; /// Convert a string or number to a floating point number, if possible. #[pyclass(name = "float")] @@ -297,6 +300,11 @@ impl PyFloat { ) } + #[pymethod(name = "__pos__")] + fn pos(&self, _vm: &VirtualMachine) -> f64 { + self.value + } + #[pymethod(name = "__neg__")] fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value @@ -429,6 +437,11 @@ impl PyFloat { zelf } + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_float(self.value) + } + #[pyproperty(name = "real")] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { zelf @@ -467,6 +480,70 @@ impl PyFloat { let denom = vm.ctx.new_int(ratio.denom().clone()); Ok(vm.ctx.new_tuple(vec![numer, denom])) } + + #[pymethod] + fn fromhex(repr: PyStringRef, vm: &VirtualMachine) -> PyResult { + hexf::parse_hexf64(&repr.value, false).or_else(|_| match repr.value.as_ref() { + "nan" => Ok(std::f64::NAN), + "inf" => Ok(std::f64::INFINITY), + "-inf" => Ok(std::f64::NEG_INFINITY), + _ => Err(vm.new_value_error("invalid hexadecimal floating-point string".to_string())), + }) + } + + #[pymethod] + fn hex(&self, _vm: &VirtualMachine) -> String { + to_hex(self.value) + } +} + +fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt), + value if value.is_infinite() => format!("{}inf", sign_fmt), + value if value.is_nan() => "nan".to_string(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}0x{:x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::thread_rng().gen::<[u64; 1]>(); + let f = f64::from_bits(bytes[0]); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } +} + +pub fn ufrexp(value: f64) -> (f64, i32) { + if 0.0 == value { + (0.0, 0i32) + } else { + let bits = value.to_bits(); + let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; + let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52); + (f64::from_bits(mantissa_bits), exponent) + } } pub type PyFloatRef = PyRef; diff --git a/vm/src/obj/objframe.rs b/vm/src/obj/objframe.rs index b4ed15f74..6669e3bb9 100644 --- a/vm/src/obj/objframe.rs +++ b/vm/src/obj/objframe.rs @@ -18,6 +18,7 @@ pub fn init(context: &PyContext) { } impl FrameRef { + #[allow(clippy::new_ret_no_self)] fn new(_class: FrameRef, vm: &VirtualMachine) -> PyResult { Err(vm.new_type_error("Cannot directly create frame object".to_string())) } diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 403fd08ef..2b703d30e 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,5 +1,4 @@ use std::fmt; -use std::hash::{Hash, Hasher}; use num_bigint::{BigInt, Sign}; use num_integer::Integer; @@ -7,6 +6,7 @@ use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -14,6 +14,7 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use super::objbyteinner::PyByteInner; +use super::objbytes::PyBytes; use super::objstr::{PyString, PyStringRef}; use super::objtype; use crate::obj::objtype::PyClassRef; @@ -111,8 +112,9 @@ impl_try_from_object_int!( (u64, to_u64), ); +#[allow(clippy::collapsible_if)] fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { - Ok(if int2.value.is_negative() { + let result = if int2.value.is_negative() { let v1 = int1.float(vm)?; let v2 = int2.float(vm)?; vm.ctx.new_float(v1.pow(v2)) @@ -132,7 +134,8 @@ fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { // practically, exp over u64 is not possible to calculate anyway vm.ctx.not_implemented() } - }) + }; + Ok(result) } #[pyimpl] @@ -400,10 +403,11 @@ impl PyInt { } #[pymethod(name = "__hash__")] - fn hash(&self, _vm: &VirtualMachine) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() + pub fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + match self.value.to_i64() { + Some(value) => (value % pyhash::MODULUS as i64), + None => (&self.value % pyhash::MODULUS).to_i64().unwrap(), + } } #[pymethod(name = "__abs__")] @@ -492,6 +496,7 @@ impl PyInt { } #[pymethod] + #[allow(clippy::match_bool)] fn from_bytes( bytes: PyByteInner, byteorder: PyStringRef, @@ -526,7 +531,75 @@ impl PyInt { } Ok(x) } + #[pymethod] + #[allow(clippy::match_bool)] + fn to_bytes( + &self, + length: PyIntRef, + byteorder: PyStringRef, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let mut signed = false; + let value = self.as_bigint(); + for (key, value) in kwargs.into_iter() { + if key == "signed" { + signed = match_class!(value, + b @ PyInt => !b.as_bigint().is_zero(), + _ => false, + ); + } + } + if value.sign() == Sign::Minus && !signed { + return Err(vm.new_overflow_error("can't convert negative int to unsigned".to_string())); + } + let byte_len; + if let Some(temp) = length.as_bigint().to_usize() { + byte_len = temp; + } else { + return Err(vm.new_value_error("length parameter is illegal".to_string())); + } + + let mut origin_bytes = match byteorder.value.as_str() { + "big" => match signed { + true => value.to_signed_bytes_be(), + false => value.to_bytes_be().1, + }, + "little" => match signed { + true => value.to_signed_bytes_le(), + false => value.to_bytes_le().1, + }, + _ => { + return Err( + vm.new_value_error("byteorder must be either 'little' or 'big'".to_string()) + ); + } + }; + let origin_len = origin_bytes.len(); + if origin_len > byte_len { + return Err(vm.new_value_error("int too big to convert".to_string())); + } + + let mut append_bytes = match value.sign() { + Sign::Minus => vec![255u8; byte_len - origin_len], + _ => vec![0u8; byte_len - origin_len], + }; + let mut bytes = vec![]; + match byteorder.value.as_str() { + "big" => { + bytes = append_bytes; + bytes.append(&mut origin_bytes); + } + "little" => { + bytes = origin_bytes; + bytes.append(&mut append_bytes); + } + _ => (), + } + + Ok(PyBytes::new(bytes)) + } #[pyproperty] fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { zelf diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 6b00b6828..77c1b4c9f 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -17,8 +17,8 @@ use super::objbool; //use super::objint; use super::objiter; use super::objsequence::{ - get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - SequenceIndex, + get_elements_cell, get_elements_list, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, + seq_mul, SequenceIndex, }; use super::objslice::PySliceRef; use super::objtype; @@ -129,7 +129,7 @@ impl PyListRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let e1 = self.elements.borrow(); - let e2 = get_elements(&other); + let e2 = get_elements_list(&other); let elements = e1.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_list(elements)) } else { @@ -141,7 +141,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { self.elements .borrow_mut() - .extend_from_slice(&get_elements(&other)); + .extend_from_slice(&get_elements_list(&other)); Ok(self.into_object()) } else { Ok(vm.ctx.not_implemented()) @@ -387,7 +387,7 @@ impl PyListRef { Ok(s) } - fn hash(self, vm: &VirtualMachine) -> PyResult { + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { Err(vm.new_type_error("unhashable type".to_string())) } @@ -490,7 +490,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_equal(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -501,7 +501,7 @@ impl PyListRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_lt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -512,7 +512,7 @@ impl PyListRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_gt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -523,7 +523,7 @@ impl PyListRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_ge(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -534,7 +534,7 @@ impl PyListRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_le(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 88040ae14..8c2af26f0 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,18 +1,37 @@ use crate::obj::objbyteinner::try_as_byte; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass(name = "memoryview")] +#[derive(Debug)] +pub struct PyMemoryView { + obj_ref: PyObjectRef, +} + pub type PyMemoryViewRef = PyRef; -#[derive(Debug)] -pub struct PyMemoryView { - obj: PyObjectRef, -} - +#[pyimpl] impl PyMemoryView { pub fn get_obj_value(&self) -> Option> { - try_as_byte(&self.obj) + try_as_byte(&self.obj_ref) + } + + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + bytes_object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + PyMemoryView { + obj_ref: bytes_object.clone(), + } + .into_ref_with_type(vm, cls) + } + + #[pyproperty] + fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef { + self.obj_ref.clone() } } @@ -22,18 +41,6 @@ impl PyValue for PyMemoryView { } } -pub fn new_memory_view( - cls: PyClassRef, - bytes_object: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult { - vm.set_attr(cls.as_object(), "obj", bytes_object.clone())?; - PyMemoryView { obj: bytes_object }.into_ref_with_type(vm, cls) -} - pub fn init(ctx: &PyContext) { - let memoryview_type = &ctx.memoryview_type; - extend_class!(ctx, memoryview_type, { - "__new__" => ctx.new_rustfunc(new_memory_view) - }); + PyMemoryView::extend_class(ctx, &ctx.memoryview_type) } diff --git a/vm/src/obj/objnamespace.rs b/vm/src/obj/objnamespace.rs new file mode 100644 index 000000000..f8bfc9369 --- /dev/null +++ b/vm/src/obj/objnamespace.rs @@ -0,0 +1,32 @@ +use crate::function::KwArgs; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +/// A simple attribute-based namespace. +/// +/// SimpleNamespace(**kwargs) +#[pyclass(name = "SimpleNamespace")] +#[derive(Debug)] +pub struct PyNamespace; + +impl PyValue for PyNamespace { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.namespace_type() + } +} + +#[pyimpl] +impl PyNamespace { + #[pymethod(name = "__init__")] + fn init(zelf: PyRef, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult<()> { + for (name, value) in kwargs.into_iter() { + vm.set_attr(zelf.as_object(), name, value)?; + } + Ok(()) + } +} + +pub fn init(context: &PyContext) { + PyNamespace::extend_class(context, &context.namespace_type); +} diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 31e52fc99..77830a402 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -5,6 +5,7 @@ use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TryFromObject, TypeProtocol, @@ -55,8 +56,8 @@ fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> Py vm.ctx.not_implemented() } -fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> u64 { - zelf.get_id() as u64 +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> pyhash::PyHash { + zelf.get_id() as pyhash::PyHash } fn object_setattr( diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index af2034370..d4226d51c 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -348,18 +348,19 @@ pub fn get_elements_cell<'a>(obj: &'a PyObjectRef) -> &'a RefCell() { return &list.elements; } - if let Some(tuple) = obj.payload::() { - return &tuple.elements; + panic!("Cannot extract elements from non-sequence"); +} + +pub fn get_elements_list<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { + if let Some(list) = obj.payload::() { + return list.elements.borrow(); } panic!("Cannot extract elements from non-sequence"); } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - if let Some(list) = obj.payload::() { - return list.elements.borrow(); - } +pub fn get_elements_tuple<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow(); + return &tuple.elements; } panic!("Cannot extract elements from non-sequence"); } @@ -368,9 +369,6 @@ pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut() { return list.elements.borrow_mut(); } - if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow_mut(); - } panic!("Cannot extract elements from non-sequence"); } diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b4f3bfabc..b7f0276ce 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -3,29 +3,38 @@ */ use std::cell::{Cell, RefCell}; -use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; -use std::hash::{Hash, Hasher}; +use crate::dictdatatype; use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; -use super::objbool; -use super::objint; use super::objlist::PyListIterator; use super::objtype; use super::objtype::PyClassRef; +pub type SetContentType = dictdatatype::Dict<()>; + +/// set() -> new empty set object +/// set(iterable) -> new set object +/// +/// Build an unordered collection of unique elements. +#[pyclass] #[derive(Default)] pub struct PySet { inner: RefCell, } pub type PySetRef = PyRef; +/// frozenset() -> empty frozenset object +/// frozenset(iterable) -> frozenset object +/// +/// Build an immutable unordered collection of unique elements. +#[pyclass] #[derive(Default)] pub struct PyFrozenSet { inner: PySetInner, @@ -60,44 +69,41 @@ impl PyValue for PyFrozenSet { #[derive(Default, Clone)] struct PySetInner { - elements: HashMap, + content: SetContentType, } impl PySetInner { - fn new(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - for item in iterable.iter(vm)? { - insert_into_set(vm, &mut elements, &item?)?; - } - elements - } - }; + fn new(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut set = PySetInner::default(); + for item in iterable.iter(vm)? { + set.add(&item?, vm)?; + } + Ok(set) + } - Ok(PySetInner { elements }) + fn from_arg(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(iterable) = iterable { + Self::new(iterable, vm) + } else { + Ok(PySetInner::default()) + } } fn len(&self) -> usize { - self.elements.len() + self.content.len() } + fn copy(&self) -> PySetInner { PySetInner { - elements: self.elements.clone(), + content: self.content.clone(), } } - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for element in self.elements.iter() { - let value = vm._eq(needle.clone(), element.1.clone())?; - if objbool::get_value(&value) { - return Ok(true); - } - } - Ok(false) + fn contains(&self, needle: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.contains(vm, needle) } + #[inline] fn _compare_inner( &self, other: &PySetInner, @@ -105,26 +111,13 @@ impl PySetInner { swap: bool, vm: &VirtualMachine, ) -> PyResult { - let get_zelf = |swap: bool| -> &PySetInner { - if swap { - other - } else { - self - } - }; - let get_other = |swap: bool| -> &PySetInner { - if swap { - self - } else { - other - } - }; + let (zelf, other) = if swap { (other, self) } else { (self, other) }; - if size_func(get_zelf(swap).len(), get_other(swap).len()) { + if size_func(zelf.len(), other.len()) { return Ok(vm.new_bool(false)); } - for element in get_other(swap).elements.iter() { - if !get_zelf(swap).contains(element.1.clone(), vm)? { + for key in other.content.keys() { + if !zelf.contains(&key, vm)? { return Ok(vm.new_bool(false)); } } @@ -177,55 +170,60 @@ impl PySetInner { } fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.clone(); + let mut set = self.clone(); for item in other.iter(vm)? { - insert_into_set(vm, &mut elements, &item?)?; + set.add(&item?, vm)?; } - Ok(PySetInner { elements }) + Ok(set) } fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = HashMap::new(); + let mut set = PySetInner::default(); for item in other.iter(vm)? { let obj = item?; - if self.contains(obj.clone(), vm)? { - insert_into_set(vm, &mut elements, &obj)?; + if self.contains(&obj, vm)? { + set.add(&obj, vm)?; } } - Ok(PySetInner { elements }) + Ok(set) } fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.clone(); + let mut set = self.copy(); for item in other.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { - remove_from_set(vm, &mut elements, &obj)?; - } + set.content.delete_if_exists(vm, &item?)?; } - Ok(PySetInner { elements }) + Ok(set) } fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { let mut new_inner = self.clone(); for item in other.iter(vm)? { - let obj = item?; - if !self.contains(obj.clone(), vm)? { - new_inner.add(&obj, vm)?; - } else { - new_inner.remove(&obj, vm)?; - } + new_inner.content.delete_or_insert(vm, &item?, ())? } Ok(new_inner) } + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in other.iter(vm)? { + if !self.contains(&item?, vm)? { + return Ok(false); + } + } + Ok(true) + } + + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let other_set = PySetInner::new(other, vm)?; + self.le(&other_set, vm) + } + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { for item in other.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { + if self.contains(&item?, vm)? { return Ok(false); } } @@ -233,7 +231,7 @@ impl PySetInner { } fn iter(&self, vm: &VirtualMachine) -> PyListIterator { - let items = self.elements.values().cloned().collect(); + let items = self.content.keys().collect(); let set_list = vm.ctx.new_list(items); PyListIterator { position: Cell::new(0), @@ -243,228 +241,236 @@ impl PySetInner { fn repr(&self, vm: &VirtualMachine) -> PyResult { let mut str_parts = vec![]; - for elem in self.elements.values() { - let part = vm.to_repr(elem)?; + for key in self.content.keys() { + let part = vm.to_repr(&key)?; str_parts.push(part.value.clone()); } Ok(format!("{{{}}}", str_parts.join(", "))) } - fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - insert_into_set(vm, &mut self.elements, &item) + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.insert(vm, item, ()) } - fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - remove_from_set(vm, &mut self.elements, &item) + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.delete(vm, item) } - fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, &mut self.elements, &item, &discard) + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.delete_if_exists(vm, item) } fn clear(&mut self) { - self.elements.clear(); + self.content.clear() } fn pop(&mut self, vm: &VirtualMachine) -> PyResult { - let elements = &mut self.elements; - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), + if let Some((key, _)) = self.content.pop_front() { + Ok(key) + } else { + Err(vm.new_key_error("pop from an empty set".to_string())) } } - fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { for item in iterable.iter(vm)? { - insert_into_set(vm, &mut self.elements, &item?)?; + self.add(&item?, vm)?; } - Ok(vm.get_none()) + Ok(()) } - fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { let temp_inner = self.copy(); self.clear(); for item in iterable.iter(vm)? { let obj = item?; - if temp_inner.contains(obj.clone(), vm)? { + if temp_inner.contains(&obj, vm)? { self.add(&obj, vm)?; } } - Ok(vm.get_none()) + Ok(()) } - fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { for item in iterable.iter(vm)? { - let obj = item?; - if self.contains(obj.clone(), vm)? { - self.remove(&obj, vm)?; - } + self.content.delete_if_exists(vm, &item?)?; } - Ok(vm.get_none()) + Ok(()) } fn symmetric_difference_update( &mut self, iterable: PyIterable, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult<()> { for item in iterable.iter(vm)? { - let obj = item?; - if !self.contains(obj.clone(), vm)? { - self.add(&obj, vm)?; - } else { - self.remove(&obj, vm)?; - } + self.content.delete_or_insert(vm, &item?, ())?; } - Ok(vm.get_none()) + Ok(()) } } -impl PySetRef { +macro_rules! try_set_inner { + ($vm:expr, $other:expr, $op:expr) => { + match_class!($other, + set @ PySet => $op(&*set.inner.borrow()), + frozen @ PyFrozenSet => $op(&frozen.inner), + _ => Ok($vm.ctx.not_implemented()), + ); + }; +} + +#[pyimpl] +impl PySet { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PySet { - inner: RefCell::new(PySetInner::new(iterable, vm)?), + Self { + inner: RefCell::new(PySetInner::from_arg(iterable, vm)?), } .into_ref_with_type(vm, cls) } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.inner.borrow().len() } - fn copy(self, _vm: &VirtualMachine) -> PySet { - PySet { + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { inner: RefCell::new(self.inner.borrow().copy()), } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow().contains(needle, vm) + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(&needle, vm) } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().eq(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().eq(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().eq(other, vm)) } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().ge(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().ge(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().ge(other, vm)) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().gt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().gt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().gt(other, vm)) } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().le(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().le(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().le(other, vm)) } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.borrow().lt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.borrow().lt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().lt(other, vm)) } - fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().union(other, vm)?), - }, - PySet::class(vm), - None, - )) + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().union(other, vm)?), + }) } - fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), - }, - PySet::class(vm), - None, - )) + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), + }) } - fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().difference(other, vm)?), - }, - PySet::class(vm), - None, - )) + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), + }) } - fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PySet { - inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), - }, - PySet::class(vm), - None, - )) + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), + }) } - fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issuperset(other, vm) + } + + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow().isdisjoint(other, vm) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } - fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } - fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } - fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.borrow().iter(vm) } - fn repr(self, vm: &VirtualMachine) -> PyResult { - let inner = self.inner.borrow(); + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = zelf.inner.borrow(); let s = if inner.len() == 0 { "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { inner.repr(vm)? } else { "set(...)".to_string() @@ -472,210 +478,245 @@ impl PySetRef { Ok(vm.new_str(s)) } - fn add(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().add(&item, vm) + #[pymethod] + pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().add(&item, vm)?; + Ok(()) } - fn remove(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.inner.borrow_mut().remove(&item, vm) } - fn discard(self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().discard(&item, vm) + #[pymethod] + fn discard(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().discard(&item, vm)?; + Ok(()) } - fn clear(self, _vm: &VirtualMachine) { + #[pymethod] + fn clear(&self, _vm: &VirtualMachine) { self.inner.borrow_mut().clear() } - fn pop(self, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().pop(vm) } - fn ior(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner.borrow_mut().update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + #[pymethod(name = "__ior__")] + fn ior(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner.borrow_mut().update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) } - fn update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().update(iterable, vm)?; Ok(vm.get_none()) } - fn intersection_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn intersection_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().intersection_update(iterable, vm)?; Ok(vm.get_none()) } - fn iand(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__iand__")] + fn iand(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .intersection_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) } - fn difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.borrow_mut().difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn isub(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__isub__")] + fn isub(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .difference_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) } - fn symmetric_difference_update(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn symmetric_difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner .borrow_mut() .symmetric_difference_update(iterable, vm)?; Ok(vm.get_none()) } - fn ixor(self, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { - self.inner + #[pymethod(name = "__ixor__")] + fn ixor(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner .borrow_mut() .symmetric_difference_update(iterable.iterable, vm)?; - Ok(self.as_object().clone()) + Ok(zelf.as_object().clone()) + } + + #[pymethod(name = "__hash__")] + fn hash(&self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type".to_string())) } } -impl PyFrozenSetRef { +#[pyimpl] +impl PyFrozenSet { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - PyFrozenSet { - inner: PySetInner::new(iterable, vm)?, + Self { + inner: PySetInner::from_arg(iterable, vm)?, } .into_ref_with_type(vm, cls) } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.inner.len() } - fn copy(self, _vm: &VirtualMachine) -> PyFrozenSet { - PyFrozenSet { + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { inner: self.inner.copy(), } } - fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner.contains(needle, vm) + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.contains(&needle, vm) } - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.eq(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.eq(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.eq(other, vm)) } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.ge(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.ge(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.ge(other, vm)) } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.gt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.gt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.gt(other, vm)) } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.le(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.le(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.le(other, vm)) } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match_class!(other, - set @ PySet => self.inner.lt(&set.inner.borrow(), vm), - frozen @ PyFrozenSet => self.inner.lt(&frozen.inner, vm), - other => {return Err(vm.new_type_error(format!("{} is not a subtype of set or frozenset", other.class())));}, - ) + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.lt(other, vm)) } - fn union(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.union(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.union(other, vm)?, + }) } - fn intersection(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.intersection(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.intersection(other, vm)?, + }) } - fn difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.difference(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.difference(other, vm)?, + }) } - fn symmetric_difference(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new( - PyFrozenSet { - inner: self.inner.symmetric_difference(other, vm)?, - }, - PyFrozenSet::class(vm), - None, - )) + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.symmetric_difference(other, vm)?, + }) } - fn isdisjoint(self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issuperset(other, vm) + } + + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { self.inner.isdisjoint(other, vm) } - fn or(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.union(other.iterable, vm) } - fn and(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.intersection(other.iterable, vm) } - fn sub(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.difference(other.iterable, vm) } - fn xor(self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { self.symmetric_difference(other.iterable, vm) } - fn iter(self, vm: &VirtualMachine) -> PyListIterator { + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { self.inner.iter(vm) } - fn repr(self, vm: &VirtualMachine) -> PyResult { - let inner = &self.inner; + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = &zelf.inner; let s = if inner.len() == 0 { "frozenset()".to_string() - } else if let Some(_guard) = ReprGuard::enter(self.as_object()) { + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { format!("frozenset({})", inner.repr(vm)?) } else { "frozenset(...)".to_string() @@ -684,60 +725,6 @@ impl PyFrozenSetRef { } } -fn perform_action_with_hash( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, - f: &Fn(&VirtualMachine, &mut HashMap, u64, &PyObjectRef) -> PyResult, -) -> PyResult { - let hash: PyObjectRef = vm.call_method(item, "__hash__", vec![])?; - - let hash_value = objint::get_value(&hash); - let mut hasher = DefaultHasher::new(); - hash_value.hash(&mut hasher); - let key = hasher.finish(); - f(vm, elements, key, item) -} - -fn insert_into_set( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, -) -> PyResult { - fn insert( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - elements.insert(key, value.clone()); - Ok(vm.get_none()) - } - perform_action_with_hash(vm, elements, item, &insert) -} - -fn remove_from_set( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, -) -> PyResult { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, elements, item, &remove) -} - struct SetIterable { iterable: PyIterable, } @@ -759,86 +746,7 @@ impl TryFromObject for SetIterable { } } -fn set_hash(_zelf: PySetRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("unhashable type".to_string())) -} - pub fn init(context: &PyContext) { - let set_type = &context.set_type; - - let set_doc = "set() -> new empty set object\n\ - set(iterable) -> new set object\n\n\ - Build an unordered collection of unique elements."; - - extend_class!(context, set_type, { - "__hash__" => context.new_rustfunc(set_hash), - "__contains__" => context.new_rustfunc(PySetRef::contains), - "__len__" => context.new_rustfunc(PySetRef::len), - "__new__" => context.new_rustfunc(PySetRef::new), - "__repr__" => context.new_rustfunc(PySetRef::repr), - "__eq__" => context.new_rustfunc(PySetRef::eq), - "__ge__" => context.new_rustfunc(PySetRef::ge), - "__gt__" => context.new_rustfunc(PySetRef::gt), - "__le__" => context.new_rustfunc(PySetRef::le), - "__lt__" => context.new_rustfunc(PySetRef::lt), - "issubset" => context.new_rustfunc(PySetRef::le), - "issuperset" => context.new_rustfunc(PySetRef::ge), - "union" => context.new_rustfunc(PySetRef::union), - "__or__" => context.new_rustfunc(PySetRef::or), - "intersection" => context.new_rustfunc(PySetRef::intersection), - "__and__" => context.new_rustfunc(PySetRef::and), - "difference" => context.new_rustfunc(PySetRef::difference), - "__sub__" => context.new_rustfunc(PySetRef::sub), - "symmetric_difference" => context.new_rustfunc(PySetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PySetRef::xor), - "__doc__" => context.new_str(set_doc.to_string()), - "add" => context.new_rustfunc(PySetRef::add), - "remove" => context.new_rustfunc(PySetRef::remove), - "discard" => context.new_rustfunc(PySetRef::discard), - "clear" => context.new_rustfunc(PySetRef::clear), - "copy" => context.new_rustfunc(PySetRef::copy), - "pop" => context.new_rustfunc(PySetRef::pop), - "update" => context.new_rustfunc(PySetRef::update), - "__ior__" => context.new_rustfunc(PySetRef::ior), - "intersection_update" => context.new_rustfunc(PySetRef::intersection_update), - "__iand__" => context.new_rustfunc(PySetRef::iand), - "difference_update" => context.new_rustfunc(PySetRef::difference_update), - "__isub__" => context.new_rustfunc(PySetRef::isub), - "symmetric_difference_update" => context.new_rustfunc(PySetRef::symmetric_difference_update), - "__ixor__" => context.new_rustfunc(PySetRef::ixor), - "__iter__" => context.new_rustfunc(PySetRef::iter), - "isdisjoint" => context.new_rustfunc(PySetRef::isdisjoint), - }); - - let frozenset_type = &context.frozenset_type; - - let frozenset_doc = "frozenset() -> empty frozenset object\n\ - frozenset(iterable) -> frozenset object\n\n\ - Build an immutable unordered collection of unique elements."; - - extend_class!(context, frozenset_type, { - "__new__" => context.new_rustfunc(PyFrozenSetRef::new), - "__eq__" => context.new_rustfunc(PyFrozenSetRef::eq), - "__ge__" => context.new_rustfunc(PyFrozenSetRef::ge), - "__gt__" => context.new_rustfunc(PyFrozenSetRef::gt), - "__le__" => context.new_rustfunc(PyFrozenSetRef::le), - "__lt__" => context.new_rustfunc(PyFrozenSetRef::lt), - "issubset" => context.new_rustfunc(PyFrozenSetRef::le), - "issuperset" => context.new_rustfunc(PyFrozenSetRef::ge), - "union" => context.new_rustfunc(PyFrozenSetRef::union), - "__or__" => context.new_rustfunc(PyFrozenSetRef::or), - "intersection" => context.new_rustfunc(PyFrozenSetRef::intersection), - "__and__" => context.new_rustfunc(PyFrozenSetRef::and), - "difference" => context.new_rustfunc(PyFrozenSetRef::difference), - "__sub__" => context.new_rustfunc(PyFrozenSetRef::sub), - "symmetric_difference" => context.new_rustfunc(PyFrozenSetRef::symmetric_difference), - "__xor__" => context.new_rustfunc(PyFrozenSetRef::xor), - "__contains__" => context.new_rustfunc(PyFrozenSetRef::contains), - "__len__" => context.new_rustfunc(PyFrozenSetRef::len), - "__doc__" => context.new_str(frozenset_doc.to_string()), - "__repr__" => context.new_rustfunc(PyFrozenSetRef::repr), - "copy" => context.new_rustfunc(PyFrozenSetRef::copy), - "__iter__" => context.new_rustfunc(PyFrozenSetRef::iter), - "isdisjoint" => context.new_rustfunc(PyFrozenSetRef::isdisjoint), - }); + PySet::extend_class(context, &context.set_type); + PyFrozenSet::extend_class(context, &context.frozenset_type); } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index ab7525e74..40d191296 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,7 +1,6 @@ extern crate unicode_xid; use std::fmt; -use std::hash::{Hash, Hasher}; use std::ops::Range; use std::str::FromStr; use std::string::ToString; @@ -13,12 +12,14 @@ use unicode_xid::UnicodeXID; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::function::{single_or_tuple_any, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; +use super::objbytes::PyBytes; use super::objdict::PyDict; use super::objint::{self, PyInt}; use super::objnone::PyNone; @@ -172,10 +173,8 @@ impl PyString { } #[pymethod(name = "__hash__")] - fn hash(&self, _vm: &VirtualMachine) -> usize { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() as usize + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_value(&self.value) } #[pymethod(name = "__len__")] @@ -185,22 +184,17 @@ impl PyString { #[pymethod(name = "__mul__")] fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &self.value; - let multiplier = objint::get_value(&val).to_i32().unwrap(); - let capacity = if multiplier > 0 { - multiplier.to_usize().unwrap() * value.len() - } else { - 0 - }; - let mut result = String::with_capacity(capacity); - for _x in 0..multiplier { - result.push_str(value.as_str()); - } - Ok(result) - } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) + if !objtype::isinstance(&val, &vm.ctx.int_type()) { + return Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))); } + objint::get_value(&val) + .to_isize() + .map(|multiplier| multiplier.max(0)) + .and_then(|multiplier| multiplier.to_usize()) + .map(|multiplier| self.value.repeat(multiplier)) + .ok_or_else(|| { + vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_string()) + }) } #[pymethod(name = "__rmul__")] @@ -303,10 +297,13 @@ impl PyString { let num_splits = num .into_option() .unwrap_or_else(|| value.split(pattern).count()); - let elements = value + let mut elements: Vec<_> = value .rsplitn(num_splits + 1, pattern) .map(|o| vm.ctx.new_str(o.to_string())) .collect(); + // Unlike Python rsplit, Rust rsplitn returns an iterator that + // starts from the end of the string. + elements.reverse(); vm.ctx.new_list(elements) } @@ -866,7 +863,7 @@ impl PyString { None => false, }; // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(|c| UnicodeXID::is_xid_continue(c)) + is_identifier_start && chars.all(UnicodeXID::is_xid_continue) } // https://docs.python.org/3/library/stdtypes.html#str.translate @@ -879,18 +876,18 @@ impl PyString { match table.get_item(c as u32, vm) { Ok(value) => { if let Some(text) = value.payload::() { - translated.extend(text.value.chars()); - } else if let Some(_) = value.payload::() { - // Do Nothing + translated.push_str(&text.value); } else if let Some(bigint) = value.payload::() { match bigint.as_bigint().to_u32().and_then(std::char::from_u32) { Some(ch) => translated.push(ch as char), None => { - return Err(vm.new_value_error(format!( - "character mapping must be in range(0x110000)" - ))); + return Err(vm.new_value_error( + "character mapping must be in range(0x110000)".to_owned(), + )); } } + } else if let Some(_) = value.payload::() { + // Do Nothing } else { return Err(vm.new_type_error( "character mapping must return integer, None or str".to_owned(), @@ -961,6 +958,31 @@ impl PyString { } } } + + #[pymethod] + fn encode( + &self, + encoding: OptionalArg, + _errors: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let encoding = encoding.map_or_else( + || Ok("utf-8".to_string()), + |v| { + if objtype::isinstance(&v, &vm.ctx.str_type()) { + Ok(get_value(&v)) + } else { + Err(vm.new_type_error(format!( + "encode() argument 1 must be str, not {}", + v.class().name + ))) + } + }, + )?; + + let encoded = PyBytes::from_string(&self.value, &encoding, vm)?; + Ok(encoded.into_pyobject(vm)?) + } } impl PyValue for PyString { diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 42999189f..40a5749ee 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -1,23 +1,21 @@ -use std::cell::{Cell, RefCell}; +use std::cell::Cell; use std::fmt; -use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; +use crate::pyhash; use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; use super::objiter; use super::objsequence::{ - get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, + get_elements_tuple, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; use super::objtype::{self, PyClassRef}; pub struct PyTuple { // TODO: shouldn't be public - // TODO: tuples are immutable, remove this RefCell - pub elements: RefCell>, + pub elements: Vec, } impl fmt::Debug for PyTuple { @@ -29,9 +27,7 @@ impl fmt::Debug for PyTuple { impl From> for PyTuple { fn from(elements: Vec) -> Self { - PyTuple { - elements: RefCell::new(elements), - } + PyTuple { elements } } } @@ -41,14 +37,19 @@ impl PyValue for PyTuple { } } +impl PyTuple { + pub fn fast_getitem(&self, idx: usize) -> PyObjectRef { + self.elements[idx].clone() + } +} + pub type PyTupleRef = PyRef; impl PyTupleRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_lt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_lt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -57,9 +58,8 @@ impl PyTupleRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_gt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_gt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -68,9 +68,8 @@ impl PyTupleRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_ge(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_ge(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -79,9 +78,8 @@ impl PyTupleRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_le(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_le(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -90,9 +88,8 @@ impl PyTupleRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let e1 = self.elements.borrow(); - let e2 = get_elements(&other); - let elements = e1.iter().chain(e2.iter()).cloned().collect(); + let e2 = get_elements_tuple(&other); + let elements = self.elements.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_tuple(elements)) } else { Ok(vm.ctx.not_implemented()) @@ -100,12 +97,12 @@ impl PyTupleRef { } fn bool(self, _vm: &VirtualMachine) -> bool { - !self.elements.borrow().is_empty() + !self.elements.is_empty() } fn count(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut count: usize = 0; - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { count += 1; } else { @@ -120,23 +117,16 @@ impl PyTupleRef { fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = &self.elements.borrow(); - let other = get_elements(&other); - let res = seq_equal(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_equal(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) } } - fn hash(self, vm: &VirtualMachine) -> PyResult { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - for element in self.elements.borrow().iter() { - let hash_result = vm.call_method(element, "__hash__", vec![])?; - let element_hash = objint::get_value(&hash_result); - element_hash.hash(&mut hasher); - } - Ok(hasher.finish()) + fn hash(self, vm: &VirtualMachine) -> PyResult { + pyhash::hash_iter(self.elements.iter(), vm) } fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { @@ -147,13 +137,13 @@ impl PyTupleRef { } fn len(self, _vm: &VirtualMachine) -> usize { - self.elements.borrow().len() + self.elements.len() } fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; - for elem in self.elements.borrow().iter() { + for elem in self.elements.iter() { let s = vm.to_repr(elem)?; str_parts.push(s.value.clone()); } @@ -170,21 +160,16 @@ impl PyTupleRef { } fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements.borrow(), counter); + let new_elements = seq_mul(&self.elements, counter); vm.ctx.new_tuple(new_elements) } fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - get_item( - vm, - self.as_object(), - &self.elements.borrow(), - needle.clone(), - ) + get_item(vm, self.as_object(), &self.elements, needle.clone()) } fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for (index, element) in self.elements.borrow().iter().enumerate() { + for (index, element) in self.elements.iter().enumerate() { if element.is(&needle) { return Ok(index); } @@ -197,7 +182,7 @@ impl PyTupleRef { } fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { return Ok(true); } @@ -241,8 +226,8 @@ impl PyValue for PyTupleIterator { impl PyTupleIterator { #[pymethod(name = "__next__")] fn next(&self, vm: &VirtualMachine) -> PyResult { - if self.position.get() < self.tuple.elements.borrow().len() { - let ret = self.tuple.elements.borrow()[self.position.get()].clone(); + if self.position.get() < self.tuple.elements.len() { + let ret = self.tuple.elements[self.position.get()].clone(); self.position.set(self.position.get() + 1); Ok(ret) } else { diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index c66e57354..0416d665b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -248,7 +248,7 @@ pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let (typ, name, bases, dict) = args.bind(vm)?; type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { - Err(vm.new_type_error(format!(": type_new: {:?}", args))) + Err(vm.new_type_error("type() takes 1 or 3 arguments".to_string())) } } @@ -289,7 +289,7 @@ fn type_dict_setter(_instance: PyClassRef, _value: PyObjectRef, vm: &VirtualMach )) } -// This is the internal get_attr implementation for fast lookup on a class. +/// This is the internal get_attr implementation for fast lookup on a class. pub fn class_get_attr(class: &PyClassRef, attr_name: &str) -> Option { if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { return Some(item); diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs index 735670f4e..5d4f68d9d 100644 --- a/vm/src/obj/objweakproxy.rs +++ b/vm/src/obj/objweakproxy.rs @@ -1,9 +1,10 @@ use super::objweakref::PyWeak; use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass] #[derive(Debug)] pub struct PyWeakProxy { weak: PyWeak, @@ -17,21 +18,27 @@ impl PyValue for PyWeakProxy { pub type PyWeakProxyRef = PyRef; -impl PyWeakProxyRef { - // TODO callbacks +#[pyimpl] +impl PyWeakProxy { + // TODO: callbacks + #[pymethod(name = "__new__")] fn create( cls: PyClassRef, referent: PyObjectRef, - _callback: OptionalArg, + callback: OptionalArg, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { + if callback.is_present() { + panic!("Passed a callback to weakproxy, but weakproxy does not yet support proxies."); + } PyWeakProxy { weak: PyWeak::downgrade(&referent), } .into_ref_with_type(vm, cls) } - fn getattr(self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__getattr__")] + fn getattr(&self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self.weak.upgrade() { Some(obj) => vm.get_attribute(obj, attr_name), None => Err(vm.new_exception( @@ -40,11 +47,19 @@ impl PyWeakProxyRef { )), } } + + #[pymethod(name = "__setattr__")] + fn setattr(&self, attr_name: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.set_attr(&obj, attr_name, value), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } } pub fn init(context: &PyContext) { - extend_class!(context, &context.weakproxy_type, { - "__new__" => context.new_rustfunc(PyWeakProxyRef::create), - "__getattr__" => context.new_rustfunc(PyWeakProxyRef::getattr), - }); + PyWeakProxy::extend_class(&context, &context.weakproxy_type); } diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs new file mode 100644 index 000000000..8bbd82168 --- /dev/null +++ b/vm/src/pyhash.rs @@ -0,0 +1,83 @@ +use std::hash::{Hash, Hasher}; + +use crate::obj::objfloat; +use crate::pyobject::PyObjectRef; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +pub type PyHash = i64; +pub type PyUHash = u64; + +/// Prime multiplier used in string and various other hashes. +pub const MULTIPLIER: PyHash = 1_000_003; // 0xf4243 +/// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 +pub const BITS: usize = 61; +pub const MODULUS: PyUHash = (1 << BITS) - 1; +pub const INF: PyHash = 314_159; +pub const NAN: PyHash = 0; +pub const IMAG: PyHash = MULTIPLIER; + +// pub const CUTOFF: usize = 7; + +pub fn hash_float(value: f64) -> PyHash { + // cpython _Py_HashDouble + if !value.is_finite() { + return if value.is_infinite() { + if value > 0.0 { + INF + } else { + -INF + } + } else { + NAN + }; + } + + let frexp = objfloat::ufrexp(value); + + // process 28 bits at a time; this should work well both for binary + // and hexadecimal floating point. + let mut m = frexp.0; + let mut e = frexp.1; + let mut x: PyUHash = 0; + while m != 0.0 { + x = ((x << 28) & MODULUS) | x >> (BITS - 28); + m *= 268_435_456.0; // 2**28 + e -= 28; + let y = m as PyUHash; // pull out integer part + m -= y as f64; + x += y; + if x >= MODULUS { + x -= MODULUS; + } + } + + // adjust for the exponent; first reduce it modulo BITS + const BITS32: i32 = BITS as i32; + e = if e >= 0 { + e % BITS32 + } else { + BITS32 - 1 - ((-1 - e) % BITS32) + }; + x = ((x << e) & MODULUS) | x >> (BITS32 - e); + + x as PyHash * value.signum() as PyHash +} + +pub fn hash_value(data: &T) -> PyHash { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + hasher.finish() as PyHash +} + +pub fn hash_iter<'a, I: std::iter::Iterator>( + iter: I, + vm: &VirtualMachine, +) -> PyResult { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for element in iter { + let item_hash = vm._hash(&element)?; + item_hash.hash(&mut hasher); + } + Ok(hasher.finish() as PyHash) +} diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 722588b3f..3023573b0 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -40,6 +40,7 @@ use crate::obj::objmap; use crate::obj::objmappingproxy; use crate::obj::objmemory; use crate::obj::objmodule::{self, PyModule}; +use crate::obj::objnamespace::{self, PyNamespace}; use crate::obj::objnone::{self, PyNone, PyNoneRef}; use crate::obj::objobject; use crate::obj::objproperty; @@ -56,6 +57,7 @@ use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; +use indexmap::IndexMap; /* Python objects and references. @@ -85,6 +87,7 @@ pub type PyResult = Result; // A valid value, o /// For attributes we do not use a dict, but a hashmap. This is probably /// faster, unordered, and only supports strings as keys. +/// TODO: class attributes should maintain insertion order (use IndexMap here) pub type PyAttributes = HashMap; impl fmt::Display for PyObject { @@ -158,6 +161,7 @@ pub struct PyContext { pub property_type: PyClassRef, pub readonly_property_type: PyClassRef, pub module_type: PyClassRef, + pub namespace_type: PyClassRef, pub bound_method_type: PyClassRef, pub weakref_type: PyClassRef, pub weakproxy_type: PyClassRef, @@ -250,6 +254,7 @@ impl PyContext { let dict_type = create_type("dict", &type_type, &object_type); let module_type = create_type("module", &type_type, &object_type); + let namespace_type = create_type("SimpleNamespace", &type_type, &object_type); let classmethod_type = create_type("classmethod", &type_type, &object_type); let staticmethod_type = create_type("staticmethod", &type_type, &object_type); let function_type = create_type("function", &type_type, &object_type); @@ -290,7 +295,7 @@ impl PyContext { let zip_type = create_type("zip", &type_type, &object_type); let bool_type = create_type("bool", &type_type, &int_type); let memoryview_type = create_type("memoryview", &type_type, &object_type); - let code_type = create_type("code", &type_type, &int_type); + let code_type = create_type("code", &type_type, &object_type); let range_type = create_type("range", &type_type, &object_type); let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); @@ -366,6 +371,7 @@ impl PyContext { readonly_property_type, generator_type, module_type, + namespace_type, bound_method_type, weakref_type, weakproxy_type, @@ -407,6 +413,7 @@ impl PyContext { objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); + objnamespace::init(&context); objmappingproxy::init(&context); exceptions::init(&context); context @@ -464,6 +471,10 @@ impl PyContext { self.module_type.clone() } + pub fn namespace_type(&self) -> PyClassRef { + self.namespace_type.clone() + } + pub fn set_type(&self) -> PyClassRef { self.set_type.clone() } @@ -658,6 +669,10 @@ impl PyContext { ) } + pub fn new_namespace(&self) -> PyObjectRef { + PyObject::new(PyNamespace, self.namespace_type(), Some(self.new_dict())) + } + pub fn new_rustfunc(&self, f: F) -> PyObjectRef where F: IntoPyNativeFunc, @@ -819,6 +834,7 @@ impl PyRef { pub fn as_object(&self) -> &PyObjectRef { &self.obj } + pub fn into_object(self) -> PyObjectRef { self.obj } @@ -1075,7 +1091,7 @@ impl PyIterable { self.method.clone(), PyFuncArgs { args: vec![], - kwargs: vec![], + kwargs: IndexMap::new(), }, )?; @@ -1224,6 +1240,7 @@ impl PyObject where T: Sized + PyObjectPayload, { + #[allow(clippy::new_ret_no_self)] pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ, dict, payload }.into_ref() } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 2ac36c81e..2b79dd706 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -457,7 +457,7 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult fn map_ast( f: fn(vm: &VirtualMachine, &T) -> PyResult, vm: &VirtualMachine, - items: &Vec, + items: &[T], ) -> PyResult { let list: PyResult> = items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs index 97fb76f8e..ff748aa01 100644 --- a/vm/src/stdlib/binascii.rs +++ b/vm/src/stdlib/binascii.rs @@ -1,7 +1,10 @@ use crate::function::PyFuncArgs; use crate::obj::objbytes; +use crate::obj::objint; use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; +use crc::{crc32, Hasher32}; +use num_traits::ToPrimitive; fn hex_nibble(n: u8) -> u8 { match n { @@ -56,6 +59,26 @@ fn binascii_unhexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(unhex)) } +fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(data, Some(vm.ctx.bytes_type()))], + optional = [(value, None)] + ); + + let bytes = objbytes::get_value(data); + let crc = match value { + None => 0u32, + Some(value) => objint::get_value(&value).to_u32().unwrap(), + }; + + let mut digest = crc32::Digest::new_with_initial(crc32::IEEE, crc); + digest.write(&bytes); + + Ok(vm.ctx.new_int(digest.sum32())) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -64,5 +87,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "b2a_hex" => ctx.new_rustfunc(binascii_hexlify), "unhexlify" => ctx.new_rustfunc(binascii_unhexlify), "a2b_hex" => ctx.new_rustfunc(binascii_unhexlify), + "crc32" => ctx.new_rustfunc(binascii_crc32), }) } diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs new file mode 100644 index 000000000..ae40c16c5 --- /dev/null +++ b/vm/src/stdlib/imp.rs @@ -0,0 +1,97 @@ +use crate::compile; +use crate::import::import_file; +use crate::obj::objcode::PyCodeRef; +use crate::obj::objmodule::PyModuleRef; +use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(vec![])) +} + +fn imp_acquire_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_release_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { + vm.stdlib_inits.borrow().contains_key(name.as_str()) +} + +fn imp_is_frozen(name: PyStringRef, vm: &VirtualMachine) -> bool { + vm.frozen.borrow().contains_key(name.as_str()) +} + +fn imp_create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + + let name = &objstr::get_value(&vm.get_attribute(spec.clone(), "name")?); + + if let Ok(module) = sys_modules.get_item(name, vm) { + Ok(module) + } else { + if let Some(make_module_func) = vm.stdlib_inits.borrow().get(name) { + Ok(make_module_func(vm)) + } else { + Ok(vm.get_none()) + } + } +} + +fn imp_exec_builtin(_mod: PyModuleRef, _vm: &VirtualMachine) -> i32 { + // TOOD: Should we do something here? + 0 +} + +fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + if let Some(frozen) = vm.frozen.borrow().get(name.as_str()) { + compile::compile(vm, frozen, &compile::Mode::Exec, "frozen".to_string()) + .map_err(|err| vm.new_syntax_error(&err)) + } else { + Err(vm.new_import_error(format!("No such frozen object named {}", name.as_str()))) + } +} + +fn imp_init_frozen(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + if let Some(frozen) = vm.frozen.borrow().get(name.as_str()) { + import_file(vm, name.as_str(), "frozen".to_string(), frozen.to_string()) + } else { + Err(vm.new_import_error(format!("No such frozen object named {}", name.as_str()))) + } +} + +fn imp_is_frozen_package(_name: PyStringRef, _vm: &VirtualMachine) -> bool { + // TODO: Support frozen package. + false +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_imp", { + "extension_suffixes" => ctx.new_rustfunc(imp_extension_suffixes), + "acquire_lock" => ctx.new_rustfunc(imp_acquire_lock), + "release_lock" => ctx.new_rustfunc(imp_release_lock), + "lock_held" => ctx.new_rustfunc(imp_lock_held), + "is_builtin" => ctx.new_rustfunc(imp_is_builtin), + "is_frozen" => ctx.new_rustfunc(imp_is_frozen), + "create_builtin" => ctx.new_rustfunc(imp_create_builtin), + "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), + "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), + "init_frozen" => ctx.new_rustfunc(imp_init_frozen), + "is_frozen_package" => ctx.new_rustfunc(imp_is_frozen_package), + }); + + module +} diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index e9b49ec44..da8bedf53 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,9 +1,7 @@ /* * I/O core tools. */ - use std::cell::RefCell; -use std::collections::HashSet; use std::fs::File; use std::io::prelude::*; use std::io::BufReader; @@ -294,6 +292,63 @@ fn text_io_base_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn split_mode_string(mode_string: String) -> Result<(String, String), String> { + let mut mode: char = '\0'; + let mut typ: char = '\0'; + let mut plus_is_set = false; + + for ch in mode_string.chars() { + match ch { + '+' => { + if plus_is_set { + return Err(format!("invalid mode: '{}'", mode_string)); + } + plus_is_set = true; + } + 't' | 'b' => { + if typ != '\0' { + if typ == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err("can't have text and binary mode at once".to_string()); + } + } + typ = ch; + } + 'a' | 'r' | 'w' => { + if mode != '\0' { + if mode == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err( + "must have exactly one of create/read/write/append mode".to_string() + ); + } + } + mode = ch; + } + _ => return Err(format!("invalid mode: '{}'", mode_string)), + } + } + + if mode == '\0' { + return Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string(), + ); + } + let mut mode = mode.to_string(); + if plus_is_set { + mode.push('+'); + } + if typ == '\0' { + typ = 't'; + } + Ok((mode, typ.to_string())) +} + pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -301,69 +356,60 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(file, Some(vm.ctx.str_type()))], optional = [(mode, Some(vm.ctx.str_type()))] ); - - let module = import::import_module(vm, PathBuf::default(), "io").unwrap(); - - //mode is optional: 'rt' is the default mode (open from reading text) - let rust_mode = mode.map_or("rt".to_string(), |m| objstr::get_value(m)); - - let mut raw_modes = HashSet::new(); - - //add raw modes - raw_modes.insert("a".to_string()); - raw_modes.insert("r".to_string()); - raw_modes.insert("x".to_string()); - raw_modes.insert("w".to_string()); - - //This is not a terribly elegant way to separate the file mode from - //the "type" flag - this should be improved. The intention here is to - //match a valid flag for the file_io_init call: - //https://docs.python.org/3/library/io.html#io.FileIO - let modes: Vec = rust_mode - .chars() - .filter(|a| raw_modes.contains(&a.to_string())) - .collect(); - - if modes.is_empty() || modes.len() > 1 { - return Err(vm.new_value_error("Invalid Mode".to_string())); - } - - //Class objects (potentially) consumed by io.open - //RawIO: FileIO - //Buffered: BufferedWriter, BufferedReader - //Text: TextIOWrapper - let file_io_class = vm.get_attribute(module.clone(), "FileIO").unwrap(); - let buffered_writer_class = vm.get_attribute(module.clone(), "BufferedWriter").unwrap(); - let buffered_reader_class = vm.get_attribute(module.clone(), "BufferedReader").unwrap(); - let text_io_wrapper_class = vm.get_attribute(module, "TextIOWrapper").unwrap(); - - //Construct a FileIO (subclass of RawIOBase) - //This is subsequently consumed by a Buffered Class. - let file_args = vec![file.clone(), vm.ctx.new_str(modes[0].to_string())]; - let file_io = vm.invoke(file_io_class, file_args)?; - - //Create Buffered class to consume FileIO. The type of buffered class depends on - //the operation in the mode. - //There are 3 possible classes here, each inheriting from the RawBaseIO - // creating || writing || appending => BufferedWriter - let buffered = if rust_mode.contains('w') { - vm.invoke(buffered_writer_class, vec![file_io.clone()]) - // reading => BufferedReader - } else { - vm.invoke(buffered_reader_class, vec![file_io.clone()]) - //TODO: updating => PyBufferedRandom + // mode is optional: 'rt' is the default mode (open from reading text) + let mode_string = mode.map_or("rt".to_string(), |m| objstr::get_value(m)); + let (mode, typ) = match split_mode_string(mode_string) { + Ok((mode, typ)) => (mode, typ), + Err(error_message) => { + return Err(vm.new_value_error(error_message)); + } }; - if rust_mode.contains('t') { - //If the mode is text this buffer type is consumed on construction of - //a TextIOWrapper which is subsequently returned. - vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) - } else { + let io_module = import::import_module(vm, PathBuf::default(), "io").unwrap(); + + // Construct a FileIO (subclass of RawIOBase) + // This is subsequently consumed by a Buffered Class. + let file_io_class = vm.get_attribute(io_module.clone(), "FileIO").unwrap(); + let file_io_obj = vm.invoke( + file_io_class, + vec![file.clone(), vm.ctx.new_str(mode.clone())], + )?; + + // Create Buffered class to consume FileIO. The type of buffered class depends on + // the operation in the mode. + // There are 3 possible classes here, each inheriting from the RawBaseIO + // creating || writing || appending => BufferedWriter + let buffered = match mode.chars().next().unwrap() { + 'w' => { + let buffered_writer_class = vm + .get_attribute(io_module.clone(), "BufferedWriter") + .unwrap(); + vm.invoke(buffered_writer_class, vec![file_io_obj.clone()]) + } + 'r' => { + let buffered_reader_class = vm + .get_attribute(io_module.clone(), "BufferedReader") + .unwrap(); + vm.invoke(buffered_reader_class, vec![file_io_obj.clone()]) + } + //TODO: updating => PyBufferedRandom + _ => unimplemented!("'a' mode is not yet implemented"), + }; + + let io_obj = match typ.chars().next().unwrap() { + // If the mode is text this buffer type is consumed on construction of + // a TextIOWrapper which is subsequently returned. + 't' => { + let text_io_wrapper_class = vm.get_attribute(io_module, "TextIOWrapper").unwrap(); + vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) + } // If the mode is binary this Buffered class is returned directly at // this point. - //For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field - buffered - } + // For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field + 'b' => buffered, + _ => unreachable!(), + }; + io_obj } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { @@ -377,7 +423,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { }); // IOBase Subclasses - let raw_io_base = py_class!(ctx, "RawIOBase", ctx.object(), {}); + let raw_io_base = py_class!(ctx, "RawIOBase", io_base.clone(), {}); let buffered_io_base = py_class!(ctx, "BufferedIOBase", io_base.clone(), { "__init__" => ctx.new_rustfunc(buffered_io_base_init) @@ -439,3 +485,90 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "BytesIO" => bytes_io, }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_mode_split_into(mode_string: &str, expected_mode: &str, expected_typ: &str) { + let (mode, typ) = split_mode_string(mode_string.to_string()).unwrap(); + assert_eq!(mode, expected_mode); + assert_eq!(typ, expected_typ); + } + + #[test] + fn test_split_mode_valid_cases() { + assert_mode_split_into("r", "r", "t"); + assert_mode_split_into("rb", "r", "b"); + assert_mode_split_into("rt", "r", "t"); + assert_mode_split_into("r+t", "r+", "t"); + assert_mode_split_into("w+t", "w+", "t"); + assert_mode_split_into("r+b", "r+", "b"); + assert_mode_split_into("w+b", "w+", "b"); + } + + #[test] + fn test_invalid_mode() { + assert_eq!( + split_mode_string("rbsss".to_string()), + Err("invalid mode: 'rbsss'".to_string()) + ); + assert_eq!( + split_mode_string("rrb".to_string()), + Err("invalid mode: 'rrb'".to_string()) + ); + assert_eq!( + split_mode_string("rbb".to_string()), + Err("invalid mode: 'rbb'".to_string()) + ); + } + + #[test] + fn test_mode_not_specified() { + assert_eq!( + split_mode_string("".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("b".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("t".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + } + + #[test] + fn test_text_and_binary_at_once() { + assert_eq!( + split_mode_string("rbt".to_string()), + Err("can't have text and binary mode at once".to_string()) + ); + } + + #[test] + fn test_exactly_one_mode() { + assert_eq!( + split_mode_string("rwb".to_string()), + Err("must have exactly one of create/read/write/append mode".to_string()) + ); + } + + #[test] + fn test_at_most_one_plus() { + assert_eq!( + split_mode_string("a++".to_string()), + Err("invalid mode: 'a++'".to_string()) + ); + } +} diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index 3856fe92f..732020480 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -3,14 +3,76 @@ use std::cmp::Ordering; use std::ops::{AddAssign, SubAssign}; use num_bigint::BigInt; +use num_traits::ToPrimitive; -use crate::function::OptionalArg; +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objbool; +use crate::obj::objint; use crate::obj::objint::{PyInt, PyIntRef}; -use crate::obj::objiter::new_stop_iteration; +use crate::obj::objiter::{call_next, get_iter, new_stop_iteration}; +use crate::obj::objtype; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass(name = "chain")] +#[derive(Debug)] +struct PyItertoolsChain { + iterables: Vec, + cur: RefCell<(usize, Option)>, +} + +impl PyValue for PyItertoolsChain { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "chain") + } +} + +#[pyimpl] +impl PyItertoolsChain { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyItertoolsChain { + iterables: args.args, + cur: RefCell::new((0, None)), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let (ref mut cur_idx, ref mut cur_iter) = *self.cur.borrow_mut(); + while *cur_idx < self.iterables.len() { + if cur_iter.is_none() { + *cur_iter = Some(get_iter(vm, &self.iterables[*cur_idx])?); + } + + // can't be directly inside the 'match' clause, otherwise the borrows collide. + let obj = call_next(vm, cur_iter.as_ref().unwrap()); + match obj { + Ok(ok) => return Ok(ok), + Err(err) => { + if objtype::isinstance(&err, &vm.ctx.exceptions.stop_iteration) { + *cur_idx += 1; + *cur_iter = None; + } else { + return Err(err); + } + } + } + } + + Err(new_stop_iteration(vm)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + #[pyclass] #[derive(Debug)] struct PyItertoolsCount { @@ -27,6 +89,7 @@ impl PyValue for PyItertoolsCount { #[pyimpl] impl PyItertoolsCount { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, start: OptionalArg, @@ -44,7 +107,7 @@ impl PyItertoolsCount { Ok(PyItertoolsCount { cur: RefCell::new(start), - step: step, + step, } .into_ref(vm) .into_object()) @@ -79,6 +142,7 @@ impl PyValue for PyItertoolsRepeat { #[pyimpl] impl PyItertoolsRepeat { #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] fn new( _cls: PyClassRef, object: PyObjectRef, @@ -92,7 +156,7 @@ impl PyItertoolsRepeat { Ok(PyItertoolsRepeat { object: object.clone(), - times: times, + times, } .into_ref(vm) .into_object()) @@ -121,17 +185,261 @@ impl PyItertoolsRepeat { } } +#[pyclass(name = "starmap")] +#[derive(Debug)] +struct PyItertoolsStarmap { + function: PyObjectRef, + iter: PyObjectRef, +} + +impl PyValue for PyItertoolsStarmap { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "starmap") + } +} + +#[pyimpl] +impl PyItertoolsStarmap { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + function: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsStarmap { function, iter } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let obj = call_next(vm, &self.iter)?; + + vm.invoke(self.function.clone(), vm.extract_elements(&obj)?) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass] +#[derive(Debug)] +struct PyItertoolsTakewhile { + predicate: PyObjectRef, + iterable: PyObjectRef, + stop_flag: RefCell, +} + +impl PyValue for PyItertoolsTakewhile { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "takewhile") + } +} + +#[pyimpl] +impl PyItertoolsTakewhile { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + predicate: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsTakewhile { + predicate, + iterable: iter, + stop_flag: RefCell::new(false), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if *self.stop_flag.borrow() { + return Err(new_stop_iteration(vm)); + } + + // might be StopIteration or anything else, which is propaged upwwards + let obj = call_next(vm, &self.iterable)?; + + let verdict = vm.invoke(self.predicate.clone(), vec![obj.clone()])?; + let verdict = objbool::boolval(vm, verdict)?; + if verdict { + Ok(obj) + } else { + *self.stop_flag.borrow_mut() = true; + Err(new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass(name = "islice")] +#[derive(Debug)] +struct PyItertoolsIslice { + iterable: PyObjectRef, + cur: RefCell, + next: RefCell, + stop: Option, + step: usize, +} + +impl PyValue for PyItertoolsIslice { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "islice") + } +} + +fn pyobject_to_opt_usize(obj: PyObjectRef, vm: &VirtualMachine) -> Option { + let is_int = objtype::isinstance(&obj, &vm.ctx.int_type()); + if is_int { + objint::get_value(&obj).to_usize() + } else { + None + } +} + +#[pyimpl] +impl PyItertoolsIslice { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let (iter, start, stop, step) = match args.args.len() { + 0 | 1 => { + return Err(vm.new_type_error(format!( + "islice expected at least 2 arguments, got {}", + args.args.len() + ))); + } + + 2 => { + let (iter, stop): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + + (iter, 0usize, stop, 1usize) + } + _ => { + let (iter, start, stop, step): ( + PyObjectRef, + PyObjectRef, + PyObjectRef, + PyObjectRef, + ) = args.bind(vm)?; + + let start = if !start.is(&vm.get_none()) { + pyobject_to_opt_usize(start, &vm).ok_or_else(|| { + vm.new_value_error( + "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize.".to_string(), + ) + })? + } else { + 0usize + }; + + let step = if !step.is(&vm.get_none()) { + pyobject_to_opt_usize(step, &vm).ok_or_else(|| { + vm.new_value_error( + "Step for islice() must be a positive integer or None.".to_string(), + ) + })? + } else { + 1usize + }; + + (iter, start, stop, step) + } + }; + + let stop = if !stop.is(&vm.get_none()) { + Some(pyobject_to_opt_usize(stop, &vm).ok_or_else(|| { + vm.new_value_error( + "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." + .to_string(), + ) + })?) + } else { + None + }; + + let iter = get_iter(vm, &iter)?; + + Ok(PyItertoolsIslice { + iterable: iter, + cur: RefCell::new(0), + next: RefCell::new(start), + stop, + step, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + while *self.cur.borrow() < *self.next.borrow() { + call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + } + + if let Some(stop) = self.stop { + if *self.cur.borrow() >= stop { + return Err(new_stop_iteration(vm)); + } + } + + let obj = call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + + // TODO is this overflow check required? attempts to copy CPython. + let (next, ovf) = (*self.next.borrow()).overflowing_add(self.step); + *self.next.borrow_mut() = if ovf { self.stop.unwrap() } else { next }; + + Ok(obj) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let chain = PyItertoolsChain::make_class(ctx); + let count = ctx.new_class("count", ctx.object()); PyItertoolsCount::extend_class(ctx, &count); let repeat = ctx.new_class("repeat", ctx.object()); PyItertoolsRepeat::extend_class(ctx, &repeat); + let starmap = PyItertoolsStarmap::make_class(ctx); + + let takewhile = ctx.new_class("takewhile", ctx.object()); + PyItertoolsTakewhile::extend_class(ctx, &takewhile); + + let islice = PyItertoolsIslice::make_class(ctx); + py_module!(vm, "itertools", { + "chain" => chain, "count" => count, "repeat" => repeat, + "starmap" => starmap, + "takewhile" => takewhile, + "islice" => islice, }) } diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index d57dd14a9..4b0ac1ca9 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -57,10 +57,11 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { serializer.serialize_i64(v.to_i64().unwrap()) // Although this may seem nice, it does not give the right result: // v.serialize(serializer) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) - || objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) - { - let elements = objsequence::get_elements(self.pyobject); + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) { + let elements = objsequence::get_elements_list(self.pyobject); + serialize_seq_elements(serializer, &elements) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) { + let elements = objsequence::get_elements_tuple(self.pyobject); serialize_seq_elements(serializer, &elements) } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs new file mode 100644 index 000000000..e7afc86d2 --- /dev/null +++ b/vm/src/stdlib/marshal.rs @@ -0,0 +1,24 @@ +use crate::bytecode; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::obj::objcode::{PyCode, PyCodeRef}; +use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult { + PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm) +} + +fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let code = bincode::deserialize::(&code_bytes).unwrap(); + let pycode = PyCode { code }; + pycode.into_pyobject(vm) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "marshal", { + "loads" => ctx.new_rustfunc(marshal_loads), + "dumps" => ctx.new_rustfunc(marshal_dumps) + }) +} diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 58b3ee953..c8ee63247 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -209,6 +209,23 @@ fn math_floor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn math_frexp(value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + objfloat::try_float(&value, vm)?.map_or_else( + || Err(vm.new_type_error(format!("must be real number, not {}", value.class()))), + |value| { + let (m, e) = if value.is_finite() { + let (m, e) = objfloat::ufrexp(value); + (m * value.signum(), e) + } else { + (value, 0) + }; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(m), vm.ctx.new_int(e)])) + }, + ) +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -256,6 +273,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "gamma" => ctx.new_rustfunc(math_gamma), "lgamma" => ctx.new_rustfunc(math_lgamma), + "frexp" => ctx.new_rustfunc(math_frexp), + // Rounding functions: "trunc" => ctx.new_rustfunc(math_trunc), "ceil" => ctx.new_rustfunc(math_ceil), diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 830aa2dca..7c72a1b61 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,9 +1,11 @@ mod ast; mod binascii; mod dis; +mod imp; mod itertools; pub(crate) mod json; mod keyword; +mod marshal; mod math; mod platform; mod pystruct; @@ -14,7 +16,7 @@ mod string; mod thread; mod time_module; mod tokenize; -mod types; +mod warnings; mod weakref; use std::collections::HashMap; @@ -24,6 +26,8 @@ use crate::vm::VirtualMachine; pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; +#[cfg(all(unix, not(target_os = "android")))] +mod pwd; use crate::pyobject::PyObjectRef; @@ -40,6 +44,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("itertools".to_string(), Box::new(itertools::make_module)); modules.insert("json".to_string(), Box::new(json::make_module)); modules.insert("keyword".to_string(), Box::new(keyword::make_module)); + modules.insert("marshal".to_string(), Box::new(marshal::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); modules.insert("platform".to_string(), Box::new(platform::make_module)); modules.insert("re".to_string(), Box::new(re::make_module)); @@ -49,8 +54,9 @@ pub fn get_module_inits() -> HashMap { modules.insert("_thread".to_string(), Box::new(thread::make_module)); modules.insert("time".to_string(), Box::new(time_module::make_module)); modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); - modules.insert("types".to_string(), Box::new(types::make_module)); modules.insert("_weakref".to_string(), Box::new(weakref::make_module)); + modules.insert("_imp".to_string(), Box::new(imp::make_module)); + modules.insert("_warnings".to_string(), Box::new(warnings::make_module)); // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] @@ -60,5 +66,11 @@ pub fn get_module_inits() -> HashMap { modules.insert("socket".to_string(), Box::new(socket::make_module)); } + // Unix-only + #[cfg(all(unix, not(target_os = "android")))] + { + modules.insert("pwd".to_string(), Box::new(pwd::make_module)); + } + modules } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 4394bdb89..0893c1e6d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -7,17 +7,16 @@ use std::{env, fs}; use num_traits::cast::ToPrimitive; -use crate::function::PyFuncArgs; +use crate::function::{IntoPyNativeFunc, PyFuncArgs}; use crate::obj::objbytes::PyBytesRef; use crate::obj::objdict::PyDictRef; -use crate::obj::objint; -use crate::obj::objint::PyIntRef; +use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objiter; -use crate::obj::objstr; -use crate::obj::objstr::PyStringRef; -use crate::obj::objtype::PyClassRef; +use crate::obj::objset::PySet; +use crate::obj::objstr::{self, PyString, PyStringRef}; +use crate::obj::objtype::{self, PyClassRef}; use crate::pyobject::{ - ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, + ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; @@ -61,6 +60,14 @@ pub fn raw_file_number(handle: File) -> i64 { unimplemented!(); } +fn make_path(_vm: &VirtualMachine, path: PyStringRef, dir_fd: &DirFd) -> PyStringRef { + if dir_fd.dir_fd.is_some() { + unimplemented!(); + } else { + path + } +} + pub fn os_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(fileno, Some(vm.ctx.int_type()))]); @@ -81,10 +88,19 @@ pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [ (name, Some(vm.ctx.str_type())), (mode, Some(vm.ctx.int_type())) - ] + ], + optional = [(dir_fd, Some(vm.ctx.int_type()))] ); - let fname = objstr::get_value(&name); + let name = name.clone().downcast::().unwrap(); + let dir_fd = if let Some(obj) = dir_fd { + DirFd { + dir_fd: Some(obj.clone().downcast::().unwrap()), + } + } else { + DirFd::default() + }; + let fname = &make_path(vm, name, &dir_fd).value; let handle = match objint::get_value(mode).to_u16().unwrap() { 0 => OpenOptions::new().read(true).open(&fname), @@ -159,11 +175,13 @@ fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { Ok(vm.ctx.new_int(written)) } -fn os_remove(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_remove(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::remove_file(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_mkdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_mkdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::create_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } @@ -171,7 +189,8 @@ fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { fs::create_dir_all(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } -fn os_rmdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_rmdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); fs::remove_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } @@ -219,6 +238,12 @@ impl PyValue for DirEntry { } } +#[derive(FromArgs, Default)] +struct DirFd { + #[pyarg(keyword_only, default = "None")] + dir_fd: Option, +} + #[derive(FromArgs)] struct FollowSymlinks { #[pyarg(keyword_only, default = "true")] @@ -234,6 +259,7 @@ impl DirEntryRef { self.entry.path().to_str().unwrap().to_string() } + #[allow(clippy::match_bool)] fn perform_on_metadata( self, follow_symlinks: FollowSymlinks, @@ -272,8 +298,13 @@ impl DirEntryRef { .is_symlink()) } - fn stat(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { - os_stat(self.path(vm).try_into_ref(vm)?, follow_symlinks, vm) + fn stat( + self, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + os_stat(self.path(vm).try_into_ref(vm)?, dir_fd, follow_symlinks, vm) } } @@ -404,6 +435,7 @@ fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { #[cfg(unix)] macro_rules! os_unix_stat_inner { ( $path:expr, $follow_symlinks:expr, $vm:expr ) => {{ + #[allow(clippy::match_bool)] fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { let meta = match follow_symlinks { true => fs::metadata(path)?, @@ -432,30 +464,36 @@ macro_rules! os_unix_stat_inner { #[cfg(target_os = "linux")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::linux::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "macos")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::macos::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } #[cfg(target_os = "android")] fn os_stat( path: PyStringRef, + dir_fd: DirFd, follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { use std::os::android::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); os_unix_stat_inner!(path, follow_symlinks, vm) } @@ -483,6 +521,7 @@ fn attributes_to_mode(attr: u32) -> u32 { #[cfg(windows)] fn os_stat( path: PyStringRef, + _dir_fd: DirFd, // TODO: error follow_symlinks: FollowSymlinks, vm: &VirtualMachine, ) -> PyResult { @@ -523,13 +562,24 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { } #[cfg(unix)] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { use std::os::unix::fs as unix_fs; + let dst = make_path(vm, dst, &dir_fd); unix_fs::symlink(&src.value, &dst.value).map_err(|s| vm.new_os_error(s.to_string())) } #[cfg(windows)] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + _dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { use std::os::windows::fs as win_fs; let ret = match fs::metadata(&dst.value) { Ok(meta) => { @@ -547,7 +597,12 @@ fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResu } #[cfg(all(not(unix), not(windows)))] -fn os_symlink(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { unimplemented!(); } @@ -564,6 +619,19 @@ fn os_chdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { env::set_current_dir(&path.value).map_err(|s| vm.new_os_error(s.to_string())) } +fn os_fspath(path: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::issubclass(&path.class(), &vm.ctx.str_type()) + || objtype::issubclass(&path.class(), &vm.ctx.bytes_type()) + { + Ok(path) + } else { + Err(vm.new_type_error(format!( + "expected str or bytes object, not {}", + path.class() + ))) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; @@ -600,36 +668,117 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "st_mtime" => ctx.new_property(StatResultRef::st_mtime), }); - py_module!(vm, "_os", { - "open" => ctx.new_rustfunc(os_open), + struct SupportFunc<'a> { + name: &'a str, + func_obj: PyObjectRef, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + }; + impl<'a> SupportFunc<'a> { + fn new( + vm: &VirtualMachine, + name: &'a str, + func: F, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + ) -> Self + where + F: IntoPyNativeFunc, + { + let func_obj = vm.ctx.new_rustfunc(func); + Self { + name, + func_obj, + fd, + dir_fd, + follow_symlinks, + } + } + } + let support_funcs = vec![ + SupportFunc::new(vm, "open", os_open, None, Some(false), None), + // access Some Some None + SupportFunc::new(vm, "chdir", os_chdir, Some(false), None, None), + // chflags Some, None Some + // chmod Some Some Some + // chown Some Some Some + // chroot Some None None + SupportFunc::new(vm, "listdir", os_listdir, Some(false), None, None), + SupportFunc::new(vm, "mkdir", os_mkdir, Some(false), Some(false), None), + // mkfifo Some Some None + // mknod Some Some None + // pathconf Some None None + // readlink Some Some None + SupportFunc::new(vm, "remove", os_remove, Some(false), Some(false), None), + // rename Some Some None + // replace Some Some None + SupportFunc::new(vm, "rmdir", os_rmdir, Some(false), Some(false), None), + SupportFunc::new(vm, "scandir", os_scandir, Some(false), None, None), + SupportFunc::new(vm, "stat", os_stat, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "symlink", os_symlink, None, Some(false), None), + // truncate Some None None + SupportFunc::new(vm, "unlink", os_remove, Some(false), Some(false), None), + // utime Some Some Some + ]; + let supports_fd = PySet::default().into_ref(vm); + let supports_dir_fd = PySet::default().into_ref(vm); + let supports_follow_symlinks = PySet::default().into_ref(vm); + + let module = py_module!(vm, "_os", { "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), "fsync" => ctx.new_rustfunc(os_fsync), "read" => ctx.new_rustfunc(os_read), "write" => ctx.new_rustfunc(os_write), - "remove" => ctx.new_rustfunc(os_remove), - "unlink" => ctx.new_rustfunc(os_remove), - "mkdir" => ctx.new_rustfunc(os_mkdir), "mkdirs" => ctx.new_rustfunc(os_mkdirs), - "rmdir" => ctx.new_rustfunc(os_rmdir), - "listdir" => ctx.new_rustfunc(os_listdir), "putenv" => ctx.new_rustfunc(os_putenv), "unsetenv" => ctx.new_rustfunc(os_unsetenv), "environ" => environ, "name" => ctx.new_str(os_name), - "scandir" => ctx.new_rustfunc(os_scandir), "ScandirIter" => scandir_iter, "DirEntry" => dir_entry, "stat_result" => stat_result, - "stat" => ctx.new_rustfunc(os_stat), - "symlink" => ctx.new_rustfunc(os_symlink), "getcwd" => ctx.new_rustfunc(os_getcwd), "chdir" => ctx.new_rustfunc(os_chdir), + "fspath" => ctx.new_rustfunc(os_fspath), "O_RDONLY" => ctx.new_int(0), "O_WRONLY" => ctx.new_int(1), "O_RDWR" => ctx.new_int(2), "O_NONBLOCK" => ctx.new_int(4), "O_APPEND" => ctx.new_int(8), "O_CREAT" => ctx.new_int(512) - }) + }); + + for support in support_funcs { + if support.fd.unwrap_or(false) { + supports_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.dir_fd.unwrap_or(false) { + supports_dir_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.follow_symlinks.unwrap_or(false) { + supports_follow_symlinks + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + vm.set_attr(&module, support.name, support.func_obj) + .unwrap(); + } + + extend_module!(vm, module, { + "supports_fd" => supports_fd.into_object(), + "supports_dir_fd" => supports_dir_fd.into_object(), + "supports_follow_symlinks" => supports_follow_symlinks.into_object(), + }); + + module } diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs new file mode 100644 index 000000000..1f5092540 --- /dev/null +++ b/vm/src/stdlib/pwd.rs @@ -0,0 +1,85 @@ +use pwd::Passwd; + +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +impl PyValue for Passwd { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("pwd", "struct_passwd") + } +} + +type PasswdRef = PyRef; + +impl PasswdRef { + fn pw_name(self, _vm: &VirtualMachine) -> String { + self.name.clone() + } + + fn pw_passwd(self, _vm: &VirtualMachine) -> Option { + self.passwd.clone() + } + + fn pw_uid(self, _vm: &VirtualMachine) -> u32 { + self.uid + } + + fn pw_gid(self, _vm: &VirtualMachine) -> u32 { + self.gid + } + + fn pw_gecos(self, _vm: &VirtualMachine) -> Option { + self.gecos.clone() + } + + fn pw_dir(self, _vm: &VirtualMachine) -> String { + self.dir.clone() + } + + fn pw_shell(self, _vm: &VirtualMachine) -> String { + self.shell.clone() + } +} + +fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + match Passwd::from_name(&name.value) { + Ok(Some(passwd)) => Ok(passwd), + _ => { + let name_repr = vm.to_repr(name.as_object())?; + let message = format!("getpwnam(): name not found: {}", name_repr); + Err(vm.new_key_error(message)) + } + } +} + +fn pwd_getpwuid(uid: u32, vm: &VirtualMachine) -> PyResult { + match Passwd::from_uid(uid) { + Some(passwd) => Ok(passwd), + _ => { + let message = format!("getpwuid(): uid not found: {}", uid); + Err(vm.new_key_error(message)) + } + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let passwd_type = py_class!(ctx, "struct_passwd", ctx.object(), { + "pw_name" => ctx.new_property(PasswdRef::pw_name), + "pw_passwd" => ctx.new_property(PasswdRef::pw_passwd), + "pw_uid" => ctx.new_property(PasswdRef::pw_uid), + "pw_gid" => ctx.new_property(PasswdRef::pw_gid), + "pw_gecos" => ctx.new_property(PasswdRef::pw_gecos), + "pw_dir" => ctx.new_property(PasswdRef::pw_dir), + "pw_shell" => ctx.new_property(PasswdRef::pw_shell), + }); + + py_module!(vm, "pwd", { + "struct_passwd" => passwd_type, + "getpwnam" => ctx.new_rustfunc(pwd_getpwnam), + "getpwuid" => ctx.new_rustfunc(pwd_getpwuid), + }) +} diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index b9f921502..f42738980 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -79,6 +79,10 @@ fn re_compile(pattern: PyStringRef, vm: &VirtualMachine) -> PyResult { make_regex(vm, &pattern.value) } +fn re_escape(pattern: PyStringRef, _vm: &VirtualMachine) -> String { + regex::escape(&pattern.value) +} + impl PyRegexRef { fn match_(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { do_match(vm, &self, &text.value) @@ -113,6 +117,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { py_module!(vm, "re", { "compile" => ctx.new_rustfunc(re_compile), + "escape" => ctx.new_rustfunc(re_escape), "Match" => match_type, "match" => ctx.new_rustfunc(re_match), "Pattern" => pattern_type, diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index edab574b3..68934bd01 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -218,7 +218,7 @@ impl SocketRef { } } - fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) -> () {} + fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) {} fn accept(self, vm: &VirtualMachine) -> PyResult { let ret = match self.con.borrow_mut().as_mut() { @@ -310,7 +310,7 @@ impl SocketRef { } } - fn close(self, _vm: &VirtualMachine) -> () { + fn close(self, _vm: &VirtualMachine) { self.con.borrow_mut().take(); } @@ -349,14 +349,14 @@ impl Address { impl TryFromObject for Address { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { let tuple = PyTupleRef::try_from_object(vm, obj)?; - if tuple.elements.borrow().len() != 2 { + if tuple.elements.len() != 2 { Err(vm.new_type_error("Address tuple should have only 2 values".to_string())) } else { Ok(Address { - host: PyStringRef::try_from_object(vm, tuple.elements.borrow()[0].clone())? + host: PyStringRef::try_from_object(vm, tuple.elements[0].clone())? .value .to_string(), - port: PyIntRef::try_from_object(vm, tuple.elements.borrow()[1].clone())? + port: PyIntRef::try_from_object(vm, tuple.elements[1].clone())? .as_bigint() .to_usize() .unwrap(), diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs deleted file mode 100644 index e224a93c6..000000000 --- a/vm/src/stdlib/types.rs +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Dynamic type creation and names for built in types. - */ - -use crate::function::OptionalArg; -use crate::obj::objdict::PyDict; -use crate::obj::objstr::PyStringRef; -use crate::obj::objtype; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::{PyIterable, PyObjectRef, PyResult, PyValue, TryFromObject}; -use crate::VirtualMachine; - -fn types_new_class( - name: PyStringRef, - bases: OptionalArg>, - vm: &VirtualMachine, -) -> PyResult { - // TODO kwds and exec_body parameter - - let bases = match bases { - OptionalArg::Present(bases) => bases, - OptionalArg::Missing => PyIterable::try_from_object(vm, vm.ctx.new_tuple(vec![]))?, - }; - let dict = PyDict::default().into_ref(vm); - objtype::type_new_class(vm, vm.ctx.type_type(), name, bases, dict) -} - -pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - - py_module!(vm, "types", { - "new_class" => ctx.new_rustfunc(types_new_class), - "FunctionType" => ctx.function_type(), - "MethodType" => ctx.bound_method_type(), - "LambdaType" => ctx.function_type(), - "CodeType" => ctx.code_type(), - "FrameType" => ctx.frame_type() - }) -} diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs new file mode 100644 index 000000000..cb97ead48 --- /dev/null +++ b/vm/src/stdlib/warnings.rs @@ -0,0 +1,37 @@ +use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +#[derive(FromArgs)] +struct WarnArgs { + #[pyarg(positional_only, optional = false)] + message: PyStringRef, + #[pyarg(positional_or_keyword, optional = true)] + category: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + stacklevel: OptionalArg, +} + +fn warnings_warn(args: WarnArgs, _vm: &VirtualMachine) { + // TODO: Implement correctly + let level = match args.stacklevel { + OptionalArg::Present(l) => l, + OptionalArg::Missing => 1, + }; + eprintln!( + "Warning: {} , category: {:?}, level: {}", + args.message.as_str(), + args.category, + level + ) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_warnings", { + "warn" => ctx.new_rustfunc(warnings_warn), + }); + + module +} diff --git a/vm/src/symboltable.rs b/vm/src/symboltable.rs index ea9ae52a0..9687b0ec7 100644 --- a/vm/src/symboltable.rs +++ b/vm/src/symboltable.rs @@ -16,7 +16,8 @@ pub fn make_symbol_table(program: &ast::Program) -> Result, @@ -101,7 +102,6 @@ fn analyze_symbol_table( symbol_scope: &SymbolScope, parent_symbol_scope: Option<&SymbolScope>, ) -> SymbolTableResult { - // println!("Analyzing {:?}, parent={:?} symbols={:?}", symbol_scope, parent_symbol_scope, symbol_scope.symbols); // Analyze sub scopes: for sub_scope in &symbol_scope.sub_scopes { analyze_symbol_table(&sub_scope, Some(symbol_scope))?; @@ -115,6 +115,7 @@ fn analyze_symbol_table( Ok(()) } +#[allow(clippy::single_match)] fn analyze_symbol( symbol_name: &str, symbol_role: &SymbolRole, @@ -157,7 +158,6 @@ impl SymbolTableBuilder { } pub fn enter_scope(&mut self) { - // Create new scope and push into scope stack. let scope = SymbolScope::new(); self.scopes.push(scope); } @@ -299,13 +299,15 @@ impl SymbolTableBuilder { ast::Statement::Import { import_parts } => { for part in import_parts { if let Some(alias) = &part.alias { + // `import mymodule as myalias` + // `from mymodule import myimportname as myalias` self.register_name(alias, SymbolRole::Assigned)?; + } else if let Some(symbol) = &part.symbol { + // `from mymodule import myimport` + self.register_name(symbol, SymbolRole::Assigned)?; } else { - if let Some(symbol) = &part.symbol { - self.register_name(symbol, SymbolRole::Assigned)?; - } else { - self.register_name(&part.module, SymbolRole::Assigned)?; - } + // `import module` + self.register_name(&part.module, SymbolRole::Assigned)?; } } } @@ -494,7 +496,7 @@ impl SymbolTableBuilder { } fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { - // Evaulate eventual default parameters: + // Evaluate eventual default parameters: self.scan_expressions(&args.defaults)?; for kw_default in &args.kw_defaults { if let Some(expression) = kw_default { @@ -541,13 +543,13 @@ impl SymbolTableBuilder { Ok(()) } + #[allow(clippy::single_match)] fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { let scope_depth = self.scopes.len(); let current_scope = self.scopes.last_mut().unwrap(); let location = Default::default(); - if let Some(_old_role) = current_scope.symbols.get(name) { + if current_scope.symbols.contains_key(name) { // Role already set.. - // debug!("TODO: {:?}", old_role); match role { SymbolRole::Global => { return Err(SymbolTableError { diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 8c6bafe74..f1cc0afcb 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -4,7 +4,7 @@ use std::{env, mem}; use crate::frame::FrameRef; use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objstr::PyStringRef; -use crate::pyobject::{IntoPyObject, ItemProtocol, PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* @@ -27,6 +27,44 @@ fn getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult PyResult { arg_check!(vm, args, required = [(object, None)]); let size = Rc::strong_count(&object); @@ -40,6 +78,21 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } +fn sys_getfilesystemencoding(_vm: &VirtualMachine) -> String { + // TODO: implmement non-utf-8 mode. + "utf-8".to_string() +} + +#[cfg(not(windows))] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogateescape".to_string() +} + +#[cfg(windows)] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogatepass".to_string() +} + // TODO implement string interning, this will be key for performance fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { value @@ -48,17 +101,38 @@ fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { let ctx = &vm.ctx; - let path_list = match env::var_os("PYTHONPATH") { - Some(paths) => env::split_paths(&paths) + let flags_type = SysFlags::make_class(ctx); + // TODO parse command line arguments and environment variables to populate SysFlags + let flags = SysFlags::default() + .into_struct_sequence(vm, flags_type) + .unwrap(); + + // TODO Add crate version to this namespace + let implementation = py_namespace!(vm, { + "name" => ctx.new_str("RustPython".to_string()), + "cache_tag" => ctx.new_str("rustpython-01".to_string()), + }); + + let path_list = if cfg!(target_arch = "wasm32") { + vec![] + } else { + let get_paths = |paths| match paths { + Some(paths) => env::split_paths(paths), + None => env::split_paths(""), + }; + + let rustpy_path = env::var_os("RUSTPYTHONPATH"); + let py_path = env::var_os("PYTHONPATH"); + get_paths(rustpy_path.as_ref()) + .chain(get_paths(py_path.as_ref())) .map(|path| { ctx.new_str( - path.to_str() - .expect("PYTHONPATH isn't valid unicode") - .to_string(), + path.into_os_string() + .into_string() + .expect("PYTHONPATH isn't valid unicode"), ) }) - .collect(), - None => vec![], + .collect() }; let path = ctx.new_list(path_list); @@ -152,8 +226,12 @@ settrace() -- set the global debug tracing function extend_module!(vm, module, { "argv" => argv(ctx), "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), + "flags" => flags, "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "implementation" => implementation, + "getfilesystemencoding" => ctx.new_rustfunc(sys_getfilesystemencoding), + "getfilesystemencodeerrors" => ctx.new_rustfunc(sys_getfilesystemencodeerrors), "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 814631015..eeba4a10d 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -16,6 +16,7 @@ use crate::builtins; use crate::bytecode; use crate::error::CompileError; use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; +use crate::frozen; use crate::function::PyFuncArgs; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; @@ -23,12 +24,14 @@ use crate::obj::objcode::PyCodeRef; use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; use crate::obj::objgenerator::PyGenerator; +use crate::obj::objint::PyInt; use crate::obj::objiter; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, @@ -51,6 +54,7 @@ pub struct VirtualMachine { pub frames: RefCell>, pub wasm_id: Option, pub exceptions: RefCell>, + pub frozen: RefCell>, } impl VirtualMachine { @@ -63,6 +67,7 @@ impl VirtualMachine { let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); + let frozen = RefCell::new(frozen::get_module_inits()); let vm = VirtualMachine { builtins: builtins.clone(), sys_module: sysmod.clone(), @@ -71,6 +76,7 @@ impl VirtualMachine { frames: RefCell::new(vec![]), wasm_id: None, exceptions: RefCell::new(vec![]), + frozen, }; builtins::make_module(&vm, builtins.clone()); @@ -165,14 +171,13 @@ impl VirtualMachine { self.invoke(exc_type.into_object(), args) } + /// Create Python instance of `exc_type` with message pub fn new_exception(&self, exc_type: PyClassRef, msg: String) -> PyObjectRef { // TODO: exc_type may be user-defined exception, so we should return PyResult // TODO: maybe there is a clearer way to create an instance: info!("New exception created: {}", msg); let pymsg = self.new_str(msg); let args: Vec = vec![pymsg]; - - // Call function: self.invoke(exc_type.into_object(), args).unwrap() } @@ -250,6 +255,11 @@ impl VirtualMachine { syntax_error } + pub fn new_import_error(&self, msg: String) -> PyObjectRef { + let import_error = self.ctx.exceptions.import_error.clone(); + self.new_exception(import_error, msg) + } + pub fn new_scope_with_builtins(&self) -> Scope { Scope::with_builtins(None, self.ctx.new_dict(), self) } @@ -396,13 +406,13 @@ impl VirtualMachine { scope: &Scope, defaults: &Option, kw_only_defaults: &Option, - args: PyFuncArgs, + func_args: PyFuncArgs, ) -> PyResult { - let scope = scope.child_scope(&self.ctx); + let scope = scope.new_child_scope(&self.ctx); self.fill_locals_from_args( &code.code, &scope.get_locals(), - args, + func_args, defaults, kw_only_defaults, )?; @@ -426,8 +436,8 @@ impl VirtualMachine { ) -> PyResult { if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope - .child_scope_with_locals(cells) - .child_scope_with_locals(locals); + .new_child_scope_with_locals(cells) + .new_child_scope_with_locals(locals); let frame = Frame::new(code.clone(), scope).into_ref(self); return self.run_frame_full(frame); } @@ -441,11 +451,11 @@ impl VirtualMachine { &self, code_object: &bytecode::CodeObject, locals: &PyDictRef, - args: PyFuncArgs, + func_args: PyFuncArgs, defaults: &Option, kw_only_defaults: &Option, ) -> PyResult<()> { - let nargs = args.args.len(); + let nargs = func_args.args.len(); let nexpected_args = code_object.arg_names.len(); // This parses the arguments from args and kwargs into @@ -463,7 +473,7 @@ impl VirtualMachine { // Copy positional arguments into local variables for i in 0..n { let arg_name = &code_object.arg_names[i]; - let arg = &args.args[i]; + let arg = &func_args.args[i]; locals.set_item(arg_name, arg.clone(), self)?; } @@ -472,7 +482,7 @@ impl VirtualMachine { bytecode::Varargs::Named(ref vararg_name) => { let mut last_args = vec![]; for i in n..nargs { - let arg = &args.args[i]; + let arg = &func_args.args[i]; last_args.push(arg.clone()); } let vararg_value = self.ctx.new_tuple(last_args); @@ -502,7 +512,7 @@ impl VirtualMachine { }; // Handle keyword arguments - for (name, value) in args.kwargs { + for (name, value) in func_args.kwargs { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { @@ -525,7 +535,7 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.borrow().len()); + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception @@ -545,7 +555,7 @@ impl VirtualMachine { ))); } if let Some(defaults) = defaults { - let defaults = defaults.elements.borrow(); + let defaults = &defaults.elements; // We have sufficient defaults, so iterate over the corresponding names and use // the default if we don't already have a value for (default_index, i) in (required_args..nexpected_args).enumerate() { @@ -578,10 +588,10 @@ impl VirtualMachine { pub fn extract_elements(&self, value: &PyObjectRef) -> PyResult> { // Extract elements from item, if possible: - let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) - || objtype::isinstance(value, &self.ctx.list_type()) - { - objsequence::get_elements(value).to_vec() + let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) { + objsequence::get_elements_tuple(value).to_vec() + } else if objtype::isinstance(value, &self.ctx.list_type()) { + objsequence::get_elements_list(value).to_vec() } else { let iter = objiter::get_iter(self, value)?; objiter::get_all(self, &iter)? @@ -912,6 +922,15 @@ impl VirtualMachine { }) } + pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { + let hash_obj = self.call_method(obj, "__hash__", vec![])?; + if objtype::isinstance(&hash_obj, &self.ctx.int_type()) { + Ok(hash_obj.payload::().unwrap().hash(self)) + } else { + Err(self.new_type_error("__hash__ method should return an integer".to_string())) + } + } + // https://docs.python.org/3/reference/expressions.html#membership-test-operations fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { let iter = objiter::get_iter(self, &haystack)?; @@ -937,7 +956,7 @@ impl VirtualMachine { } } - pub fn push_exception(&self, exc: PyObjectRef) -> () { + pub fn push_exception(&self, exc: PyObjectRef) { self.exceptions.borrow_mut().push(exc) } diff --git a/wasm/demo/src/main.js b/wasm/demo/src/main.js index f5034f62a..9b0fca8ef 100644 --- a/wasm/demo/src/main.js +++ b/wasm/demo/src/main.js @@ -82,12 +82,6 @@ snippets.addEventListener('change', updateSnippet); // option selected for the `select`, but the textarea won't be updated) updateSnippet(); -const prompt = '>>>>> '; - -const term = new Terminal(); -term.open(document.getElementById('terminal')); -term.write(prompt); - function removeNonAscii(str) { if (str === null || str === '') return false; else str = str.toString(); @@ -99,38 +93,80 @@ function printToConsole(data) { term.write(removeNonAscii(data) + '\r\n'); } +const term = new Terminal(); +term.open(document.getElementById('terminal')); + const terminalVM = rp.vmStore.init('term_vm'); terminalVM.setStdout(printToConsole); -var input = ''; +function getPrompt(name = 'ps1') { + terminalVM.exec(` +try: + import sys as __sys + __prompt = __sys.${name} +except: + __prompt = '' +finally: + del __sys +`); + return String(terminalVM.eval('__prompt')); +} + +term.write(getPrompt()); + +function resetInput() { + continuedInput = []; + input = ''; + continuing = false; +} + +let continuedInput, input, continuing; +resetInput(); + +let ps2; + term.on('data', data => { const code = data.charCodeAt(0); if (code == 13) { // CR - if (input[input.length - 1] == ':') { - input += data; - term.write('\r\n.....'); - } else { - term.write('\r\n'); - try { - terminalVM.execSingle(input); - } catch (err) { - if (err instanceof WebAssembly.RuntimeError) { - err = window.__RUSTPYTHON_ERROR || err; - } - printToConsole(err); + term.write('\r\n'); + continuedInput.push(input); + if (continuing) { + if (input === '') { + continuing = false; + } else { + input = ''; + term.write(ps2); + return; } - term.write(prompt); - input = ''; } - } else if (code == 127) { + try { + terminalVM.execSingle(continuedInput.join('\n')); + } catch (err) { + if (err instanceof SyntaxError && err.message.includes('EOF')) { + ps2 = getPrompt('ps2'); + term.write(ps2); + continuing = true; + input = ''; + return; + } else if (err instanceof WebAssembly.RuntimeError) { + err = window.__RUSTPYTHON_ERROR || err; + } + printToConsole(err); + } + resetInput(); + term.write(getPrompt()); + } else if (code == 127 || code == 8) { + // Backspace if (input.length > 0) { term.write('\b \b'); input = input.slice(0, -1); } - } else if (code < 32 || code == 127) { + } else if (code < 32) { // Control - return; + term.write('\r\n' + getPrompt()); + input = ''; + continuedInput = []; } else { // Visible term.write(data); diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 6f40fb182..a65141c57 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -40,10 +40,10 @@ pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { }); if let Ok(tb) = vm.get_attribute(py_err.clone(), "__traceback__") { if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let elements = objsequence::get_elements(&tb).to_vec(); + let elements = objsequence::get_elements_list(&tb).to_vec(); if let Some(top) = elements.get(0) { if objtype::isinstance(&top, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&top); + let element = objsequence::get_elements_list(&top); if let Some(lineno) = objint::to_int(vm, &element[1], 10) .ok() @@ -93,7 +93,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { let (key, val) = pair?; py_func_args .kwargs - .push((js_sys::JsString::from(key).into(), js_to_py(vm, val))); + .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); } } let result = vm.invoke(py_obj.clone(), py_func_args); diff --git a/wasm/tests/.travis-runner.sh b/wasm/tests/.travis-runner.sh index add96c6b2..40e1dee50 100755 --- a/wasm/tests/.travis-runner.sh +++ b/wasm/tests/.travis-runner.sh @@ -20,4 +20,4 @@ export PATH=$PATH:$PWD/geckodriver pip install pipenv (cd wasm/tests; pipenv install) -(cd wasm/demo; npm install; npm run ci) +(cd wasm/demo; npm install; npm run build; npm run ci)