diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7bd1504fc..d30eb909f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -25,15 +25,21 @@ jobs:
- name: Cache cargo dependencies
uses: actions/cache@v2
with:
- key: ${{ runner.os }}-rust_tests-${{ hashFiles('Cargo.lock') }}
- path: target
- restore-keys: |
- ${{ runner.os }}-rust_tests-
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
- name: run rust tests
uses: actions-rs/cargo@v1
with:
command: test
args: --verbose ${{ env.CARGO_ARGS }}
+ - name: check compilation without threading
+ uses: actions-rs/cargo@v1
+ with:
+ command: check
+ args: ${{ env.CARGO_ARGS }} --no-default-features
snippets_cpython:
name: Run snippets and cpython tests
@@ -51,10 +57,11 @@ jobs:
- name: Cache cargo dependencies
uses: actions/cache@v2
with:
- key: ${{ runner.os }}-snippets-${{ hashFiles('Cargo.lock') }}
- path: target
- restore-keys: |
- ${{ runner.os }}-snippets-
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-release-${{ hashFiles('**/Cargo.lock') }}
- name: build rustpython
uses: actions-rs/cargo@v1
with:
@@ -77,9 +84,14 @@ jobs:
run: target/release/rustpython -m test -v
env:
RUSTPYTHONPATH: ${{ github.workspace }}/Lib
- if: runner.os != 'Windows'
+ if: runner.os == 'Linux'
+ - name: run cpython tests (macOS lightweight)
+ run: target/release/rustpython -m test -x test_argparse -x test_json -x test_bytes -x test_long -v
+ env:
+ RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+ if: runner.os == 'macOS'
- format:
+ lint:
name: Check Rust code with rustfmt and clippy
runs-on: ubuntu-latest
steps:
@@ -100,12 +112,6 @@ jobs:
with:
command: clippy
args: ${{ env.CARGO_ARGS }} -- -Dwarnings
-
- lint:
- name: Lint Python code with flake8
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@master
- uses: actions/setup-python@v1
with:
python-version: 3.8
@@ -116,16 +122,20 @@ jobs:
wasm:
name: Check the WASM package and demo
+ needs: rust_tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Cache cargo dependencies
uses: actions/cache@v2
with:
- key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }}
- path: target
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-wasm_opt3-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
- ${{ runner.os }}-wasm-
+ ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
- name: install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: install geckodriver
diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile
index 546e02fd1..0a54e9d39 100644
--- a/.gitpod.Dockerfile
+++ b/.gitpod.Dockerfile
@@ -11,4 +11,11 @@ RUN rm -rf ~/.rustup && \
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
rustup target add wasm32-unknown-unknown
+RUN sudo apt-get -q update \
+ && sudo apt-get install -yq \
+ libpython3.6 \
+ rust-lldb \
+ && sudo rm -rf /var/lib/apt/lists/*
+ENV RUST_LLDB=/usr/bin/lldb-8
+
USER root
diff --git a/.gitpod.yml b/.gitpod.yml
index 527c76d39..7f2eea913 100644
--- a/.gitpod.yml
+++ b/.gitpod.yml
@@ -1,2 +1,6 @@
image:
file: .gitpod.Dockerfile
+
+vscode:
+ extensions:
+ - vadimcn.vscode-lldb@1.5.3:vTh/rWhvJ5nQpeAVsD20QA==
\ No newline at end of file
diff --git a/.theia/launch.json b/.theia/launch.json
new file mode 100644
index 000000000..699c72ee9
--- /dev/null
+++ b/.theia/launch.json
@@ -0,0 +1,16 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "type": "lldb",
+ "request": "launch",
+ "name": "Debug Rust Code",
+ //"preLaunchTask": "cargo",
+ "program": "${workspaceFolder}/target/debug/rustpython",
+ "cwd": "${workspaceFolder}",
+ //"valuesFormatting": "parseText"
+ }
+ ]
+}
diff --git a/.theia/settings.json b/.theia/settings.json
new file mode 100644
index 000000000..83db8fc48
--- /dev/null
+++ b/.theia/settings.json
@@ -0,0 +1,8 @@
+{
+ "cpp.buildConfigurations": [
+ {
+ "name": "",
+ "directory": ""
+ },
+ ]
+}
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 156c88a12..efe6c395c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7,16 +7,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
[[package]]
-name = "adler32"
-version = "1.0.4"
+name = "adler"
+version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
+checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+
+[[package]]
+name = "adler32"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d"
[[package]]
name = "aho-corasick"
-version = "0.7.10"
+version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada"
+checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
dependencies = [
"memchr",
]
@@ -47,8 +53,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0609c78bd572f4edc74310dfb63a01f5609d53fa8b4dd7c4d98aef3b3e8d72d1"
dependencies = [
"proc-macro-hack",
- "quote 1.0.3",
- "syn 1.0.17",
+ "quote",
+ "syn",
]
[[package]]
@@ -57,15 +63,6 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
-[[package]]
-name = "arrayvec"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
-dependencies = [
- "nodrop",
-]
-
[[package]]
name = "arrayvec"
version = "0.5.1"
@@ -92,40 +89,12 @@ dependencies = [
"winapi",
]
-[[package]]
-name = "autocfg"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
-
[[package]]
name = "autocfg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
-[[package]]
-name = "backtrace"
-version = "0.3.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4ed64ae6d9ebfd9893193c4b2532b1292ec97bd8271c9d7d0fa90cd78a34cba"
-dependencies = [
- "backtrace-sys",
- "cfg-if",
- "libc",
- "rustc-demangle",
-]
-
-[[package]]
-name = "backtrace-sys"
-version = "0.1.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118"
-dependencies = [
- "cc",
- "libc",
-]
-
[[package]]
name = "base64"
version = "0.11.0"
@@ -134,28 +103,28 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]]
name = "bincode"
-version = "1.2.1"
+version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf"
+checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d"
dependencies = [
- "byteorder 1.3.4",
+ "byteorder",
"serde",
]
[[package]]
name = "bit-set"
-version = "0.5.1"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
+checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
-version = "0.5.1"
+version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
+checksum = "5f0dc55f2d8a1a85650ac47858bb001b4c0dd73d79e3c455a842925e68d29cd3"
[[package]]
name = "bitflags"
@@ -182,7 +151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
dependencies = [
"arrayref",
- "arrayvec 0.5.1",
+ "arrayvec",
"constant_time_eq",
]
@@ -194,7 +163,7 @@ checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
- "byteorder 1.3.4",
+ "byteorder",
"generic-array",
]
@@ -209,9 +178,9 @@ dependencies = [
[[package]]
name = "bstr"
-version = "0.2.12"
+version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41"
+checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
dependencies = [
"lazy_static 1.4.0",
"memchr",
@@ -227,9 +196,9 @@ checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
[[package]]
name = "bumpalo"
-version = "3.2.1"
+version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187"
+checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
[[package]]
name = "byte-tools"
@@ -237,12 +206,6 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
-[[package]]
-name = "byteorder"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
-
[[package]]
name = "byteorder"
version = "1.3.4"
@@ -261,9 +224,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.0.50"
+version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
+checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518"
[[package]]
name = "cfg-if"
@@ -273,9 +236,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "chrono"
-version = "0.4.11"
+version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2"
+checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6"
dependencies = [
"js-sys",
"num-integer",
@@ -286,9 +249,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "2.33.0"
+version = "2.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
+checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129"
dependencies = [
"ansi_term",
"atty",
@@ -299,15 +262,6 @@ dependencies = [
"vec_map",
]
-[[package]]
-name = "cloudabi"
-version = "0.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
-dependencies = [
- "bitflags",
-]
-
[[package]]
name = "cloudabi"
version = "0.1.0"
@@ -358,7 +312,7 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"cfg-if",
"lazy_static 1.4.0",
]
@@ -401,9 +355,9 @@ version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f"
dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
@@ -432,16 +386,6 @@ dependencies = [
"winapi",
]
-[[package]]
-name = "dirs"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c4ef5a8b902d393339e2a2c7fe573af92ce7e0ee5a3ff827b4c9ad7e07e4fa1"
-dependencies = [
- "cfg-if",
- "dirs-sys",
-]
-
[[package]]
name = "dirs-next"
version = "1.0.1"
@@ -452,18 +396,6 @@ dependencies = [
"dirs-sys-next",
]
-[[package]]
-name = "dirs-sys"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_users",
- "winapi",
-]
-
[[package]]
name = "dirs-sys-next"
version = "0.1.0"
@@ -477,9 +409,9 @@ dependencies = [
[[package]]
name = "dns-lookup"
-version = "1.0.1"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13988670860b076248c74e1b54444efc4f1dec70c8bb25da4b7c0024396b72bf"
+checksum = "4cbea23c15f7e685a7ad01c994407f3849fa7c88556aaafe4054215b6df93f78"
dependencies = [
"libc",
"socket2",
@@ -506,9 +438,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]]
name = "ena"
-version = "0.13.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36"
+checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
dependencies = [
"log",
]
@@ -532,28 +464,6 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
-[[package]]
-name = "failure"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b"
-dependencies = [
- "backtrace",
- "failure_derive",
-]
-
-[[package]]
-name = "failure_derive"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231"
-dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
- "synstructure",
-]
-
[[package]]
name = "fake-simd"
version = "0.1.2"
@@ -562,9 +472,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fixedbitset"
-version = "0.1.9"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
+checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
[[package]]
name = "flame"
@@ -581,13 +491,13 @@ dependencies = [
[[package]]
name = "flamer"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2add1a5e84b1ed7b5d00cdc21789a28e0a8f4e427b677313c773880ba3c4dac"
+checksum = "36b732da54fd4ea34452f2431cf464ac7be94ca4b339c9cd3d3d12eb06fe7aab"
dependencies = [
"flame",
- "quote 0.6.13",
- "syn 0.15.44",
+ "quote",
+ "syn",
]
[[package]]
@@ -604,9 +514,9 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.0.14"
+version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
+checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e"
dependencies = [
"cfg-if",
"crc32fast",
@@ -617,9 +527,9 @@ dependencies = [
[[package]]
name = "fnv"
-version = "1.0.6"
+version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
@@ -636,12 +546,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-[[package]]
-name = "fuchsia-cprng"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-
[[package]]
name = "futures"
version = "0.1.29"
@@ -650,9 +554,9 @@ checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef"
[[package]]
name = "generational-arena"
-version = "0.2.3"
+version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "921c3803adaeb9f9639de5149d9f0f9f4b79f00c423915b701db2e02ed80b9ce"
+checksum = "8e1d3b771574f62d0548cee0ad9057857e9fc25d7a3335f140c84f6acd0bf601"
dependencies = [
"cfg-if",
]
@@ -689,10 +593,19 @@ dependencies = [
]
[[package]]
-name = "hermit-abi"
-version = "0.1.10"
+name = "hashbrown"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e"
+checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9"
dependencies = [
"libc",
]
@@ -720,18 +633,24 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "1.3.2"
+version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292"
+checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
+ "hashbrown",
]
[[package]]
name = "instant"
-version = "0.1.4"
+version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7777a24a1ce5de49fcdde84ec46efa487c3af49d5b6e6e0a50367cc5c1096182"
+checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
[[package]]
name = "is-macro"
@@ -741,31 +660,31 @@ checksum = "04807f3dc9e3ea39af3f8469a5297267faf94859637afb836b33f47d9b2650ee"
dependencies = [
"Inflector",
"pmutil",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
name = "itertools"
-version = "0.8.2"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
+checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
dependencies = [
"either",
]
[[package]]
name = "itoa"
-version = "0.4.5"
+version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
+checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
[[package]]
name = "js-sys"
-version = "0.3.37"
+version = "0.3.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055"
+checksum = "52732a3d3ad72c58ad2dc70624f9c17b46ecd0943b9a4f1ee37c4c18c5d983e2"
dependencies = [
"wasm-bindgen",
]
@@ -778,9 +697,9 @@ checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7"
[[package]]
name = "lalrpop"
-version = "0.17.2"
+version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64dc3698e75d452867d9bd86f4a723f452ce9d01fe1d55990b79f0c790aa67db"
+checksum = "d6f55673d283313791404be21209bb433f128f7e5c451986df107eb5fdbd68d2"
dependencies = [
"ascii-canvas",
"atty",
@@ -798,14 +717,14 @@ dependencies = [
"sha2",
"string_cache",
"term",
- "unicode-xid 0.1.0",
+ "unicode-xid",
]
[[package]]
name = "lalrpop-util"
-version = "0.17.2"
+version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c277d18683b36349ab5cd030158b54856fca6bb2d5dc5263b06288f486958b7c"
+checksum = "f7e88f15a7d31dfa8fb607986819039127f0161058a3b248a146142d276cbd28"
[[package]]
name = "lazy_static"
@@ -819,35 +738,24 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-[[package]]
-name = "lexical"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0afaeae1c07c575338ef6809875bfea8daa9ea8b2ee381ef1f93ba0c6e32f003"
-dependencies = [
- "cfg-if",
- "lexical-core",
- "rustc_version",
-]
-
[[package]]
name = "lexical-core"
-version = "0.6.2"
+version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7043aa5c05dd34fb73b47acb8c3708eac428de4545ea3682ed2f11293ebd890"
+checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
dependencies = [
- "arrayvec 0.4.12",
+ "arrayvec",
+ "bitflags",
"cfg-if",
- "rustc_version",
"ryu",
"static_assertions",
]
[[package]]
name = "libc"
-version = "0.2.71"
+version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49"
+checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9"
[[package]]
name = "libz-sys"
@@ -863,31 +771,27 @@ dependencies = [
[[package]]
name = "lock_api"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e"
+checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
-version = "0.4.8"
+version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
+checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
dependencies = [
"cfg-if",
]
[[package]]
-name = "lz4-compress"
-version = "0.1.1"
+name = "lz4-compression"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f966533a922a9bba9e95e594c1fdb3b9bf5fdcdb11e37e51ad84cd76e468b91"
-dependencies = [
- "byteorder 0.5.3",
- "quick-error",
-]
+checksum = "761104bf97f13a3caf47d822498a0760a10d00d220148bac2669f63fc3bb8270"
[[package]]
name = "maplit"
@@ -920,11 +824,11 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]]
name = "miniz_oxide"
-version = "0.3.6"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5"
+checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f"
dependencies = [
- "adler32",
+ "adler",
]
[[package]]
@@ -933,8 +837,8 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c674293daac706360a8fa633c802ca15d27ee4a52394f12ecec2f6d2aa5508bf"
dependencies = [
- "rand 0.7.3",
- "rand_core 0.5.1",
+ "rand",
+ "rand_core",
]
[[package]]
@@ -956,19 +860,13 @@ dependencies = [
"void",
]
-[[package]]
-name = "nodrop"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
-
[[package]]
name = "num-bigint"
-version = "0.2.6"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+checksum = "b7f3fc75e3697059fb1bc465e3d8cca6cf92f56854f201158b3f9c77d5a3cfa0"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"num-integer",
"num-traits",
"serde",
@@ -980,39 +878,39 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"num-traits",
"serde",
]
[[package]]
name = "num-integer"
-version = "0.1.42"
+version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba"
+checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"num-traits",
]
[[package]]
name = "num-iter"
-version = "0.1.40"
+version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00"
+checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
-version = "0.2.4"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
+checksum = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"num-bigint",
"num-integer",
"num-traits",
@@ -1020,18 +918,18 @@ dependencies = [
[[package]]
name = "num-traits"
-version = "0.2.11"
+version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096"
+checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
]
[[package]]
name = "num_cpus"
-version = "1.12.0"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
+checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
@@ -1054,16 +952,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d"
dependencies = [
"proc-macro-crate",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
name = "once_cell"
-version = "1.3.1"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b"
+checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
[[package]]
name = "opaque-debug"
@@ -1073,9 +971,9 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl"
-version = "0.10.29"
+version = "0.10.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd"
+checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
dependencies = [
"bitflags",
"cfg-if",
@@ -1093,20 +991,20 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
[[package]]
name = "openssl-src"
-version = "111.8.1+1.1.1f"
+version = "111.10.2+1.1.1g"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f04f0299a91de598dde58d2e99101895498dcf3d58896a3297798f28b27c8b72"
+checksum = "a287fdb22e32b5b60624d4a5a7a02dbe82777f730ec0dbc42a0554326fef5a70"
dependencies = [
"cc",
]
[[package]]
name = "openssl-sys"
-version = "0.9.55"
+version = "0.9.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8"
+checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
dependencies = [
- "autocfg 1.0.0",
+ "autocfg",
"cc",
"libc",
"openssl-src",
@@ -1114,12 +1012,6 @@ dependencies = [
"vcpkg",
]
-[[package]]
-name = "ordermap"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063"
-
[[package]]
name = "parking_lot"
version = "0.11.0"
@@ -1138,7 +1030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
dependencies = [
"cfg-if",
- "cloudabi 0.1.0",
+ "cloudabi",
"instant",
"libc",
"redox_syscall",
@@ -1148,9 +1040,9 @@ dependencies = [
[[package]]
name = "paste"
-version = "0.1.10"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab4fb1930692d1b6a9cfabdde3d06ea0a7d186518e2f4d67660d8970e2fa647a"
+checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880"
dependencies = [
"paste-impl",
"proc-macro-hack",
@@ -1158,50 +1050,37 @@ dependencies = [
[[package]]
name = "paste-impl"
-version = "0.1.10"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a"
+checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6"
dependencies = [
"proc-macro-hack",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
]
[[package]]
name = "petgraph"
-version = "0.4.13"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f"
+checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
dependencies = [
"fixedbitset",
- "ordermap",
-]
-
-[[package]]
-name = "phf_generator"
-version = "0.7.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
-dependencies = [
- "phf_shared",
- "rand 0.6.5",
+ "indexmap",
]
[[package]]
name = "phf_shared"
-version = "0.7.24"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
dependencies = [
"siphasher",
]
[[package]]
name = "pkg-config"
-version = "0.3.17"
+version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
+checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33"
[[package]]
name = "pmutil"
@@ -1209,16 +1088,16 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004"
dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
name = "ppv-lite86"
-version = "0.2.6"
+version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
+checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea"
[[package]]
name = "precomputed-hash"
@@ -1228,45 +1107,26 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro-crate"
-version = "0.1.4"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e"
+checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
dependencies = [
"toml",
]
[[package]]
name = "proc-macro-hack"
-version = "0.5.15"
+version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63"
+checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
[[package]]
name = "proc-macro2"
-version = "0.4.30"
+version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
+checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
dependencies = [
- "unicode-xid 0.1.0",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3"
-dependencies = [
- "unicode-xid 0.2.0",
-]
-
-[[package]]
-name = "pwd"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dd32d8bece608e144ca20251e714ed107cdecdabb20c2d383cfc687825106a5"
-dependencies = [
- "failure",
- "libc",
+ "unicode-xid",
]
[[package]]
@@ -1287,39 +1147,11 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quote"
-version = "0.6.13"
+version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
+checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
dependencies = [
- "proc-macro2 0.4.30",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f"
-dependencies = [
- "proc-macro2 1.0.10",
-]
-
-[[package]]
-name = "rand"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
-dependencies = [
- "autocfg 0.1.7",
- "libc",
- "rand_chacha 0.1.1",
- "rand_core 0.4.2",
- "rand_hc 0.1.0",
- "rand_isaac",
- "rand_jitter",
- "rand_os",
- "rand_pcg",
- "rand_xorshift",
- "winapi",
+ "proc-macro2",
]
[[package]]
@@ -1330,19 +1162,9 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"libc",
- "rand_chacha 0.2.2",
- "rand_core 0.5.1",
- "rand_hc 0.2.0",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
-dependencies = [
- "autocfg 0.1.7",
- "rand_core 0.3.1",
+ "rand_chacha",
+ "rand_core",
+ "rand_hc",
]
[[package]]
@@ -1352,24 +1174,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
- "rand_core 0.5.1",
+ "rand_core",
]
-[[package]]
-name = "rand_core"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
-dependencies = [
- "rand_core 0.4.2",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
-
[[package]]
name = "rand_core"
version = "0.5.1"
@@ -1379,91 +1186,20 @@ dependencies = [
"getrandom",
]
-[[package]]
-name = "rand_hc"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
-dependencies = [
- "rand_core 0.3.1",
-]
-
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
- "rand_core 0.5.1",
-]
-
-[[package]]
-name = "rand_isaac"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
-dependencies = [
- "rand_core 0.3.1",
-]
-
-[[package]]
-name = "rand_jitter"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
-dependencies = [
- "libc",
- "rand_core 0.4.2",
- "winapi",
-]
-
-[[package]]
-name = "rand_os"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
-dependencies = [
- "cloudabi 0.0.3",
- "fuchsia-cprng",
- "libc",
- "rand_core 0.4.2",
- "rdrand",
- "winapi",
-]
-
-[[package]]
-name = "rand_pcg"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
-dependencies = [
- "autocfg 0.1.7",
- "rand_core 0.4.2",
-]
-
-[[package]]
-name = "rand_xorshift"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
-dependencies = [
- "rand_core 0.3.1",
-]
-
-[[package]]
-name = "rdrand"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
-dependencies = [
- "rand_core 0.3.1",
+ "rand_core",
]
[[package]]
name = "redox_syscall"
-version = "0.1.56"
+version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_users"
@@ -1478,9 +1214,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.3.6"
+version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3"
+checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [
"aho-corasick",
"memchr",
@@ -1494,14 +1230,14 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
dependencies = [
- "byteorder 1.3.4",
+ "byteorder",
]
[[package]]
name = "regex-syntax"
-version = "0.6.17"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae"
+checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
[[package]]
name = "result-like"
@@ -1524,12 +1260,6 @@ dependencies = [
"crossbeam-utils",
]
-[[package]]
-name = "rustc-demangle"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
-
[[package]]
name = "rustc_version"
version = "0.2.3"
@@ -1564,6 +1294,7 @@ dependencies = [
"num-traits",
"rustpython-compiler",
"rustpython-parser",
+ "rustpython-pylib",
"rustpython-vm",
"rustyline",
]
@@ -1575,17 +1306,30 @@ dependencies = [
"bincode",
"bitflags",
"itertools",
- "lz4-compress",
+ "lz4-compression",
"num-bigint",
"num-complex",
"serde",
]
+[[package]]
+name = "rustpython-common"
+version = "0.0.0"
+dependencies = [
+ "cfg-if",
+ "hexf-parse",
+ "lexical-core",
+ "num-bigint",
+ "num-traits",
+ "parking_lot",
+ "rand",
+]
+
[[package]]
name = "rustpython-compiler"
version = "0.1.2"
dependencies = [
- "arrayvec 0.5.1",
+ "arrayvec",
"indexmap",
"itertools",
"log",
@@ -1600,11 +1344,11 @@ version = "0.1.2"
dependencies = [
"maplit",
"once_cell",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
+ "proc-macro2",
+ "quote",
"rustpython-bytecode",
"rustpython-compiler",
- "syn 1.0.17",
+ "syn",
]
[[package]]
@@ -1625,7 +1369,6 @@ dependencies = [
name = "rustpython-pylib"
version = "0.1.0"
dependencies = [
- "maplit",
"rustpython-bytecode",
"rustpython-derive",
]
@@ -1640,7 +1383,7 @@ dependencies = [
"bitflags",
"blake2",
"bstr",
- "byteorder 1.3.4",
+ "byteorder",
"caseless",
"cfg-if",
"chrono",
@@ -1663,7 +1406,7 @@ dependencies = [
"indexmap",
"is-macro",
"itertools",
- "lexical",
+ "lexical-core",
"libc",
"libz-sys",
"log",
@@ -1685,13 +1428,13 @@ dependencies = [
"openssl-sys",
"parking_lot",
"paste",
- "pwd",
- "rand 0.7.3",
- "rand_core 0.5.1",
+ "rand",
+ "rand_core",
"regex",
"result-like",
"rustc_version_runtime",
"rustpython-bytecode",
+ "rustpython-common",
"rustpython-compiler",
"rustpython-derive",
"rustpython-parser",
@@ -1722,13 +1465,6 @@ dependencies = [
"winreg",
]
-[[package]]
-name = "rustpython_freeze"
-version = "0.1.2"
-dependencies = [
- "rustpython-vm",
-]
-
[[package]]
name = "rustpython_wasm"
version = "0.1.2"
@@ -1737,6 +1473,7 @@ dependencies = [
"futures",
"generational-arena",
"js-sys",
+ "parking_lot",
"rustpython-compiler",
"rustpython-parser",
"rustpython-vm",
@@ -1749,16 +1486,17 @@ dependencies = [
[[package]]
name = "rustyline"
-version = "6.1.0"
+version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6155a1169c281a8b75ee943a83eef0e2bb63aff057688c2111be319e51fc9f63"
+checksum = "3358c21cbbc1a751892528db4e1de4b7a2b6a73f001e215aaba97d712cfa9777"
dependencies = [
"cfg-if",
- "dirs 2.0.1",
+ "dirs-next",
"libc",
"log",
"memchr",
"nix",
+ "scopeguard",
"unicode-segmentation",
"unicode-width",
"utf8parse",
@@ -1767,15 +1505,15 @@ dependencies = [
[[package]]
name = "ryu"
-version = "1.0.3"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "schannel"
-version = "0.1.18"
+version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19"
+checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [
"lazy_static 1.4.0",
"winapi",
@@ -1804,9 +1542,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
-version = "1.0.106"
+version = "1.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399"
+checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
dependencies = [
"serde_derive",
]
@@ -1825,20 +1563,20 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.106"
+version = "1.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c"
+checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
name = "serde_json"
-version = "1.0.51"
+version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9"
+checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3"
dependencies = [
"itoa",
"ryu",
@@ -1859,9 +1597,9 @@ dependencies = [
[[package]]
name = "sha2"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0"
+checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
dependencies = [
"block-buffer",
"digest",
@@ -1884,9 +1622,9 @@ dependencies = [
[[package]]
name = "siphasher"
-version = "0.2.3"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
+checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7"
[[package]]
name = "smallbox"
@@ -1896,9 +1634,9 @@ checksum = "aa0eec6425219a55511a35afac416a2eecc62f5fa1bec14f9b97de0caa7daebe"
[[package]]
name = "smallvec"
-version = "1.4.0"
+version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4"
+checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f"
[[package]]
name = "socket2"
@@ -1914,9 +1652,9 @@ dependencies = [
[[package]]
name = "static_assertions"
-version = "0.3.4"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "statrs"
@@ -1924,43 +1662,22 @@ version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cce16f6de653e88beca7bd13780d08e09d4489dbca1f9210e041bc4852481382"
dependencies = [
- "rand 0.7.3",
+ "rand",
]
[[package]]
name = "string_cache"
-version = "0.7.5"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67"
+checksum = "2940c75beb4e3bf3a494cef919a747a2cb81e52571e212bfbd185074add7208a"
dependencies = [
"lazy_static 1.4.0",
"new_debug_unreachable",
"phf_shared",
"precomputed-hash",
"serde",
- "string_cache_codegen",
- "string_cache_shared",
]
-[[package]]
-name = "string_cache_codegen"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6"
-dependencies = [
- "phf_generator",
- "phf_shared",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "string_cache_shared",
-]
-
-[[package]]
-name = "string_cache_shared"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
-
[[package]]
name = "strsim"
version = "0.8.0"
@@ -1981,36 +1698,13 @@ checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee"
[[package]]
name = "syn"
-version = "0.15.44"
+version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
+checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0"
dependencies = [
- "proc-macro2 0.4.30",
- "quote 0.6.13",
- "unicode-xid 0.1.0",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03"
-dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "unicode-xid 0.2.0",
-]
-
-[[package]]
-name = "synstructure"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
-dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
- "unicode-xid 0.2.0",
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
]
[[package]]
@@ -2019,8 +1713,8 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42"
dependencies = [
- "byteorder 1.3.4",
- "dirs 1.0.5",
+ "byteorder",
+ "dirs",
"winapi",
]
@@ -2072,6 +1766,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "tinyvec"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed"
+
[[package]]
name = "toml"
version = "0.5.6"
@@ -2083,9 +1783,9 @@ dependencies = [
[[package]]
name = "typenum"
-version = "1.11.2"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9"
+checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
[[package]]
name = "uname"
@@ -2220,11 +1920,11 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56"
[[package]]
name = "unicode-normalization"
-version = "0.1.12"
+version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4"
+checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977"
dependencies = [
- "smallvec",
+ "tinyvec",
]
[[package]]
@@ -2235,21 +1935,15 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
[[package]]
name = "unicode-width"
-version = "0.1.7"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
-version = "0.1.0"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "unicode_names2"
@@ -2275,15 +1969,15 @@ dependencies = [
[[package]]
name = "vcpkg"
-version = "0.2.8"
+version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
+checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
[[package]]
name = "vec_map"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "void"
@@ -2305,9 +1999,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasm-bindgen"
-version = "0.2.60"
+version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f"
+checksum = "f3edbcc9536ab7eababcc6d2374a0b7bfe13a2b6d562c5e07f370456b1a8f33d"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@@ -2315,16 +2009,16 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.60"
+version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd"
+checksum = "89ed2fb8c84bfad20ea66b26a3743f3e7ba8735a69fe7d95118c33ec8fc1244d"
dependencies = [
"bumpalo",
"lazy_static 1.4.0",
"log",
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
"wasm-bindgen-shared",
]
@@ -2343,38 +2037,38 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.60"
+version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4"
+checksum = "eb071268b031a64d92fc6cf691715ca5a40950694d8f683c5bb43db7c730929e"
dependencies = [
- "quote 1.0.3",
+ "quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.60"
+version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931"
+checksum = "cf592c807080719d1ff2f245a687cbadb3ed28b2077ed7084b47aba8b691f2c6"
dependencies = [
- "proc-macro2 1.0.10",
- "quote 1.0.3",
- "syn 1.0.17",
+ "proc-macro2",
+ "quote",
+ "syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.60"
+version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639"
+checksum = "72b6c0220ded549d63860c78c38f3bcc558d1ca3f4efa74942c536ddbbb55e87"
[[package]]
name = "web-sys"
-version = "0.3.37"
+version = "0.3.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb"
+checksum = "8be2398f326b7ba09815d0b403095f34dd708579220d099caae89be0b32137b2"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -2382,9 +2076,9 @@ dependencies = [
[[package]]
name = "winapi"
-version = "0.3.8"
+version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
@@ -2398,9 +2092,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
-version = "0.1.4"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
diff --git a/Cargo.toml b/Cargo.toml
index 629001891..d99b23c33 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,17 +6,20 @@ edition = "2018"
description = "A python interpreter written in rust."
repository = "https://github.com/RustPython/RustPython"
license = "MIT"
+include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
[workspace]
-members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate"]
+members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common"]
[[bench]]
name = "bench"
path = "./benchmarks/bench.rs"
[features]
+default = ["threading", "pylib"]
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+threading = ["rustpython-vm/threading"]
ssl = ["rustpython-vm/ssl"]
@@ -24,9 +27,10 @@ ssl = ["rustpython-vm/ssl"]
log = "0.4"
env_logger = "0.7"
clap = "2.33"
-rustpython-compiler = {path = "compiler", version = "0.1.1"}
-rustpython-parser = {path = "parser", version = "0.1.1"}
-rustpython-vm = {path = "vm", version = "0.1.1"}
+rustpython-compiler = { path = "compiler", version = "0.1.1" }
+rustpython-parser = { path = "parser", version = "0.1.1" }
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
+pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true }
dirs = { package = "dirs-next", version = "1.0" }
num-traits = "0.2.8"
cfg-if = "0.1"
@@ -45,6 +49,9 @@ version = "0.2"
name = "rustpython"
path = "src/main.rs"
+[profile.dev.package."*"]
+opt-level = 3
+
[patch.crates-io]
# REDOX START, Uncommment when you want to compile/check with redoxer
# # following patches are just waiting on a new version to be released to crates.io
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py
new file mode 100644
index 000000000..db6674ea2
--- /dev/null
+++ b/Lib/_osx_support.py
@@ -0,0 +1,502 @@
+"""Shared OS X support functions."""
+
+import os
+import re
+import sys
+
+__all__ = [
+ 'compiler_fixup',
+ 'customize_config_vars',
+ 'customize_compiler',
+ 'get_platform_osx',
+]
+
+# configuration variables that may contain universal build flags,
+# like "-arch" or "-isdkroot", that may need customization for
+# the user environment
+_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
+ 'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
+ 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
+ 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS')
+
+# configuration variables that may contain compiler calls
+_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
+
+# prefix added to original configuration variable names
+_INITPRE = '_OSX_SUPPORT_INITIAL_'
+
+
+def _find_executable(executable, path=None):
+ """Tries to find 'executable' in the directories listed in 'path'.
+
+ A string listing directories separated by 'os.pathsep'; defaults to
+ os.environ['PATH']. Returns the complete filename or None if not found.
+ """
+ if path is None:
+ path = os.environ['PATH']
+
+ paths = path.split(os.pathsep)
+ base, ext = os.path.splitext(executable)
+
+ if (sys.platform == 'win32') and (ext != '.exe'):
+ executable = executable + '.exe'
+
+ if not os.path.isfile(executable):
+ for p in paths:
+ f = os.path.join(p, executable)
+ if os.path.isfile(f):
+ # the file exists, we have a shot at spawn working
+ return f
+ return None
+ else:
+ return executable
+
+
+def _read_output(commandstring):
+ """Output from successful command execution or None"""
+ # Similar to os.popen(commandstring, "r").read(),
+ # but without actually using os.popen because that
+ # function is not usable during python bootstrap.
+ # tempfile is also not available then.
+ import contextlib
+ try:
+ import tempfile
+ fp = tempfile.NamedTemporaryFile()
+ except ImportError:
+ fp = open("/tmp/_osx_support.%s"%(
+ os.getpid(),), "w+b")
+
+ with contextlib.closing(fp) as fp:
+ cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+ return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
+
+
+def _find_build_tool(toolname):
+ """Find a build tool on current path or using xcrun"""
+ return (_find_executable(toolname)
+ or _read_output("/usr/bin/xcrun -find %s" % (toolname,))
+ or ''
+ )
+
+_SYSTEM_VERSION = None
+
+def _get_system_version():
+ """Return the OS X system version as a string"""
+ # Reading this plist is a documented way to get the system
+ # version (see the documentation for the Gestalt Manager)
+ # We avoid using platform.mac_ver to avoid possible bootstrap issues during
+ # the build of Python itself (distutils is used to build standard library
+ # extensions).
+
+ global _SYSTEM_VERSION
+
+ if _SYSTEM_VERSION is None:
+ _SYSTEM_VERSION = ''
+ try:
+ f = open('/System/Library/CoreServices/SystemVersion.plist')
+ except OSError:
+ # We're on a plain darwin box, fall back to the default
+ # behaviour.
+ pass
+ else:
+ try:
+ m = re.search(r'ProductUserVisibleVersion\s*'
+ r'(.*?)', f.read())
+ finally:
+ f.close()
+ if m is not None:
+ _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2])
+ # else: fall back to the default behaviour
+
+ return _SYSTEM_VERSION
+
+def _remove_original_values(_config_vars):
+ """Remove original unmodified values for testing"""
+ # This is needed for higher-level cross-platform tests of get_platform.
+ for k in list(_config_vars):
+ if k.startswith(_INITPRE):
+ del _config_vars[k]
+
+def _save_modified_value(_config_vars, cv, newvalue):
+ """Save modified and original unmodified value of configuration var"""
+
+ oldvalue = _config_vars.get(cv, '')
+ if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars):
+ _config_vars[_INITPRE + cv] = oldvalue
+ _config_vars[cv] = newvalue
+
+def _supports_universal_builds():
+ """Returns True if universal builds are supported on this system"""
+ # As an approximation, we assume that if we are running on 10.4 or above,
+ # then we are running with an Xcode environment that supports universal
+ # builds, in particular -isysroot and -arch arguments to the compiler. This
+ # is in support of allowing 10.4 universal builds to run on 10.3.x systems.
+
+ osx_version = _get_system_version()
+ if osx_version:
+ try:
+ osx_version = tuple(int(i) for i in osx_version.split('.'))
+ except ValueError:
+ osx_version = ''
+ return bool(osx_version >= (10, 4)) if osx_version else False
+
+
+def _find_appropriate_compiler(_config_vars):
+ """Find appropriate C compiler for extension module builds"""
+
+ # Issue #13590:
+ # The OSX location for the compiler varies between OSX
+ # (or rather Xcode) releases. With older releases (up-to 10.5)
+ # the compiler is in /usr/bin, with newer releases the compiler
+ # can only be found inside Xcode.app if the "Command Line Tools"
+ # are not installed.
+ #
+ # Furthermore, the compiler that can be used varies between
+ # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
+ # as the compiler, after that 'clang' should be used because
+ # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
+ # miscompiles Python.
+
+ # skip checks if the compiler was overridden with a CC env variable
+ if 'CC' in os.environ:
+ return _config_vars
+
+ # The CC config var might contain additional arguments.
+ # Ignore them while searching.
+ cc = oldcc = _config_vars['CC'].split()[0]
+ if not _find_executable(cc):
+ # Compiler is not found on the shell search PATH.
+ # Now search for clang, first on PATH (if the Command LIne
+ # Tools have been installed in / or if the user has provided
+ # another location via CC). If not found, try using xcrun
+ # to find an uninstalled clang (within a selected Xcode).
+
+ # NOTE: Cannot use subprocess here because of bootstrap
+ # issues when building Python itself (and os.popen is
+ # implemented on top of subprocess and is therefore not
+ # usable as well)
+
+ cc = _find_build_tool('clang')
+
+ elif os.path.basename(cc).startswith('gcc'):
+ # Compiler is GCC, check if it is LLVM-GCC
+ data = _read_output("'%s' --version"
+ % (cc.replace("'", "'\"'\"'"),))
+ if data and 'llvm-gcc' in data:
+ # Found LLVM-GCC, fall back to clang
+ cc = _find_build_tool('clang')
+
+ if not cc:
+ raise SystemError(
+ "Cannot locate working compiler")
+
+ if cc != oldcc:
+ # Found a replacement compiler.
+ # Modify config vars using new compiler, if not already explicitly
+ # overridden by an env variable, preserving additional arguments.
+ for cv in _COMPILER_CONFIG_VARS:
+ if cv in _config_vars and cv not in os.environ:
+ cv_split = _config_vars[cv].split()
+ cv_split[0] = cc if cv != 'CXX' else cc + '++'
+ _save_modified_value(_config_vars, cv, ' '.join(cv_split))
+
+ return _config_vars
+
+
+def _remove_universal_flags(_config_vars):
+ """Remove all universal build arguments from config vars"""
+
+ for cv in _UNIVERSAL_CONFIG_VARS:
+ # Do not alter a config var explicitly overridden by env var
+ if cv in _config_vars and cv not in os.environ:
+ flags = _config_vars[cv]
+ flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
+ flags = re.sub('-isysroot [^ \t]*', ' ', flags)
+ _save_modified_value(_config_vars, cv, flags)
+
+ return _config_vars
+
+
+def _remove_unsupported_archs(_config_vars):
+ """Remove any unsupported archs from config vars"""
+ # Different Xcode releases support different sets for '-arch'
+ # flags. In particular, Xcode 4.x no longer supports the
+ # PPC architectures.
+ #
+ # This code automatically removes '-arch ppc' and '-arch ppc64'
+ # when these are not supported. That makes it possible to
+ # build extensions on OSX 10.7 and later with the prebuilt
+ # 32-bit installer on the python.org website.
+
+ # skip checks if the compiler was overridden with a CC env variable
+ if 'CC' in os.environ:
+ return _config_vars
+
+ if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None:
+ # NOTE: Cannot use subprocess here because of bootstrap
+ # issues when building Python itself
+ status = os.system(
+ """echo 'int main{};' | """
+ """'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null"""
+ %(_config_vars['CC'].replace("'", "'\"'\"'"),))
+ if status:
+ # The compile failed for some reason. Because of differences
+ # across Xcode and compiler versions, there is no reliable way
+ # to be sure why it failed. Assume here it was due to lack of
+ # PPC support and remove the related '-arch' flags from each
+ # config variables not explicitly overridden by an environment
+ # variable. If the error was for some other reason, we hope the
+ # failure will show up again when trying to compile an extension
+ # module.
+ for cv in _UNIVERSAL_CONFIG_VARS:
+ if cv in _config_vars and cv not in os.environ:
+ flags = _config_vars[cv]
+ flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags)
+ _save_modified_value(_config_vars, cv, flags)
+
+ return _config_vars
+
+
+def _override_all_archs(_config_vars):
+ """Allow override of all archs with ARCHFLAGS env var"""
+ # NOTE: This name was introduced by Apple in OSX 10.5 and
+ # is used by several scripting languages distributed with
+ # that OS release.
+ if 'ARCHFLAGS' in os.environ:
+ arch = os.environ['ARCHFLAGS']
+ for cv in _UNIVERSAL_CONFIG_VARS:
+ if cv in _config_vars and '-arch' in _config_vars[cv]:
+ flags = _config_vars[cv]
+ flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
+ flags = flags + ' ' + arch
+ _save_modified_value(_config_vars, cv, flags)
+
+ return _config_vars
+
+
+def _check_for_unavailable_sdk(_config_vars):
+ """Remove references to any SDKs not available"""
+ # If we're on OSX 10.5 or later and the user tries to
+ # compile an extension using an SDK that is not present
+ # on the current machine it is better to not use an SDK
+ # than to fail. This is particularly important with
+ # the standalone Command Line Tools alternative to a
+ # full-blown Xcode install since the CLT packages do not
+ # provide SDKs. If the SDK is not present, it is assumed
+ # that the header files and dev libs have been installed
+ # to /usr and /System/Library by either a standalone CLT
+ # package or the CLT component within Xcode.
+ cflags = _config_vars.get('CFLAGS', '')
+ m = re.search(r'-isysroot\s+(\S+)', cflags)
+ if m is not None:
+ sdk = m.group(1)
+ if not os.path.exists(sdk):
+ for cv in _UNIVERSAL_CONFIG_VARS:
+ # Do not alter a config var explicitly overridden by env var
+ if cv in _config_vars and cv not in os.environ:
+ flags = _config_vars[cv]
+ flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
+ _save_modified_value(_config_vars, cv, flags)
+
+ return _config_vars
+
+
+def compiler_fixup(compiler_so, cc_args):
+ """
+ This function will strip '-isysroot PATH' and '-arch ARCH' from the
+ compile flags if the user has specified one them in extra_compile_flags.
+
+ This is needed because '-arch ARCH' adds another architecture to the
+ build, without a way to remove an architecture. Furthermore GCC will
+ barf if multiple '-isysroot' arguments are present.
+ """
+ stripArch = stripSysroot = False
+
+ compiler_so = list(compiler_so)
+
+ if not _supports_universal_builds():
+ # OSX before 10.4.0, these don't support -arch and -isysroot at
+ # all.
+ stripArch = stripSysroot = True
+ else:
+ stripArch = '-arch' in cc_args
+ stripSysroot = '-isysroot' in cc_args
+
+ if stripArch or 'ARCHFLAGS' in os.environ:
+ while True:
+ try:
+ index = compiler_so.index('-arch')
+ # Strip this argument and the next one:
+ del compiler_so[index:index+2]
+ except ValueError:
+ break
+
+ if 'ARCHFLAGS' in os.environ and not stripArch:
+ # User specified different -arch flags in the environ,
+ # see also distutils.sysconfig
+ compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
+
+ if stripSysroot:
+ while True:
+ try:
+ index = compiler_so.index('-isysroot')
+ # Strip this argument and the next one:
+ del compiler_so[index:index+2]
+ except ValueError:
+ break
+
+ # Check if the SDK that is used during compilation actually exists,
+ # the universal build requires the usage of a universal SDK and not all
+ # users have that installed by default.
+ sysroot = None
+ if '-isysroot' in cc_args:
+ idx = cc_args.index('-isysroot')
+ sysroot = cc_args[idx+1]
+ elif '-isysroot' in compiler_so:
+ idx = compiler_so.index('-isysroot')
+ sysroot = compiler_so[idx+1]
+
+ if sysroot and not os.path.isdir(sysroot):
+ from distutils import log
+ log.warn("Compiling with an SDK that doesn't seem to exist: %s",
+ sysroot)
+ log.warn("Please check your Xcode installation")
+
+ return compiler_so
+
+
+def customize_config_vars(_config_vars):
+ """Customize Python build configuration variables.
+
+ Called internally from sysconfig with a mutable mapping
+ containing name/value pairs parsed from the configured
+ makefile used to build this interpreter. Returns
+ the mapping updated as needed to reflect the environment
+ in which the interpreter is running; in the case of
+ a Python from a binary installer, the installed
+ environment may be very different from the build
+ environment, i.e. different OS levels, different
+ built tools, different available CPU architectures.
+
+ This customization is performed whenever
+ distutils.sysconfig.get_config_vars() is first
+ called. It may be used in environments where no
+ compilers are present, i.e. when installing pure
+ Python dists. Customization of compiler paths
+ and detection of unavailable archs is deferred
+ until the first extension module build is
+ requested (in distutils.sysconfig.customize_compiler).
+
+ Currently called from distutils.sysconfig
+ """
+
+ if not _supports_universal_builds():
+ # On Mac OS X before 10.4, check if -arch and -isysroot
+ # are in CFLAGS or LDFLAGS and remove them if they are.
+ # This is needed when building extensions on a 10.3 system
+ # using a universal build of python.
+ _remove_universal_flags(_config_vars)
+
+ # Allow user to override all archs with ARCHFLAGS env var
+ _override_all_archs(_config_vars)
+
+ # Remove references to sdks that are not found
+ _check_for_unavailable_sdk(_config_vars)
+
+ return _config_vars
+
+
+def customize_compiler(_config_vars):
+ """Customize compiler path and configuration variables.
+
+ This customization is performed when the first
+ extension module build is requested
+ in distutils.sysconfig.customize_compiler).
+ """
+
+ # Find a compiler to use for extension module builds
+ _find_appropriate_compiler(_config_vars)
+
+ # Remove ppc arch flags if not supported here
+ _remove_unsupported_archs(_config_vars)
+
+ # Allow user to override all archs with ARCHFLAGS env var
+ _override_all_archs(_config_vars)
+
+ return _config_vars
+
+
+def get_platform_osx(_config_vars, osname, release, machine):
+ """Filter values for get_platform()"""
+ # called from get_platform() in sysconfig and distutils.util
+ #
+ # For our purposes, we'll assume that the system version from
+ # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
+ # to. This makes the compatibility story a bit more sane because the
+ # machine is going to compile and link as if it were
+ # MACOSX_DEPLOYMENT_TARGET.
+
+ macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
+ macrelease = _get_system_version() or macver
+ macver = macver or macrelease
+
+ if macver:
+ release = macver
+ osname = "macosx"
+
+ # Use the original CFLAGS value, if available, so that we
+ # return the same machine type for the platform string.
+ # Otherwise, distutils may consider this a cross-compiling
+ # case and disallow installs.
+ cflags = _config_vars.get(_INITPRE+'CFLAGS',
+ _config_vars.get('CFLAGS', ''))
+ if macrelease:
+ try:
+ macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+ except ValueError:
+ macrelease = (10, 0)
+ else:
+ # assume no universal support
+ macrelease = (10, 0)
+
+ if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
+ # The universal build will build fat binaries, but not on
+ # systems before 10.4
+
+ machine = 'fat'
+
+ archs = re.findall(r'-arch\s+(\S+)', cflags)
+ archs = tuple(sorted(set(archs)))
+
+ if len(archs) == 1:
+ machine = archs[0]
+ elif archs == ('i386', 'ppc'):
+ machine = 'fat'
+ elif archs == ('i386', 'x86_64'):
+ machine = 'intel'
+ elif archs == ('i386', 'ppc', 'x86_64'):
+ machine = 'fat3'
+ elif archs == ('ppc64', 'x86_64'):
+ machine = 'fat64'
+ elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
+ machine = 'universal'
+ else:
+ raise ValueError(
+ "Don't know machine value for archs=%r" % (archs,))
+
+ elif machine == 'i386':
+ # On OSX the machine type returned by uname is always the
+ # 32-bit variant, even if the executable architecture is
+ # the 64-bit variant
+ if sys.maxsize >= 2**32:
+ machine = 'x86_64'
+
+ elif machine in ('PowerPC', 'Power_Macintosh'):
+ # Pick a sane name for the PPC architecture.
+ # See 'i386' case
+ if sys.maxsize >= 2**32:
+ machine = 'ppc64'
+ else:
+ machine = 'ppc'
+
+ return (osname, release, machine)
diff --git a/Lib/cgi.py b/Lib/cgi.py
new file mode 100755
index 000000000..c22c71b38
--- /dev/null
+++ b/Lib/cgi.py
@@ -0,0 +1,992 @@
+#! /usr/local/bin/python
+
+# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is
+# intentionally NOT "/usr/bin/env python". On many systems
+# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI
+# scripts, and /usr/local/bin is the default directory where Python is
+# installed, so /usr/bin/env would be unable to find python. Granted,
+# binary installations by Linux vendors often install Python in
+# /usr/bin. So let those vendors patch cgi.py to match their choice
+# of installation.
+
+"""Support module for CGI (Common Gateway Interface) scripts.
+
+This module defines a number of utilities for use by CGI scripts
+written in Python.
+"""
+
+# History
+# -------
+#
+# Michael McLay started this module. Steve Majewski changed the
+# interface to SvFormContentDict and FormContentDict. The multipart
+# parsing was inspired by code submitted by Andreas Paepcke. Guido van
+# Rossum rewrote, reformatted and documented the module and is currently
+# responsible for its maintenance.
+#
+
+__version__ = "2.6"
+
+
+# Imports
+# =======
+
+from io import StringIO, BytesIO, TextIOWrapper
+from collections.abc import Mapping
+import sys
+import os
+import urllib.parse
+from email.parser import FeedParser
+from email.message import Message
+import html
+import locale
+import tempfile
+
+__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
+ "parse_header", "test", "print_exception", "print_environ",
+ "print_form", "print_directory", "print_arguments",
+ "print_environ_usage"]
+
+# Logging support
+# ===============
+
+logfile = "" # Filename to log to, if not empty
+logfp = None # File object to log to, if not None
+
+def initlog(*allargs):
+ """Write a log message, if there is a log file.
+
+ Even though this function is called initlog(), you should always
+ use log(); log is a variable that is set either to initlog
+ (initially), to dolog (once the log file has been opened), or to
+ nolog (when logging is disabled).
+
+ The first argument is a format string; the remaining arguments (if
+ any) are arguments to the % operator, so e.g.
+ log("%s: %s", "a", "b")
+ will write "a: b" to the log file, followed by a newline.
+
+ If the global logfp is not None, it should be a file object to
+ which log data is written.
+
+ If the global logfp is None, the global logfile may be a string
+ giving a filename to open, in append mode. This file should be
+ world writable!!! If the file can't be opened, logging is
+ silently disabled (since there is no safe place where we could
+ send an error message).
+
+ """
+ global log, logfile, logfp
+ if logfile and not logfp:
+ try:
+ logfp = open(logfile, "a")
+ except OSError:
+ pass
+ if not logfp:
+ log = nolog
+ else:
+ log = dolog
+ log(*allargs)
+
+def dolog(fmt, *args):
+ """Write a log message to the log file. See initlog() for docs."""
+ logfp.write(fmt%args + "\n")
+
+def nolog(*allargs):
+ """Dummy function, assigned to log when logging is disabled."""
+ pass
+
+def closelog():
+ """Close the log file."""
+ global log, logfile, logfp
+ logfile = ''
+ if logfp:
+ logfp.close()
+ logfp = None
+ log = initlog
+
+log = initlog # The current logging function
+
+
+# Parsing functions
+# =================
+
+# Maximum input we will accept when REQUEST_METHOD is POST
+# 0 ==> unlimited input
+maxlen = 0
+
+def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
+ """Parse a query in the environment or from a file (default stdin)
+
+ Arguments, all optional:
+
+ fp : file pointer; default: sys.stdin.buffer
+
+ environ : environment dictionary; default: os.environ
+
+ keep_blank_values: flag indicating whether blank values in
+ percent-encoded forms should be treated as blank strings.
+ A true value indicates that blanks should be retained as
+ blank strings. The default false value indicates that
+ blank values are to be ignored and treated as if they were
+ not included.
+
+ strict_parsing: flag indicating what to do with parsing errors.
+ If false (the default), errors are silently ignored.
+ If true, errors raise a ValueError exception.
+ """
+ if fp is None:
+ fp = sys.stdin
+
+ # field keys and values (except for files) are returned as strings
+ # an encoding is required to decode the bytes read from self.fp
+ if hasattr(fp,'encoding'):
+ encoding = fp.encoding
+ else:
+ encoding = 'latin-1'
+
+ # fp.read() must return bytes
+ if isinstance(fp, TextIOWrapper):
+ fp = fp.buffer
+
+ if not 'REQUEST_METHOD' in environ:
+ environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone
+ if environ['REQUEST_METHOD'] == 'POST':
+ ctype, pdict = parse_header(environ['CONTENT_TYPE'])
+ if ctype == 'multipart/form-data':
+ return parse_multipart(fp, pdict)
+ elif ctype == 'application/x-www-form-urlencoded':
+ clength = int(environ['CONTENT_LENGTH'])
+ if maxlen and clength > maxlen:
+ raise ValueError('Maximum content length exceeded')
+ qs = fp.read(clength).decode(encoding)
+ else:
+ qs = '' # Unknown content-type
+ if 'QUERY_STRING' in environ:
+ if qs: qs = qs + '&'
+ qs = qs + environ['QUERY_STRING']
+ elif sys.argv[1:]:
+ if qs: qs = qs + '&'
+ qs = qs + sys.argv[1]
+ environ['QUERY_STRING'] = qs # XXX Shouldn't, really
+ elif 'QUERY_STRING' in environ:
+ qs = environ['QUERY_STRING']
+ else:
+ if sys.argv[1:]:
+ qs = sys.argv[1]
+ else:
+ qs = ""
+ environ['QUERY_STRING'] = qs # XXX Shouldn't, really
+ return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
+ encoding=encoding)
+
+
+def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
+ """Parse multipart input.
+
+ Arguments:
+ fp : input file
+ pdict: dictionary containing other parameters of content-type header
+ encoding, errors: request encoding and error handler, passed to
+ FieldStorage
+
+ Returns a dictionary just like parse_qs(): keys are the field names, each
+ value is a list of values for that field. For non-file fields, the value
+ is a list of strings.
+ """
+ # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always
+ # represented as 7bit US-ASCII.
+ boundary = pdict['boundary'].decode('ascii')
+ ctype = "multipart/form-data; boundary={}".format(boundary)
+ headers = Message()
+ headers.set_type(ctype)
+ headers['Content-Length'] = pdict['CONTENT-LENGTH']
+ fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
+ environ={'REQUEST_METHOD': 'POST'})
+ return {k: fs.getlist(k) for k in fs}
+
+def _parseparam(s):
+ while s[:1] == ';':
+ s = s[1:]
+ end = s.find(';')
+ while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
+ end = s.find(';', end + 1)
+ if end < 0:
+ end = len(s)
+ f = s[:end]
+ yield f.strip()
+ s = s[end:]
+
+def parse_header(line):
+ """Parse a Content-type like header.
+
+ Return the main content-type and a dictionary of options.
+
+ """
+ parts = _parseparam(';' + line)
+ key = parts.__next__()
+ pdict = {}
+ for p in parts:
+ i = p.find('=')
+ if i >= 0:
+ name = p[:i].strip().lower()
+ value = p[i+1:].strip()
+ if len(value) >= 2 and value[0] == value[-1] == '"':
+ value = value[1:-1]
+ value = value.replace('\\\\', '\\').replace('\\"', '"')
+ pdict[name] = value
+ return key, pdict
+
+
+# Classes for field storage
+# =========================
+
+class MiniFieldStorage:
+
+ """Like FieldStorage, for use when no file uploads are possible."""
+
+ # Dummy attributes
+ filename = None
+ list = None
+ type = None
+ file = None
+ type_options = {}
+ disposition = None
+ disposition_options = {}
+ headers = {}
+
+ def __init__(self, name, value):
+ """Constructor from field name and value."""
+ self.name = name
+ self.value = value
+ # self.file = StringIO(value)
+
+ def __repr__(self):
+ """Return printable representation."""
+ return "MiniFieldStorage(%r, %r)" % (self.name, self.value)
+
+
+class FieldStorage:
+
+ """Store a sequence of fields, reading multipart/form-data.
+
+ This class provides naming, typing, files stored on disk, and
+ more. At the top level, it is accessible like a dictionary, whose
+ keys are the field names. (Note: None can occur as a field name.)
+ The items are either a Python list (if there's multiple values) or
+ another FieldStorage or MiniFieldStorage object. If it's a single
+ object, it has the following attributes:
+
+ name: the field name, if specified; otherwise None
+
+ filename: the filename, if specified; otherwise None; this is the
+ client side filename, *not* the file name on which it is
+ stored (that's a temporary file you don't deal with)
+
+ value: the value as a *string*; for file uploads, this
+ transparently reads the file every time you request the value
+ and returns *bytes*
+
+ file: the file(-like) object from which you can read the data *as
+ bytes* ; None if the data is stored a simple string
+
+ type: the content-type, or None if not specified
+
+ type_options: dictionary of options specified on the content-type
+ line
+
+ disposition: content-disposition, or None if not specified
+
+ disposition_options: dictionary of corresponding options
+
+ headers: a dictionary(-like) object (sometimes email.message.Message or a
+ subclass thereof) containing *all* headers
+
+ The class is subclassable, mostly for the purpose of overriding
+ the make_file() method, which is called internally to come up with
+ a file open for reading and writing. This makes it possible to
+ override the default choice of storing all files in a temporary
+ directory and unlinking them as soon as they have been opened.
+
+ """
+ def __init__(self, fp=None, headers=None, outerboundary=b'',
+ environ=os.environ, keep_blank_values=0, strict_parsing=0,
+ limit=None, encoding='utf-8', errors='replace',
+ max_num_fields=None):
+ """Constructor. Read multipart/* until last part.
+
+ Arguments, all optional:
+
+ fp : file pointer; default: sys.stdin.buffer
+ (not used when the request method is GET)
+ Can be :
+ 1. a TextIOWrapper object
+ 2. an object whose read() and readline() methods return bytes
+
+ headers : header dictionary-like object; default:
+ taken from environ as per CGI spec
+
+ outerboundary : terminating multipart boundary
+ (for internal use only)
+
+ environ : environment dictionary; default: os.environ
+
+ keep_blank_values: flag indicating whether blank values in
+ percent-encoded forms should be treated as blank strings.
+ A true value indicates that blanks should be retained as
+ blank strings. The default false value indicates that
+ blank values are to be ignored and treated as if they were
+ not included.
+
+ strict_parsing: flag indicating what to do with parsing errors.
+ If false (the default), errors are silently ignored.
+ If true, errors raise a ValueError exception.
+
+ limit : used internally to read parts of multipart/form-data forms,
+ to exit from the reading loop when reached. It is the difference
+ between the form content-length and the number of bytes already
+ read
+
+ encoding, errors : the encoding and error handler used to decode the
+ binary stream to strings. Must be the same as the charset defined
+ for the page sending the form (content-type : meta http-equiv or
+ header)
+
+ max_num_fields: int. If set, then __init__ throws a ValueError
+ if there are more than n fields read by parse_qsl().
+
+ """
+ method = 'GET'
+ self.keep_blank_values = keep_blank_values
+ self.strict_parsing = strict_parsing
+ self.max_num_fields = max_num_fields
+ if 'REQUEST_METHOD' in environ:
+ method = environ['REQUEST_METHOD'].upper()
+ self.qs_on_post = None
+ if method == 'GET' or method == 'HEAD':
+ if 'QUERY_STRING' in environ:
+ qs = environ['QUERY_STRING']
+ elif sys.argv[1:]:
+ qs = sys.argv[1]
+ else:
+ qs = ""
+ qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape')
+ fp = BytesIO(qs)
+ if headers is None:
+ headers = {'content-type':
+ "application/x-www-form-urlencoded"}
+ if headers is None:
+ headers = {}
+ if method == 'POST':
+ # Set default content-type for POST to what's traditional
+ headers['content-type'] = "application/x-www-form-urlencoded"
+ if 'CONTENT_TYPE' in environ:
+ headers['content-type'] = environ['CONTENT_TYPE']
+ if 'QUERY_STRING' in environ:
+ self.qs_on_post = environ['QUERY_STRING']
+ if 'CONTENT_LENGTH' in environ:
+ headers['content-length'] = environ['CONTENT_LENGTH']
+ else:
+ if not (isinstance(headers, (Mapping, Message))):
+ raise TypeError("headers must be mapping or an instance of "
+ "email.message.Message")
+ self.headers = headers
+ if fp is None:
+ self.fp = sys.stdin.buffer
+ # self.fp.read() must return bytes
+ elif isinstance(fp, TextIOWrapper):
+ self.fp = fp.buffer
+ else:
+ if not (hasattr(fp, 'read') and hasattr(fp, 'readline')):
+ raise TypeError("fp must be file pointer")
+ self.fp = fp
+
+ self.encoding = encoding
+ self.errors = errors
+
+ if not isinstance(outerboundary, bytes):
+ raise TypeError('outerboundary must be bytes, not %s'
+ % type(outerboundary).__name__)
+ self.outerboundary = outerboundary
+
+ self.bytes_read = 0
+ self.limit = limit
+
+ # Process content-disposition header
+ cdisp, pdict = "", {}
+ if 'content-disposition' in self.headers:
+ cdisp, pdict = parse_header(self.headers['content-disposition'])
+ self.disposition = cdisp
+ self.disposition_options = pdict
+ self.name = None
+ if 'name' in pdict:
+ self.name = pdict['name']
+ self.filename = None
+ if 'filename' in pdict:
+ self.filename = pdict['filename']
+ self._binary_file = self.filename is not None
+
+ # Process content-type header
+ #
+ # Honor any existing content-type header. But if there is no
+ # content-type header, use some sensible defaults. Assume
+ # outerboundary is "" at the outer level, but something non-false
+ # inside a multi-part. The default for an inner part is text/plain,
+ # but for an outer part it should be urlencoded. This should catch
+ # bogus clients which erroneously forget to include a content-type
+ # header.
+ #
+ # See below for what we do if there does exist a content-type header,
+ # but it happens to be something we don't understand.
+ if 'content-type' in self.headers:
+ ctype, pdict = parse_header(self.headers['content-type'])
+ elif self.outerboundary or method != 'POST':
+ ctype, pdict = "text/plain", {}
+ else:
+ ctype, pdict = 'application/x-www-form-urlencoded', {}
+ self.type = ctype
+ self.type_options = pdict
+ if 'boundary' in pdict:
+ self.innerboundary = pdict['boundary'].encode(self.encoding,
+ self.errors)
+ else:
+ self.innerboundary = b""
+
+ clen = -1
+ if 'content-length' in self.headers:
+ try:
+ clen = int(self.headers['content-length'])
+ except ValueError:
+ pass
+ if maxlen and clen > maxlen:
+ raise ValueError('Maximum content length exceeded')
+ self.length = clen
+ if self.limit is None and clen >= 0:
+ self.limit = clen
+
+ self.list = self.file = None
+ self.done = 0
+ if ctype == 'application/x-www-form-urlencoded':
+ self.read_urlencoded()
+ elif ctype[:10] == 'multipart/':
+ self.read_multi(environ, keep_blank_values, strict_parsing)
+ else:
+ self.read_single()
+
+ def __del__(self):
+ try:
+ self.file.close()
+ except AttributeError:
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.file.close()
+
+ def __repr__(self):
+ """Return a printable representation."""
+ return "FieldStorage(%r, %r, %r)" % (
+ self.name, self.filename, self.value)
+
+ def __iter__(self):
+ return iter(self.keys())
+
+ def __getattr__(self, name):
+ if name != 'value':
+ raise AttributeError(name)
+ if self.file:
+ self.file.seek(0)
+ value = self.file.read()
+ self.file.seek(0)
+ elif self.list is not None:
+ value = self.list
+ else:
+ value = None
+ return value
+
+ def __getitem__(self, key):
+ """Dictionary style indexing."""
+ if self.list is None:
+ raise TypeError("not indexable")
+ found = []
+ for item in self.list:
+ if item.name == key: found.append(item)
+ if not found:
+ raise KeyError(key)
+ if len(found) == 1:
+ return found[0]
+ else:
+ return found
+
+ def getvalue(self, key, default=None):
+ """Dictionary style get() method, including 'value' lookup."""
+ if key in self:
+ value = self[key]
+ if isinstance(value, list):
+ return [x.value for x in value]
+ else:
+ return value.value
+ else:
+ return default
+
+ def getfirst(self, key, default=None):
+ """ Return the first value received."""
+ if key in self:
+ value = self[key]
+ if isinstance(value, list):
+ return value[0].value
+ else:
+ return value.value
+ else:
+ return default
+
+ def getlist(self, key):
+ """ Return list of received values."""
+ if key in self:
+ value = self[key]
+ if isinstance(value, list):
+ return [x.value for x in value]
+ else:
+ return [value.value]
+ else:
+ return []
+
+ def keys(self):
+ """Dictionary style keys() method."""
+ if self.list is None:
+ raise TypeError("not indexable")
+ return list(set(item.name for item in self.list))
+
+ def __contains__(self, key):
+ """Dictionary style __contains__ method."""
+ if self.list is None:
+ raise TypeError("not indexable")
+ return any(item.name == key for item in self.list)
+
+ def __len__(self):
+ """Dictionary style len(x) support."""
+ return len(self.keys())
+
+ def __bool__(self):
+ if self.list is None:
+ raise TypeError("Cannot be converted to bool.")
+ return bool(self.list)
+
+ def read_urlencoded(self):
+ """Internal: read data in query string format."""
+ qs = self.fp.read(self.length)
+ if not isinstance(qs, bytes):
+ raise ValueError("%s should return bytes, got %s" \
+ % (self.fp, type(qs).__name__))
+ qs = qs.decode(self.encoding, self.errors)
+ if self.qs_on_post:
+ qs += '&' + self.qs_on_post
+ query = urllib.parse.parse_qsl(
+ qs, self.keep_blank_values, self.strict_parsing,
+ encoding=self.encoding, errors=self.errors,
+ max_num_fields=self.max_num_fields)
+ self.list = [MiniFieldStorage(key, value) for key, value in query]
+ self.skip_lines()
+
+ FieldStorageClass = None
+
+ def read_multi(self, environ, keep_blank_values, strict_parsing):
+ """Internal: read a part that is itself multipart."""
+ ib = self.innerboundary
+ if not valid_boundary(ib):
+ raise ValueError('Invalid boundary in multipart form: %r' % (ib,))
+ self.list = []
+ if self.qs_on_post:
+ query = urllib.parse.parse_qsl(
+ self.qs_on_post, self.keep_blank_values, self.strict_parsing,
+ encoding=self.encoding, errors=self.errors,
+ max_num_fields=self.max_num_fields)
+ self.list.extend(MiniFieldStorage(key, value) for key, value in query)
+
+ klass = self.FieldStorageClass or self.__class__
+ first_line = self.fp.readline() # bytes
+ if not isinstance(first_line, bytes):
+ raise ValueError("%s should return bytes, got %s" \
+ % (self.fp, type(first_line).__name__))
+ self.bytes_read += len(first_line)
+
+ # Ensure that we consume the file until we've hit our inner boundary
+ while (first_line.strip() != (b"--" + self.innerboundary) and
+ first_line):
+ first_line = self.fp.readline()
+ self.bytes_read += len(first_line)
+
+ # Propagate max_num_fields into the sub class appropriately
+ max_num_fields = self.max_num_fields
+ if max_num_fields is not None:
+ max_num_fields -= len(self.list)
+
+ while True:
+ parser = FeedParser()
+ hdr_text = b""
+ while True:
+ data = self.fp.readline()
+ hdr_text += data
+ if not data.strip():
+ break
+ if not hdr_text:
+ break
+ # parser takes strings, not bytes
+ self.bytes_read += len(hdr_text)
+ parser.feed(hdr_text.decode(self.encoding, self.errors))
+ headers = parser.close()
+
+ # Some clients add Content-Length for part headers, ignore them
+ if 'content-length' in headers:
+ del headers['content-length']
+
+ limit = None if self.limit is None \
+ else self.limit - self.bytes_read
+ part = klass(self.fp, headers, ib, environ, keep_blank_values,
+ strict_parsing, limit,
+ self.encoding, self.errors, max_num_fields)
+
+ if max_num_fields is not None:
+ max_num_fields -= 1
+ if part.list:
+ max_num_fields -= len(part.list)
+ if max_num_fields < 0:
+ raise ValueError('Max number of fields exceeded')
+
+ self.bytes_read += part.bytes_read
+ self.list.append(part)
+ if part.done or self.bytes_read >= self.length > 0:
+ break
+ self.skip_lines()
+
+ def read_single(self):
+ """Internal: read an atomic part."""
+ if self.length >= 0:
+ self.read_binary()
+ self.skip_lines()
+ else:
+ self.read_lines()
+ self.file.seek(0)
+
+ bufsize = 8*1024 # I/O buffering size for copy to file
+
+ def read_binary(self):
+ """Internal: read binary data."""
+ self.file = self.make_file()
+ todo = self.length
+ if todo >= 0:
+ while todo > 0:
+ data = self.fp.read(min(todo, self.bufsize)) # bytes
+ if not isinstance(data, bytes):
+ raise ValueError("%s should return bytes, got %s"
+ % (self.fp, type(data).__name__))
+ self.bytes_read += len(data)
+ if not data:
+ self.done = -1
+ break
+ self.file.write(data)
+ todo = todo - len(data)
+
+ def read_lines(self):
+ """Internal: read lines until EOF or outerboundary."""
+ if self._binary_file:
+ self.file = self.__file = BytesIO() # store data as bytes for files
+ else:
+ self.file = self.__file = StringIO() # as strings for other fields
+ if self.outerboundary:
+ self.read_lines_to_outerboundary()
+ else:
+ self.read_lines_to_eof()
+
+ def __write(self, line):
+ """line is always bytes, not string"""
+ if self.__file is not None:
+ if self.__file.tell() + len(line) > 1000:
+ self.file = self.make_file()
+ data = self.__file.getvalue()
+ self.file.write(data)
+ self.__file = None
+ if self._binary_file:
+ # keep bytes
+ self.file.write(line)
+ else:
+ # decode to string
+ self.file.write(line.decode(self.encoding, self.errors))
+
+ def read_lines_to_eof(self):
+ """Internal: read lines until EOF."""
+ while 1:
+ line = self.fp.readline(1<<16) # bytes
+ self.bytes_read += len(line)
+ if not line:
+ self.done = -1
+ break
+ self.__write(line)
+
+ def read_lines_to_outerboundary(self):
+ """Internal: read lines until outerboundary.
+ Data is read as bytes: boundaries and line ends must be converted
+ to bytes for comparisons.
+ """
+ next_boundary = b"--" + self.outerboundary
+ last_boundary = next_boundary + b"--"
+ delim = b""
+ last_line_lfend = True
+ _read = 0
+ while 1:
+ if self.limit is not None and _read >= self.limit:
+ break
+ line = self.fp.readline(1<<16) # bytes
+ self.bytes_read += len(line)
+ _read += len(line)
+ if not line:
+ self.done = -1
+ break
+ if delim == b"\r":
+ line = delim + line
+ delim = b""
+ if line.startswith(b"--") and last_line_lfend:
+ strippedline = line.rstrip()
+ if strippedline == next_boundary:
+ break
+ if strippedline == last_boundary:
+ self.done = 1
+ break
+ odelim = delim
+ if line.endswith(b"\r\n"):
+ delim = b"\r\n"
+ line = line[:-2]
+ last_line_lfend = True
+ elif line.endswith(b"\n"):
+ delim = b"\n"
+ line = line[:-1]
+ last_line_lfend = True
+ elif line.endswith(b"\r"):
+ # We may interrupt \r\n sequences if they span the 2**16
+ # byte boundary
+ delim = b"\r"
+ line = line[:-1]
+ last_line_lfend = False
+ else:
+ delim = b""
+ last_line_lfend = False
+ self.__write(odelim + line)
+
+ def skip_lines(self):
+ """Internal: skip lines until outer boundary if defined."""
+ if not self.outerboundary or self.done:
+ return
+ next_boundary = b"--" + self.outerboundary
+ last_boundary = next_boundary + b"--"
+ last_line_lfend = True
+ while True:
+ line = self.fp.readline(1<<16)
+ self.bytes_read += len(line)
+ if not line:
+ self.done = -1
+ break
+ if line.endswith(b"--") and last_line_lfend:
+ strippedline = line.strip()
+ if strippedline == next_boundary:
+ break
+ if strippedline == last_boundary:
+ self.done = 1
+ break
+ last_line_lfend = line.endswith(b'\n')
+
+ def make_file(self):
+ """Overridable: return a readable & writable file.
+
+ The file will be used as follows:
+ - data is written to it
+ - seek(0)
+ - data is read from it
+
+ The file is opened in binary mode for files, in text mode
+ for other fields
+
+ This version opens a temporary file for reading and writing,
+ and immediately deletes (unlinks) it. The trick (on Unix!) is
+ that the file can still be used, but it can't be opened by
+ another process, and it will automatically be deleted when it
+ is closed or when the current process terminates.
+
+ If you want a more permanent file, you derive a class which
+ overrides this method. If you want a visible temporary file
+ that is nevertheless automatically deleted when the script
+ terminates, try defining a __del__ method in a derived class
+ which unlinks the temporary files you have created.
+
+ """
+ if self._binary_file:
+ return tempfile.TemporaryFile("wb+")
+ else:
+ return tempfile.TemporaryFile("w+",
+ encoding=self.encoding, newline = '\n')
+
+
+# Test/debug code
+# ===============
+
+def test(environ=os.environ):
+ """Robust test CGI script, usable as main program.
+
+ Write minimal HTTP headers and dump all information provided to
+ the script in HTML form.
+
+ """
+ print("Content-type: text/html")
+ print()
+ sys.stderr = sys.stdout
+ try:
+ form = FieldStorage() # Replace with other classes to test those
+ print_directory()
+ print_arguments()
+ print_form(form)
+ print_environ(environ)
+ print_environ_usage()
+ def f():
+ exec("testing print_exception() -- italics?")
+ def g(f=f):
+ f()
+ print("What follows is a test, not an actual exception:
")
+ g()
+ except:
+ print_exception()
+
+ print("Second try with a small maxlen...
")
+
+ global maxlen
+ maxlen = 50
+ try:
+ form = FieldStorage() # Replace with other classes to test those
+ print_directory()
+ print_arguments()
+ print_form(form)
+ print_environ(environ)
+ except:
+ print_exception()
+
+def print_exception(type=None, value=None, tb=None, limit=None):
+ if type is None:
+ type, value, tb = sys.exc_info()
+ import traceback
+ print()
+ print("Traceback (most recent call last):
")
+ list = traceback.format_tb(tb, limit) + \
+ traceback.format_exception_only(type, value)
+ print("%s%s
" % (
+ html.escape("".join(list[:-1])),
+ html.escape(list[-1]),
+ ))
+ del tb
+
+def print_environ(environ=os.environ):
+ """Dump the shell environment as HTML."""
+ keys = sorted(environ.keys())
+ print()
+ print("Shell Environment:
")
+ print("")
+ for key in keys:
+ print("- ", html.escape(key), "
- ", html.escape(environ[key]))
+ print("
")
+ print()
+
+def print_form(form):
+ """Dump the contents of a form as HTML."""
+ keys = sorted(form.keys())
+ print()
+ print("Form Contents:
")
+ if not keys:
+ print("No form fields.")
+ print("
")
+ for key in keys:
+ print("- " + html.escape(key) + ":", end=' ')
+ value = form[key]
+ print("" + html.escape(repr(type(value))) + "")
+ print("
- " + html.escape(repr(value)))
+ print("
")
+ print()
+
+def print_directory():
+ """Dump the current directory as HTML."""
+ print()
+ print("Current Working Directory:
")
+ try:
+ pwd = os.getcwd()
+ except OSError as msg:
+ print("OSError:", html.escape(str(msg)))
+ else:
+ print(html.escape(pwd))
+ print()
+
+def print_arguments():
+ print()
+ print("Command Line Arguments:
")
+ print()
+ print(sys.argv)
+ print()
+
+def print_environ_usage():
+ """Dump a list of environment variables used by CGI as HTML."""
+ print("""
+These environment variables could have been set:
+
+- AUTH_TYPE
+
- CONTENT_LENGTH
+
- CONTENT_TYPE
+
- DATE_GMT
+
- DATE_LOCAL
+
- DOCUMENT_NAME
+
- DOCUMENT_ROOT
+
- DOCUMENT_URI
+
- GATEWAY_INTERFACE
+
- LAST_MODIFIED
+
- PATH
+
- PATH_INFO
+
- PATH_TRANSLATED
+
- QUERY_STRING
+
- REMOTE_ADDR
+
- REMOTE_HOST
+
- REMOTE_IDENT
+
- REMOTE_USER
+
- REQUEST_METHOD
+
- SCRIPT_NAME
+
- SERVER_NAME
+
- SERVER_PORT
+
- SERVER_PROTOCOL
+
- SERVER_ROOT
+
- SERVER_SOFTWARE
+
+In addition, HTTP headers sent by the server may be passed in the
+environment as well. Here are some common variable names:
+
+- HTTP_ACCEPT
+
- HTTP_CONNECTION
+
- HTTP_HOST
+
- HTTP_PRAGMA
+
- HTTP_REFERER
+
- HTTP_USER_AGENT
+
+""")
+
+
+# Utilities
+# =========
+
+def valid_boundary(s):
+ import re
+ if isinstance(s, bytes):
+ _vb_pattern = b"^[ -~]{0,200}[!-~]$"
+ else:
+ _vb_pattern = "^[ -~]{0,200}[!-~]$"
+ return re.match(_vb_pattern, s)
+
+# Invoke mainline
+# ===============
+
+# Call test() when this file is run as a script (not imported as a module)
+if __name__ == '__main__':
+ test()
diff --git a/Lib/imp.py b/Lib/imp.py
new file mode 100644
index 000000000..31f8c7663
--- /dev/null
+++ b/Lib/imp.py
@@ -0,0 +1,345 @@
+"""This module provides the components needed to build your own __import__
+function. Undocumented functions are obsolete.
+
+In most cases it is preferred you consider using the importlib module's
+functionality over this module.
+
+"""
+# (Probably) need to stay in _imp
+from _imp import (lock_held, acquire_lock, release_lock,
+ get_frozen_object, is_frozen_package,
+ init_frozen, is_builtin, is_frozen,
+ _fix_co_filename)
+try:
+ from _imp import create_dynamic
+except ImportError:
+ # Platform doesn't support dynamic loading.
+ create_dynamic = None
+
+from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
+from importlib._bootstrap_external import SourcelessFileLoader
+
+from importlib import machinery
+from importlib import util
+import importlib
+import os
+import sys
+import tokenize
+import types
+import warnings
+
+warnings.warn("the imp module is deprecated in favour of importlib; "
+ "see the module's documentation for alternative uses",
+ DeprecationWarning, stacklevel=2)
+
+# DEPRECATED
+SEARCH_ERROR = 0
+PY_SOURCE = 1
+PY_COMPILED = 2
+C_EXTENSION = 3
+PY_RESOURCE = 4
+PKG_DIRECTORY = 5
+C_BUILTIN = 6
+PY_FROZEN = 7
+PY_CODERESOURCE = 8
+IMP_HOOK = 9
+
+
+def new_module(name):
+ """**DEPRECATED**
+
+ Create a new module.
+
+ The module is not entered into sys.modules.
+
+ """
+ return types.ModuleType(name)
+
+
+def get_magic():
+ """**DEPRECATED**
+
+ Return the magic number for .pyc files.
+ """
+ return util.MAGIC_NUMBER
+
+
+def get_tag():
+ """Return the magic tag for .pyc files."""
+ return sys.implementation.cache_tag
+
+
+def cache_from_source(path, debug_override=None):
+ """**DEPRECATED**
+
+ Given the path to a .py file, return the path to its .pyc file.
+
+ The .py file does not need to exist; this simply returns the path to the
+ .pyc file calculated as if the .py file were imported.
+
+ If debug_override is not None, then it must be a boolean and is used in
+ place of sys.flags.optimize.
+
+ If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ return util.cache_from_source(path, debug_override)
+
+
+def source_from_cache(path):
+ """**DEPRECATED**
+
+ Given the path to a .pyc. file, return the path to its .py file.
+
+ The .pyc file does not need to exist; this simply returns the path to
+ the .py file calculated to correspond to the .pyc file. If path does
+ not conform to PEP 3147 format, ValueError will be raised. If
+ sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ return util.source_from_cache(path)
+
+
+def get_suffixes():
+ """**DEPRECATED**"""
+ extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
+ source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
+ bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
+
+ return extensions + source + bytecode
+
+
+class NullImporter:
+
+ """**DEPRECATED**
+
+ Null import object.
+
+ """
+
+ def __init__(self, path):
+ if path == '':
+ raise ImportError('empty pathname', path='')
+ elif os.path.isdir(path):
+ raise ImportError('existing directory', path=path)
+
+ def find_module(self, fullname):
+ """Always returns None."""
+ return None
+
+
+class _HackedGetData:
+
+ """Compatibility support for 'file' arguments of various load_*()
+ functions."""
+
+ def __init__(self, fullname, path, file=None):
+ super().__init__(fullname, path)
+ self.file = file
+
+ def get_data(self, path):
+ """Gross hack to contort loader to deal w/ load_*()'s bad API."""
+ if self.file and path == self.path:
+ # The contract of get_data() requires us to return bytes. Reopen the
+ # file in binary mode if needed.
+ if not self.file.closed:
+ file = self.file
+ if 'b' not in file.mode:
+ file.close()
+ if self.file.closed:
+ self.file = file = open(self.path, 'rb')
+
+ with file:
+ return file.read()
+ else:
+ return super().get_data(path)
+
+
+class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
+
+ """Compatibility support for implementing load_source()."""
+
+
+def load_source(name, pathname, file=None):
+ loader = _LoadSourceCompatibility(name, pathname, file)
+ spec = util.spec_from_file_location(name, pathname, loader=loader)
+ if name in sys.modules:
+ module = _exec(spec, sys.modules[name])
+ else:
+ module = _load(spec)
+ # To allow reloading to potentially work, use a non-hacked loader which
+ # won't rely on a now-closed file object.
+ module.__loader__ = machinery.SourceFileLoader(name, pathname)
+ module.__spec__.loader = module.__loader__
+ return module
+
+
+class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
+
+ """Compatibility support for implementing load_compiled()."""
+
+
+def load_compiled(name, pathname, file=None):
+ """**DEPRECATED**"""
+ loader = _LoadCompiledCompatibility(name, pathname, file)
+ spec = util.spec_from_file_location(name, pathname, loader=loader)
+ if name in sys.modules:
+ module = _exec(spec, sys.modules[name])
+ else:
+ module = _load(spec)
+ # To allow reloading to potentially work, use a non-hacked loader which
+ # won't rely on a now-closed file object.
+ module.__loader__ = SourcelessFileLoader(name, pathname)
+ module.__spec__.loader = module.__loader__
+ return module
+
+
+def load_package(name, path):
+ """**DEPRECATED**"""
+ if os.path.isdir(path):
+ extensions = (machinery.SOURCE_SUFFIXES[:] +
+ machinery.BYTECODE_SUFFIXES[:])
+ for extension in extensions:
+ init_path = os.path.join(path, '__init__' + extension)
+ if os.path.exists(init_path):
+ path = init_path
+ break
+ else:
+ raise ValueError('{!r} is not a package'.format(path))
+ spec = util.spec_from_file_location(name, path,
+ submodule_search_locations=[])
+ if name in sys.modules:
+ return _exec(spec, sys.modules[name])
+ else:
+ return _load(spec)
+
+
+def load_module(name, file, filename, details):
+ """**DEPRECATED**
+
+ Load a module, given information returned by find_module().
+
+ The module name must include the full package name, if any.
+
+ """
+ suffix, mode, type_ = details
+ if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+ raise ValueError('invalid file open mode {!r}'.format(mode))
+ elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
+ msg = 'file object required for import (type code {})'.format(type_)
+ raise ValueError(msg)
+ elif type_ == PY_SOURCE:
+ return load_source(name, filename, file)
+ elif type_ == PY_COMPILED:
+ return load_compiled(name, filename, file)
+ elif type_ == C_EXTENSION and load_dynamic is not None:
+ if file is None:
+ with open(filename, 'rb') as opened_file:
+ return load_dynamic(name, filename, opened_file)
+ else:
+ return load_dynamic(name, filename, file)
+ elif type_ == PKG_DIRECTORY:
+ return load_package(name, filename)
+ elif type_ == C_BUILTIN:
+ return init_builtin(name)
+ elif type_ == PY_FROZEN:
+ return init_frozen(name)
+ else:
+ msg = "Don't know how to import {} (type code {})".format(name, type_)
+ raise ImportError(msg, name=name)
+
+
+def find_module(name, path=None):
+ """**DEPRECATED**
+
+ Search for a module.
+
+ If path is omitted or None, search for a built-in, frozen or special
+ module and continue search in sys.path. The module name cannot
+ contain '.'; to search for a submodule of a package, pass the
+ submodule name and the package's __path__.
+
+ """
+ if not isinstance(name, str):
+ raise TypeError("'name' must be a str, not {}".format(type(name)))
+ elif not isinstance(path, (type(None), list)):
+ # Backwards-compatibility
+ raise RuntimeError("'path' must be None or a list, "
+ "not {}".format(type(path)))
+
+ if path is None:
+ if is_builtin(name):
+ return None, None, ('', '', C_BUILTIN)
+ elif is_frozen(name):
+ return None, None, ('', '', PY_FROZEN)
+ else:
+ path = sys.path
+
+ for entry in path:
+ package_directory = os.path.join(entry, name)
+ for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
+ package_file_name = '__init__' + suffix
+ file_path = os.path.join(package_directory, package_file_name)
+ if os.path.isfile(file_path):
+ return None, package_directory, ('', '', PKG_DIRECTORY)
+ for suffix, mode, type_ in get_suffixes():
+ file_name = name + suffix
+ file_path = os.path.join(entry, file_name)
+ if os.path.isfile(file_path):
+ break
+ else:
+ continue
+ break # Break out of outer loop when breaking out of inner loop.
+ else:
+ raise ImportError(_ERR_MSG.format(name), name=name)
+
+ encoding = None
+ if 'b' not in mode:
+ with open(file_path, 'rb') as file:
+ encoding = tokenize.detect_encoding(file.readline)[0]
+ file = open(file_path, mode, encoding=encoding)
+ return file, file_path, (suffix, mode, type_)
+
+
+def reload(module):
+ """**DEPRECATED**
+
+ Reload the module and return it.
+
+ The module must have been successfully imported before.
+
+ """
+ return importlib.reload(module)
+
+
+def init_builtin(name):
+ """**DEPRECATED**
+
+ Load and return a built-in module by name, or None is such module doesn't
+ exist
+ """
+ try:
+ return _builtin_from_name(name)
+ except ImportError:
+ return None
+
+
+if create_dynamic:
+ def load_dynamic(name, path, file=None):
+ """**DEPRECATED**
+
+ Load an extension module.
+ """
+ import importlib.machinery
+ loader = importlib.machinery.ExtensionFileLoader(name, path)
+
+ # Issue #24748: Skip the sys.modules check in _load_module_shim;
+ # always load new extension
+ spec = importlib.machinery.ModuleSpec(
+ name=name, loader=loader, origin=path)
+ return _load(spec)
+
+else:
+ load_dynamic = None
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index b6a9f82e0..0c73c505f 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -48,8 +48,8 @@ else:
sys.modules['importlib._bootstrap_external'] = _bootstrap_external
# To simplify imports in test code
-_w_long = _bootstrap_external._w_long
-_r_long = _bootstrap_external._r_long
+_pack_uint32 = _bootstrap_external._pack_uint32
+_unpack_uint32 = _bootstrap_external._unpack_uint32
# Fully bootstrapped at this point, import whatever you like, circular
# dependencies and startup overhead minimisation permitting :)
@@ -79,7 +79,8 @@ def find_loader(name, path=None):
This function is deprecated in favor of importlib.util.find_spec().
"""
- warnings.warn('Use importlib.util.find_spec() instead.',
+ warnings.warn('Deprecated since Python 3.4. '
+ 'Use importlib.util.find_spec() instead.',
DeprecationWarning, stacklevel=2)
try:
loader = sys.modules[name].__loader__
@@ -136,7 +137,7 @@ def reload(module):
"""
if not module or not isinstance(module, types.ModuleType):
- raise TypeError("reload() argument must be module")
+ raise TypeError("reload() argument must be a module")
try:
name = module.__spec__.name
except AttributeError:
@@ -163,6 +164,8 @@ def reload(module):
pkgpath = None
target = module
spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
+ if spec is None:
+ raise ModuleNotFoundError(f"spec not found for the module {name!r}", name=name)
_bootstrap._exec(spec, module)
# The module may have replaced itself in sys.modules!
return sys.modules[name]
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index d60cb7691..a4e6a7176 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -26,16 +26,6 @@ _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
+ _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
-def _w_long(x):
- """Convert a 32-bit integer to little-endian."""
- return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
-
-
-def _r_long(int_bytes):
- """Convert 4 bytes in little-endian to an integer."""
- return int.from_bytes(int_bytes, 'little')
-
-
def _make_relax_case():
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
@@ -1597,7 +1587,10 @@ def _setup(_bootstrap_module):
setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
# Directly load the _thread module (needed during bootstrap).
- thread_module = _bootstrap._builtin_from_name('_thread')
+ try:
+ thread_module = _bootstrap._builtin_from_name('_thread')
+ except ImportError:
+ thread_module = None
setattr(self_module, '_thread', thread_module)
# Directly load the _weakref module (needed during bootstrap).
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index daff681e6..4b2d3de6d 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -13,6 +13,7 @@ try:
except ImportError as exc:
_frozen_importlib_external = _bootstrap_external
import abc
+import warnings
def _register(abstract_cls, *classes):
@@ -34,6 +35,8 @@ class Finder(metaclass=abc.ABCMeta):
reimplementations of the import system. Otherwise, finder
implementations should derive from the more specific MetaPathFinder
or PathEntryFinder ABCs.
+
+ Deprecated since Python 3.3
"""
@abc.abstractmethod
@@ -57,11 +60,16 @@ class MetaPathFinder(Finder):
If no module is found, return None. The fullname is a str and
the path is a list of strings or None.
- This method is deprecated in favor of finder.find_spec(). If find_spec()
- exists then backwards-compatible functionality is provided for this
- method.
+ This method is deprecated since Python 3.4 in favor of
+ finder.find_spec(). If find_spec() exists then backwards-compatible
+ functionality is provided for this method.
"""
+ warnings.warn("MetaPathFinder.find_module() is deprecated since Python "
+ "3.4 in favor of MetaPathFinder.find_spec() "
+ "(available since 3.4)",
+ DeprecationWarning,
+ stacklevel=2)
if not hasattr(self, 'find_spec'):
return None
found = self.find_spec(fullname, path)
@@ -94,10 +102,15 @@ class PathEntryFinder(Finder):
The portion will be discarded if another path entry finder
locates the module as a normal module or package.
- This method is deprecated in favor of finder.find_spec(). If find_spec()
- is provided than backwards-compatible functionality is provided.
-
+ This method is deprecated since Python 3.4 in favor of
+ finder.find_spec(). If find_spec() is provided than backwards-compatible
+ functionality is provided.
"""
+ warnings.warn("PathEntryFinder.find_loader() is deprecated since Python "
+ "3.4 in favor of PathEntryFinder.find_spec() "
+ "(available since 3.4)",
+ DeprecationWarning,
+ stacklevel=2)
if not hasattr(self, 'find_spec'):
return None, []
found = self.find_spec(fullname)
@@ -180,7 +193,7 @@ class ResourceLoader(Loader):
def get_data(self, path):
"""Abstract method which when implemented should return the bytes for
the specified path. The path must be a str."""
- raise IOError
+ raise OSError
class InspectLoader(Loader):
@@ -302,7 +315,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
def path_mtime(self, path):
"""Return the (int) modification time for the path (str)."""
if self.path_stats.__func__ is SourceLoader.path_stats:
- raise IOError
+ raise OSError
return int(self.path_stats(path)['mtime'])
def path_stats(self, path):
@@ -313,7 +326,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
- 'size' (optional) is the size in bytes of the source code.
"""
if self.path_mtime.__func__ is SourceLoader.path_mtime:
- raise IOError
+ raise OSError
return {'mtime': self.path_mtime(path)}
def set_data(self, path, data):
@@ -327,3 +340,49 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
"""
_register(SourceLoader, machinery.SourceFileLoader)
+
+
+class ResourceReader(metaclass=abc.ABCMeta):
+
+ """Abstract base class to provide resource-reading support.
+
+ Loaders that support resource reading are expected to implement
+ the ``get_resource_reader(fullname)`` method and have it either return None
+ or an object compatible with this ABC.
+ """
+
+ @abc.abstractmethod
+ def open_resource(self, resource):
+ """Return an opened, file-like object for binary reading.
+
+ The 'resource' argument is expected to represent only a file name
+ and thus not contain any subdirectory components.
+
+ If the resource cannot be found, FileNotFoundError is raised.
+ """
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def resource_path(self, resource):
+ """Return the file system path to the specified resource.
+
+ The 'resource' argument is expected to represent only a file name
+ and thus not contain any subdirectory components.
+
+ If the resource does not exist on the file system, raise
+ FileNotFoundError.
+ """
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def is_resource(self, name):
+ """Return True if the named 'name' is consider a resource."""
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def contents(self):
+ """Return an iterable of strings over the contents of the package."""
+ return []
+
+
+_register(ResourceReader, machinery.SourceFileLoader)
diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py
new file mode 100644
index 000000000..fc3a1c9ca
--- /dev/null
+++ b/Lib/importlib/resources.py
@@ -0,0 +1,259 @@
+import os
+import tempfile
+
+from . import abc as resources_abc
+from contextlib import contextmanager, suppress
+from importlib import import_module
+from importlib.abc import ResourceLoader
+from io import BytesIO, TextIOWrapper
+from pathlib import Path
+from types import ModuleType
+from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401
+from typing import cast
+from typing.io import BinaryIO, TextIO
+from zipimport import ZipImportError
+
+
+__all__ = [
+ 'Package',
+ 'Resource',
+ 'contents',
+ 'is_resource',
+ 'open_binary',
+ 'open_text',
+ 'path',
+ 'read_binary',
+ 'read_text',
+ ]
+
+
+Package = Union[str, ModuleType]
+Resource = Union[str, os.PathLike]
+
+
+def _get_package(package) -> ModuleType:
+ """Take a package name or module object and return the module.
+
+ If a name, the module is imported. If the passed or imported module
+ object is not a package, raise an exception.
+ """
+ if hasattr(package, '__spec__'):
+ if package.__spec__.submodule_search_locations is None:
+ raise TypeError('{!r} is not a package'.format(
+ package.__spec__.name))
+ else:
+ return package
+ else:
+ module = import_module(package)
+ if module.__spec__.submodule_search_locations is None:
+ raise TypeError('{!r} is not a package'.format(package))
+ else:
+ return module
+
+
+def _normalize_path(path) -> str:
+ """Normalize a path by ensuring it is a string.
+
+ If the resulting string contains path separators, an exception is raised.
+ """
+ parent, file_name = os.path.split(path)
+ if parent:
+ raise ValueError('{!r} must be only a file name'.format(path))
+ else:
+ return file_name
+
+
+def _get_resource_reader(
+ package: ModuleType) -> Optional[resources_abc.ResourceReader]:
+ # Return the package's loader if it's a ResourceReader. We can't use
+ # a issubclass() check here because apparently abc.'s __subclasscheck__()
+ # hook wants to create a weak reference to the object, but
+ # zipimport.zipimporter does not support weak references, resulting in a
+ # TypeError. That seems terrible.
+ spec = package.__spec__
+ if hasattr(spec.loader, 'get_resource_reader'):
+ return cast(resources_abc.ResourceReader,
+ spec.loader.get_resource_reader(spec.name))
+ return None
+
+
+def _check_location(package):
+ if package.__spec__.origin is None or not package.__spec__.has_location:
+ raise FileNotFoundError(f'Package has no location {package!r}')
+
+
+def open_binary(package: Package, resource: Resource) -> BinaryIO:
+ """Return a file-like object opened for binary reading of the resource."""
+ resource = _normalize_path(resource)
+ package = _get_package(package)
+ reader = _get_resource_reader(package)
+ if reader is not None:
+ return reader.open_resource(resource)
+ _check_location(package)
+ absolute_package_path = os.path.abspath(package.__spec__.origin)
+ package_path = os.path.dirname(absolute_package_path)
+ full_path = os.path.join(package_path, resource)
+ try:
+ return open(full_path, mode='rb')
+ except OSError:
+ # Just assume the loader is a resource loader; all the relevant
+ # importlib.machinery loaders are and an AttributeError for
+ # get_data() will make it clear what is needed from the loader.
+ loader = cast(ResourceLoader, package.__spec__.loader)
+ data = None
+ if hasattr(package.__spec__.loader, 'get_data'):
+ with suppress(OSError):
+ data = loader.get_data(full_path)
+ if data is None:
+ package_name = package.__spec__.name
+ message = '{!r} resource not found in {!r}'.format(
+ resource, package_name)
+ raise FileNotFoundError(message)
+ else:
+ return BytesIO(data)
+
+
+def open_text(package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict') -> TextIO:
+ """Return a file-like object opened for text reading of the resource."""
+ resource = _normalize_path(resource)
+ package = _get_package(package)
+ reader = _get_resource_reader(package)
+ if reader is not None:
+ return TextIOWrapper(reader.open_resource(resource), encoding, errors)
+ _check_location(package)
+ absolute_package_path = os.path.abspath(package.__spec__.origin)
+ package_path = os.path.dirname(absolute_package_path)
+ full_path = os.path.join(package_path, resource)
+ try:
+ return open(full_path, mode='r', encoding=encoding, errors=errors)
+ except OSError:
+ # Just assume the loader is a resource loader; all the relevant
+ # importlib.machinery loaders are and an AttributeError for
+ # get_data() will make it clear what is needed from the loader.
+ loader = cast(ResourceLoader, package.__spec__.loader)
+ data = None
+ if hasattr(package.__spec__.loader, 'get_data'):
+ with suppress(OSError):
+ data = loader.get_data(full_path)
+ if data is None:
+ package_name = package.__spec__.name
+ message = '{!r} resource not found in {!r}'.format(
+ resource, package_name)
+ raise FileNotFoundError(message)
+ else:
+ return TextIOWrapper(BytesIO(data), encoding, errors)
+
+
+def read_binary(package: Package, resource: Resource) -> bytes:
+ """Return the binary contents of the resource."""
+ resource = _normalize_path(resource)
+ package = _get_package(package)
+ with open_binary(package, resource) as fp:
+ return fp.read()
+
+
+def read_text(package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict') -> str:
+ """Return the decoded string of the resource.
+
+ The decoding-related arguments have the same semantics as those of
+ bytes.decode().
+ """
+ resource = _normalize_path(resource)
+ package = _get_package(package)
+ with open_text(package, resource, encoding, errors) as fp:
+ return fp.read()
+
+
+@contextmanager
+def path(package: Package, resource: Resource) -> Iterator[Path]:
+ """A context manager providing a file path object to the resource.
+
+ If the resource does not already exist on its own on the file system,
+ a temporary file will be created. If the file was created, the file
+ will be deleted upon exiting the context manager (no exception is
+ raised if the file was deleted prior to the context manager
+ exiting).
+ """
+ resource = _normalize_path(resource)
+ package = _get_package(package)
+ reader = _get_resource_reader(package)
+ if reader is not None:
+ try:
+ yield Path(reader.resource_path(resource))
+ return
+ except FileNotFoundError:
+ pass
+ else:
+ _check_location(package)
+ # Fall-through for both the lack of resource_path() *and* if
+ # resource_path() raises FileNotFoundError.
+ package_directory = Path(package.__spec__.origin).parent
+ file_path = package_directory / resource
+ if file_path.exists():
+ yield file_path
+ else:
+ with open_binary(package, resource) as fp:
+ data = fp.read()
+ # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
+ # blocks due to the need to close the temporary file to work on
+ # Windows properly.
+ fd, raw_path = tempfile.mkstemp()
+ try:
+ os.write(fd, data)
+ os.close(fd)
+ yield Path(raw_path)
+ finally:
+ try:
+ os.remove(raw_path)
+ except FileNotFoundError:
+ pass
+
+
+def is_resource(package: Package, name: str) -> bool:
+ """True if 'name' is a resource inside 'package'.
+
+ Directories are *not* resources.
+ """
+ package = _get_package(package)
+ _normalize_path(name)
+ reader = _get_resource_reader(package)
+ if reader is not None:
+ return reader.is_resource(name)
+ try:
+ package_contents = set(contents(package))
+ except (NotADirectoryError, FileNotFoundError):
+ return False
+ if name not in package_contents:
+ return False
+ # Just because the given file_name lives as an entry in the package's
+ # contents doesn't necessarily mean it's a resource. Directories are not
+ # resources, so let's try to find out if it's a directory or not.
+ path = Path(package.__spec__.origin).parent / name
+ return path.is_file()
+
+
+def contents(package: Package) -> Iterable[str]:
+ """Return an iterable of entries in 'package'.
+
+ Note that not all entries are resources. Specifically, directories are
+ not considered resources. Use `is_resource()` on each entry returned here
+ to check if it is a resource or not.
+ """
+ package = _get_package(package)
+ reader = _get_resource_reader(package)
+ if reader is not None:
+ return reader.contents()
+ # Is the package a namespace package? By definition, namespace packages
+ # cannot have resources. We could use _check_location() and catch the
+ # exception, but that's extra work, so just inline the check.
+ elif package.__spec__.origin is None or not package.__spec__.has_location:
+ return ()
+ else:
+ package_directory = Path(package.__spec__.origin).parent
+ return os.listdir(package_directory)
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 6bdf0d445..201e0f4cb 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -5,18 +5,25 @@ from ._bootstrap import _resolve_name
from ._bootstrap import spec_from_loader
from ._bootstrap import _find_spec
from ._bootstrap_external import MAGIC_NUMBER
+from ._bootstrap_external import _RAW_MAGIC_NUMBER
from ._bootstrap_external import cache_from_source
from ._bootstrap_external import decode_source
from ._bootstrap_external import source_from_cache
from ._bootstrap_external import spec_from_file_location
from contextlib import contextmanager
+import _imp
import functools
import sys
import types
import warnings
+def source_hash(source_bytes):
+ "Return the hash of *source_bytes* as used in hash-based pyc files."
+ return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes)
+
+
def resolve_name(name, package):
"""Resolve a relative module name to an absolute one."""
if not name.startswith('.'):
@@ -84,11 +91,16 @@ def find_spec(name, package=None):
if fullname not in sys.modules:
parent_name = fullname.rpartition('.')[0]
if parent_name:
- # Use builtins.__import__() in case someone replaced it.
parent = __import__(parent_name, fromlist=['__path__'])
- return _find_spec(fullname, parent.__path__)
+ try:
+ parent_path = parent.__path__
+ except AttributeError as e:
+ raise ModuleNotFoundError(
+ f"__path__ attribute not found on {parent_name!r} "
+ f"while trying to find {fullname!r}", name=fullname) from e
else:
- return _find_spec(fullname, None)
+ parent_path = None
+ return _find_spec(fullname, parent_path)
else:
module = sys.modules[fullname]
if module is None:
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index 11c5b505c..8e9dd57a5 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -3,6 +3,7 @@
This module has intimate knowledge of the format of .pyc files.
"""
+import enum
import importlib._bootstrap_external
import importlib.machinery
import importlib.util
@@ -11,7 +12,7 @@ import os.path
import sys
import traceback
-__all__ = ["compile", "main", "PyCompileError"]
+__all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"]
class PyCompileError(Exception):
@@ -62,7 +63,21 @@ class PyCompileError(Exception):
return self.msg
-def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
+class PycInvalidationMode(enum.Enum):
+ TIMESTAMP = 1
+ CHECKED_HASH = 2
+ UNCHECKED_HASH = 3
+
+
+def _get_default_invalidation_mode():
+ if os.environ.get('SOURCE_DATE_EPOCH'):
+ return PycInvalidationMode.CHECKED_HASH
+ else:
+ return PycInvalidationMode.TIMESTAMP
+
+
+def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1,
+ invalidation_mode=None):
"""Byte-compile one Python source file to Python bytecode.
:param file: The source file name.
@@ -79,6 +94,7 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
:param optimize: The optimization level for the compiler. Valid values
are -1, 0, 1 and 2. A value of -1 means to use the optimization
level of the current interpreter, as given by -O command line options.
+ :param invalidation_mode:
:return: Path to the resulting byte compiled file.
@@ -103,6 +119,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
the resulting file would be regular and thus not the same type of file as
it was previously.
"""
+ if invalidation_mode is None:
+ invalidation_mode = _get_default_invalidation_mode()
if cfile is None:
if optimize >= 0:
optimization = optimize if optimize >= 1 else ''
@@ -136,9 +154,17 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
os.makedirs(dirname)
except FileExistsError:
pass
- source_stats = loader.path_stats(file)
- bytecode = importlib._bootstrap_external._code_to_bytecode(
+ if invalidation_mode == PycInvalidationMode.TIMESTAMP:
+ source_stats = loader.path_stats(file)
+ bytecode = importlib._bootstrap_external._code_to_timestamp_pyc(
code, source_stats['mtime'], source_stats['size'])
+ else:
+ source_hash = importlib.util.source_hash(source_bytes)
+ bytecode = importlib._bootstrap_external._code_to_hash_pyc(
+ code,
+ source_hash,
+ (invalidation_mode == PycInvalidationMode.CHECKED_HASH),
+ )
mode = importlib._bootstrap_external._calc_mode(file)
importlib._bootstrap_external._write_atomic(cfile, bytecode, mode)
return cfile
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 175f34945..544d88620 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2568,6 +2568,31 @@ def skip_if_buggy_ucrt_strfptime(test):
_buggy_ucrt = False
return unittest.skip("buggy MSVC UCRT strptime/strftime")(test) if _buggy_ucrt else test
+
+def skip_if_broken_multiprocessing_synchronize():
+ """
+ Skip tests if the multiprocessing.synchronize module is missing, if there
+ is no available semaphore implementation, or if creating a lock raises an
+ OSError (on Linux only).
+ """
+
+ # Skip tests if the _multiprocessing extension is missing.
+ import_module('_multiprocessing')
+
+ # Skip tests if there is no available semaphore implementation:
+ # multiprocessing.synchronize requires _multiprocessing.SemLock.
+ synchronize = import_module('multiprocessing.synchronize')
+
+ if sys.platform == "linux":
+ try:
+ # bpo-38377: On Linux, creating a semaphore fails with OSError
+ # if the current user does not have the permission to create
+ # a file in /dev/shm/ directory.
+ synchronize.Lock(ctx=None)
+ except OSError as exc:
+ raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}")
+
+
class PythonSymlink:
"""Creates a symlink for the current Python executable"""
def __init__(self, link=None):
diff --git a/Lib/test/test___future__.py b/Lib/test/test___future__.py
new file mode 100644
index 000000000..559a1873a
--- /dev/null
+++ b/Lib/test/test___future__.py
@@ -0,0 +1,61 @@
+import unittest
+import __future__
+
+GOOD_SERIALS = ("alpha", "beta", "candidate", "final")
+
+features = __future__.all_feature_names
+
+class FutureTest(unittest.TestCase):
+
+ def test_names(self):
+ # Verify that all_feature_names appears correct.
+ given_feature_names = features[:]
+ for name in dir(__future__):
+ obj = getattr(__future__, name, None)
+ if obj is not None and isinstance(obj, __future__._Feature):
+ self.assertTrue(
+ name in given_feature_names,
+ "%r should have been in all_feature_names" % name
+ )
+ given_feature_names.remove(name)
+ self.assertEqual(len(given_feature_names), 0,
+ "all_feature_names has too much: %r" % given_feature_names)
+
+ def test_attributes(self):
+ for feature in features:
+ value = getattr(__future__, feature)
+
+ optional = value.getOptionalRelease()
+ mandatory = value.getMandatoryRelease()
+
+ a = self.assertTrue
+ e = self.assertEqual
+ def check(t, name):
+ a(isinstance(t, tuple), "%s isn't tuple" % name)
+ e(len(t), 5, "%s isn't 5-tuple" % name)
+ (major, minor, micro, level, serial) = t
+ a(isinstance(major, int), "%s major isn't int" % name)
+ a(isinstance(minor, int), "%s minor isn't int" % name)
+ a(isinstance(micro, int), "%s micro isn't int" % name)
+ a(isinstance(level, str),
+ "%s level isn't string" % name)
+ a(level in GOOD_SERIALS,
+ "%s level string has unknown value" % name)
+ a(isinstance(serial, int), "%s serial isn't int" % name)
+
+ check(optional, "optional")
+ if mandatory is not None:
+ check(mandatory, "mandatory")
+ a(optional < mandatory,
+ "optional not less than mandatory, and mandatory not None")
+
+ a(hasattr(value, "compiler_flag"),
+ "feature is missing a .compiler_flag attr")
+ # Make sure the compile accepts the flag.
+ compile("", "", "exec", value.compiler_flag)
+ a(isinstance(getattr(value, "compiler_flag"), int),
+ ".compiler_flag isn't int")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py
new file mode 100644
index 000000000..2e06f0d16
--- /dev/null
+++ b/Lib/test/test_abstract_numbers.py
@@ -0,0 +1,44 @@
+"""Unit tests for numbers.py."""
+
+import math
+import operator
+import unittest
+from numbers import Complex, Real, Rational, Integral
+
+class TestNumbers(unittest.TestCase):
+ def test_int(self):
+ self.assertTrue(issubclass(int, Integral))
+ self.assertTrue(issubclass(int, Complex))
+
+ self.assertEqual(7, int(7).real)
+ self.assertEqual(0, int(7).imag)
+ self.assertEqual(7, int(7).conjugate())
+ self.assertEqual(-7, int(-7).conjugate())
+ self.assertEqual(7, int(7).numerator)
+ self.assertEqual(1, int(7).denominator)
+
+ def test_float(self):
+ self.assertFalse(issubclass(float, Rational))
+ self.assertTrue(issubclass(float, Real))
+
+ self.assertEqual(7.3, float(7.3).real)
+ self.assertEqual(0, float(7.3).imag)
+ self.assertEqual(7.3, float(7.3).conjugate())
+ self.assertEqual(-7.3, float(-7.3).conjugate())
+
+ def test_complex(self):
+ self.assertFalse(issubclass(complex, Real))
+ self.assertTrue(issubclass(complex, Complex))
+
+ c1, c2 = complex(3, 2), complex(4,1)
+ # XXX: This is not ideal, but see the comment in math_trunc().
+ self.assertRaises(TypeError, math.trunc, c1)
+ self.assertRaises(TypeError, operator.mod, c1, c2)
+ self.assertRaises(TypeError, divmod, c1, c2)
+ self.assertRaises(TypeError, operator.floordiv, c1, c2)
+ self.assertRaises(TypeError, float, c1)
+ self.assertRaises(TypeError, int, c1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
index 7cca83d78..d1b2f03f4 100644
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -918,8 +918,6 @@ class BaseTest:
del a[start:stop:step]
self.assertEqual(a, array.array(self.typecode, L))
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_index(self):
example = 2*self.example
a = array.array(self.typecode, example)
@@ -929,8 +927,6 @@ class BaseTest:
self.assertRaises(ValueError, a.index, None)
self.assertRaises(ValueError, a.index, self.outside)
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_count(self):
example = 2*self.example
a = array.array(self.typecode, example)
@@ -940,8 +936,6 @@ class BaseTest:
self.assertEqual(a.count(self.outside), 0)
self.assertEqual(a.count(None), 0)
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_remove(self):
for x in self.example:
example = 2*self.example
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 54834fa1a..546c03371 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -446,8 +446,6 @@ class BuiltinTest(unittest.TestCase):
# test that object has a __dir__()
self.assertEqual(sorted([].__dir__()), dir([]))
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_divmod(self):
self.assertEqual(divmod(12, 7), (1, 5))
self.assertEqual(divmod(-12, 7), (-2, 2))
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
new file mode 100644
index 000000000..8ea2484ae
--- /dev/null
+++ b/Lib/test/test_cgi.py
@@ -0,0 +1,609 @@
+import cgi
+import os
+import sys
+import tempfile
+import unittest
+from collections import namedtuple
+from io import StringIO, BytesIO
+from test import support
+
+class HackedSysModule:
+ # The regression test will have real values in sys.argv, which
+ # will completely confuse the test of the cgi module
+ argv = []
+ stdin = sys.stdin
+
+cgi.sys = HackedSysModule()
+
+class ComparableException:
+ def __init__(self, err):
+ self.err = err
+
+ def __str__(self):
+ return str(self.err)
+
+ def __eq__(self, anExc):
+ if not isinstance(anExc, Exception):
+ return NotImplemented
+ return (self.err.__class__ == anExc.__class__ and
+ self.err.args == anExc.args)
+
+ def __getattr__(self, attr):
+ return getattr(self.err, attr)
+
+def do_test(buf, method):
+ env = {}
+ if method == "GET":
+ fp = None
+ env['REQUEST_METHOD'] = 'GET'
+ env['QUERY_STRING'] = buf
+ elif method == "POST":
+ fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes
+ env['REQUEST_METHOD'] = 'POST'
+ env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded'
+ env['CONTENT_LENGTH'] = str(len(buf))
+ else:
+ raise ValueError("unknown method: %s" % method)
+ try:
+ return cgi.parse(fp, env, strict_parsing=1)
+ except Exception as err:
+ return ComparableException(err)
+
+parse_strict_test_cases = [
+ ("", ValueError("bad query field: ''")),
+ ("&", ValueError("bad query field: ''")),
+ ("&&", ValueError("bad query field: ''")),
+ (";", ValueError("bad query field: ''")),
+ (";&;", ValueError("bad query field: ''")),
+ # Should the next few really be valid?
+ ("=", {}),
+ ("=&=", {}),
+ ("=;=", {}),
+ # This rest seem to make sense
+ ("=a", {'': ['a']}),
+ ("&=a", ValueError("bad query field: ''")),
+ ("=a&", ValueError("bad query field: ''")),
+ ("=&a", ValueError("bad query field: 'a'")),
+ ("b=a", {'b': ['a']}),
+ ("b+=a", {'b ': ['a']}),
+ ("a=b=a", {'a': ['b=a']}),
+ ("a=+b=a", {'a': [' b=a']}),
+ ("&b=a", ValueError("bad query field: ''")),
+ ("b&=a", ValueError("bad query field: 'b'")),
+ ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
+ ("a=a+b&a=b+a", {'a': ['a b', 'b a']}),
+ ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+ ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+ ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+ ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env",
+ {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'],
+ 'cuyer': ['r'],
+ 'expire': ['964546263'],
+ 'kid': ['130003.300038'],
+ 'lobale': ['en-US'],
+ 'order_id': ['0bb2e248638833d48cb7fed300000f1b'],
+ 'ss': ['env'],
+ 'view': ['bustomer'],
+ }),
+
+ ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse",
+ {'SUBMIT': ['Browse'],
+ '_assigned_to': ['31392'],
+ '_category': ['100'],
+ '_status': ['1'],
+ 'group_id': ['5470'],
+ 'set': ['custom'],
+ })
+ ]
+
+def norm(seq):
+ return sorted(seq, key=repr)
+
+def first_elts(list):
+ return [p[0] for p in list]
+
+def first_second_elts(list):
+ return [(p[0], p[1][0]) for p in list]
+
+def gen_result(data, environ):
+ encoding = 'latin-1'
+ fake_stdin = BytesIO(data.encode(encoding))
+ fake_stdin.seek(0)
+ form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding)
+
+ result = {}
+ for k, v in dict(form).items():
+ result[k] = isinstance(v, list) and form.getlist(k) or v.value
+
+ return result
+
+class CgiTests(unittest.TestCase):
+
+ def test_parse_multipart(self):
+ fp = BytesIO(POSTDATA.encode('latin1'))
+ env = {'boundary': BOUNDARY.encode('latin1'),
+ 'CONTENT-LENGTH': '558'}
+ result = cgi.parse_multipart(fp, env)
+ expected = {'submit': [' Add '], 'id': ['1234'],
+ 'file': [b'Testing 123.\n'], 'title': ['']}
+ self.assertEqual(result, expected)
+
+ # TODO RUSTPYTHON - see https://github.com/RustPython/RustPython/issues/935
+ @unittest.expectedFailure
+ def test_parse_multipart_invalid_encoding(self):
+ BOUNDARY = "JfISa01"
+ POSTDATA = """--JfISa01
+Content-Disposition: form-data; name="submit-name"
+Content-Length: 3
+
+\u2603
+--JfISa01"""
+ fp = BytesIO(POSTDATA.encode('utf8'))
+ env = {'boundary': BOUNDARY.encode('latin1'),
+ 'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))}
+ result = cgi.parse_multipart(fp, env, encoding="ascii",
+ errors="surrogateescape")
+ expected = {'submit-name': ["\udce2\udc98\udc83"]}
+ self.assertEqual(result, expected)
+ self.assertEqual("\u2603".encode('utf8'),
+ result["submit-name"][0].encode('utf8', 'surrogateescape'))
+
+ def test_fieldstorage_properties(self):
+ fs = cgi.FieldStorage()
+ self.assertFalse(fs)
+ self.assertIn("FieldStorage", repr(fs))
+ self.assertEqual(list(fs), list(fs.keys()))
+ fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue'))
+ self.assertTrue(fs)
+
+ def test_fieldstorage_invalid(self):
+ self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj",
+ environ={"REQUEST_METHOD":"PUT"})
+ self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar")
+ fs = cgi.FieldStorage(headers={'content-type':'text/plain'})
+ self.assertRaises(TypeError, bool, fs)
+
+ def test_strict(self):
+ for orig, expect in parse_strict_test_cases:
+ # Test basic parsing
+ d = do_test(orig, "GET")
+ self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig))
+ d = do_test(orig, "POST")
+ self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig))
+
+ env = {'QUERY_STRING': orig}
+ fs = cgi.FieldStorage(environ=env)
+ if isinstance(expect, dict):
+ # test dict interface
+ self.assertEqual(len(expect), len(fs))
+ self.assertCountEqual(expect.keys(), fs.keys())
+ ##self.assertEqual(norm(expect.values()), norm(fs.values()))
+ ##self.assertEqual(norm(expect.items()), norm(fs.items()))
+ self.assertEqual(fs.getvalue("nonexistent field", "default"), "default")
+ # test individual fields
+ for key in expect.keys():
+ expect_val = expect[key]
+ self.assertIn(key, fs)
+ if len(expect_val) > 1:
+ self.assertEqual(fs.getvalue(key), expect_val)
+ else:
+ self.assertEqual(fs.getvalue(key), expect_val[0])
+
+ def test_log(self):
+ cgi.log("Testing")
+
+ cgi.logfp = StringIO()
+ cgi.initlog("%s", "Testing initlog 1")
+ cgi.log("%s", "Testing log 2")
+ self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n")
+ if os.path.exists(os.devnull):
+ cgi.logfp = None
+ cgi.logfile = os.devnull
+ cgi.initlog("%s", "Testing log 3")
+ self.addCleanup(cgi.closelog)
+ cgi.log("Testing log 4")
+
+ # TODO RUSTPYTHON
+ @unittest.expectedFailure
+ def test_fieldstorage_readline(self):
+ # FieldStorage uses readline, which has the capacity to read all
+ # contents of the input file into memory; we use readline's size argument
+ # to prevent that for files that do not contain any newlines in
+ # non-GET/HEAD requests
+ class TestReadlineFile:
+ def __init__(self, file):
+ self.file = file
+ self.numcalls = 0
+
+ def readline(self, size=None):
+ self.numcalls += 1
+ if size:
+ return self.file.readline(size)
+ else:
+ return self.file.readline()
+
+ def __getattr__(self, name):
+ file = self.__dict__['file']
+ a = getattr(file, name)
+ if not isinstance(a, int):
+ setattr(self, name, a)
+ return a
+
+ f = TestReadlineFile(tempfile.TemporaryFile("wb+"))
+ self.addCleanup(f.close)
+ f.write(b'x' * 256 * 1024)
+ f.seek(0)
+ env = {'REQUEST_METHOD':'PUT'}
+ fs = cgi.FieldStorage(fp=f, environ=env)
+ self.addCleanup(fs.file.close)
+ # if we're not chunking properly, readline is only called twice
+ # (by read_binary); if we are chunking properly, it will be called 5 times
+ # as long as the chunksize is 1 << 16.
+ self.assertGreater(f.numcalls, 2)
+ f.close()
+
+ def test_fieldstorage_multipart(self):
+ #Test basic FieldStorage multipart parsing
+ env = {
+ 'REQUEST_METHOD': 'POST',
+ 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+ 'CONTENT_LENGTH': '558'}
+ fp = BytesIO(POSTDATA.encode('latin-1'))
+ fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+ self.assertEqual(len(fs.list), 4)
+ expect = [{'name':'id', 'filename':None, 'value':'1234'},
+ {'name':'title', 'filename':None, 'value':''},
+ {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'},
+ {'name':'submit', 'filename':None, 'value':' Add '}]
+ for x in range(len(fs.list)):
+ for k, exp in expect[x].items():
+ got = getattr(fs.list[x], k)
+ self.assertEqual(got, exp)
+
+ def test_fieldstorage_multipart_leading_whitespace(self):
+ env = {
+ 'REQUEST_METHOD': 'POST',
+ 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+ 'CONTENT_LENGTH': '560'}
+ # Add some leading whitespace to our post data that will cause the
+ # first line to not be the innerboundary.
+ fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1'))
+ fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+ self.assertEqual(len(fs.list), 4)
+ expect = [{'name':'id', 'filename':None, 'value':'1234'},
+ {'name':'title', 'filename':None, 'value':''},
+ {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'},
+ {'name':'submit', 'filename':None, 'value':' Add '}]
+ for x in range(len(fs.list)):
+ for k, exp in expect[x].items():
+ got = getattr(fs.list[x], k)
+ self.assertEqual(got, exp)
+
+ def test_fieldstorage_multipart_non_ascii(self):
+ #Test basic FieldStorage multipart parsing
+ env = {'REQUEST_METHOD':'POST',
+ 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+ 'CONTENT_LENGTH':'558'}
+ for encoding in ['iso-8859-1','utf-8']:
+ fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding))
+ fs = cgi.FieldStorage(fp, environ=env,encoding=encoding)
+ self.assertEqual(len(fs.list), 1)
+ expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}]
+ for x in range(len(fs.list)):
+ for k, exp in expect[x].items():
+ got = getattr(fs.list[x], k)
+ self.assertEqual(got, exp)
+
+ # TODO RUSTPYTHON
+ @unittest.expectedFailure
+ def test_fieldstorage_multipart_maxline(self):
+ # Issue #18167
+ maxline = 1 << 16
+ self.maxDiff = None
+ def check(content):
+ data = """---123
+Content-Disposition: form-data; name="upload"; filename="fake.txt"
+Content-Type: text/plain
+
+%s
+---123--
+""".replace('\n', '\r\n') % content
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'multipart/form-data; boundary=-123',
+ 'REQUEST_METHOD': 'POST',
+ }
+ self.assertEqual(gen_result(data, environ),
+ {'upload': content.encode('latin1')})
+ check('x' * (maxline - 1))
+ check('x' * (maxline - 1) + '\r')
+ check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1))
+
+ def test_fieldstorage_multipart_w3c(self):
+ # Test basic FieldStorage multipart parsing (W3C sample)
+ env = {
+ 'REQUEST_METHOD': 'POST',
+ 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3),
+ 'CONTENT_LENGTH': str(len(POSTDATA_W3))}
+ fp = BytesIO(POSTDATA_W3.encode('latin-1'))
+ fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+ self.assertEqual(len(fs.list), 2)
+ self.assertEqual(fs.list[0].name, 'submit-name')
+ self.assertEqual(fs.list[0].value, 'Larry')
+ self.assertEqual(fs.list[1].name, 'files')
+ files = fs.list[1].value
+ self.assertEqual(len(files), 2)
+ expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'},
+ {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}]
+ for x in range(len(files)):
+ for k, exp in expect[x].items():
+ got = getattr(files[x], k)
+ self.assertEqual(got, exp)
+
+ def test_fieldstorage_part_content_length(self):
+ BOUNDARY = "JfISa01"
+ POSTDATA = """--JfISa01
+Content-Disposition: form-data; name="submit-name"
+Content-Length: 5
+
+Larry
+--JfISa01"""
+ env = {
+ 'REQUEST_METHOD': 'POST',
+ 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+ 'CONTENT_LENGTH': str(len(POSTDATA))}
+ fp = BytesIO(POSTDATA.encode('latin-1'))
+ fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+ self.assertEqual(len(fs.list), 1)
+ self.assertEqual(fs.list[0].name, 'submit-name')
+ self.assertEqual(fs.list[0].value, 'Larry')
+
+ def test_field_storage_multipart_no_content_length(self):
+ fp = BytesIO(b"""--MyBoundary
+Content-Disposition: form-data; name="my-arg"; filename="foo"
+
+Test
+
+--MyBoundary--
+""")
+ env = {
+ "REQUEST_METHOD": "POST",
+ "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary",
+ "wsgi.input": fp,
+ }
+ fields = cgi.FieldStorage(fp, environ=env)
+
+ self.assertEqual(len(fields["my-arg"].file.read()), 5)
+
+ def test_fieldstorage_as_context_manager(self):
+ fp = BytesIO(b'x' * 10)
+ env = {'REQUEST_METHOD': 'PUT'}
+ with cgi.FieldStorage(fp=fp, environ=env) as fs:
+ content = fs.file.read()
+ self.assertFalse(fs.file.closed)
+ self.assertTrue(fs.file.closed)
+ self.assertEqual(content, 'x' * 10)
+ with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'):
+ fs.file.read()
+
+ _qs_result = {
+ 'key1': 'value1',
+ 'key2': ['value2x', 'value2y'],
+ 'key3': 'value3',
+ 'key4': 'value4'
+ }
+ def testQSAndUrlEncode(self):
+ data = "key2=value2x&key3=value3&key4=value4"
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'application/x-www-form-urlencoded',
+ 'QUERY_STRING': 'key1=value1&key2=value2y',
+ 'REQUEST_METHOD': 'POST',
+ }
+ v = gen_result(data, environ)
+ self.assertEqual(self._qs_result, v)
+
+ def test_max_num_fields(self):
+ # For application/x-www-form-urlencoded
+ data = '&'.join(['a=a']*11)
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'application/x-www-form-urlencoded',
+ 'REQUEST_METHOD': 'POST',
+ }
+
+ with self.assertRaises(ValueError):
+ cgi.FieldStorage(
+ fp=BytesIO(data.encode()),
+ environ=environ,
+ max_num_fields=10,
+ )
+
+ # For multipart/form-data
+ data = """---123
+Content-Disposition: form-data; name="a"
+
+3
+---123
+Content-Type: application/x-www-form-urlencoded
+
+a=4
+---123
+Content-Type: application/x-www-form-urlencoded
+
+a=5
+---123--
+"""
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'multipart/form-data; boundary=-123',
+ 'QUERY_STRING': 'a=1&a=2',
+ 'REQUEST_METHOD': 'POST',
+ }
+
+ # 2 GET entities
+ # 1 top level POST entities
+ # 1 entity within the second POST entity
+ # 1 entity within the third POST entity
+ with self.assertRaises(ValueError):
+ cgi.FieldStorage(
+ fp=BytesIO(data.encode()),
+ environ=environ,
+ max_num_fields=4,
+ )
+ cgi.FieldStorage(
+ fp=BytesIO(data.encode()),
+ environ=environ,
+ max_num_fields=5,
+ )
+
+ def testQSAndFormData(self):
+ data = """---123
+Content-Disposition: form-data; name="key2"
+
+value2y
+---123
+Content-Disposition: form-data; name="key3"
+
+value3
+---123
+Content-Disposition: form-data; name="key4"
+
+value4
+---123--
+"""
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'multipart/form-data; boundary=-123',
+ 'QUERY_STRING': 'key1=value1&key2=value2x',
+ 'REQUEST_METHOD': 'POST',
+ }
+ v = gen_result(data, environ)
+ self.assertEqual(self._qs_result, v)
+
+ def testQSAndFormDataFile(self):
+ data = """---123
+Content-Disposition: form-data; name="key2"
+
+value2y
+---123
+Content-Disposition: form-data; name="key3"
+
+value3
+---123
+Content-Disposition: form-data; name="key4"
+
+value4
+---123
+Content-Disposition: form-data; name="upload"; filename="fake.txt"
+Content-Type: text/plain
+
+this is the content of the fake file
+
+---123--
+"""
+ environ = {
+ 'CONTENT_LENGTH': str(len(data)),
+ 'CONTENT_TYPE': 'multipart/form-data; boundary=-123',
+ 'QUERY_STRING': 'key1=value1&key2=value2x',
+ 'REQUEST_METHOD': 'POST',
+ }
+ result = self._qs_result.copy()
+ result.update({
+ 'upload': b'this is the content of the fake file\n'
+ })
+ v = gen_result(data, environ)
+ self.assertEqual(result, v)
+
+ def test_parse_header(self):
+ self.assertEqual(
+ cgi.parse_header("text/plain"),
+ ("text/plain", {}))
+ self.assertEqual(
+ cgi.parse_header("text/vnd.just.made.this.up ; "),
+ ("text/vnd.just.made.this.up", {}))
+ self.assertEqual(
+ cgi.parse_header("text/plain;charset=us-ascii"),
+ ("text/plain", {"charset": "us-ascii"}))
+ self.assertEqual(
+ cgi.parse_header('text/plain ; charset="us-ascii"'),
+ ("text/plain", {"charset": "us-ascii"}))
+ self.assertEqual(
+ cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'),
+ ("text/plain", {"charset": "us-ascii", "another": "opt"}))
+ self.assertEqual(
+ cgi.parse_header('attachment; filename="silly.txt"'),
+ ("attachment", {"filename": "silly.txt"}))
+ self.assertEqual(
+ cgi.parse_header('attachment; filename="strange;name"'),
+ ("attachment", {"filename": "strange;name"}))
+ self.assertEqual(
+ cgi.parse_header('attachment; filename="strange;name";size=123;'),
+ ("attachment", {"filename": "strange;name", "size": "123"}))
+ self.assertEqual(
+ cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'),
+ ("form-data", {"name": "files", "filename": 'fo"o;bar'}))
+
+ def test_all(self):
+ blacklist = {"logfile", "logfp", "initlog", "dolog", "nolog",
+ "closelog", "log", "maxlen", "valid_boundary"}
+ support.check__all__(self, cgi, blacklist=blacklist)
+
+
+BOUNDARY = "---------------------------721837373350705526688164684"
+
+POSTDATA = """-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="id"
+
+1234
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="title"
+
+
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="file"; filename="test.txt"
+Content-Type: text/plain
+
+Testing 123.
+
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="submit"
+
+ Add\x20
+-----------------------------721837373350705526688164684--
+"""
+
+POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="id"
+
+\xe7\xf1\x80
+-----------------------------721837373350705526688164684
+"""
+
+# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4
+BOUNDARY_W3 = "AaB03x"
+POSTDATA_W3 = """--AaB03x
+Content-Disposition: form-data; name="submit-name"
+
+Larry
+--AaB03x
+Content-Disposition: form-data; name="files"
+Content-Type: multipart/mixed; boundary=BbC04y
+
+--BbC04y
+Content-Disposition: file; filename="file1.txt"
+Content-Type: text/plain
+
+... contents of file1.txt ...
+--BbC04y
+Content-Disposition: file; filename="file2.gif"
+Content-Type: image/gif
+Content-Transfer-Encoding: binary
+
+...contents of file2.gif...
+--BbC04y--
+--AaB03x--
+"""
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py
new file mode 100644
index 000000000..96e0c30da
--- /dev/null
+++ b/Lib/test/test_cmd.py
@@ -0,0 +1,242 @@
+"""
+Test script for the 'cmd' module
+Original by Michael Schneider
+"""
+
+
+import cmd
+import sys
+import unittest
+import io
+from test import support
+
+class samplecmdclass(cmd.Cmd):
+ """
+ Instance the sampleclass:
+ >>> mycmd = samplecmdclass()
+
+ Test for the function parseline():
+ >>> mycmd.parseline("")
+ (None, None, '')
+ >>> mycmd.parseline("?")
+ ('help', '', 'help ')
+ >>> mycmd.parseline("?help")
+ ('help', 'help', 'help help')
+ >>> mycmd.parseline("!")
+ ('shell', '', 'shell ')
+ >>> mycmd.parseline("!command")
+ ('shell', 'command', 'shell command')
+ >>> mycmd.parseline("func")
+ ('func', '', 'func')
+ >>> mycmd.parseline("func arg1")
+ ('func', 'arg1', 'func arg1')
+
+
+ Test for the function onecmd():
+ >>> mycmd.onecmd("")
+ >>> mycmd.onecmd("add 4 5")
+ 9
+ >>> mycmd.onecmd("")
+ 9
+ >>> mycmd.onecmd("test")
+ *** Unknown syntax: test
+
+ Test for the function emptyline():
+ >>> mycmd.emptyline()
+ *** Unknown syntax: test
+
+ Test for the function default():
+ >>> mycmd.default("default")
+ *** Unknown syntax: default
+
+ Test for the function completedefault():
+ >>> mycmd.completedefault()
+ This is the completedefault method
+ >>> mycmd.completenames("a")
+ ['add']
+
+ Test for the function completenames():
+ >>> mycmd.completenames("12")
+ []
+ >>> mycmd.completenames("help")
+ ['help']
+
+ Test for the function complete_help():
+ >>> mycmd.complete_help("a")
+ ['add']
+ >>> mycmd.complete_help("he")
+ ['help']
+ >>> mycmd.complete_help("12")
+ []
+ >>> sorted(mycmd.complete_help(""))
+ ['add', 'exit', 'help', 'shell']
+
+ Test for the function do_help():
+ >>> mycmd.do_help("testet")
+ *** No help on testet
+ >>> mycmd.do_help("add")
+ help text for add
+ >>> mycmd.onecmd("help add")
+ help text for add
+ >>> mycmd.do_help("")
+
+ Documented commands (type help ):
+ ========================================
+ add help
+
+ Undocumented commands:
+ ======================
+ exit shell
+
+
+ Test for the function print_topics():
+ >>> mycmd.print_topics("header", ["command1", "command2"], 2 ,10)
+ header
+ ======
+ command1
+ command2
+
+
+ Test for the function columnize():
+ >>> mycmd.columnize([str(i) for i in range(20)])
+ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
+ >>> mycmd.columnize([str(i) for i in range(20)], 10)
+ 0 7 14
+ 1 8 15
+ 2 9 16
+ 3 10 17
+ 4 11 18
+ 5 12 19
+ 6 13
+
+ This is an interactive test, put some commands in the cmdqueue attribute
+ and let it execute
+ This test includes the preloop(), postloop(), default(), emptyline(),
+ parseline(), do_help() functions
+ >>> mycmd.use_rawinput=0
+ >>> mycmd.cmdqueue=["", "add", "add 4 5", "help", "help add","exit"]
+ >>> mycmd.cmdloop()
+ Hello from preloop
+ help text for add
+ *** invalid number of arguments
+ 9
+
+ Documented commands (type help ):
+ ========================================
+ add help
+
+ Undocumented commands:
+ ======================
+ exit shell
+
+ help text for add
+ Hello from postloop
+ """
+
+ def preloop(self):
+ print("Hello from preloop")
+
+ def postloop(self):
+ print("Hello from postloop")
+
+ def completedefault(self, *ignored):
+ print("This is the completedefault method")
+
+ def complete_command(self):
+ print("complete command")
+
+ def do_shell(self, s):
+ pass
+
+ def do_add(self, s):
+ l = s.split()
+ if len(l) != 2:
+ print("*** invalid number of arguments")
+ return
+ try:
+ l = [int(i) for i in l]
+ except ValueError:
+ print("*** arguments should be numbers")
+ return
+ print(l[0]+l[1])
+
+ def help_add(self):
+ print("help text for add")
+ return
+
+ def do_exit(self, arg):
+ return True
+
+
+class TestAlternateInput(unittest.TestCase):
+
+ class simplecmd(cmd.Cmd):
+
+ def do_print(self, args):
+ print(args, file=self.stdout)
+
+ def do_EOF(self, args):
+ return True
+
+
+ class simplecmd2(simplecmd):
+
+ def do_EOF(self, args):
+ print('*** Unknown syntax: EOF', file=self.stdout)
+ return True
+
+
+ def test_file_with_missing_final_nl(self):
+ input = io.StringIO("print test\nprint test2")
+ output = io.StringIO()
+ cmd = self.simplecmd(stdin=input, stdout=output)
+ cmd.use_rawinput = False
+ cmd.cmdloop()
+ self.assertMultiLineEqual(output.getvalue(),
+ ("(Cmd) test\n"
+ "(Cmd) test2\n"
+ "(Cmd) "))
+
+
+ def test_input_reset_at_EOF(self):
+ input = io.StringIO("print test\nprint test2")
+ output = io.StringIO()
+ cmd = self.simplecmd2(stdin=input, stdout=output)
+ cmd.use_rawinput = False
+ cmd.cmdloop()
+ self.assertMultiLineEqual(output.getvalue(),
+ ("(Cmd) test\n"
+ "(Cmd) test2\n"
+ "(Cmd) *** Unknown syntax: EOF\n"))
+ input = io.StringIO("print \n\n")
+ output = io.StringIO()
+ cmd.stdin = input
+ cmd.stdout = output
+ cmd.cmdloop()
+ self.assertMultiLineEqual(output.getvalue(),
+ ("(Cmd) \n"
+ "(Cmd) \n"
+ "(Cmd) *** Unknown syntax: EOF\n"))
+
+
+def test_main(verbose=None):
+ from test import test_cmd
+ support.run_doctest(test_cmd, verbose)
+ support.run_unittest(TestAlternateInput)
+
+def test_coverage(coverdir):
+ trace = support.import_module('trace')
+ tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,],
+ trace=0, count=1)
+ tracer.run('import importlib; importlib.reload(cmd); test_main()')
+ r=tracer.results()
+ print("Writing coverage results...")
+ r.write_results(show_missing=True, summary=True, coverdir=coverdir)
+
+if __name__ == "__main__":
+ if "-c" in sys.argv:
+ test_coverage('/tmp/cmd.cover')
+ elif "-i" in sys.argv:
+ samplecmdclass().cmdloop()
+ else:
+ test_main()
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
new file mode 100644
index 000000000..8bbba86a4
--- /dev/null
+++ b/Lib/test/test_dis.py
@@ -0,0 +1,60 @@
+import subprocess
+import sys
+import unittest
+
+# This only tests that it prints something in order
+# to avoid changing this test if the bytecode changes
+
+# These tests start a new process instead of redirecting stdout because
+# stdout is being written to by rust code, which currently can't be
+# redirected by reassigning sys.stdout
+
+
+class TestDis(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.setup = """
+import dis
+def tested_func(): pass
+"""
+ cls.command = (sys.executable, "-c")
+
+ def test_dis(self):
+ test_code = f"""
+{self.setup}
+dis.dis(tested_func)
+dis.dis("x = 2; print(x)")
+"""
+
+ result = subprocess.run(
+ self.command + (test_code,), capture_output=True
+ )
+ self.assertNotEqual("", result.stdout.decode())
+ self.assertEqual("", result.stderr.decode())
+
+ def test_disassemble(self):
+ test_code = f"""
+{self.setup}
+dis.disassemble(tested_func)
+"""
+ result = subprocess.run(
+ self.command + (test_code,), capture_output=True
+ )
+ # In CPython this would raise an AttributeError, not a
+ # TypeError because dis is implemented in python in CPython and
+ # as such the type mismatch wouldn't be caught immeadiately
+ self.assertIn("TypeError", result.stderr.decode())
+
+ test_code = f"""
+{self.setup}
+dis.disassemble(tested_func.__code__)
+"""
+ result = subprocess.run(
+ self.command + (test_code,), capture_output=True
+ )
+ self.assertNotEqual("", result.stdout.decode())
+ self.assertEqual("", result.stderr.decode())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py
index 63f66fc7c..4232d85cc 100644
--- a/Lib/test/test_fractions.py
+++ b/Lib/test/test_fractions.py
@@ -392,8 +392,6 @@ class FractionTest(unittest.TestCase):
self.assertTypedEquals(F(-2, 10), round(F(-15, 100), 1))
self.assertTypedEquals(F(-2, 10), round(F(-25, 100), 1))
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def testArithmetic(self):
self.assertEqual(F(1, 2), F(1, 10) + F(2, 5))
self.assertEqual(F(-3, 10), F(1, 10) - F(2, 5))
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
new file mode 100644
index 000000000..b42f14afb
--- /dev/null
+++ b/Lib/test/test_imp.py
@@ -0,0 +1,472 @@
+import importlib
+import importlib.util
+import os
+import os.path
+import py_compile
+import sys
+from test import support
+from test.support import script_helper
+import unittest
+import warnings
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ import imp
+import _imp
+
+
+def requires_load_dynamic(meth):
+ """Decorator to skip a test if not running under CPython or lacking
+ imp.load_dynamic()."""
+ meth = support.cpython_only(meth)
+ return unittest.skipIf(not hasattr(imp, 'load_dynamic'),
+ 'imp.load_dynamic() required')(meth)
+
+
+class LockTests(unittest.TestCase):
+
+ """Very basic test of import lock functions."""
+
+ def verify_lock_state(self, expected):
+ self.assertEqual(imp.lock_held(), expected,
+ "expected imp.lock_held() to be %r" % expected)
+ def testLock(self):
+ LOOPS = 50
+
+ # The import lock may already be held, e.g. if the test suite is run
+ # via "import test.autotest".
+ lock_held_at_start = imp.lock_held()
+ self.verify_lock_state(lock_held_at_start)
+
+ for i in range(LOOPS):
+ imp.acquire_lock()
+ self.verify_lock_state(True)
+
+ for i in range(LOOPS):
+ imp.release_lock()
+
+ # The original state should be restored now.
+ self.verify_lock_state(lock_held_at_start)
+
+ if not lock_held_at_start:
+ try:
+ imp.release_lock()
+ except RuntimeError:
+ pass
+ else:
+ self.fail("release_lock() without lock should raise "
+ "RuntimeError")
+
+class ImportTests(unittest.TestCase):
+ # TODO: RustPython
+ # def setUp(self):
+ # mod = importlib.import_module('test.encoded_modules')
+ # self.test_strings = mod.test_strings
+ # self.test_path = mod.__path__
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_import_encoded_module(self):
+ for modname, encoding, teststr in self.test_strings:
+ mod = importlib.import_module('test.encoded_modules.'
+ 'module_' + modname)
+ self.assertEqual(teststr, mod.test)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_find_module_encoding(self):
+ for mod, encoding, _ in self.test_strings:
+ with imp.find_module('module_' + mod, self.test_path)[0] as fd:
+ self.assertEqual(fd.encoding, encoding)
+
+ path = [os.path.dirname(__file__)]
+ with self.assertRaises(SyntaxError):
+ imp.find_module('badsyntax_pep3120', path)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_issue1267(self):
+ for mod, encoding, _ in self.test_strings:
+ fp, filename, info = imp.find_module('module_' + mod,
+ self.test_path)
+ with fp:
+ self.assertNotEqual(fp, None)
+ self.assertEqual(fp.encoding, encoding)
+ self.assertEqual(fp.tell(), 0)
+ self.assertEqual(fp.readline(), '# test %s encoding\n'
+ % encoding)
+
+ fp, filename, info = imp.find_module("tokenize")
+ with fp:
+ self.assertNotEqual(fp, None)
+ self.assertEqual(fp.encoding, "utf-8")
+ self.assertEqual(fp.tell(), 0)
+ self.assertEqual(fp.readline(),
+ '"""Tokenization help for Python programs.\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_issue3594(self):
+ temp_mod_name = 'test_imp_helper'
+ sys.path.insert(0, '.')
+ try:
+ with open(temp_mod_name + '.py', 'w') as file:
+ file.write("# coding: cp1252\nu = 'test.test_imp'\n")
+ file, filename, info = imp.find_module(temp_mod_name)
+ file.close()
+ self.assertEqual(file.encoding, 'cp1252')
+ finally:
+ del sys.path[0]
+ support.unlink(temp_mod_name + '.py')
+ support.unlink(temp_mod_name + '.pyc')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_issue5604(self):
+ # Test cannot cover imp.load_compiled function.
+ # Martin von Loewis note what shared library cannot have non-ascii
+ # character because init_xxx function cannot be compiled
+ # and issue never happens for dynamic modules.
+ # But sources modified to follow generic way for processing paths.
+
+ # the return encoding could be uppercase or None
+ fs_encoding = sys.getfilesystemencoding()
+
+ # covers utf-8 and Windows ANSI code pages
+ # one non-space symbol from every page
+ # (http://en.wikipedia.org/wiki/Code_page)
+ known_locales = {
+ 'utf-8' : b'\xc3\xa4',
+ 'cp1250' : b'\x8C',
+ 'cp1251' : b'\xc0',
+ 'cp1252' : b'\xc0',
+ 'cp1253' : b'\xc1',
+ 'cp1254' : b'\xc0',
+ 'cp1255' : b'\xe0',
+ 'cp1256' : b'\xe0',
+ 'cp1257' : b'\xc0',
+ 'cp1258' : b'\xc0',
+ }
+
+ if sys.platform == 'darwin':
+ self.assertEqual(fs_encoding, 'utf-8')
+ # Mac OS X uses the Normal Form D decomposition
+ # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
+ special_char = b'a\xcc\x88'
+ else:
+ special_char = known_locales.get(fs_encoding)
+
+ if not special_char:
+ self.skipTest("can't run this test with %s as filesystem encoding"
+ % fs_encoding)
+ decoded_char = special_char.decode(fs_encoding)
+ temp_mod_name = 'test_imp_helper_' + decoded_char
+ test_package_name = 'test_imp_helper_package_' + decoded_char
+ init_file_name = os.path.join(test_package_name, '__init__.py')
+ try:
+ # if the curdir is not in sys.path the test fails when run with
+ # ./python ./Lib/test/regrtest.py test_imp
+ sys.path.insert(0, os.curdir)
+ with open(temp_mod_name + '.py', 'w') as file:
+ file.write('a = 1\n')
+ file, filename, info = imp.find_module(temp_mod_name)
+ with file:
+ self.assertIsNotNone(file)
+ self.assertTrue(filename[:-3].endswith(temp_mod_name))
+ self.assertEqual(info[0], '.py')
+ self.assertEqual(info[1], 'r')
+ self.assertEqual(info[2], imp.PY_SOURCE)
+
+ mod = imp.load_module(temp_mod_name, file, filename, info)
+ self.assertEqual(mod.a, 1)
+
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
+ self.assertEqual(mod.a, 1)
+
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ if not sys.dont_write_bytecode:
+ mod = imp.load_compiled(
+ temp_mod_name,
+ imp.cache_from_source(temp_mod_name + '.py'))
+ self.assertEqual(mod.a, 1)
+
+ if not os.path.exists(test_package_name):
+ os.mkdir(test_package_name)
+ with open(init_file_name, 'w') as file:
+ file.write('b = 2\n')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ package = imp.load_package(test_package_name, test_package_name)
+ self.assertEqual(package.b, 2)
+ finally:
+ del sys.path[0]
+ for ext in ('.py', '.pyc'):
+ support.unlink(temp_mod_name + ext)
+ support.unlink(init_file_name + ext)
+ support.rmtree(test_package_name)
+ support.rmtree('__pycache__')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_issue9319(self):
+ path = os.path.dirname(__file__)
+ self.assertRaises(SyntaxError,
+ imp.find_module, "badsyntax_pep3120", [path])
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_load_from_source(self):
+ # Verify that the imp module can correctly load and find .py files
+ # XXX (ncoghlan): It would be nice to use support.CleanImport
+ # here, but that breaks because the os module registers some
+ # handlers in copy_reg on import. Since CleanImport doesn't
+ # revert that registration, the module is left in a broken
+ # state after reversion. Reinitialising the module contents
+ # and just reverting os.environ to its previous state is an OK
+ # workaround
+ orig_path = os.path
+ orig_getenv = os.getenv
+ with support.EnvironmentVarGuard():
+ x = imp.find_module("os")
+ self.addCleanup(x[0].close)
+ new_os = imp.load_module("os", *x)
+ self.assertIs(os, new_os)
+ self.assertIs(orig_path, new_os.path)
+ self.assertIsNot(orig_getenv, new_os.getenv)
+
+ @requires_load_dynamic
+ def test_issue15828_load_extensions(self):
+ # Issue 15828 picked up that the adapter between the old imp API
+ # and importlib couldn't handle C extensions
+ example = "_heapq"
+ x = imp.find_module(example)
+ file_ = x[0]
+ if file_ is not None:
+ self.addCleanup(file_.close)
+ mod = imp.load_module(example, *x)
+ self.assertEqual(mod.__name__, example)
+
+ @requires_load_dynamic
+ def test_issue16421_multiple_modules_in_one_dll(self):
+ # Issue 16421: loading several modules from the same compiled file fails
+ m = '_testimportmultiple'
+ fileobj, pathname, description = imp.find_module(m)
+ fileobj.close()
+ mod0 = imp.load_dynamic(m, pathname)
+ mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname)
+ mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname)
+ self.assertEqual(mod0.__name__, m)
+ self.assertEqual(mod1.__name__, '_testimportmultiple_foo')
+ self.assertEqual(mod2.__name__, '_testimportmultiple_bar')
+ with self.assertRaises(ImportError):
+ imp.load_dynamic('nonexistent', pathname)
+
+ @requires_load_dynamic
+ def test_load_dynamic_ImportError_path(self):
+ # Issue #1559549 added `name` and `path` attributes to ImportError
+ # in order to provide better detail. Issue #10854 implemented those
+ # attributes on import failures of extensions on Windows.
+ path = 'bogus file path'
+ name = 'extension'
+ with self.assertRaises(ImportError) as err:
+ imp.load_dynamic(name, path)
+ self.assertIn(path, err.exception.path)
+ self.assertEqual(name, err.exception.name)
+
+ @requires_load_dynamic
+ def test_load_module_extension_file_is_None(self):
+ # When loading an extension module and the file is None, open one
+ # on the behalf of imp.load_dynamic().
+ # Issue #15902
+ name = '_testimportmultiple'
+ found = imp.find_module(name)
+ if found[0] is not None:
+ found[0].close()
+ if found[2][2] != imp.C_EXTENSION:
+ self.skipTest("found module doesn't appear to be a C extension")
+ imp.load_module(name, None, *found[1:])
+
+ @requires_load_dynamic
+ def test_issue24748_load_module_skips_sys_modules_check(self):
+ name = 'test.imp_dummy'
+ try:
+ del sys.modules[name]
+ except KeyError:
+ pass
+ try:
+ module = importlib.import_module(name)
+ spec = importlib.util.find_spec('_testmultiphase')
+ module = imp.load_dynamic(name, spec.origin)
+ self.assertEqual(module.__name__, name)
+ self.assertEqual(module.__spec__.name, name)
+ self.assertEqual(module.__spec__.origin, spec.origin)
+ self.assertRaises(AttributeError, getattr, module, 'dummy_name')
+ self.assertEqual(module.int_const, 1969)
+ self.assertIs(sys.modules[name], module)
+ finally:
+ try:
+ del sys.modules[name]
+ except KeyError:
+ pass
+
+ @unittest.skipIf(sys.dont_write_bytecode,
+ "test meaningful only when writing bytecode")
+ def test_bug7732(self):
+ with support.temp_cwd():
+ source = support.TESTFN + '.py'
+ os.mkdir(source)
+ self.assertRaisesRegex(ImportError, '^No module',
+ imp.find_module, support.TESTFN, ["."])
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_multiple_calls_to_get_data(self):
+ # Issue #18755: make sure multiple calls to get_data() can succeed.
+ loader = imp._LoadSourceCompatibility('imp', imp.__file__,
+ open(imp.__file__))
+ loader.get_data(imp.__file__) # File should be closed
+ loader.get_data(imp.__file__) # Will need to create a newly opened file
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_load_source(self):
+ # Create a temporary module since load_source(name) modifies
+ # sys.modules[name] attributes like __loader___
+ modname = f"tmp{__name__}"
+ mod = type(sys.modules[__name__])(modname)
+ with support.swap_item(sys.modules, modname, mod):
+ with self.assertRaisesRegex(ValueError, 'embedded null'):
+ imp.load_source(modname, __file__ + "\0")
+
+ @support.cpython_only
+ def test_issue31315(self):
+ # There shouldn't be an assertion failure in imp.create_dynamic(),
+ # when spec.name is not a string.
+ create_dynamic = support.get_attribute(imp, 'create_dynamic')
+ class BadSpec:
+ name = None
+ origin = 'foo'
+ with self.assertRaises(TypeError):
+ create_dynamic(BadSpec())
+
+ def test_issue_35321(self):
+ # Both _frozen_importlib and _frozen_importlib_external
+ # should have a spec origin of "frozen" and
+ # no need to clean up imports in this case.
+
+ import _frozen_importlib_external
+ self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen")
+
+ import _frozen_importlib
+ self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_source_hash(self):
+ self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab')
+ self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_pyc_invalidation_mode_from_cmdline(self):
+ cases = [
+ ([], "default"),
+ (["--check-hash-based-pycs", "default"], "default"),
+ (["--check-hash-based-pycs", "always"], "always"),
+ (["--check-hash-based-pycs", "never"], "never"),
+ ]
+ for interp_args, expected in cases:
+ args = interp_args + [
+ "-c",
+ "import _imp; print(_imp.check_hash_based_pycs)",
+ ]
+ res = script_helper.assert_python_ok(*args)
+ self.assertEqual(res.out.strip().decode('utf-8'), expected)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_find_and_load_checked_pyc(self):
+ # issue 34056
+ with support.temp_cwd():
+ with open('mymod.py', 'wb') as fp:
+ fp.write(b'x = 42\n')
+ py_compile.compile(
+ 'mymod.py',
+ doraise=True,
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+ )
+ file, path, description = imp.find_module('mymod', path=['.'])
+ mod = imp.load_module('mymod', file, path, description)
+ self.assertEqual(mod.x, 42)
+
+
+class ReloadTests(unittest.TestCase):
+
+ """Very basic tests to make sure that imp.reload() operates just like
+ reload()."""
+
+ def test_source(self):
+ # XXX (ncoghlan): It would be nice to use test.support.CleanImport
+ # here, but that breaks because the os module registers some
+ # handlers in copy_reg on import. Since CleanImport doesn't
+ # revert that registration, the module is left in a broken
+ # state after reversion. Reinitialising the module contents
+ # and just reverting os.environ to its previous state is an OK
+ # workaround
+ with support.EnvironmentVarGuard():
+ import os
+ imp.reload(os)
+
+ def test_extension(self):
+ with support.CleanImport('time'):
+ import time
+ imp.reload(time)
+
+ def test_builtin(self):
+ with support.CleanImport('marshal'):
+ import marshal
+ imp.reload(marshal)
+
+ def test_with_deleted_parent(self):
+ # see #18681
+ from html import parser
+ html = sys.modules.pop('html')
+ def cleanup():
+ sys.modules['html'] = html
+ self.addCleanup(cleanup)
+ with self.assertRaisesRegex(ImportError, 'html'):
+ imp.reload(parser)
+
+
+class PEP3147Tests(unittest.TestCase):
+ """Tests of PEP 3147."""
+
+ tag = imp.get_tag()
+
+ @unittest.skipUnless(sys.implementation.cache_tag is not None,
+ 'requires sys.implementation.cache_tag not be None')
+ def test_cache_from_source(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyc file (i.e. under __pycache__).
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ self.assertEqual(imp.cache_from_source(path, True), expect)
+
+ @unittest.skipUnless(sys.implementation.cache_tag is not None,
+ 'requires sys.implementation.cache_tag to not be '
+ 'None')
+ def test_source_from_cache(self):
+ # Given the path to a PEP 3147 defined .pyc file, return the path to
+ # its source. This tests the good path.
+ path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ self.assertEqual(imp.source_from_cache(path), expect)
+
+
+class NullImporterTests(unittest.TestCase):
+ @unittest.skipIf(support.TESTFN_UNENCODABLE is None,
+ "Need an undecodeable filename")
+ def test_unencodeable(self):
+ name = support.TESTFN_UNENCODABLE
+ os.mkdir(name)
+ try:
+ self.assertRaises(ImportError, imp.NullImporter, name)
+ finally:
+ os.rmdir(name)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_importlib/__init__.py b/Lib/test/test_importlib/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/__main__.py b/Lib/test/test_importlib/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/abc.py b/Lib/test/test_importlib/abc.py
new file mode 100644
index 000000000..5d4b95876
--- /dev/null
+++ b/Lib/test/test_importlib/abc.py
@@ -0,0 +1,93 @@
+import abc
+
+
+class FinderTests(metaclass=abc.ABCMeta):
+
+ """Basic tests for a finder to pass."""
+
+ @abc.abstractmethod
+ def test_module(self):
+ # Test importing a top-level module.
+ pass
+
+ @abc.abstractmethod
+ def test_package(self):
+ # Test importing a package.
+ pass
+
+ @abc.abstractmethod
+ def test_module_in_package(self):
+ # Test importing a module contained within a package.
+ # A value for 'path' should be used if for a meta_path finder.
+ pass
+
+ @abc.abstractmethod
+ def test_package_in_package(self):
+ # Test importing a subpackage.
+ # A value for 'path' should be used if for a meta_path finder.
+ pass
+
+ @abc.abstractmethod
+ def test_package_over_module(self):
+ # Test that packages are chosen over modules.
+ pass
+
+ @abc.abstractmethod
+ def test_failure(self):
+ # Test trying to find a module that cannot be handled.
+ pass
+
+
+class LoaderTests(metaclass=abc.ABCMeta):
+
+ @abc.abstractmethod
+ def test_module(self):
+ """A module should load without issue.
+
+ After the loader returns the module should be in sys.modules.
+
+ Attributes to verify:
+
+ * __file__
+ * __loader__
+ * __name__
+ * No __path__
+
+ """
+ pass
+
+ @abc.abstractmethod
+ def test_package(self):
+ """Loading a package should work.
+
+ After the loader returns the module should be in sys.modules.
+
+ Attributes to verify:
+
+ * __name__
+ * __file__
+ * __package__
+ * __path__
+ * __loader__
+
+ """
+ pass
+
+ @abc.abstractmethod
+ def test_lacking_parent(self):
+ """A loader should not be dependent on it's parent package being
+ imported."""
+ pass
+
+ @abc.abstractmethod
+ def test_state_after_failure(self):
+ """If a module is already in sys.modules and a reload fails
+ (e.g. a SyntaxError), the module should be in the state it was before
+ the reload began."""
+ pass
+
+ @abc.abstractmethod
+ def test_unloadable(self):
+ """Test ImportError is raised when the loader is asked to load a module
+ it can't."""
+ pass
diff --git a/Lib/test/test_importlib/builtin/__init__.py b/Lib/test/test_importlib/builtin/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/builtin/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/builtin/__main__.py b/Lib/test/test_importlib/builtin/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/builtin/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py
new file mode 100644
index 000000000..084f3de6b
--- /dev/null
+++ b/Lib/test/test_importlib/builtin/test_finder.py
@@ -0,0 +1,90 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import sys
+import unittest
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class FindSpecTests(abc.FinderTests):
+
+ """Test find_spec() for built-in modules."""
+
+ def test_module(self):
+ # Common case.
+ with util.uncache(util.BUILTINS.good_name):
+ found = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name)
+ self.assertTrue(found)
+ self.assertEqual(found.origin, 'built-in')
+
+ # Built-in modules cannot be a package.
+ test_package = None
+
+ # Built-in modules cannot be in a package.
+ test_module_in_package = None
+
+ # Built-in modules cannot be a package.
+ test_package_in_package = None
+
+ # Built-in modules cannot be a package.
+ test_package_over_module = None
+
+ def test_failure(self):
+ name = 'importlib'
+ assert name not in sys.builtin_module_names
+ spec = self.machinery.BuiltinImporter.find_spec(name)
+ self.assertIsNone(spec)
+
+ def test_ignore_path(self):
+ # The value for 'path' should always trigger a failed import.
+ with util.uncache(util.BUILTINS.good_name):
+ spec = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name,
+ ['pkg'])
+ self.assertIsNone(spec)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, machinery=machinery)
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class FinderTests(abc.FinderTests):
+
+ """Test find_module() for built-in modules."""
+
+ def test_module(self):
+ # Common case.
+ with util.uncache(util.BUILTINS.good_name):
+ found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name)
+ self.assertTrue(found)
+ self.assertTrue(hasattr(found, 'load_module'))
+
+ # Built-in modules cannot be a package.
+ test_package = test_package_in_package = test_package_over_module = None
+
+ # Built-in modules cannot be in a package.
+ test_module_in_package = None
+
+ def test_failure(self):
+ assert 'importlib' not in sys.builtin_module_names
+ loader = self.machinery.BuiltinImporter.find_module('importlib')
+ self.assertIsNone(loader)
+
+ def test_ignore_path(self):
+ # The value for 'path' should always trigger a failed import.
+ with util.uncache(util.BUILTINS.good_name):
+ loader = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name,
+ ['pkg'])
+ self.assertIsNone(loader)
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py
new file mode 100644
index 000000000..b1349ec5d
--- /dev/null
+++ b/Lib/test/test_importlib/builtin/test_loader.py
@@ -0,0 +1,108 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import sys
+import types
+import unittest
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class LoaderTests(abc.LoaderTests):
+
+ """Test load_module() for built-in modules."""
+
+ def setUp(self):
+ self.verification = {'__name__': 'errno', '__package__': '',
+ '__loader__': self.machinery.BuiltinImporter}
+
+ def verify(self, module):
+ """Verify that the module matches against what it should have."""
+ self.assertIsInstance(module, types.ModuleType)
+ for attr, value in self.verification.items():
+ self.assertEqual(getattr(module, attr), value)
+ self.assertIn(module.__name__, sys.modules)
+
+ def load_module(self, name):
+ return self.machinery.BuiltinImporter.load_module(name)
+
+ def test_module(self):
+ # Common case.
+ with util.uncache(util.BUILTINS.good_name):
+ module = self.load_module(util.BUILTINS.good_name)
+ self.verify(module)
+
+ # Built-in modules cannot be a package.
+ test_package = test_lacking_parent = None
+
+ # No way to force an import failure.
+ test_state_after_failure = None
+
+ def test_module_reuse(self):
+ # Test that the same module is used in a reload.
+ with util.uncache(util.BUILTINS.good_name):
+ module1 = self.load_module(util.BUILTINS.good_name)
+ module2 = self.load_module(util.BUILTINS.good_name)
+ self.assertIs(module1, module2)
+
+ def test_unloadable(self):
+ name = 'dssdsdfff'
+ assert name not in sys.builtin_module_names
+ with self.assertRaises(ImportError) as cm:
+ self.load_module(name)
+ self.assertEqual(cm.exception.name, name)
+
+ def test_already_imported(self):
+ # Using the name of a module already imported but not a built-in should
+ # still fail.
+ module_name = 'builtin_reload_test'
+ assert module_name not in sys.builtin_module_names
+ with util.uncache(module_name):
+ module = types.ModuleType(module_name)
+ sys.modules[module_name] = module
+ with self.assertRaises(ImportError) as cm:
+ self.load_module(module_name)
+ self.assertEqual(cm.exception.name, module_name)
+
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class InspectLoaderTests:
+
+ """Tests for InspectLoader methods for BuiltinImporter."""
+
+ def test_get_code(self):
+ # There is no code object.
+ result = self.machinery.BuiltinImporter.get_code(util.BUILTINS.good_name)
+ self.assertIsNone(result)
+
+ def test_get_source(self):
+ # There is no source.
+ result = self.machinery.BuiltinImporter.get_source(util.BUILTINS.good_name)
+ self.assertIsNone(result)
+
+ def test_is_package(self):
+ # Cannot be a package.
+ result = self.machinery.BuiltinImporter.is_package(util.BUILTINS.good_name)
+ self.assertFalse(result)
+
+ @unittest.skipIf(util.BUILTINS.bad_name is None, 'all modules are built in')
+ def test_not_builtin(self):
+ # Modules not built-in should raise ImportError.
+ for meth_name in ('get_code', 'get_source', 'is_package'):
+ method = getattr(self.machinery.BuiltinImporter, meth_name)
+ with self.assertRaises(ImportError) as cm:
+ method(util.BUILTINS.bad_name)
+
+
+(Frozen_InspectLoaderTests,
+ Source_InspectLoaderTests
+ ) = util.test_both(InspectLoaderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file
new file mode 100644
index 000000000..eaf36c1da
Binary files /dev/null and b/Lib/test/test_importlib/data01/binary.file differ
diff --git a/Lib/test/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file
new file mode 100644
index 000000000..eaf36c1da
Binary files /dev/null and b/Lib/test/test_importlib/data01/subdirectory/binary.file differ
diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file
new file mode 100644
index 000000000..2cb772295
Binary files /dev/null and b/Lib/test/test_importlib/data01/utf-16.file differ
diff --git a/Lib/test/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file
new file mode 100644
index 000000000..1c0132ad9
--- /dev/null
+++ b/Lib/test/test_importlib/data01/utf-8.file
@@ -0,0 +1 @@
+Hello, UTF-8 world!
diff --git a/Lib/test/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt
new file mode 100644
index 000000000..61a813e40
--- /dev/null
+++ b/Lib/test/test_importlib/data02/one/resource1.txt
@@ -0,0 +1 @@
+one resource
diff --git a/Lib/test/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt
new file mode 100644
index 000000000..a80ce46ea
--- /dev/null
+++ b/Lib/test/test_importlib/data02/two/resource2.txt
@@ -0,0 +1 @@
+two resource
diff --git a/Lib/test/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/extension/__init__.py b/Lib/test/test_importlib/extension/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/extension/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/extension/__main__.py b/Lib/test/test_importlib/extension/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/extension/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py
new file mode 100644
index 000000000..0dd9c8615
--- /dev/null
+++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py
@@ -0,0 +1,46 @@
+from importlib import _bootstrap_external
+from test import support
+import unittest
+
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+
+@unittest.skipIf(util.EXTENSIONS.filename is None, '_testcapi not available')
+@util.case_insensitive_tests
+class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase):
+
+ def find_module(self):
+ good_name = util.EXTENSIONS.name
+ bad_name = good_name.upper()
+ assert good_name != bad_name
+ finder = self.machinery.FileFinder(util.EXTENSIONS.path,
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))
+ return finder.find_module(bad_name)
+
+ def test_case_sensitive(self):
+ with support.EnvironmentVarGuard() as env:
+ env.unset('PYTHONCASEOK')
+ self.caseok_env_changed(should_exist=False)
+ loader = self.find_module()
+ self.assertIsNone(loader)
+
+ def test_case_insensitivity(self):
+ with support.EnvironmentVarGuard() as env:
+ env.set('PYTHONCASEOK', '1')
+ self.caseok_env_changed(should_exist=True)
+ loader = self.find_module()
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+
+(Frozen_ExtensionCaseSensitivity,
+ Source_ExtensionCaseSensitivity
+ ) = util.test_both(ExtensionModuleCaseSensitivityTest, importlib=importlib,
+ machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py
new file mode 100644
index 000000000..89855d686
--- /dev/null
+++ b/Lib/test/test_importlib/extension/test_finder.py
@@ -0,0 +1,46 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+import warnings
+
+
+class FinderTests(abc.FinderTests):
+
+ """Test the finder for extension modules."""
+
+ def find_module(self, fullname):
+ importer = self.machinery.FileFinder(util.EXTENSIONS.path,
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return importer.find_module(fullname)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_module(self):
+ self.assertTrue(self.find_module(util.EXTENSIONS.name))
+
+ # No extension module as an __init__ available for testing.
+ test_package = test_package_in_package = None
+
+ # No extension module in a package available for testing.
+ test_module_in_package = None
+
+ # Extension modules cannot be an __init__ for a package.
+ test_package_over_module = None
+
+ def test_failure(self):
+ self.assertIsNone(self.find_module('asdfjkl;'))
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py
new file mode 100644
index 000000000..6707fc1ba
--- /dev/null
+++ b/Lib/test/test_importlib/extension/test_loader.py
@@ -0,0 +1,306 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import os.path
+import sys
+import types
+import unittest
+import importlib.util
+import importlib
+from test.support.script_helper import assert_python_failure
+
+class LoaderTests(abc.LoaderTests):
+
+ """Test load_module() for extension modules."""
+
+ def setUp(self):
+ self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+ util.EXTENSIONS.file_path)
+
+ def load_module(self, fullname):
+ return self.loader.load_module(fullname)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_load_module_API(self):
+ # Test the default argument for load_module().
+ self.loader.load_module()
+ self.loader.load_module(None)
+ with self.assertRaises(ImportError):
+ self.load_module('XXX')
+
+ def test_equality(self):
+ other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+ util.EXTENSIONS.file_path)
+ self.assertEqual(self.loader, other)
+
+ def test_inequality(self):
+ other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name,
+ util.EXTENSIONS.file_path)
+ self.assertNotEqual(self.loader, other)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_module(self):
+ with util.uncache(util.EXTENSIONS.name):
+ module = self.load_module(util.EXTENSIONS.name)
+ for attr, value in [('__name__', util.EXTENSIONS.name),
+ ('__file__', util.EXTENSIONS.file_path),
+ ('__package__', '')]:
+ self.assertEqual(getattr(module, attr), value)
+ self.assertIn(util.EXTENSIONS.name, sys.modules)
+ self.assertIsInstance(module.__loader__,
+ self.machinery.ExtensionFileLoader)
+
+ # No extension module as __init__ available for testing.
+ test_package = None
+
+ # No extension module in a package available for testing.
+ test_lacking_parent = None
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_module_reuse(self):
+ with util.uncache(util.EXTENSIONS.name):
+ module1 = self.load_module(util.EXTENSIONS.name)
+ module2 = self.load_module(util.EXTENSIONS.name)
+ self.assertIs(module1, module2)
+
+ # No easy way to trigger a failure after a successful import.
+ test_state_after_failure = None
+
+ def test_unloadable(self):
+ name = 'asdfjkl;'
+ with self.assertRaises(ImportError) as cm:
+ self.load_module(name)
+ self.assertEqual(cm.exception.name, name)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_is_package(self):
+ self.assertFalse(self.loader.is_package(util.EXTENSIONS.name))
+ for suffix in self.machinery.EXTENSION_SUFFIXES:
+ path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
+ loader = self.machinery.ExtensionFileLoader('pkg', path)
+ self.assertTrue(loader.is_package('pkg'))
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+@unittest.skip("TODO: RUSTPYTHON")
+class MultiPhaseExtensionModuleTests(abc.LoaderTests):
+ """Test loading extension modules with multi-phase initialization (PEP 489)
+ """
+
+ def setUp(self):
+ self.name = '_testmultiphase'
+ finder = self.machinery.FileFinder(None)
+ self.spec = importlib.util.find_spec(self.name)
+ assert self.spec
+ self.loader = self.machinery.ExtensionFileLoader(
+ self.name, self.spec.origin)
+
+ # No extension module as __init__ available for testing.
+ test_package = None
+
+ # No extension module in a package available for testing.
+ test_lacking_parent = None
+
+ # Handling failure on reload is the up to the module.
+ test_state_after_failure = None
+
+ def test_module(self):
+ '''Test loading an extension module'''
+ with util.uncache(self.name):
+ module = self.load_module()
+ for attr, value in [('__name__', self.name),
+ ('__file__', self.spec.origin),
+ ('__package__', '')]:
+ self.assertEqual(getattr(module, attr), value)
+ with self.assertRaises(AttributeError):
+ module.__path__
+ self.assertIs(module, sys.modules[self.name])
+ self.assertIsInstance(module.__loader__,
+ self.machinery.ExtensionFileLoader)
+
+ def test_functionality(self):
+ '''Test basic functionality of stuff defined in an extension module'''
+ with util.uncache(self.name):
+ module = self.load_module()
+ self.assertIsInstance(module, types.ModuleType)
+ ex = module.Example()
+ self.assertEqual(ex.demo('abcd'), 'abcd')
+ self.assertEqual(ex.demo(), None)
+ with self.assertRaises(AttributeError):
+ ex.abc
+ ex.abc = 0
+ self.assertEqual(ex.abc, 0)
+ self.assertEqual(module.foo(9, 9), 18)
+ self.assertIsInstance(module.Str(), str)
+ self.assertEqual(module.Str(1) + '23', '123')
+ with self.assertRaises(module.error):
+ raise module.error()
+ self.assertEqual(module.int_const, 1969)
+ self.assertEqual(module.str_const, 'something different')
+
+ def test_reload(self):
+ '''Test that reload didn't re-set the module's attributes'''
+ with util.uncache(self.name):
+ module = self.load_module()
+ ex_class = module.Example
+ importlib.reload(module)
+ self.assertIs(ex_class, module.Example)
+
+ def test_try_registration(self):
+ '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work'''
+ module = self.load_module()
+ with self.subTest('PyState_FindModule'):
+ self.assertEqual(module.call_state_registration_func(0), None)
+ with self.subTest('PyState_AddModule'):
+ with self.assertRaises(SystemError):
+ module.call_state_registration_func(1)
+ with self.subTest('PyState_RemoveModule'):
+ with self.assertRaises(SystemError):
+ module.call_state_registration_func(2)
+
+ def load_module(self):
+ '''Load the module from the test extension'''
+ return self.loader.load_module(self.name)
+
+ def load_module_by_name(self, fullname):
+ '''Load a module from the test extension by name'''
+ origin = self.spec.origin
+ loader = self.machinery.ExtensionFileLoader(fullname, origin)
+ spec = importlib.util.spec_from_loader(fullname, loader)
+ module = importlib.util.module_from_spec(spec)
+ loader.exec_module(module)
+ return module
+
+ def test_load_submodule(self):
+ '''Test loading a simulated submodule'''
+ module = self.load_module_by_name('pkg.' + self.name)
+ self.assertIsInstance(module, types.ModuleType)
+ self.assertEqual(module.__name__, 'pkg.' + self.name)
+ self.assertEqual(module.str_const, 'something different')
+
+ def test_load_short_name(self):
+ '''Test loading module with a one-character name'''
+ module = self.load_module_by_name('x')
+ self.assertIsInstance(module, types.ModuleType)
+ self.assertEqual(module.__name__, 'x')
+ self.assertEqual(module.str_const, 'something different')
+ self.assertNotIn('x', sys.modules)
+
+ def test_load_twice(self):
+ '''Test that 2 loads result in 2 module objects'''
+ module1 = self.load_module_by_name(self.name)
+ module2 = self.load_module_by_name(self.name)
+ self.assertIsNot(module1, module2)
+
+ def test_unloadable(self):
+ '''Test nonexistent module'''
+ name = 'asdfjkl;'
+ with self.assertRaises(ImportError) as cm:
+ self.load_module_by_name(name)
+ self.assertEqual(cm.exception.name, name)
+
+ def test_unloadable_nonascii(self):
+ '''Test behavior with nonexistent module with non-ASCII name'''
+ name = 'fo\xf3'
+ with self.assertRaises(ImportError) as cm:
+ self.load_module_by_name(name)
+ self.assertEqual(cm.exception.name, name)
+
+ def test_nonmodule(self):
+ '''Test returning a non-module object from create works'''
+ name = self.name + '_nonmodule'
+ mod = self.load_module_by_name(name)
+ self.assertNotEqual(type(mod), type(unittest))
+ self.assertEqual(mod.three, 3)
+
+ # issue 27782
+ def test_nonmodule_with_methods(self):
+ '''Test creating a non-module object with methods defined'''
+ name = self.name + '_nonmodule_with_methods'
+ mod = self.load_module_by_name(name)
+ self.assertNotEqual(type(mod), type(unittest))
+ self.assertEqual(mod.three, 3)
+ self.assertEqual(mod.bar(10, 1), 9)
+
+ def test_null_slots(self):
+ '''Test that NULL slots aren't a problem'''
+ name = self.name + '_null_slots'
+ module = self.load_module_by_name(name)
+ self.assertIsInstance(module, types.ModuleType)
+ self.assertEqual(module.__name__, name)
+
+ def test_bad_modules(self):
+ '''Test SystemError is raised for misbehaving extensions'''
+ for name_base in [
+ 'bad_slot_large',
+ 'bad_slot_negative',
+ 'create_int_with_state',
+ 'negative_size',
+ 'export_null',
+ 'export_uninitialized',
+ 'export_raise',
+ 'export_unreported_exception',
+ 'create_null',
+ 'create_raise',
+ 'create_unreported_exception',
+ 'nonmodule_with_exec_slots',
+ 'exec_err',
+ 'exec_raise',
+ 'exec_unreported_exception',
+ ]:
+ with self.subTest(name_base):
+ name = self.name + '_' + name_base
+ with self.assertRaises(SystemError):
+ self.load_module_by_name(name)
+
+ def test_nonascii(self):
+ '''Test that modules with non-ASCII names can be loaded'''
+ # punycode behaves slightly differently in some-ASCII and no-ASCII
+ # cases, so test both
+ cases = [
+ (self.name + '_zkou\u0161ka_na\u010dten\xed', 'Czech'),
+ ('\uff3f\u30a4\u30f3\u30dd\u30fc\u30c8\u30c6\u30b9\u30c8',
+ 'Japanese'),
+ ]
+ for name, lang in cases:
+ with self.subTest(name):
+ module = self.load_module_by_name(name)
+ self.assertEqual(module.__name__, name)
+ self.assertEqual(module.__doc__, "Module named in %s" % lang)
+
+ @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'),
+ '--with-pydebug has to be enabled for this test')
+ def test_bad_traverse(self):
+ ''' Issue #32374: Test that traverse fails when accessing per-module
+ state before Py_mod_exec was executed.
+ (Multiphase initialization modules only)
+ '''
+ script = """if True:
+ try:
+ from test import support
+ import importlib.util as util
+ spec = util.find_spec('_testmultiphase')
+ spec.name = '_testmultiphase_with_bad_traverse'
+
+ with support.SuppressCrashReport():
+ m = spec.loader.create_module(spec)
+ except:
+ # Prevent Python-level exceptions from
+ # ending the process with non-zero status
+ # (We are testing for a crash in C-code)
+ pass"""
+ assert_python_failure("-c", script)
+
+
+(Frozen_MultiPhaseExtensionModuleTests,
+ Source_MultiPhaseExtensionModuleTests
+ ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py
new file mode 100644
index 000000000..a4b5a64aa
--- /dev/null
+++ b/Lib/test/test_importlib/extension/test_path_hook.py
@@ -0,0 +1,31 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class PathHookTests:
+
+ """Test the path hook for extension modules."""
+ # XXX Should it only succeed for pre-existing directories?
+ # XXX Should it only work for directories containing an extension module?
+
+ def hook(self, entry):
+ return self.machinery.FileFinder.path_hook(
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))(entry)
+
+ def test_success(self):
+ # Path hook should handle a directory where a known extension module
+ # exists.
+ self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_module'))
+
+
+(Frozen_PathHooksTests,
+ Source_PathHooksTests
+ ) = util.test_both(PathHookTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/frozen/__init__.py b/Lib/test/test_importlib/frozen/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/frozen/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/frozen/__main__.py b/Lib/test/test_importlib/frozen/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/frozen/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
new file mode 100644
index 000000000..4c224cc66
--- /dev/null
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -0,0 +1,92 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class FindSpecTests(abc.FinderTests):
+
+ """Test finding frozen modules."""
+
+ def find(self, name, path=None):
+ finder = self.machinery.FrozenImporter
+ return finder.find_spec(name, path)
+
+ def test_module(self):
+ name = '__hello__'
+ spec = self.find(name)
+ self.assertEqual(spec.origin, 'frozen')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_package(self):
+ spec = self.find('__phello__')
+ self.assertIsNotNone(spec)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_module_in_package(self):
+ spec = self.find('__phello__.spam', ['__phello__'])
+ self.assertIsNotNone(spec)
+
+ # No frozen package within another package to test with.
+ test_package_in_package = None
+
+ # No easy way to test.
+ test_package_over_module = None
+
+ def test_failure(self):
+ spec = self.find('')
+ self.assertIsNone(spec)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, machinery=machinery)
+
+
+class FinderTests(abc.FinderTests):
+
+ """Test finding frozen modules."""
+
+ def find(self, name, path=None):
+ finder = self.machinery.FrozenImporter
+ return finder.find_module(name, path)
+
+ def test_module(self):
+ name = '__hello__'
+ loader = self.find(name)
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_package(self):
+ loader = self.find('__phello__')
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_module_in_package(self):
+ loader = self.find('__phello__.spam', ['__phello__'])
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # No frozen package within another package to test with.
+ test_package_in_package = None
+
+ # No easy way to test.
+ test_package_over_module = None
+
+ def test_failure(self):
+ loader = self.find('')
+ self.assertIsNone(loader)
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
new file mode 100644
index 000000000..a8e55286a
--- /dev/null
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -0,0 +1,230 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+from test.support import captured_stdout
+import types
+import unittest
+import warnings
+
+
+class ExecModuleTests(abc.LoaderTests):
+
+ def exec_module(self, name):
+ with util.uncache(name), captured_stdout() as stdout:
+ spec = self.machinery.ModuleSpec(
+ name, self.machinery.FrozenImporter, origin='frozen',
+ is_package=self.machinery.FrozenImporter.is_package(name))
+ module = types.ModuleType(name)
+ module.__spec__ = spec
+ assert not hasattr(module, 'initialized')
+ self.machinery.FrozenImporter.exec_module(module)
+ self.assertTrue(module.initialized)
+ self.assertTrue(hasattr(module, '__spec__'))
+ self.assertEqual(module.__spec__.origin, 'frozen')
+ return module, stdout.getvalue()
+
+ def test_module(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+ self.assertEqual(output, 'Hello world!\n')
+ self.assertTrue(hasattr(module, '__spec__'))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_package(self):
+ name = '__phello__'
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ 'for {name}.{attr}, {given!r} != {expected!r}'.format(
+ name=name, attr=attr, given=attr_value,
+ expected=value))
+ self.assertEqual(output, 'Hello world!\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_lacking_parent(self):
+ name = '__phello__.spam'
+ with util.uncache('__phello__'):
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ 'for {name}.{attr}, {given} != {expected!r}'.format(
+ name=name, attr=attr, given=attr_value,
+ expected=value))
+ self.assertEqual(output, 'Hello world!\n')
+
+ def test_module_repr(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ repr_str = self.machinery.FrozenImporter.module_repr(module)
+ self.assertEqual(repr_str,
+ "")
+
+ def test_module_repr_indirect(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ self.assertEqual(repr(module),
+ "")
+
+ # No way to trigger an error in a frozen module.
+ test_state_after_failure = None
+
+ def test_unloadable(self):
+ assert self.machinery.FrozenImporter.find_module('_not_real') is None
+ with self.assertRaises(ImportError) as cm:
+ self.exec_module('_not_real')
+ self.assertEqual(cm.exception.name, '_not_real')
+
+
+(Frozen_ExecModuleTests,
+ Source_ExecModuleTests
+ ) = util.test_both(ExecModuleTests, machinery=machinery)
+
+
+class LoaderTests(abc.LoaderTests):
+
+ def test_module(self):
+ with util.uncache('__hello__'), captured_stdout() as stdout:
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__hello__')
+ check = {'__name__': '__hello__',
+ '__package__': '',
+ '__loader__': self.machinery.FrozenImporter,
+ }
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+ self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+ self.assertFalse(hasattr(module, '__file__'))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_package(self):
+ with util.uncache('__phello__'), captured_stdout() as stdout:
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__phello__')
+ check = {'__name__': '__phello__',
+ '__package__': '__phello__',
+ '__path__': [],
+ '__loader__': self.machinery.FrozenImporter,
+ }
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ "for __phello__.%s, %r != %r" %
+ (attr, attr_value, value))
+ self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+ self.assertFalse(hasattr(module, '__file__'))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_lacking_parent(self):
+ with util.uncache('__phello__', '__phello__.spam'), \
+ captured_stdout() as stdout:
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__phello__.spam')
+ check = {'__name__': '__phello__.spam',
+ '__package__': '__phello__',
+ '__loader__': self.machinery.FrozenImporter,
+ }
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ "for __phello__.spam.%s, %r != %r" %
+ (attr, attr_value, value))
+ self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+ self.assertFalse(hasattr(module, '__file__'))
+
+ def test_module_reuse(self):
+ with util.uncache('__hello__'), captured_stdout() as stdout:
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module1 = self.machinery.FrozenImporter.load_module('__hello__')
+ module2 = self.machinery.FrozenImporter.load_module('__hello__')
+ self.assertIs(module1, module2)
+ self.assertEqual(stdout.getvalue(),
+ 'Hello world!\nHello world!\n')
+
+ def test_module_repr(self):
+ with util.uncache('__hello__'), captured_stdout():
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__hello__')
+ repr_str = self.machinery.FrozenImporter.module_repr(module)
+ self.assertEqual(repr_str,
+ "")
+
+ def test_module_repr_indirect(self):
+ with util.uncache('__hello__'), captured_stdout():
+ module = self.machinery.FrozenImporter.load_module('__hello__')
+ self.assertEqual(repr(module),
+ "")
+
+ # No way to trigger an error in a frozen module.
+ test_state_after_failure = None
+
+ def test_unloadable(self):
+ assert self.machinery.FrozenImporter.find_module('_not_real') is None
+ with self.assertRaises(ImportError) as cm:
+ self.machinery.FrozenImporter.load_module('_not_real')
+ self.assertEqual(cm.exception.name, '_not_real')
+
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+
+class InspectLoaderTests:
+
+ """Tests for the InspectLoader methods for FrozenImporter."""
+
+ def test_get_code(self):
+ # Make sure that the code object is good.
+ name = '__hello__'
+ with captured_stdout() as stdout:
+ code = self.machinery.FrozenImporter.get_code(name)
+ mod = types.ModuleType(name)
+ exec(code, mod.__dict__)
+ self.assertTrue(hasattr(mod, 'initialized'))
+ self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+
+ def test_get_source(self):
+ # Should always return None.
+ result = self.machinery.FrozenImporter.get_source('__hello__')
+ self.assertIsNone(result)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_is_package(self):
+ # Should be able to tell what is a package.
+ test_for = (('__hello__', False), ('__phello__', True),
+ ('__phello__.spam', False))
+ for name, is_package in test_for:
+ result = self.machinery.FrozenImporter.is_package(name)
+ self.assertEqual(bool(result), is_package)
+
+ def test_failure(self):
+ # Raise ImportError for modules that are not frozen.
+ for meth_name in ('get_code', 'get_source', 'is_package'):
+ method = getattr(self.machinery.FrozenImporter, meth_name)
+ with self.assertRaises(ImportError) as cm:
+ method('importlib')
+ self.assertEqual(cm.exception.name, 'importlib')
+
+(Frozen_ILTests,
+ Source_ILTests
+ ) = util.test_both(InspectLoaderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/__init__.py b/Lib/test/test_importlib/import_/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/import_/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/import_/__main__.py b/Lib/test/test_importlib/import_/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/import_/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py
new file mode 100644
index 000000000..4b18093cf
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test___loader__.py
@@ -0,0 +1,75 @@
+from importlib import machinery
+import sys
+import types
+import unittest
+
+from .. import util
+
+
+class SpecLoaderMock:
+
+ def find_spec(self, fullname, path=None, target=None):
+ return machinery.ModuleSpec(fullname, self)
+
+ def create_module(self, spec):
+ return None
+
+ def exec_module(self, module):
+ pass
+
+
+class SpecLoaderAttributeTests:
+
+ def test___loader__(self):
+ loader = SpecLoaderMock()
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+
+(Frozen_SpecTests,
+ Source_SpecTests
+ ) = util.test_both(SpecLoaderAttributeTests, __import__=util.__import__)
+
+
+class LoaderMock:
+
+ def find_module(self, fullname, path=None):
+ return self
+
+ def load_module(self, fullname):
+ sys.modules[fullname] = self.module
+ return self.module
+
+
+class LoaderAttributeTests:
+
+ def test___loader___missing(self):
+ module = types.ModuleType('blah')
+ try:
+ del module.__loader__
+ except AttributeError:
+ pass
+ loader = LoaderMock()
+ loader.module = module
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+ def test___loader___is_None(self):
+ module = types.ModuleType('blah')
+ module.__loader__ = None
+ loader = LoaderMock()
+ loader.module = module
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ returned_module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+
+(Frozen_Tests,
+ Source_Tests
+ ) = util.test_both(LoaderAttributeTests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py
new file mode 100644
index 000000000..4b1766452
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test___package__.py
@@ -0,0 +1,167 @@
+"""PEP 366 ("Main module explicit relative imports") specifies the
+semantics for the __package__ attribute on modules. This attribute is
+used, when available, to detect which package a module belongs to (instead
+of using the typical __path__/__name__ test).
+
+"""
+import unittest
+import warnings
+from .. import util
+
+
+class Using__package__:
+
+ """Use of __package__ supersedes the use of __name__/__path__ to calculate
+ what package a module belongs to. The basic algorithm is [__package__]::
+
+ def resolve_name(name, package, level):
+ level -= 1
+ base = package.rsplit('.', level)[0]
+ return '{0}.{1}'.format(base, name)
+
+ But since there is no guarantee that __package__ has been set (or not been
+ set to None [None]), there has to be a way to calculate the attribute's value
+ [__name__]::
+
+ def calc_package(caller_name, has___path__):
+ if has__path__:
+ return caller_name
+ else:
+ return caller_name.rsplit('.', 1)[0]
+
+ Then the normal algorithm for relative name imports can proceed as if
+ __package__ had been set.
+
+ """
+
+ def import_module(self, globals_):
+ with self.mock_modules('pkg.__init__', 'pkg.fake') as importer:
+ with util.import_state(meta_path=[importer]):
+ self.__import__('pkg.fake')
+ module = self.__import__('',
+ globals=globals_,
+ fromlist=['attr'], level=2)
+ return module
+
+ def test_using___package__(self):
+ # [__package__]
+ module = self.import_module({'__package__': 'pkg.fake'})
+ self.assertEqual(module.__name__, 'pkg')
+
+ def test_using___name__(self):
+ # [__name__]
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ module = self.import_module({'__name__': 'pkg.fake',
+ '__path__': []})
+ self.assertEqual(module.__name__, 'pkg')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_warn_when_using___name__(self):
+ with self.assertWarns(ImportWarning):
+ self.import_module({'__name__': 'pkg.fake', '__path__': []})
+
+ def test_None_as___package__(self):
+ # [None]
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ module = self.import_module({
+ '__name__': 'pkg.fake', '__path__': [], '__package__': None })
+ self.assertEqual(module.__name__, 'pkg')
+
+ def test_spec_fallback(self):
+ # If __package__ isn't defined, fall back on __spec__.parent.
+ module = self.import_module({'__spec__': FakeSpec('pkg.fake')})
+ self.assertEqual(module.__name__, 'pkg')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_warn_when_package_and_spec_disagree(self):
+ # Raise an ImportWarning if __package__ != __spec__.parent.
+ with self.assertWarns(ImportWarning):
+ self.import_module({'__package__': 'pkg.fake',
+ '__spec__': FakeSpec('pkg.fakefake')})
+
+ def test_bad__package__(self):
+ globals = {'__package__': ''}
+ with self.assertRaises(ModuleNotFoundError):
+ self.__import__('', globals, {}, ['relimport'], 1)
+
+ def test_bunk__package__(self):
+ globals = {'__package__': 42}
+ with self.assertRaises(TypeError):
+ self.__import__('', globals, {}, ['relimport'], 1)
+
+
+class FakeSpec:
+ def __init__(self, parent):
+ self.parent = parent
+
+
+class Using__package__PEP302(Using__package__):
+ mock_modules = util.mock_modules
+
+
+(Frozen_UsingPackagePEP302,
+ Source_UsingPackagePEP302
+ ) = util.test_both(Using__package__PEP302, __import__=util.__import__)
+
+
+class Using__package__PEP451(Using__package__):
+ mock_modules = util.mock_spec
+
+
+(Frozen_UsingPackagePEP451,
+ Source_UsingPackagePEP451
+ ) = util.test_both(Using__package__PEP451, __import__=util.__import__)
+
+
+class Setting__package__:
+
+ """Because __package__ is a new feature, it is not always set by a loader.
+ Import will set it as needed to help with the transition to relying on
+ __package__.
+
+ For a top-level module, __package__ is set to None [top-level]. For a
+ package __name__ is used for __package__ [package]. For submodules the
+ value is __name__.rsplit('.', 1)[0] [submodule].
+
+ """
+
+ __import__ = util.__import__['Source']
+
+ # [top-level]
+ def test_top_level(self):
+ with self.mock_modules('top_level') as mock:
+ with util.import_state(meta_path=[mock]):
+ del mock['top_level'].__package__
+ module = self.__import__('top_level')
+ self.assertEqual(module.__package__, '')
+
+ # [package]
+ def test_package(self):
+ with self.mock_modules('pkg.__init__') as mock:
+ with util.import_state(meta_path=[mock]):
+ del mock['pkg'].__package__
+ module = self.__import__('pkg')
+ self.assertEqual(module.__package__, 'pkg')
+
+ # [submodule]
+ def test_submodule(self):
+ with self.mock_modules('pkg.__init__', 'pkg.mod') as mock:
+ with util.import_state(meta_path=[mock]):
+ del mock['pkg.mod'].__package__
+ pkg = self.__import__('pkg.mod')
+ module = getattr(pkg, 'mod')
+ self.assertEqual(module.__package__, 'pkg')
+
+class Setting__package__PEP302(Setting__package__, unittest.TestCase):
+ mock_modules = util.mock_modules
+
+class Setting__package__PEP451(Setting__package__, unittest.TestCase):
+ mock_modules = util.mock_spec
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
new file mode 100644
index 000000000..0cd9de4da
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -0,0 +1,119 @@
+from .. import util
+
+from importlib import machinery
+import sys
+import types
+import unittest
+
+PKG_NAME = 'fine'
+SUBMOD_NAME = 'fine.bogus'
+
+
+class BadSpecFinderLoader:
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ if fullname == SUBMOD_NAME:
+ spec = machinery.ModuleSpec(fullname, cls)
+ return spec
+
+ @staticmethod
+ def create_module(spec):
+ return None
+
+ @staticmethod
+ def exec_module(module):
+ if module.__name__ == SUBMOD_NAME:
+ raise ImportError('I cannot be loaded!')
+
+
+class BadLoaderFinder:
+ @classmethod
+ def find_module(cls, fullname, path):
+ if fullname == SUBMOD_NAME:
+ return cls
+
+ @classmethod
+ def load_module(cls, fullname):
+ if fullname == SUBMOD_NAME:
+ raise ImportError('I cannot be loaded!')
+
+
+class APITest:
+
+ """Test API-specific details for __import__ (e.g. raising the right
+ exception when passing in an int for the module name)."""
+
+ def test_raises_ModuleNotFoundError(self):
+ with self.assertRaises(ModuleNotFoundError):
+ util.import_importlib('some module that does not exist')
+
+ def test_name_requires_rparition(self):
+ # Raise TypeError if a non-string is passed in for the module name.
+ with self.assertRaises(TypeError):
+ self.__import__(42)
+
+ def test_negative_level(self):
+ # Raise ValueError when a negative level is specified.
+ # PEP 328 did away with sys.module None entries and the ambiguity of
+ # absolute/relative imports.
+ with self.assertRaises(ValueError):
+ self.__import__('os', globals(), level=-1)
+
+ def test_nonexistent_fromlist_entry(self):
+ # If something in fromlist doesn't exist, that's okay.
+ # issue15715
+ mod = types.ModuleType(PKG_NAME)
+ mod.__path__ = ['XXX']
+ with util.import_state(meta_path=[self.bad_finder_loader]):
+ with util.uncache(PKG_NAME):
+ sys.modules[PKG_NAME] = mod
+ self.__import__(PKG_NAME, fromlist=['not here'])
+
+ def test_fromlist_load_error_propagates(self):
+ # If something in fromlist triggers an exception not related to not
+ # existing, let that exception propagate.
+ # issue15316
+ mod = types.ModuleType(PKG_NAME)
+ mod.__path__ = ['XXX']
+ with util.import_state(meta_path=[self.bad_finder_loader]):
+ with util.uncache(PKG_NAME):
+ sys.modules[PKG_NAME] = mod
+ with self.assertRaises(ImportError):
+ self.__import__(PKG_NAME,
+ fromlist=[SUBMOD_NAME.rpartition('.')[-1]])
+
+ def test_blocked_fromlist(self):
+ # If fromlist entry is None, let a ModuleNotFoundError propagate.
+ # issue31642
+ mod = types.ModuleType(PKG_NAME)
+ mod.__path__ = []
+ with util.import_state(meta_path=[self.bad_finder_loader]):
+ with util.uncache(PKG_NAME, SUBMOD_NAME):
+ sys.modules[PKG_NAME] = mod
+ sys.modules[SUBMOD_NAME] = None
+ with self.assertRaises(ModuleNotFoundError) as cm:
+ self.__import__(PKG_NAME,
+ fromlist=[SUBMOD_NAME.rpartition('.')[-1]])
+ self.assertEqual(cm.exception.name, SUBMOD_NAME)
+
+
+class OldAPITests(APITest):
+ bad_finder_loader = BadLoaderFinder
+
+
+(Frozen_OldAPITests,
+ Source_OldAPITests
+ ) = util.test_both(OldAPITests, __import__=util.__import__)
+
+
+class SpecAPITests(APITest):
+ bad_finder_loader = BadSpecFinderLoader
+
+
+(Frozen_SpecAPITests,
+ Source_SpecAPITests
+ ) = util.test_both(SpecAPITests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py
new file mode 100644
index 000000000..0630e10bd
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_caching.py
@@ -0,0 +1,94 @@
+"""Test that sys.modules is used properly by import."""
+from .. import util
+import sys
+from types import MethodType
+import unittest
+
+
+class UseCache:
+
+ """When it comes to sys.modules, import prefers it over anything else.
+
+ Once a name has been resolved, sys.modules is checked to see if it contains
+ the module desired. If so, then it is returned [use cache]. If it is not
+ found, then the proper steps are taken to perform the import, but
+ sys.modules is still used to return the imported module (e.g., not what a
+ loader returns) [from cache on return]. This also applies to imports of
+ things contained within a package and thus get assigned as an attribute
+ [from cache to attribute] or pulled in thanks to a fromlist import
+ [from cache for fromlist]. But if sys.modules contains None then
+ ImportError is raised [None in cache].
+
+ """
+
+ def test_using_cache(self):
+ # [use cache]
+ module_to_use = "some module found!"
+ with util.uncache('some_module'):
+ sys.modules['some_module'] = module_to_use
+ module = self.__import__('some_module')
+ self.assertEqual(id(module_to_use), id(module))
+
+ def test_None_in_cache(self):
+ #[None in cache]
+ name = 'using_None'
+ with util.uncache(name):
+ sys.modules[name] = None
+ with self.assertRaises(ImportError) as cm:
+ self.__import__(name)
+ self.assertEqual(cm.exception.name, name)
+
+
+(Frozen_UseCache,
+ Source_UseCache
+ ) = util.test_both(UseCache, __import__=util.__import__)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class ImportlibUseCache(UseCache, unittest.TestCase):
+
+ # Pertinent only to PEP 302; exec_module() doesn't return a module.
+
+ __import__ = util.__import__['Source']
+
+ def create_mock(self, *names, return_=None):
+ mock = util.mock_modules(*names)
+ original_load = mock.load_module
+ def load_module(self, fullname):
+ original_load(fullname)
+ return return_
+ mock.load_module = MethodType(load_module, mock)
+ return mock
+
+ # __import__ inconsistent between loaders and built-in import when it comes
+ # to when to use the module in sys.modules and when not to.
+ def test_using_cache_after_loader(self):
+ # [from cache on return]
+ with self.create_mock('module') as mock:
+ with util.import_state(meta_path=[mock]):
+ module = self.__import__('module')
+ self.assertEqual(id(module), id(sys.modules['module']))
+
+ # See test_using_cache_after_loader() for reasoning.
+ def test_using_cache_for_assigning_to_attribute(self):
+ # [from cache to attribute]
+ with self.create_mock('pkg.__init__', 'pkg.module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg.module')
+ self.assertTrue(hasattr(module, 'module'))
+ self.assertEqual(id(module.module),
+ id(sys.modules['pkg.module']))
+
+ # See test_using_cache_after_loader() for reasoning.
+ def test_using_cache_for_fromlist(self):
+ # [from cache for fromlist]
+ with self.create_mock('pkg.__init__', 'pkg.module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg', fromlist=['module'])
+ self.assertTrue(hasattr(module, 'module'))
+ self.assertEqual(id(module.module),
+ id(sys.modules['pkg.module']))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py
new file mode 100644
index 000000000..018c17217
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_fromlist.py
@@ -0,0 +1,175 @@
+"""Test that the semantics relating to the 'fromlist' argument are correct."""
+from .. import util
+import warnings
+import unittest
+
+
+class ReturnValue:
+
+ """The use of fromlist influences what import returns.
+
+ If direct ``import ...`` statement is used, the root module or package is
+ returned [import return]. But if fromlist is set, then the specified module
+ is actually returned (whether it is a relative import or not)
+ [from return].
+
+ """
+
+ def test_return_from_import(self):
+ # [import return]
+ with util.mock_spec('pkg.__init__', 'pkg.module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg.module')
+ self.assertEqual(module.__name__, 'pkg')
+
+ def test_return_from_from_import(self):
+ # [from return]
+ with util.mock_modules('pkg.__init__', 'pkg.module')as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg.module', fromlist=['attr'])
+ self.assertEqual(module.__name__, 'pkg.module')
+
+
+(Frozen_ReturnValue,
+ Source_ReturnValue
+ ) = util.test_both(ReturnValue, __import__=util.__import__)
+
+
+class HandlingFromlist:
+
+ """Using fromlist triggers different actions based on what is being asked
+ of it.
+
+ If fromlist specifies an object on a module, nothing special happens
+ [object case]. This is even true if the object does not exist [bad object].
+
+ If a package is being imported, then what is listed in fromlist may be
+ treated as a module to be imported [module]. And this extends to what is
+ contained in __all__ when '*' is imported [using *]. And '*' does not need
+ to be the only name in the fromlist [using * with others].
+
+ """
+
+ def test_object(self):
+ # [object case]
+ with util.mock_modules('module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('module', fromlist=['attr'])
+ self.assertEqual(module.__name__, 'module')
+
+ def test_nonexistent_object(self):
+ # [bad object]
+ with util.mock_modules('module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('module', fromlist=['non_existent'])
+ self.assertEqual(module.__name__, 'module')
+ self.assertFalse(hasattr(module, 'non_existent'))
+
+ def test_module_from_package(self):
+ # [module]
+ with util.mock_modules('pkg.__init__', 'pkg.module') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg', fromlist=['module'])
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'module'))
+ self.assertEqual(module.module.__name__, 'pkg.module')
+
+ def test_nonexistent_from_package(self):
+ with util.mock_modules('pkg.__init__') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg', fromlist=['non_existent'])
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertFalse(hasattr(module, 'non_existent'))
+
+ def test_module_from_package_triggers_ModuleNotFoundError(self):
+ # If a submodule causes an ModuleNotFoundError because it tries
+ # to import a module which doesn't exist, that should let the
+ # ModuleNotFoundError propagate.
+ def module_code():
+ import i_do_not_exist
+ with util.mock_modules('pkg.__init__', 'pkg.mod',
+ module_code={'pkg.mod': module_code}) as importer:
+ with util.import_state(meta_path=[importer]):
+ with self.assertRaises(ModuleNotFoundError) as exc:
+ self.__import__('pkg', fromlist=['mod'])
+ self.assertEqual('i_do_not_exist', exc.exception.name)
+
+ def test_empty_string(self):
+ with util.mock_modules('pkg.__init__', 'pkg.mod') as importer:
+ with util.import_state(meta_path=[importer]):
+ module = self.__import__('pkg.mod', fromlist=[''])
+ self.assertEqual(module.__name__, 'pkg.mod')
+
+ def basic_star_test(self, fromlist=['*']):
+ # [using *]
+ with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
+ with util.import_state(meta_path=[mock]):
+ mock['pkg'].__all__ = ['module']
+ module = self.__import__('pkg', fromlist=fromlist)
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'module'))
+ self.assertEqual(module.module.__name__, 'pkg.module')
+
+ def test_using_star(self):
+ # [using *]
+ self.basic_star_test()
+
+ def test_fromlist_as_tuple(self):
+ self.basic_star_test(('*',))
+
+ def test_star_with_others(self):
+ # [using * with others]
+ context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2')
+ with context as mock:
+ with util.import_state(meta_path=[mock]):
+ mock['pkg'].__all__ = ['module1']
+ module = self.__import__('pkg', fromlist=['module2', '*'])
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'module1'))
+ self.assertTrue(hasattr(module, 'module2'))
+ self.assertEqual(module.module1.__name__, 'pkg.module1')
+ self.assertEqual(module.module2.__name__, 'pkg.module2')
+
+ def test_nonexistent_in_all(self):
+ with util.mock_modules('pkg.__init__') as importer:
+ with util.import_state(meta_path=[importer]):
+ importer['pkg'].__all__ = ['non_existent']
+ module = self.__import__('pkg', fromlist=['*'])
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertFalse(hasattr(module, 'non_existent'))
+
+ def test_star_in_all(self):
+ with util.mock_modules('pkg.__init__') as importer:
+ with util.import_state(meta_path=[importer]):
+ importer['pkg'].__all__ = ['*']
+ module = self.__import__('pkg', fromlist=['*'])
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertFalse(hasattr(module, '*'))
+
+ def test_invalid_type(self):
+ with util.mock_modules('pkg.__init__') as importer:
+ with util.import_state(meta_path=[importer]), \
+ warnings.catch_warnings():
+ warnings.simplefilter('error', BytesWarning)
+ with self.assertRaisesRegex(TypeError, r'\bfrom\b'):
+ self.__import__('pkg', fromlist=[b'attr'])
+ with self.assertRaisesRegex(TypeError, r'\bfrom\b'):
+ self.__import__('pkg', fromlist=iter([b'attr']))
+
+ def test_invalid_type_in_all(self):
+ with util.mock_modules('pkg.__init__') as importer:
+ with util.import_state(meta_path=[importer]), \
+ warnings.catch_warnings():
+ warnings.simplefilter('error', BytesWarning)
+ importer['pkg'].__all__ = [b'attr']
+ with self.assertRaisesRegex(TypeError, r'\bpkg\.__all__\b'):
+ self.__import__('pkg', fromlist=['*'])
+
+
+(Frozen_FromList,
+ Source_FromList
+ ) = util.test_both(HandlingFromlist, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py
new file mode 100644
index 000000000..469684886
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_meta_path.py
@@ -0,0 +1,128 @@
+from .. import util
+import importlib._bootstrap
+import sys
+from types import MethodType
+import unittest
+import warnings
+
+
+class CallingOrder:
+
+ """Calls to the importers on sys.meta_path happen in order that they are
+ specified in the sequence, starting with the first importer
+ [first called], and then continuing on down until one is found that doesn't
+ return None [continuing]."""
+
+
+ def test_first_called(self):
+ # [first called]
+ mod = 'top_level'
+ with util.mock_spec(mod) as first, util.mock_spec(mod) as second:
+ with util.import_state(meta_path=[first, second]):
+ self.assertIs(self.__import__(mod), first.modules[mod])
+
+ def test_continuing(self):
+ # [continuing]
+ mod_name = 'for_real'
+ with util.mock_spec('nonexistent') as first, \
+ util.mock_spec(mod_name) as second:
+ first.find_spec = lambda self, fullname, path=None, parent=None: None
+ with util.import_state(meta_path=[first, second]):
+ self.assertIs(self.__import__(mod_name), second.modules[mod_name])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_empty(self):
+ # Raise an ImportWarning if sys.meta_path is empty.
+ module_name = 'nothing'
+ try:
+ del sys.modules[module_name]
+ except KeyError:
+ pass
+ with util.import_state(meta_path=[]):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self.assertIsNone(importlib._bootstrap._find_spec('nothing',
+ None))
+ self.assertEqual(len(w), 1)
+ self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
+
+(Frozen_CallingOrder,
+ Source_CallingOrder
+ ) = util.test_both(CallingOrder, __import__=util.__import__)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class CallSignature:
+
+ """If there is no __path__ entry on the parent module, then 'path' is None
+ [no path]. Otherwise, the value for __path__ is passed in for the 'path'
+ argument [path set]."""
+
+ def log_finder(self, importer):
+ fxn = getattr(importer, self.finder_name)
+ log = []
+ def wrapper(self, *args, **kwargs):
+ log.append([args, kwargs])
+ return fxn(*args, **kwargs)
+ return log, wrapper
+
+ def test_no_path(self):
+ # [no path]
+ mod_name = 'top_level'
+ assert '.' not in mod_name
+ with self.mock_modules(mod_name) as importer:
+ log, wrapped_call = self.log_finder(importer)
+ setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
+ with util.import_state(meta_path=[importer]):
+ self.__import__(mod_name)
+ assert len(log) == 1
+ args = log[0][0]
+ # Assuming all arguments are positional.
+ self.assertEqual(args[0], mod_name)
+ self.assertIsNone(args[1])
+
+ def test_with_path(self):
+ # [path set]
+ pkg_name = 'pkg'
+ mod_name = pkg_name + '.module'
+ path = [42]
+ assert '.' in mod_name
+ with self.mock_modules(pkg_name+'.__init__', mod_name) as importer:
+ importer.modules[pkg_name].__path__ = path
+ log, wrapped_call = self.log_finder(importer)
+ setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
+ with util.import_state(meta_path=[importer]):
+ self.__import__(mod_name)
+ assert len(log) == 2
+ args = log[1][0]
+ kwargs = log[1][1]
+ # Assuming all arguments are positional.
+ self.assertFalse(kwargs)
+ self.assertEqual(args[0], mod_name)
+ self.assertIs(args[1], path)
+
+
+class CallSignaturePEP302(CallSignature):
+ mock_modules = util.mock_modules
+ finder_name = 'find_module'
+
+
+(Frozen_CallSignaturePEP302,
+ Source_CallSignaturePEP302
+ ) = util.test_both(CallSignaturePEP302, __import__=util.__import__)
+
+
+class CallSignaturePEP451(CallSignature):
+ mock_modules = util.mock_spec
+ finder_name = 'find_spec'
+
+
+(Frozen_CallSignaturePEP451,
+ Source_CallSignaturePEP451
+ ) = util.test_both(CallSignaturePEP451, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py
new file mode 100644
index 000000000..24396044a
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_packages.py
@@ -0,0 +1,110 @@
+from .. import util
+import sys
+import unittest
+from test import support
+
+
+class ParentModuleTests:
+
+ """Importing a submodule should import the parent modules."""
+
+ def test_import_parent(self):
+ with util.mock_spec('pkg.__init__', 'pkg.module') as mock:
+ with util.import_state(meta_path=[mock]):
+ module = self.__import__('pkg.module')
+ self.assertIn('pkg', sys.modules)
+
+ def test_bad_parent(self):
+ with util.mock_spec('pkg.module') as mock:
+ with util.import_state(meta_path=[mock]):
+ with self.assertRaises(ImportError) as cm:
+ self.__import__('pkg.module')
+ self.assertEqual(cm.exception.name, 'pkg')
+
+ def test_raising_parent_after_importing_child(self):
+ def __init__():
+ import pkg.module
+ 1/0
+ mock = util.mock_spec('pkg.__init__', 'pkg.module',
+ module_code={'pkg': __init__})
+ with mock:
+ with util.import_state(meta_path=[mock]):
+ with self.assertRaises(ZeroDivisionError):
+ self.__import__('pkg')
+ self.assertNotIn('pkg', sys.modules)
+ self.assertIn('pkg.module', sys.modules)
+ with self.assertRaises(ZeroDivisionError):
+ self.__import__('pkg.module')
+ self.assertNotIn('pkg', sys.modules)
+ self.assertIn('pkg.module', sys.modules)
+
+ def test_raising_parent_after_relative_importing_child(self):
+ def __init__():
+ from . import module
+ 1/0
+ mock = util.mock_spec('pkg.__init__', 'pkg.module',
+ module_code={'pkg': __init__})
+ with mock:
+ with util.import_state(meta_path=[mock]):
+ with self.assertRaises((ZeroDivisionError, ImportError)):
+ # This raises ImportError on the "from . import module"
+ # line, not sure why.
+ self.__import__('pkg')
+ self.assertNotIn('pkg', sys.modules)
+ with self.assertRaises((ZeroDivisionError, ImportError)):
+ self.__import__('pkg.module')
+ self.assertNotIn('pkg', sys.modules)
+ # XXX False
+ #self.assertIn('pkg.module', sys.modules)
+
+ def test_raising_parent_after_double_relative_importing_child(self):
+ def __init__():
+ from ..subpkg import module
+ 1/0
+ mock = util.mock_spec('pkg.__init__', 'pkg.subpkg.__init__',
+ 'pkg.subpkg.module',
+ module_code={'pkg.subpkg': __init__})
+ with mock:
+ with util.import_state(meta_path=[mock]):
+ with self.assertRaises((ZeroDivisionError, ImportError)):
+ # This raises ImportError on the "from ..subpkg import module"
+ # line, not sure why.
+ self.__import__('pkg.subpkg')
+ self.assertNotIn('pkg.subpkg', sys.modules)
+ with self.assertRaises((ZeroDivisionError, ImportError)):
+ self.__import__('pkg.subpkg.module')
+ self.assertNotIn('pkg.subpkg', sys.modules)
+ # XXX False
+ #self.assertIn('pkg.subpkg.module', sys.modules)
+
+ def test_module_not_package(self):
+ # Try to import a submodule from a non-package should raise ImportError.
+ assert not hasattr(sys, '__path__')
+ with self.assertRaises(ImportError) as cm:
+ self.__import__('sys.no_submodules_here')
+ self.assertEqual(cm.exception.name, 'sys.no_submodules_here')
+
+ def test_module_not_package_but_side_effects(self):
+ # If a module injects something into sys.modules as a side-effect, then
+ # pick up on that fact.
+ name = 'mod'
+ subname = name + '.b'
+ def module_injection():
+ sys.modules[subname] = 'total bunk'
+ mock_spec = util.mock_spec('mod',
+ module_code={'mod': module_injection})
+ with mock_spec as mock:
+ with util.import_state(meta_path=[mock]):
+ try:
+ submodule = self.__import__(subname)
+ finally:
+ support.unload(subname)
+
+
+(Frozen_ParentTests,
+ Source_ParentTests
+ ) = util.test_both(ParentModuleTests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py
new file mode 100644
index 000000000..7beaee1ca
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_path.py
@@ -0,0 +1,280 @@
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+import os
+import sys
+import tempfile
+from types import ModuleType
+import unittest
+import warnings
+import zipimport
+
+
+class FinderTests:
+
+ """Tests for PathFinder."""
+
+ find = None
+ check_found = None
+
+ def test_failure(self):
+ # Test None returned upon not finding a suitable loader.
+ module = ''
+ with util.import_state():
+ self.assertIsNone(self.find(module))
+
+ def test_sys_path(self):
+ # Test that sys.path is used when 'path' is None.
+ # Implicitly tests that sys.path_importer_cache is used.
+ module = ''
+ path = ''
+ importer = util.mock_spec(module)
+ with util.import_state(path_importer_cache={path: importer},
+ path=[path]):
+ found = self.find(module)
+ self.check_found(found, importer)
+
+ def test_path(self):
+ # Test that 'path' is used when set.
+ # Implicitly tests that sys.path_importer_cache is used.
+ module = ''
+ path = ''
+ importer = util.mock_spec(module)
+ with util.import_state(path_importer_cache={path: importer}):
+ found = self.find(module, [path])
+ self.check_found(found, importer)
+
+ def test_empty_list(self):
+ # An empty list should not count as asking for sys.path.
+ module = 'module'
+ path = ''
+ importer = util.mock_spec(module)
+ with util.import_state(path_importer_cache={path: importer},
+ path=[path]):
+ self.assertIsNone(self.find('module', []))
+
+ def test_path_hooks(self):
+ # Test that sys.path_hooks is used.
+ # Test that sys.path_importer_cache is set.
+ module = ''
+ path = ''
+ importer = util.mock_spec(module)
+ hook = util.mock_path_hook(path, importer=importer)
+ with util.import_state(path_hooks=[hook]):
+ found = self.find(module, [path])
+ self.check_found(found, importer)
+ self.assertIn(path, sys.path_importer_cache)
+ self.assertIs(sys.path_importer_cache[path], importer)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_empty_path_hooks(self):
+ # Test that if sys.path_hooks is empty a warning is raised,
+ # sys.path_importer_cache gets None set, and PathFinder returns None.
+ path_entry = 'bogus_path'
+ with util.import_state(path_importer_cache={}, path_hooks=[],
+ path=[path_entry]):
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ self.assertIsNone(self.find('os'))
+ self.assertIsNone(sys.path_importer_cache[path_entry])
+ self.assertEqual(len(w), 1)
+ self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
+ def test_path_importer_cache_empty_string(self):
+ # The empty string should create a finder using the cwd.
+ path = ''
+ module = ''
+ importer = util.mock_spec(module)
+ hook = util.mock_path_hook(os.getcwd(), importer=importer)
+ with util.import_state(path=[path], path_hooks=[hook]):
+ found = self.find(module)
+ self.check_found(found, importer)
+ self.assertIn(os.getcwd(), sys.path_importer_cache)
+
+ def test_None_on_sys_path(self):
+ # Putting None in sys.path[0] caused an import regression from Python
+ # 3.2: http://bugs.python.org/issue16514
+ new_path = sys.path[:]
+ new_path.insert(0, None)
+ new_path_importer_cache = sys.path_importer_cache.copy()
+ new_path_importer_cache.pop(None, None)
+ new_path_hooks = [zipimport.zipimporter,
+ self.machinery.FileFinder.path_hook(
+ *self.importlib._bootstrap_external._get_supported_file_loaders())]
+ missing = object()
+ email = sys.modules.pop('email', missing)
+ try:
+ with util.import_state(meta_path=sys.meta_path[:],
+ path=new_path,
+ path_importer_cache=new_path_importer_cache,
+ path_hooks=new_path_hooks):
+ module = self.importlib.import_module('email')
+ self.assertIsInstance(module, ModuleType)
+ finally:
+ if email is not missing:
+ sys.modules['email'] = email
+
+ def test_finder_with_find_module(self):
+ class TestFinder:
+ def find_module(self, fullname):
+ return self.to_return
+ failing_finder = TestFinder()
+ failing_finder.to_return = None
+ path = 'testing path'
+ with util.import_state(path_importer_cache={path: failing_finder}):
+ self.assertIsNone(
+ self.machinery.PathFinder.find_spec('whatever', [path]))
+ success_finder = TestFinder()
+ success_finder.to_return = __loader__
+ with util.import_state(path_importer_cache={path: success_finder}):
+ spec = self.machinery.PathFinder.find_spec('whatever', [path])
+ self.assertEqual(spec.loader, __loader__)
+
+ def test_finder_with_find_loader(self):
+ class TestFinder:
+ loader = None
+ portions = []
+ def find_loader(self, fullname):
+ return self.loader, self.portions
+ path = 'testing path'
+ with util.import_state(path_importer_cache={path: TestFinder()}):
+ self.assertIsNone(
+ self.machinery.PathFinder.find_spec('whatever', [path]))
+ success_finder = TestFinder()
+ success_finder.loader = __loader__
+ with util.import_state(path_importer_cache={path: success_finder}):
+ spec = self.machinery.PathFinder.find_spec('whatever', [path])
+ self.assertEqual(spec.loader, __loader__)
+
+ def test_finder_with_find_spec(self):
+ class TestFinder:
+ spec = None
+ def find_spec(self, fullname, target=None):
+ return self.spec
+ path = 'testing path'
+ with util.import_state(path_importer_cache={path: TestFinder()}):
+ self.assertIsNone(
+ self.machinery.PathFinder.find_spec('whatever', [path]))
+ success_finder = TestFinder()
+ success_finder.spec = self.machinery.ModuleSpec('whatever', __loader__)
+ with util.import_state(path_importer_cache={path: success_finder}):
+ got = self.machinery.PathFinder.find_spec('whatever', [path])
+ self.assertEqual(got, success_finder.spec)
+
+ def test_deleted_cwd(self):
+ # Issue #22834
+ old_dir = os.getcwd()
+ self.addCleanup(os.chdir, old_dir)
+ new_dir = tempfile.mkdtemp()
+ try:
+ os.chdir(new_dir)
+ try:
+ os.rmdir(new_dir)
+ except OSError:
+ # EINVAL on Solaris, EBUSY on AIX, ENOTEMPTY on Windows
+ self.skipTest("platform does not allow "
+ "the deletion of the cwd")
+ except:
+ os.chdir(old_dir)
+ os.rmdir(new_dir)
+ raise
+
+ with util.import_state(path=['']):
+ # Do not want FileNotFoundError raised.
+ self.assertIsNone(self.machinery.PathFinder.find_spec('whatever'))
+
+ def test_invalidate_caches_finders(self):
+ # Finders with an invalidate_caches() method have it called.
+ class FakeFinder:
+ def __init__(self):
+ self.called = False
+
+ def invalidate_caches(self):
+ self.called = True
+
+ cache = {'leave_alone': object(), 'finder_to_invalidate': FakeFinder()}
+ with util.import_state(path_importer_cache=cache):
+ self.machinery.PathFinder.invalidate_caches()
+ self.assertTrue(cache['finder_to_invalidate'].called)
+
+ def test_invalidate_caches_clear_out_None(self):
+ # Clear out None in sys.path_importer_cache() when invalidating caches.
+ cache = {'clear_out': None}
+ with util.import_state(path_importer_cache=cache):
+ self.machinery.PathFinder.invalidate_caches()
+ self.assertEqual(len(cache), 0)
+
+
+class FindModuleTests(FinderTests):
+ def find(self, *args, **kwargs):
+ return self.machinery.PathFinder.find_module(*args, **kwargs)
+ def check_found(self, found, importer):
+ self.assertIs(found, importer)
+
+
+(Frozen_FindModuleTests,
+ Source_FindModuleTests
+) = util.test_both(FindModuleTests, importlib=importlib, machinery=machinery)
+
+
+class FindSpecTests(FinderTests):
+ def find(self, *args, **kwargs):
+ return self.machinery.PathFinder.find_spec(*args, **kwargs)
+ def check_found(self, found, importer):
+ self.assertIs(found.loader, importer)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, importlib=importlib, machinery=machinery)
+
+
+class PathEntryFinderTests:
+
+ def test_finder_with_failing_find_spec(self):
+ # PathEntryFinder with find_module() defined should work.
+ # Issue #20763.
+ class Finder:
+ path_location = 'test_finder_with_find_module'
+ def __init__(self, path):
+ if path != self.path_location:
+ raise ImportError
+
+ @staticmethod
+ def find_module(fullname):
+ return None
+
+
+ with util.import_state(path=[Finder.path_location]+sys.path[:],
+ path_hooks=[Finder]):
+ self.machinery.PathFinder.find_spec('importlib')
+
+ def test_finder_with_failing_find_module(self):
+ # PathEntryFinder with find_module() defined should work.
+ # Issue #20763.
+ class Finder:
+ path_location = 'test_finder_with_find_module'
+ def __init__(self, path):
+ if path != self.path_location:
+ raise ImportError
+
+ @staticmethod
+ def find_module(fullname):
+ return None
+
+
+ with util.import_state(path=[Finder.path_location]+sys.path[:],
+ path_hooks=[Finder]):
+ self.machinery.PathFinder.find_module('importlib')
+
+
+(Frozen_PEFTests,
+ Source_PEFTests
+ ) = util.test_both(PathEntryFinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py
new file mode 100644
index 000000000..8a95a3210
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test_relative_imports.py
@@ -0,0 +1,232 @@
+"""Test relative imports (PEP 328)."""
+from .. import util
+import unittest
+import warnings
+
+
+class RelativeImports:
+
+ """PEP 328 introduced relative imports. This allows for imports to occur
+ from within a package without having to specify the actual package name.
+
+ A simple example is to import another module within the same package
+ [module from module]::
+
+ # From pkg.mod1 with pkg.mod2 being a module.
+ from . import mod2
+
+ This also works for getting an attribute from a module that is specified
+ in a relative fashion [attr from module]::
+
+ # From pkg.mod1.
+ from .mod2 import attr
+
+ But this is in no way restricted to working between modules; it works
+ from [package to module],::
+
+ # From pkg, importing pkg.module which is a module.
+ from . import module
+
+ [module to package],::
+
+ # Pull attr from pkg, called from pkg.module which is a module.
+ from . import attr
+
+ and [package to package]::
+
+ # From pkg.subpkg1 (both pkg.subpkg[1,2] are packages).
+ from .. import subpkg2
+
+ The number of dots used is in no way restricted [deep import]::
+
+ # Import pkg.attr from pkg.pkg1.pkg2.pkg3.pkg4.pkg5.
+ from ...... import attr
+
+ To prevent someone from accessing code that is outside of a package, one
+ cannot reach the location containing the root package itself::
+
+ # From pkg.__init__ [too high from package]
+ from .. import top_level
+
+ # From pkg.module [too high from module]
+ from .. import top_level
+
+ Relative imports are the only type of import that allow for an empty
+ module name for an import [empty name].
+
+ """
+
+ def relative_import_test(self, create, globals_, callback):
+ """Abstract out boilerplace for setting up for an import test."""
+ uncache_names = []
+ for name in create:
+ if not name.endswith('.__init__'):
+ uncache_names.append(name)
+ else:
+ uncache_names.append(name[:-len('.__init__')])
+ with util.mock_spec(*create) as importer:
+ with util.import_state(meta_path=[importer]):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ for global_ in globals_:
+ with util.uncache(*uncache_names):
+ callback(global_)
+
+
+ def test_module_from_module(self):
+ # [module from module]
+ create = 'pkg.__init__', 'pkg.mod2'
+ globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
+ def callback(global_):
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['mod2'], level=1)
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'mod2'))
+ self.assertEqual(module.mod2.attr, 'pkg.mod2')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_attr_from_module(self):
+ # [attr from module]
+ create = 'pkg.__init__', 'pkg.mod2'
+ globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
+ def callback(global_):
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('mod2', global_, fromlist=['attr'],
+ level=1)
+ self.assertEqual(module.__name__, 'pkg.mod2')
+ self.assertEqual(module.attr, 'pkg.mod2')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_package_to_module(self):
+ # [package to module]
+ create = 'pkg.__init__', 'pkg.module'
+ globals_ = ({'__package__': 'pkg'},
+ {'__name__': 'pkg', '__path__': ['blah']})
+ def callback(global_):
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['module'],
+ level=1)
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'module'))
+ self.assertEqual(module.module.attr, 'pkg.module')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_module_to_package(self):
+ # [module to package]
+ create = 'pkg.__init__', 'pkg.module'
+ globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
+ def callback(global_):
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['attr'], level=1)
+ self.assertEqual(module.__name__, 'pkg')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_package_to_package(self):
+ # [package to package]
+ create = ('pkg.__init__', 'pkg.subpkg1.__init__',
+ 'pkg.subpkg2.__init__')
+ globals_ = ({'__package__': 'pkg.subpkg1'},
+ {'__name__': 'pkg.subpkg1', '__path__': ['blah']})
+ def callback(global_):
+ module = self.__import__('', global_, fromlist=['subpkg2'],
+ level=2)
+ self.assertEqual(module.__name__, 'pkg')
+ self.assertTrue(hasattr(module, 'subpkg2'))
+ self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__')
+
+ def test_deep_import(self):
+ # [deep import]
+ create = ['pkg.__init__']
+ for count in range(1,6):
+ create.append('{0}.pkg{1}.__init__'.format(
+ create[-1][:-len('.__init__')], count))
+ globals_ = ({'__package__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5'},
+ {'__name__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5',
+ '__path__': ['blah']})
+ def callback(global_):
+ self.__import__(globals_[0]['__package__'])
+ module = self.__import__('', global_, fromlist=['attr'], level=6)
+ self.assertEqual(module.__name__, 'pkg')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_too_high_from_package(self):
+ # [too high from package]
+ create = ['top_level', 'pkg.__init__']
+ globals_ = ({'__package__': 'pkg'},
+ {'__name__': 'pkg', '__path__': ['blah']})
+ def callback(global_):
+ self.__import__('pkg')
+ with self.assertRaises(ValueError):
+ self.__import__('', global_, fromlist=['top_level'],
+ level=2)
+ self.relative_import_test(create, globals_, callback)
+
+ def test_too_high_from_module(self):
+ # [too high from module]
+ create = ['top_level', 'pkg.__init__', 'pkg.module']
+ globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
+ def callback(global_):
+ self.__import__('pkg')
+ with self.assertRaises(ValueError):
+ self.__import__('', global_, fromlist=['top_level'],
+ level=2)
+ self.relative_import_test(create, globals_, callback)
+
+ def test_empty_name_w_level_0(self):
+ # [empty name]
+ with self.assertRaises(ValueError):
+ self.__import__('')
+
+ def test_import_from_different_package(self):
+ # Test importing from a different package than the caller.
+ # in pkg.subpkg1.mod
+ # from ..subpkg2 import mod
+ create = ['__runpy_pkg__.__init__',
+ '__runpy_pkg__.__runpy_pkg__.__init__',
+ '__runpy_pkg__.uncle.__init__',
+ '__runpy_pkg__.uncle.cousin.__init__',
+ '__runpy_pkg__.uncle.cousin.nephew']
+ globals_ = {'__package__': '__runpy_pkg__.__runpy_pkg__'}
+ def callback(global_):
+ self.__import__('__runpy_pkg__.__runpy_pkg__')
+ module = self.__import__('uncle.cousin', globals_, {},
+ fromlist=['nephew'],
+ level=2)
+ self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_import_relative_import_no_fromlist(self):
+ # Import a relative module w/ no fromlist.
+ create = ['crash.__init__', 'crash.mod']
+ globals_ = [{'__package__': 'crash', '__name__': 'crash'}]
+ def callback(global_):
+ self.__import__('crash')
+ mod = self.__import__('mod', global_, {}, [], 1)
+ self.assertEqual(mod.__name__, 'crash.mod')
+ self.relative_import_test(create, globals_, callback)
+
+ def test_relative_import_no_globals(self):
+ # No globals for a relative import is an error.
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ with self.assertRaises(KeyError):
+ self.__import__('sys', level=1)
+
+ def test_relative_import_no_package(self):
+ with self.assertRaises(ImportError):
+ self.__import__('a', {'__package__': '', '__spec__': None},
+ level=1)
+
+ def test_relative_import_no_package_exists_absolute(self):
+ with self.assertRaises(ImportError):
+ self.__import__('sys', {'__package__': '', '__spec__': None},
+ level=1)
+
+
+(Frozen_RelativeImports,
+ Source_RelativeImports
+ ) = util.test_both(RelativeImports, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py
new file mode 100644
index 000000000..3080f6f8f
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py
@@ -0,0 +1 @@
+attr = 'both_portions foo one'
diff --git a/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py
new file mode 100644
index 000000000..4131d3d4b
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py
@@ -0,0 +1 @@
+attr = 'both_portions foo two'
diff --git a/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip b/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip
new file mode 100644
index 000000000..836a9106b
Binary files /dev/null and b/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip differ
diff --git a/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py
new file mode 100644
index 000000000..43cbedbbd
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py
@@ -0,0 +1 @@
+attr = 'in module'
diff --git a/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip b/Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip
new file mode 100644
index 000000000..8d22406f2
Binary files /dev/null and b/Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip differ
diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py
new file mode 100644
index 000000000..d8f5c831f
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py
@@ -0,0 +1 @@
+attr = 'portion1 foo one'
diff --git a/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py
new file mode 100644
index 000000000..d8f5c831f
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py
@@ -0,0 +1 @@
+attr = 'portion1 foo one'
diff --git a/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py
new file mode 100644
index 000000000..d092e1e99
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py
@@ -0,0 +1 @@
+attr = 'portion2 foo two'
diff --git a/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py
new file mode 100644
index 000000000..2776fcdfd
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py
@@ -0,0 +1 @@
+attr = 'parent child one'
diff --git a/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py
new file mode 100644
index 000000000..8b037bcb0
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py
@@ -0,0 +1 @@
+attr = 'parent child two'
diff --git a/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py
new file mode 100644
index 000000000..f8abfe1c1
--- /dev/null
+++ b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py
@@ -0,0 +1 @@
+attr = 'parent child three'
diff --git a/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip b/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip
new file mode 100644
index 000000000..3b866c914
Binary files /dev/null and b/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip differ
diff --git a/Lib/test/test_importlib/source/__init__.py b/Lib/test/test_importlib/source/__init__.py
new file mode 100644
index 000000000..4b16ecc31
--- /dev/null
+++ b/Lib/test/test_importlib/source/__init__.py
@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+ return load_package_tests(os.path.dirname(__file__), *args)
diff --git a/Lib/test/test_importlib/source/__main__.py b/Lib/test/test_importlib/source/__main__.py
new file mode 100644
index 000000000..40a23a297
--- /dev/null
+++ b/Lib/test/test_importlib/source/__main__.py
@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()
diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
new file mode 100644
index 000000000..12ce0cb93
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -0,0 +1,85 @@
+"""Test case-sensitivity (PEP 235)."""
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+import os
+from test import support as test_support
+import unittest
+
+
+@util.case_insensitive_tests
+class CaseSensitivityTest(util.CASEOKTestBase):
+
+ """PEP 235 dictates that on case-preserving, case-insensitive file systems
+ that imports are case-sensitive unless the PYTHONCASEOK environment
+ variable is set."""
+
+ name = 'MoDuLe'
+ assert name != name.lower()
+
+ def finder(self, path):
+ return self.machinery.FileFinder(path,
+ (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES),
+ (self.machinery.SourcelessFileLoader,
+ self.machinery.BYTECODE_SUFFIXES))
+
+ def sensitivity_test(self):
+ """Look for a module with matching and non-matching sensitivity."""
+ sensitive_pkg = 'sensitive.{0}'.format(self.name)
+ insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
+ context = util.create_modules(insensitive_pkg, sensitive_pkg)
+ with context as mapping:
+ sensitive_path = os.path.join(mapping['.root'], 'sensitive')
+ insensitive_path = os.path.join(mapping['.root'], 'insensitive')
+ sensitive_finder = self.finder(sensitive_path)
+ insensitive_finder = self.finder(insensitive_path)
+ return self.find(sensitive_finder), self.find(insensitive_finder)
+
+ def test_sensitive(self):
+ with test_support.EnvironmentVarGuard() as env:
+ env.unset('PYTHONCASEOK')
+ self.caseok_env_changed(should_exist=False)
+ sensitive, insensitive = self.sensitivity_test()
+ self.assertIsNotNone(sensitive)
+ self.assertIn(self.name, sensitive.get_filename(self.name))
+ self.assertIsNone(insensitive)
+
+ def test_insensitive(self):
+ with test_support.EnvironmentVarGuard() as env:
+ env.set('PYTHONCASEOK', '1')
+ self.caseok_env_changed(should_exist=True)
+ sensitive, insensitive = self.sensitivity_test()
+ self.assertIsNotNone(sensitive)
+ self.assertIn(self.name, sensitive.get_filename(self.name))
+ self.assertIsNotNone(insensitive)
+ self.assertIn(self.name, insensitive.get_filename(self.name))
+
+
+class CaseSensitivityTestPEP302(CaseSensitivityTest):
+ def find(self, finder):
+ return finder.find_module(self.name)
+
+
+(Frozen_CaseSensitivityTestPEP302,
+ Source_CaseSensitivityTestPEP302
+ ) = util.test_both(CaseSensitivityTestPEP302, importlib=importlib,
+ machinery=machinery)
+
+
+class CaseSensitivityTestPEP451(CaseSensitivityTest):
+ def find(self, finder):
+ found = finder.find_spec(self.name)
+ return found.loader if found is not None else found
+
+
+(Frozen_CaseSensitivityTestPEP451,
+ Source_CaseSensitivityTestPEP451
+ ) = util.test_both(CaseSensitivityTestPEP451, importlib=importlib,
+ machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
new file mode 100644
index 000000000..79187e696
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -0,0 +1,804 @@
+from .. import abc
+from .. import util
+
+importlib = util.import_importlib('importlib')
+importlib_abc = util.import_importlib('importlib.abc')
+machinery = util.import_importlib('importlib.machinery')
+importlib_util = util.import_importlib('importlib.util')
+
+import errno
+import marshal
+import os
+import py_compile
+import shutil
+import stat
+import sys
+import types
+import unittest
+import warnings
+
+from test.support import make_legacy_pyc, unload
+
+from test.test_py_compile import without_source_date_epoch
+from test.test_py_compile import SourceDateEpochTestMeta
+
+
+class SimpleTest(abc.LoaderTests):
+
+ """Should have no issue importing a source module [basic]. And if there is
+ a syntax error, it should raise a SyntaxError [syntax error].
+
+ """
+
+ def setUp(self):
+ self.name = 'spam'
+ self.filepath = os.path.join('ham', self.name + '.py')
+ self.loader = self.machinery.SourceFileLoader(self.name, self.filepath)
+
+ def test_load_module_API(self):
+ class Tester(self.abc.FileLoader):
+ def get_source(self, _): return 'attr = 42'
+ def is_package(self, _): return False
+
+ loader = Tester('blah', 'blah.py')
+ self.addCleanup(unload, 'blah')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module() # Should not raise an exception.
+
+ def test_get_filename_API(self):
+ # If fullname is not set then assume self.path is desired.
+ class Tester(self.abc.FileLoader):
+ def get_code(self, _): pass
+ def get_source(self, _): pass
+ def is_package(self, _): pass
+ def module_repr(self, _): pass
+
+ path = 'some_path'
+ name = 'some_name'
+ loader = Tester(name, path)
+ self.assertEqual(path, loader.get_filename(name))
+ self.assertEqual(path, loader.get_filename())
+ self.assertEqual(path, loader.get_filename(None))
+ with self.assertRaises(ImportError):
+ loader.get_filename(name + 'XXX')
+
+ def test_equality(self):
+ other = self.machinery.SourceFileLoader(self.name, self.filepath)
+ self.assertEqual(self.loader, other)
+
+ def test_inequality(self):
+ other = self.machinery.SourceFileLoader('_' + self.name, self.filepath)
+ self.assertNotEqual(self.loader, other)
+
+ # [basic]
+ def test_module(self):
+ with util.create_modules('_temp') as mapping:
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
+ self.assertIn('_temp', sys.modules)
+ check = {'__name__': '_temp', '__file__': mapping['_temp'],
+ '__package__': ''}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+ def test_package(self):
+ with util.create_modules('_pkg.__init__') as mapping:
+ loader = self.machinery.SourceFileLoader('_pkg',
+ mapping['_pkg.__init__'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_pkg')
+ self.assertIn('_pkg', sys.modules)
+ check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
+ '__path__': [os.path.dirname(mapping['_pkg.__init__'])],
+ '__package__': '_pkg'}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+
+ def test_lacking_parent(self):
+ with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
+ loader = self.machinery.SourceFileLoader('_pkg.mod',
+ mapping['_pkg.mod'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_pkg.mod')
+ self.assertIn('_pkg.mod', sys.modules)
+ check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
+ '__package__': '_pkg'}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+
+ def fake_mtime(self, fxn):
+ """Fake mtime to always be higher than expected."""
+ return lambda name: fxn(name) + 1
+
+ def test_module_reuse(self):
+ with util.create_modules('_temp') as mapping:
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
+ module_id = id(module)
+ module_dict_id = id(module.__dict__)
+ with open(mapping['_temp'], 'w') as file:
+ file.write("testing_var = 42\n")
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
+ self.assertIn('testing_var', module.__dict__,
+ "'testing_var' not in "
+ "{0}".format(list(module.__dict__.keys())))
+ self.assertEqual(module, sys.modules['_temp'])
+ self.assertEqual(id(module), module_id)
+ self.assertEqual(id(module.__dict__), module_dict_id)
+
+ def test_state_after_failure(self):
+ # A failed reload should leave the original module intact.
+ attributes = ('__file__', '__path__', '__package__')
+ value = ''
+ name = '_temp'
+ with util.create_modules(name) as mapping:
+ orig_module = types.ModuleType(name)
+ for attr in attributes:
+ setattr(orig_module, attr, value)
+ with open(mapping[name], 'w') as file:
+ file.write('+++ bad syntax +++')
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with self.assertRaises(SyntaxError):
+ loader.exec_module(orig_module)
+ for attr in attributes:
+ self.assertEqual(getattr(orig_module, attr), value)
+ with self.assertRaises(SyntaxError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module(name)
+ for attr in attributes:
+ self.assertEqual(getattr(orig_module, attr), value)
+
+ # [syntax error]
+ def test_bad_syntax(self):
+ with util.create_modules('_temp') as mapping:
+ with open(mapping['_temp'], 'w') as file:
+ file.write('=')
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with self.assertRaises(SyntaxError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module('_temp')
+ self.assertNotIn('_temp', sys.modules)
+
+ def test_file_from_empty_string_dir(self):
+ # Loading a module found from an empty string entry on sys.path should
+ # not only work, but keep all attributes relative.
+ file_path = '_temp.py'
+ with open(file_path, 'w') as file:
+ file.write("# test file for importlib")
+ try:
+ with util.uncache('_temp'):
+ loader = self.machinery.SourceFileLoader('_temp', file_path)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ mod = loader.load_module('_temp')
+ self.assertEqual(file_path, mod.__file__)
+ self.assertEqual(self.util.cache_from_source(file_path),
+ mod.__cached__)
+ finally:
+ os.unlink(file_path)
+ pycache = os.path.dirname(self.util.cache_from_source(file_path))
+ if os.path.exists(pycache):
+ shutil.rmtree(pycache)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ @util.writes_bytecode_files
+ def test_timestamp_overflow(self):
+ # When a modification timestamp is larger than 2**32, it should be
+ # truncated rather than raise an OverflowError.
+ with util.create_modules('_temp') as mapping:
+ source = mapping['_temp']
+ compiled = self.util.cache_from_source(source)
+ with open(source, 'w') as f:
+ f.write("x = 5")
+ try:
+ os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5))
+ except OverflowError:
+ self.skipTest("cannot set modification time to large integer")
+ except OSError as e:
+ if e.errno != getattr(errno, 'EOVERFLOW', None):
+ raise
+ self.skipTest("cannot set modification time to large integer ({})".format(e))
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ # PEP 451
+ module = types.ModuleType('_temp')
+ module.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(module)
+ self.assertEqual(module.x, 5)
+ self.assertTrue(os.path.exists(compiled))
+ os.unlink(compiled)
+ # PEP 302
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ mod = loader.load_module('_temp')
+ # Sanity checks.
+ self.assertEqual(mod.__cached__, compiled)
+ self.assertEqual(mod.x, 5)
+ # The pyc file was created.
+ self.assertTrue(os.path.exists(compiled))
+
+ def test_unloadable(self):
+ loader = self.machinery.SourceFileLoader('good name', {})
+ module = types.ModuleType('bad name')
+ module.__spec__ = self.machinery.ModuleSpec('bad name', loader)
+ with self.assertRaises(ImportError):
+ loader.exec_module(module)
+ with self.assertRaises(ImportError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module('bad name')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ @util.writes_bytecode_files
+ def test_checked_hash_based_pyc(self):
+ with util.create_modules('_temp') as mapping:
+ source = mapping['_temp']
+ pyc = self.util.cache_from_source(source)
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "old"')
+ os.utime(source, (50, 50))
+ py_compile.compile(
+ source,
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+ )
+ loader = self.machinery.SourceFileLoader('_temp', source)
+ mod = types.ModuleType('_temp')
+ mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+ # Write a new source with the same mtime and size as before.
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "new"')
+ os.utime(source, (50, 50))
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'new')
+ with open(pyc, 'rb') as fp:
+ data = fp.read()
+ self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
+ self.assertEqual(
+ self.util.source_hash(b'state = "new"'),
+ data[8:16],
+ )
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ @util.writes_bytecode_files
+ def test_overridden_checked_hash_based_pyc(self):
+ with util.create_modules('_temp') as mapping, \
+ unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
+ source = mapping['_temp']
+ pyc = self.util.cache_from_source(source)
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "old"')
+ os.utime(source, (50, 50))
+ py_compile.compile(
+ source,
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+ )
+ loader = self.machinery.SourceFileLoader('_temp', source)
+ mod = types.ModuleType('_temp')
+ mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+ # Write a new source with the same mtime and size as before.
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "new"')
+ os.utime(source, (50, 50))
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ @util.writes_bytecode_files
+ def test_unchecked_hash_based_pyc(self):
+ with util.create_modules('_temp') as mapping:
+ source = mapping['_temp']
+ pyc = self.util.cache_from_source(source)
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "old"')
+ os.utime(source, (50, 50))
+ py_compile.compile(
+ source,
+ invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+ )
+ loader = self.machinery.SourceFileLoader('_temp', source)
+ mod = types.ModuleType('_temp')
+ mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+ # Update the source file, which should be ignored.
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "new"')
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+ with open(pyc, 'rb') as fp:
+ data = fp.read()
+ self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1)
+ self.assertEqual(
+ self.util.source_hash(b'state = "old"'),
+ data[8:16],
+ )
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ @util.writes_bytecode_files
+ def test_overiden_unchecked_hash_based_pyc(self):
+ with util.create_modules('_temp') as mapping, \
+ unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
+ source = mapping['_temp']
+ pyc = self.util.cache_from_source(source)
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "old"')
+ os.utime(source, (50, 50))
+ py_compile.compile(
+ source,
+ invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+ )
+ loader = self.machinery.SourceFileLoader('_temp', source)
+ mod = types.ModuleType('_temp')
+ mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'old')
+ # Update the source file, which should be ignored.
+ with open(source, 'wb') as fp:
+ fp.write(b'state = "new"')
+ loader.exec_module(mod)
+ self.assertEqual(mod.state, 'new')
+ with open(pyc, 'rb') as fp:
+ data = fp.read()
+ self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1)
+ self.assertEqual(
+ self.util.source_hash(b'state = "new"'),
+ data[8:16],
+ )
+
+
+(Frozen_SimpleTest,
+ Source_SimpleTest
+ ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery,
+ abc=importlib_abc, util=importlib_util)
+
+
+class SourceDateEpochTestMeta(SourceDateEpochTestMeta,
+ type(Source_SimpleTest)):
+ pass
+
+
+class SourceDateEpoch_SimpleTest(Source_SimpleTest,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
+class BadBytecodeTest:
+
+ def import_(self, file, module_name):
+ raise NotImplementedError
+
+ def manipulate_bytecode(self,
+ name, mapping, manipulator, *,
+ del_source=False,
+ invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP):
+ """Manipulate the bytecode of a module by passing it into a callable
+ that returns what to use as the new bytecode."""
+ try:
+ del sys.modules['_temp']
+ except KeyError:
+ pass
+ py_compile.compile(mapping[name], invalidation_mode=invalidation_mode)
+ if not del_source:
+ bytecode_path = self.util.cache_from_source(mapping[name])
+ else:
+ os.unlink(mapping[name])
+ bytecode_path = make_legacy_pyc(mapping[name])
+ if manipulator:
+ with open(bytecode_path, 'rb') as file:
+ bc = file.read()
+ new_bc = manipulator(bc)
+ with open(bytecode_path, 'wb') as file:
+ if new_bc is not None:
+ file.write(new_bc)
+ return bytecode_path
+
+ def _test_empty_file(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: b'',
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ @util.writes_bytecode_files
+ def _test_partial_magic(self, test, *, del_source=False):
+ # When their are less than 4 bytes to a .pyc, regenerate it if
+ # possible, else raise ImportError.
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:3],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_magic_only(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:4],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_flags(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:7],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_hash(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode(
+ '_temp',
+ mapping,
+ lambda bc: bc[:13],
+ del_source=del_source,
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+ )
+ test('_temp', mapping, bc_path)
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode(
+ '_temp',
+ mapping,
+ lambda bc: bc[:13],
+ del_source=del_source,
+ invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+ )
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_timestamp(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:11],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_partial_size(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:15],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
+ def _test_no_marshal(self, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:16],
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bc_path
+ with self.assertRaises(EOFError):
+ self.import_(file_path, '_temp')
+
+ def _test_non_code_marshal(self, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:16] + marshal.dumps(b'abcd'),
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
+ with self.assertRaises(ImportError) as cm:
+ self.import_(file_path, '_temp')
+ self.assertEqual(cm.exception.name, '_temp')
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ def _test_bad_marshal(self, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bytecode_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:16] + b'',
+ del_source=del_source)
+ file_path = mapping['_temp'] if not del_source else bytecode_path
+ with self.assertRaises(EOFError):
+ self.import_(file_path, '_temp')
+
+ def _test_bad_magic(self, test, *, del_source=False):
+ with util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: b'\x00\x00\x00\x00' + bc[4:])
+ test('_temp', mapping, bc_path)
+
+
+class BadBytecodeTestPEP451(BadBytecodeTest):
+
+ def import_(self, file, module_name):
+ loader = self.loader(module_name, file)
+ module = types.ModuleType(module_name)
+ module.__spec__ = self.util.spec_from_loader(module_name, loader)
+ loader.exec_module(module)
+
+
+class BadBytecodeTestPEP302(BadBytecodeTest):
+
+ def import_(self, file, module_name):
+ loader = self.loader(module_name, file)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module(module_name)
+ self.assertIn(module_name, sys.modules)
+
+
+class SourceLoaderBadBytecodeTest:
+
+ @classmethod
+ def setUpClass(cls):
+ cls.loader = cls.machinery.SourceFileLoader
+
+ @util.writes_bytecode_files
+ def test_empty_file(self):
+ # When a .pyc is empty, regenerate it if possible, else raise
+ # ImportError.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_empty_file(test)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_partial_magic(test)
+
+ @util.writes_bytecode_files
+ def test_magic_only(self):
+ # When there is only the magic number, regenerate the .pyc if possible,
+ # else raise EOFError.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_magic_only(test)
+
+ @util.writes_bytecode_files
+ def test_bad_magic(self):
+ # When the magic number is different, the bytecode should be
+ # regenerated.
+ def test(name, mapping, bytecode_path):
+ self.import_(mapping[name], name)
+ with open(bytecode_path, 'rb') as bytecode_file:
+ self.assertEqual(bytecode_file.read(4),
+ self.util.MAGIC_NUMBER)
+
+ self._test_bad_magic(test)
+
+ @util.writes_bytecode_files
+ def test_partial_timestamp(self):
+ # When the timestamp is partial, regenerate the .pyc, else
+ # raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_partial_timestamp(test)
+
+ @util.writes_bytecode_files
+ def test_partial_flags(self):
+ # When the flags is partial, regenerate the .pyc, else raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_partial_flags(test)
+
+ @util.writes_bytecode_files
+ def test_partial_hash(self):
+ # When the hash is partial, regenerate the .pyc, else raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_partial_hash(test)
+
+ @util.writes_bytecode_files
+ def test_partial_size(self):
+ # When the size is partial, regenerate the .pyc, else
+ # raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 16)
+
+ self._test_partial_size(test)
+
+ @util.writes_bytecode_files
+ def test_no_marshal(self):
+ # When there is only the magic number and timestamp, raise EOFError.
+ self._test_no_marshal()
+
+ @util.writes_bytecode_files
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal()
+ # XXX ImportError when sourceless
+
+ # [bad marshal]
+ @util.writes_bytecode_files
+ def test_bad_marshal(self):
+ # Bad marshal data should raise a ValueError.
+ self._test_bad_marshal()
+
+ # [bad timestamp]
+ @util.writes_bytecode_files
+ @without_source_date_epoch
+ def test_old_timestamp(self):
+ # When the timestamp is older than the source, bytecode should be
+ # regenerated.
+ zeros = b'\x00\x00\x00\x00'
+ with util.create_modules('_temp') as mapping:
+ py_compile.compile(mapping['_temp'])
+ bytecode_path = self.util.cache_from_source(mapping['_temp'])
+ with open(bytecode_path, 'r+b') as bytecode_file:
+ bytecode_file.seek(8)
+ bytecode_file.write(zeros)
+ self.import_(mapping['_temp'], '_temp')
+ source_mtime = os.path.getmtime(mapping['_temp'])
+ source_timestamp = self.importlib._pack_uint32(source_mtime)
+ with open(bytecode_path, 'rb') as bytecode_file:
+ bytecode_file.seek(8)
+ self.assertEqual(bytecode_file.read(4), source_timestamp)
+
+ # [bytecode read-only]
+ @util.writes_bytecode_files
+ def test_read_only_bytecode(self):
+ # When bytecode is read-only but should be rewritten, fail silently.
+ with util.create_modules('_temp') as mapping:
+ # Create bytecode that will need to be re-created.
+ py_compile.compile(mapping['_temp'])
+ bytecode_path = self.util.cache_from_source(mapping['_temp'])
+ with open(bytecode_path, 'r+b') as bytecode_file:
+ bytecode_file.seek(0)
+ bytecode_file.write(b'\x00\x00\x00\x00')
+ # Make the bytecode read-only.
+ os.chmod(bytecode_path,
+ stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+ try:
+ # Should not raise OSError!
+ self.import_(mapping['_temp'], '_temp')
+ finally:
+ # Make writable for eventual clean-up.
+ os.chmod(bytecode_path, stat.S_IWUSR)
+
+
+# TODO: RustPython
+# class SourceLoaderBadBytecodeTestPEP451(
+# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451):
+# pass
+
+
+# (Frozen_SourceBadBytecodePEP451,
+# Source_SourceBadBytecodePEP451
+# ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib,
+# machinery=machinery, abc=importlib_abc,
+# util=importlib_util)
+
+
+# class SourceLoaderBadBytecodeTestPEP302(
+# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302):
+# pass
+
+
+# (Frozen_SourceBadBytecodePEP302,
+# Source_SourceBadBytecodePEP302
+# ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib,
+# machinery=machinery, abc=importlib_abc,
+# util=importlib_util)
+
+
+class SourcelessLoaderBadBytecodeTest:
+
+ @classmethod
+ def setUpClass(cls):
+ cls.loader = cls.machinery.SourcelessFileLoader
+
+ def test_empty_file(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ self._test_empty_file(test, del_source=True)
+
+ def test_partial_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+ self._test_partial_magic(test, del_source=True)
+
+ def test_magic_only(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_magic_only(test, del_source=True)
+
+ def test_bad_magic(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(ImportError) as cm:
+ self.import_(bytecode_path, name)
+ self.assertEqual(cm.exception.name, name)
+ self.assertEqual(cm.exception.path, bytecode_path)
+
+ self._test_bad_magic(test, del_source=True)
+
+ def test_partial_timestamp(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_timestamp(test, del_source=True)
+
+ def test_partial_flags(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_flags(test, del_source=True)
+
+ def test_partial_hash(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_hash(test, del_source=True)
+
+ def test_partial_size(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_size(test, del_source=True)
+
+ def test_no_marshal(self):
+ self._test_no_marshal(del_source=True)
+
+ def test_non_code_marshal(self):
+ self._test_non_code_marshal(del_source=True)
+
+
+# TODO: RustPython
+# class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest,
+# BadBytecodeTestPEP451):
+# pass
+
+
+# (Frozen_SourcelessBadBytecodePEP451,
+# Source_SourcelessBadBytecodePEP451
+# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib,
+# machinery=machinery, abc=importlib_abc,
+# util=importlib_util)
+
+
+# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
+# BadBytecodeTestPEP302):
+# pass
+
+
+# (Frozen_SourcelessBadBytecodePEP302,
+# Source_SourcelessBadBytecodePEP302
+# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib,
+# machinery=machinery, abc=importlib_abc,
+# util=importlib_util)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py
new file mode 100644
index 000000000..1a76fc02d
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_finder.py
@@ -0,0 +1,237 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import errno
+import os
+import py_compile
+import stat
+import sys
+import tempfile
+from test.support import make_legacy_pyc
+import unittest
+import warnings
+
+
+class FinderTests(abc.FinderTests):
+
+ """For a top-level module, it should just be found directly in the
+ directory being searched. This is true for a directory with source
+ [top-level source], bytecode [top-level bc], or both [top-level both].
+ There is also the possibility that it is a package [top-level package], in
+ which case there will be a directory with the module name and an
+ __init__.py file. If there is a directory without an __init__.py an
+ ImportWarning is returned [empty dir].
+
+ For sub-modules and sub-packages, the same happens as above but only use
+ the tail end of the name [sub module] [sub package] [sub empty].
+
+ When there is a conflict between a package and module having the same name
+ in the same directory, the package wins out [package over module]. This is
+ so that imports of modules within the package can occur rather than trigger
+ an import error.
+
+ When there is a package and module with the same name, always pick the
+ package over the module [package over module]. This is so that imports from
+ the package have the possibility of succeeding.
+
+ """
+
+ def get_finder(self, root):
+ loader_details = [(self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES),
+ (self.machinery.SourcelessFileLoader,
+ self.machinery.BYTECODE_SUFFIXES)]
+ return self.machinery.FileFinder(root, *loader_details)
+
+ def import_(self, root, module):
+ finder = self.get_finder(root)
+ return self._find(finder, module, loader_only=True)
+
+ def run_test(self, test, create=None, *, compile_=None, unlink=None):
+ """Test the finding of 'test' with the creation of modules listed in
+ 'create'.
+
+ Any names listed in 'compile_' are byte-compiled. Modules
+ listed in 'unlink' have their source files deleted.
+
+ """
+ if create is None:
+ create = {test}
+ with util.create_modules(*create) as mapping:
+ if compile_:
+ for name in compile_:
+ py_compile.compile(mapping[name])
+ if unlink:
+ for name in unlink:
+ os.unlink(mapping[name])
+ try:
+ make_legacy_pyc(mapping[name])
+ except OSError as error:
+ # Some tests do not set compile_=True so the source
+ # module will not get compiled and there will be no
+ # PEP 3147 pyc file to rename.
+ if error.errno != errno.ENOENT:
+ raise
+ loader = self.import_(mapping['.root'], test)
+ self.assertTrue(hasattr(loader, 'load_module'))
+ return loader
+
+ def test_module(self):
+ # [top-level source]
+ self.run_test('top_level')
+ # [top-level bc]
+ self.run_test('top_level', compile_={'top_level'},
+ unlink={'top_level'})
+ # [top-level both]
+ self.run_test('top_level', compile_={'top_level'})
+
+ # [top-level package]
+ def test_package(self):
+ # Source.
+ self.run_test('pkg', {'pkg.__init__'})
+ # Bytecode.
+ self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'},
+ unlink={'pkg.__init__'})
+ # Both.
+ self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'})
+
+ # [sub module]
+ def test_module_in_package(self):
+ with util.create_modules('pkg.__init__', 'pkg.sub') as mapping:
+ pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+ loader = self.import_(pkg_dir, 'pkg.sub')
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # [sub package]
+ def test_package_in_package(self):
+ context = util.create_modules('pkg.__init__', 'pkg.sub.__init__')
+ with context as mapping:
+ pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+ loader = self.import_(pkg_dir, 'pkg.sub')
+ self.assertTrue(hasattr(loader, 'load_module'))
+
+ # [package over modules]
+ def test_package_over_module(self):
+ name = '_temp'
+ loader = self.run_test(name, {'{0}.__init__'.format(name), name})
+ self.assertIn('__init__', loader.get_filename(name))
+
+ def test_failure(self):
+ with util.create_modules('blah') as mapping:
+ nothing = self.import_(mapping['.root'], 'sdfsadsadf')
+ self.assertIsNone(nothing)
+
+ def test_empty_string_for_dir(self):
+ # The empty string from sys.path means to search in the cwd.
+ finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
+ with open('mod.py', 'w') as file:
+ file.write("# test file for importlib")
+ try:
+ loader = self._find(finder, 'mod', loader_only=True)
+ self.assertTrue(hasattr(loader, 'load_module'))
+ finally:
+ os.unlink('mod.py')
+
+ def test_invalidate_caches(self):
+ # invalidate_caches() should reset the mtime.
+ finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
+ finder._path_mtime = 42
+ finder.invalidate_caches()
+ self.assertEqual(finder._path_mtime, -1)
+
+ # Regression test for http://bugs.python.org/issue14846
+ def test_dir_removal_handling(self):
+ mod = 'mod'
+ with util.create_modules(mod) as mapping:
+ finder = self.get_finder(mapping['.root'])
+ found = self._find(finder, 'mod', loader_only=True)
+ self.assertIsNotNone(found)
+ found = self._find(finder, 'mod', loader_only=True)
+ self.assertIsNone(found)
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ 'os.chmod() does not support the needed arguments under Windows')
+ def test_no_read_directory(self):
+ # Issue #16730
+ tempdir = tempfile.TemporaryDirectory()
+ original_mode = os.stat(tempdir.name).st_mode
+ def cleanup(tempdir):
+ """Cleanup function for the temporary directory.
+
+ Since we muck with the permissions, we want to set them back to
+ their original values to make sure the directory can be properly
+ cleaned up.
+
+ """
+ os.chmod(tempdir.name, original_mode)
+ # If this is not explicitly called then the __del__ method is used,
+ # but since already mucking around might as well explicitly clean
+ # up.
+ tempdir.__exit__(None, None, None)
+ self.addCleanup(cleanup, tempdir)
+ os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR)
+ finder = self.get_finder(tempdir.name)
+ found = self._find(finder, 'doesnotexist')
+ self.assertEqual(found, self.NOT_FOUND)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_ignore_file(self):
+ # If a directory got changed to a file from underneath us, then don't
+ # worry about looking for submodules.
+ with tempfile.NamedTemporaryFile() as file_obj:
+ finder = self.get_finder(file_obj.name)
+ found = self._find(finder, 'doesnotexist')
+ self.assertEqual(found, self.NOT_FOUND)
+
+
+class FinderTestsPEP451(FinderTests):
+
+ NOT_FOUND = None
+
+ def _find(self, finder, name, loader_only=False):
+ spec = finder.find_spec(name)
+ return spec.loader if spec is not None else spec
+
+
+(Frozen_FinderTestsPEP451,
+ Source_FinderTestsPEP451
+ ) = util.test_both(FinderTestsPEP451, machinery=machinery)
+
+
+class FinderTestsPEP420(FinderTests):
+
+ NOT_FOUND = (None, [])
+
+ def _find(self, finder, name, loader_only=False):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ loader_portions = finder.find_loader(name)
+ return loader_portions[0] if loader_only else loader_portions
+
+
+(Frozen_FinderTestsPEP420,
+ Source_FinderTestsPEP420
+ ) = util.test_both(FinderTestsPEP420, machinery=machinery)
+
+
+class FinderTestsPEP302(FinderTests):
+
+ NOT_FOUND = None
+
+ def _find(self, finder, name, loader_only=False):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ return finder.find_module(name)
+
+
+(Frozen_FinderTestsPEP302,
+ Source_FinderTestsPEP302
+ ) = util.test_both(FinderTestsPEP302, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py
new file mode 100644
index 000000000..795d436c3
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_path_hook.py
@@ -0,0 +1,41 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class PathHookTest:
+
+ """Test the path hook for source."""
+
+ def path_hook(self):
+ return self.machinery.FileFinder.path_hook((self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
+
+ def test_success(self):
+ with util.create_modules('dummy') as mapping:
+ self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
+ 'find_spec'))
+
+ def test_success_legacy(self):
+ with util.create_modules('dummy') as mapping:
+ self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
+ 'find_module'))
+
+ def test_empty_string(self):
+ # The empty string represents the cwd.
+ self.assertTrue(hasattr(self.path_hook()(''), 'find_spec'))
+
+ def test_empty_string_legacy(self):
+ # The empty string represents the cwd.
+ self.assertTrue(hasattr(self.path_hook()(''), 'find_module'))
+
+
+(Frozen_PathHookTest,
+ Source_PathHooktest
+ ) = util.test_both(PathHookTest, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py
new file mode 100644
index 000000000..f2cfa3c98
--- /dev/null
+++ b/Lib/test/test_importlib/source/test_source_encoding.py
@@ -0,0 +1,181 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import codecs
+import importlib.util
+import re
+import types
+# Because sys.path gets essentially blanked, need to have unicodedata already
+# imported for the parser to use.
+import unicodedata
+import unittest
+import warnings
+
+
+CODING_RE = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
+
+
+class EncodingTest:
+
+ """PEP 3120 makes UTF-8 the default encoding for source code
+ [default encoding].
+
+ PEP 263 specifies how that can change on a per-file basis. Either the first
+ or second line can contain the encoding line [encoding first line]
+ encoding second line]. If the file has the BOM marker it is considered UTF-8
+ implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is
+ an error [BOM and utf-8][BOM conflict].
+
+ """
+
+ variable = '\u00fc'
+ character = '\u00c9'
+ source_line = "{0} = '{1}'\n".format(variable, character)
+ module_name = '_temp'
+
+ def run_test(self, source):
+ with util.create_modules(self.module_name) as mapping:
+ with open(mapping[self.module_name], 'wb') as file:
+ file.write(source)
+ loader = self.machinery.SourceFileLoader(self.module_name,
+ mapping[self.module_name])
+ return self.load(loader)
+
+ def create_source(self, encoding):
+ encoding_line = "# coding={0}".format(encoding)
+ assert CODING_RE.match(encoding_line)
+ source_lines = [encoding_line.encode('utf-8')]
+ source_lines.append(self.source_line.encode(encoding))
+ return b'\n'.join(source_lines)
+
+ def test_non_obvious_encoding(self):
+ # Make sure that an encoding that has never been a standard one for
+ # Python works.
+ encoding_line = "# coding=koi8-r"
+ assert CODING_RE.match(encoding_line)
+ source = "{0}\na=42\n".format(encoding_line).encode("koi8-r")
+ self.run_test(source)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ # [default encoding]
+ def test_default_encoding(self):
+ self.run_test(self.source_line.encode('utf-8'))
+
+ # [encoding first line]
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_encoding_on_first_line(self):
+ encoding = 'Latin-1'
+ source = self.create_source(encoding)
+ self.run_test(source)
+
+ # [encoding second line]
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_encoding_on_second_line(self):
+ source = b"#/usr/bin/python\n" + self.create_source('Latin-1')
+ self.run_test(source)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ # [BOM]
+ def test_bom(self):
+ self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8'))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ # [BOM and utf-8]
+ def test_bom_and_utf_8(self):
+ source = codecs.BOM_UTF8 + self.create_source('utf-8')
+ self.run_test(source)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ # [BOM conflict]
+ def test_bom_conflict(self):
+ source = codecs.BOM_UTF8 + self.create_source('latin-1')
+ with self.assertRaises(SyntaxError):
+ self.run_test(source)
+
+
+class EncodingTestPEP451(EncodingTest):
+
+ def load(self, loader):
+ module = types.ModuleType(self.module_name)
+ module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader)
+ loader.exec_module(module)
+ return module
+
+
+(Frozen_EncodingTestPEP451,
+ Source_EncodingTestPEP451
+ ) = util.test_both(EncodingTestPEP451, machinery=machinery)
+
+
+class EncodingTestPEP302(EncodingTest):
+
+ def load(self, loader):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return loader.load_module(self.module_name)
+
+
+(Frozen_EncodingTestPEP302,
+ Source_EncodingTestPEP302
+ ) = util.test_both(EncodingTestPEP302, machinery=machinery)
+
+
+class LineEndingTest:
+
+ r"""Source written with the three types of line endings (\n, \r\n, \r)
+ need to be readable [cr][crlf][lf]."""
+
+ def run_test(self, line_ending):
+ module_name = '_temp'
+ source_lines = [b"a = 42", b"b = -13", b'']
+ source = line_ending.join(source_lines)
+ with util.create_modules(module_name) as mapping:
+ with open(mapping[module_name], 'wb') as file:
+ file.write(source)
+ loader = self.machinery.SourceFileLoader(module_name,
+ mapping[module_name])
+ return self.load(loader, module_name)
+
+ # [cr]
+ def test_cr(self):
+ self.run_test(b'\r')
+
+ # [crlf]
+ def test_crlf(self):
+ self.run_test(b'\r\n')
+
+ # [lf]
+ def test_lf(self):
+ self.run_test(b'\n')
+
+
+class LineEndingTestPEP451(LineEndingTest):
+
+ def load(self, loader, module_name):
+ module = types.ModuleType(module_name)
+ module.__spec__ = importlib.util.spec_from_loader(module_name, loader)
+ loader.exec_module(module)
+ return module
+
+
+(Frozen_LineEndingTestPEP451,
+ Source_LineEndingTestPEP451
+ ) = util.test_both(LineEndingTestPEP451, machinery=machinery)
+
+
+class LineEndingTestPEP302(LineEndingTest):
+
+ def load(self, loader, module_name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return loader.load_module(module_name)
+
+
+(Frozen_LineEndingTestPEP302,
+ Source_LineEndingTestPEP302
+ ) = util.test_both(LineEndingTestPEP302, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py
new file mode 100644
index 000000000..11250ab06
--- /dev/null
+++ b/Lib/test/test_importlib/test_abc.py
@@ -0,0 +1,1004 @@
+import io
+import marshal
+import os
+import sys
+from test import support
+import types
+import unittest
+from unittest import mock
+import warnings
+
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+abc = test_util.import_importlib('importlib.abc')
+machinery = test_util.import_importlib('importlib.machinery')
+util = test_util.import_importlib('importlib.util')
+
+
+##### Inheritance ##############################################################
+class InheritanceTests:
+
+ """Test that the specified class is a subclass/superclass of the expected
+ classes."""
+
+ subclasses = []
+ superclasses = []
+
+ def setUp(self):
+ self.superclasses = [getattr(self.abc, class_name)
+ for class_name in self.superclass_names]
+ if hasattr(self, 'subclass_names'):
+ # Because test.support.import_fresh_module() creates a new
+ # importlib._bootstrap per module, inheritance checks fail when
+ # checking across module boundaries (i.e. the _bootstrap in abc is
+ # not the same as the one in machinery). That means stealing one of
+ # the modules from the other to make sure the same instance is used.
+ machinery = self.abc.machinery
+ self.subclasses = [getattr(machinery, class_name)
+ for class_name in self.subclass_names]
+ assert self.subclasses or self.superclasses, self.__class__
+ self.__test = getattr(self.abc, self._NAME)
+
+ def test_subclasses(self):
+ # Test that the expected subclasses inherit.
+ for subclass in self.subclasses:
+ self.assertTrue(issubclass(subclass, self.__test),
+ "{0} is not a subclass of {1}".format(subclass, self.__test))
+
+ def test_superclasses(self):
+ # Test that the class inherits from the expected superclasses.
+ for superclass in self.superclasses:
+ self.assertTrue(issubclass(self.__test, superclass),
+ "{0} is not a superclass of {1}".format(superclass, self.__test))
+
+
+class MetaPathFinder(InheritanceTests):
+ superclass_names = ['Finder']
+ subclass_names = ['BuiltinImporter', 'FrozenImporter', 'PathFinder',
+ 'WindowsRegistryFinder']
+
+
+(Frozen_MetaPathFinderInheritanceTests,
+ Source_MetaPathFinderInheritanceTests
+ ) = test_util.test_both(MetaPathFinder, abc=abc)
+
+
+class PathEntryFinder(InheritanceTests):
+ superclass_names = ['Finder']
+ subclass_names = ['FileFinder']
+
+
+(Frozen_PathEntryFinderInheritanceTests,
+ Source_PathEntryFinderInheritanceTests
+ ) = test_util.test_both(PathEntryFinder, abc=abc)
+
+
+class ResourceLoader(InheritanceTests):
+ superclass_names = ['Loader']
+
+
+(Frozen_ResourceLoaderInheritanceTests,
+ Source_ResourceLoaderInheritanceTests
+ ) = test_util.test_both(ResourceLoader, abc=abc)
+
+
+class InspectLoader(InheritanceTests):
+ superclass_names = ['Loader']
+ subclass_names = ['BuiltinImporter', 'FrozenImporter', 'ExtensionFileLoader']
+
+
+(Frozen_InspectLoaderInheritanceTests,
+ Source_InspectLoaderInheritanceTests
+ ) = test_util.test_both(InspectLoader, abc=abc)
+
+
+class ExecutionLoader(InheritanceTests):
+ superclass_names = ['InspectLoader']
+ subclass_names = ['ExtensionFileLoader']
+
+
+(Frozen_ExecutionLoaderInheritanceTests,
+ Source_ExecutionLoaderInheritanceTests
+ ) = test_util.test_both(ExecutionLoader, abc=abc)
+
+
+class FileLoader(InheritanceTests):
+ superclass_names = ['ResourceLoader', 'ExecutionLoader']
+ subclass_names = ['SourceFileLoader', 'SourcelessFileLoader']
+
+
+(Frozen_FileLoaderInheritanceTests,
+ Source_FileLoaderInheritanceTests
+ ) = test_util.test_both(FileLoader, abc=abc)
+
+
+class SourceLoader(InheritanceTests):
+ superclass_names = ['ResourceLoader', 'ExecutionLoader']
+ subclass_names = ['SourceFileLoader']
+
+
+(Frozen_SourceLoaderInheritanceTests,
+ Source_SourceLoaderInheritanceTests
+ ) = test_util.test_both(SourceLoader, abc=abc)
+
+
+##### Default return values ####################################################
+
+def make_abc_subclasses(base_class, name=None, inst=False, **kwargs):
+ if name is None:
+ name = base_class.__name__
+ base = {kind: getattr(splitabc, name)
+ for kind, splitabc in abc.items()}
+ return {cls._KIND: cls() if inst else cls
+ for cls in test_util.split_frozen(base_class, base, **kwargs)}
+
+
+class ABCTestHarness:
+
+ @property
+ def ins(self):
+ # Lazily set ins on the class.
+ cls = self.SPLIT[self._KIND]
+ ins = cls()
+ self.__class__.ins = ins
+ return ins
+
+
+class MetaPathFinder:
+
+ def find_module(self, fullname, path):
+ return super().find_module(fullname, path)
+
+
+class MetaPathFinderDefaultsTests(ABCTestHarness):
+
+ SPLIT = make_abc_subclasses(MetaPathFinder)
+
+ def test_find_module(self):
+ # Default should return None.
+ with self.assertWarns(DeprecationWarning):
+ found = self.ins.find_module('something', None)
+ self.assertIsNone(found)
+
+ def test_invalidate_caches(self):
+ # Calling the method is a no-op.
+ self.ins.invalidate_caches()
+
+
+(Frozen_MPFDefaultTests,
+ Source_MPFDefaultTests
+ ) = test_util.test_both(MetaPathFinderDefaultsTests)
+
+
+class PathEntryFinder:
+
+ def find_loader(self, fullname):
+ return super().find_loader(fullname)
+
+
+class PathEntryFinderDefaultsTests(ABCTestHarness):
+
+ SPLIT = make_abc_subclasses(PathEntryFinder)
+
+ def test_find_loader(self):
+ with self.assertWarns(DeprecationWarning):
+ found = self.ins.find_loader('something')
+ self.assertEqual(found, (None, []))
+
+ def find_module(self):
+ self.assertEqual(None, self.ins.find_module('something'))
+
+ def test_invalidate_caches(self):
+ # Should be a no-op.
+ self.ins.invalidate_caches()
+
+
+(Frozen_PEFDefaultTests,
+ Source_PEFDefaultTests
+ ) = test_util.test_both(PathEntryFinderDefaultsTests)
+
+
+class Loader:
+
+ def load_module(self, fullname):
+ return super().load_module(fullname)
+
+
+class LoaderDefaultsTests(ABCTestHarness):
+
+ SPLIT = make_abc_subclasses(Loader)
+
+ def test_create_module(self):
+ spec = 'a spec'
+ self.assertIsNone(self.ins.create_module(spec))
+
+ def test_load_module(self):
+ with self.assertRaises(ImportError):
+ self.ins.load_module('something')
+
+ def test_module_repr(self):
+ mod = types.ModuleType('blah')
+ with self.assertRaises(NotImplementedError):
+ self.ins.module_repr(mod)
+ original_repr = repr(mod)
+ mod.__loader__ = self.ins
+ # Should still return a proper repr.
+ self.assertTrue(repr(mod))
+
+
+(Frozen_LDefaultTests,
+ SourceLDefaultTests
+ ) = test_util.test_both(LoaderDefaultsTests)
+
+
+class ResourceLoader(Loader):
+
+ def get_data(self, path):
+ return super().get_data(path)
+
+
+class ResourceLoaderDefaultsTests(ABCTestHarness):
+
+ SPLIT = make_abc_subclasses(ResourceLoader)
+
+ def test_get_data(self):
+ with self.assertRaises(IOError):
+ self.ins.get_data('/some/path')
+
+
+(Frozen_RLDefaultTests,
+ Source_RLDefaultTests
+ ) = test_util.test_both(ResourceLoaderDefaultsTests)
+
+
+class InspectLoader(Loader):
+
+ def is_package(self, fullname):
+ return super().is_package(fullname)
+
+ def get_source(self, fullname):
+ return super().get_source(fullname)
+
+
+SPLIT_IL = make_abc_subclasses(InspectLoader)
+
+
+class InspectLoaderDefaultsTests(ABCTestHarness):
+
+ SPLIT = SPLIT_IL
+
+ def test_is_package(self):
+ with self.assertRaises(ImportError):
+ self.ins.is_package('blah')
+
+ def test_get_source(self):
+ with self.assertRaises(ImportError):
+ self.ins.get_source('blah')
+
+
+(Frozen_ILDefaultTests,
+ Source_ILDefaultTests
+ ) = test_util.test_both(InspectLoaderDefaultsTests)
+
+
+class ExecutionLoader(InspectLoader):
+
+ def get_filename(self, fullname):
+ return super().get_filename(fullname)
+
+
+SPLIT_EL = make_abc_subclasses(ExecutionLoader)
+
+
+class ExecutionLoaderDefaultsTests(ABCTestHarness):
+
+ SPLIT = SPLIT_EL
+
+ def test_get_filename(self):
+ with self.assertRaises(ImportError):
+ self.ins.get_filename('blah')
+
+
+(Frozen_ELDefaultTests,
+ Source_ELDefaultsTests
+ ) = test_util.test_both(InspectLoaderDefaultsTests)
+
+
+class ResourceReader:
+
+ def open_resource(self, *args, **kwargs):
+ return super().open_resource(*args, **kwargs)
+
+ def resource_path(self, *args, **kwargs):
+ return super().resource_path(*args, **kwargs)
+
+ def is_resource(self, *args, **kwargs):
+ return super().is_resource(*args, **kwargs)
+
+ def contents(self, *args, **kwargs):
+ return super().contents(*args, **kwargs)
+
+
+class ResourceReaderDefaultsTests(ABCTestHarness):
+
+ SPLIT = make_abc_subclasses(ResourceReader)
+
+ def test_open_resource(self):
+ with self.assertRaises(FileNotFoundError):
+ self.ins.open_resource('dummy_file')
+
+ def test_resource_path(self):
+ with self.assertRaises(FileNotFoundError):
+ self.ins.resource_path('dummy_file')
+
+ def test_is_resource(self):
+ with self.assertRaises(FileNotFoundError):
+ self.ins.is_resource('dummy_file')
+
+ def test_contents(self):
+ self.assertEqual([], list(self.ins.contents()))
+
+(Frozen_RRDefaultTests,
+ Source_RRDefaultsTests
+ ) = test_util.test_both(ResourceReaderDefaultsTests)
+
+
+##### MetaPathFinder concrete methods ##########################################
+class MetaPathFinderFindModuleTests:
+
+ @classmethod
+ def finder(cls, spec):
+ class MetaPathSpecFinder(cls.abc.MetaPathFinder):
+
+ def find_spec(self, fullname, path, target=None):
+ self.called_for = fullname, path
+ return spec
+
+ return MetaPathSpecFinder()
+
+ def test_no_spec(self):
+ finder = self.finder(None)
+ path = ['a', 'b', 'c']
+ name = 'blah'
+ with self.assertWarns(DeprecationWarning):
+ found = finder.find_module(name, path)
+ self.assertIsNone(found)
+ self.assertEqual(name, finder.called_for[0])
+ self.assertEqual(path, finder.called_for[1])
+
+ def test_spec(self):
+ loader = object()
+ spec = self.util.spec_from_loader('blah', loader)
+ finder = self.finder(spec)
+ with self.assertWarns(DeprecationWarning):
+ found = finder.find_module('blah', None)
+ self.assertIs(found, spec.loader)
+
+
+(Frozen_MPFFindModuleTests,
+ Source_MPFFindModuleTests
+ ) = test_util.test_both(MetaPathFinderFindModuleTests, abc=abc, util=util)
+
+
+##### PathEntryFinder concrete methods #########################################
+class PathEntryFinderFindLoaderTests:
+
+ @classmethod
+ def finder(cls, spec):
+ class PathEntrySpecFinder(cls.abc.PathEntryFinder):
+
+ def find_spec(self, fullname, target=None):
+ self.called_for = fullname
+ return spec
+
+ return PathEntrySpecFinder()
+
+ def test_no_spec(self):
+ finder = self.finder(None)
+ name = 'blah'
+ with self.assertWarns(DeprecationWarning):
+ found = finder.find_loader(name)
+ self.assertIsNone(found[0])
+ self.assertEqual([], found[1])
+ self.assertEqual(name, finder.called_for)
+
+ def test_spec_with_loader(self):
+ loader = object()
+ spec = self.util.spec_from_loader('blah', loader)
+ finder = self.finder(spec)
+ with self.assertWarns(DeprecationWarning):
+ found = finder.find_loader('blah')
+ self.assertIs(found[0], spec.loader)
+
+ def test_spec_with_portions(self):
+ spec = self.machinery.ModuleSpec('blah', None)
+ paths = ['a', 'b', 'c']
+ spec.submodule_search_locations = paths
+ finder = self.finder(spec)
+ with self.assertWarns(DeprecationWarning):
+ found = finder.find_loader('blah')
+ self.assertIsNone(found[0])
+ self.assertEqual(paths, found[1])
+
+
+(Frozen_PEFFindLoaderTests,
+ Source_PEFFindLoaderTests
+ ) = test_util.test_both(PathEntryFinderFindLoaderTests, abc=abc, util=util,
+ machinery=machinery)
+
+
+##### Loader concrete methods ##################################################
+class LoaderLoadModuleTests:
+
+ def loader(self):
+ class SpecLoader(self.abc.Loader):
+ found = None
+ def exec_module(self, module):
+ self.found = module
+
+ def is_package(self, fullname):
+ """Force some non-default module state to be set."""
+ return True
+
+ return SpecLoader()
+
+ def test_fresh(self):
+ loader = self.loader()
+ name = 'blah'
+ with test_util.uncache(name):
+ loader.load_module(name)
+ module = loader.found
+ self.assertIs(sys.modules[name], module)
+ self.assertEqual(loader, module.__loader__)
+ self.assertEqual(loader, module.__spec__.loader)
+ self.assertEqual(name, module.__name__)
+ self.assertEqual(name, module.__spec__.name)
+ self.assertIsNotNone(module.__path__)
+ self.assertIsNotNone(module.__path__,
+ module.__spec__.submodule_search_locations)
+
+ def test_reload(self):
+ name = 'blah'
+ loader = self.loader()
+ module = types.ModuleType(name)
+ module.__spec__ = self.util.spec_from_loader(name, loader)
+ module.__loader__ = loader
+ with test_util.uncache(name):
+ sys.modules[name] = module
+ loader.load_module(name)
+ found = loader.found
+ self.assertIs(found, sys.modules[name])
+ self.assertIs(module, sys.modules[name])
+
+
+(Frozen_LoaderLoadModuleTests,
+ Source_LoaderLoadModuleTests
+ ) = test_util.test_both(LoaderLoadModuleTests, abc=abc, util=util)
+
+
+##### InspectLoader concrete methods ###########################################
+class InspectLoaderSourceToCodeTests:
+
+ def source_to_module(self, data, path=None):
+ """Help with source_to_code() tests."""
+ module = types.ModuleType('blah')
+ loader = self.InspectLoaderSubclass()
+ if path is None:
+ code = loader.source_to_code(data)
+ else:
+ code = loader.source_to_code(data, path)
+ exec(code, module.__dict__)
+ return module
+
+ def test_source_to_code_source(self):
+ # Since compile() can handle strings, so should source_to_code().
+ source = 'attr = 42'
+ module = self.source_to_module(source)
+ self.assertTrue(hasattr(module, 'attr'))
+ self.assertEqual(module.attr, 42)
+
+ def test_source_to_code_bytes(self):
+ # Since compile() can handle bytes, so should source_to_code().
+ source = b'attr = 42'
+ module = self.source_to_module(source)
+ self.assertTrue(hasattr(module, 'attr'))
+ self.assertEqual(module.attr, 42)
+
+ def test_source_to_code_path(self):
+ # Specifying a path should set it for the code object.
+ path = 'path/to/somewhere'
+ loader = self.InspectLoaderSubclass()
+ code = loader.source_to_code('', path)
+ self.assertEqual(code.co_filename, path)
+
+ def test_source_to_code_no_path(self):
+ # Not setting a path should still work and be set to since that
+ # is a pre-existing practice as a default to compile().
+ loader = self.InspectLoaderSubclass()
+ code = loader.source_to_code('')
+ self.assertEqual(code.co_filename, '')
+
+
+(Frozen_ILSourceToCodeTests,
+ Source_ILSourceToCodeTests
+ ) = test_util.test_both(InspectLoaderSourceToCodeTests,
+ InspectLoaderSubclass=SPLIT_IL)
+
+
+class InspectLoaderGetCodeTests:
+
+ def test_get_code(self):
+ # Test success.
+ module = types.ModuleType('blah')
+ with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked:
+ mocked.return_value = 'attr = 42'
+ loader = self.InspectLoaderSubclass()
+ code = loader.get_code('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+ def test_get_code_source_is_None(self):
+ # If get_source() is None then this should be None.
+ with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked:
+ mocked.return_value = None
+ loader = self.InspectLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertIsNone(code)
+
+ def test_get_code_source_not_found(self):
+ # If there is no source then there is no code object.
+ loader = self.InspectLoaderSubclass()
+ with self.assertRaises(ImportError):
+ loader.get_code('blah')
+
+
+(Frozen_ILGetCodeTests,
+ Source_ILGetCodeTests
+ ) = test_util.test_both(InspectLoaderGetCodeTests,
+ InspectLoaderSubclass=SPLIT_IL)
+
+
+class InspectLoaderLoadModuleTests:
+
+ """Test InspectLoader.load_module()."""
+
+ module_name = 'blah'
+
+ def setUp(self):
+ support.unload(self.module_name)
+ self.addCleanup(support.unload, self.module_name)
+
+ def load(self, loader):
+ spec = self.util.spec_from_loader(self.module_name, loader)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return self.init._bootstrap._load_unlocked(spec)
+
+ def mock_get_code(self):
+ return mock.patch.object(self.InspectLoaderSubclass, 'get_code')
+
+ def test_get_code_ImportError(self):
+ # If get_code() raises ImportError, it should propagate.
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.side_effect = ImportError
+ with self.assertRaises(ImportError):
+ loader = self.InspectLoaderSubclass()
+ self.load(loader)
+
+ def test_get_code_None(self):
+ # If get_code() returns None, raise ImportError.
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.return_value = None
+ with self.assertRaises(ImportError):
+ loader = self.InspectLoaderSubclass()
+ self.load(loader)
+
+ def test_module_returned(self):
+ # The loaded module should be returned.
+ code = compile('attr = 42', '', 'exec')
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.return_value = code
+ loader = self.InspectLoaderSubclass()
+ module = self.load(loader)
+ self.assertEqual(module, sys.modules[self.module_name])
+
+
+(Frozen_ILLoadModuleTests,
+ Source_ILLoadModuleTests
+ ) = test_util.test_both(InspectLoaderLoadModuleTests,
+ InspectLoaderSubclass=SPLIT_IL,
+ init=init,
+ util=util)
+
+
+##### ExecutionLoader concrete methods #########################################
+class ExecutionLoaderGetCodeTests:
+
+ def mock_methods(self, *, get_source=False, get_filename=False):
+ source_mock_context, filename_mock_context = None, None
+ if get_source:
+ source_mock_context = mock.patch.object(self.ExecutionLoaderSubclass,
+ 'get_source')
+ if get_filename:
+ filename_mock_context = mock.patch.object(self.ExecutionLoaderSubclass,
+ 'get_filename')
+ return source_mock_context, filename_mock_context
+
+ def test_get_code(self):
+ path = 'blah.py'
+ source_mock_context, filename_mock_context = self.mock_methods(
+ get_source=True, get_filename=True)
+ with source_mock_context as source_mock, filename_mock_context as name_mock:
+ source_mock.return_value = 'attr = 42'
+ name_mock.return_value = path
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertEqual(code.co_filename, path)
+ module = types.ModuleType('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+ def test_get_code_source_is_None(self):
+ # If get_source() is None then this should be None.
+ source_mock_context, _ = self.mock_methods(get_source=True)
+ with source_mock_context as mocked:
+ mocked.return_value = None
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertIsNone(code)
+
+ def test_get_code_source_not_found(self):
+ # If there is no source then there is no code object.
+ loader = self.ExecutionLoaderSubclass()
+ with self.assertRaises(ImportError):
+ loader.get_code('blah')
+
+ def test_get_code_no_path(self):
+ # If get_filename() raises ImportError then simply skip setting the path
+ # on the code object.
+ source_mock_context, filename_mock_context = self.mock_methods(
+ get_source=True, get_filename=True)
+ with source_mock_context as source_mock, filename_mock_context as name_mock:
+ source_mock.return_value = 'attr = 42'
+ name_mock.side_effect = ImportError
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertEqual(code.co_filename, '')
+ module = types.ModuleType('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+
+(Frozen_ELGetCodeTests,
+ Source_ELGetCodeTests
+ ) = test_util.test_both(ExecutionLoaderGetCodeTests,
+ ExecutionLoaderSubclass=SPLIT_EL)
+
+
+##### SourceLoader concrete methods ############################################
+class SourceOnlyLoader:
+
+ # Globals that should be defined for all modules.
+ source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, "
+ b"repr(__loader__)])")
+
+ def __init__(self, path):
+ self.path = path
+
+ def get_data(self, path):
+ if path != self.path:
+ raise IOError
+ return self.source
+
+ def get_filename(self, fullname):
+ return self.path
+
+ def module_repr(self, module):
+ return ''
+
+
+SPLIT_SOL = make_abc_subclasses(SourceOnlyLoader, 'SourceLoader')
+
+
+class SourceLoader(SourceOnlyLoader):
+
+ source_mtime = 1
+
+ def __init__(self, path, magic=None):
+ super().__init__(path)
+ self.bytecode_path = self.util.cache_from_source(self.path)
+ self.source_size = len(self.source)
+ if magic is None:
+ magic = self.util.MAGIC_NUMBER
+ data = bytearray(magic)
+ data.extend(self.init._pack_uint32(0))
+ data.extend(self.init._pack_uint32(self.source_mtime))
+ data.extend(self.init._pack_uint32(self.source_size))
+ code_object = compile(self.source, self.path, 'exec',
+ dont_inherit=True)
+ data.extend(marshal.dumps(code_object))
+ self.bytecode = bytes(data)
+ self.written = {}
+
+ def get_data(self, path):
+ if path == self.path:
+ return super().get_data(path)
+ elif path == self.bytecode_path:
+ return self.bytecode
+ else:
+ raise OSError
+
+ def path_stats(self, path):
+ if path != self.path:
+ raise IOError
+ return {'mtime': self.source_mtime, 'size': self.source_size}
+
+ def set_data(self, path, data):
+ self.written[path] = bytes(data)
+ return path == self.bytecode_path
+
+
+SPLIT_SL = make_abc_subclasses(SourceLoader, util=util, init=init)
+
+
+class SourceLoaderTestHarness:
+
+ def setUp(self, *, is_package=True, **kwargs):
+ self.package = 'pkg'
+ if is_package:
+ self.path = os.path.join(self.package, '__init__.py')
+ self.name = self.package
+ else:
+ module_name = 'mod'
+ self.path = os.path.join(self.package, '.'.join(['mod', 'py']))
+ self.name = '.'.join([self.package, module_name])
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = self.loader_mock(self.path, **kwargs)
+
+ def verify_module(self, module):
+ self.assertEqual(module.__name__, self.name)
+ self.assertEqual(module.__file__, self.path)
+ self.assertEqual(module.__cached__, self.cached)
+ self.assertEqual(module.__package__, self.package)
+ self.assertEqual(module.__loader__, self.loader)
+ values = module._.split('::')
+ self.assertEqual(values[0], self.name)
+ self.assertEqual(values[1], self.path)
+ self.assertEqual(values[2], self.cached)
+ self.assertEqual(values[3], self.package)
+ self.assertEqual(values[4], repr(self.loader))
+
+ def verify_code(self, code_object):
+ module = types.ModuleType(self.name)
+ module.__file__ = self.path
+ module.__cached__ = self.cached
+ module.__package__ = self.package
+ module.__loader__ = self.loader
+ module.__path__ = []
+ exec(code_object, module.__dict__)
+ self.verify_module(module)
+
+
+class SourceOnlyLoaderTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader for source-only loading.
+
+ Reload testing is subsumed by the tests for
+ importlib.util.module_for_loader.
+
+ """
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_get_source(self):
+ # Verify the source code is returned as a string.
+ # If an OSError is raised by get_data then raise ImportError.
+ expected_source = self.loader.source.decode('utf-8')
+ self.assertEqual(self.loader.get_source(self.name), expected_source)
+ def raise_OSError(path):
+ raise OSError
+ self.loader.get_data = raise_OSError
+ with self.assertRaises(ImportError) as cm:
+ self.loader.get_source(self.name)
+ self.assertEqual(cm.exception.name, self.name)
+
+ def test_is_package(self):
+ # Properly detect when loading a package.
+ self.setUp(is_package=False)
+ self.assertFalse(self.loader.is_package(self.name))
+ self.setUp(is_package=True)
+ self.assertTrue(self.loader.is_package(self.name))
+ self.assertFalse(self.loader.is_package(self.name + '.__init__'))
+
+ def test_get_code(self):
+ # Verify the code object is created.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_source_to_code(self):
+ # Verify the compiled code object.
+ code = self.loader.source_to_code(self.loader.source, self.path)
+ self.verify_code(code)
+
+ def test_load_module(self):
+ # Loading a module should set __name__, __loader__, __package__,
+ # __path__ (for packages), __file__, and __cached__.
+ # The module should also be put into sys.modules.
+ with test_util.uncache(self.name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertEqual(module.__path__, [os.path.dirname(self.path)])
+ self.assertIn(self.name, sys.modules)
+
+ def test_package_settings(self):
+ # __package__ needs to be set, while __path__ is set on if the module
+ # is a package.
+ # Testing the values for a package are covered by test_load_module.
+ self.setUp(is_package=False)
+ with test_util.uncache(self.name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertFalse(hasattr(module, '__path__'))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_get_source_encoding(self):
+ # Source is considered encoded in UTF-8 by default unless otherwise
+ # specified by an encoding line.
+ source = "_ = 'ü'"
+ self.loader.source = source.encode('utf-8')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+ source = "# coding: latin-1\n_ = ü"
+ self.loader.source = source.encode('latin-1')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+
+
+(Frozen_SourceOnlyLoaderTests,
+ Source_SourceOnlyLoaderTests
+ ) = test_util.test_both(SourceOnlyLoaderTests, util=util,
+ loader_mock=SPLIT_SOL)
+
+
+@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true")
+class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader's use of bytecode.
+
+ Source-only testing handled by SourceOnlyLoaderTests.
+
+ """
+
+ def verify_code(self, code_object, *, bytecode_written=False):
+ super().verify_code(code_object)
+ if bytecode_written:
+ self.assertIn(self.cached, self.loader.written)
+ data = bytearray(self.util.MAGIC_NUMBER)
+ data.extend(self.init._pack_uint32(0))
+ data.extend(self.init._pack_uint32(self.loader.source_mtime))
+ data.extend(self.init._pack_uint32(self.loader.source_size))
+ data.extend(marshal.dumps(code_object))
+ self.assertEqual(self.loader.written[self.cached], bytes(data))
+
+ def test_code_with_everything(self):
+ # When everything should work.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_no_bytecode(self):
+ # If no bytecode exists then move on to the source.
+ self.loader.bytecode_path = ""
+ # Sanity check
+ with self.assertRaises(OSError):
+ bytecode_path = self.util.cache_from_source(self.path)
+ self.loader.get_data(bytecode_path)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_code_bad_timestamp(self):
+ # Bytecode is only used when the timestamp matches the source EXACTLY.
+ for source_mtime in (0, 2):
+ assert source_mtime != self.loader.source_mtime
+ original = self.loader.source_mtime
+ self.loader.source_mtime = source_mtime
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+ self.loader.source_mtime = original
+
+ def test_code_bad_magic(self):
+ # Skip over bytecode with a bad magic number.
+ self.setUp(magic=b'0000')
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_dont_write_bytecode(self):
+ # Bytecode is not written if sys.dont_write_bytecode is true.
+ # Can assume it is false already thanks to the skipIf class decorator.
+ try:
+ sys.dont_write_bytecode = True
+ self.loader.bytecode_path = ""
+ code_object = self.loader.get_code(self.name)
+ self.assertNotIn(self.cached, self.loader.written)
+ finally:
+ sys.dont_write_bytecode = False
+
+ def test_no_set_data(self):
+ # If set_data is not defined, one can still read bytecode.
+ self.setUp(magic=b'0000')
+ original_set_data = self.loader.__class__.mro()[1].set_data
+ try:
+ del self.loader.__class__.mro()[1].set_data
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+ finally:
+ self.loader.__class__.mro()[1].set_data = original_set_data
+
+ def test_set_data_raises_exceptions(self):
+ # Raising NotImplementedError or OSError is okay for set_data.
+ def raise_exception(exc):
+ def closure(*args, **kwargs):
+ raise exc
+ return closure
+
+ self.setUp(magic=b'0000')
+ self.loader.set_data = raise_exception(NotImplementedError)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+
+(Frozen_SLBytecodeTests,
+ SourceSLBytecodeTests
+ ) = test_util.test_both(SourceLoaderBytecodeTests, init=init, util=util,
+ loader_mock=SPLIT_SL)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class SourceLoaderGetSourceTests:
+
+ """Tests for importlib.abc.SourceLoader.get_source()."""
+
+ def test_default_encoding(self):
+ # Should have no problems with UTF-8 text.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock('mod.file')
+ source = 'x = "ü"'
+ mock.source = source.encode('utf-8')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_decoded_source(self):
+ # Decoding should work.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock("mod.file")
+ source = "# coding: Latin-1\nx='ü'"
+ assert source.encode('latin-1') != source.encode('utf-8')
+ mock.source = source.encode('latin-1')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_universal_newlines(self):
+ # PEP 302 says universal newlines should be used.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock('mod.file')
+ source = "x = 42\r\ny = -13\r\n"
+ mock.source = source.encode('utf-8')
+ expect = io.IncrementalNewlineDecoder(None, True).decode(source)
+ self.assertEqual(mock.get_source(name), expect)
+
+
+(Frozen_SourceOnlyLoaderGetSourceTests,
+ Source_SourceOnlyLoaderGetSourceTests
+ ) = test_util.test_both(SourceLoaderGetSourceTests,
+ SourceOnlyLoaderMock=SPLIT_SOL)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
new file mode 100644
index 000000000..7b4137652
--- /dev/null
+++ b/Lib/test/test_importlib/test_api.py
@@ -0,0 +1,465 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+util = test_util.import_importlib('importlib.util')
+machinery = test_util.import_importlib('importlib.machinery')
+
+import os.path
+import sys
+from test import support
+import types
+import unittest
+import warnings
+
+
+class ImportModuleTests:
+
+ """Test importlib.import_module."""
+
+ def test_module_import(self):
+ # Test importing a top-level module.
+ with test_util.mock_modules('top_level') as mock:
+ with test_util.import_state(meta_path=[mock]):
+ module = self.init.import_module('top_level')
+ self.assertEqual(module.__name__, 'top_level')
+
+ def test_absolute_package_import(self):
+ # Test importing a module from a package with an absolute name.
+ pkg_name = 'pkg'
+ pkg_long_name = '{0}.__init__'.format(pkg_name)
+ name = '{0}.mod'.format(pkg_name)
+ with test_util.mock_modules(pkg_long_name, name) as mock:
+ with test_util.import_state(meta_path=[mock]):
+ module = self.init.import_module(name)
+ self.assertEqual(module.__name__, name)
+
+ def test_shallow_relative_package_import(self):
+ # Test importing a module from a package through a relative import.
+ pkg_name = 'pkg'
+ pkg_long_name = '{0}.__init__'.format(pkg_name)
+ module_name = 'mod'
+ absolute_name = '{0}.{1}'.format(pkg_name, module_name)
+ relative_name = '.{0}'.format(module_name)
+ with test_util.mock_modules(pkg_long_name, absolute_name) as mock:
+ with test_util.import_state(meta_path=[mock]):
+ self.init.import_module(pkg_name)
+ module = self.init.import_module(relative_name, pkg_name)
+ self.assertEqual(module.__name__, absolute_name)
+
+ def test_deep_relative_package_import(self):
+ modules = ['a.__init__', 'a.b.__init__', 'a.c']
+ with test_util.mock_modules(*modules) as mock:
+ with test_util.import_state(meta_path=[mock]):
+ self.init.import_module('a')
+ self.init.import_module('a.b')
+ module = self.init.import_module('..c', 'a.b')
+ self.assertEqual(module.__name__, 'a.c')
+
+ def test_absolute_import_with_package(self):
+ # Test importing a module from a package with an absolute name with
+ # the 'package' argument given.
+ pkg_name = 'pkg'
+ pkg_long_name = '{0}.__init__'.format(pkg_name)
+ name = '{0}.mod'.format(pkg_name)
+ with test_util.mock_modules(pkg_long_name, name) as mock:
+ with test_util.import_state(meta_path=[mock]):
+ self.init.import_module(pkg_name)
+ module = self.init.import_module(name, pkg_name)
+ self.assertEqual(module.__name__, name)
+
+ def test_relative_import_wo_package(self):
+ # Relative imports cannot happen without the 'package' argument being
+ # set.
+ with self.assertRaises(TypeError):
+ self.init.import_module('.support')
+
+
+ def test_loaded_once(self):
+ # Issue #13591: Modules should only be loaded once when
+ # initializing the parent package attempts to import the
+ # module currently being imported.
+ b_load_count = 0
+ def load_a():
+ self.init.import_module('a.b')
+ def load_b():
+ nonlocal b_load_count
+ b_load_count += 1
+ code = {'a': load_a, 'a.b': load_b}
+ modules = ['a.__init__', 'a.b']
+ with test_util.mock_modules(*modules, module_code=code) as mock:
+ with test_util.import_state(meta_path=[mock]):
+ self.init.import_module('a.b')
+ self.assertEqual(b_load_count, 1)
+
+
+(Frozen_ImportModuleTests,
+ Source_ImportModuleTests
+ ) = test_util.test_both(ImportModuleTests, init=init)
+
+
+class FindLoaderTests:
+
+ FakeMetaFinder = None
+
+ def test_sys_modules(self):
+ # If a module with __loader__ is in sys.modules, then return it.
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ loader = 'a loader!'
+ module.__loader__ = loader
+ sys.modules[name] = module
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ found = self.init.find_loader(name)
+ self.assertEqual(loader, found)
+
+ def test_sys_modules_loader_is_None(self):
+ # If sys.modules[name].__loader__ is None, raise ValueError.
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ module.__loader__ = None
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.init.find_loader(name)
+
+ def test_sys_modules_loader_is_not_set(self):
+ # Should raise ValueError
+ # Issue #17099
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ try:
+ del module.__loader__
+ except AttributeError:
+ pass
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.init.find_loader(name)
+
+ def test_success(self):
+ # Return the loader found on sys.meta_path.
+ name = 'some_mod'
+ with test_util.uncache(name):
+ with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual((name, None), self.init.find_loader(name))
+
+ def test_success_path(self):
+ # Searching on a path should work.
+ name = 'some_mod'
+ path = 'path to some place'
+ with test_util.uncache(name):
+ with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual((name, path),
+ self.init.find_loader(name, path))
+
+ def test_nothing(self):
+ # None is returned upon failure to find a loader.
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule'))
+
+
+class FindLoaderPEP451Tests(FindLoaderTests):
+
+ class FakeMetaFinder:
+ @staticmethod
+ def find_spec(name, path=None, target=None):
+ return machinery['Source'].ModuleSpec(name, (name, path))
+
+
+(Frozen_FindLoaderPEP451Tests,
+ Source_FindLoaderPEP451Tests
+ ) = test_util.test_both(FindLoaderPEP451Tests, init=init)
+
+
+class FindLoaderPEP302Tests(FindLoaderTests):
+
+ class FakeMetaFinder:
+ @staticmethod
+ def find_module(name, path=None):
+ return name, path
+
+
+(Frozen_FindLoaderPEP302Tests,
+ Source_FindLoaderPEP302Tests
+ ) = test_util.test_both(FindLoaderPEP302Tests, init=init)
+
+
+class ReloadTests:
+
+ def test_reload_modules(self):
+ for mod in ('tokenize', 'time', 'marshal'):
+ with self.subTest(module=mod):
+ with support.CleanImport(mod):
+ module = self.init.import_module(mod)
+ self.init.reload(module)
+
+ def test_module_replaced(self):
+ def code():
+ import sys
+ module = type(sys)('top_level')
+ module.spam = 3
+ sys.modules['top_level'] = module
+ mock = test_util.mock_modules('top_level',
+ module_code={'top_level': code})
+ with mock:
+ with test_util.import_state(meta_path=[mock]):
+ module = self.init.import_module('top_level')
+ reloaded = self.init.reload(module)
+ actual = sys.modules['top_level']
+ self.assertEqual(actual.spam, 3)
+ self.assertEqual(reloaded.spam, 3)
+
+ def test_reload_missing_loader(self):
+ with support.CleanImport('types'):
+ import types
+ loader = types.__loader__
+ del types.__loader__
+ reloaded = self.init.reload(types)
+
+ self.assertIs(reloaded, types)
+ self.assertIs(sys.modules['types'], types)
+ self.assertEqual(reloaded.__loader__.path, loader.path)
+
+ def test_reload_loader_replaced(self):
+ with support.CleanImport('types'):
+ import types
+ types.__loader__ = None
+ self.init.invalidate_caches()
+ reloaded = self.init.reload(types)
+
+ self.assertIsNot(reloaded.__loader__, None)
+ self.assertIs(reloaded, types)
+ self.assertIs(sys.modules['types'], types)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_reload_location_changed(self):
+ name = 'spam'
+ with support.temp_cwd(None) as cwd:
+ with test_util.uncache('spam'):
+ with support.DirsOnSysPath(cwd):
+ # Start as a plain module.
+ self.init.invalidate_caches()
+ path = os.path.join(cwd, name + '.py')
+ cached = self.util.cache_from_source(path)
+ expected = {'__name__': name,
+ '__package__': '',
+ '__file__': path,
+ '__cached__': cached,
+ '__doc__': None,
+ }
+ support.create_empty_file(path)
+ module = self.init.import_module(name)
+ ns = vars(module).copy()
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ ns.pop('__builtins__', None) # An implementation detail.
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(loader.path, path)
+ self.assertEqual(ns, expected)
+
+ # Change to a package.
+ self.init.invalidate_caches()
+ init_path = os.path.join(cwd, name, '__init__.py')
+ cached = self.util.cache_from_source(init_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__file__': init_path,
+ '__cached__': cached,
+ '__path__': [os.path.dirname(init_path)],
+ '__doc__': None,
+ }
+ os.mkdir(name)
+ os.rename(path, init_path)
+ reloaded = self.init.reload(module)
+ ns = vars(reloaded).copy()
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ ns.pop('__builtins__', None) # An implementation detail.
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(reloaded, module)
+ self.assertEqual(loader.path, init_path)
+ self.maxDiff = None
+ self.assertEqual(ns, expected)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_reload_namespace_changed(self):
+ name = 'spam'
+ with support.temp_cwd(None) as cwd:
+ with test_util.uncache('spam'):
+ with support.DirsOnSysPath(cwd):
+ # Start as a namespace package.
+ self.init.invalidate_caches()
+ bad_path = os.path.join(cwd, name, '__init.py')
+ cached = self.util.cache_from_source(bad_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__doc__': None,
+ '__file__': None,
+ }
+ os.mkdir(name)
+ with open(bad_path, 'w') as init_file:
+ init_file.write('eggs = None')
+ module = self.init.import_module(name)
+ ns = vars(module).copy()
+ loader = ns.pop('__loader__')
+ path = ns.pop('__path__')
+ spec = ns.pop('__spec__')
+ ns.pop('__builtins__', None) # An implementation detail.
+ self.assertEqual(spec.name, name)
+ self.assertIsNotNone(spec.loader)
+ self.assertIsNotNone(loader)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(set(path),
+ set([os.path.dirname(bad_path)]))
+ with self.assertRaises(AttributeError):
+ # a NamespaceLoader
+ loader.path
+ self.assertEqual(ns, expected)
+
+ # Change to a regular package.
+ self.init.invalidate_caches()
+ init_path = os.path.join(cwd, name, '__init__.py')
+ cached = self.util.cache_from_source(init_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__file__': init_path,
+ '__cached__': cached,
+ '__path__': [os.path.dirname(init_path)],
+ '__doc__': None,
+ 'eggs': None,
+ }
+ os.rename(bad_path, init_path)
+ reloaded = self.init.reload(module)
+ ns = vars(reloaded).copy()
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ ns.pop('__builtins__', None) # An implementation detail.
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(reloaded, module)
+ self.assertEqual(loader.path, init_path)
+ self.assertEqual(ns, expected)
+
+ def test_reload_submodule(self):
+ # See #19851.
+ name = 'spam'
+ subname = 'ham'
+ with test_util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = test_util.submodule(name, subname, pkg_dir)
+ ham = self.init.import_module(fullname)
+ reloaded = self.init.reload(ham)
+ self.assertIs(reloaded, ham)
+
+ def test_module_missing_spec(self):
+ #Test that reload() throws ModuleNotFounderror when reloading
+ # a module who's missing a spec. (bpo-29851)
+ name = 'spam'
+ with test_util.uncache(name):
+ module = sys.modules[name] = types.ModuleType(name)
+ # Sanity check by attempting an import.
+ module = self.init.import_module(name)
+ self.assertIsNone(module.__spec__)
+ with self.assertRaises(ModuleNotFoundError):
+ self.init.reload(module)
+
+
+(Frozen_ReloadTests,
+ Source_ReloadTests
+ ) = test_util.test_both(ReloadTests, init=init, util=util)
+
+
+class InvalidateCacheTests:
+
+ def test_method_called(self):
+ # If defined the method should be called.
+ class InvalidatingNullFinder:
+ def __init__(self, *ignored):
+ self.called = False
+ def find_module(self, *args):
+ return None
+ def invalidate_caches(self):
+ self.called = True
+
+ key = 'gobledeegook'
+ meta_ins = InvalidatingNullFinder()
+ path_ins = InvalidatingNullFinder()
+ sys.meta_path.insert(0, meta_ins)
+ self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+ sys.path_importer_cache[key] = path_ins
+ self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
+ self.init.invalidate_caches()
+ self.assertTrue(meta_ins.called)
+ self.assertTrue(path_ins.called)
+
+ def test_method_lacking(self):
+ # There should be no issues if the method is not defined.
+ key = 'gobbledeegook'
+ sys.path_importer_cache[key] = None
+ self.addCleanup(lambda: sys.path_importer_cache.pop(key, None))
+ self.init.invalidate_caches() # Shouldn't trigger an exception.
+
+
+(Frozen_InvalidateCacheTests,
+ Source_InvalidateCacheTests
+ ) = test_util.test_both(InvalidateCacheTests, init=init)
+
+
+class FrozenImportlibTests(unittest.TestCase):
+
+ def test_no_frozen_importlib(self):
+ # Should be able to import w/o _frozen_importlib being defined.
+ # Can't do an isinstance() check since separate copies of importlib
+ # may have been used for import, so just check the name is not for the
+ # frozen loader.
+ source_init = init['Source']
+ self.assertNotEqual(source_init.__loader__.__class__.__name__,
+ 'FrozenImporter')
+
+
+class StartupTests:
+
+ def test_everyone_has___loader__(self):
+ # Issue #17098: all modules should have __loader__ defined.
+ for name, module in sys.modules.items():
+ if isinstance(module, types.ModuleType):
+ with self.subTest(name=name):
+ self.assertTrue(hasattr(module, '__loader__'),
+ '{!r} lacks a __loader__ attribute'.format(name))
+ if self.machinery.BuiltinImporter.find_module(name):
+ self.assertIsNot(module.__loader__, None)
+ elif self.machinery.FrozenImporter.find_module(name):
+ self.assertIsNot(module.__loader__, None)
+
+ def test_everyone_has___spec__(self):
+ for name, module in sys.modules.items():
+ if isinstance(module, types.ModuleType):
+ with self.subTest(name=name):
+ self.assertTrue(hasattr(module, '__spec__'))
+ if self.machinery.BuiltinImporter.find_module(name):
+ self.assertIsNot(module.__spec__, None)
+ elif self.machinery.FrozenImporter.find_module(name):
+ self.assertIsNot(module.__spec__, None)
+
+
+(Frozen_StartupTests,
+ Source_StartupTests
+ ) = test_util.test_both(StartupTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py
new file mode 100644
index 000000000..4b3be72f3
--- /dev/null
+++ b/Lib/test/test_importlib/test_lazy.py
@@ -0,0 +1,146 @@
+import importlib
+from importlib import abc
+from importlib import util
+import sys
+import types
+import unittest
+
+from . import util as test_util
+
+
+class CollectInit:
+
+ def __init__(self, *args, **kwargs):
+ self.args = args
+ self.kwargs = kwargs
+
+ def exec_module(self, module):
+ return self
+
+
+class LazyLoaderFactoryTests(unittest.TestCase):
+
+ def test_init(self):
+ factory = util.LazyLoader.factory(CollectInit)
+ # E.g. what importlib.machinery.FileFinder instantiates loaders with
+ # plus keyword arguments.
+ lazy_loader = factory('module name', 'module path', kw='kw')
+ loader = lazy_loader.loader
+ self.assertEqual(('module name', 'module path'), loader.args)
+ self.assertEqual({'kw': 'kw'}, loader.kwargs)
+
+ def test_validation(self):
+ # No exec_module(), no lazy loading.
+ with self.assertRaises(TypeError):
+ util.LazyLoader.factory(object)
+
+
+class TestingImporter(abc.MetaPathFinder, abc.Loader):
+
+ module_name = 'lazy_loader_test'
+ mutated_name = 'changed'
+ loaded = None
+ source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name)
+
+ def find_spec(self, name, path, target=None):
+ if name != self.module_name:
+ return None
+ return util.spec_from_loader(name, util.LazyLoader(self))
+
+ def exec_module(self, module):
+ exec(self.source_code, module.__dict__)
+ self.loaded = module
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class LazyLoaderTests(unittest.TestCase):
+
+ def test_init(self):
+ with self.assertRaises(TypeError):
+ # Classes that don't define exec_module() trigger TypeError.
+ util.LazyLoader(object)
+
+ def new_module(self, source_code=None):
+ loader = TestingImporter()
+ if source_code is not None:
+ loader.source_code = source_code
+ spec = util.spec_from_loader(TestingImporter.module_name,
+ util.LazyLoader(loader))
+ module = spec.loader.create_module(spec)
+ if module is None:
+ module = types.ModuleType(TestingImporter.module_name)
+ module.__spec__ = spec
+ module.__loader__ = spec.loader
+ spec.loader.exec_module(module)
+ # Module is now lazy.
+ self.assertIsNone(loader.loaded)
+ return module
+
+ def test_e2e(self):
+ # End-to-end test to verify the load is in fact lazy.
+ importer = TestingImporter()
+ assert importer.loaded is None
+ with test_util.uncache(importer.module_name):
+ with test_util.import_state(meta_path=[importer]):
+ module = importlib.import_module(importer.module_name)
+ self.assertIsNone(importer.loaded)
+ # Trigger load.
+ self.assertEqual(module.__loader__, importer)
+ self.assertIsNotNone(importer.loaded)
+ self.assertEqual(module, importer.loaded)
+
+ def test_attr_unchanged(self):
+ # An attribute only mutated as a side-effect of import should not be
+ # changed needlessly.
+ module = self.new_module()
+ self.assertEqual(TestingImporter.mutated_name, module.__name__)
+
+ def test_new_attr(self):
+ # A new attribute should persist.
+ module = self.new_module()
+ module.new_attr = 42
+ self.assertEqual(42, module.new_attr)
+
+ def test_mutated_preexisting_attr(self):
+ # Changing an attribute that already existed on the module --
+ # e.g. __name__ -- should persist.
+ module = self.new_module()
+ module.__name__ = 'bogus'
+ self.assertEqual('bogus', module.__name__)
+
+ def test_mutated_attr(self):
+ # Changing an attribute that comes into existence after an import
+ # should persist.
+ module = self.new_module()
+ module.attr = 6
+ self.assertEqual(6, module.attr)
+
+ def test_delete_eventual_attr(self):
+ # Deleting an attribute should stay deleted.
+ module = self.new_module()
+ del module.attr
+ self.assertFalse(hasattr(module, 'attr'))
+
+ def test_delete_preexisting_attr(self):
+ module = self.new_module()
+ del module.__name__
+ self.assertFalse(hasattr(module, '__name__'))
+
+ def test_module_substitution_error(self):
+ with test_util.uncache(TestingImporter.module_name):
+ fresh_module = types.ModuleType(TestingImporter.module_name)
+ sys.modules[TestingImporter.module_name] = fresh_module
+ module = self.new_module()
+ with self.assertRaisesRegex(ValueError, "substituted"):
+ module.__name__
+
+ def test_module_already_in_sys(self):
+ with test_util.uncache(TestingImporter.module_name):
+ module = self.new_module()
+ sys.modules[TestingImporter.module_name] = module
+ # Force the load; just care that no exception is raised.
+ module.__name__
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py
new file mode 100644
index 000000000..db494c339
--- /dev/null
+++ b/Lib/test/test_importlib/test_locks.py
@@ -0,0 +1,153 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+
+import sys
+import threading
+import weakref
+
+from test import support
+from test import lock_tests
+
+
+class ModuleLockAsRLockTests:
+ locktype = classmethod(lambda cls: cls.LockType("some_lock"))
+
+ # _is_owned() unsupported
+ test__is_owned = None
+ # acquire(blocking=False) unsupported
+ test_try_acquire = None
+ test_try_acquire_contended = None
+ # `with` unsupported
+ test_with = None
+ # acquire(timeout=...) unsupported
+ test_timeout = None
+ # _release_save() unsupported
+ test_release_save_unacquired = None
+ # lock status in repr unsupported
+ test_repr = None
+ test_locked_repr = None
+
+LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock
+ for kind, splitinit in init.items()}
+
+(Frozen_ModuleLockAsRLockTests,
+ Source_ModuleLockAsRLockTests
+ ) = test_util.test_both(ModuleLockAsRLockTests, lock_tests.RLockTests,
+ LockType=LOCK_TYPES)
+
+
+class DeadlockAvoidanceTests:
+
+ def setUp(self):
+ try:
+ self.old_switchinterval = sys.getswitchinterval()
+ support.setswitchinterval(0.000001)
+ except AttributeError:
+ self.old_switchinterval = None
+
+ def tearDown(self):
+ if self.old_switchinterval is not None:
+ sys.setswitchinterval(self.old_switchinterval)
+
+ def run_deadlock_avoidance_test(self, create_deadlock):
+ NLOCKS = 10
+ locks = [self.LockType(str(i)) for i in range(NLOCKS)]
+ pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)]
+ if create_deadlock:
+ NTHREADS = NLOCKS
+ else:
+ NTHREADS = NLOCKS - 1
+ barrier = threading.Barrier(NTHREADS)
+ results = []
+
+ def _acquire(lock):
+ """Try to acquire the lock. Return True on success,
+ False on deadlock."""
+ try:
+ lock.acquire()
+ except self.DeadlockError:
+ return False
+ else:
+ return True
+
+ def f():
+ a, b = pairs.pop()
+ ra = _acquire(a)
+ barrier.wait()
+ rb = _acquire(b)
+ results.append((ra, rb))
+ if rb:
+ b.release()
+ if ra:
+ a.release()
+ lock_tests.Bunch(f, NTHREADS).wait_for_finished()
+ self.assertEqual(len(results), NTHREADS)
+ return results
+
+ def test_deadlock(self):
+ results = self.run_deadlock_avoidance_test(True)
+ # At least one of the threads detected a potential deadlock on its
+ # second acquire() call. It may be several of them, because the
+ # deadlock avoidance mechanism is conservative.
+ nb_deadlocks = results.count((True, False))
+ self.assertGreaterEqual(nb_deadlocks, 1)
+ self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks)
+
+ def test_no_deadlock(self):
+ results = self.run_deadlock_avoidance_test(False)
+ self.assertEqual(results.count((True, False)), 0)
+ self.assertEqual(results.count((True, True)), len(results))
+
+
+DEADLOCK_ERRORS = {kind: splitinit._bootstrap._DeadlockError
+ for kind, splitinit in init.items()}
+
+(Frozen_DeadlockAvoidanceTests,
+ Source_DeadlockAvoidanceTests
+ ) = test_util.test_both(DeadlockAvoidanceTests,
+ LockType=LOCK_TYPES,
+ DeadlockError=DEADLOCK_ERRORS)
+
+
+class LifetimeTests:
+
+ @property
+ def bootstrap(self):
+ return self.init._bootstrap
+
+ def test_lock_lifetime(self):
+ name = "xyzzy"
+ self.assertNotIn(name, self.bootstrap._module_locks)
+ lock = self.bootstrap._get_module_lock(name)
+ self.assertIn(name, self.bootstrap._module_locks)
+ wr = weakref.ref(lock)
+ del lock
+ support.gc_collect()
+ self.assertNotIn(name, self.bootstrap._module_locks)
+ self.assertIsNone(wr())
+
+ def test_all_locks(self):
+ support.gc_collect()
+ self.assertEqual(0, len(self.bootstrap._module_locks),
+ self.bootstrap._module_locks)
+
+# TODO: RustPython
+# (Frozen_LifetimeTests,
+# Source_LifetimeTests
+# ) = test_util.test_both(LifetimeTests, init=init)
+
+
+@support.reap_threads
+def test_main():
+ support.run_unittest(Frozen_ModuleLockAsRLockTests,
+ Source_ModuleLockAsRLockTests,
+ Frozen_DeadlockAvoidanceTests,
+ Source_DeadlockAvoidanceTests,
+ # Frozen_LifetimeTests,
+ # Source_LifetimeTests
+ )
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py
new file mode 100644
index 000000000..a8f95a035
--- /dev/null
+++ b/Lib/test/test_importlib/test_namespace_pkgs.py
@@ -0,0 +1,343 @@
+import contextlib
+import importlib
+import os
+import sys
+import unittest
+
+from test.test_importlib import util
+
+# needed tests:
+#
+# need to test when nested, so that the top-level path isn't sys.path
+# need to test dynamic path detection, both at top-level and nested
+# with dynamic path, check when a loader is returned on path reload (that is,
+# trying to switch from a namespace package to a regular package)
+
+
+@contextlib.contextmanager
+def sys_modules_context():
+ """
+ Make sure sys.modules is the same object and has the same content
+ when exiting the context as when entering.
+
+ Similar to importlib.test.util.uncache, but doesn't require explicit
+ names.
+ """
+ sys_modules_saved = sys.modules
+ sys_modules_copy = sys.modules.copy()
+ try:
+ yield
+ finally:
+ sys.modules = sys_modules_saved
+ sys.modules.clear()
+ sys.modules.update(sys_modules_copy)
+
+
+@contextlib.contextmanager
+def namespace_tree_context(**kwargs):
+ """
+ Save import state and sys.modules cache and restore it on exit.
+ Typical usage:
+
+ >>> with namespace_tree_context(path=['/tmp/xxyy/portion1',
+ ... '/tmp/xxyy/portion2']):
+ ... pass
+ """
+ # use default meta_path and path_hooks unless specified otherwise
+ kwargs.setdefault('meta_path', sys.meta_path)
+ kwargs.setdefault('path_hooks', sys.path_hooks)
+ import_context = util.import_state(**kwargs)
+ with import_context, sys_modules_context():
+ yield
+
+class NamespacePackageTest(unittest.TestCase):
+ """
+ Subclasses should define self.root and self.paths (under that root)
+ to be added to sys.path.
+ """
+ root = os.path.join(os.path.dirname(__file__), 'namespace_pkgs')
+
+ def setUp(self):
+ self.resolved_paths = [
+ os.path.join(self.root, path) for path in self.paths
+ ]
+ self.ctx = namespace_tree_context(path=self.resolved_paths)
+ self.ctx.__enter__()
+
+ def tearDown(self):
+ # TODO: will we ever want to pass exc_info to __exit__?
+ self.ctx.__exit__(None, None, None)
+
+
+class SingleNamespacePackage(NamespacePackageTest):
+ paths = ['portion1']
+
+ def test_simple_package(self):
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ def test_cant_import_other(self):
+ with self.assertRaises(ImportError):
+ import foo.two
+
+ def test_module_repr(self):
+ import foo.one
+ self.assertEqual(repr(foo), "")
+
+
+class DynamicPathNamespacePackage(NamespacePackageTest):
+ paths = ['portion1']
+
+ def test_dynamic_path(self):
+ # Make sure only 'foo.one' can be imported
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ with self.assertRaises(ImportError):
+ import foo.two
+
+ # Now modify sys.path
+ sys.path.append(os.path.join(self.root, 'portion2'))
+
+ # And make sure foo.two is now importable
+ import foo.two
+ self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class CombinedNamespacePackages(NamespacePackageTest):
+ paths = ['both_portions']
+
+ def test_imports(self):
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'both_portions foo one')
+ self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SeparatedNamespacePackages(NamespacePackageTest):
+ paths = ['portion1', 'portion2']
+
+ def test_imports(self):
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+ self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
+ paths = ['portion1', 'both_portions']
+
+ def test_first_path_wins(self):
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+ self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+ def test_first_path_wins_again(self):
+ sys.path.reverse()
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'both_portions foo one')
+ self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+ def test_first_path_wins_importing_second_first(self):
+ import foo.two
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+ self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SingleZipNamespacePackage(NamespacePackageTest):
+ paths = ['top_level_portion1.zip']
+
+ def test_simple_package(self):
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ def test_cant_import_other(self):
+ with self.assertRaises(ImportError):
+ import foo.two
+
+
+class SeparatedZipNamespacePackages(NamespacePackageTest):
+ paths = ['top_level_portion1.zip', 'portion2']
+
+ def test_imports(self):
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+ self.assertEqual(foo.two.attr, 'portion2 foo two')
+ self.assertIn('top_level_portion1.zip', foo.one.__file__)
+ self.assertNotIn('.zip', foo.two.__file__)
+
+
+class SingleNestedZipNamespacePackage(NamespacePackageTest):
+ paths = ['nested_portion1.zip/nested_portion1']
+
+ def test_simple_package(self):
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ def test_cant_import_other(self):
+ with self.assertRaises(ImportError):
+ import foo.two
+
+
+class SeparatedNestedZipNamespacePackages(NamespacePackageTest):
+ paths = ['nested_portion1.zip/nested_portion1', 'portion2']
+
+ def test_imports(self):
+ import foo.one
+ import foo.two
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+ self.assertEqual(foo.two.attr, 'portion2 foo two')
+ fn = os.path.join('nested_portion1.zip', 'nested_portion1')
+ self.assertIn(fn, foo.one.__file__)
+ self.assertNotIn('.zip', foo.two.__file__)
+
+
+class LegacySupport(NamespacePackageTest):
+ paths = ['not_a_namespace_pkg', 'portion1', 'portion2', 'both_portions']
+
+ def test_non_namespace_package_takes_precedence(self):
+ import foo.one
+ with self.assertRaises(ImportError):
+ import foo.two
+ self.assertIn('__init__', foo.__file__)
+ self.assertNotIn('namespace', str(foo.__loader__).lower())
+
+
+class DynamicPathCalculation(NamespacePackageTest):
+ paths = ['project1', 'project2']
+
+ def test_project3_fails(self):
+ import parent.child.one
+ self.assertEqual(len(parent.__path__), 2)
+ self.assertEqual(len(parent.child.__path__), 2)
+ import parent.child.two
+ self.assertEqual(len(parent.__path__), 2)
+ self.assertEqual(len(parent.child.__path__), 2)
+
+ self.assertEqual(parent.child.one.attr, 'parent child one')
+ self.assertEqual(parent.child.two.attr, 'parent child two')
+
+ with self.assertRaises(ImportError):
+ import parent.child.three
+
+ self.assertEqual(len(parent.__path__), 2)
+ self.assertEqual(len(parent.child.__path__), 2)
+
+ def test_project3_succeeds(self):
+ import parent.child.one
+ self.assertEqual(len(parent.__path__), 2)
+ self.assertEqual(len(parent.child.__path__), 2)
+ import parent.child.two
+ self.assertEqual(len(parent.__path__), 2)
+ self.assertEqual(len(parent.child.__path__), 2)
+
+ self.assertEqual(parent.child.one.attr, 'parent child one')
+ self.assertEqual(parent.child.two.attr, 'parent child two')
+
+ with self.assertRaises(ImportError):
+ import parent.child.three
+
+ # now add project3
+ sys.path.append(os.path.join(self.root, 'project3'))
+ import parent.child.three
+
+ # the paths dynamically get longer, to include the new directories
+ self.assertEqual(len(parent.__path__), 3)
+ self.assertEqual(len(parent.child.__path__), 3)
+
+ self.assertEqual(parent.child.three.attr, 'parent child three')
+
+
+class ZipWithMissingDirectory(NamespacePackageTest):
+ paths = ['missing_directory.zip']
+
+ @unittest.expectedFailure
+ def test_missing_directory(self):
+ # This will fail because missing_directory.zip contains:
+ # Length Date Time Name
+ # --------- ---------- ----- ----
+ # 29 2012-05-03 18:13 foo/one.py
+ # 0 2012-05-03 20:57 bar/
+ # 38 2012-05-03 20:57 bar/two.py
+ # --------- -------
+ # 67 3 files
+
+ # Because there is no 'foo/', the zipimporter currently doesn't
+ # know that foo is a namespace package
+
+ import foo.one
+
+ def test_present_directory(self):
+ # This succeeds because there is a "bar/" in the zip file
+ import bar.two
+ self.assertEqual(bar.two.attr, 'missing_directory foo two')
+
+
+class ModuleAndNamespacePackageInSameDir(NamespacePackageTest):
+ paths = ['module_and_namespace_package']
+
+ def test_module_before_namespace_package(self):
+ # Make sure we find the module in preference to the
+ # namespace package.
+ import a_test
+ self.assertEqual(a_test.attr, 'in module')
+
+
+class ReloadTests(NamespacePackageTest):
+ paths = ['portion1']
+
+ def test_simple_package(self):
+ import foo.one
+ foo = importlib.reload(foo)
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ def test_cant_import_other(self):
+ import foo
+ with self.assertRaises(ImportError):
+ import foo.two
+ foo = importlib.reload(foo)
+ with self.assertRaises(ImportError):
+ import foo.two
+
+ def test_dynamic_path(self):
+ import foo.one
+ with self.assertRaises(ImportError):
+ import foo.two
+
+ # Now modify sys.path and reload.
+ sys.path.append(os.path.join(self.root, 'portion2'))
+ foo = importlib.reload(foo)
+
+ # And make sure foo.two is now importable
+ import foo.two
+ self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class LoaderTests(NamespacePackageTest):
+ paths = ['portion1']
+
+ def test_namespace_loader_consistency(self):
+ # bpo-32303
+ import foo
+ self.assertEqual(foo.__loader__, foo.__spec__.loader)
+ self.assertIsNotNone(foo.__loader__)
+
+ def test_namespace_origin_consistency(self):
+ # bpo-32305
+ import foo
+ self.assertIsNone(foo.__spec__.origin)
+ self.assertIsNone(foo.__file__)
+
+ def test_path_indexable(self):
+ # bpo-35843
+ import foo
+ expected_path = os.path.join(self.root, 'portion1', 'foo')
+ self.assertEqual(foo.__path__[0], expected_path)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py
new file mode 100644
index 000000000..4c740095e
--- /dev/null
+++ b/Lib/test/test_importlib/test_open.py
@@ -0,0 +1,75 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+ def execute(self, package, path):
+ with resources.open_binary(package, path):
+ pass
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+ def execute(self, package, path):
+ with resources.open_text(package, path):
+ pass
+
+
+class OpenTests:
+ def test_open_binary(self):
+ with resources.open_binary(self.data, 'binary.file') as fp:
+ result = fp.read()
+ self.assertEqual(result, b'\x00\x01\x02\x03')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_open_text_default_encoding(self):
+ with resources.open_text(self.data, 'utf-8.file') as fp:
+ result = fp.read()
+ self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_open_text_given_encoding(self):
+ with resources.open_text(
+ self.data, 'utf-16.file', 'utf-16', 'strict') as fp:
+ result = fp.read()
+ self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_open_text_with_errors(self):
+ # Raises UnicodeError without the 'errors' argument.
+ with resources.open_text(
+ self.data, 'utf-16.file', 'utf-8', 'strict') as fp:
+ self.assertRaises(UnicodeError, fp.read)
+ with resources.open_text(
+ self.data, 'utf-16.file', 'utf-8', 'ignore') as fp:
+ result = fp.read()
+ self.assertEqual(
+ result,
+ 'H\x00e\x00l\x00l\x00o\x00,\x00 '
+ '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+ '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+ def test_open_binary_FileNotFoundError(self):
+ self.assertRaises(
+ FileNotFoundError,
+ resources.open_binary, self.data, 'does-not-exist')
+
+ def test_open_text_FileNotFoundError(self):
+ self.assertRaises(
+ FileNotFoundError,
+ resources.open_text, self.data, 'does-not-exist')
+
+
+class OpenDiskTests(OpenTests, unittest.TestCase):
+ def setUp(self):
+ self.data = data01
+
+
+class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
+ pass
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py
new file mode 100644
index 000000000..562a566e0
--- /dev/null
+++ b/Lib/test/test_importlib/test_path.py
@@ -0,0 +1,40 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonTests(util.CommonResourceTests, unittest.TestCase):
+ def execute(self, package, path):
+ with resources.path(package, path):
+ pass
+
+
+class PathTests:
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_reading(self):
+ # Path should be readable.
+ # Test also implicitly verifies the returned object is a pathlib.Path
+ # instance.
+ with resources.path(self.data, 'utf-8.file') as path:
+ # pathlib.Path.read_text() was introduced in Python 3.5.
+ with path.open('r', encoding='utf-8') as file:
+ text = file.read()
+ self.assertEqual('Hello, UTF-8 world!\n', text)
+
+
+class PathDiskTests(PathTests, unittest.TestCase):
+ data = data01
+
+
+class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase):
+ def test_remove_in_context_manager(self):
+ # It is not an error if the file that was temporarily stashed on the
+ # file system is removed inside the `with` stanza.
+ with resources.path(self.data, 'utf-8.file') as path:
+ path.unlink()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py
new file mode 100644
index 000000000..52cc24501
--- /dev/null
+++ b/Lib/test/test_importlib/test_read.py
@@ -0,0 +1,65 @@
+import unittest
+
+from importlib import import_module, resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+ def execute(self, package, path):
+ resources.read_binary(package, path)
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+ def execute(self, package, path):
+ resources.read_text(package, path)
+
+
+class ReadTests:
+ def test_read_binary(self):
+ result = resources.read_binary(self.data, 'binary.file')
+ self.assertEqual(result, b'\0\1\2\3')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_read_text_default_encoding(self):
+ result = resources.read_text(self.data, 'utf-8.file')
+ self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_read_text_given_encoding(self):
+ result = resources.read_text(
+ self.data, 'utf-16.file', encoding='utf-16')
+ self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_read_text_with_errors(self):
+ # Raises UnicodeError without the 'errors' argument.
+ self.assertRaises(
+ UnicodeError, resources.read_text, self.data, 'utf-16.file')
+ result = resources.read_text(self.data, 'utf-16.file', errors='ignore')
+ self.assertEqual(
+ result,
+ 'H\x00e\x00l\x00l\x00o\x00,\x00 '
+ '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+ '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+
+class ReadDiskTests(ReadTests, unittest.TestCase):
+ data = data01
+
+
+class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase):
+ def test_read_submodule_resource(self):
+ submodule = import_module('ziptestdata.subdirectory')
+ result = resources.read_binary(
+ submodule, 'binary.file')
+ self.assertEqual(result, b'\0\1\2\3')
+
+ def test_read_submodule_resource_by_name(self):
+ result = resources.read_binary(
+ 'ziptestdata.subdirectory', 'binary.file')
+ self.assertEqual(result, b'\0\1\2\3')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py
new file mode 100644
index 000000000..4de6d9ac3
--- /dev/null
+++ b/Lib/test/test_importlib/test_resource.py
@@ -0,0 +1,167 @@
+import sys
+import unittest
+
+from . import data01
+from . import zipdata01, zipdata02
+from . import util
+from importlib import resources, import_module
+
+
+class ResourceTests:
+ # Subclasses are expected to set the `data` attribute.
+
+ def test_is_resource_good_path(self):
+ self.assertTrue(resources.is_resource(self.data, 'binary.file'))
+
+ def test_is_resource_missing(self):
+ self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
+
+ def test_is_resource_subresource_directory(self):
+ # Directories are not resources.
+ self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
+
+ def test_contents(self):
+ contents = set(resources.contents(self.data))
+ # There may be cruft in the directory listing of the data directory.
+ # Under Python 3 we could have a __pycache__ directory, and under
+ # Python 2 we could have .pyc files. These are both artifacts of the
+ # test suite importing these modules and writing these caches. They
+ # aren't germane to this test, so just filter them out.
+ contents.discard('__pycache__')
+ contents.discard('__init__.pyc')
+ contents.discard('__init__.pyo')
+ self.assertEqual(contents, {
+ '__init__.py',
+ 'subdirectory',
+ 'utf-8.file',
+ 'binary.file',
+ 'utf-16.file',
+ })
+
+
+class ResourceDiskTests(ResourceTests, unittest.TestCase):
+ def setUp(self):
+ self.data = data01
+
+
+class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase):
+ pass
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class ResourceLoaderTests(unittest.TestCase):
+ def test_resource_contents(self):
+ package = util.create_package(
+ file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+ self.assertEqual(
+ set(resources.contents(package)),
+ {'A', 'B', 'C'})
+
+ def test_resource_is_resource(self):
+ package = util.create_package(
+ file=data01, path=data01.__file__,
+ contents=['A', 'B', 'C', 'D/E', 'D/F'])
+ self.assertTrue(resources.is_resource(package, 'B'))
+
+ def test_resource_directory_is_not_resource(self):
+ package = util.create_package(
+ file=data01, path=data01.__file__,
+ contents=['A', 'B', 'C', 'D/E', 'D/F'])
+ self.assertFalse(resources.is_resource(package, 'D'))
+
+ def test_resource_missing_is_not_resource(self):
+ package = util.create_package(
+ file=data01, path=data01.__file__,
+ contents=['A', 'B', 'C', 'D/E', 'D/F'])
+ self.assertFalse(resources.is_resource(package, 'Z'))
+
+
+class ResourceCornerCaseTests(unittest.TestCase):
+ def test_package_has_no_reader_fallback(self):
+ # Test odd ball packages which:
+ # 1. Do not have a ResourceReader as a loader
+ # 2. Are not on the file system
+ # 3. Are not in a zip file
+ module = util.create_package(
+ file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+ # Give the module a dummy loader.
+ module.__loader__ = object()
+ # Give the module a dummy origin.
+ module.__file__ = '/path/which/shall/not/be/named'
+ if sys.version_info >= (3,):
+ module.__spec__.loader = module.__loader__
+ module.__spec__.origin = module.__file__
+ self.assertFalse(resources.is_resource(module, 'A'))
+
+
+class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase):
+ ZIP_MODULE = zipdata02 # type: ignore
+
+ def test_unrelated_contents(self):
+ # https://gitlab.com/python-devs/importlib_resources/issues/44
+ #
+ # Here we have a zip file with two unrelated subpackages. The bug
+ # reports that getting the contents of a resource returns unrelated
+ # files.
+ self.assertEqual(
+ set(resources.contents('ziptestdata.one')),
+ {'__init__.py', 'resource1.txt'})
+ self.assertEqual(
+ set(resources.contents('ziptestdata.two')),
+ {'__init__.py', 'resource2.txt'})
+
+
+class SubdirectoryResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase):
+ ZIP_MODULE = zipdata01 # type: ignore
+
+ def test_is_submodule_resource(self):
+ submodule = import_module('ziptestdata.subdirectory')
+ self.assertTrue(
+ resources.is_resource(submodule, 'binary.file'))
+
+ def test_read_submodule_resource_by_name(self):
+ self.assertTrue(
+ resources.is_resource('ziptestdata.subdirectory', 'binary.file'))
+
+ def test_submodule_contents(self):
+ submodule = import_module('ziptestdata.subdirectory')
+ self.assertEqual(
+ set(resources.contents(submodule)),
+ {'__init__.py', 'binary.file'})
+
+ def test_submodule_contents_by_name(self):
+ self.assertEqual(
+ set(resources.contents('ziptestdata.subdirectory')),
+ {'__init__.py', 'binary.file'})
+
+
+class NamespaceTest(unittest.TestCase):
+ def test_namespaces_cannot_have_resources(self):
+ contents = resources.contents('test.test_importlib.data03.namespace')
+ self.assertFalse(list(contents))
+ # Even though there is a file in the namespace directory, it is not
+ # considered a resource, since namespace packages can't have them.
+ self.assertFalse(resources.is_resource(
+ 'test.test_importlib.data03.namespace',
+ 'resource1.txt'))
+ # We should get an exception if we try to read it or open it.
+ self.assertRaises(
+ FileNotFoundError,
+ resources.open_text,
+ 'test.test_importlib.data03.namespace', 'resource1.txt')
+ self.assertRaises(
+ FileNotFoundError,
+ resources.open_binary,
+ 'test.test_importlib.data03.namespace', 'resource1.txt')
+ self.assertRaises(
+ FileNotFoundError,
+ resources.read_text,
+ 'test.test_importlib.data03.namespace', 'resource1.txt')
+ self.assertRaises(
+ FileNotFoundError,
+ resources.read_binary,
+ 'test.test_importlib.data03.namespace', 'resource1.txt')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py
new file mode 100644
index 000000000..5a16a03de
--- /dev/null
+++ b/Lib/test/test_importlib/test_spec.py
@@ -0,0 +1,819 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+machinery = test_util.import_importlib('importlib.machinery')
+util = test_util.import_importlib('importlib.util')
+
+import os.path
+import pathlib
+from test.support import CleanImport
+import unittest
+import sys
+import warnings
+
+
+
+class TestLoader:
+
+ def __init__(self, path=None, is_package=None):
+ self.path = path
+ self.package = is_package
+
+ def __repr__(self):
+ return ''
+
+ def __getattr__(self, name):
+ if name == 'get_filename' and self.path is not None:
+ return self._get_filename
+ if name == 'is_package':
+ return self._is_package
+ raise AttributeError(name)
+
+ def _get_filename(self, name):
+ return self.path
+
+ def _is_package(self, name):
+ return self.package
+
+ def create_module(self, spec):
+ return None
+
+
+class NewLoader(TestLoader):
+
+ EGGS = 1
+
+ def exec_module(self, module):
+ module.eggs = self.EGGS
+
+
+class LegacyLoader(TestLoader):
+
+ HAM = -1
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+
+ frozen_util = util['Frozen']
+
+ @frozen_util.module_for_loader
+ def load_module(self, module):
+ module.ham = self.HAM
+ return module
+
+
+class ModuleSpecTests:
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+ self.loc_spec._set_fileattr = True
+
+ def test_default(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_no_loader(self):
+ spec = self.machinery.ModuleSpec(self.name, None)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertIs(spec.loader, None)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_is_package_false(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_is_package_true(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_has_location_setter(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin='somewhere')
+ self.assertFalse(spec.has_location)
+ spec.has_location = True
+ self.assertTrue(spec.has_location)
+
+ def test_equality(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ origin=None,
+ submodule_search_locations=None,
+ has_location=False,
+ cached=None,
+ )
+
+ self.assertTrue(self.spec == other)
+
+ def test_equality_location(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ origin=self.path,
+ submodule_search_locations=None,
+ has_location=True,
+ cached=self.cached,
+ )
+
+ self.assertEqual(self.loc_spec, other)
+
+ def test_inequality(self):
+ other = type(sys.implementation)(name='ham',
+ loader=self.loader,
+ origin=None,
+ submodule_search_locations=None,
+ has_location=False,
+ cached=None,
+ )
+
+ self.assertNotEqual(self.spec, other)
+
+ def test_inequality_incomplete(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ )
+
+ self.assertNotEqual(self.spec, other)
+
+ def test_package(self):
+ spec = self.machinery.ModuleSpec('spam.eggs', self.loader)
+
+ self.assertEqual(spec.parent, 'spam')
+
+ def test_package_is_package(self):
+ spec = self.machinery.ModuleSpec('spam.eggs', self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.parent, 'spam.eggs')
+
+ # cached
+
+ def test_cached_set(self):
+ before = self.spec.cached
+ self.spec.cached = 'there'
+ after = self.spec.cached
+
+ self.assertIs(before, None)
+ self.assertEqual(after, 'there')
+
+ def test_cached_no_origin(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+ self.assertIs(spec.cached, None)
+
+ def test_cached_with_origin_not_location(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+
+ self.assertIs(spec.cached, None)
+
+ def test_cached_source(self):
+ expected = self.util.cache_from_source(self.path)
+
+ self.assertEqual(self.loc_spec.cached, expected)
+
+ def test_cached_source_unknown_suffix(self):
+ self.loc_spec.origin = 'spam.spamspamspam'
+
+ self.assertIs(self.loc_spec.cached, None)
+
+ def test_cached_source_missing_cache_tag(self):
+ original = sys.implementation.cache_tag
+ sys.implementation.cache_tag = None
+ try:
+ cached = self.loc_spec.cached
+ finally:
+ sys.implementation.cache_tag = original
+
+ self.assertIs(cached, None)
+
+ def test_cached_sourceless(self):
+ self.loc_spec.origin = 'spam.pyc'
+
+ self.assertEqual(self.loc_spec.cached, 'spam.pyc')
+
+
+(Frozen_ModuleSpecTests,
+ Source_ModuleSpecTests
+ ) = test_util.test_both(ModuleSpecTests, util=util, machinery=machinery)
+
+
+class ModuleSpecMethodsTests:
+
+ @property
+ def bootstrap(self):
+ return self.init._bootstrap
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+ self.loc_spec._set_fileattr = True
+
+ # exec()
+
+ def test_exec(self):
+ self.spec.loader = NewLoader()
+ module = self.util.module_from_spec(self.spec)
+ sys.modules[self.name] = module
+ self.assertFalse(hasattr(module, 'eggs'))
+ self.bootstrap._exec(self.spec, module)
+
+ self.assertEqual(module.eggs, 1)
+
+ # load()
+
+ def test_load(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(loaded, installed)
+
+ def test_load_replaced(self):
+ replacement = object()
+ class ReplacingLoader(TestLoader):
+ def exec_module(self, module):
+ sys.modules[module.__name__] = replacement
+ self.spec.loader = ReplacingLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ installed = sys.modules[self.spec.name]
+
+ self.assertIs(loaded, replacement)
+ self.assertIs(installed, replacement)
+
+ def test_load_failed(self):
+ class FailedLoader(TestLoader):
+ def exec_module(self, module):
+ raise RuntimeError
+ self.spec.loader = FailedLoader()
+ with CleanImport(self.spec.name):
+ with self.assertRaises(RuntimeError):
+ loaded = self.bootstrap._load(self.spec)
+ self.assertNotIn(self.spec.name, sys.modules)
+
+ def test_load_failed_removed(self):
+ class FailedLoader(TestLoader):
+ def exec_module(self, module):
+ del sys.modules[module.__name__]
+ raise RuntimeError
+ self.spec.loader = FailedLoader()
+ with CleanImport(self.spec.name):
+ with self.assertRaises(RuntimeError):
+ loaded = self.bootstrap._load(self.spec)
+ self.assertNotIn(self.spec.name, sys.modules)
+
+ def test_load_legacy(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+
+ self.assertEqual(loaded.ham, -1)
+
+ def test_load_legacy_attributes(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+
+ self.assertIs(loaded.__loader__, self.spec.loader)
+ self.assertEqual(loaded.__package__, self.spec.parent)
+ self.assertIs(loaded.__spec__, self.spec)
+
+ def test_load_legacy_attributes_immutable(self):
+ module = object()
+ class ImmutableLoader(TestLoader):
+ def load_module(self, name):
+ sys.modules[name] = module
+ return module
+ self.spec.loader = ImmutableLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+
+ self.assertIs(sys.modules[self.spec.name], module)
+
+ # reload()
+
+ def test_reload(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ reloaded = self.bootstrap._exec(self.spec, loaded)
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(reloaded, loaded)
+ self.assertIs(installed, loaded)
+
+ def test_reload_modified(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ loaded.eggs = 2
+ reloaded = self.bootstrap._exec(self.spec, loaded)
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(reloaded, loaded)
+
+ def test_reload_extra_attributes(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ loaded.available = False
+ reloaded = self.bootstrap._exec(self.spec, loaded)
+
+ self.assertFalse(loaded.available)
+ self.assertIs(reloaded, loaded)
+
+ def test_reload_init_module_attrs(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ loaded.__name__ = 'ham'
+ del loaded.__loader__
+ del loaded.__package__
+ del loaded.__spec__
+ self.bootstrap._exec(self.spec, loaded)
+
+ self.assertEqual(loaded.__name__, self.spec.name)
+ self.assertIs(loaded.__loader__, self.spec.loader)
+ self.assertEqual(loaded.__package__, self.spec.parent)
+ self.assertIs(loaded.__spec__, self.spec)
+ self.assertFalse(hasattr(loaded, '__path__'))
+ self.assertFalse(hasattr(loaded, '__file__'))
+ self.assertFalse(hasattr(loaded, '__cached__'))
+
+ def test_reload_legacy(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._load(self.spec)
+ reloaded = self.bootstrap._exec(self.spec, loaded)
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.ham, -1)
+ self.assertIs(reloaded, loaded)
+ self.assertIs(installed, loaded)
+
+
+(Frozen_ModuleSpecMethodsTests,
+ Source_ModuleSpecMethodsTests
+ ) = test_util.test_both(ModuleSpecMethodsTests, init=init, util=util,
+ machinery=machinery)
+
+
+class ModuleReprTests:
+
+ @property
+ def bootstrap(self):
+ return self.init._bootstrap
+
+ def setUp(self):
+ self.module = type(os)('spam')
+ self.spec = self.machinery.ModuleSpec('spam', TestLoader())
+
+ def test_module___loader___module_repr(self):
+ class Loader:
+ def module_repr(self, module):
+ return ''.format(module.__name__)
+ self.module.__loader__ = Loader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '')
+
+ def test_module___loader___module_repr_bad(self):
+ class Loader(TestLoader):
+ def module_repr(self, module):
+ raise Exception
+ self.module.__loader__ = Loader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ ')>'.format('spam'))
+
+ def test_module___spec__(self):
+ origin = 'in a hole, in the ground'
+ self.spec.origin = origin
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, ''.format('spam', origin))
+
+ def test_module___spec___location(self):
+ location = 'in_a_galaxy_far_far_away.py'
+ self.spec.origin = location
+ self.spec._set_fileattr = True
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ ''.format('spam', location))
+
+ def test_module___spec___no_origin(self):
+ self.spec.loader = TestLoader()
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ ')>'.format('spam'))
+
+ def test_module___spec___no_origin_no_loader(self):
+ self.spec.loader = None
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, ''.format('spam'))
+
+ def test_module_no_name(self):
+ del self.module.__name__
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, ''.format('?'))
+
+ def test_module_with_file(self):
+ filename = 'e/i/e/i/o/spam.py'
+ self.module.__file__ = filename
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ ''.format('spam', filename))
+
+ def test_module_no_file(self):
+ self.module.__loader__ = TestLoader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ ')>'.format('spam'))
+
+ def test_module_no_file_no_loader(self):
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, ''.format('spam'))
+
+
+(Frozen_ModuleReprTests,
+ Source_ModuleReprTests
+ ) = test_util.test_both(ModuleReprTests, init=init, util=util,
+ machinery=machinery)
+
+
+class FactoryTests:
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.fileloader = TestLoader(self.path)
+ self.pkgloader = TestLoader(self.path, True)
+
+ # spec_from_loader()
+
+ def test_spec_from_loader_default(self):
+ spec = self.util.spec_from_loader(self.name, self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_default_with_bad_is_package(self):
+ class Loader:
+ def is_package(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_origin(self):
+ origin = 'somewhere over the rainbow'
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ origin=origin)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, origin)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_false(self):
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_true(self):
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_origin_and_is_package(self):
+ origin = 'where the streets have no name'
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ origin=origin, is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, origin)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_with_loader_false(self):
+ loader = TestLoader(is_package=False)
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_with_loader_true(self):
+ loader = TestLoader(is_package=True)
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_default_with_file_loader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_loader_is_package_false_with_fileloader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_loader_is_package_true_with_fileloader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ # spec_from_file_location()
+
+ def test_spec_from_file_location_default(self):
+ spec = self.util.spec_from_file_location(self.name, self.path)
+
+ self.assertEqual(spec.name, self.name)
+ # Need to use a circuitous route to get at importlib.machinery to make
+ # sure the same class object is used in the isinstance() check as
+ # would have been used to create the loader.
+ self.assertIsInstance(spec.loader,
+ self.util.abc.machinery.SourceFileLoader)
+ self.assertEqual(spec.loader.name, self.name)
+ self.assertEqual(spec.loader.path, self.path)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_path_like_arg(self):
+ spec = self.util.spec_from_file_location(self.name,
+ pathlib.PurePath(self.path))
+ self.assertEqual(spec.origin, self.path)
+
+ def test_spec_from_file_location_default_without_location(self):
+ spec = self.util.spec_from_file_location(self.name)
+
+ self.assertIs(spec, None)
+
+ def test_spec_from_file_location_default_bad_suffix(self):
+ spec = self.util.spec_from_file_location(self.name, 'spam.eggs')
+
+ self.assertIs(spec, None)
+
+ def test_spec_from_file_location_loader_no_location(self):
+ spec = self.util.spec_from_file_location(self.name,
+ loader=self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_loader_no_location_no_get_filename(self):
+ spec = self.util.spec_from_file_location(self.name,
+ loader=self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertEqual(spec.origin, '')
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_loader_no_location_bad_get_filename(self):
+ class Loader:
+ def get_filename(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, '')
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_none(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=None)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_empty(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=[])
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_not_empty(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=['eggs'])
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, ['eggs'])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.pkgloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.pkgloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_not_package(self):
+ class Loader:
+ def is_package(self, name):
+ return False
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_no_is_package(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_bad_is_package(self):
+ class Loader:
+ def is_package(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+
+(Frozen_FactoryTests,
+ Source_FactoryTests
+ ) = test_util.test_both(FactoryTests, util=util, machinery=machinery)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py
new file mode 100644
index 000000000..df7125bfb
--- /dev/null
+++ b/Lib/test/test_importlib/test_util.py
@@ -0,0 +1,888 @@
+from . import util
+abc = util.import_importlib('importlib.abc')
+init = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+importlib_util = util.import_importlib('importlib.util')
+
+import contextlib
+import importlib.util
+import os
+import pathlib
+import string
+import sys
+from test import support
+import types
+import unittest
+import unittest.mock
+import warnings
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class DecodeSourceBytesTests:
+
+ source = "string ='ü'"
+
+ def test_ut8_default(self):
+ source_bytes = self.source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes), self.source)
+
+ def test_specified_encoding(self):
+ source = '# coding=latin-1\n' + self.source
+ source_bytes = source.encode('latin-1')
+ assert source_bytes != source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes), source)
+
+ def test_universal_newlines(self):
+ source = '\r\n'.join([self.source, self.source])
+ source_bytes = source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes),
+ '\n'.join([self.source, self.source]))
+
+
+(Frozen_DecodeSourceBytesTests,
+ Source_DecodeSourceBytesTests
+ ) = util.test_both(DecodeSourceBytesTests, util=importlib_util)
+
+
+class ModuleFromSpecTests:
+
+ def test_no_create_module(self):
+ class Loader:
+ def exec_module(self, module):
+ pass
+ spec = self.machinery.ModuleSpec('test', Loader())
+ with self.assertRaises(ImportError):
+ module = self.util.module_from_spec(spec)
+
+ def test_create_module_returns_None(self):
+ class Loader(self.abc.Loader):
+ def create_module(self, spec):
+ return None
+ spec = self.machinery.ModuleSpec('test', Loader())
+ module = self.util.module_from_spec(spec)
+ self.assertIsInstance(module, types.ModuleType)
+ self.assertEqual(module.__name__, spec.name)
+
+ def test_create_module(self):
+ name = 'already set'
+ class CustomModule(types.ModuleType):
+ pass
+ class Loader(self.abc.Loader):
+ def create_module(self, spec):
+ module = CustomModule(spec.name)
+ module.__name__ = name
+ return module
+ spec = self.machinery.ModuleSpec('test', Loader())
+ module = self.util.module_from_spec(spec)
+ self.assertIsInstance(module, CustomModule)
+ self.assertEqual(module.__name__, name)
+
+ def test___name__(self):
+ spec = self.machinery.ModuleSpec('test', object())
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__name__, spec.name)
+
+ def test___spec__(self):
+ spec = self.machinery.ModuleSpec('test', object())
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__spec__, spec)
+
+ def test___loader__(self):
+ loader = object()
+ spec = self.machinery.ModuleSpec('test', loader)
+ module = self.util.module_from_spec(spec)
+ self.assertIs(module.__loader__, loader)
+
+ def test___package__(self):
+ spec = self.machinery.ModuleSpec('test.pkg', object())
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__package__, spec.parent)
+
+ def test___path__(self):
+ spec = self.machinery.ModuleSpec('test', object(), is_package=True)
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__path__, spec.submodule_search_locations)
+
+ def test___file__(self):
+ spec = self.machinery.ModuleSpec('test', object(), origin='some/path')
+ spec.has_location = True
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__file__, spec.origin)
+
+ def test___cached__(self):
+ spec = self.machinery.ModuleSpec('test', object())
+ spec.cached = 'some/path'
+ spec.has_location = True
+ module = self.util.module_from_spec(spec)
+ self.assertEqual(module.__cached__, spec.cached)
+
+(Frozen_ModuleFromSpecTests,
+ Source_ModuleFromSpecTests
+) = util.test_both(ModuleFromSpecTests, abc=abc, machinery=machinery,
+ util=importlib_util)
+
+
+class ModuleForLoaderTests:
+
+ """Tests for importlib.util.module_for_loader."""
+
+ @classmethod
+ def module_for_loader(cls, func):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return cls.util.module_for_loader(func)
+
+ def test_warning(self):
+ # Should raise a PendingDeprecationWarning when used.
+ with warnings.catch_warnings():
+ warnings.simplefilter('error', DeprecationWarning)
+ with self.assertRaises(DeprecationWarning):
+ func = self.util.module_for_loader(lambda x: x)
+
+ def return_module(self, name):
+ fxn = self.module_for_loader(lambda self, module: module)
+ return fxn(self, name)
+
+ def raise_exception(self, name):
+ def to_wrap(self, module):
+ raise ImportError
+ fxn = self.module_for_loader(to_wrap)
+ try:
+ fxn(self, name)
+ except ImportError:
+ pass
+
+ def test_new_module(self):
+ # Test that when no module exists in sys.modules a new module is
+ # created.
+ module_name = 'a.b.c'
+ with util.uncache(module_name):
+ module = self.return_module(module_name)
+ self.assertIn(module_name, sys.modules)
+ self.assertIsInstance(module, types.ModuleType)
+ self.assertEqual(module.__name__, module_name)
+
+ def test_reload(self):
+ # Test that a module is reused if already in sys.modules.
+ class FakeLoader:
+ def is_package(self, name):
+ return True
+ @self.module_for_loader
+ def load_module(self, module):
+ return module
+ name = 'a.b.c'
+ module = types.ModuleType('a.b.c')
+ module.__loader__ = 42
+ module.__package__ = 42
+ with util.uncache(name):
+ sys.modules[name] = module
+ loader = FakeLoader()
+ returned_module = loader.load_module(name)
+ self.assertIs(returned_module, sys.modules[name])
+ self.assertEqual(module.__loader__, loader)
+ self.assertEqual(module.__package__, name)
+
+ def test_new_module_failure(self):
+ # Test that a module is removed from sys.modules if added but an
+ # exception is raised.
+ name = 'a.b.c'
+ with util.uncache(name):
+ self.raise_exception(name)
+ self.assertNotIn(name, sys.modules)
+
+ def test_reload_failure(self):
+ # Test that a failure on reload leaves the module in-place.
+ name = 'a.b.c'
+ module = types.ModuleType(name)
+ with util.uncache(name):
+ sys.modules[name] = module
+ self.raise_exception(name)
+ self.assertIs(module, sys.modules[name])
+
+ def test_decorator_attrs(self):
+ def fxn(self, module): pass
+ wrapped = self.module_for_loader(fxn)
+ self.assertEqual(wrapped.__name__, fxn.__name__)
+ self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
+
+ def test_false_module(self):
+ # If for some odd reason a module is considered false, still return it
+ # from sys.modules.
+ class FalseModule(types.ModuleType):
+ def __bool__(self): return False
+
+ name = 'mod'
+ module = FalseModule(name)
+ with util.uncache(name):
+ self.assertFalse(module)
+ sys.modules[name] = module
+ given = self.return_module(name)
+ self.assertIs(given, module)
+
+ def test_attributes_set(self):
+ # __name__, __loader__, and __package__ should be set (when
+ # is_package() is defined; undefined implicitly tested elsewhere).
+ class FakeLoader:
+ def __init__(self, is_package):
+ self._pkg = is_package
+ def is_package(self, name):
+ return self._pkg
+ @self.module_for_loader
+ def load_module(self, module):
+ return module
+
+ name = 'pkg.mod'
+ with util.uncache(name):
+ loader = FakeLoader(False)
+ module = loader.load_module(name)
+ self.assertEqual(module.__name__, name)
+ self.assertIs(module.__loader__, loader)
+ self.assertEqual(module.__package__, 'pkg')
+
+ name = 'pkg.sub'
+ with util.uncache(name):
+ loader = FakeLoader(True)
+ module = loader.load_module(name)
+ self.assertEqual(module.__name__, name)
+ self.assertIs(module.__loader__, loader)
+ self.assertEqual(module.__package__, name)
+
+
+(Frozen_ModuleForLoaderTests,
+ Source_ModuleForLoaderTests
+ ) = util.test_both(ModuleForLoaderTests, util=importlib_util)
+
+
+class SetPackageTests:
+
+ """Tests for importlib.util.set_package."""
+
+ def verify(self, module, expect):
+ """Verify the module has the expected value for __package__ after
+ passing through set_package."""
+ fxn = lambda: module
+ wrapped = self.util.set_package(fxn)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ wrapped()
+ self.assertTrue(hasattr(module, '__package__'))
+ self.assertEqual(expect, module.__package__)
+
+ def test_top_level(self):
+ # __package__ should be set to the empty string if a top-level module.
+ # Implicitly tests when package is set to None.
+ module = types.ModuleType('module')
+ module.__package__ = None
+ self.verify(module, '')
+
+ def test_package(self):
+ # Test setting __package__ for a package.
+ module = types.ModuleType('pkg')
+ module.__path__ = ['']
+ module.__package__ = None
+ self.verify(module, 'pkg')
+
+ def test_submodule(self):
+ # Test __package__ for a module in a package.
+ module = types.ModuleType('pkg.mod')
+ module.__package__ = None
+ self.verify(module, 'pkg')
+
+ def test_setting_if_missing(self):
+ # __package__ should be set if it is missing.
+ module = types.ModuleType('mod')
+ if hasattr(module, '__package__'):
+ delattr(module, '__package__')
+ self.verify(module, '')
+
+ def test_leaving_alone(self):
+ # If __package__ is set and not None then leave it alone.
+ for value in (True, False):
+ module = types.ModuleType('mod')
+ module.__package__ = value
+ self.verify(module, value)
+
+ def test_decorator_attrs(self):
+ def fxn(module): pass
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ wrapped = self.util.set_package(fxn)
+ self.assertEqual(wrapped.__name__, fxn.__name__)
+ self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
+
+
+(Frozen_SetPackageTests,
+ Source_SetPackageTests
+ ) = util.test_both(SetPackageTests, util=importlib_util)
+
+
+class SetLoaderTests:
+
+ """Tests importlib.util.set_loader()."""
+
+ @property
+ def DummyLoader(self):
+ # Set DummyLoader on the class lazily.
+ class DummyLoader:
+ @self.util.set_loader
+ def load_module(self, module):
+ return self.module
+ self.__class__.DummyLoader = DummyLoader
+ return DummyLoader
+
+ def test_no_attribute(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ try:
+ del loader.module.__loader__
+ except AttributeError:
+ pass
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(loader, loader.load_module('blah').__loader__)
+
+ def test_attribute_is_None(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ loader.module.__loader__ = None
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(loader, loader.load_module('blah').__loader__)
+
+ def test_not_reset(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ loader.module.__loader__ = 42
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(42, loader.load_module('blah').__loader__)
+
+
+(Frozen_SetLoaderTests,
+ Source_SetLoaderTests
+ ) = util.test_both(SetLoaderTests, util=importlib_util)
+
+
+class ResolveNameTests:
+
+ """Tests importlib.util.resolve_name()."""
+
+ def test_absolute(self):
+ # bacon
+ self.assertEqual('bacon', self.util.resolve_name('bacon', None))
+
+ def test_absolute_within_package(self):
+ # bacon in spam
+ self.assertEqual('bacon', self.util.resolve_name('bacon', 'spam'))
+
+ def test_no_package(self):
+ # .bacon in ''
+ with self.assertRaises(ValueError):
+ self.util.resolve_name('.bacon', '')
+
+ def test_in_package(self):
+ # .bacon in spam
+ self.assertEqual('spam.eggs.bacon',
+ self.util.resolve_name('.bacon', 'spam.eggs'))
+
+ def test_other_package(self):
+ # ..bacon in spam.bacon
+ self.assertEqual('spam.bacon',
+ self.util.resolve_name('..bacon', 'spam.eggs'))
+
+ def test_escape(self):
+ # ..bacon in spam
+ with self.assertRaises(ValueError):
+ self.util.resolve_name('..bacon', 'spam')
+
+
+(Frozen_ResolveNameTests,
+ Source_ResolveNameTests
+ ) = util.test_both(ResolveNameTests, util=importlib_util)
+
+
+class FindSpecTests:
+
+ class FakeMetaFinder:
+ @staticmethod
+ def find_spec(name, path=None, target=None): return name, path, target
+
+ def test_sys_modules(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ loader = 'a loader!'
+ spec = self.machinery.ModuleSpec(name, loader)
+ module.__loader__ = loader
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+
+ def test_sys_modules_without___loader__(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ del module.__loader__
+ loader = 'a loader!'
+ spec = self.machinery.ModuleSpec(name, loader)
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+
+ def test_sys_modules_spec_is_None(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ module.__spec__ = None
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ self.util.find_spec(name)
+
+ def test_sys_modules_loader_is_None(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ spec = self.machinery.ModuleSpec(name, None)
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+
+ def test_sys_modules_spec_is_not_set(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ try:
+ del module.__spec__
+ except AttributeError:
+ pass
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ self.util.find_spec(name)
+
+ def test_success(self):
+ name = 'some_mod'
+ with util.uncache(name):
+ with util.import_state(meta_path=[self.FakeMetaFinder]):
+ self.assertEqual((name, None, None),
+ self.util.find_spec(name))
+
+ def test_nothing(self):
+ # None is returned upon failure to find a loader.
+ self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule'))
+
+ def test_find_submodule(self):
+ name = 'spam'
+ subname = 'ham'
+ with util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = util.submodule(name, subname, pkg_dir)
+ spec = self.util.find_spec(fullname)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_submodule_parent_already_imported(self):
+ name = 'spam'
+ subname = 'ham'
+ with util.temp_module(name, pkg=True) as pkg_dir:
+ self.init.import_module(name)
+ fullname, _ = util.submodule(name, subname, pkg_dir)
+ spec = self.util.find_spec(fullname)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_relative_module(self):
+ name = 'spam'
+ subname = 'ham'
+ with util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = util.submodule(name, subname, pkg_dir)
+ relname = '.' + subname
+ spec = self.util.find_spec(relname, name)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_relative_module_missing_package(self):
+ name = 'spam'
+ subname = 'ham'
+ with util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = util.submodule(name, subname, pkg_dir)
+ relname = '.' + subname
+ with self.assertRaises(ValueError):
+ self.util.find_spec(relname)
+ self.assertNotIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+
+ def test_find_submodule_in_module(self):
+ # ModuleNotFoundError raised when a module is specified as
+ # a parent instead of a package.
+ with self.assertRaises(ModuleNotFoundError):
+ self.util.find_spec('module.name')
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, init=init, util=importlib_util,
+ machinery=machinery)
+
+
+class MagicNumberTests:
+
+ def test_length(self):
+ # Should be 4 bytes.
+ self.assertEqual(len(self.util.MAGIC_NUMBER), 4)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_incorporates_rn(self):
+ # The magic number uses \r\n to come out wrong when splitting on lines.
+ self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n'))
+
+
+(Frozen_MagicNumberTests,
+ Source_MagicNumberTests
+ ) = util.test_both(MagicNumberTests, util=importlib_util)
+
+
+class PEP3147Tests:
+
+ """Tests of PEP 3147-related functions: cache_from_source and source_from_cache."""
+
+ tag = sys.implementation.cache_tag
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag not be None')
+ def test_cache_from_source(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyc file (i.e. under __pycache__).
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ def test_cache_from_source_no_cache_tag(self):
+ # No cache tag means NotImplementedError.
+ with support.swap_attr(sys.implementation, 'cache_tag', None):
+ with self.assertRaises(NotImplementedError):
+ self.util.cache_from_source('whatever.py')
+
+ def test_cache_from_source_no_dot(self):
+ # Directory with a dot, filename without dot.
+ path = os.path.join('foo.bar', 'file')
+ expect = os.path.join('foo.bar', '__pycache__',
+ 'file{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_cache_from_source_debug_override(self):
+ # Given the path to a .py file, return the path to its PEP 3147/PEP 488
+ # defined .pyc file (i.e. under __pycache__).
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ self.assertEqual(self.util.cache_from_source(path, False),
+ self.util.cache_from_source(path, optimization=1))
+ self.assertEqual(self.util.cache_from_source(path, True),
+ self.util.cache_from_source(path, optimization=''))
+ with warnings.catch_warnings():
+ warnings.simplefilter('error')
+ with self.assertRaises(DeprecationWarning):
+ self.util.cache_from_source(path, False)
+ with self.assertRaises(DeprecationWarning):
+ self.util.cache_from_source(path, True)
+
+ def test_cache_from_source_cwd(self):
+ path = 'foo.py'
+ expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ def test_cache_from_source_override(self):
+ # When debug_override is not None, it can be any true-ish or false-ish
+ # value.
+ path = os.path.join('foo', 'bar', 'baz.py')
+ # However if the bool-ishness can't be determined, the exception
+ # propagates.
+ class Bearish:
+ def __bool__(self): raise RuntimeError
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ self.assertEqual(self.util.cache_from_source(path, []),
+ self.util.cache_from_source(path, optimization=1))
+ self.assertEqual(self.util.cache_from_source(path, [17]),
+ self.util.cache_from_source(path, optimization=''))
+ with self.assertRaises(RuntimeError):
+ self.util.cache_from_source('/foo/bar/baz.py', Bearish())
+
+
+ def test_cache_from_source_optimization_empty_string(self):
+ # Setting 'optimization' to '' leads to no optimization tag (PEP 488).
+ path = 'foo.py'
+ expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ def test_cache_from_source_optimization_None(self):
+ # Setting 'optimization' to None uses the interpreter's optimization.
+ # (PEP 488)
+ path = 'foo.py'
+ optimization_level = sys.flags.optimize
+ almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag))
+ if optimization_level == 0:
+ expect = almost_expect + '.pyc'
+ elif optimization_level <= 2:
+ expect = almost_expect + '.opt-{}.pyc'.format(optimization_level)
+ else:
+ msg = '{!r} is a non-standard optimization level'.format(optimization_level)
+ self.skipTest(msg)
+ self.assertEqual(self.util.cache_from_source(path, optimization=None),
+ expect)
+
+ def test_cache_from_source_optimization_set(self):
+ # The 'optimization' parameter accepts anything that has a string repr
+ # that passes str.alnum().
+ path = 'foo.py'
+ valid_characters = string.ascii_letters + string.digits
+ almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag))
+ got = self.util.cache_from_source(path, optimization=valid_characters)
+ # Test all valid characters are accepted.
+ self.assertEqual(got,
+ almost_expect + '.opt-{}.pyc'.format(valid_characters))
+ # str() should be called on argument.
+ self.assertEqual(self.util.cache_from_source(path, optimization=42),
+ almost_expect + '.opt-42.pyc')
+ # Invalid characters raise ValueError.
+ with self.assertRaises(ValueError):
+ self.util.cache_from_source(path, optimization='path/is/bad')
+
+ def test_cache_from_source_debug_override_optimization_both_set(self):
+ # Can only set one of the optimization-related parameters.
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ with self.assertRaises(TypeError):
+ self.util.cache_from_source('foo.py', False, optimization='')
+
+ @unittest.skipUnless(os.sep == '\\' and os.altsep == '/',
+ 'test meaningful only where os.altsep is defined')
+ def test_sep_altsep_and_sep_cache_from_source(self):
+ # Windows path and PEP 3147 where sep is right of altsep.
+ self.assertEqual(
+ self.util.cache_from_source('\\foo\\bar\\baz/qux.py', optimization=''),
+ '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag not be None')
+ def test_cache_from_source_path_like_arg(self):
+ path = pathlib.PurePath('foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_source_from_cache(self):
+ # Given the path to a PEP 3147 defined .pyc file, return the path to
+ # its source. This tests the good path.
+ path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ self.assertEqual(self.util.source_from_cache(path), expect)
+
+ def test_source_from_cache_no_cache_tag(self):
+ # If sys.implementation.cache_tag is None, raise NotImplementedError.
+ path = os.path.join('blah', '__pycache__', 'whatever.pyc')
+ with support.swap_attr(sys.implementation, 'cache_tag', None):
+ with self.assertRaises(NotImplementedError):
+ self.util.source_from_cache(path)
+
+ def test_source_from_cache_bad_path(self):
+ # When the path to a pyc file is not in PEP 3147 format, a ValueError
+ # is raised.
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, '/foo/bar/bazqux.pyc')
+
+ def test_source_from_cache_no_slash(self):
+ # No slashes at all in path -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, 'foo.cpython-32.pyc')
+
+ def test_source_from_cache_too_few_dots(self):
+ # Too few dots in final path component -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, '__pycache__/foo.pyc')
+
+ def test_source_from_cache_too_many_dots(self):
+ with self.assertRaises(ValueError):
+ self.util.source_from_cache(
+ '__pycache__/foo.cpython-32.opt-1.foo.pyc')
+
+ def test_source_from_cache_not_opt(self):
+ # Non-`opt-` path component -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache,
+ '__pycache__/foo.cpython-32.foo.pyc')
+
+ def test_source_from_cache_no__pycache__(self):
+ # Another problem with the path -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache,
+ '/foo/bar/foo.cpython-32.foo.pyc')
+
+ def test_source_from_cache_optimized_bytecode(self):
+ # Optimized bytecode is not an issue.
+ path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag))
+ self.assertEqual(self.util.source_from_cache(path), 'foo.py')
+
+ def test_source_from_cache_missing_optimization(self):
+ # An empty optimization level is a no-no.
+ path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag))
+ with self.assertRaises(ValueError):
+ self.util.source_from_cache(path)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_source_from_cache_path_like_arg(self):
+ path = pathlib.PurePath('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ self.assertEqual(self.util.source_from_cache(path), expect)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_cache_from_source_respects_pycache_prefix(self):
+ # If pycache_prefix is set, cache_from_source will return a bytecode
+ # path inside that directory (in a subdirectory mirroring the .py file's
+ # path) rather than in a __pycache__ dir next to the py file.
+ pycache_prefixes = [
+ os.path.join(os.path.sep, 'tmp', 'bytecode'),
+ os.path.join(os.path.sep, 'tmp', '\u2603'), # non-ASCII in path!
+ os.path.join(os.path.sep, 'tmp', 'trailing-slash') + os.path.sep,
+ ]
+ drive = ''
+ if os.name == 'nt':
+ drive = 'C:'
+ pycache_prefixes = [
+ f'{drive}{prefix}' for prefix in pycache_prefixes]
+ pycache_prefixes += [r'\\?\C:\foo', r'\\localhost\c$\bar']
+ for pycache_prefix in pycache_prefixes:
+ with self.subTest(path=pycache_prefix):
+ path = drive + os.path.join(
+ os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join(
+ pycache_prefix, 'foo', 'bar', 'baz',
+ 'qux.{}.pyc'.format(self.tag))
+ with util.temporary_pycache_prefix(pycache_prefix):
+ self.assertEqual(
+ self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_cache_from_source_respects_pycache_prefix_relative(self):
+ # If the .py path we are given is relative, we will resolve to an
+ # absolute path before prefixing with pycache_prefix, to avoid any
+ # possible ambiguity.
+ pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ root = os.path.splitdrive(os.getcwd())[0] + os.path.sep
+ expect = os.path.join(
+ pycache_prefix,
+ os.path.relpath(os.getcwd(), root),
+ 'foo', 'bar', 'baz', f'qux.{self.tag}.pyc')
+ with util.temporary_pycache_prefix(pycache_prefix):
+ self.assertEqual(
+ self.util.cache_from_source(path, optimization=''),
+ expect)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_source_from_cache_inside_pycache_prefix(self):
+ # If pycache_prefix is set and the cache path we get is inside it,
+ # we return an absolute path to the py file based on the remainder of
+ # the path within pycache_prefix.
+ pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+ path = os.path.join(pycache_prefix, 'foo', 'bar', 'baz',
+ f'qux.{self.tag}.pyc')
+ expect = os.path.join(os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
+ with util.temporary_pycache_prefix(pycache_prefix):
+ self.assertEqual(self.util.source_from_cache(path), expect)
+
+ @unittest.skipIf(sys.implementation.cache_tag is None,
+ 'requires sys.implementation.cache_tag to not be None')
+ def test_source_from_cache_outside_pycache_prefix(self):
+ # If pycache_prefix is set but the cache path we get is not inside
+ # it, just ignore it and handle the cache path according to the default
+ # behavior.
+ pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+ path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ f'qux.{self.tag}.pyc')
+ expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ with util.temporary_pycache_prefix(pycache_prefix):
+ self.assertEqual(self.util.source_from_cache(path), expect)
+
+
+(Frozen_PEP3147Tests,
+ Source_PEP3147Tests
+ ) = util.test_both(PEP3147Tests, util=importlib_util)
+
+
+class MagicNumberTests(unittest.TestCase):
+ """
+ Test release compatibility issues relating to importlib
+ """
+ @unittest.skipUnless(
+ sys.version_info.releaselevel in ('candidate', 'final'),
+ 'only applies to candidate or final python release levels'
+ )
+ def test_magic_number(self):
+ """
+ Each python minor release should generally have a MAGIC_NUMBER
+ that does not change once the release reaches candidate status.
+
+ Once a release reaches candidate status, the value of the constant
+ EXPECTED_MAGIC_NUMBER in this test should be changed.
+ This test will then check that the actual MAGIC_NUMBER matches
+ the expected value for the release.
+
+ In exceptional cases, it may be required to change the MAGIC_NUMBER
+ for a maintenance release. In this case the change should be
+ discussed in python-dev. If a change is required, community
+ stakeholders such as OS package maintainers must be notified
+ in advance. Such exceptional releases will then require an
+ adjustment to this test case.
+ """
+ EXPECTED_MAGIC_NUMBER = 3410
+ actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little')
+
+ msg = (
+ "To avoid breaking backwards compatibility with cached bytecode "
+ "files that can't be automatically regenerated by the current "
+ "user, candidate and final releases require the current "
+ "importlib.util.MAGIC_NUMBER to match the expected "
+ "magic number in this test. Set the expected "
+ "magic number in this test to the current MAGIC_NUMBER to "
+ "continue with the release.\n\n"
+ "Changing the MAGIC_NUMBER for a maintenance release "
+ "requires discussion in python-dev and notification of "
+ "community stakeholders."
+ )
+ self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py
new file mode 100644
index 000000000..005b685cc
--- /dev/null
+++ b/Lib/test/test_importlib/test_windows.py
@@ -0,0 +1,109 @@
+from . import util as test_util
+machinery = test_util.import_importlib('importlib.machinery')
+
+import os
+import re
+import sys
+import unittest
+from test import support
+from distutils.util import get_platform
+from contextlib import contextmanager
+from .util import temp_module
+
+support.import_module('winreg', required_on=['win'])
+from winreg import (
+ CreateKey, HKEY_CURRENT_USER,
+ SetValue, REG_SZ, KEY_ALL_ACCESS,
+ EnumKey, CloseKey, DeleteKey, OpenKey
+)
+
+def delete_registry_tree(root, subkey):
+ try:
+ hkey = OpenKey(root, subkey, access=KEY_ALL_ACCESS)
+ except OSError:
+ # subkey does not exist
+ return
+ while True:
+ try:
+ subsubkey = EnumKey(hkey, 0)
+ except OSError:
+ # no more subkeys
+ break
+ delete_registry_tree(hkey, subsubkey)
+ CloseKey(hkey)
+ DeleteKey(root, subkey)
+
+@contextmanager
+def setup_module(machinery, name, path=None):
+ if machinery.WindowsRegistryFinder.DEBUG_BUILD:
+ root = machinery.WindowsRegistryFinder.REGISTRY_KEY_DEBUG
+ else:
+ root = machinery.WindowsRegistryFinder.REGISTRY_KEY
+ key = root.format(fullname=name,
+ sys_version='%d.%d' % sys.version_info[:2])
+ try:
+ with temp_module(name, "a = 1") as location:
+ subkey = CreateKey(HKEY_CURRENT_USER, key)
+ if path is None:
+ path = location + ".py"
+ SetValue(subkey, "", REG_SZ, path)
+ yield
+ finally:
+ if machinery.WindowsRegistryFinder.DEBUG_BUILD:
+ key = os.path.dirname(key)
+ delete_registry_tree(HKEY_CURRENT_USER, key)
+
+
+@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows')
+class WindowsRegistryFinderTests:
+ # The module name is process-specific, allowing for
+ # simultaneous runs of the same test on a single machine.
+ test_module = "spamham{}".format(os.getpid())
+
+ def test_find_spec_missing(self):
+ spec = self.machinery.WindowsRegistryFinder.find_spec('spam')
+ self.assertIs(spec, None)
+
+ def test_find_module_missing(self):
+ loader = self.machinery.WindowsRegistryFinder.find_module('spam')
+ self.assertIs(loader, None)
+
+ def test_module_found(self):
+ with setup_module(self.machinery, self.test_module):
+ loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module)
+ spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module)
+ self.assertIsNot(loader, None)
+ self.assertIsNot(spec, None)
+
+ def test_module_not_found(self):
+ with setup_module(self.machinery, self.test_module, path="."):
+ loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module)
+ spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module)
+ self.assertIsNone(loader)
+ self.assertIsNone(spec)
+
+(Frozen_WindowsRegistryFinderTests,
+ Source_WindowsRegistryFinderTests
+ ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery)
+
+@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows')
+class WindowsExtensionSuffixTests:
+ def test_tagged_suffix(self):
+ suffixes = self.machinery.EXTENSION_SUFFIXES
+ expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info,
+ re.sub('[^a-zA-Z0-9]', '_', get_platform()))
+ try:
+ untagged_i = suffixes.index(".pyd")
+ except ValueError:
+ untagged_i = suffixes.index("_d.pyd")
+ expected_tag = "_d" + expected_tag
+
+ self.assertIn(expected_tag, suffixes)
+
+ # Ensure the tags are in the correct order
+ tagged_i = suffixes.index(expected_tag)
+ self.assertLess(tagged_i, untagged_i)
+
+(Frozen_WindowsExtensionSuffixTests,
+ Source_WindowsExtensionSuffixTests
+ ) = test_util.test_both(WindowsExtensionSuffixTests, machinery=machinery)
diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py
new file mode 100644
index 000000000..6ecf4f419
--- /dev/null
+++ b/Lib/test/test_importlib/util.py
@@ -0,0 +1,575 @@
+import abc
+import builtins
+import contextlib
+import errno
+import functools
+import importlib
+from importlib import machinery, util, invalidate_caches
+from importlib.abc import ResourceReader
+import io
+import os
+import os.path
+from pathlib import Path, PurePath
+from test import support
+import unittest
+import sys
+import tempfile
+import types
+
+from . import data01
+from . import zipdata01
+
+
+BUILTINS = types.SimpleNamespace()
+BUILTINS.good_name = None
+BUILTINS.bad_name = None
+if 'errno' in sys.builtin_module_names:
+ BUILTINS.good_name = 'errno'
+if 'importlib' not in sys.builtin_module_names:
+ BUILTINS.bad_name = 'importlib'
+
+EXTENSIONS = types.SimpleNamespace()
+EXTENSIONS.path = None
+EXTENSIONS.ext = None
+EXTENSIONS.filename = None
+EXTENSIONS.file_path = None
+EXTENSIONS.name = '_testcapi'
+
+def _extension_details():
+ global EXTENSIONS
+ for path in sys.path:
+ for ext in machinery.EXTENSION_SUFFIXES:
+ filename = EXTENSIONS.name + ext
+ file_path = os.path.join(path, filename)
+ if os.path.exists(file_path):
+ EXTENSIONS.path = path
+ EXTENSIONS.ext = ext
+ EXTENSIONS.filename = filename
+ EXTENSIONS.file_path = file_path
+ return
+
+_extension_details()
+
+
+def import_importlib(module_name):
+ """Import a module from importlib both w/ and w/o _frozen_importlib."""
+ fresh = ('importlib',) if '.' in module_name else ()
+ frozen = support.import_fresh_module(module_name)
+ source = support.import_fresh_module(module_name, fresh=fresh,
+ blocked=('_frozen_importlib', '_frozen_importlib_external'))
+ return {'Frozen': frozen, 'Source': source}
+
+
+def specialize_class(cls, kind, base=None, **kwargs):
+ # XXX Support passing in submodule names--load (and cache) them?
+ # That would clean up the test modules a bit more.
+ if base is None:
+ base = unittest.TestCase
+ elif not isinstance(base, type):
+ base = base[kind]
+ name = '{}_{}'.format(kind, cls.__name__)
+ bases = (cls, base)
+ specialized = types.new_class(name, bases)
+ specialized.__module__ = cls.__module__
+ specialized._NAME = cls.__name__
+ specialized._KIND = kind
+ for attr, values in kwargs.items():
+ value = values[kind]
+ setattr(specialized, attr, value)
+ return specialized
+
+
+def split_frozen(cls, base=None, **kwargs):
+ frozen = specialize_class(cls, 'Frozen', base, **kwargs)
+ source = specialize_class(cls, 'Source', base, **kwargs)
+ return frozen, source
+
+
+def test_both(test_class, base=None, **kwargs):
+ return split_frozen(test_class, base, **kwargs)
+
+
+CASE_INSENSITIVE_FS = True
+# Windows is the only OS that is *always* case-insensitive
+# (OS X *can* be case-sensitive).
+if sys.platform not in ('win32', 'cygwin'):
+ changed_name = __file__.upper()
+ if changed_name == __file__:
+ changed_name = __file__.lower()
+ if not os.path.exists(changed_name):
+ CASE_INSENSITIVE_FS = False
+
+source_importlib = import_importlib('importlib')['Source']
+__import__ = {'Frozen': staticmethod(builtins.__import__),
+ 'Source': staticmethod(source_importlib.__import__)}
+
+
+def case_insensitive_tests(test):
+ """Class decorator that nullifies tests requiring a case-insensitive
+ file system."""
+ return unittest.skipIf(not CASE_INSENSITIVE_FS,
+ "requires a case-insensitive filesystem")(test)
+
+
+def submodule(parent, name, pkg_dir, content=''):
+ path = os.path.join(pkg_dir, name + '.py')
+ with open(path, 'w') as subfile:
+ subfile.write(content)
+ return '{}.{}'.format(parent, name), path
+
+
+@contextlib.contextmanager
+def uncache(*names):
+ """Uncache a module from sys.modules.
+
+ A basic sanity check is performed to prevent uncaching modules that either
+ cannot/shouldn't be uncached.
+
+ """
+ for name in names:
+ if name in ('sys', 'marshal', 'imp'):
+ raise ValueError(
+ "cannot uncache {0}".format(name))
+ try:
+ del sys.modules[name]
+ except KeyError:
+ pass
+ try:
+ yield
+ finally:
+ for name in names:
+ try:
+ del sys.modules[name]
+ except KeyError:
+ pass
+
+
+@contextlib.contextmanager
+def temp_module(name, content='', *, pkg=False):
+ conflicts = [n for n in sys.modules if n.partition('.')[0] == name]
+ with support.temp_cwd(None) as cwd:
+ with uncache(name, *conflicts):
+ with support.DirsOnSysPath(cwd):
+ invalidate_caches()
+
+ location = os.path.join(cwd, name)
+ if pkg:
+ modpath = os.path.join(location, '__init__.py')
+ os.mkdir(name)
+ else:
+ modpath = location + '.py'
+ if content is None:
+ # Make sure the module file gets created.
+ content = ''
+ if content is not None:
+ # not a namespace package
+ with open(modpath, 'w') as modfile:
+ modfile.write(content)
+ yield location
+
+
+@contextlib.contextmanager
+def import_state(**kwargs):
+ """Context manager to manage the various importers and stored state in the
+ sys module.
+
+ The 'modules' attribute is not supported as the interpreter state stores a
+ pointer to the dict that the interpreter uses internally;
+ reassigning to sys.modules does not have the desired effect.
+
+ """
+ originals = {}
+ try:
+ for attr, default in (('meta_path', []), ('path', []),
+ ('path_hooks', []),
+ ('path_importer_cache', {})):
+ originals[attr] = getattr(sys, attr)
+ if attr in kwargs:
+ new_value = kwargs[attr]
+ del kwargs[attr]
+ else:
+ new_value = default
+ setattr(sys, attr, new_value)
+ if len(kwargs):
+ raise ValueError(
+ 'unrecognized arguments: {0}'.format(kwargs.keys()))
+ yield
+ finally:
+ for attr, value in originals.items():
+ setattr(sys, attr, value)
+
+
+class _ImporterMock:
+
+ """Base class to help with creating importer mocks."""
+
+ def __init__(self, *names, module_code={}):
+ self.modules = {}
+ self.module_code = {}
+ for name in names:
+ if not name.endswith('.__init__'):
+ import_name = name
+ else:
+ import_name = name[:-len('.__init__')]
+ if '.' not in name:
+ package = None
+ elif import_name == name:
+ package = name.rsplit('.', 1)[0]
+ else:
+ package = import_name
+ module = types.ModuleType(import_name)
+ module.__loader__ = self
+ module.__file__ = ''
+ module.__package__ = package
+ module.attr = name
+ if import_name != name:
+ module.__path__ = ['']
+ self.modules[import_name] = module
+ if import_name in module_code:
+ self.module_code[import_name] = module_code[import_name]
+
+ def __getitem__(self, name):
+ return self.modules[name]
+
+ def __enter__(self):
+ self._uncache = uncache(*self.modules.keys())
+ self._uncache.__enter__()
+ return self
+
+ def __exit__(self, *exc_info):
+ self._uncache.__exit__(None, None, None)
+
+
+class mock_modules(_ImporterMock):
+
+ """Importer mock using PEP 302 APIs."""
+
+ def find_module(self, fullname, path=None):
+ if fullname not in self.modules:
+ return None
+ else:
+ return self
+
+ def load_module(self, fullname):
+ if fullname not in self.modules:
+ raise ImportError
+ else:
+ sys.modules[fullname] = self.modules[fullname]
+ if fullname in self.module_code:
+ try:
+ self.module_code[fullname]()
+ except Exception:
+ del sys.modules[fullname]
+ raise
+ return self.modules[fullname]
+
+
+class mock_spec(_ImporterMock):
+
+ """Importer mock using PEP 451 APIs."""
+
+ def find_spec(self, fullname, path=None, parent=None):
+ try:
+ module = self.modules[fullname]
+ except KeyError:
+ return None
+ spec = util.spec_from_file_location(
+ fullname, module.__file__, loader=self,
+ submodule_search_locations=getattr(module, '__path__', None))
+ return spec
+
+ def create_module(self, spec):
+ if spec.name not in self.modules:
+ raise ImportError
+ return self.modules[spec.name]
+
+ def exec_module(self, module):
+ try:
+ self.module_code[module.__spec__.name]()
+ except KeyError:
+ pass
+
+
+def writes_bytecode_files(fxn):
+ """Decorator to protect sys.dont_write_bytecode from mutation and to skip
+ tests that require it to be set to False."""
+ if sys.dont_write_bytecode:
+ return lambda *args, **kwargs: None
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ original = sys.dont_write_bytecode
+ sys.dont_write_bytecode = False
+ try:
+ to_return = fxn(*args, **kwargs)
+ finally:
+ sys.dont_write_bytecode = original
+ return to_return
+ return wrapper
+
+
+def ensure_bytecode_path(bytecode_path):
+ """Ensure that the __pycache__ directory for PEP 3147 pyc file exists.
+
+ :param bytecode_path: File system path to PEP 3147 pyc file.
+ """
+ try:
+ os.mkdir(os.path.dirname(bytecode_path))
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+
+
+@contextlib.contextmanager
+def temporary_pycache_prefix(prefix):
+ """Adjust and restore sys.pycache_prefix."""
+ _orig_prefix = sys.pycache_prefix
+ sys.pycache_prefix = prefix
+ try:
+ yield
+ finally:
+ sys.pycache_prefix = _orig_prefix
+
+
+@contextlib.contextmanager
+def create_modules(*names):
+ """Temporarily create each named module with an attribute (named 'attr')
+ that contains the name passed into the context manager that caused the
+ creation of the module.
+
+ All files are created in a temporary directory returned by
+ tempfile.mkdtemp(). This directory is inserted at the beginning of
+ sys.path. When the context manager exits all created files (source and
+ bytecode) are explicitly deleted.
+
+ No magic is performed when creating packages! This means that if you create
+ a module within a package you must also create the package's __init__ as
+ well.
+
+ """
+ source = 'attr = {0!r}'
+ created_paths = []
+ mapping = {}
+ state_manager = None
+ uncache_manager = None
+ try:
+ temp_dir = tempfile.mkdtemp()
+ mapping['.root'] = temp_dir
+ import_names = set()
+ for name in names:
+ if not name.endswith('__init__'):
+ import_name = name
+ else:
+ import_name = name[:-len('.__init__')]
+ import_names.add(import_name)
+ if import_name in sys.modules:
+ del sys.modules[import_name]
+ name_parts = name.split('.')
+ file_path = temp_dir
+ for directory in name_parts[:-1]:
+ file_path = os.path.join(file_path, directory)
+ if not os.path.exists(file_path):
+ os.mkdir(file_path)
+ created_paths.append(file_path)
+ file_path = os.path.join(file_path, name_parts[-1] + '.py')
+ with open(file_path, 'w') as file:
+ file.write(source.format(name))
+ created_paths.append(file_path)
+ mapping[name] = file_path
+ uncache_manager = uncache(*import_names)
+ uncache_manager.__enter__()
+ state_manager = import_state(path=[temp_dir])
+ state_manager.__enter__()
+ yield mapping
+ finally:
+ if state_manager is not None:
+ state_manager.__exit__(None, None, None)
+ if uncache_manager is not None:
+ uncache_manager.__exit__(None, None, None)
+ support.rmtree(temp_dir)
+
+
+def mock_path_hook(*entries, importer):
+ """A mock sys.path_hooks entry."""
+ def hook(entry):
+ if entry not in entries:
+ raise ImportError
+ return importer
+ return hook
+
+
+class CASEOKTestBase:
+
+ def caseok_env_changed(self, *, should_exist):
+ possibilities = b'PYTHONCASEOK', 'PYTHONCASEOK'
+ if any(x in self.importlib._bootstrap_external._os.environ
+ for x in possibilities) != should_exist:
+ self.skipTest('os.environ changes not reflected in _os.environ')
+
+
+def create_package(file, path, is_package=True, contents=()):
+ class Reader(ResourceReader):
+ def get_resource_reader(self, package):
+ return self
+
+ def open_resource(self, path):
+ self._path = path
+ if isinstance(file, Exception):
+ raise file
+ else:
+ return file
+
+ def resource_path(self, path_):
+ self._path = path_
+ if isinstance(path, Exception):
+ raise path
+ else:
+ return path
+
+ def is_resource(self, path_):
+ self._path = path_
+ if isinstance(path, Exception):
+ raise path
+ for entry in contents:
+ parts = entry.split('/')
+ if len(parts) == 1 and parts[0] == path_:
+ return True
+ return False
+
+ def contents(self):
+ if isinstance(path, Exception):
+ raise path
+ # There's no yield from in baseball, er, Python 2.
+ for entry in contents:
+ yield entry
+
+ name = 'testingpackage'
+ # Unforunately importlib.util.module_from_spec() was not introduced until
+ # Python 3.5.
+ module = types.ModuleType(name)
+ loader = Reader()
+ spec = machinery.ModuleSpec(
+ name, loader,
+ origin='does-not-exist',
+ is_package=is_package)
+ module.__spec__ = spec
+ module.__loader__ = loader
+ return module
+
+
+class CommonResourceTests(abc.ABC):
+ @abc.abstractmethod
+ def execute(self, package, path):
+ raise NotImplementedError
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_package_name(self):
+ # Passing in the package name should succeed.
+ self.execute(data01.__name__, 'utf-8.file')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_package_object(self):
+ # Passing in the package itself should succeed.
+ self.execute(data01, 'utf-8.file')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_string_path(self):
+ # Passing in a string for the path should succeed.
+ path = 'utf-8.file'
+ self.execute(data01, path)
+
+ @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support')
+ def test_pathlib_path(self):
+ # Passing in a pathlib.PurePath object for the path should succeed.
+ path = PurePath('utf-8.file')
+ self.execute(data01, path)
+
+ def test_absolute_path(self):
+ # An absolute path is a ValueError.
+ path = Path(__file__)
+ full_path = path.parent/'utf-8.file'
+ with self.assertRaises(ValueError):
+ self.execute(data01, full_path)
+
+ def test_relative_path(self):
+ # A reative path is a ValueError.
+ with self.assertRaises(ValueError):
+ self.execute(data01, '../data01/utf-8.file')
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_importing_module_as_side_effect(self):
+ # The anchor package can already be imported.
+ del sys.modules[data01.__name__]
+ self.execute(data01.__name__, 'utf-8.file')
+
+ def test_non_package_by_name(self):
+ # The anchor package cannot be a module.
+ with self.assertRaises(TypeError):
+ self.execute(__name__, 'utf-8.file')
+
+ def test_non_package_by_package(self):
+ # The anchor package cannot be a module.
+ with self.assertRaises(TypeError):
+ module = sys.modules['test.test_importlib.util']
+ self.execute(module, 'utf-8.file')
+
+ @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2')
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_resource_opener(self):
+ bytes_data = io.BytesIO(b'Hello, world!')
+ package = create_package(file=bytes_data, path=FileNotFoundError())
+ self.execute(package, 'utf-8.file')
+ self.assertEqual(package.__loader__._path, 'utf-8.file')
+
+ @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2')
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_resource_path(self):
+ bytes_data = io.BytesIO(b'Hello, world!')
+ path = __file__
+ package = create_package(file=bytes_data, path=path)
+ self.execute(package, 'utf-8.file')
+ self.assertEqual(package.__loader__._path, 'utf-8.file')
+
+ def test_useless_loader(self):
+ package = create_package(file=FileNotFoundError(),
+ path=FileNotFoundError())
+ with self.assertRaises(FileNotFoundError):
+ self.execute(package, 'utf-8.file')
+
+
+class ZipSetupBase:
+ ZIP_MODULE = None
+
+ @classmethod
+ def setUpClass(cls):
+ data_path = Path(cls.ZIP_MODULE.__file__)
+ data_dir = data_path.parent
+ cls._zip_path = str(data_dir / 'ziptestdata.zip')
+ sys.path.append(cls._zip_path)
+ cls.data = importlib.import_module('ziptestdata')
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ sys.path.remove(cls._zip_path)
+ except ValueError:
+ pass
+
+ try:
+ del sys.path_importer_cache[cls._zip_path]
+ del sys.modules[cls.data.__name__]
+ except KeyError:
+ pass
+
+ try:
+ del cls.data
+ del cls._zip_path
+ except AttributeError:
+ pass
+
+ def setUp(self):
+ modules = support.modules_setup()
+ self.addCleanup(support.modules_cleanup, *modules)
+
+
+class ZipSetup(ZipSetupBase):
+ ZIP_MODULE = zipdata01 # type: ignore
diff --git a/Lib/test/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip
new file mode 100644
index 000000000..8d8fa97f1
Binary files /dev/null and b/Lib/test/test_importlib/zipdata01/ziptestdata.zip differ
diff --git a/Lib/test/test_importlib/zipdata02/__init__.py b/Lib/test/test_importlib/zipdata02/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Lib/test/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip
new file mode 100644
index 000000000..6f348899a
Binary files /dev/null and b/Lib/test/test_importlib/zipdata02/ziptestdata.zip differ
diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index e5e7f0eba..4f1aa68e4 100644
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -226,8 +226,6 @@ class TestCase(unittest.TestCase):
self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6])
# Test a new_style class with __iter__ but no next() method
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_new_style_iter_class(self):
class IterClass(object):
def __iter__(self):
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
new file mode 100644
index 000000000..422fa904c
--- /dev/null
+++ b/Lib/test/test_itertools.py
@@ -0,0 +1,2608 @@
+import unittest
+from test import support
+from itertools import *
+import weakref
+from decimal import Decimal
+from fractions import Fraction
+import operator
+import random
+import copy
+import pickle
+from functools import reduce
+import sys
+import struct
+import threading
+maxsize = support.MAX_Py_ssize_t
+minsize = -maxsize-1
+
+def lzip(*args):
+ return list(zip(*args))
+
+def onearg(x):
+ 'Test function of one argument'
+ return 2*x
+
+def errfunc(*args):
+ 'Test function that raises an error'
+ raise ValueError
+
+def gen3():
+ 'Non-restartable source sequence'
+ for i in (0, 1, 2):
+ yield i
+
+def isEven(x):
+ 'Test predicate'
+ return x%2==0
+
+def isOdd(x):
+ 'Test predicate'
+ return x%2==1
+
+def tupleize(*args):
+ return args
+
+def irange(n):
+ for i in range(n):
+ yield i
+
+class StopNow:
+ 'Class emulating an empty iterable.'
+ def __iter__(self):
+ return self
+ def __next__(self):
+ raise StopIteration
+
+def take(n, seq):
+ 'Convenience function for partially consuming a long of infinite iterable'
+ return list(islice(seq, n))
+
+def prod(iterable):
+ return reduce(operator.mul, iterable, 1)
+
+def fact(n):
+ 'Factorial'
+ return prod(range(1, n+1))
+
+# root level methods for pickling ability
+def testR(r):
+ return r[0]
+
+def testR2(r):
+ return r[2]
+
+def underten(x):
+ return x<10
+
+picklecopiers = [lambda s, proto=proto: pickle.loads(pickle.dumps(s, proto))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1)]
+
+class TestBasicOps(unittest.TestCase):
+
+ def pickletest(self, protocol, it, stop=4, take=1, compare=None):
+ """Test that an iterator is the same after pickling, also when part-consumed"""
+ def expand(it, i=0):
+ # Recursively expand iterables, within sensible bounds
+ if i > 10:
+ raise RuntimeError("infinite recursion encountered")
+ if isinstance(it, str):
+ return it
+ try:
+ l = list(islice(it, stop))
+ except TypeError:
+ return it # can't expand it
+ return [expand(e, i+1) for e in l]
+
+ # Test the initial copy against the original
+ dump = pickle.dumps(it, protocol)
+ i2 = pickle.loads(dump)
+ self.assertEqual(type(it), type(i2))
+ a, b = expand(it), expand(i2)
+ self.assertEqual(a, b)
+ if compare:
+ c = expand(compare)
+ self.assertEqual(a, c)
+
+ # Take from the copy, and create another copy and compare them.
+ i3 = pickle.loads(dump)
+ took = 0
+ try:
+ for i in range(take):
+ next(i3)
+ took += 1
+ except StopIteration:
+ pass #in case there is less data than 'take'
+ dump = pickle.dumps(i3, protocol)
+ i4 = pickle.loads(dump)
+ a, b = expand(i3), expand(i4)
+ self.assertEqual(a, b)
+ if compare:
+ c = expand(compare[took:])
+ self.assertEqual(a, c);
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_accumulate(self):
+ self.assertEqual(list(accumulate(range(10))), # one positional arg
+ [0, 1, 3, 6, 10, 15, 21, 28, 36, 45])
+ self.assertEqual(list(accumulate(iterable=range(10))), # kw arg
+ [0, 1, 3, 6, 10, 15, 21, 28, 36, 45])
+ for typ in int, complex, Decimal, Fraction: # multiple types
+ self.assertEqual(
+ list(accumulate(map(typ, range(10)))),
+ list(map(typ, [0, 1, 3, 6, 10, 15, 21, 28, 36, 45])))
+ self.assertEqual(list(accumulate('abc')), ['a', 'ab', 'abc']) # works with non-numeric
+ self.assertEqual(list(accumulate([])), []) # empty iterable
+ self.assertEqual(list(accumulate([7])), [7]) # iterable of length one
+ self.assertRaises(TypeError, accumulate, range(10), 5, 6) # too many args
+ self.assertRaises(TypeError, accumulate) # too few args
+ self.assertRaises(TypeError, accumulate, x=range(10)) # unexpected kwd arg
+ self.assertRaises(TypeError, list, accumulate([1, []])) # args that don't add
+
+ s = [2, 8, 9, 5, 7, 0, 3, 4, 1, 6]
+ self.assertEqual(list(accumulate(s, min)),
+ [2, 2, 2, 2, 2, 0, 0, 0, 0, 0])
+ self.assertEqual(list(accumulate(s, max)),
+ [2, 8, 9, 9, 9, 9, 9, 9, 9, 9])
+ self.assertEqual(list(accumulate(s, operator.mul)),
+ [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0])
+ with self.assertRaises(TypeError):
+ list(accumulate(s, chr)) # unary-operation
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, accumulate(range(10))) # test pickling
+ self.pickletest(proto, accumulate(range(10), initial=7))
+ self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16])
+ self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116])
+ self.assertEqual(list(accumulate([], initial=100)), [100])
+ with self.assertRaises(TypeError):
+ list(accumulate([10, 20], 100))
+
+ def test_chain(self):
+
+ def chain2(*iterables):
+ 'Pure python version in the docs'
+ for it in iterables:
+ for element in it:
+ yield element
+
+ for c in (chain, chain2):
+ self.assertEqual(list(c('abc', 'def')), list('abcdef'))
+ self.assertEqual(list(c('abc')), list('abc'))
+ self.assertEqual(list(c('')), [])
+ self.assertEqual(take(4, c('abc', 'def')), list('abcd'))
+ self.assertRaises(TypeError, list,c(2, 3))
+
+ def test_chain_from_iterable(self):
+ self.assertEqual(list(chain.from_iterable(['abc', 'def'])), list('abcdef'))
+ self.assertEqual(list(chain.from_iterable(['abc'])), list('abc'))
+ self.assertEqual(list(chain.from_iterable([''])), [])
+ self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd'))
+ self.assertRaises(TypeError, list, chain.from_iterable([2, 3]))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_chain_reducible(self):
+ for oper in [copy.deepcopy] + picklecopiers:
+ it = chain('abc', 'def')
+ self.assertEqual(list(oper(it)), list('abcdef'))
+ self.assertEqual(next(it), 'a')
+ self.assertEqual(list(oper(it)), list('bcdef'))
+
+ self.assertEqual(list(oper(chain(''))), [])
+ self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd'))
+ self.assertRaises(TypeError, list, oper(chain(2, 3)))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef'))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_chain_setstate(self):
+ self.assertRaises(TypeError, chain().__setstate__, ())
+ self.assertRaises(TypeError, chain().__setstate__, [])
+ self.assertRaises(TypeError, chain().__setstate__, 0)
+ self.assertRaises(TypeError, chain().__setstate__, ([],))
+ self.assertRaises(TypeError, chain().__setstate__, (iter([]), []))
+ it = chain()
+ it.__setstate__((iter(['abc', 'def']),))
+ self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f'])
+ it = chain()
+ it.__setstate__((iter(['abc', 'def']), iter(['ghi'])))
+ self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f'])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_combinations(self):
+ self.assertRaises(TypeError, combinations, 'abc') # missing r argument
+ self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments
+ self.assertRaises(TypeError, combinations, None) # pool is not iterable
+ self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative
+
+ for op in [lambda a:a] + picklecopiers:
+ self.assertEqual(list(op(combinations('abc', 32))), []) # r > n
+
+ self.assertEqual(list(op(combinations('ABCD', 2))),
+ [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+ testIntermediate = combinations('ABCD', 2)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+
+ self.assertEqual(list(op(combinations(range(4), 3))),
+ [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+ testIntermediate = combinations(range(4), 3)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [(0,1,3), (0,2,3), (1,2,3)])
+
+
+ def combinations1(iterable, r):
+ 'Pure python version shown in the docs'
+ pool = tuple(iterable)
+ n = len(pool)
+ if r > n:
+ return
+ indices = list(range(r))
+ yield tuple(pool[i] for i in indices)
+ while 1:
+ for i in reversed(range(r)):
+ if indices[i] != i + n - r:
+ break
+ else:
+ return
+ indices[i] += 1
+ for j in range(i+1, r):
+ indices[j] = indices[j-1] + 1
+ yield tuple(pool[i] for i in indices)
+
+ def combinations2(iterable, r):
+ 'Pure python version shown in the docs'
+ pool = tuple(iterable)
+ n = len(pool)
+ for indices in permutations(range(n), r):
+ if sorted(indices) == list(indices):
+ yield tuple(pool[i] for i in indices)
+
+ def combinations3(iterable, r):
+ 'Pure python version from cwr()'
+ pool = tuple(iterable)
+ n = len(pool)
+ for indices in combinations_with_replacement(range(n), r):
+ if len(set(indices)) == r:
+ yield tuple(pool[i] for i in indices)
+
+ for n in range(7):
+ values = [5*x-12 for x in range(n)]
+ for r in range(n+2):
+ result = list(combinations(values, r))
+ self.assertEqual(len(result), 0 if r>n else fact(n) / fact(r) / fact(n-r)) # right number of combs
+ self.assertEqual(len(result), len(set(result))) # no repeats
+ self.assertEqual(result, sorted(result)) # lexicographic order
+ for c in result:
+ self.assertEqual(len(c), r) # r-length combinations
+ self.assertEqual(len(set(c)), r) # no duplicate elements
+ self.assertEqual(list(c), sorted(c)) # keep original ordering
+ self.assertTrue(all(e in values for e in c)) # elements taken from input iterable
+ self.assertEqual(list(c),
+ [e for e in values if e in c]) # comb is a subsequence of the input iterable
+ self.assertEqual(result, list(combinations1(values, r))) # matches first pure python version
+ self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
+ self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, combinations(values, r)) # test pickling
+
+ @support.bigaddrspacetest
+ def test_combinations_overflow(self):
+ with self.assertRaises((OverflowError, MemoryError)):
+ combinations("AA", 2**29)
+
+ # Test implementation detail: tuple re-use
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_combinations_tuple_reuse(self):
+ self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
+ self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_combinations_with_replacement(self):
+ cwr = combinations_with_replacement
+ self.assertRaises(TypeError, cwr, 'abc') # missing r argument
+ self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
+ self.assertRaises(TypeError, cwr, None) # pool is not iterable
+ self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative
+
+ for op in [lambda a:a] + picklecopiers:
+ self.assertEqual(list(op(cwr('ABC', 2))),
+ [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+ testIntermediate = cwr('ABC', 2)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
+
+ def cwr1(iterable, r):
+ 'Pure python version shown in the docs'
+ # number items returned: (n+r-1)! / r! / (n-1)! when n>0
+ pool = tuple(iterable)
+ n = len(pool)
+ if not n and r:
+ return
+ indices = [0] * r
+ yield tuple(pool[i] for i in indices)
+ while 1:
+ for i in reversed(range(r)):
+ if indices[i] != n - 1:
+ break
+ else:
+ return
+ indices[i:] = [indices[i] + 1] * (r - i)
+ yield tuple(pool[i] for i in indices)
+
+ def cwr2(iterable, r):
+ 'Pure python version shown in the docs'
+ pool = tuple(iterable)
+ n = len(pool)
+ for indices in product(range(n), repeat=r):
+ if sorted(indices) == list(indices):
+ yield tuple(pool[i] for i in indices)
+
+ def numcombs(n, r):
+ if not n:
+ return 0 if r else 1
+ return fact(n+r-1) / fact(r)/ fact(n-1)
+
+ for n in range(7):
+ values = [5*x-12 for x in range(n)]
+ for r in range(n+2):
+ result = list(cwr(values, r))
+
+ self.assertEqual(len(result), numcombs(n, r)) # right number of combs
+ self.assertEqual(len(result), len(set(result))) # no repeats
+ self.assertEqual(result, sorted(result)) # lexicographic order
+
+ regular_combs = list(combinations(values, r)) # compare to combs without replacement
+ if n == 0 or r <= 1:
+ self.assertEqual(result, regular_combs) # cases that should be identical
+ else:
+ self.assertTrue(set(result) >= set(regular_combs)) # rest should be supersets of regular combs
+
+ for c in result:
+ self.assertEqual(len(c), r) # r-length combinations
+ noruns = [k for k,v in groupby(c)] # combo without consecutive repeats
+ self.assertEqual(len(noruns), len(set(noruns))) # no repeats other than consecutive
+ self.assertEqual(list(c), sorted(c)) # keep original ordering
+ self.assertTrue(all(e in values for e in c)) # elements taken from input iterable
+ self.assertEqual(noruns,
+ [e for e in values if e in c]) # comb is a subsequence of the input iterable
+ self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version
+ self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, cwr(values,r)) # test pickling
+
+ @support.bigaddrspacetest
+ def test_combinations_with_replacement_overflow(self):
+ with self.assertRaises((OverflowError, MemoryError)):
+ combinations_with_replacement("AA", 2**30)
+
+ # Test implementation detail: tuple re-use
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_combinations_with_replacement_tuple_reuse(self):
+ cwr = combinations_with_replacement
+ self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1)
+ self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1)
+
+ # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615'
+ @unittest.skip("panics")
+ def test_permutations(self):
+ self.assertRaises(TypeError, permutations) # too few arguments
+ self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments
+ self.assertRaises(TypeError, permutations, None) # pool is not iterable
+ self.assertRaises(ValueError, permutations, 'abc', -2) # r is negative
+ self.assertEqual(list(permutations('abc', 32)), []) # r > n
+ self.assertRaises(TypeError, permutations, 'abc', 's') # r is not an int or None
+ self.assertEqual(list(permutations(range(3), 2)),
+ [(0,1), (0,2), (1,0), (1,2), (2,0), (2,1)])
+
+ def permutations1(iterable, r=None):
+ 'Pure python version shown in the docs'
+ pool = tuple(iterable)
+ n = len(pool)
+ r = n if r is None else r
+ if r > n:
+ return
+ indices = list(range(n))
+ cycles = list(range(n-r+1, n+1))[::-1]
+ yield tuple(pool[i] for i in indices[:r])
+ while n:
+ for i in reversed(range(r)):
+ cycles[i] -= 1
+ if cycles[i] == 0:
+ indices[i:] = indices[i+1:] + indices[i:i+1]
+ cycles[i] = n - i
+ else:
+ j = cycles[i]
+ indices[i], indices[-j] = indices[-j], indices[i]
+ yield tuple(pool[i] for i in indices[:r])
+ break
+ else:
+ return
+
+ def permutations2(iterable, r=None):
+ 'Pure python version shown in the docs'
+ pool = tuple(iterable)
+ n = len(pool)
+ r = n if r is None else r
+ for indices in product(range(n), repeat=r):
+ if len(set(indices)) == r:
+ yield tuple(pool[i] for i in indices)
+
+ for n in range(7):
+ values = [5*x-12 for x in range(n)]
+ for r in range(n+2):
+ result = list(permutations(values, r))
+ self.assertEqual(len(result), 0 if r>n else fact(n) / fact(n-r)) # right number of perms
+ self.assertEqual(len(result), len(set(result))) # no repeats
+ self.assertEqual(result, sorted(result)) # lexicographic order
+ for p in result:
+ self.assertEqual(len(p), r) # r-length permutations
+ self.assertEqual(len(set(p)), r) # no duplicate elements
+ self.assertTrue(all(e in values for e in p)) # elements taken from input iterable
+ self.assertEqual(result, list(permutations1(values, r))) # matches first pure python version
+ self.assertEqual(result, list(permutations2(values, r))) # matches second pure python version
+ if r == n:
+ self.assertEqual(result, list(permutations(values, None))) # test r as None
+ self.assertEqual(result, list(permutations(values))) # test default r
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, permutations(values, r)) # test pickling
+
+ @support.bigaddrspacetest
+ def test_permutations_overflow(self):
+ with self.assertRaises((OverflowError, MemoryError)):
+ permutations("A", 2**30)
+
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_permutations_tuple_reuse(self):
+ self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
+ self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1)
+
+ # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615'
+ @unittest.skip("panics")
+ def test_combinatorics(self):
+ # Test relationships between product(), permutations(),
+ # combinations() and combinations_with_replacement().
+
+ for n in range(6):
+ s = 'ABCDEFG'[:n]
+ for r in range(8):
+ prod = list(product(s, repeat=r))
+ cwr = list(combinations_with_replacement(s, r))
+ perm = list(permutations(s, r))
+ comb = list(combinations(s, r))
+
+ # Check size
+ self.assertEqual(len(prod), n**r)
+ self.assertEqual(len(cwr), (fact(n+r-1) / fact(r)/ fact(n-1)) if n else (not r))
+ self.assertEqual(len(perm), 0 if r>n else fact(n) / fact(n-r))
+ self.assertEqual(len(comb), 0 if r>n else fact(n) / fact(r) / fact(n-r))
+
+ # Check lexicographic order without repeated tuples
+ self.assertEqual(prod, sorted(set(prod)))
+ self.assertEqual(cwr, sorted(set(cwr)))
+ self.assertEqual(perm, sorted(set(perm)))
+ self.assertEqual(comb, sorted(set(comb)))
+
+ # Check interrelationships
+ self.assertEqual(cwr, [t for t in prod if sorted(t)==list(t)]) # cwr: prods which are sorted
+ self.assertEqual(perm, [t for t in prod if len(set(t))==r]) # perm: prods with no dups
+ self.assertEqual(comb, [t for t in perm if sorted(t)==list(t)]) # comb: perms that are sorted
+ self.assertEqual(comb, [t for t in cwr if len(set(t))==r]) # comb: cwrs without dups
+ self.assertEqual(comb, list(filter(set(cwr).__contains__, perm))) # comb: perm that is a cwr
+ self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm
+ self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_compress(self):
+ self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF'))
+ self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF'))
+ self.assertEqual(list(compress('ABCDEF', [0,0,0,0,0,0])), list(''))
+ self.assertEqual(list(compress('ABCDEF', [1,1,1,1,1,1])), list('ABCDEF'))
+ self.assertEqual(list(compress('ABCDEF', [1,0,1])), list('AC'))
+ self.assertEqual(list(compress('ABC', [0,1,1,1,1,1])), list('BC'))
+ n = 10000
+ data = chain.from_iterable(repeat(range(6), n))
+ selectors = chain.from_iterable(repeat((0, 1)))
+ self.assertEqual(list(compress(data, selectors)), [1,3,5] * n)
+ self.assertRaises(TypeError, compress, None, range(6)) # 1st arg not iterable
+ self.assertRaises(TypeError, compress, range(6), None) # 2nd arg not iterable
+ self.assertRaises(TypeError, compress, range(6)) # too few args
+ self.assertRaises(TypeError, compress, range(6), None) # too many args
+
+ # check copy, deepcopy, pickle
+ for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers:
+ for data, selectors, result1, result2 in [
+ ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'),
+ ('ABCDEF', [0,0,0,0,0,0], '', ''),
+ ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'),
+ ('ABCDEF', [1,0,1], 'AC', 'C'),
+ ('ABC', [0,1,1,1,1,1], 'BC', 'C'),
+ ]:
+
+ self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1))
+ self.assertEqual(list(op(compress(data, selectors))), list(result1))
+ testIntermediate = compress(data, selectors)
+ if result1:
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)), list(result2))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_count(self):
+ self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)])
+ self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)])
+ self.assertEqual(take(2, lzip('abc',count(3))), [('a', 3), ('b', 4)])
+ self.assertEqual(take(2, zip('abc',count(-1))), [('a', -1), ('b', 0)])
+ self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)])
+ self.assertRaises(TypeError, count, 2, 3, 4)
+ self.assertRaises(TypeError, count, 'a')
+ self.assertEqual(take(10, count(maxsize-5)),
+ list(range(maxsize-5, maxsize+5)))
+ self.assertEqual(take(10, count(-maxsize-5)),
+ list(range(-maxsize-5, -maxsize+5)))
+ self.assertEqual(take(3, count(3.25)), [3.25, 4.25, 5.25])
+ self.assertEqual(take(3, count(3.25-4j)), [3.25-4j, 4.25-4j, 5.25-4j])
+ self.assertEqual(take(3, count(Decimal('1.1'))),
+ [Decimal('1.1'), Decimal('2.1'), Decimal('3.1')])
+ self.assertEqual(take(3, count(Fraction(2, 3))),
+ [Fraction(2, 3), Fraction(5, 3), Fraction(8, 3)])
+ BIGINT = 1<<1000
+ self.assertEqual(take(3, count(BIGINT)), [BIGINT, BIGINT+1, BIGINT+2])
+ c = count(3)
+ self.assertEqual(repr(c), 'count(3)')
+ next(c)
+ self.assertEqual(repr(c), 'count(4)')
+ c = count(-9)
+ self.assertEqual(repr(c), 'count(-9)')
+ next(c)
+ self.assertEqual(next(c), -8)
+ self.assertEqual(repr(count(10.25)), 'count(10.25)')
+ self.assertEqual(repr(count(10.0)), 'count(10.0)')
+ self.assertEqual(type(next(count(10.0))), float)
+ for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5):
+ # Test repr
+ r1 = repr(count(i))
+ r2 = 'count(%r)'.__mod__(i)
+ self.assertEqual(r1, r2)
+
+ # check copy, deepcopy, pickle
+ for value in -3, 3, maxsize-5, maxsize+5:
+ c = count(value)
+ self.assertEqual(next(copy.copy(c)), value)
+ self.assertEqual(next(copy.deepcopy(c)), value)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, count(value))
+
+ #check proper internal error handling for large "step' sizes
+ count(1, maxsize+5); sys.exc_info()
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_count_with_stride(self):
+ self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)])
+ self.assertEqual(lzip('abc',count(start=2,step=3)),
+ [('a', 2), ('b', 5), ('c', 8)])
+ self.assertEqual(lzip('abc',count(step=-1)),
+ [('a', 0), ('b', -1), ('c', -2)])
+ self.assertRaises(TypeError, count, 'a', 'b')
+ self.assertEqual(lzip('abc',count(2,0)), [('a', 2), ('b', 2), ('c', 2)])
+ self.assertEqual(lzip('abc',count(2,1)), [('a', 2), ('b', 3), ('c', 4)])
+ self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)])
+ self.assertEqual(take(20, count(maxsize-15, 3)), take(20, range(maxsize-15, maxsize+100, 3)))
+ self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3)))
+ self.assertEqual(take(3, count(10, maxsize+5)),
+ list(range(10, 10+3*(maxsize+5), maxsize+5)))
+ self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5])
+ self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j])
+ self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))),
+ [Decimal('1.1'), Decimal('1.2'), Decimal('1.3')])
+ self.assertEqual(take(3, count(Fraction(2,3), Fraction(1,7))),
+ [Fraction(2,3), Fraction(17,21), Fraction(20,21)])
+ BIGINT = 1<<1000
+ self.assertEqual(take(3, count(step=BIGINT)), [0, BIGINT, 2*BIGINT])
+ self.assertEqual(repr(take(3, count(10, 2.5))), repr([10, 12.5, 15.0]))
+ c = count(3, 5)
+ self.assertEqual(repr(c), 'count(3, 5)')
+ next(c)
+ self.assertEqual(repr(c), 'count(8, 5)')
+ c = count(-9, 0)
+ self.assertEqual(repr(c), 'count(-9, 0)')
+ next(c)
+ self.assertEqual(repr(c), 'count(-9, 0)')
+ c = count(-9, -3)
+ self.assertEqual(repr(c), 'count(-9, -3)')
+ next(c)
+ self.assertEqual(repr(c), 'count(-12, -3)')
+ self.assertEqual(repr(c), 'count(-12, -3)')
+ self.assertEqual(repr(count(10.5, 1.25)), 'count(10.5, 1.25)')
+ self.assertEqual(repr(count(10.5, 1)), 'count(10.5)') # suppress step=1 when it's an int
+ self.assertEqual(repr(count(10.5, 1.00)), 'count(10.5, 1.0)') # do show float values lilke 1.0
+ self.assertEqual(repr(count(10, 1.00)), 'count(10, 1.0)')
+ c = count(10, 1.0)
+ self.assertEqual(type(next(c)), int)
+ self.assertEqual(type(next(c)), float)
+ for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5):
+ for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5):
+ # Test repr
+ r1 = repr(count(i, j))
+ if j == 1:
+ r2 = ('count(%r)' % i)
+ else:
+ r2 = ('count(%r, %r)' % (i, j))
+ self.assertEqual(r1, r2)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, count(i, j))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_cycle(self):
+ self.assertEqual(take(10, cycle('abc')), list('abcabcabca'))
+ self.assertEqual(list(cycle('')), [])
+ self.assertRaises(TypeError, cycle)
+ self.assertRaises(TypeError, cycle, 5)
+ self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0])
+
+ # check copy, deepcopy, pickle
+ c = cycle('abc')
+ self.assertEqual(next(c), 'a')
+ #simple copy currently not supported, because __reduce__ returns
+ #an internal iterator
+ #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab'))
+ self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab'))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))),
+ list('bcabcabcab'))
+ next(c)
+ self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))),
+ list('cabcabcabc'))
+ next(c)
+ next(c)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, cycle('abc'))
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ # test with partial consumed input iterable
+ it = iter('abcde')
+ c = cycle(it)
+ _ = [next(c) for i in range(2)] # consume 2 of 5 inputs
+ p = pickle.dumps(c, proto)
+ d = pickle.loads(p) # rebuild the cycle object
+ self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab'))
+
+ # test with completely consumed input iterable
+ it = iter('abcde')
+ c = cycle(it)
+ _ = [next(c) for i in range(7)] # consume 7 of 5 inputs
+ p = pickle.dumps(c, proto)
+ d = pickle.loads(p) # rebuild the cycle object
+ self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab'))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_cycle_setstate(self):
+ # Verify both modes for restoring state
+
+ # Mode 0 is efficient. It uses an incompletely consumed input
+ # iterator to build a cycle object and then passes in state with
+ # a list of previously consumed values. There is no data
+ # overlap between the two.
+ c = cycle('defg')
+ c.__setstate__((list('abc'), 0))
+ self.assertEqual(take(20, c), list('defgabcdefgabcdefgab'))
+
+ # Mode 1 is inefficient. It starts with a cycle object built
+ # from an iterator over the remaining elements in a partial
+ # cycle and then passes in state with all of the previously
+ # seen values (this overlaps values included in the iterator).
+ c = cycle('defg')
+ c.__setstate__((list('abcdefg'), 1))
+ self.assertEqual(take(20, c), list('defgabcdefgabcdefgab'))
+
+ # The first argument to setstate needs to be a tuple
+ with self.assertRaises(TypeError):
+ cycle('defg').__setstate__([list('abcdefg'), 0])
+
+ # The first argument in the setstate tuple must be a list
+ with self.assertRaises(TypeError):
+ c = cycle('defg')
+ c.__setstate__((tuple('defg'), 0))
+ take(20, c)
+
+ # The second argument in the setstate tuple must be an int
+ with self.assertRaises(TypeError):
+ cycle('defg').__setstate__((list('abcdefg'), 'x'))
+
+ self.assertRaises(TypeError, cycle('').__setstate__, ())
+ self.assertRaises(TypeError, cycle('').__setstate__, ([],))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_groupby(self):
+ # Check whether it accepts arguments correctly
+ self.assertEqual([], list(groupby([])))
+ self.assertEqual([], list(groupby([], key=id)))
+ self.assertRaises(TypeError, list, groupby('abc', []))
+ self.assertRaises(TypeError, groupby, None)
+ self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10)
+
+ # Check normal input
+ s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22),
+ (2,15,22), (3,16,23), (3,17,23)]
+ dup = []
+ for k, g in groupby(s, lambda r:r[0]):
+ for elem in g:
+ self.assertEqual(k, elem[0])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
+ # Check normal pickled
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ dup = []
+ for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)):
+ for elem in g:
+ self.assertEqual(k, elem[0])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
+ # Check nested case
+ dup = []
+ for k, g in groupby(s, testR):
+ for ik, ig in groupby(g, testR2):
+ for elem in ig:
+ self.assertEqual(k, elem[0])
+ self.assertEqual(ik, elem[2])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
+ # Check nested and pickled
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ dup = []
+ for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)):
+ for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)):
+ for elem in ig:
+ self.assertEqual(k, elem[0])
+ self.assertEqual(ik, elem[2])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
+
+ # Check case where inner iterator is not used
+ keys = [k for k, g in groupby(s, testR)]
+ expectedkeys = set([r[0] for r in s])
+ self.assertEqual(set(keys), expectedkeys)
+ self.assertEqual(len(keys), len(expectedkeys))
+
+ # Check case where inner iterator is used after advancing the groupby
+ # iterator
+ s = list(zip('AABBBAAAA', range(9)))
+ it = groupby(s, testR)
+ _, g1 = next(it)
+ _, g2 = next(it)
+ _, g3 = next(it)
+ self.assertEqual(list(g1), [])
+ self.assertEqual(list(g2), [])
+ self.assertEqual(next(g3), ('A', 5))
+ list(it) # exhaust the groupby iterator
+ self.assertEqual(list(g3), [])
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ it = groupby(s, testR)
+ _, g = next(it)
+ next(it)
+ next(it)
+ self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), [])
+
+ # Exercise pipes and filters style
+ s = 'abracadabra'
+ # sort s | uniq
+ r = [k for k, g in groupby(sorted(s))]
+ self.assertEqual(r, ['a', 'b', 'c', 'd', 'r'])
+ # sort s | uniq -d
+ r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))]
+ self.assertEqual(r, ['a', 'b', 'r'])
+ # sort s | uniq -c
+ r = [(len(list(g)), k) for k, g in groupby(sorted(s))]
+ self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')])
+ # sort s | uniq -c | sort -rn | head -3
+ r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3]
+ self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')])
+
+ # iter.__next__ failure
+ class ExpectedError(Exception):
+ pass
+ def delayed_raise(n=0):
+ for i in range(n):
+ yield 'yo'
+ raise ExpectedError
+ def gulp(iterable, keyp=None, func=list):
+ return [func(g) for k, g in groupby(iterable, keyp)]
+
+ # iter.__next__ failure on outer object
+ self.assertRaises(ExpectedError, gulp, delayed_raise(0))
+ # iter.__next__ failure on inner object
+ self.assertRaises(ExpectedError, gulp, delayed_raise(1))
+
+ # __eq__ failure
+ class DummyCmp:
+ def __eq__(self, dst):
+ raise ExpectedError
+ s = [DummyCmp(), DummyCmp(), None]
+
+ # __eq__ failure on outer object
+ self.assertRaises(ExpectedError, gulp, s, func=id)
+ # __eq__ failure on inner object
+ self.assertRaises(ExpectedError, gulp, s)
+
+ # keyfunc failure
+ def keyfunc(obj):
+ if keyfunc.skip > 0:
+ keyfunc.skip -= 1
+ return obj
+ else:
+ raise ExpectedError
+
+ # keyfunc failure on outer object
+ keyfunc.skip = 0
+ self.assertRaises(ExpectedError, gulp, [None], keyfunc)
+ keyfunc.skip = 1
+ self.assertRaises(ExpectedError, gulp, [None, None], keyfunc)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_filter(self):
+ self.assertEqual(list(filter(isEven, range(6))), [0,2,4])
+ self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2])
+ self.assertEqual(list(filter(bool, [0,1,0,2,0])), [1,2])
+ self.assertEqual(take(4, filter(isEven, count())), [0,2,4,6])
+ self.assertRaises(TypeError, filter)
+ self.assertRaises(TypeError, filter, lambda x:x)
+ self.assertRaises(TypeError, filter, lambda x:x, range(6), 7)
+ self.assertRaises(TypeError, filter, isEven, 3)
+ self.assertRaises(TypeError, next, filter(range(6), range(6)))
+
+ # check copy, deepcopy, pickle
+ ans = [0,2,4]
+
+ c = filter(isEven, range(6))
+ self.assertEqual(list(copy.copy(c)), ans)
+ c = filter(isEven, range(6))
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ c = filter(isEven, range(6))
+ self.assertEqual(list(pickle.loads(pickle.dumps(c, proto))), ans)
+ next(c)
+ self.assertEqual(list(pickle.loads(pickle.dumps(c, proto))), ans[1:])
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ c = filter(isEven, range(6))
+ self.pickletest(proto, c)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_filterfalse(self):
+ self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5])
+ self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0])
+ self.assertEqual(list(filterfalse(bool, [0,1,0,2,0])), [0,0,0])
+ self.assertEqual(take(4, filterfalse(isEven, count())), [1,3,5,7])
+ self.assertRaises(TypeError, filterfalse)
+ self.assertRaises(TypeError, filterfalse, lambda x:x)
+ self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7)
+ self.assertRaises(TypeError, filterfalse, isEven, 3)
+ self.assertRaises(TypeError, next, filterfalse(range(6), range(6)))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, filterfalse(isEven, range(6)))
+
+ def test_zip(self):
+ # XXX This is rather silly now that builtin zip() calls zip()...
+ ans = [(x,y) for x, y in zip('abc',count())]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+ self.assertEqual(list(zip('abc', range(6))), lzip('abc', range(6)))
+ self.assertEqual(list(zip('abcdef', range(3))), lzip('abcdef', range(3)))
+ self.assertEqual(take(3,zip('abcdef', count())), lzip('abcdef', range(3)))
+ self.assertEqual(list(zip('abcdef')), lzip('abcdef'))
+ self.assertEqual(list(zip()), lzip())
+ self.assertRaises(TypeError, zip, 3)
+ self.assertRaises(TypeError, zip, range(3), 3)
+ self.assertEqual([tuple(list(pair)) for pair in zip('abc', 'def')],
+ lzip('abc', 'def'))
+ self.assertEqual([pair for pair in zip('abc', 'def')],
+ lzip('abc', 'def'))
+
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_zip_tuple_reuse(self):
+ ids = list(map(id, zip('abc', 'def')))
+ self.assertEqual(min(ids), max(ids))
+ ids = list(map(id, list(zip('abc', 'def'))))
+ self.assertEqual(len(dict.fromkeys(ids)), len(ids))
+
+ # check copy, deepcopy, pickle
+ ans = [(x,y) for x, y in copy.copy(zip('abc',count()))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ testIntermediate = zip('abc',count())
+ next(testIntermediate)
+ ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))]
+ self.assertEqual(ans, [('b', 1), ('c', 2)])
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, zip('abc', count()))
+
+ def test_ziplongest(self):
+ for args in [
+ ['abc', range(6)],
+ [range(6), 'abc'],
+ [range(1000), range(2000,2100), range(3000,3050)],
+ [range(1000), range(0), range(3000,3050), range(1200), range(1500)],
+ [range(1000), range(0), range(3000,3050), range(1200), range(1500), range(0)],
+ ]:
+ target = [tuple([arg[i] if i < len(arg) else None for arg in args])
+ for i in range(max(map(len, args)))]
+ self.assertEqual(list(zip_longest(*args)), target)
+ self.assertEqual(list(zip_longest(*args, **{})), target)
+ target = [tuple((e is None and 'X' or e) for e in t) for t in target] # Replace None fills with 'X'
+ self.assertEqual(list(zip_longest(*args, **dict(fillvalue='X'))), target)
+
+ self.assertEqual(take(3,zip_longest('abcdef', count())), list(zip('abcdef', range(3)))) # take 3 from infinite input
+
+ self.assertEqual(list(zip_longest()), list(zip()))
+ self.assertEqual(list(zip_longest([])), list(zip([])))
+ self.assertEqual(list(zip_longest('abcdef')), list(zip('abcdef')))
+
+ self.assertEqual(list(zip_longest('abc', 'defg', **{})),
+ list(zip(list('abc')+[None], 'defg'))) # empty keyword dict
+ self.assertRaises(TypeError, zip_longest, 3)
+ self.assertRaises(TypeError, zip_longest, range(3), 3)
+
+ for stmt in [
+ "zip_longest('abc', fv=1)",
+ "zip_longest('abc', fillvalue=1, bogus_keyword=None)",
+ ]:
+ try:
+ eval(stmt, globals(), locals())
+ except TypeError:
+ pass
+ else:
+ self.fail('Did not raise Type in: ' + stmt)
+
+ self.assertEqual([tuple(list(pair)) for pair in zip_longest('abc', 'def')],
+ list(zip('abc', 'def')))
+ self.assertEqual([pair for pair in zip_longest('abc', 'def')],
+ list(zip('abc', 'def')))
+
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_zip_longest_tuple_reuse(self):
+ ids = list(map(id, zip_longest('abc', 'def')))
+ self.assertEqual(min(ids), max(ids))
+ ids = list(map(id, list(zip_longest('abc', 'def'))))
+ self.assertEqual(len(dict.fromkeys(ids)), len(ids))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_zip_longest_pickling(self):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, zip_longest("abc", "def"))
+ self.pickletest(proto, zip_longest("abc", "defgh"))
+ self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1))
+ self.pickletest(proto, zip_longest("", "defgh"))
+
+ def test_zip_longest_bad_iterable(self):
+ exception = TypeError()
+
+ class BadIterable:
+ def __iter__(self):
+ raise exception
+
+ with self.assertRaises(TypeError) as cm:
+ zip_longest(BadIterable())
+
+ self.assertIs(cm.exception, exception)
+
+ def test_bug_7244(self):
+
+ class Repeater:
+ # this class is similar to itertools.repeat
+ def __init__(self, o, t, e):
+ self.o = o
+ self.t = int(t)
+ self.e = e
+ def __iter__(self): # its iterator is itself
+ return self
+ def __next__(self):
+ if self.t > 0:
+ self.t -= 1
+ return self.o
+ else:
+ raise self.e
+
+ # Formerly this code in would fail in debug mode
+ # with Undetected Error and Stop Iteration
+ r1 = Repeater(1, 3, StopIteration)
+ r2 = Repeater(2, 4, StopIteration)
+ def run(r1, r2):
+ result = []
+ for i, j in zip_longest(r1, r2, fillvalue=0):
+ with support.captured_output('stdout'):
+ print((i, j))
+ result.append((i, j))
+ return result
+ self.assertEqual(run(r1, r2), [(1,2), (1,2), (1,2), (0,2)])
+
+ # Formerly, the RuntimeError would be lost
+ # and StopIteration would stop as expected
+ r1 = Repeater(1, 3, RuntimeError)
+ r2 = Repeater(2, 4, StopIteration)
+ it = zip_longest(r1, r2, fillvalue=0)
+ self.assertEqual(next(it), (1, 2))
+ self.assertEqual(next(it), (1, 2))
+ self.assertEqual(next(it), (1, 2))
+ self.assertRaises(RuntimeError, next, it)
+
+ # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615'
+ @unittest.skip("panics")
+ def test_product(self):
+ for args, result in [
+ ([], [()]), # zero iterables
+ (['ab'], [('a',), ('b',)]), # one iterable
+ ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables
+ ([range(0), range(2), range(3)], []), # first iterable with zero length
+ ([range(2), range(0), range(3)], []), # middle iterable with zero length
+ ([range(2), range(3), range(0)], []), # last iterable with zero length
+ ]:
+ self.assertEqual(list(product(*args)), result)
+ for r in range(4):
+ self.assertEqual(list(product(*(args*r))),
+ list(product(*args, **dict(repeat=r))))
+ self.assertEqual(len(list(product(*[range(7)]*6))), 7**6)
+ self.assertRaises(TypeError, product, range(6), None)
+
+ def product1(*args, **kwds):
+ pools = list(map(tuple, args)) * kwds.get('repeat', 1)
+ n = len(pools)
+ if n == 0:
+ yield ()
+ return
+ if any(len(pool) == 0 for pool in pools):
+ return
+ indices = [0] * n
+ yield tuple(pool[i] for pool, i in zip(pools, indices))
+ while 1:
+ for i in reversed(range(n)): # right to left
+ if indices[i] == len(pools[i]) - 1:
+ continue
+ indices[i] += 1
+ for j in range(i+1, n):
+ indices[j] = 0
+ yield tuple(pool[i] for pool, i in zip(pools, indices))
+ break
+ else:
+ return
+
+ def product2(*args, **kwds):
+ 'Pure python version used in docs'
+ pools = list(map(tuple, args)) * kwds.get('repeat', 1)
+ result = [[]]
+ for pool in pools:
+ result = [x+[y] for x in result for y in pool]
+ for prod in result:
+ yield tuple(prod)
+
+ argtypes = ['', 'abc', '', range(0), range(4), dict(a=1, b=2, c=3),
+ set('abcdefg'), range(11), tuple(range(13))]
+ for i in range(100):
+ args = [random.choice(argtypes) for j in range(random.randrange(5))]
+ expected_len = prod(map(len, args))
+ self.assertEqual(len(list(product(*args))), expected_len)
+ self.assertEqual(list(product(*args)), list(product1(*args)))
+ self.assertEqual(list(product(*args)), list(product2(*args)))
+ args = map(iter, args)
+ self.assertEqual(len(list(product(*args))), expected_len)
+
+ @support.bigaddrspacetest
+ def test_product_overflow(self):
+ with self.assertRaises((OverflowError, MemoryError)):
+ product(*(['ab']*2**5), repeat=2**25)
+
+ @support.impl_detail("tuple reuse is specific to CPython")
+ def test_product_tuple_reuse(self):
+ self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
+ self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
+
+ # TODO: RUSTPYTHON - panics with 'index out of bounds: the len is 0 but the index is 18446744073709551615'
+ @unittest.skip("panics")
+ def test_product_pickling(self):
+ # check copy, deepcopy, pickle
+ for args, result in [
+ ([], [()]), # zero iterables
+ (['ab'], [('a',), ('b',)]), # one iterable
+ ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables
+ ([range(0), range(2), range(3)], []), # first iterable with zero length
+ ([range(2), range(0), range(3)], []), # middle iterable with zero length
+ ([range(2), range(3), range(0)], []), # last iterable with zero length
+ ]:
+ self.assertEqual(list(copy.copy(product(*args))), result)
+ self.assertEqual(list(copy.deepcopy(product(*args))), result)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, product(*args))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_product_issue_25021(self):
+ # test that indices are properly clamped to the length of the tuples
+ p = product((1, 2),(3,))
+ p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped
+ self.assertEqual(next(p), (2, 3))
+ # test that empty tuple in the list will result in an immediate StopIteration
+ p = product((1, 2), (), (3,))
+ p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped
+ self.assertRaises(StopIteration, next, p)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_repeat(self):
+ self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a'])
+ self.assertEqual(lzip(range(3),repeat('a')),
+ [(0, 'a'), (1, 'a'), (2, 'a')])
+ self.assertEqual(list(repeat('a', 3)), ['a', 'a', 'a'])
+ self.assertEqual(take(3, repeat('a')), ['a', 'a', 'a'])
+ self.assertEqual(list(repeat('a', 0)), [])
+ self.assertEqual(list(repeat('a', -3)), [])
+ self.assertRaises(TypeError, repeat)
+ self.assertRaises(TypeError, repeat, None, 3, 4)
+ self.assertRaises(TypeError, repeat, None, 'a')
+ r = repeat(1+0j)
+ self.assertEqual(repr(r), 'repeat((1+0j))')
+ r = repeat(1+0j, 5)
+ self.assertEqual(repr(r), 'repeat((1+0j), 5)')
+ list(r)
+ self.assertEqual(repr(r), 'repeat((1+0j), 0)')
+
+ # check copy, deepcopy, pickle
+ c = repeat(object='a', times=10)
+ self.assertEqual(next(c), 'a')
+ self.assertEqual(take(2, copy.copy(c)), list('a' * 2))
+ self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, repeat(object='a', times=10))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_repeat_with_negative_times(self):
+ self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)")
+ self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)")
+ self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)")
+ self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)")
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_map(self):
+ self.assertEqual(list(map(operator.pow, range(3), range(1,7))),
+ [0**1, 1**2, 2**3])
+ self.assertEqual(list(map(tupleize, 'abc', range(5))),
+ [('a',0),('b',1),('c',2)])
+ self.assertEqual(list(map(tupleize, 'abc', count())),
+ [('a',0),('b',1),('c',2)])
+ self.assertEqual(take(2,map(tupleize, 'abc', count())),
+ [('a',0),('b',1)])
+ self.assertEqual(list(map(operator.pow, [])), [])
+ self.assertRaises(TypeError, map)
+ self.assertRaises(TypeError, list, map(None, range(3), range(3)))
+ self.assertRaises(TypeError, map, operator.neg)
+ self.assertRaises(TypeError, next, map(10, range(5)))
+ self.assertRaises(ValueError, next, map(errfunc, [4], [5]))
+ self.assertRaises(TypeError, next, map(onearg, [4], [5]))
+
+ # check copy, deepcopy, pickle
+ ans = [('a',0),('b',1),('c',2)]
+
+ c = map(tupleize, 'abc', count())
+ self.assertEqual(list(copy.copy(c)), ans)
+
+ c = map(tupleize, 'abc', count())
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ c = map(tupleize, 'abc', count())
+ self.pickletest(proto, c)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_starmap(self):
+ self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))),
+ [0**1, 1**2, 2**3])
+ self.assertEqual(take(3, starmap(operator.pow, zip(count(), count(1)))),
+ [0**1, 1**2, 2**3])
+ self.assertEqual(list(starmap(operator.pow, [])), [])
+ self.assertEqual(list(starmap(operator.pow, [iter([4,5])])), [4**5])
+ self.assertRaises(TypeError, list, starmap(operator.pow, [None]))
+ self.assertRaises(TypeError, starmap)
+ self.assertRaises(TypeError, starmap, operator.pow, [(4,5)], 'extra')
+ self.assertRaises(TypeError, next, starmap(10, [(4,5)]))
+ self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)]))
+ self.assertRaises(TypeError, next, starmap(onearg, [(4,5)]))
+
+ # check copy, deepcopy, pickle
+ ans = [0**1, 1**2, 2**3]
+
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.assertEqual(list(copy.copy(c)), ans)
+
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.pickletest(proto, c)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_islice(self):
+ for args in [ # islice(args) should agree with range(args)
+ (10, 20, 3),
+ (10, 3, 20),
+ (10, 20),
+ (10, 10),
+ (10, 3),
+ (20,)
+ ]:
+ self.assertEqual(list(islice(range(100), *args)),
+ list(range(*args)))
+
+ for args, tgtargs in [ # Stop when seqn is exhausted
+ ((10, 110, 3), ((10, 100, 3))),
+ ((10, 110), ((10, 100))),
+ ((110,), (100,))
+ ]:
+ self.assertEqual(list(islice(range(100), *args)),
+ list(range(*tgtargs)))
+
+ # Test stop=None
+ self.assertEqual(list(islice(range(10), None)), list(range(10)))
+ self.assertEqual(list(islice(range(10), None, None)), list(range(10)))
+ self.assertEqual(list(islice(range(10), None, None, None)), list(range(10)))
+ self.assertEqual(list(islice(range(10), 2, None)), list(range(2, 10)))
+ self.assertEqual(list(islice(range(10), 1, None, 2)), list(range(1, 10, 2)))
+
+ # Test number of items consumed SF #1171417
+ it = iter(range(10))
+ self.assertEqual(list(islice(it, 3)), list(range(3)))
+ self.assertEqual(list(it), list(range(3, 10)))
+
+ it = iter(range(10))
+ self.assertEqual(list(islice(it, 3, 3)), [])
+ self.assertEqual(list(it), list(range(3, 10)))
+
+ # Test invalid arguments
+ ra = range(10)
+ self.assertRaises(TypeError, islice, ra)
+ self.assertRaises(TypeError, islice, ra, 1, 2, 3, 4)
+ self.assertRaises(ValueError, islice, ra, -5, 10, 1)
+ self.assertRaises(ValueError, islice, ra, 1, -5, -1)
+ self.assertRaises(ValueError, islice, ra, 1, 10, -1)
+ self.assertRaises(ValueError, islice, ra, 1, 10, 0)
+ self.assertRaises(ValueError, islice, ra, 'a')
+ self.assertRaises(ValueError, islice, ra, 'a', 1)
+ self.assertRaises(ValueError, islice, ra, 1, 'a')
+ self.assertRaises(ValueError, islice, ra, 'a', 1, 1)
+ self.assertRaises(ValueError, islice, ra, 1, 'a', 1)
+ self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
+
+ # Issue #10323: Less islice in a predictable state
+ c = count()
+ self.assertEqual(list(islice(c, 1, 3, 50)), [1])
+ self.assertEqual(next(c), 3)
+
+ # check copy, deepcopy, pickle
+ for args in [ # islice(args) should agree with range(args)
+ (10, 20, 3),
+ (10, 3, 20),
+ (10, 20),
+ (10, 3),
+ (20,)
+ ]:
+ self.assertEqual(list(copy.copy(islice(range(100), *args))),
+ list(range(*args)))
+ self.assertEqual(list(copy.deepcopy(islice(range(100), *args))),
+ list(range(*args)))
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, islice(range(100), *args))
+
+ # Issue #21321: check source iterator is not referenced
+ # from islice() after the latter has been exhausted
+ it = (x for x in (1, 2))
+ wr = weakref.ref(it)
+ it = islice(it, 1)
+ self.assertIsNotNone(wr())
+ list(it) # exhaust the iterator
+ support.gc_collect()
+ self.assertIsNone(wr())
+
+ # Issue #30537: islice can accept integer-like objects as
+ # arguments
+ class IntLike(object):
+ def __init__(self, val):
+ self.val = val
+ def __index__(self):
+ return self.val
+ self.assertEqual(list(islice(range(100), IntLike(10))), list(range(10)))
+ self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50))),
+ list(range(10, 50)))
+ self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))),
+ list(range(10,50,5)))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_takewhile(self):
+ data = [1, 3, 5, 20, 2, 4, 6, 8]
+ self.assertEqual(list(takewhile(underten, data)), [1, 3, 5])
+ self.assertEqual(list(takewhile(underten, [])), [])
+ self.assertRaises(TypeError, takewhile)
+ self.assertRaises(TypeError, takewhile, operator.pow)
+ self.assertRaises(TypeError, takewhile, operator.pow, [(4,5)], 'extra')
+ self.assertRaises(TypeError, next, takewhile(10, [(4,5)]))
+ self.assertRaises(ValueError, next, takewhile(errfunc, [(4,5)]))
+ t = takewhile(bool, [1, 1, 1, 0, 0, 0])
+ self.assertEqual(list(t), [1, 1, 1])
+ self.assertRaises(StopIteration, next, t)
+
+ # check copy, deepcopy, pickle
+ self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5])
+ self.assertEqual(list(copy.deepcopy(takewhile(underten, data))),
+ [1, 3, 5])
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, takewhile(underten, data))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_dropwhile(self):
+ data = [1, 3, 5, 20, 2, 4, 6, 8]
+ self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8])
+ self.assertEqual(list(dropwhile(underten, [])), [])
+ self.assertRaises(TypeError, dropwhile)
+ self.assertRaises(TypeError, dropwhile, operator.pow)
+ self.assertRaises(TypeError, dropwhile, operator.pow, [(4,5)], 'extra')
+ self.assertRaises(TypeError, next, dropwhile(10, [(4,5)]))
+ self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)]))
+
+ # check copy, deepcopy, pickle
+ self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8])
+ self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))),
+ [20, 2, 4, 6, 8])
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, dropwhile(underten, data))
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_tee(self):
+ n = 200
+
+ a, b = tee([]) # test empty iterator
+ self.assertEqual(list(a), [])
+ self.assertEqual(list(b), [])
+
+ a, b = tee(irange(n)) # test 100% interleaved
+ self.assertEqual(lzip(a,b), lzip(range(n), range(n)))
+
+ a, b = tee(irange(n)) # test 0% interleaved
+ self.assertEqual(list(a), list(range(n)))
+ self.assertEqual(list(b), list(range(n)))
+
+ a, b = tee(irange(n)) # test dealloc of leading iterator
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ del a
+ self.assertEqual(list(b), list(range(n)))
+
+ a, b = tee(irange(n)) # test dealloc of trailing iterator
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ del b
+ self.assertEqual(list(a), list(range(100, n)))
+
+ for j in range(5): # test randomly interleaved
+ order = [0]*n + [1]*n
+ random.shuffle(order)
+ lists = ([], [])
+ its = tee(irange(n))
+ for i in order:
+ value = next(its[i])
+ lists[i].append(value)
+ self.assertEqual(lists[0], list(range(n)))
+ self.assertEqual(lists[1], list(range(n)))
+
+ # test argument format checking
+ self.assertRaises(TypeError, tee)
+ self.assertRaises(TypeError, tee, 3)
+ self.assertRaises(TypeError, tee, [1,2], 'x')
+ self.assertRaises(TypeError, tee, [1,2], 3, 'x')
+
+ # tee object should be instantiable
+ a, b = tee('abc')
+ c = type(a)('def')
+ self.assertEqual(list(c), list('def'))
+
+ # test long-lagged and multi-way split
+ a, b, c = tee(range(2000), 3)
+ for i in range(100):
+ self.assertEqual(next(a), i)
+ self.assertEqual(list(b), list(range(2000)))
+ self.assertEqual([next(c), next(c)], list(range(2)))
+ self.assertEqual(list(a), list(range(100,2000)))
+ self.assertEqual(list(c), list(range(2,2000)))
+
+ # test values of n
+ self.assertRaises(TypeError, tee, 'abc', 'invalid')
+ self.assertRaises(ValueError, tee, [], -1)
+ for n in range(5):
+ result = tee('abc', n)
+ self.assertEqual(type(result), tuple)
+ self.assertEqual(len(result), n)
+ self.assertEqual([list(x) for x in result], [list('abc')]*n)
+
+ # tee pass-through to copyable iterator
+ a, b = tee('abc')
+ c, d = tee(a)
+ self.assertTrue(a is c)
+
+ # test tee_new
+ t1, t2 = tee('abc')
+ tnew = type(t1)
+ self.assertRaises(TypeError, tnew)
+ self.assertRaises(TypeError, tnew, 10)
+ t3 = tnew(t1)
+ self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc'))
+
+ # test that tee objects are weak referencable
+ a, b = tee(range(10))
+ p = weakref.proxy(a)
+ self.assertEqual(getattr(p, '__class__'), type(b))
+ del a
+ self.assertRaises(ReferenceError, getattr, p, '__class__')
+
+ ans = list('abc')
+ long_ans = list(range(10000))
+
+ # check copy
+ a, b = tee('abc')
+ self.assertEqual(list(copy.copy(a)), ans)
+ self.assertEqual(list(copy.copy(b)), ans)
+ a, b = tee(list(range(10000)))
+ self.assertEqual(list(copy.copy(a)), long_ans)
+ self.assertEqual(list(copy.copy(b)), long_ans)
+
+ # check partially consumed copy
+ a, b = tee('abc')
+ take(2, a)
+ take(1, b)
+ self.assertEqual(list(copy.copy(a)), ans[2:])
+ self.assertEqual(list(copy.copy(b)), ans[1:])
+ self.assertEqual(list(a), ans[2:])
+ self.assertEqual(list(b), ans[1:])
+ a, b = tee(range(10000))
+ take(100, a)
+ take(60, b)
+ self.assertEqual(list(copy.copy(a)), long_ans[100:])
+ self.assertEqual(list(copy.copy(b)), long_ans[60:])
+ self.assertEqual(list(a), long_ans[100:])
+ self.assertEqual(list(b), long_ans[60:])
+
+ # check deepcopy
+ a, b = tee('abc')
+ self.assertEqual(list(copy.deepcopy(a)), ans)
+ self.assertEqual(list(copy.deepcopy(b)), ans)
+ self.assertEqual(list(a), ans)
+ self.assertEqual(list(b), ans)
+ a, b = tee(range(10000))
+ self.assertEqual(list(copy.deepcopy(a)), long_ans)
+ self.assertEqual(list(copy.deepcopy(b)), long_ans)
+ self.assertEqual(list(a), long_ans)
+ self.assertEqual(list(b), long_ans)
+
+ # check partially consumed deepcopy
+ a, b = tee('abc')
+ take(2, a)
+ take(1, b)
+ self.assertEqual(list(copy.deepcopy(a)), ans[2:])
+ self.assertEqual(list(copy.deepcopy(b)), ans[1:])
+ self.assertEqual(list(a), ans[2:])
+ self.assertEqual(list(b), ans[1:])
+ a, b = tee(range(10000))
+ take(100, a)
+ take(60, b)
+ self.assertEqual(list(copy.deepcopy(a)), long_ans[100:])
+ self.assertEqual(list(copy.deepcopy(b)), long_ans[60:])
+ self.assertEqual(list(a), long_ans[100:])
+ self.assertEqual(list(b), long_ans[60:])
+
+ # check pickle
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ self.pickletest(proto, iter(tee('abc')))
+ a, b = tee('abc')
+ self.pickletest(proto, a, compare=ans)
+ self.pickletest(proto, b, compare=ans)
+
+ # Issue 13454: Crash when deleting backward iterator from tee()
+ # TODO: RUSTPYTHON
+ @unittest.skip("hangs")
+ def test_tee_del_backward(self):
+ forward, backward = tee(repeat(None, 20000000))
+ try:
+ any(forward) # exhaust the iterator
+ del backward
+ except:
+ del forward, backward
+ raise
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_tee_reenter(self):
+ class I:
+ first = True
+ def __iter__(self):
+ return self
+ def __next__(self):
+ first = self.first
+ self.first = False
+ if first:
+ return next(b)
+
+ a, b = tee(I())
+ with self.assertRaisesRegex(RuntimeError, "tee"):
+ next(a)
+
+ # TODO: RUSTPYTHON - hangs
+ @unittest.skip("hangs")
+ def test_tee_concurrent(self):
+ start = threading.Event()
+ finish = threading.Event()
+ class I:
+ def __iter__(self):
+ return self
+ def __next__(self):
+ start.set()
+ finish.wait()
+
+ a, b = tee(I())
+ thread = threading.Thread(target=next, args=[a])
+ thread.start()
+ try:
+ start.wait()
+ with self.assertRaisesRegex(RuntimeError, "tee"):
+ next(b)
+ finally:
+ finish.set()
+ thread.join()
+
+ def test_StopIteration(self):
+ self.assertRaises(StopIteration, next, zip())
+
+ for f in (chain, cycle, zip, groupby):
+ self.assertRaises(StopIteration, next, f([]))
+ self.assertRaises(StopIteration, next, f(StopNow()))
+
+ self.assertRaises(StopIteration, next, islice([], None))
+ self.assertRaises(StopIteration, next, islice(StopNow(), None))
+
+ p, q = tee([])
+ self.assertRaises(StopIteration, next, p)
+ self.assertRaises(StopIteration, next, q)
+ p, q = tee(StopNow())
+ self.assertRaises(StopIteration, next, p)
+ self.assertRaises(StopIteration, next, q)
+
+ self.assertRaises(StopIteration, next, repeat(None, 0))
+
+ for f in (filter, filterfalse, map, takewhile, dropwhile, starmap):
+ self.assertRaises(StopIteration, next, f(lambda x:x, []))
+ self.assertRaises(StopIteration, next, f(lambda x:x, StopNow()))
+
+class TestExamples(unittest.TestCase):
+
+ def test_accumulate(self):
+ self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_accumulate_reducible(self):
+ # check copy, deepcopy, pickle
+ data = [1, 2, 3, 4, 5]
+ accumulated = [1, 3, 6, 10, 15]
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ it = accumulate(data)
+ self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:])
+ self.assertEqual(next(it), 1)
+ self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:])
+ it = accumulate(data)
+ self.assertEqual(next(it), 1)
+ self.assertEqual(list(copy.deepcopy(it)), accumulated[1:])
+ self.assertEqual(list(copy.copy(it)), accumulated[1:])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_accumulate_reducible_none(self):
+ # Issue #25718: total is None
+ it = accumulate([None, None, None], operator.is_)
+ self.assertEqual(next(it), None)
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ it_copy = pickle.loads(pickle.dumps(it, proto))
+ self.assertEqual(list(it_copy), [True, False])
+ self.assertEqual(list(copy.deepcopy(it)), [True, False])
+ self.assertEqual(list(copy.copy(it)), [True, False])
+
+ def test_chain(self):
+ self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF')
+
+ def test_chain_from_iterable(self):
+ self.assertEqual(''.join(chain.from_iterable(['ABC', 'DEF'])), 'ABCDEF')
+
+ def test_combinations(self):
+ self.assertEqual(list(combinations('ABCD', 2)),
+ [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+ self.assertEqual(list(combinations(range(4), 3)),
+ [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+
+ def test_combinations_with_replacement(self):
+ self.assertEqual(list(combinations_with_replacement('ABC', 2)),
+ [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
+ def test_compress(self):
+ self.assertEqual(list(compress('ABCDEF', [1,0,1,0,1,1])), list('ACEF'))
+
+ def test_count(self):
+ self.assertEqual(list(islice(count(10), 5)), [10, 11, 12, 13, 14])
+
+ def test_cycle(self):
+ self.assertEqual(list(islice(cycle('ABCD'), 12)), list('ABCDABCDABCD'))
+
+ def test_dropwhile(self):
+ self.assertEqual(list(dropwhile(lambda x: x<5, [1,4,6,4,1])), [6,4,1])
+
+ def test_groupby(self):
+ self.assertEqual([k for k, g in groupby('AAAABBBCCDAABBB')],
+ list('ABCDAB'))
+ self.assertEqual([(list(g)) for k, g in groupby('AAAABBBCCD')],
+ [list('AAAA'), list('BBB'), list('CC'), list('D')])
+
+ def test_filter(self):
+ self.assertEqual(list(filter(lambda x: x%2, range(10))), [1,3,5,7,9])
+
+ def test_filterfalse(self):
+ self.assertEqual(list(filterfalse(lambda x: x%2, range(10))), [0,2,4,6,8])
+
+ def test_map(self):
+ self.assertEqual(list(map(pow, (2,3,10), (5,2,3))), [32, 9, 1000])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_islice(self):
+ self.assertEqual(list(islice('ABCDEFG', 2)), list('AB'))
+ self.assertEqual(list(islice('ABCDEFG', 2, 4)), list('CD'))
+ self.assertEqual(list(islice('ABCDEFG', 2, None)), list('CDEFG'))
+ self.assertEqual(list(islice('ABCDEFG', 0, None, 2)), list('ACEG'))
+
+ def test_zip(self):
+ self.assertEqual(list(zip('ABCD', 'xy')), [('A', 'x'), ('B', 'y')])
+
+ def test_zip_longest(self):
+ self.assertEqual(list(zip_longest('ABCD', 'xy', fillvalue='-')),
+ [('A', 'x'), ('B', 'y'), ('C', '-'), ('D', '-')])
+
+ def test_permutations(self):
+ self.assertEqual(list(permutations('ABCD', 2)),
+ list(map(tuple, 'AB AC AD BA BC BD CA CB CD DA DB DC'.split())))
+ self.assertEqual(list(permutations(range(3))),
+ [(0,1,2), (0,2,1), (1,0,2), (1,2,0), (2,0,1), (2,1,0)])
+
+ def test_product(self):
+ self.assertEqual(list(product('ABCD', 'xy')),
+ list(map(tuple, 'Ax Ay Bx By Cx Cy Dx Dy'.split())))
+ self.assertEqual(list(product(range(2), repeat=3)),
+ [(0,0,0), (0,0,1), (0,1,0), (0,1,1),
+ (1,0,0), (1,0,1), (1,1,0), (1,1,1)])
+
+ def test_repeat(self):
+ self.assertEqual(list(repeat(10, 3)), [10, 10, 10])
+
+ def test_stapmap(self):
+ self.assertEqual(list(starmap(pow, [(2,5), (3,2), (10,3)])),
+ [32, 9, 1000])
+
+ def test_takewhile(self):
+ self.assertEqual(list(takewhile(lambda x: x<5, [1,4,6,4,1])), [1,4])
+
+
+class TestPurePythonRoughEquivalents(unittest.TestCase):
+
+ @staticmethod
+ def islice(iterable, *args):
+ s = slice(*args)
+ start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1
+ it = iter(range(start, stop, step))
+ try:
+ nexti = next(it)
+ except StopIteration:
+ # Consume *iterable* up to the *start* position.
+ for i, element in zip(range(start), iterable):
+ pass
+ return
+ try:
+ for i, element in enumerate(iterable):
+ if i == nexti:
+ yield element
+ nexti = next(it)
+ except StopIteration:
+ # Consume to *stop*.
+ for i, element in zip(range(i + 1, stop), iterable):
+ pass
+
+ def test_islice_recipe(self):
+ self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB'))
+ self.assertEqual(list(self.islice('ABCDEFG', 2, 4)), list('CD'))
+ self.assertEqual(list(self.islice('ABCDEFG', 2, None)), list('CDEFG'))
+ self.assertEqual(list(self.islice('ABCDEFG', 0, None, 2)), list('ACEG'))
+ # Test items consumed.
+ it = iter(range(10))
+ self.assertEqual(list(self.islice(it, 3)), list(range(3)))
+ self.assertEqual(list(it), list(range(3, 10)))
+ it = iter(range(10))
+ self.assertEqual(list(self.islice(it, 3, 3)), [])
+ self.assertEqual(list(it), list(range(3, 10)))
+ # Test that slice finishes in predictable state.
+ c = count()
+ self.assertEqual(list(self.islice(c, 1, 3, 50)), [1])
+ self.assertEqual(next(c), 3)
+
+
+class TestGC(unittest.TestCase):
+
+ def makecycle(self, iterator, container):
+ container.append(iterator)
+ next(iterator)
+ del container, iterator
+
+ def test_accumulate(self):
+ a = []
+ self.makecycle(accumulate([1,2,a,3]), a)
+
+ def test_chain(self):
+ a = []
+ self.makecycle(chain(a), a)
+
+ def test_chain_from_iterable(self):
+ a = []
+ self.makecycle(chain.from_iterable([a]), a)
+
+ def test_combinations(self):
+ a = []
+ self.makecycle(combinations([1,2,a,3], 3), a)
+
+ def test_combinations_with_replacement(self):
+ a = []
+ self.makecycle(combinations_with_replacement([1,2,a,3], 3), a)
+
+ def test_compress(self):
+ a = []
+ self.makecycle(compress('ABCDEF', [1,0,1,0,1,0]), a)
+
+ def test_count(self):
+ a = []
+ Int = type('Int', (int,), dict(x=a))
+ self.makecycle(count(Int(0), Int(1)), a)
+
+ def test_cycle(self):
+ a = []
+ self.makecycle(cycle([a]*2), a)
+
+ def test_dropwhile(self):
+ a = []
+ self.makecycle(dropwhile(bool, [0, a, a]), a)
+
+ def test_groupby(self):
+ a = []
+ self.makecycle(groupby([a]*2, lambda x:x), a)
+
+ def test_issue2246(self):
+ # Issue 2246 -- the _grouper iterator was not included in GC
+ n = 10
+ keyfunc = lambda x: x
+ for i, j in groupby(range(n), key=keyfunc):
+ keyfunc.__dict__.setdefault('x',[]).append(j)
+
+ def test_filter(self):
+ a = []
+ self.makecycle(filter(lambda x:True, [a]*2), a)
+
+ def test_filterfalse(self):
+ a = []
+ self.makecycle(filterfalse(lambda x:False, a), a)
+
+ def test_zip(self):
+ a = []
+ self.makecycle(zip([a]*2, [a]*3), a)
+
+ def test_zip_longest(self):
+ a = []
+ self.makecycle(zip_longest([a]*2, [a]*3), a)
+ b = [a, None]
+ self.makecycle(zip_longest([a]*2, [a]*3, fillvalue=b), a)
+
+ def test_map(self):
+ a = []
+ self.makecycle(map(lambda x:x, [a]*2), a)
+
+ def test_islice(self):
+ a = []
+ self.makecycle(islice([a]*2, None), a)
+
+ def test_permutations(self):
+ a = []
+ self.makecycle(permutations([1,2,a,3], 3), a)
+
+ def test_product(self):
+ a = []
+ self.makecycle(product([1,2,a,3], repeat=3), a)
+
+ def test_repeat(self):
+ a = []
+ self.makecycle(repeat(a), a)
+
+ def test_starmap(self):
+ a = []
+ self.makecycle(starmap(lambda *t: t, [(a,a)]*2), a)
+
+ def test_takewhile(self):
+ a = []
+ self.makecycle(takewhile(bool, [1, 0, a, a]), a)
+
+def R(seqn):
+ 'Regular generator'
+ for i in seqn:
+ yield i
+
+class G:
+ 'Sequence using __getitem__'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ def __getitem__(self, i):
+ return self.seqn[i]
+
+class I:
+ 'Sequence using iterator protocol'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ self.i = 0
+ def __iter__(self):
+ return self
+ def __next__(self):
+ if self.i >= len(self.seqn): raise StopIteration
+ v = self.seqn[self.i]
+ self.i += 1
+ return v
+
+class Ig:
+ 'Sequence using iterator protocol defined with a generator'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ self.i = 0
+ def __iter__(self):
+ for val in self.seqn:
+ yield val
+
+class X:
+ 'Missing __getitem__ and __iter__'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ self.i = 0
+ def __next__(self):
+ if self.i >= len(self.seqn): raise StopIteration
+ v = self.seqn[self.i]
+ self.i += 1
+ return v
+
+class N:
+ 'Iterator missing __next__()'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ self.i = 0
+ def __iter__(self):
+ return self
+
+class E:
+ 'Test propagation of exceptions'
+ def __init__(self, seqn):
+ self.seqn = seqn
+ self.i = 0
+ def __iter__(self):
+ return self
+ def __next__(self):
+ 3 // 0
+
+class S:
+ 'Test immediate stop'
+ def __init__(self, seqn):
+ pass
+ def __iter__(self):
+ return self
+ def __next__(self):
+ raise StopIteration
+
+def L(seqn):
+ 'Test multiple tiers of iterators'
+ return chain(map(lambda x:x, R(Ig(G(seqn)))))
+
+
+class TestVariousIteratorArgs(unittest.TestCase):
+ def test_accumulate(self):
+ s = [1,2,3,4,5]
+ r = [1,3,6,10,15]
+ n = len(s)
+ for g in (G, I, Ig, L, R):
+ self.assertEqual(list(accumulate(g(s))), r)
+ self.assertEqual(list(accumulate(S(s))), [])
+ self.assertRaises(TypeError, accumulate, X(s))
+ self.assertRaises(TypeError, accumulate, N(s))
+ self.assertRaises(ZeroDivisionError, list, accumulate(E(s)))
+
+ def test_chain(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(chain(g(s))), list(g(s)))
+ self.assertEqual(list(chain(g(s), g(s))), list(g(s))+list(g(s)))
+ self.assertRaises(TypeError, list, chain(X(s)))
+ self.assertRaises(TypeError, list, chain(N(s)))
+ self.assertRaises(ZeroDivisionError, list, chain(E(s)))
+
+ def test_compress(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ n = len(s)
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(compress(g(s), repeat(1))), list(g(s)))
+ self.assertRaises(TypeError, compress, X(s), repeat(1))
+ self.assertRaises(TypeError, compress, N(s), repeat(1))
+ self.assertRaises(ZeroDivisionError, list, compress(E(s), repeat(1)))
+
+ def test_product(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ self.assertRaises(TypeError, product, X(s))
+ self.assertRaises(TypeError, product, N(s))
+ self.assertRaises(ZeroDivisionError, product, E(s))
+
+ def test_cycle(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ tgtlen = len(s) * 3
+ expected = list(g(s))*3
+ actual = list(islice(cycle(g(s)), tgtlen))
+ self.assertEqual(actual, expected)
+ self.assertRaises(TypeError, cycle, X(s))
+ self.assertRaises(TypeError, cycle, N(s))
+ self.assertRaises(ZeroDivisionError, list, cycle(E(s)))
+
+ def test_groupby(self):
+ for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual([k for k, sb in groupby(g(s))], list(g(s)))
+ self.assertRaises(TypeError, groupby, X(s))
+ self.assertRaises(TypeError, groupby, N(s))
+ self.assertRaises(ZeroDivisionError, list, groupby(E(s)))
+
+ def test_filter(self):
+ for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(filter(isEven, g(s))),
+ [x for x in g(s) if isEven(x)])
+ self.assertRaises(TypeError, filter, isEven, X(s))
+ self.assertRaises(TypeError, filter, isEven, N(s))
+ self.assertRaises(ZeroDivisionError, list, filter(isEven, E(s)))
+
+ def test_filterfalse(self):
+ for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(filterfalse(isEven, g(s))),
+ [x for x in g(s) if isOdd(x)])
+ self.assertRaises(TypeError, filterfalse, isEven, X(s))
+ self.assertRaises(TypeError, filterfalse, isEven, N(s))
+ self.assertRaises(ZeroDivisionError, list, filterfalse(isEven, E(s)))
+
+ def test_zip(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(zip(g(s))), lzip(g(s)))
+ self.assertEqual(list(zip(g(s), g(s))), lzip(g(s), g(s)))
+ self.assertRaises(TypeError, zip, X(s))
+ self.assertRaises(TypeError, zip, N(s))
+ self.assertRaises(ZeroDivisionError, list, zip(E(s)))
+
+ def test_ziplongest(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(zip_longest(g(s))), list(zip(g(s))))
+ self.assertEqual(list(zip_longest(g(s), g(s))), list(zip(g(s), g(s))))
+ self.assertRaises(TypeError, zip_longest, X(s))
+ self.assertRaises(TypeError, zip_longest, N(s))
+ self.assertRaises(ZeroDivisionError, list, zip_longest(E(s)))
+
+ def test_map(self):
+ for s in (range(10), range(0), range(100), (7,11), range(20,50,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(map(onearg, g(s))),
+ [onearg(x) for x in g(s)])
+ self.assertEqual(list(map(operator.pow, g(s), g(s))),
+ [x**x for x in g(s)])
+ self.assertRaises(TypeError, map, onearg, X(s))
+ self.assertRaises(TypeError, map, onearg, N(s))
+ self.assertRaises(ZeroDivisionError, list, map(onearg, E(s)))
+
+ def test_islice(self):
+ for s in ("12345", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ self.assertEqual(list(islice(g(s),1,None,2)), list(g(s))[1::2])
+ self.assertRaises(TypeError, islice, X(s), 10)
+ self.assertRaises(TypeError, islice, N(s), 10)
+ self.assertRaises(ZeroDivisionError, list, islice(E(s), 10))
+
+ def test_starmap(self):
+ for s in (range(10), range(0), range(100), (7,11), range(20,50,5)):
+ for g in (G, I, Ig, S, L, R):
+ ss = lzip(s, s)
+ self.assertEqual(list(starmap(operator.pow, g(ss))),
+ [x**x for x in g(s)])
+ self.assertRaises(TypeError, starmap, operator.pow, X(ss))
+ self.assertRaises(TypeError, starmap, operator.pow, N(ss))
+ self.assertRaises(ZeroDivisionError, list, starmap(operator.pow, E(ss)))
+
+ def test_takewhile(self):
+ for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ tgt = []
+ for elem in g(s):
+ if not isEven(elem): break
+ tgt.append(elem)
+ self.assertEqual(list(takewhile(isEven, g(s))), tgt)
+ self.assertRaises(TypeError, takewhile, isEven, X(s))
+ self.assertRaises(TypeError, takewhile, isEven, N(s))
+ self.assertRaises(ZeroDivisionError, list, takewhile(isEven, E(s)))
+
+ def test_dropwhile(self):
+ for s in (range(10), range(0), range(1000), (7,11), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ tgt = []
+ for elem in g(s):
+ if not tgt and isOdd(elem): continue
+ tgt.append(elem)
+ self.assertEqual(list(dropwhile(isOdd, g(s))), tgt)
+ self.assertRaises(TypeError, dropwhile, isOdd, X(s))
+ self.assertRaises(TypeError, dropwhile, isOdd, N(s))
+ self.assertRaises(ZeroDivisionError, list, dropwhile(isOdd, E(s)))
+
+ def test_tee(self):
+ for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)):
+ for g in (G, I, Ig, S, L, R):
+ it1, it2 = tee(g(s))
+ self.assertEqual(list(it1), list(g(s)))
+ self.assertEqual(list(it2), list(g(s)))
+ self.assertRaises(TypeError, tee, X(s))
+ self.assertRaises(TypeError, tee, N(s))
+ self.assertRaises(ZeroDivisionError, list, tee(E(s))[0])
+
+class LengthTransparency(unittest.TestCase):
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_repeat(self):
+ self.assertEqual(operator.length_hint(repeat(None, 50)), 50)
+ self.assertEqual(operator.length_hint(repeat(None, 0)), 0)
+ self.assertEqual(operator.length_hint(repeat(None), 12), 12)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_repeat_with_negative_times(self):
+ self.assertEqual(operator.length_hint(repeat(None, -1)), 0)
+ self.assertEqual(operator.length_hint(repeat(None, -2)), 0)
+ self.assertEqual(operator.length_hint(repeat(None, times=-1)), 0)
+ self.assertEqual(operator.length_hint(repeat(None, times=-2)), 0)
+
+class RegressionTests(unittest.TestCase):
+
+ def test_sf_793826(self):
+ # Fix Armin Rigo's successful efforts to wreak havoc
+
+ def mutatingtuple(tuple1, f, tuple2):
+ # this builds a tuple t which is a copy of tuple1,
+ # then calls f(t), then mutates t to be equal to tuple2
+ # (needs len(tuple1) == len(tuple2)).
+ def g(value, first=[1]):
+ if first:
+ del first[:]
+ f(next(z))
+ return value
+ items = list(tuple2)
+ items[1:1] = list(tuple1)
+ gen = map(g, items)
+ z = zip(*[gen]*len(tuple1))
+ next(z)
+
+ def f(t):
+ global T
+ T = t
+ first[:] = list(T)
+
+ first = []
+ mutatingtuple((1,2,3), f, (4,5,6))
+ second = list(T)
+ self.assertEqual(first, second)
+
+
+ def test_sf_950057(self):
+ # Make sure that chain() and cycle() catch exceptions immediately
+ # rather than when shifting between input sources
+
+ def gen1():
+ hist.append(0)
+ yield 1
+ hist.append(1)
+ raise AssertionError
+ hist.append(2)
+
+ def gen2(x):
+ hist.append(3)
+ yield 2
+ hist.append(4)
+
+ hist = []
+ self.assertRaises(AssertionError, list, chain(gen1(), gen2(False)))
+ self.assertEqual(hist, [0,1])
+
+ hist = []
+ self.assertRaises(AssertionError, list, chain(gen1(), gen2(True)))
+ self.assertEqual(hist, [0,1])
+
+ hist = []
+ self.assertRaises(AssertionError, list, cycle(gen1()))
+ self.assertEqual(hist, [0,1])
+
+ @support.skip_if_pgo_task
+ def test_long_chain_of_empty_iterables(self):
+ # Make sure itertools.chain doesn't run into recursion limits when
+ # dealing with long chains of empty iterables. Even with a high
+ # number this would probably only fail in Py_DEBUG mode.
+ it = chain.from_iterable(() for unused in range(10000000))
+ with self.assertRaises(StopIteration):
+ next(it)
+
+ def test_issue30347_1(self):
+ def f(n):
+ if n == 5:
+ list(b)
+ return n != 6
+ for (k, b) in groupby(range(10), f):
+ list(b) # shouldn't crash
+
+ def test_issue30347_2(self):
+ class K:
+ def __init__(self, v):
+ pass
+ def __eq__(self, other):
+ nonlocal i
+ i += 1
+ if i == 1:
+ next(g, None)
+ return True
+ i = 0
+ g = next(groupby(range(10), K))[1]
+ for j in range(2):
+ next(g, None) # shouldn't crash
+
+
+class SubclassWithKwargsTest(unittest.TestCase):
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_keywords_in_subclass(self):
+ # count is not subclassable...
+ for cls in (repeat, zip, filter, filterfalse, chain, map,
+ starmap, islice, takewhile, dropwhile, cycle, compress):
+ class Subclass(cls):
+ def __init__(self, newarg=None, *args):
+ cls.__init__(self, *args)
+ try:
+ Subclass(newarg=1)
+ except TypeError as err:
+ # we expect type errors because of wrong argument count
+ self.assertNotIn("keyword arguments", err.args[0])
+
+@support.cpython_only
+class SizeofTest(unittest.TestCase):
+ def setUp(self):
+ self.ssize_t = struct.calcsize('n')
+
+ check_sizeof = support.check_sizeof
+
+ def test_product_sizeof(self):
+ basesize = support.calcobjsize('3Pi')
+ check = self.check_sizeof
+ check(product('ab', '12'), basesize + 2 * self.ssize_t)
+ check(product(*(('abc',) * 10)), basesize + 10 * self.ssize_t)
+
+ def test_combinations_sizeof(self):
+ basesize = support.calcobjsize('3Pni')
+ check = self.check_sizeof
+ check(combinations('abcd', 3), basesize + 3 * self.ssize_t)
+ check(combinations(range(10), 4), basesize + 4 * self.ssize_t)
+
+ def test_combinations_with_replacement_sizeof(self):
+ cwr = combinations_with_replacement
+ basesize = support.calcobjsize('3Pni')
+ check = self.check_sizeof
+ check(cwr('abcd', 3), basesize + 3 * self.ssize_t)
+ check(cwr(range(10), 4), basesize + 4 * self.ssize_t)
+
+ def test_permutations_sizeof(self):
+ basesize = support.calcobjsize('4Pni')
+ check = self.check_sizeof
+ check(permutations('abcd'),
+ basesize + 4 * self.ssize_t + 4 * self.ssize_t)
+ check(permutations('abcd', 3),
+ basesize + 4 * self.ssize_t + 3 * self.ssize_t)
+ check(permutations('abcde', 3),
+ basesize + 5 * self.ssize_t + 3 * self.ssize_t)
+ check(permutations(range(10), 4),
+ basesize + 10 * self.ssize_t + 4 * self.ssize_t)
+
+
+libreftest = """ Doctest for examples in the library reference: libitertools.tex
+
+
+>>> amounts = [120.15, 764.05, 823.14]
+>>> for checknum, amount in zip(count(1200), amounts):
+... print('Check %d is for $%.2f' % (checknum, amount))
+...
+Check 1200 is for $120.15
+Check 1201 is for $764.05
+Check 1202 is for $823.14
+
+>>> import operator
+>>> for cube in map(operator.pow, range(1,4), repeat(3)):
+... print(cube)
+...
+1
+8
+27
+
+>>> reportlines = ['EuroPython', 'Roster', '', 'alex', '', 'laura', '', 'martin', '', 'walter', '', 'samuele']
+>>> for name in islice(reportlines, 3, None, 2):
+... print(name.title())
+...
+Alex
+Laura
+Martin
+Walter
+Samuele
+
+>>> from operator import itemgetter
+>>> d = dict(a=1, b=2, c=1, d=2, e=1, f=2, g=3)
+>>> di = sorted(sorted(d.items()), key=itemgetter(1))
+>>> for k, g in groupby(di, itemgetter(1)):
+... print(k, list(map(itemgetter(0), g)))
+...
+1 ['a', 'c', 'e']
+2 ['b', 'd', 'f']
+3 ['g']
+
+# Find runs of consecutive numbers using groupby. The key to the solution
+# is differencing with a range so that consecutive numbers all appear in
+# same group.
+>>> data = [ 1, 4,5,6, 10, 15,16,17,18, 22, 25,26,27,28]
+>>> for k, g in groupby(enumerate(data), lambda t:t[0]-t[1]):
+... print(list(map(operator.itemgetter(1), g)))
+...
+[1]
+[4, 5, 6]
+[10]
+[15, 16, 17, 18]
+[22]
+[25, 26, 27, 28]
+
+>>> def take(n, iterable):
+... "Return first n items of the iterable as a list"
+... return list(islice(iterable, n))
+
+>>> def prepend(value, iterator):
+... "Prepend a single value in front of an iterator"
+... # prepend(1, [2, 3, 4]) -> 1 2 3 4
+... return chain([value], iterator)
+
+>>> def enumerate(iterable, start=0):
+... return zip(count(start), iterable)
+
+>>> def tabulate(function, start=0):
+... "Return function(0), function(1), ..."
+... return map(function, count(start))
+
+>>> import collections
+>>> def consume(iterator, n=None):
+... "Advance the iterator n-steps ahead. If n is None, consume entirely."
+... # Use functions that consume iterators at C speed.
+... if n is None:
+... # feed the entire iterator into a zero-length deque
+... collections.deque(iterator, maxlen=0)
+... else:
+... # advance to the empty slice starting at position n
+... next(islice(iterator, n, n), None)
+
+>>> def nth(iterable, n, default=None):
+... "Returns the nth item or a default value"
+... return next(islice(iterable, n, None), default)
+
+>>> def all_equal(iterable):
+... "Returns True if all the elements are equal to each other"
+... g = groupby(iterable)
+... return next(g, True) and not next(g, False)
+
+>>> def quantify(iterable, pred=bool):
+... "Count how many times the predicate is true"
+... return sum(map(pred, iterable))
+
+>>> def padnone(iterable):
+... "Returns the sequence elements and then returns None indefinitely"
+... return chain(iterable, repeat(None))
+
+>>> def ncycles(iterable, n):
+... "Returns the sequence elements n times"
+... return chain(*repeat(iterable, n))
+
+>>> def dotproduct(vec1, vec2):
+... return sum(map(operator.mul, vec1, vec2))
+
+>>> def flatten(listOfLists):
+... return list(chain.from_iterable(listOfLists))
+
+>>> def repeatfunc(func, times=None, *args):
+... "Repeat calls to func with specified arguments."
+... " Example: repeatfunc(random.random)"
+... if times is None:
+... return starmap(func, repeat(args))
+... else:
+... return starmap(func, repeat(args, times))
+
+>>> def pairwise(iterable):
+... "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+... a, b = tee(iterable)
+... try:
+... next(b)
+... except StopIteration:
+... pass
+... return zip(a, b)
+
+>>> def grouper(n, iterable, fillvalue=None):
+... "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
+... args = [iter(iterable)] * n
+... return zip_longest(*args, fillvalue=fillvalue)
+
+>>> def roundrobin(*iterables):
+... "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
+... # Recipe credited to George Sakkis
+... pending = len(iterables)
+... nexts = cycle(iter(it).__next__ for it in iterables)
+... while pending:
+... try:
+... for next in nexts:
+... yield next()
+... except StopIteration:
+... pending -= 1
+... nexts = cycle(islice(nexts, pending))
+
+>>> def powerset(iterable):
+... "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"
+... s = list(iterable)
+... return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))
+
+>>> def unique_everseen(iterable, key=None):
+... "List unique elements, preserving order. Remember all elements ever seen."
+... # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+... # unique_everseen('ABBCcAD', str.lower) --> A B C D
+... seen = set()
+... seen_add = seen.add
+... if key is None:
+... for element in iterable:
+... if element not in seen:
+... seen_add(element)
+... yield element
+... else:
+... for element in iterable:
+... k = key(element)
+... if k not in seen:
+... seen_add(k)
+... yield element
+
+>>> def unique_justseen(iterable, key=None):
+... "List unique elements, preserving order. Remember only the element just seen."
+... # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B
+... # unique_justseen('ABBCcAD', str.lower) --> A B C A D
+... return map(next, map(itemgetter(1), groupby(iterable, key)))
+
+>>> def first_true(iterable, default=False, pred=None):
+... '''Returns the first true value in the iterable.
+...
+... If no true value is found, returns *default*
+...
+... If *pred* is not None, returns the first item
+... for which pred(item) is true.
+...
+... '''
+... # first_true([a,b,c], x) --> a or b or c or x
+... # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x
+... return next(filter(pred, iterable), default)
+
+>>> def nth_combination(iterable, r, index):
+... 'Equivalent to list(combinations(iterable, r))[index]'
+... pool = tuple(iterable)
+... n = len(pool)
+... if r < 0 or r > n:
+... raise ValueError
+... c = 1
+... k = min(r, n-r)
+... for i in range(1, k+1):
+... c = c * (n - k + i) // i
+... if index < 0:
+... index += c
+... if index < 0 or index >= c:
+... raise IndexError
+... result = []
+... while r:
+... c, n, r = c*r//n, n-1, r-1
+... while index >= c:
+... index -= c
+... c, n = c*(n-r)//n, n-1
+... result.append(pool[-1-n])
+... return tuple(result)
+
+
+This is not part of the examples but it tests to make sure the definitions
+perform as purported.
+
+>>> take(10, count())
+[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+>>> list(prepend(1, [2, 3, 4]))
+[1, 2, 3, 4]
+
+>>> list(enumerate('abc'))
+[(0, 'a'), (1, 'b'), (2, 'c')]
+
+>>> list(islice(tabulate(lambda x: 2*x), 4))
+[0, 2, 4, 6]
+
+>>> it = iter(range(10))
+>>> consume(it, 3)
+>>> next(it)
+3
+>>> consume(it)
+>>> next(it, 'Done')
+'Done'
+
+>>> nth('abcde', 3)
+'d'
+
+>>> nth('abcde', 9) is None
+True
+
+>>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')]
+[True, True, True, False, False]
+
+>>> quantify(range(99), lambda x: x%2==0)
+50
+
+>>> a = [[1, 2, 3], [4, 5, 6]]
+>>> flatten(a)
+[1, 2, 3, 4, 5, 6]
+
+>>> list(repeatfunc(pow, 5, 2, 3))
+[8, 8, 8, 8, 8]
+
+>>> import random
+>>> take(5, map(int, repeatfunc(random.random)))
+[0, 0, 0, 0, 0]
+
+>>> list(pairwise('abcd'))
+[('a', 'b'), ('b', 'c'), ('c', 'd')]
+
+>>> list(pairwise([]))
+[]
+
+>>> list(pairwise('a'))
+[]
+
+>>> list(islice(padnone('abc'), 0, 6))
+['a', 'b', 'c', None, None, None]
+
+>>> list(ncycles('abc', 3))
+['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c']
+
+>>> dotproduct([1,2,3], [4,5,6])
+32
+
+>>> list(grouper(3, 'abcdefg', 'x'))
+[('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')]
+
+>>> list(roundrobin('abc', 'd', 'ef'))
+['a', 'd', 'e', 'b', 'f', 'c']
+
+>>> list(powerset([1,2,3]))
+[(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)]
+
+>>> all(len(list(powerset(range(n)))) == 2**n for n in range(18))
+True
+
+>>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len)
+True
+
+>>> list(unique_everseen('AAAABBBCCDAABBB'))
+['A', 'B', 'C', 'D']
+
+>>> list(unique_everseen('ABBCcAD', str.lower))
+['A', 'B', 'C', 'D']
+
+>>> list(unique_justseen('AAAABBBCCDAABBB'))
+['A', 'B', 'C', 'D', 'A', 'B']
+
+>>> list(unique_justseen('ABBCcAD', str.lower))
+['A', 'B', 'C', 'A', 'D']
+
+>>> first_true('ABC0DEF1', '9', str.isdigit)
+'0'
+
+>>> population = 'ABCDEFGH'
+>>> for r in range(len(population) + 1):
+... seq = list(combinations(population, r))
+... for i in range(len(seq)):
+... assert nth_combination(population, r, i) == seq[i]
+... for i in range(-len(seq), 0):
+... assert nth_combination(population, r, i) == seq[i]
+
+
+"""
+
+__test__ = {'libreftest' : libreftest}
+
+def test_main(verbose=None):
+ test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC,
+ RegressionTests, LengthTransparency,
+ SubclassWithKwargsTest, TestExamples,
+ TestPurePythonRoughEquivalents,
+ SizeofTest)
+ support.run_unittest(*test_classes)
+
+ # verify reference counting
+ if verbose and hasattr(sys, "gettotalrefcount"):
+ import gc
+ counts = [None] * 5
+ for i in range(len(counts)):
+ support.run_unittest(*test_classes)
+ gc.collect()
+ counts[i] = sys.gettotalrefcount()
+ print(counts)
+
+ # TODO: RUSTPYTHON this hangs or is very slow
+ # doctest the examples in the library reference
+ # support.run_doctest(sys.modules[__name__], verbose)
+
+if __name__ == "__main__":
+ test_main(verbose=True)
diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py
index d8cc54050..f362f1b13 100644
--- a/Lib/test/test_json/test_tool.py
+++ b/Lib/test/test_json/test_tool.py
@@ -7,7 +7,6 @@ from test import support
from test.support.script_helper import assert_python_ok
-@unittest.skip("TODO: RUSTPYTHON, rustpython -X faulthandler")
class TestTool(unittest.TestCase):
data = """
diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py
new file mode 100644
index 000000000..c13a7c929
--- /dev/null
+++ b/Lib/test/test_pwd.py
@@ -0,0 +1,112 @@
+import sys
+import unittest
+from test import support
+
+pwd = support.import_module('pwd')
+
+@unittest.skipUnless(hasattr(pwd, 'getpwall'), 'Does not have getpwall()')
+class PwdTest(unittest.TestCase):
+
+ def test_values(self):
+ entries = pwd.getpwall()
+
+ for e in entries:
+ self.assertEqual(len(e), 7)
+ self.assertEqual(e[0], e.pw_name)
+ self.assertIsInstance(e.pw_name, str)
+ self.assertEqual(e[1], e.pw_passwd)
+ self.assertIsInstance(e.pw_passwd, str)
+ self.assertEqual(e[2], e.pw_uid)
+ self.assertIsInstance(e.pw_uid, int)
+ self.assertEqual(e[3], e.pw_gid)
+ self.assertIsInstance(e.pw_gid, int)
+ self.assertEqual(e[4], e.pw_gecos)
+ self.assertIsInstance(e.pw_gecos, str)
+ self.assertEqual(e[5], e.pw_dir)
+ self.assertIsInstance(e.pw_dir, str)
+ self.assertEqual(e[6], e.pw_shell)
+ self.assertIsInstance(e.pw_shell, str)
+
+ # The following won't work, because of duplicate entries
+ # for one uid
+ # self.assertEqual(pwd.getpwuid(e.pw_uid), e)
+ # instead of this collect all entries for one uid
+ # and check afterwards (done in test_values_extended)
+
+ def test_values_extended(self):
+ entries = pwd.getpwall()
+ entriesbyname = {}
+ entriesbyuid = {}
+
+ if len(entries) > 1000: # Huge passwd file (NIS?) -- skip this test
+ self.skipTest('passwd file is huge; extended test skipped')
+
+ for e in entries:
+ entriesbyname.setdefault(e.pw_name, []).append(e)
+ entriesbyuid.setdefault(e.pw_uid, []).append(e)
+
+ # check whether the entry returned by getpwuid()
+ # for each uid is among those from getpwall() for this uid
+ for e in entries:
+ if not e[0] or e[0] == '+':
+ continue # skip NIS entries etc.
+ self.assertIn(pwd.getpwnam(e.pw_name), entriesbyname[e.pw_name])
+ self.assertIn(pwd.getpwuid(e.pw_uid), entriesbyuid[e.pw_uid])
+
+ def test_errors(self):
+ self.assertRaises(TypeError, pwd.getpwuid)
+ self.assertRaises(TypeError, pwd.getpwuid, 3.14)
+ self.assertRaises(TypeError, pwd.getpwnam)
+ self.assertRaises(TypeError, pwd.getpwnam, 42)
+ self.assertRaises(TypeError, pwd.getpwall, 42)
+
+ # try to get some errors
+ bynames = {}
+ byuids = {}
+ for (n, p, u, g, gecos, d, s) in pwd.getpwall():
+ bynames[n] = u
+ byuids[u] = n
+
+ allnames = list(bynames.keys())
+ namei = 0
+ fakename = allnames[namei]
+ while fakename in bynames:
+ chars = list(fakename)
+ for i in range(len(chars)):
+ if chars[i] == 'z':
+ chars[i] = 'A'
+ break
+ elif chars[i] == 'Z':
+ continue
+ else:
+ chars[i] = chr(ord(chars[i]) + 1)
+ break
+ else:
+ namei = namei + 1
+ try:
+ fakename = allnames[namei]
+ except IndexError:
+ # should never happen... if so, just forget it
+ break
+ fakename = ''.join(chars)
+
+ self.assertRaises(KeyError, pwd.getpwnam, fakename)
+
+ # In some cases, byuids isn't a complete list of all users in the
+ # system, so if we try to pick a value not in byuids (via a perturbing
+ # loop, say), pwd.getpwuid() might still be able to find data for that
+ # uid. Using sys.maxint may provoke the same problems, but hopefully
+ # it will be a more repeatable failure.
+ fakeuid = sys.maxsize
+ self.assertNotIn(fakeuid, byuids)
+ self.assertRaises(KeyError, pwd.getpwuid, fakeuid)
+
+ # -1 shouldn't be a valid uid because it has a special meaning in many
+ # uid-related functions
+ self.assertRaises(KeyError, pwd.getpwuid, -1)
+ # should be out of uid_t range
+ self.assertRaises(KeyError, pwd.getpwuid, 2**128)
+ self.assertRaises(KeyError, pwd.getpwuid, -2**128)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
new file mode 100644
index 000000000..8dc52c29f
--- /dev/null
+++ b/Lib/test/test_py_compile.py
@@ -0,0 +1,215 @@
+import functools
+import importlib.util
+import os
+import py_compile
+import shutil
+import stat
+import sys
+import tempfile
+import unittest
+
+from test import support
+
+
+def without_source_date_epoch(fxn):
+ """Runs function with SOURCE_DATE_EPOCH unset."""
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ with support.EnvironmentVarGuard() as env:
+ env.unset('SOURCE_DATE_EPOCH')
+ return fxn(*args, **kwargs)
+ return wrapper
+
+
+def with_source_date_epoch(fxn):
+ """Runs function with SOURCE_DATE_EPOCH set."""
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ with support.EnvironmentVarGuard() as env:
+ env['SOURCE_DATE_EPOCH'] = '123456789'
+ return fxn(*args, **kwargs)
+ return wrapper
+
+
+# Run tests with SOURCE_DATE_EPOCH set or unset explicitly.
+class SourceDateEpochTestMeta(type(unittest.TestCase)):
+ def __new__(mcls, name, bases, dct, *, source_date_epoch):
+ cls = super().__new__(mcls, name, bases, dct)
+
+ for attr in dir(cls):
+ if attr.startswith('test_'):
+ meth = getattr(cls, attr)
+ if source_date_epoch:
+ wrapper = with_source_date_epoch(meth)
+ else:
+ wrapper = without_source_date_epoch(meth)
+ setattr(cls, attr, wrapper)
+
+ return cls
+
+
+class PyCompileTestsBase:
+
+ def setUp(self):
+ self.directory = tempfile.mkdtemp()
+ self.source_path = os.path.join(self.directory, '_test.py')
+ self.pyc_path = self.source_path + 'c'
+ self.cache_path = importlib.util.cache_from_source(self.source_path)
+ self.cwd_drive = os.path.splitdrive(os.getcwd())[0]
+ # In these tests we compute relative paths. When using Windows, the
+ # current working directory path and the 'self.source_path' might be
+ # on different drives. Therefore we need to switch to the drive where
+ # the temporary source file lives.
+ drive = os.path.splitdrive(self.source_path)[0]
+ if drive:
+ os.chdir(drive)
+ with open(self.source_path, 'w') as file:
+ file.write('x = 123\n')
+
+ def tearDown(self):
+ shutil.rmtree(self.directory)
+ if self.cwd_drive:
+ os.chdir(self.cwd_drive)
+
+ def test_absolute_path(self):
+ py_compile.compile(self.source_path, self.pyc_path)
+ self.assertTrue(os.path.exists(self.pyc_path))
+ self.assertFalse(os.path.exists(self.cache_path))
+
+ def test_do_not_overwrite_symlinks(self):
+ # In the face of a cfile argument being a symlink, bail out.
+ # Issue #17222
+ try:
+ os.symlink(self.pyc_path + '.actual', self.pyc_path)
+ except (NotImplementedError, OSError):
+ self.skipTest('need to be able to create a symlink for a file')
+ else:
+ assert os.path.islink(self.pyc_path)
+ with self.assertRaises(FileExistsError):
+ py_compile.compile(self.source_path, self.pyc_path)
+
+ @unittest.skipIf(not os.path.exists(os.devnull) or os.path.isfile(os.devnull),
+ 'requires os.devnull and for it to be a non-regular file')
+ def test_do_not_overwrite_nonregular_files(self):
+ # In the face of a cfile argument being a non-regular file, bail out.
+ # Issue #17222
+ with self.assertRaises(FileExistsError):
+ py_compile.compile(self.source_path, os.devnull)
+
+ def test_cache_path(self):
+ py_compile.compile(self.source_path)
+ self.assertTrue(os.path.exists(self.cache_path))
+
+ def test_cwd(self):
+ with support.change_cwd(self.directory):
+ py_compile.compile(os.path.basename(self.source_path),
+ os.path.basename(self.pyc_path))
+ self.assertTrue(os.path.exists(self.pyc_path))
+ self.assertFalse(os.path.exists(self.cache_path))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_relative_path(self):
+ py_compile.compile(os.path.relpath(self.source_path),
+ os.path.relpath(self.pyc_path))
+ self.assertTrue(os.path.exists(self.pyc_path))
+ self.assertFalse(os.path.exists(self.cache_path))
+
+ @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
+ 'non-root user required')
+ @unittest.skipIf(os.name == 'nt',
+ 'cannot control directory permissions on Windows')
+ def test_exceptions_propagate(self):
+ # Make sure that exceptions raised thanks to issues with writing
+ # bytecode.
+ # http://bugs.python.org/issue17244
+ mode = os.stat(self.directory)
+ os.chmod(self.directory, stat.S_IREAD)
+ try:
+ with self.assertRaises(IOError):
+ py_compile.compile(self.source_path, self.pyc_path)
+ finally:
+ os.chmod(self.directory, mode.st_mode)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_bad_coding(self):
+ bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py')
+ with support.captured_stderr():
+ self.assertIsNone(py_compile.compile(bad_coding, doraise=False))
+ self.assertFalse(os.path.exists(
+ importlib.util.cache_from_source(bad_coding)))
+
+ def test_source_date_epoch(self):
+ py_compile.compile(self.source_path, self.pyc_path)
+ self.assertTrue(os.path.exists(self.pyc_path))
+ self.assertFalse(os.path.exists(self.cache_path))
+ with open(self.pyc_path, 'rb') as fp:
+ flags = importlib._bootstrap_external._classify_pyc(
+ fp.read(), 'test', {})
+ if os.environ.get('SOURCE_DATE_EPOCH'):
+ expected_flags = 0b11
+ else:
+ expected_flags = 0b00
+
+ self.assertEqual(flags, expected_flags)
+
+ @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O')
+ def test_double_dot_no_clobber(self):
+ # http://bugs.python.org/issue22966
+ # py_compile foo.bar.py -> __pycache__/foo.cpython-34.pyc
+ weird_path = os.path.join(self.directory, 'foo.bar.py')
+ cache_path = importlib.util.cache_from_source(weird_path)
+ pyc_path = weird_path + 'c'
+ head, tail = os.path.split(cache_path)
+ penultimate_tail = os.path.basename(head)
+ self.assertEqual(
+ os.path.join(penultimate_tail, tail),
+ os.path.join(
+ '__pycache__',
+ 'foo.bar.{}.pyc'.format(sys.implementation.cache_tag)))
+ with open(weird_path, 'w') as file:
+ file.write('x = 123\n')
+ py_compile.compile(weird_path)
+ self.assertTrue(os.path.exists(cache_path))
+ self.assertFalse(os.path.exists(pyc_path))
+
+ def test_optimization_path(self):
+ # Specifying optimized bytecode should lead to a path reflecting that.
+ self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_invalidation_mode(self):
+ py_compile.compile(
+ self.source_path,
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+ )
+ with open(self.cache_path, 'rb') as fp:
+ flags = importlib._bootstrap_external._classify_pyc(
+ fp.read(), 'test', {})
+ self.assertEqual(flags, 0b11)
+ py_compile.compile(
+ self.source_path,
+ invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+ )
+ with open(self.cache_path, 'rb') as fp:
+ flags = importlib._bootstrap_external._classify_pyc(
+ fp.read(), 'test', {})
+ self.assertEqual(flags, 0b1)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class PyCompileTestsWithSourceEpoch(PyCompileTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
+class PyCompileTestsWithoutSourceEpoch(PyCompileTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=False):
+ pass
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
new file mode 100644
index 000000000..71f192f90
--- /dev/null
+++ b/Lib/test/test_repl.py
@@ -0,0 +1,98 @@
+"""Test the interactive interpreter."""
+
+import sys
+import os
+import unittest
+import subprocess
+from textwrap import dedent
+from test.support import cpython_only, SuppressCrashReport
+from test.support.script_helper import kill_python
+
+def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
+ """Run the Python REPL with the given arguments.
+
+ kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
+ object.
+ """
+
+ # To run the REPL without using a terminal, spawn python with the command
+ # line option '-i' and the process name set to ''.
+ # The directory of argv[0] must match the directory of the Python
+ # executable for the Popen() call to python to succeed as the directory
+ # path may be used by Py_GetPath() to build the default module search
+ # path.
+ stdin_fname = os.path.join(os.path.dirname(sys.executable), "")
+ cmd_line = [stdin_fname, '-E', '-i']
+ cmd_line.extend(args)
+
+ # Set TERM=vt100, for the rationale see the comments in spawn_python() of
+ # test.support.script_helper.
+ env = kw.setdefault('env', dict(os.environ))
+ env['TERM'] = 'vt100'
+ return subprocess.Popen(cmd_line, executable=sys.executable,
+ stdin=subprocess.PIPE,
+ stdout=stdout, stderr=stderr,
+ **kw)
+
+class TestInteractiveInterpreter(unittest.TestCase):
+
+ @cpython_only
+ def test_no_memory(self):
+ # Issue #30696: Fix the interactive interpreter looping endlessly when
+ # no memory. Check also that the fix does not break the interactive
+ # loop when an exception is raised.
+ user_input = """
+ import sys, _testcapi
+ 1/0
+ print('After the exception.')
+ _testcapi.set_nomemory(0)
+ sys.exit(0)
+ """
+ user_input = dedent(user_input)
+ user_input = user_input.encode()
+ p = spawn_repl()
+ with SuppressCrashReport():
+ p.stdin.write(user_input)
+ output = kill_python(p)
+ self.assertIn(b'After the exception.', output)
+ # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
+ self.assertIn(p.returncode, (1, 120))
+
+ @cpython_only
+ def test_multiline_string_parsing(self):
+ # bpo-39209: Multiline string tokens need to be handled in the tokenizer
+ # in two places: the interactive path and the non-interactive path.
+ user_input = '''\
+ x = """
+
+
+
+
+ 0KiB
+ 0
+ 1.3
+ 0
+
+
+ 16738211KiB
+ 237.15
+ 1.3
+ 0
+
+ never
+ none
+
+
+ """
+ '''
+ user_input = dedent(user_input)
+ user_input = user_input.encode()
+ p = spawn_repl()
+ with SuppressCrashReport():
+ p.stdin.write(user_input)
+ output = kill_python(p)
+ self.assertEqual(p.returncode, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py
new file mode 100644
index 000000000..242d211e5
--- /dev/null
+++ b/Lib/test/test_string.py
@@ -0,0 +1,488 @@
+import unittest
+import string
+from string import Template
+
+
+class ModuleTest(unittest.TestCase):
+
+ def test_attrs(self):
+ # While the exact order of the items in these attributes is not
+ # technically part of the "language spec", in practice there is almost
+ # certainly user code that depends on the order, so de-facto it *is*
+ # part of the spec.
+ self.assertEqual(string.whitespace, ' \t\n\r\x0b\x0c')
+ self.assertEqual(string.ascii_lowercase, 'abcdefghijklmnopqrstuvwxyz')
+ self.assertEqual(string.ascii_uppercase, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+ self.assertEqual(string.ascii_letters, string.ascii_lowercase + string.ascii_uppercase)
+ self.assertEqual(string.digits, '0123456789')
+ self.assertEqual(string.hexdigits, string.digits + 'abcdefABCDEF')
+ self.assertEqual(string.octdigits, '01234567')
+ self.assertEqual(string.punctuation, '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~')
+ self.assertEqual(string.printable, string.digits + string.ascii_lowercase + string.ascii_uppercase + string.punctuation + string.whitespace)
+
+ def test_capwords(self):
+ self.assertEqual(string.capwords('abc def ghi'), 'Abc Def Ghi')
+ self.assertEqual(string.capwords('abc\tdef\nghi'), 'Abc Def Ghi')
+ self.assertEqual(string.capwords('abc\t def \nghi'), 'Abc Def Ghi')
+ self.assertEqual(string.capwords('ABC DEF GHI'), 'Abc Def Ghi')
+ self.assertEqual(string.capwords('ABC-DEF-GHI', '-'), 'Abc-Def-Ghi')
+ self.assertEqual(string.capwords('ABC-def DEF-ghi GHI'), 'Abc-def Def-ghi Ghi')
+ self.assertEqual(string.capwords(' aBc DeF '), 'Abc Def')
+ self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def')
+ self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t')
+
+ def test_basic_formatter(self):
+ fmt = string.Formatter()
+ self.assertEqual(fmt.format("foo"), "foo")
+ self.assertEqual(fmt.format("foo{0}", "bar"), "foobar")
+ self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
+ self.assertRaises(TypeError, fmt.format)
+ self.assertRaises(TypeError, string.Formatter.format)
+
+ def test_format_keyword_arguments(self):
+ fmt = string.Formatter()
+ self.assertEqual(fmt.format("-{arg}-", arg='test'), '-test-')
+ self.assertRaises(KeyError, fmt.format, "-{arg}-")
+ self.assertEqual(fmt.format("-{self}-", self='test'), '-test-')
+ self.assertRaises(KeyError, fmt.format, "-{self}-")
+ self.assertEqual(fmt.format("-{format_string}-", format_string='test'),
+ '-test-')
+ self.assertRaises(KeyError, fmt.format, "-{format_string}-")
+ with self.assertRaisesRegex(TypeError, "format_string"):
+ fmt.format(format_string="-{arg}-", arg='test')
+
+ def test_auto_numbering(self):
+ fmt = string.Formatter()
+ self.assertEqual(fmt.format('foo{}{}', 'bar', 6),
+ 'foo{}{}'.format('bar', 6))
+ self.assertEqual(fmt.format('foo{1}{num}{1}', None, 'bar', num=6),
+ 'foo{1}{num}{1}'.format(None, 'bar', num=6))
+ self.assertEqual(fmt.format('{:^{}}', 'bar', 6),
+ '{:^{}}'.format('bar', 6))
+ self.assertEqual(fmt.format('{:^{}} {}', 'bar', 6, 'X'),
+ '{:^{}} {}'.format('bar', 6, 'X'))
+ self.assertEqual(fmt.format('{:^{pad}}{}', 'foo', 'bar', pad=6),
+ '{:^{pad}}{}'.format('foo', 'bar', pad=6))
+
+ with self.assertRaises(ValueError):
+ fmt.format('foo{1}{}', 'bar', 6)
+
+ with self.assertRaises(ValueError):
+ fmt.format('foo{}{1}', 'bar', 6)
+
+ def test_conversion_specifiers(self):
+ fmt = string.Formatter()
+ self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-")
+ self.assertEqual(fmt.format("{0!s}", 'test'), 'test')
+ self.assertRaises(ValueError, fmt.format, "{0!h}", 'test')
+ # issue13579
+ self.assertEqual(fmt.format("{0!a}", 42), '42')
+ self.assertEqual(fmt.format("{0!a}", string.ascii_letters),
+ "'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'")
+ self.assertEqual(fmt.format("{0!a}", chr(255)), "'\\xff'")
+ self.assertEqual(fmt.format("{0!a}", chr(256)), "'\\u0100'")
+
+ def test_name_lookup(self):
+ fmt = string.Formatter()
+ class AnyAttr:
+ def __getattr__(self, attr):
+ return attr
+ x = AnyAttr()
+ self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack')
+ with self.assertRaises(AttributeError):
+ fmt.format("{0.lumber}{0.jack}", '')
+
+ def test_index_lookup(self):
+ fmt = string.Formatter()
+ lookup = ["eggs", "and", "spam"]
+ self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs')
+ with self.assertRaises(IndexError):
+ fmt.format("{0[2]}{0[0]}", [])
+ with self.assertRaises(KeyError):
+ fmt.format("{0[2]}{0[0]}", {})
+
+ def test_override_get_value(self):
+ class NamespaceFormatter(string.Formatter):
+ def __init__(self, namespace={}):
+ string.Formatter.__init__(self)
+ self.namespace = namespace
+
+ def get_value(self, key, args, kwds):
+ if isinstance(key, str):
+ try:
+ # Check explicitly passed arguments first
+ return kwds[key]
+ except KeyError:
+ return self.namespace[key]
+ else:
+ string.Formatter.get_value(key, args, kwds)
+
+ fmt = NamespaceFormatter({'greeting':'hello'})
+ self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!')
+
+
+ def test_override_format_field(self):
+ class CallFormatter(string.Formatter):
+ def format_field(self, value, format_spec):
+ return format(value(), format_spec)
+
+ fmt = CallFormatter()
+ self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*')
+
+
+ def test_override_convert_field(self):
+ class XFormatter(string.Formatter):
+ def convert_field(self, value, conversion):
+ if conversion == 'x':
+ return None
+ return super().convert_field(value, conversion)
+
+ fmt = XFormatter()
+ self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None")
+
+
+ def test_override_parse(self):
+ class BarFormatter(string.Formatter):
+ # returns an iterable that contains tuples of the form:
+ # (literal_text, field_name, format_spec, conversion)
+ def parse(self, format_string):
+ for field in format_string.split('|'):
+ if field[0] == '+':
+ # it's markup
+ field_name, _, format_spec = field[1:].partition(':')
+ yield '', field_name, format_spec, None
+ else:
+ yield field, None, None, None
+
+ fmt = BarFormatter()
+ self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *')
+
+ def test_check_unused_args(self):
+ class CheckAllUsedFormatter(string.Formatter):
+ def check_unused_args(self, used_args, args, kwargs):
+ # Track which arguments actually got used
+ unused_args = set(kwargs.keys())
+ unused_args.update(range(0, len(args)))
+
+ for arg in used_args:
+ unused_args.remove(arg)
+
+ if unused_args:
+ raise ValueError("unused arguments")
+
+ fmt = CheckAllUsedFormatter()
+ self.assertEqual(fmt.format("{0}", 10), "10")
+ self.assertEqual(fmt.format("{0}{i}", 10, i=100), "10100")
+ self.assertEqual(fmt.format("{0}{i}{1}", 10, 20, i=100), "1010020")
+ self.assertRaises(ValueError, fmt.format, "{0}{i}{1}", 10, 20, i=100, j=0)
+ self.assertRaises(ValueError, fmt.format, "{0}", 10, 20)
+ self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100)
+ self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100)
+
+ def test_vformat_recursion_limit(self):
+ fmt = string.Formatter()
+ args = ()
+ kwargs = dict(i=100)
+ with self.assertRaises(ValueError) as err:
+ fmt._vformat("{i}", args, kwargs, set(), -1)
+ self.assertIn("recursion", str(err.exception))
+
+
+# Template tests (formerly housed in test_pep292.py)
+
+class Bag:
+ pass
+
+class Mapping:
+ def __getitem__(self, name):
+ obj = self
+ for part in name.split('.'):
+ try:
+ obj = getattr(obj, part)
+ except AttributeError:
+ raise KeyError(name)
+ return obj
+
+
+class TestTemplate(unittest.TestCase):
+ def test_regular_templates(self):
+ s = Template('$who likes to eat a bag of $what worth $$100')
+ self.assertEqual(s.substitute(dict(who='tim', what='ham')),
+ 'tim likes to eat a bag of ham worth $100')
+ self.assertRaises(KeyError, s.substitute, dict(who='tim'))
+ self.assertRaises(TypeError, Template.substitute)
+
+ def test_regular_templates_with_braces(self):
+ s = Template('$who likes ${what} for ${meal}')
+ d = dict(who='tim', what='ham', meal='dinner')
+ self.assertEqual(s.substitute(d), 'tim likes ham for dinner')
+ self.assertRaises(KeyError, s.substitute,
+ dict(who='tim', what='ham'))
+
+ def test_regular_templates_with_upper_case(self):
+ s = Template('$WHO likes ${WHAT} for ${MEAL}')
+ d = dict(WHO='tim', WHAT='ham', MEAL='dinner')
+ self.assertEqual(s.substitute(d), 'tim likes ham for dinner')
+
+ def test_regular_templates_with_non_letters(self):
+ s = Template('$_wh0_ likes ${_w_h_a_t_} for ${mea1}')
+ d = dict(_wh0_='tim', _w_h_a_t_='ham', mea1='dinner')
+ self.assertEqual(s.substitute(d), 'tim likes ham for dinner')
+
+ def test_escapes(self):
+ eq = self.assertEqual
+ s = Template('$who likes to eat a bag of $$what worth $$100')
+ eq(s.substitute(dict(who='tim', what='ham')),
+ 'tim likes to eat a bag of $what worth $100')
+ s = Template('$who likes $$')
+ eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $')
+
+ def test_percents(self):
+ eq = self.assertEqual
+ s = Template('%(foo)s $foo ${foo}')
+ d = dict(foo='baz')
+ eq(s.substitute(d), '%(foo)s baz baz')
+ eq(s.safe_substitute(d), '%(foo)s baz baz')
+
+ def test_stringification(self):
+ eq = self.assertEqual
+ s = Template('tim has eaten $count bags of ham today')
+ d = dict(count=7)
+ eq(s.substitute(d), 'tim has eaten 7 bags of ham today')
+ eq(s.safe_substitute(d), 'tim has eaten 7 bags of ham today')
+ s = Template('tim has eaten ${count} bags of ham today')
+ eq(s.substitute(d), 'tim has eaten 7 bags of ham today')
+
+ def test_tupleargs(self):
+ eq = self.assertEqual
+ s = Template('$who ate ${meal}')
+ d = dict(who=('tim', 'fred'), meal=('ham', 'kung pao'))
+ eq(s.substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')")
+ eq(s.safe_substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')")
+
+ def test_SafeTemplate(self):
+ eq = self.assertEqual
+ s = Template('$who likes ${what} for ${meal}')
+ eq(s.safe_substitute(dict(who='tim')), 'tim likes ${what} for ${meal}')
+ eq(s.safe_substitute(dict(what='ham')), '$who likes ham for ${meal}')
+ eq(s.safe_substitute(dict(what='ham', meal='dinner')),
+ '$who likes ham for dinner')
+ eq(s.safe_substitute(dict(who='tim', what='ham')),
+ 'tim likes ham for ${meal}')
+ eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')),
+ 'tim likes ham for dinner')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_invalid_placeholders(self):
+ raises = self.assertRaises
+ s = Template('$who likes $')
+ raises(ValueError, s.substitute, dict(who='tim'))
+ s = Template('$who likes ${what)')
+ raises(ValueError, s.substitute, dict(who='tim'))
+ s = Template('$who likes $100')
+ raises(ValueError, s.substitute, dict(who='tim'))
+ # Template.idpattern should match to only ASCII characters.
+ # https://bugs.python.org/issue31672
+ s = Template("$who likes $\u0131") # (DOTLESS I)
+ raises(ValueError, s.substitute, dict(who='tim'))
+ s = Template("$who likes $\u0130") # (LATIN CAPITAL LETTER I WITH DOT ABOVE)
+ raises(ValueError, s.substitute, dict(who='tim'))
+
+ def test_idpattern_override(self):
+ class PathPattern(Template):
+ idpattern = r'[_a-z][._a-z0-9]*'
+ m = Mapping()
+ m.bag = Bag()
+ m.bag.foo = Bag()
+ m.bag.foo.who = 'tim'
+ m.bag.what = 'ham'
+ s = PathPattern('$bag.foo.who likes to eat a bag of $bag.what')
+ self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_flags_override(self):
+ class MyPattern(Template):
+ flags = 0
+ s = MyPattern('$wHO likes ${WHAT} for ${meal}')
+ d = dict(wHO='tim', WHAT='ham', meal='dinner', w='fred')
+ self.assertRaises(ValueError, s.substitute, d)
+ self.assertEqual(s.safe_substitute(d), 'fredHO likes ${WHAT} for dinner')
+
+ def test_idpattern_override_inside_outside(self):
+ # bpo-1198569: Allow the regexp inside and outside braces to be
+ # different when deriving from Template.
+ class MyPattern(Template):
+ idpattern = r'[a-z]+'
+ braceidpattern = r'[A-Z]+'
+ flags = 0
+ m = dict(foo='foo', BAR='BAR')
+ s = MyPattern('$foo ${BAR}')
+ self.assertEqual(s.substitute(m), 'foo BAR')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_idpattern_override_inside_outside_invalid_unbraced(self):
+ # bpo-1198569: Allow the regexp inside and outside braces to be
+ # different when deriving from Template.
+ class MyPattern(Template):
+ idpattern = r'[a-z]+'
+ braceidpattern = r'[A-Z]+'
+ flags = 0
+ m = dict(foo='foo', BAR='BAR')
+ s = MyPattern('$FOO')
+ self.assertRaises(ValueError, s.substitute, m)
+ s = MyPattern('${bar}')
+ self.assertRaises(ValueError, s.substitute, m)
+
+ def test_pattern_override(self):
+ class MyPattern(Template):
+ pattern = r"""
+ (?P@{2}) |
+ @(?P[_a-z][._a-z0-9]*) |
+ @{(?P[_a-z][._a-z0-9]*)} |
+ (?P@)
+ """
+ m = Mapping()
+ m.bag = Bag()
+ m.bag.foo = Bag()
+ m.bag.foo.who = 'tim'
+ m.bag.what = 'ham'
+ s = MyPattern('@bag.foo.who likes to eat a bag of @bag.what')
+ self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham')
+
+ class BadPattern(Template):
+ pattern = r"""
+ (?P.*) |
+ (?P@{2}) |
+ @(?P[_a-z][._a-z0-9]*) |
+ @{(?P[_a-z][._a-z0-9]*)} |
+ (?P@) |
+ """
+ s = BadPattern('@bag.foo.who likes to eat a bag of @bag.what')
+ self.assertRaises(ValueError, s.substitute, {})
+ self.assertRaises(ValueError, s.safe_substitute, {})
+
+ def test_braced_override(self):
+ class MyTemplate(Template):
+ pattern = r"""
+ \$(?:
+ (?P$) |
+ (?P[_a-z][_a-z0-9]*) |
+ @@(?P[_a-z][_a-z0-9]*)@@ |
+ (?P) |
+ )
+ """
+
+ tmpl = 'PyCon in $@@location@@'
+ t = MyTemplate(tmpl)
+ self.assertRaises(KeyError, t.substitute, {})
+ val = t.substitute({'location': 'Cleveland'})
+ self.assertEqual(val, 'PyCon in Cleveland')
+
+ def test_braced_override_safe(self):
+ class MyTemplate(Template):
+ pattern = r"""
+ \$(?:
+ (?P$) |
+ (?P[_a-z][_a-z0-9]*) |
+ @@(?P[_a-z][_a-z0-9]*)@@ |
+ (?P) |
+ )
+ """
+
+ tmpl = 'PyCon in $@@location@@'
+ t = MyTemplate(tmpl)
+ self.assertEqual(t.safe_substitute(), tmpl)
+ val = t.safe_substitute({'location': 'Cleveland'})
+ self.assertEqual(val, 'PyCon in Cleveland')
+
+ def test_invalid_with_no_lines(self):
+ # The error formatting for invalid templates
+ # has a special case for no data that the default
+ # pattern can't trigger (always has at least '$')
+ # So we craft a pattern that is always invalid
+ # with no leading data.
+ class MyTemplate(Template):
+ pattern = r"""
+ (?P) |
+ unreachable(
+ (?P) |
+ (?P) |
+ (?P)
+ )
+ """
+ s = MyTemplate('')
+ with self.assertRaises(ValueError) as err:
+ s.substitute({})
+ self.assertIn('line 1, col 1', str(err.exception))
+
+ def test_unicode_values(self):
+ s = Template('$who likes $what')
+ d = dict(who='t\xffm', what='f\xfe\fed')
+ self.assertEqual(s.substitute(d), 't\xffm likes f\xfe\x0ced')
+
+ def test_keyword_arguments(self):
+ eq = self.assertEqual
+ s = Template('$who likes $what')
+ eq(s.substitute(who='tim', what='ham'), 'tim likes ham')
+ eq(s.substitute(dict(who='tim'), what='ham'), 'tim likes ham')
+ eq(s.substitute(dict(who='fred', what='kung pao'),
+ who='tim', what='ham'),
+ 'tim likes ham')
+ s = Template('the mapping is $mapping')
+ eq(s.substitute(dict(foo='none'), mapping='bozo'),
+ 'the mapping is bozo')
+ eq(s.substitute(dict(mapping='one'), mapping='two'),
+ 'the mapping is two')
+
+ s = Template('the self is $self')
+ eq(s.substitute(self='bozo'), 'the self is bozo')
+
+ def test_keyword_arguments_safe(self):
+ eq = self.assertEqual
+ raises = self.assertRaises
+ s = Template('$who likes $what')
+ eq(s.safe_substitute(who='tim', what='ham'), 'tim likes ham')
+ eq(s.safe_substitute(dict(who='tim'), what='ham'), 'tim likes ham')
+ eq(s.safe_substitute(dict(who='fred', what='kung pao'),
+ who='tim', what='ham'),
+ 'tim likes ham')
+ s = Template('the mapping is $mapping')
+ eq(s.safe_substitute(dict(foo='none'), mapping='bozo'),
+ 'the mapping is bozo')
+ eq(s.safe_substitute(dict(mapping='one'), mapping='two'),
+ 'the mapping is two')
+ d = dict(mapping='one')
+ raises(TypeError, s.substitute, d, {})
+ raises(TypeError, s.safe_substitute, d, {})
+
+ s = Template('the self is $self')
+ eq(s.safe_substitute(self='bozo'), 'the self is bozo')
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_delimiter_override(self):
+ eq = self.assertEqual
+ raises = self.assertRaises
+ class AmpersandTemplate(Template):
+ delimiter = '&'
+ s = AmpersandTemplate('this &gift is for &{who} &&')
+ eq(s.substitute(gift='bud', who='you'), 'this bud is for you &')
+ raises(KeyError, s.substitute)
+ eq(s.safe_substitute(gift='bud', who='you'), 'this bud is for you &')
+ eq(s.safe_substitute(), 'this &gift is for &{who} &')
+ s = AmpersandTemplate('this &gift is for &{who} &')
+ raises(ValueError, s.substitute, dict(gift='bud', who='you'))
+ eq(s.safe_substitute(), 'this &gift is for &{who} &')
+
+ class PieDelims(Template):
+ delimiter = '@'
+ s = PieDelims('@who likes to eat a bag of @{what} worth $100')
+ self.assertEqual(s.substitute(dict(who='tim', what='ham')),
+ 'tim likes to eat a bag of ham worth $100')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py
new file mode 100644
index 000000000..9579fde93
--- /dev/null
+++ b/Lib/test/test_subclassinit.py
@@ -0,0 +1,290 @@
+import types
+import unittest
+
+
+class Test(unittest.TestCase):
+ def test_init_subclass(self):
+ class A:
+ initialized = False
+
+ def __init_subclass__(cls):
+ super().__init_subclass__()
+ cls.initialized = True
+
+ class B(A):
+ pass
+
+ self.assertFalse(A.initialized)
+ self.assertTrue(B.initialized)
+
+ def test_init_subclass_dict(self):
+ class A(dict):
+ initialized = False
+
+ def __init_subclass__(cls):
+ super().__init_subclass__()
+ cls.initialized = True
+
+ class B(A):
+ pass
+
+ self.assertFalse(A.initialized)
+ self.assertTrue(B.initialized)
+
+ def test_init_subclass_kwargs(self):
+ class A:
+ def __init_subclass__(cls, **kwargs):
+ cls.kwargs = kwargs
+
+ class B(A, x=3):
+ pass
+
+ self.assertEqual(B.kwargs, dict(x=3))
+
+ def test_init_subclass_error(self):
+ class A:
+ def __init_subclass__(cls):
+ raise RuntimeError
+
+ with self.assertRaises(RuntimeError):
+ class B(A):
+ pass
+
+ def test_init_subclass_wrong(self):
+ class A:
+ def __init_subclass__(cls, whatever):
+ pass
+
+ with self.assertRaises(TypeError):
+ class B(A):
+ pass
+
+ def test_init_subclass_skipped(self):
+ class BaseWithInit:
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ cls.initialized = cls
+
+ class BaseWithoutInit(BaseWithInit):
+ pass
+
+ class A(BaseWithoutInit):
+ pass
+
+ self.assertIs(A.initialized, A)
+ self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit)
+
+ def test_init_subclass_diamond(self):
+ class Base:
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ cls.calls = []
+
+ class Left(Base):
+ pass
+
+ class Middle:
+ def __init_subclass__(cls, middle, **kwargs):
+ super().__init_subclass__(**kwargs)
+ cls.calls += [middle]
+
+ class Right(Base):
+ def __init_subclass__(cls, right="right", **kwargs):
+ super().__init_subclass__(**kwargs)
+ cls.calls += [right]
+
+ class A(Left, Middle, Right, middle="middle"):
+ pass
+
+ self.assertEqual(A.calls, ["right", "middle"])
+ self.assertEqual(Left.calls, [])
+ self.assertEqual(Right.calls, [])
+
+ def test_set_name(self):
+ class Descriptor:
+ def __set_name__(self, owner, name):
+ self.owner = owner
+ self.name = name
+
+ class A:
+ d = Descriptor()
+
+ self.assertEqual(A.d.name, "d")
+ self.assertIs(A.d.owner, A)
+
+ #TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_set_name_metaclass(self):
+ class Meta(type):
+ def __new__(cls, name, bases, ns):
+ ret = super().__new__(cls, name, bases, ns)
+ self.assertEqual(ret.d.name, "d")
+ self.assertIs(ret.d.owner, ret)
+ return 0
+
+ class Descriptor:
+ def __set_name__(self, owner, name):
+ self.owner = owner
+ self.name = name
+
+ class A(metaclass=Meta):
+ d = Descriptor()
+ self.assertEqual(A, 0)
+
+ def test_set_name_error(self):
+ class Descriptor:
+ def __set_name__(self, owner, name):
+ 1/0
+
+ with self.assertRaises(RuntimeError) as cm:
+ class NotGoingToWork:
+ attr = Descriptor()
+
+ exc = cm.exception
+ self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+ self.assertRegex(str(exc), r'\battr\b')
+ self.assertRegex(str(exc), r'\bDescriptor\b')
+ self.assertIsInstance(exc.__cause__, ZeroDivisionError)
+
+ def test_set_name_wrong(self):
+ class Descriptor:
+ def __set_name__(self):
+ pass
+
+ with self.assertRaises(RuntimeError) as cm:
+ class NotGoingToWork:
+ attr = Descriptor()
+
+ exc = cm.exception
+ self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+ self.assertRegex(str(exc), r'\battr\b')
+ self.assertRegex(str(exc), r'\bDescriptor\b')
+ self.assertIsInstance(exc.__cause__, TypeError)
+
+ def test_set_name_lookup(self):
+ resolved = []
+ class NonDescriptor:
+ def __getattr__(self, name):
+ resolved.append(name)
+
+ class A:
+ d = NonDescriptor()
+
+ self.assertNotIn('__set_name__', resolved,
+ '__set_name__ is looked up in instance dict')
+
+ def test_set_name_init_subclass(self):
+ class Descriptor:
+ def __set_name__(self, owner, name):
+ self.owner = owner
+ self.name = name
+
+ class Meta(type):
+ def __new__(cls, name, bases, ns):
+ self = super().__new__(cls, name, bases, ns)
+ self.meta_owner = self.owner
+ self.meta_name = self.name
+ return self
+
+ class A:
+ def __init_subclass__(cls):
+ cls.owner = cls.d.owner
+ cls.name = cls.d.name
+
+ class B(A, metaclass=Meta):
+ d = Descriptor()
+
+ self.assertIs(B.owner, B)
+ self.assertEqual(B.name, 'd')
+ self.assertIs(B.meta_owner, B)
+ self.assertEqual(B.name, 'd')
+
+ #TODO: RUSTPYTHON
+ @unittest.skip("infinite loops")
+ def test_set_name_modifying_dict(self):
+ notified = []
+ class Descriptor:
+ def __set_name__(self, owner, name):
+ setattr(owner, name + 'x', None)
+ notified.append(name)
+
+ class A:
+ a = Descriptor()
+ b = Descriptor()
+ c = Descriptor()
+ d = Descriptor()
+ e = Descriptor()
+
+ self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e'])
+
+ #TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_errors(self):
+ class MyMeta(type):
+ pass
+
+ with self.assertRaises(TypeError):
+ class MyClass(metaclass=MyMeta, otherarg=1):
+ pass
+
+ with self.assertRaises(TypeError):
+ types.new_class("MyClass", (object,),
+ dict(metaclass=MyMeta, otherarg=1))
+ types.prepare_class("MyClass", (object,),
+ dict(metaclass=MyMeta, otherarg=1))
+
+ class MyMeta(type):
+ def __init__(self, name, bases, namespace, otherarg):
+ super().__init__(name, bases, namespace)
+
+ with self.assertRaises(TypeError):
+ class MyClass(metaclass=MyMeta, otherarg=1):
+ pass
+
+ class MyMeta(type):
+ def __new__(cls, name, bases, namespace, otherarg):
+ return super().__new__(cls, name, bases, namespace)
+
+ def __init__(self, name, bases, namespace, otherarg):
+ super().__init__(name, bases, namespace)
+ self.otherarg = otherarg
+
+ class MyClass(metaclass=MyMeta, otherarg=1):
+ pass
+
+ self.assertEqual(MyClass.otherarg, 1)
+
+ def test_errors_changed_pep487(self):
+ # These tests failed before Python 3.6, PEP 487
+ class MyMeta(type):
+ def __new__(cls, name, bases, namespace):
+ return super().__new__(cls, name=name, bases=bases,
+ dict=namespace)
+
+ with self.assertRaises(TypeError):
+ class MyClass(metaclass=MyMeta):
+ pass
+
+ class MyMeta(type):
+ def __new__(cls, name, bases, namespace, otherarg):
+ self = super().__new__(cls, name, bases, namespace)
+ self.otherarg = otherarg
+ return self
+
+ class MyClass(metaclass=MyMeta, otherarg=1):
+ pass
+
+ self.assertEqual(MyClass.otherarg, 1)
+
+ def test_type(self):
+ t = type('NewClass', (object,), {})
+ self.assertIsInstance(t, type)
+ self.assertEqual(t.__name__, 'NewClass')
+
+ with self.assertRaises(TypeError):
+ type(name='NewClass', bases=(object,), dict={})
+
+
+if __name__ == "__main__":
+ unittest.main()
+
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
new file mode 100644
index 000000000..8e83bc72f
--- /dev/null
+++ b/Lib/test/test_symtable.py
@@ -0,0 +1,251 @@
+"""
+Test the API of the symtable module.
+"""
+import symtable
+import unittest
+
+
+
+TEST_CODE = """
+import sys
+
+glob = 42
+some_var = 12
+
+class Mine:
+ instance_var = 24
+ def a_method(p1, p2):
+ pass
+
+def spam(a, b, *var, **kw):
+ global bar
+ bar = 47
+ some_var = 10
+ x = 23
+ glob
+ def internal():
+ return x
+ def other_internal():
+ nonlocal some_var
+ some_var = 3
+ return some_var
+ return internal
+
+def foo():
+ pass
+
+def namespace_test(): pass
+def namespace_test(): pass
+"""
+
+
+def find_block(block, name):
+ for ch in block.get_children():
+ if ch.get_name() == name:
+ return ch
+
+
+class SymtableTest(unittest.TestCase):
+
+ top = symtable.symtable(TEST_CODE, "?", "exec")
+ # These correspond to scopes in TEST_CODE
+ Mine = find_block(top, "Mine")
+ a_method = find_block(Mine, "a_method")
+ spam = find_block(top, "spam")
+ internal = find_block(spam, "internal")
+ other_internal = find_block(spam, "other_internal")
+ foo = find_block(top, "foo")
+
+ def test_type(self):
+ self.assertEqual(self.top.get_type(), "module")
+ self.assertEqual(self.Mine.get_type(), "class")
+ self.assertEqual(self.a_method.get_type(), "function")
+ self.assertEqual(self.spam.get_type(), "function")
+ self.assertEqual(self.internal.get_type(), "function")
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_id(self):
+ self.assertGreater(self.top.get_id(), 0)
+ self.assertGreater(self.Mine.get_id(), 0)
+ self.assertGreater(self.a_method.get_id(), 0)
+ self.assertGreater(self.spam.get_id(), 0)
+ self.assertGreater(self.internal.get_id(), 0)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_optimized(self):
+ self.assertFalse(self.top.is_optimized())
+
+ self.assertTrue(self.spam.is_optimized())
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_nested(self):
+ self.assertFalse(self.top.is_nested())
+ self.assertFalse(self.Mine.is_nested())
+ self.assertFalse(self.spam.is_nested())
+ self.assertTrue(self.internal.is_nested())
+
+ def test_children(self):
+ self.assertTrue(self.top.has_children())
+ self.assertTrue(self.Mine.has_children())
+ self.assertFalse(self.foo.has_children())
+
+ def test_lineno(self):
+ self.assertEqual(self.top.get_lineno(), 0)
+ self.assertEqual(self.spam.get_lineno(), 12)
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_function_info(self):
+ func = self.spam
+ self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"])
+ expected = ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x']
+ self.assertEqual(sorted(func.get_locals()), expected)
+ self.assertEqual(sorted(func.get_globals()), ["bar", "glob"])
+ self.assertEqual(self.internal.get_frees(), ("x",))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_globals(self):
+ self.assertTrue(self.spam.lookup("glob").is_global())
+ self.assertFalse(self.spam.lookup("glob").is_declared_global())
+ self.assertTrue(self.spam.lookup("bar").is_global())
+ self.assertTrue(self.spam.lookup("bar").is_declared_global())
+ self.assertFalse(self.internal.lookup("x").is_global())
+ self.assertFalse(self.Mine.lookup("instance_var").is_global())
+ self.assertTrue(self.spam.lookup("bar").is_global())
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_nonlocal(self):
+ self.assertFalse(self.spam.lookup("some_var").is_nonlocal())
+ self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal())
+ expected = ("some_var",)
+ self.assertEqual(self.other_internal.get_nonlocals(), expected)
+
+ def test_local(self):
+ self.assertTrue(self.spam.lookup("x").is_local())
+ self.assertFalse(self.spam.lookup("bar").is_local())
+
+ def test_free(self):
+ self.assertTrue(self.internal.lookup("x").is_free())
+
+ def test_referenced(self):
+ self.assertTrue(self.internal.lookup("x").is_referenced())
+ self.assertTrue(self.spam.lookup("internal").is_referenced())
+ self.assertFalse(self.spam.lookup("x").is_referenced())
+
+ def test_parameters(self):
+ for sym in ("a", "var", "kw"):
+ self.assertTrue(self.spam.lookup(sym).is_parameter())
+ self.assertFalse(self.spam.lookup("x").is_parameter())
+
+ def test_symbol_lookup(self):
+ self.assertEqual(len(self.top.get_identifiers()),
+ len(self.top.get_symbols()))
+
+ self.assertRaises(KeyError, self.top.lookup, "not_here")
+
+ def test_namespaces(self):
+ self.assertTrue(self.top.lookup("Mine").is_namespace())
+ self.assertTrue(self.Mine.lookup("a_method").is_namespace())
+ self.assertTrue(self.top.lookup("spam").is_namespace())
+ self.assertTrue(self.spam.lookup("internal").is_namespace())
+ self.assertTrue(self.top.lookup("namespace_test").is_namespace())
+ self.assertFalse(self.spam.lookup("x").is_namespace())
+
+ # TODO(RUSTPYTHON): lookup should return same pythonref
+ # self.assertTrue(self.top.lookup("spam").get_namespace() is self.spam)
+ ns_test = self.top.lookup("namespace_test")
+ self.assertEqual(len(ns_test.get_namespaces()), 2)
+ self.assertRaises(ValueError, ns_test.get_namespace)
+
+ def test_assigned(self):
+ self.assertTrue(self.spam.lookup("x").is_assigned())
+ self.assertTrue(self.spam.lookup("bar").is_assigned())
+ self.assertTrue(self.top.lookup("spam").is_assigned())
+ self.assertTrue(self.Mine.lookup("a_method").is_assigned())
+ self.assertFalse(self.internal.lookup("x").is_assigned())
+
+ def test_annotated(self):
+ st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
+ st2 = st1.get_children()[0]
+ self.assertTrue(st2.lookup('x').is_local())
+ self.assertTrue(st2.lookup('x').is_annotated())
+ self.assertFalse(st2.lookup('x').is_global())
+ st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
+ st4 = st3.get_children()[0]
+ self.assertTrue(st4.lookup('x').is_local())
+ self.assertFalse(st4.lookup('x').is_annotated())
+
+ # Test that annotations in the global scope are valid after the
+ # variable is declared as nonlocal.
+ st5 = symtable.symtable('global x\nx: int', 'test', 'exec')
+ self.assertTrue(st5.lookup("x").is_global())
+
+ # Test that annotations for nonlocals are valid after the
+ # variable is declared as nonlocal.
+ st6 = symtable.symtable('def g():\n'
+ ' x = 2\n'
+ ' def f():\n'
+ ' nonlocal x\n'
+ ' x: int',
+ 'test', 'exec')
+
+ def test_imported(self):
+ self.assertTrue(self.top.lookup("sys").is_imported())
+
+ def test_name(self):
+ self.assertEqual(self.top.get_name(), "top")
+ self.assertEqual(self.spam.get_name(), "spam")
+ self.assertEqual(self.spam.lookup("x").get_name(), "x")
+ self.assertEqual(self.Mine.get_name(), "Mine")
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_class_info(self):
+ self.assertEqual(self.Mine.get_methods(), ('a_method',))
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_filename_correct(self):
+ ### Bug tickler: SyntaxError file name correct whether error raised
+ ### while parsing or building symbol table.
+ def checkfilename(brokencode, offset):
+ try:
+ symtable.symtable(brokencode, "spam", "exec")
+ except SyntaxError as e:
+ self.assertEqual(e.filename, "spam")
+ self.assertEqual(e.lineno, 1)
+ self.assertEqual(e.offset, offset)
+ else:
+ self.fail("no SyntaxError for %r" % (brokencode,))
+ # TODO: RUSTPYTHON, now offset get 15
+ checkfilename("def f(x): foo)(", 14) # parse-time
+ checkfilename("def f(x): global x", 11) # symtable-build-time
+ symtable.symtable("pass", b"spam", "exec")
+ with self.assertWarns(DeprecationWarning), \
+ self.assertRaises(TypeError):
+ symtable.symtable("pass", bytearray(b"spam"), "exec")
+ with self.assertWarns(DeprecationWarning):
+ symtable.symtable("pass", memoryview(b"spam"), "exec")
+ with self.assertRaises(TypeError):
+ symtable.symtable("pass", list(b"spam"), "exec")
+
+ def test_eval(self):
+ symbols = symtable.symtable("42", "?", "eval")
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_single(self):
+ symbols = symtable.symtable("42", "?", "single")
+
+ def test_exec(self):
+ symbols = symtable.symtable("def f(x): return x", "?", "exec")
+
+ @unittest.skip("TODO: RUSTPYTHON")
+ def test_bytes(self):
+ top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
+ self.assertIsNotNone(find_block(top, "Mine"))
+
+ code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
+
+ top = symtable.symtable(code, "?", "exec")
+ self.assertIsNotNone(find_block(top, "\u017d"))
+
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
new file mode 100644
index 000000000..ca107100e
--- /dev/null
+++ b/Lib/test/test_sys.py
@@ -0,0 +1,1465 @@
+from test import support
+from test.support.script_helper import assert_python_ok, assert_python_failure
+import builtins
+import codecs
+# import gc
+import locale
+import operator
+import os
+import struct
+import subprocess
+import sys
+import sysconfig
+import test.support
+import textwrap
+import unittest
+import warnings
+
+
+# count the number of test runs, used to create unique
+# strings to intern in test_intern()
+INTERN_NUMRUNS = 0
+
+
+class DisplayHookTest(unittest.TestCase):
+
+ def test_original_displayhook(self):
+ dh = sys.__displayhook__
+
+ with support.captured_stdout() as out:
+ dh(42)
+
+ self.assertEqual(out.getvalue(), "42\n")
+ self.assertEqual(builtins._, 42)
+
+ del builtins._
+
+ with support.captured_stdout() as out:
+ dh(None)
+
+ self.assertEqual(out.getvalue(), "")
+ self.assertTrue(not hasattr(builtins, "_"))
+
+ # sys.displayhook() requires arguments
+ self.assertRaises(TypeError, dh)
+
+ stdout = sys.stdout
+ try:
+ del sys.stdout
+ self.assertRaises(RuntimeError, dh, 42)
+ finally:
+ sys.stdout = stdout
+
+ def test_lost_displayhook(self):
+ displayhook = sys.displayhook
+ try:
+ del sys.displayhook
+ code = compile("42", "", "single")
+ self.assertRaises(RuntimeError, eval, code)
+ finally:
+ sys.displayhook = displayhook
+
+ def test_custom_displayhook(self):
+ def baddisplayhook(obj):
+ raise ValueError
+
+ with support.swap_attr(sys, 'displayhook', baddisplayhook):
+ code = compile("42", "", "single")
+ self.assertRaises(ValueError, eval, code)
+
+
+class ExceptHookTest(unittest.TestCase):
+
+ def test_original_excepthook(self):
+ try:
+ raise ValueError(42)
+ except ValueError as exc:
+ with support.captured_stderr() as err:
+ sys.__excepthook__(*sys.exc_info())
+
+ self.assertTrue(err.getvalue().endswith("ValueError: 42\n"))
+
+ self.assertRaises(TypeError, sys.__excepthook__)
+
+ @unittest.skip("TODO: RUSTPYTHON; SyntaxError formatting in arbitrary tracebacks")
+ def test_excepthook_bytes_filename(self):
+ # bpo-37467: sys.excepthook() must not crash if a filename
+ # is a bytes string
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', BytesWarning)
+
+ try:
+ raise SyntaxError("msg", (b"bytes_filename", 123, 0, "text"))
+ except SyntaxError as exc:
+ with support.captured_stderr() as err:
+ sys.__excepthook__(*sys.exc_info())
+
+ err = err.getvalue()
+ self.assertIn(""" File "b'bytes_filename'", line 123\n""", err)
+ self.assertIn(""" text\n""", err)
+ self.assertTrue(err.endswith("SyntaxError: msg\n"))
+
+ @unittest.skip("TODO: RUSTPYTHON; print argument error to stderr in sys.excepthook instead of throwing")
+ def test_excepthook(self):
+ with test.support.captured_output("stderr") as stderr:
+ sys.excepthook(1, '1', 1)
+ self.assertTrue("TypeError: print_exception(): Exception expected for " \
+ "value, str found" in stderr.getvalue())
+
+ # FIXME: testing the code for a lost or replaced excepthook in
+ # Python/pythonrun.c::PyErr_PrintEx() is tricky.
+
+
+class SysModuleTest(unittest.TestCase):
+
+ def tearDown(self):
+ test.support.reap_children()
+
+ def test_exit(self):
+ # call with two arguments
+ self.assertRaises(TypeError, sys.exit, 42, 42)
+
+ # call without argument
+ with self.assertRaises(SystemExit) as cm:
+ sys.exit()
+ self.assertIsNone(cm.exception.code)
+
+ rc, out, err = assert_python_ok('-c', 'import sys; sys.exit()')
+ self.assertEqual(rc, 0)
+ self.assertEqual(out, b'')
+ self.assertEqual(err, b'')
+
+ # call with integer argument
+ with self.assertRaises(SystemExit) as cm:
+ sys.exit(42)
+ self.assertEqual(cm.exception.code, 42)
+
+ # call with tuple argument with one entry
+ # entry will be unpacked
+ with self.assertRaises(SystemExit) as cm:
+ sys.exit((42,))
+ self.assertEqual(cm.exception.code, 42)
+
+ # call with string argument
+ with self.assertRaises(SystemExit) as cm:
+ sys.exit("exit")
+ self.assertEqual(cm.exception.code, "exit")
+
+ # call with tuple argument with two entries
+ with self.assertRaises(SystemExit) as cm:
+ sys.exit((17, 23))
+ self.assertEqual(cm.exception.code, (17, 23))
+
+ # test that the exit machinery handles SystemExits properly
+ rc, out, err = assert_python_failure('-c', 'raise SystemExit(47)')
+ self.assertEqual(rc, 47)
+ self.assertEqual(out, b'')
+ self.assertEqual(err, b'')
+
+ def check_exit_message(code, expected, **env_vars):
+ rc, out, err = assert_python_failure('-c', code, **env_vars)
+ self.assertEqual(rc, 1)
+ self.assertEqual(out, b'')
+ self.assertTrue(err.startswith(expected),
+ "%s doesn't start with %s" % (ascii(err), ascii(expected)))
+
+ # test that stderr buffer is flushed before the exit message is written
+ # into stderr
+ check_exit_message(
+ r'import sys; sys.stderr.write("unflushed,"); sys.exit("message")',
+ b"unflushed,message")
+
+ # test that the exit message is written with backslashreplace error
+ # handler to stderr
+ # TODO: RUSTPYTHON; allow surrogates in strings
+ # check_exit_message(
+ # r'import sys; sys.exit("surrogates:\uDCFF")',
+ # b"surrogates:\\udcff")
+
+ # test that the unicode message is encoded to the stderr encoding
+ # instead of the default encoding (utf8)
+ # TODO: RUSTPYTHON; handle PYTHONIOENCODING
+ # check_exit_message(
+ # r'import sys; sys.exit("h\xe9")',
+ # b"h\xe9", PYTHONIOENCODING='latin-1')
+
+ def test_getdefaultencoding(self):
+ self.assertRaises(TypeError, sys.getdefaultencoding, 42)
+ # can't check more than the type, as the user might have changed it
+ self.assertIsInstance(sys.getdefaultencoding(), str)
+
+ # testing sys.settrace() is done in test_sys_settrace.py
+ # testing sys.setprofile() is done in test_sys_setprofile.py
+
+ @unittest.skip("RUSTPYTHON: don't have sys.setcheckinterval")
+ def test_setcheckinterval(self):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ self.assertRaises(TypeError, sys.setcheckinterval)
+ orig = sys.getcheckinterval()
+ for n in 0, 100, 120, orig: # orig last to restore starting state
+ sys.setcheckinterval(n)
+ self.assertEqual(sys.getcheckinterval(), n)
+
+ @unittest.skip("RUSTPYTHON: don't have sys.setswitchinterval")
+ def test_switchinterval(self):
+ self.assertRaises(TypeError, sys.setswitchinterval)
+ self.assertRaises(TypeError, sys.setswitchinterval, "a")
+ self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
+ self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
+ orig = sys.getswitchinterval()
+ # sanity check
+ self.assertTrue(orig < 0.5, orig)
+ try:
+ for n in 0.00001, 0.05, 3.0, orig:
+ sys.setswitchinterval(n)
+ self.assertAlmostEqual(sys.getswitchinterval(), n)
+ finally:
+ sys.setswitchinterval(orig)
+
+ def test_recursionlimit(self):
+ self.assertRaises(TypeError, sys.getrecursionlimit, 42)
+ oldlimit = sys.getrecursionlimit()
+ self.assertRaises(TypeError, sys.setrecursionlimit)
+ self.assertRaises(ValueError, sys.setrecursionlimit, -42)
+ sys.setrecursionlimit(10000)
+ self.assertEqual(sys.getrecursionlimit(), 10000)
+ sys.setrecursionlimit(oldlimit)
+
+ def test_recursionlimit_recovery(self):
+ if hasattr(sys, 'gettrace') and sys.gettrace():
+ self.skipTest('fatal error if run with a trace function')
+
+ oldlimit = sys.getrecursionlimit()
+ def f():
+ f()
+ try:
+ for depth in (10, 25, 50, 75, 100, 250, 1000):
+ try:
+ sys.setrecursionlimit(depth)
+ except RecursionError:
+ # Issue #25274: The recursion limit is too low at the
+ # current recursion depth
+ continue
+
+ # Issue #5392: test stack overflow after hitting recursion
+ # limit twice
+ self.assertRaises(RecursionError, f)
+ self.assertRaises(RecursionError, f)
+ finally:
+ sys.setrecursionlimit(oldlimit)
+
+ @test.support.cpython_only
+ def test_setrecursionlimit_recursion_depth(self):
+ # Issue #25274: Setting a low recursion limit must be blocked if the
+ # current recursion depth is already higher than the "lower-water
+ # mark". Otherwise, it may not be possible anymore to
+ # reset the overflowed flag to 0.
+
+ from _testcapi import get_recursion_depth
+
+ def set_recursion_limit_at_depth(depth, limit):
+ recursion_depth = get_recursion_depth()
+ if recursion_depth >= depth:
+ with self.assertRaises(RecursionError) as cm:
+ sys.setrecursionlimit(limit)
+ self.assertRegex(str(cm.exception),
+ "cannot set the recursion limit to [0-9]+ "
+ "at the recursion depth [0-9]+: "
+ "the limit is too low")
+ else:
+ set_recursion_limit_at_depth(depth, limit)
+
+ oldlimit = sys.getrecursionlimit()
+ try:
+ sys.setrecursionlimit(1000)
+
+ for limit in (10, 25, 50, 75, 100, 150, 200):
+ # formula extracted from _Py_RecursionLimitLowerWaterMark()
+ if limit > 200:
+ depth = limit - 50
+ else:
+ depth = limit * 3 // 4
+ set_recursion_limit_at_depth(depth, limit)
+ finally:
+ sys.setrecursionlimit(oldlimit)
+
+ @unittest.skip("TODO: RUSTPYTHON; super recursion detection")
+ def test_recursionlimit_fatalerror(self):
+ # A fatal error occurs if a second recursion limit is hit when recovering
+ # from a first one.
+ code = textwrap.dedent("""
+ import sys
+
+ def f():
+ try:
+ f()
+ except RecursionError:
+ f()
+
+ sys.setrecursionlimit(%d)
+ f()""")
+ with test.support.SuppressCrashReport():
+ for i in (50, 1000):
+ sub = subprocess.Popen([sys.executable, '-c', code % i],
+ stderr=subprocess.PIPE)
+ err = sub.communicate()[1]
+ self.assertTrue(sub.returncode, sub.returncode)
+ self.assertIn(
+ b"Fatal Python error: Cannot recover from stack overflow",
+ err)
+
+ def test_getwindowsversion(self):
+ # Raise SkipTest if sys doesn't have getwindowsversion attribute
+ test.support.get_attribute(sys, "getwindowsversion")
+ v = sys.getwindowsversion()
+ self.assertEqual(len(v), 5)
+ self.assertIsInstance(v[0], int)
+ self.assertIsInstance(v[1], int)
+ self.assertIsInstance(v[2], int)
+ self.assertIsInstance(v[3], int)
+ self.assertIsInstance(v[4], str)
+ self.assertRaises(IndexError, operator.getitem, v, 5)
+ self.assertIsInstance(v.major, int)
+ self.assertIsInstance(v.minor, int)
+ self.assertIsInstance(v.build, int)
+ self.assertIsInstance(v.platform, int)
+ self.assertIsInstance(v.service_pack, str)
+ self.assertIsInstance(v.service_pack_minor, int)
+ self.assertIsInstance(v.service_pack_major, int)
+ self.assertIsInstance(v.suite_mask, int)
+ self.assertIsInstance(v.product_type, int)
+ self.assertEqual(v[0], v.major)
+ self.assertEqual(v[1], v.minor)
+ self.assertEqual(v[2], v.build)
+ self.assertEqual(v[3], v.platform)
+ self.assertEqual(v[4], v.service_pack)
+
+ # This is how platform.py calls it. Make sure tuple
+ # still has 5 elements
+ maj, min, buildno, plat, csd = sys.getwindowsversion()
+
+ @unittest.skip("TODO: RUSTPYTHON; sys.call_tracing")
+ def test_call_tracing(self):
+ self.assertRaises(TypeError, sys.call_tracing, type, 2)
+
+ @unittest.skipUnless(hasattr(sys, "setdlopenflags"),
+ 'test needs sys.setdlopenflags()')
+ def test_dlopenflags(self):
+ self.assertTrue(hasattr(sys, "getdlopenflags"))
+ self.assertRaises(TypeError, sys.getdlopenflags, 42)
+ oldflags = sys.getdlopenflags()
+ self.assertRaises(TypeError, sys.setdlopenflags)
+ sys.setdlopenflags(oldflags+1)
+ self.assertEqual(sys.getdlopenflags(), oldflags+1)
+ sys.setdlopenflags(oldflags)
+
+ @test.support.refcount_test
+ def test_refcount(self):
+ # n here must be a global in order for this test to pass while
+ # tracing with a python function. Tracing calls PyFrame_FastToLocals
+ # which will add a copy of any locals to the frame object, causing
+ # the reference count to increase by 2 instead of 1.
+ global n
+ self.assertRaises(TypeError, sys.getrefcount)
+ c = sys.getrefcount(None)
+ n = None
+ self.assertEqual(sys.getrefcount(None), c+1)
+ del n
+ self.assertEqual(sys.getrefcount(None), c)
+ if hasattr(sys, "gettotalrefcount"):
+ self.assertIsInstance(sys.gettotalrefcount(), int)
+
+ def test_getframe(self):
+ self.assertRaises(TypeError, sys._getframe, 42, 42)
+ self.assertRaises(ValueError, sys._getframe, 2000000000)
+ self.assertTrue(
+ SysModuleTest.test_getframe.__code__ \
+ is sys._getframe().f_code
+ )
+
+ # sys._current_frames() is a CPython-only gimmick.
+ # XXX RUSTPYTHON: above comment is from original cpython test; not sure why the cpython_only decorator wasn't added
+ @test.support.cpython_only
+ @test.support.reap_threads
+ def test_current_frames(self):
+ import threading
+ import traceback
+
+ # Spawn a thread that blocks at a known place. Then the main
+ # thread does sys._current_frames(), and verifies that the frames
+ # returned make sense.
+ entered_g = threading.Event()
+ leave_g = threading.Event()
+ thread_info = [] # the thread's id
+
+ def f123():
+ g456()
+
+ def g456():
+ thread_info.append(threading.get_ident())
+ entered_g.set()
+ leave_g.wait()
+
+ t = threading.Thread(target=f123)
+ t.start()
+ entered_g.wait()
+
+ # At this point, t has finished its entered_g.set(), although it's
+ # impossible to guess whether it's still on that line or has moved on
+ # to its leave_g.wait().
+ self.assertEqual(len(thread_info), 1)
+ thread_id = thread_info[0]
+
+ d = sys._current_frames()
+ for tid in d:
+ self.assertIsInstance(tid, int)
+ self.assertGreater(tid, 0)
+
+ main_id = threading.get_ident()
+ self.assertIn(main_id, d)
+ self.assertIn(thread_id, d)
+
+ # Verify that the captured main-thread frame is _this_ frame.
+ frame = d.pop(main_id)
+ self.assertTrue(frame is sys._getframe())
+
+ # Verify that the captured thread frame is blocked in g456, called
+ # from f123. This is a litte tricky, since various bits of
+ # threading.py are also in the thread's call stack.
+ frame = d.pop(thread_id)
+ stack = traceback.extract_stack(frame)
+ for i, (filename, lineno, funcname, sourceline) in enumerate(stack):
+ if funcname == "f123":
+ break
+ else:
+ self.fail("didn't find f123() on thread's call stack")
+
+ self.assertEqual(sourceline, "g456()")
+
+ # And the next record must be for g456().
+ filename, lineno, funcname, sourceline = stack[i+1]
+ self.assertEqual(funcname, "g456")
+ self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"])
+
+ # Reap the spawned thread.
+ leave_g.set()
+ t.join()
+
+ def test_attributes(self):
+ self.assertIsInstance(sys.api_version, int)
+ self.assertIsInstance(sys.argv, list)
+ self.assertIn(sys.byteorder, ("little", "big"))
+ self.assertIsInstance(sys.builtin_module_names, tuple)
+ self.assertIsInstance(sys.copyright, str)
+ self.assertIsInstance(sys.exec_prefix, str)
+ self.assertIsInstance(sys.base_exec_prefix, str)
+ self.assertIsInstance(sys.executable, str)
+ self.assertEqual(len(sys.float_info), 11)
+ self.assertEqual(sys.float_info.radix, 2)
+ self.assertEqual(len(sys.int_info), 2)
+ self.assertTrue(sys.int_info.bits_per_digit % 5 == 0)
+ self.assertTrue(sys.int_info.sizeof_digit >= 1)
+ self.assertEqual(type(sys.int_info.bits_per_digit), int)
+ self.assertEqual(type(sys.int_info.sizeof_digit), int)
+ self.assertIsInstance(sys.hexversion, int)
+
+ self.assertEqual(len(sys.hash_info), 9)
+ self.assertLess(sys.hash_info.modulus, 2**sys.hash_info.width)
+ # sys.hash_info.modulus should be a prime; we do a quick
+ # probable primality test (doesn't exclude the possibility of
+ # a Carmichael number)
+ for x in range(1, 100):
+ self.assertEqual(
+ pow(x, sys.hash_info.modulus-1, sys.hash_info.modulus),
+ 1,
+ "sys.hash_info.modulus {} is a non-prime".format(
+ sys.hash_info.modulus)
+ )
+ self.assertIsInstance(sys.hash_info.inf, int)
+ self.assertIsInstance(sys.hash_info.nan, int)
+ self.assertIsInstance(sys.hash_info.imag, int)
+ algo = sysconfig.get_config_var("Py_HASH_ALGORITHM")
+ if sys.hash_info.algorithm in {"fnv", "siphash24"}:
+ self.assertIn(sys.hash_info.hash_bits, {32, 64})
+ self.assertIn(sys.hash_info.seed_bits, {32, 64, 128})
+
+ if algo == 1:
+ self.assertEqual(sys.hash_info.algorithm, "siphash24")
+ elif algo == 2:
+ self.assertEqual(sys.hash_info.algorithm, "fnv")
+ else:
+ self.assertIn(sys.hash_info.algorithm, {"fnv", "siphash24"})
+ else:
+ # PY_HASH_EXTERNAL
+ # TODO: RUSTPYTHON; use siphash24
+ # self.assertEqual(algo, 0)
+ pass
+ self.assertGreaterEqual(sys.hash_info.cutoff, 0)
+ self.assertLess(sys.hash_info.cutoff, 8)
+
+ self.assertIsInstance(sys.maxsize, int)
+ self.assertIsInstance(sys.maxunicode, int)
+ self.assertEqual(sys.maxunicode, 0x10FFFF)
+ self.assertIsInstance(sys.platform, str)
+ self.assertIsInstance(sys.prefix, str)
+ self.assertIsInstance(sys.base_prefix, str)
+ self.assertIsInstance(sys.version, str)
+ vi = sys.version_info
+ self.assertIsInstance(vi[:], tuple)
+ self.assertEqual(len(vi), 5)
+ self.assertIsInstance(vi[0], int)
+ self.assertIsInstance(vi[1], int)
+ self.assertIsInstance(vi[2], int)
+ self.assertIn(vi[3], ("alpha", "beta", "candidate", "final"))
+ self.assertIsInstance(vi[4], int)
+ self.assertIsInstance(vi.major, int)
+ self.assertIsInstance(vi.minor, int)
+ self.assertIsInstance(vi.micro, int)
+ self.assertIn(vi.releaselevel, ("alpha", "beta", "candidate", "final"))
+ self.assertIsInstance(vi.serial, int)
+ self.assertEqual(vi[0], vi.major)
+ self.assertEqual(vi[1], vi.minor)
+ self.assertEqual(vi[2], vi.micro)
+ self.assertEqual(vi[3], vi.releaselevel)
+ self.assertEqual(vi[4], vi.serial)
+ self.assertTrue(vi > (1,0,0))
+ self.assertIsInstance(sys.float_repr_style, str)
+ self.assertIn(sys.float_repr_style, ('short', 'legacy'))
+ if not sys.platform.startswith('win'):
+ self.assertIsInstance(sys.abiflags, str)
+
+ @unittest.skip("TODO: RUSTPYTHON; sys.thread_info")
+ def test_thread_info(self):
+ info = sys.thread_info
+ self.assertEqual(len(info), 3)
+ self.assertIn(info.name, ('nt', 'pthread', 'solaris', None))
+ self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))
+
+ def test_43581(self):
+ # Can't use sys.stdout, as this is a StringIO object when
+ # the test runs under regrtest.
+ self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding)
+
+ @unittest.skip("TODO: RUSTPYTHON; sys.intern() string interning")
+ def test_intern(self):
+ global INTERN_NUMRUNS
+ INTERN_NUMRUNS += 1
+ self.assertRaises(TypeError, sys.intern)
+ s = "never interned before" + str(INTERN_NUMRUNS)
+ self.assertTrue(sys.intern(s) is s)
+ s2 = s.swapcase().swapcase()
+ self.assertTrue(sys.intern(s2) is s)
+
+ # Subclasses of string can't be interned, because they
+ # provide too much opportunity for insane things to happen.
+ # We don't want them in the interned dict and if they aren't
+ # actually interned, we don't want to create the appearance
+ # that they are by allowing intern() to succeed.
+ class S(str):
+ def __hash__(self):
+ return 123
+
+ self.assertRaises(TypeError, sys.intern, S("abc"))
+
+ def test_sys_flags(self):
+ self.assertTrue(sys.flags)
+ attrs = ("debug",
+ "inspect", "interactive", "optimize", "dont_write_bytecode",
+ "no_user_site", "no_site", "ignore_environment", "verbose",
+ "bytes_warning", "quiet", "hash_randomization", "isolated",
+ "dev_mode", "utf8_mode")
+ for attr in attrs:
+ self.assertTrue(hasattr(sys.flags, attr), attr)
+ attr_type = bool if attr == "dev_mode" else int
+ self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr)
+ self.assertTrue(repr(sys.flags))
+ self.assertEqual(len(sys.flags), len(attrs))
+
+ self.assertIn(sys.flags.utf8_mode, {0, 1, 2})
+
+ def assert_raise_on_new_sys_type(self, sys_attr):
+ # Users are intentionally prevented from creating new instances of
+ # sys.flags, sys.version_info, and sys.getwindowsversion.
+ attr_type = type(sys_attr)
+ with self.assertRaises(TypeError):
+ attr_type()
+ with self.assertRaises(TypeError):
+ attr_type.__new__(attr_type)
+
+ def test_sys_flags_no_instantiation(self):
+ self.assert_raise_on_new_sys_type(sys.flags)
+
+ def test_sys_version_info_no_instantiation(self):
+ self.assert_raise_on_new_sys_type(sys.version_info)
+
+ def test_sys_getwindowsversion_no_instantiation(self):
+ # Skip if not being run on Windows.
+ test.support.get_attribute(sys, "getwindowsversion")
+ self.assert_raise_on_new_sys_type(sys.getwindowsversion())
+
+ @test.support.cpython_only
+ def test_clear_type_cache(self):
+ sys._clear_type_cache()
+
+ @unittest.skip("TODO: RUSTPYTHON; PYTHONIOENCODING var")
+ def test_ioencoding(self):
+ env = dict(os.environ)
+
+ # Test character: cent sign, encoded as 0x4A (ASCII J) in CP424,
+ # not representable in ASCII.
+
+ env["PYTHONIOENCODING"] = "cp424"
+ p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
+ stdout = subprocess.PIPE, env=env)
+ out = p.communicate()[0].strip()
+ expected = ("\xa2" + os.linesep).encode("cp424")
+ self.assertEqual(out, expected)
+
+ env["PYTHONIOENCODING"] = "ascii:replace"
+ p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
+ stdout = subprocess.PIPE, env=env)
+ out = p.communicate()[0].strip()
+ self.assertEqual(out, b'?')
+
+ env["PYTHONIOENCODING"] = "ascii"
+ p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ env=env)
+ out, err = p.communicate()
+ self.assertEqual(out, b'')
+ self.assertIn(b'UnicodeEncodeError:', err)
+ self.assertIn(rb"'\xa2'", err)
+
+ env["PYTHONIOENCODING"] = "ascii:"
+ p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xa2))'],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ env=env)
+ out, err = p.communicate()
+ self.assertEqual(out, b'')
+ self.assertIn(b'UnicodeEncodeError:', err)
+ self.assertIn(rb"'\xa2'", err)
+
+ env["PYTHONIOENCODING"] = ":surrogateescape"
+ p = subprocess.Popen([sys.executable, "-c", 'print(chr(0xdcbd))'],
+ stdout=subprocess.PIPE, env=env)
+ out = p.communicate()[0].strip()
+ self.assertEqual(out, b'\xbd')
+
+ @unittest.skipUnless(test.support.FS_NONASCII,
+ 'requires OS support of non-ASCII encodings')
+ @unittest.skipUnless(sys.getfilesystemencoding() == locale.getpreferredencoding(False),
+ 'requires FS encoding to match locale')
+ def test_ioencoding_nonascii(self):
+ env = dict(os.environ)
+
+ env["PYTHONIOENCODING"] = ""
+ p = subprocess.Popen([sys.executable, "-c",
+ 'print(%a)' % test.support.FS_NONASCII],
+ stdout=subprocess.PIPE, env=env)
+ out = p.communicate()[0].strip()
+ self.assertEqual(out, os.fsencode(test.support.FS_NONASCII))
+
+ @unittest.skipIf(sys.base_prefix != sys.prefix,
+ 'Test is not venv-compatible')
+ def test_executable(self):
+ # sys.executable should be absolute
+ self.assertEqual(os.path.abspath(sys.executable), sys.executable)
+
+ # Issue #7774: Ensure that sys.executable is an empty string if argv[0]
+ # has been set to a non existent program name and Python is unable to
+ # retrieve the real program name
+
+ # For a normal installation, it should work without 'cwd'
+ # argument. For test runs in the build directory, see #7774.
+ python_dir = os.path.dirname(os.path.realpath(sys.executable))
+ p = subprocess.Popen(
+ ["nonexistent", "-c",
+ 'import sys; print(sys.executable.encode("ascii", "backslashreplace"))'],
+ executable=sys.executable, stdout=subprocess.PIPE, cwd=python_dir)
+ stdout = p.communicate()[0]
+ executable = stdout.strip().decode("ASCII")
+ p.wait()
+ self.assertIn(executable, ["b''", repr(sys.executable.encode("ascii", "backslashreplace"))])
+
+ def check_fsencoding(self, fs_encoding, expected=None):
+ self.assertIsNotNone(fs_encoding)
+ codecs.lookup(fs_encoding)
+ if expected:
+ self.assertEqual(fs_encoding, expected)
+
+ def test_getfilesystemencoding(self):
+ fs_encoding = sys.getfilesystemencoding()
+ if sys.platform == 'darwin':
+ expected = 'utf-8'
+ else:
+ expected = None
+ self.check_fsencoding(fs_encoding, expected)
+
+ def c_locale_get_error_handler(self, locale, isolated=False, encoding=None):
+ # Force the POSIX locale
+ env = os.environ.copy()
+ env["LC_ALL"] = locale
+ env["PYTHONCOERCECLOCALE"] = "0"
+ code = '\n'.join((
+ 'import sys',
+ 'def dump(name):',
+ ' std = getattr(sys, name)',
+ ' print("%s: %s" % (name, std.errors))',
+ 'dump("stdin")',
+ 'dump("stdout")',
+ 'dump("stderr")',
+ ))
+ args = [sys.executable, "-X", "utf8=0", "-c", code]
+ if isolated:
+ args.append("-I")
+ if encoding is not None:
+ env['PYTHONIOENCODING'] = encoding
+ else:
+ env.pop('PYTHONIOENCODING', None)
+ p = subprocess.Popen(args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=env,
+ universal_newlines=True)
+ stdout, stderr = p.communicate()
+ return stdout
+
+ @unittest.skip("TODO: RUSTPYTHON; surrogates in strings")
+ def check_locale_surrogateescape(self, locale):
+ out = self.c_locale_get_error_handler(locale, isolated=True)
+ self.assertEqual(out,
+ 'stdin: surrogateescape\n'
+ 'stdout: surrogateescape\n'
+ 'stderr: backslashreplace\n')
+
+ # replace the default error handler
+ out = self.c_locale_get_error_handler(locale, encoding=':ignore')
+ self.assertEqual(out,
+ 'stdin: ignore\n'
+ 'stdout: ignore\n'
+ 'stderr: backslashreplace\n')
+
+ # force the encoding
+ out = self.c_locale_get_error_handler(locale, encoding='iso8859-1')
+ self.assertEqual(out,
+ 'stdin: strict\n'
+ 'stdout: strict\n'
+ 'stderr: backslashreplace\n')
+ out = self.c_locale_get_error_handler(locale, encoding='iso8859-1:')
+ self.assertEqual(out,
+ 'stdin: strict\n'
+ 'stdout: strict\n'
+ 'stderr: backslashreplace\n')
+
+ # have no any effect
+ out = self.c_locale_get_error_handler(locale, encoding=':')
+ self.assertEqual(out,
+ 'stdin: surrogateescape\n'
+ 'stdout: surrogateescape\n'
+ 'stderr: backslashreplace\n')
+ out = self.c_locale_get_error_handler(locale, encoding='')
+ self.assertEqual(out,
+ 'stdin: surrogateescape\n'
+ 'stdout: surrogateescape\n'
+ 'stderr: backslashreplace\n')
+
+ def test_c_locale_surrogateescape(self):
+ self.check_locale_surrogateescape('C')
+
+ def test_posix_locale_surrogateescape(self):
+ self.check_locale_surrogateescape('POSIX')
+
+ def test_implementation(self):
+ # This test applies to all implementations equally.
+
+ levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF}
+
+ self.assertTrue(hasattr(sys.implementation, 'name'))
+ self.assertTrue(hasattr(sys.implementation, 'version'))
+ self.assertTrue(hasattr(sys.implementation, 'hexversion'))
+ self.assertTrue(hasattr(sys.implementation, 'cache_tag'))
+
+ version = sys.implementation.version
+ self.assertEqual(version[:2], (version.major, version.minor))
+
+ hexversion = (version.major << 24 | version.minor << 16 |
+ version.micro << 8 | levels[version.releaselevel] << 4 |
+ version.serial << 0)
+ self.assertEqual(sys.implementation.hexversion, hexversion)
+
+ # PEP 421 requires that .name be lower case.
+ self.assertEqual(sys.implementation.name,
+ sys.implementation.name.lower())
+
+ @test.support.cpython_only
+ def test_debugmallocstats(self):
+ # Test sys._debugmallocstats()
+ from test.support.script_helper import assert_python_ok
+ args = ['-c', 'import sys; sys._debugmallocstats()']
+ ret, out, err = assert_python_ok(*args)
+ self.assertIn(b"free PyDictObjects", err)
+
+ # The function has no parameter
+ self.assertRaises(TypeError, sys._debugmallocstats, True)
+
+ @unittest.skipUnless(hasattr(sys, "getallocatedblocks"),
+ "sys.getallocatedblocks unavailable on this build")
+ def test_getallocatedblocks(self):
+ try:
+ import _testcapi
+ except ImportError:
+ with_pymalloc = support.with_pymalloc()
+ else:
+ try:
+ alloc_name = _testcapi.pymem_getallocatorsname()
+ except RuntimeError as exc:
+ # "cannot get allocators name" (ex: tracemalloc is used)
+ with_pymalloc = True
+ else:
+ with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug'))
+
+ # Some sanity checks
+ a = sys.getallocatedblocks()
+ self.assertIs(type(a), int)
+ if with_pymalloc:
+ self.assertGreater(a, 0)
+ else:
+ # When WITH_PYMALLOC isn't available, we don't know anything
+ # about the underlying implementation: the function might
+ # return 0 or something greater.
+ self.assertGreaterEqual(a, 0)
+ try:
+ # While we could imagine a Python session where the number of
+ # multiple buffer objects would exceed the sharing of references,
+ # it is unlikely to happen in a normal test run.
+ self.assertLess(a, sys.gettotalrefcount())
+ except AttributeError:
+ # gettotalrefcount() not available
+ pass
+ gc.collect()
+ b = sys.getallocatedblocks()
+ self.assertLessEqual(b, a)
+ gc.collect()
+ c = sys.getallocatedblocks()
+ self.assertIn(c, range(b - 50, b + 50))
+
+ @unittest.skip("TODO: RUSTPYTHON; destructors + interpreter finalization")
+ @test.support.requires_type_collecting
+ def test_is_finalizing(self):
+ self.assertIs(sys.is_finalizing(), False)
+ # Don't use the atexit module because _Py_Finalizing is only set
+ # after calling atexit callbacks
+ code = """if 1:
+ import sys
+
+ class AtExit:
+ is_finalizing = sys.is_finalizing
+ print = print
+
+ def __del__(self):
+ self.print(self.is_finalizing(), flush=True)
+
+ # Keep a reference in the __main__ module namespace, so the
+ # AtExit destructor will be called at Python exit
+ ref = AtExit()
+ """
+ rc, stdout, stderr = assert_python_ok('-c', code)
+ self.assertEqual(stdout.rstrip(), b'True')
+
+ @unittest.skip("TODO: RUSTPYTHON; __del__ destructors/interpreter shutdown")
+ @test.support.requires_type_collecting
+ def test_issue20602(self):
+ # sys.flags and sys.float_info were wiped during shutdown.
+ code = """if 1:
+ import sys
+ class A:
+ def __del__(self, sys=sys):
+ print(sys.flags)
+ print(sys.float_info)
+ a = A()
+ """
+ rc, out, err = assert_python_ok('-c', code)
+ out = out.splitlines()
+ self.assertIn(b'sys.flags', out[0])
+ self.assertIn(b'sys.float_info', out[1])
+
+ @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'),
+ 'need sys.getandroidapilevel()')
+ def test_getandroidapilevel(self):
+ level = sys.getandroidapilevel()
+ self.assertIsInstance(level, int)
+ self.assertGreater(level, 0)
+
+ @unittest.skip("TODO: RUSTPYTHON; sys.tracebacklimit")
+ def test_sys_tracebacklimit(self):
+ code = """if 1:
+ import sys
+ def f1():
+ 1 / 0
+ def f2():
+ f1()
+ sys.tracebacklimit = %r
+ f2()
+ """
+ def check(tracebacklimit, expected):
+ p = subprocess.Popen([sys.executable, '-c', code % tracebacklimit],
+ stderr=subprocess.PIPE)
+ out = p.communicate()[1]
+ self.assertEqual(out.splitlines(), expected)
+
+ traceback = [
+ b'Traceback (most recent call last):',
+ b' File "", line 8, in ',
+ b' File "", line 6, in f2',
+ b' File "", line 4, in f1',
+ b'ZeroDivisionError: division by zero'
+ ]
+ check(10, traceback)
+ check(3, traceback)
+ check(2, traceback[:1] + traceback[2:])
+ check(1, traceback[:1] + traceback[3:])
+ check(0, [traceback[-1]])
+ check(-1, [traceback[-1]])
+ check(1<<1000, traceback)
+ check(-1<<1000, [traceback[-1]])
+ check(None, traceback)
+
+ def test_no_duplicates_in_meta_path(self):
+ self.assertEqual(len(sys.meta_path), len(set(sys.meta_path)))
+
+ @unittest.skipUnless(hasattr(sys, "_enablelegacywindowsfsencoding"),
+ 'needs sys._enablelegacywindowsfsencoding()')
+ def test__enablelegacywindowsfsencoding(self):
+ code = ('import sys',
+ 'sys._enablelegacywindowsfsencoding()',
+ 'print(sys.getfilesystemencoding(), sys.getfilesystemencodeerrors())')
+ rc, out, err = assert_python_ok('-c', '; '.join(code))
+ out = out.decode('ascii', 'replace').rstrip()
+ self.assertEqual(out, 'mbcs replace')
+
+
+@test.support.cpython_only
+class UnraisableHookTest(unittest.TestCase):
+ def write_unraisable_exc(self, exc, err_msg, obj):
+ import _testcapi
+ import types
+ err_msg2 = f"Exception ignored {err_msg}"
+ try:
+ _testcapi.write_unraisable_exc(exc, err_msg, obj)
+ return types.SimpleNamespace(exc_type=type(exc),
+ exc_value=exc,
+ exc_traceback=exc.__traceback__,
+ err_msg=err_msg2,
+ object=obj)
+ finally:
+ # Explicitly break any reference cycle
+ exc = None
+
+ def test_original_unraisablehook(self):
+ for err_msg in (None, "original hook"):
+ with self.subTest(err_msg=err_msg):
+ obj = "an object"
+
+ with test.support.captured_output("stderr") as stderr:
+ with test.support.swap_attr(sys, 'unraisablehook',
+ sys.__unraisablehook__):
+ self.write_unraisable_exc(ValueError(42), err_msg, obj)
+
+ err = stderr.getvalue()
+ if err_msg is not None:
+ self.assertIn(f'Exception ignored {err_msg}: {obj!r}\n', err)
+ else:
+ self.assertIn(f'Exception ignored in: {obj!r}\n', err)
+ self.assertIn('Traceback (most recent call last):\n', err)
+ self.assertIn('ValueError: 42\n', err)
+
+ def test_original_unraisablehook_err(self):
+ # bpo-22836: PyErr_WriteUnraisable() should give sensible reports
+ class BrokenDel:
+ def __del__(self):
+ exc = ValueError("del is broken")
+ # The following line is included in the traceback report:
+ raise exc
+
+ class BrokenStrException(Exception):
+ def __str__(self):
+ raise Exception("str() is broken")
+
+ class BrokenExceptionDel:
+ def __del__(self):
+ exc = BrokenStrException()
+ # The following line is included in the traceback report:
+ raise exc
+
+ for test_class in (BrokenDel, BrokenExceptionDel):
+ with self.subTest(test_class):
+ obj = test_class()
+ with test.support.captured_stderr() as stderr, \
+ test.support.swap_attr(sys, 'unraisablehook',
+ sys.__unraisablehook__):
+ # Trigger obj.__del__()
+ del obj
+
+ report = stderr.getvalue()
+ self.assertIn("Exception ignored", report)
+ self.assertIn(test_class.__del__.__qualname__, report)
+ self.assertIn("test_sys.py", report)
+ self.assertIn("raise exc", report)
+ if test_class is BrokenExceptionDel:
+ self.assertIn("BrokenStrException", report)
+ self.assertIn("", report)
+ else:
+ self.assertIn("ValueError", report)
+ self.assertIn("del is broken", report)
+ self.assertTrue(report.endswith("\n"))
+
+
+ def test_original_unraisablehook_wrong_type(self):
+ exc = ValueError(42)
+ with test.support.swap_attr(sys, 'unraisablehook',
+ sys.__unraisablehook__):
+ with self.assertRaises(TypeError):
+ sys.unraisablehook(exc)
+
+ def test_custom_unraisablehook(self):
+ hook_args = None
+
+ def hook_func(args):
+ nonlocal hook_args
+ hook_args = args
+
+ obj = object()
+ try:
+ with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+ expected = self.write_unraisable_exc(ValueError(42),
+ "custom hook", obj)
+ for attr in "exc_type exc_value exc_traceback err_msg object".split():
+ self.assertEqual(getattr(hook_args, attr),
+ getattr(expected, attr),
+ (hook_args, expected))
+ finally:
+ # expected and hook_args contain an exception: break reference cycle
+ expected = None
+ hook_args = None
+
+ def test_custom_unraisablehook_fail(self):
+ def hook_func(*args):
+ raise Exception("hook_func failed")
+
+ with test.support.captured_output("stderr") as stderr:
+ with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+ self.write_unraisable_exc(ValueError(42),
+ "custom hook fail", None)
+
+ err = stderr.getvalue()
+ self.assertIn(f'Exception ignored in sys.unraisablehook: '
+ f'{hook_func!r}\n',
+ err)
+ self.assertIn('Traceback (most recent call last):\n', err)
+ self.assertIn('Exception: hook_func failed\n', err)
+
+
+@test.support.cpython_only
+class SizeofTest(unittest.TestCase):
+
+ def setUp(self):
+ self.P = struct.calcsize('P')
+ self.longdigit = sys.int_info.sizeof_digit
+ import _testcapi
+ self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD
+
+ check_sizeof = test.support.check_sizeof
+
+ def test_gc_head_size(self):
+ # Check that the gc header size is added to objects tracked by the gc.
+ vsize = test.support.calcvobjsize
+ gc_header_size = self.gc_headsize
+ # bool objects are not gc tracked
+ self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
+ # but lists are
+ self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size)
+
+ def test_errors(self):
+ class BadSizeof:
+ def __sizeof__(self):
+ raise ValueError
+ self.assertRaises(ValueError, sys.getsizeof, BadSizeof())
+
+ class InvalidSizeof:
+ def __sizeof__(self):
+ return None
+ self.assertRaises(TypeError, sys.getsizeof, InvalidSizeof())
+ sentinel = ["sentinel"]
+ self.assertIs(sys.getsizeof(InvalidSizeof(), sentinel), sentinel)
+
+ class FloatSizeof:
+ def __sizeof__(self):
+ return 4.5
+ self.assertRaises(TypeError, sys.getsizeof, FloatSizeof())
+ self.assertIs(sys.getsizeof(FloatSizeof(), sentinel), sentinel)
+
+ class OverflowSizeof(int):
+ def __sizeof__(self):
+ return int(self)
+ self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)),
+ sys.maxsize + self.gc_headsize)
+ with self.assertRaises(OverflowError):
+ sys.getsizeof(OverflowSizeof(sys.maxsize + 1))
+ with self.assertRaises(ValueError):
+ sys.getsizeof(OverflowSizeof(-1))
+ with self.assertRaises((ValueError, OverflowError)):
+ sys.getsizeof(OverflowSizeof(-sys.maxsize - 1))
+
+ def test_default(self):
+ size = test.support.calcvobjsize
+ self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
+ self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)
+
+ def test_objecttypes(self):
+ # check all types defined in Objects/
+ calcsize = struct.calcsize
+ size = test.support.calcobjsize
+ vsize = test.support.calcvobjsize
+ check = self.check_sizeof
+ # bool
+ check(True, vsize('') + self.longdigit)
+ # buffer
+ # XXX
+ # builtin_function_or_method
+ check(len, size('5P'))
+ # bytearray
+ samples = [b'', b'u'*100000]
+ for sample in samples:
+ x = bytearray(sample)
+ check(x, vsize('n2Pi') + x.__alloc__())
+ # bytearray_iterator
+ check(iter(bytearray()), size('nP'))
+ # bytes
+ check(b'', vsize('n') + 1)
+ check(b'x' * 10, vsize('n') + 11)
+ # cell
+ def get_cell():
+ x = 42
+ def inner():
+ return x
+ return inner
+ check(get_cell().__closure__[0], size('P'))
+ # code
+ def check_code_size(a, expected_size):
+ self.assertGreaterEqual(sys.getsizeof(a), expected_size)
+ check_code_size(get_cell().__code__, size('6i13P'))
+ check_code_size(get_cell.__code__, size('6i13P'))
+ def get_cell2(x):
+ def inner():
+ return x
+ return inner
+ check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n'))
+ # complex
+ check(complex(0,1), size('2d'))
+ # method_descriptor (descriptor object)
+ check(str.lower, size('3PPP'))
+ # classmethod_descriptor (descriptor object)
+ # XXX
+ # member_descriptor (descriptor object)
+ import datetime
+ check(datetime.timedelta.days, size('3PP'))
+ # getset_descriptor (descriptor object)
+ import collections
+ check(collections.defaultdict.default_factory, size('3PP'))
+ # wrapper_descriptor (descriptor object)
+ check(int.__add__, size('3P2P'))
+ # method-wrapper (descriptor object)
+ check({}.__iter__, size('2P'))
+ # empty dict
+ check({}, size('nQ2P'))
+ # dict
+ check({"a": 1}, size('nQ2P') + calcsize('2nP2n') + 8 + (8*2//3)*calcsize('n2P'))
+ longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
+ check(longdict, size('nQ2P') + calcsize('2nP2n') + 16 + (16*2//3)*calcsize('n2P'))
+ # dictionary-keyview
+ check({}.keys(), size('P'))
+ # dictionary-valueview
+ check({}.values(), size('P'))
+ # dictionary-itemview
+ check({}.items(), size('P'))
+ # dictionary iterator
+ check(iter({}), size('P2nPn'))
+ # dictionary-keyiterator
+ check(iter({}.keys()), size('P2nPn'))
+ # dictionary-valueiterator
+ check(iter({}.values()), size('P2nPn'))
+ # dictionary-itemiterator
+ check(iter({}.items()), size('P2nPn'))
+ # dictproxy
+ class C(object): pass
+ check(C.__dict__, size('P'))
+ # BaseException
+ check(BaseException(), size('5Pb'))
+ # UnicodeEncodeError
+ check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pb 2P2nP'))
+ # UnicodeDecodeError
+ check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pb 2P2nP'))
+ # UnicodeTranslateError
+ check(UnicodeTranslateError("", 0, 1, ""), size('5Pb 2P2nP'))
+ # ellipses
+ check(Ellipsis, size(''))
+ # EncodingMap
+ import codecs, encodings.iso8859_3
+ x = codecs.charmap_build(encodings.iso8859_3.decoding_table)
+ check(x, size('32B2iB'))
+ # enumerate
+ check(enumerate([]), size('n3P'))
+ # reverse
+ check(reversed(''), size('nP'))
+ # float
+ check(float(0), size('d'))
+ # sys.floatinfo
+ check(sys.float_info, vsize('') + self.P * len(sys.float_info))
+ # frame
+ import inspect
+ CO_MAXBLOCKS = 20
+ x = inspect.currentframe()
+ ncells = len(x.f_code.co_cellvars)
+ nfrees = len(x.f_code.co_freevars)
+ extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
+ ncells + nfrees - 1
+ check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+ # function
+ def func(): pass
+ check(func, size('13P'))
+ class c():
+ @staticmethod
+ def foo():
+ pass
+ @classmethod
+ def bar(cls):
+ pass
+ # staticmethod
+ check(foo, size('PP'))
+ # classmethod
+ check(bar, size('PP'))
+ # generator
+ def get_gen(): yield 1
+ check(get_gen(), size('Pb2PPP4P'))
+ # iterator
+ check(iter('abc'), size('lP'))
+ # callable-iterator
+ import re
+ check(re.finditer('',''), size('2P'))
+ # list
+ samples = [[], [1,2,3], ['1', '2', '3']]
+ for sample in samples:
+ check(sample, vsize('Pn') + len(sample)*self.P)
+ # sortwrapper (list)
+ # XXX
+ # cmpwrapper (list)
+ # XXX
+ # listiterator (list)
+ check(iter([]), size('lP'))
+ # listreverseiterator (list)
+ check(reversed([]), size('nP'))
+ # int
+ check(0, vsize(''))
+ check(1, vsize('') + self.longdigit)
+ check(-1, vsize('') + self.longdigit)
+ PyLong_BASE = 2**sys.int_info.bits_per_digit
+ check(int(PyLong_BASE), vsize('') + 2*self.longdigit)
+ check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
+ check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
+ # module
+ check(unittest, size('PnPPP'))
+ # None
+ check(None, size(''))
+ # NotImplementedType
+ check(NotImplemented, size(''))
+ # object
+ check(object(), size(''))
+ # property (descriptor object)
+ class C(object):
+ def getx(self): return self.__x
+ def setx(self, value): self.__x = value
+ def delx(self): del self.__x
+ x = property(getx, setx, delx, "")
+ check(x, size('4Pi'))
+ # PyCapsule
+ # XXX
+ # rangeiterator
+ check(iter(range(1)), size('4l'))
+ # reverse
+ check(reversed(''), size('nP'))
+ # range
+ check(range(1), size('4P'))
+ check(range(66000), size('4P'))
+ # set
+ # frozenset
+ PySet_MINSIZE = 8
+ samples = [[], range(10), range(50)]
+ s = size('3nP' + PySet_MINSIZE*'nP' + '2nP')
+ for sample in samples:
+ minused = len(sample)
+ if minused == 0: tmp = 1
+ # the computation of minused is actually a bit more complicated
+ # but this suffices for the sizeof test
+ minused = minused*2
+ newsize = PySet_MINSIZE
+ while newsize <= minused:
+ newsize = newsize << 1
+ if newsize <= 8:
+ check(set(sample), s)
+ check(frozenset(sample), s)
+ else:
+ check(set(sample), s + newsize*calcsize('nP'))
+ check(frozenset(sample), s + newsize*calcsize('nP'))
+ # setiterator
+ check(iter(set()), size('P3n'))
+ # slice
+ check(slice(0), size('3P'))
+ # super
+ check(super(int), size('3P'))
+ # tuple
+ check((), vsize(''))
+ check((1,2,3), vsize('') + 3*self.P)
+ # type
+ # static type: PyTypeObject
+ fmt = 'P2nPI13Pl4Pn9Pn11PIPPP'
+ if hasattr(sys, 'getcounts'):
+ fmt += '3n2P'
+ s = vsize(fmt)
+ check(int, s)
+ # class
+ s = vsize(fmt + # PyTypeObject
+ '3P' # PyAsyncMethods
+ '36P' # PyNumberMethods
+ '3P' # PyMappingMethods
+ '10P' # PySequenceMethods
+ '2P' # PyBufferProcs
+ '4P')
+ class newstyleclass(object): pass
+ # Separate block for PyDictKeysObject with 8 keys and 5 entries
+ check(newstyleclass, s + calcsize("2nP2n0P") + 8 + 5*calcsize("n2P"))
+ # dict with shared keys
+ check(newstyleclass().__dict__, size('nQ2P') + 5*self.P)
+ o = newstyleclass()
+ o.a = o.b = o.c = o.d = o.e = o.f = o.g = o.h = 1
+ # Separate block for PyDictKeysObject with 16 keys and 10 entries
+ check(newstyleclass, s + calcsize("2nP2n0P") + 16 + 10*calcsize("n2P"))
+ # dict with shared keys
+ check(newstyleclass().__dict__, size('nQ2P') + 10*self.P)
+ # unicode
+ # each tuple contains a string and its expected character size
+ # don't put any static strings here, as they may contain
+ # wchar_t or UTF-8 representations
+ samples = ['1'*100, '\xff'*50,
+ '\u0100'*40, '\uffff'*100,
+ '\U00010000'*30, '\U0010ffff'*100]
+ asciifields = "nnbP"
+ compactfields = asciifields + "nPn"
+ unicodefields = compactfields + "P"
+ for s in samples:
+ maxchar = ord(max(s))
+ if maxchar < 128:
+ L = size(asciifields) + len(s) + 1
+ elif maxchar < 256:
+ L = size(compactfields) + len(s) + 1
+ elif maxchar < 65536:
+ L = size(compactfields) + 2*(len(s) + 1)
+ else:
+ L = size(compactfields) + 4*(len(s) + 1)
+ check(s, L)
+ # verify that the UTF-8 size is accounted for
+ s = chr(0x4000) # 4 bytes canonical representation
+ check(s, size(compactfields) + 4)
+ # compile() will trigger the generation of the UTF-8
+ # representation as a side effect
+ compile(s, "", "eval")
+ check(s, size(compactfields) + 4 + 4)
+ # TODO: add check that forces the presence of wchar_t representation
+ # TODO: add check that forces layout of unicodefields
+ # weakref
+ import weakref
+ check(weakref.ref(int), size('2Pn2P'))
+ # weakproxy
+ # XXX
+ # weakcallableproxy
+ check(weakref.proxy(int), size('2Pn2P'))
+
+ def check_slots(self, obj, base, extra):
+ expected = sys.getsizeof(base) + struct.calcsize(extra)
+ if gc.is_tracked(obj) and not gc.is_tracked(base):
+ expected += self.gc_headsize
+ self.assertEqual(sys.getsizeof(obj), expected)
+
+ def test_slots(self):
+ # check all subclassable types defined in Objects/ that allow
+ # non-empty __slots__
+ check = self.check_slots
+ class BA(bytearray):
+ __slots__ = 'a', 'b', 'c'
+ check(BA(), bytearray(), '3P')
+ class D(dict):
+ __slots__ = 'a', 'b', 'c'
+ check(D(x=[]), {'x': []}, '3P')
+ class L(list):
+ __slots__ = 'a', 'b', 'c'
+ check(L(), [], '3P')
+ class S(set):
+ __slots__ = 'a', 'b', 'c'
+ check(S(), set(), '3P')
+ class FS(frozenset):
+ __slots__ = 'a', 'b', 'c'
+ check(FS(), frozenset(), '3P')
+ from collections import OrderedDict
+ class OD(OrderedDict):
+ __slots__ = 'a', 'b', 'c'
+ check(OD(x=[]), OrderedDict(x=[]), '3P')
+
+ def test_pythontypes(self):
+ # check all types defined in Python/
+ size = test.support.calcobjsize
+ vsize = test.support.calcvobjsize
+ check = self.check_sizeof
+ # _ast.AST
+ import _ast
+ check(_ast.AST(), size('P'))
+ try:
+ raise TypeError
+ except TypeError:
+ tb = sys.exc_info()[2]
+ # traceback
+ if tb is not None:
+ check(tb, size('2P2i'))
+ # symtable entry
+ # XXX
+ # sys.flags
+ check(sys.flags, vsize('') + self.P * len(sys.flags))
+
+ def test_asyncgen_hooks(self):
+ old = sys.get_asyncgen_hooks()
+ self.assertIsNone(old.firstiter)
+ self.assertIsNone(old.finalizer)
+
+ firstiter = lambda *a: None
+ sys.set_asyncgen_hooks(firstiter=firstiter)
+ hooks = sys.get_asyncgen_hooks()
+ self.assertIs(hooks.firstiter, firstiter)
+ self.assertIs(hooks[0], firstiter)
+ self.assertIs(hooks.finalizer, None)
+ self.assertIs(hooks[1], None)
+
+ finalizer = lambda *a: None
+ sys.set_asyncgen_hooks(finalizer=finalizer)
+ hooks = sys.get_asyncgen_hooks()
+ self.assertIs(hooks.firstiter, firstiter)
+ self.assertIs(hooks[0], firstiter)
+ self.assertIs(hooks.finalizer, finalizer)
+ self.assertIs(hooks[1], finalizer)
+
+ sys.set_asyncgen_hooks(*old)
+ cur = sys.get_asyncgen_hooks()
+ self.assertIsNone(cur.firstiter)
+ self.assertIsNone(cur.finalizer)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 09782eb96..69aff1518 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1273,8 +1273,6 @@ class UnicodeTest(string_tests.CommonTest,
self.assertEqual("{!s}".format(n), 'N(data)')
self.assertRaises(TypeError, "{}".format, n)
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_format_map(self):
self.assertEqual(''.format_map({}), '')
self.assertEqual('a'.format_map({}), 'a')
@@ -2958,8 +2956,6 @@ class CAPITest(unittest.TestCase):
self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
class StringModuleTest(unittest.TestCase):
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_formatter_parser(self):
def parse(format):
return list(_string.formatter_parser(format))
@@ -2993,8 +2989,6 @@ class StringModuleTest(unittest.TestCase):
self.assertRaises(TypeError, _string.formatter_parser, 1)
- # TODO: RUSTPYTHON
- @unittest.expectedFailure
def test_formatter_field_name_split(self):
def split(name):
items = list(_string.formatter_field_name_split(name))
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
new file mode 100644
index 000000000..61234b764
--- /dev/null
+++ b/Lib/test/test_urlparse.py
@@ -0,0 +1,1261 @@
+import sys
+import unicodedata
+import unittest
+import urllib.parse
+
+RFC1808_BASE = "http://a/b/c/d;p?q#f"
+RFC2396_BASE = "http://a/b/c/d;p?q"
+RFC3986_BASE = 'http://a/b/c/d;p?q'
+SIMPLE_BASE = 'http://a/b/c/d'
+
+# Each parse_qsl testcase is a two-tuple that contains
+# a string with the query and a list with the expected result.
+
+parse_qsl_test_cases = [
+ ("", []),
+ ("&", []),
+ ("&&", []),
+ ("=", [('', '')]),
+ ("=a", [('', 'a')]),
+ ("a", [('a', '')]),
+ ("a=", [('a', '')]),
+ ("&a=b", [('a', 'b')]),
+ ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
+ ("a=1&a=2", [('a', '1'), ('a', '2')]),
+ (b"", []),
+ (b"&", []),
+ (b"&&", []),
+ (b"=", [(b'', b'')]),
+ (b"=a", [(b'', b'a')]),
+ (b"a", [(b'a', b'')]),
+ (b"a=", [(b'a', b'')]),
+ (b"&a=b", [(b'a', b'b')]),
+ (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
+ (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]),
+ (";", []),
+ (";;", []),
+ (";a=b", [('a', 'b')]),
+ ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]),
+ ("a=1;a=2", [('a', '1'), ('a', '2')]),
+ (b";", []),
+ (b";;", []),
+ (b";a=b", [(b'a', b'b')]),
+ (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
+ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
+]
+
+# Each parse_qs testcase is a two-tuple that contains
+# a string with the query and a dictionary with the expected result.
+
+parse_qs_test_cases = [
+ ("", {}),
+ ("&", {}),
+ ("&&", {}),
+ ("=", {'': ['']}),
+ ("=a", {'': ['a']}),
+ ("a", {'a': ['']}),
+ ("a=", {'a': ['']}),
+ ("&a=b", {'a': ['b']}),
+ ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
+ ("a=1&a=2", {'a': ['1', '2']}),
+ (b"", {}),
+ (b"&", {}),
+ (b"&&", {}),
+ (b"=", {b'': [b'']}),
+ (b"=a", {b'': [b'a']}),
+ (b"a", {b'a': [b'']}),
+ (b"a=", {b'a': [b'']}),
+ (b"&a=b", {b'a': [b'b']}),
+ (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
+ (b"a=1&a=2", {b'a': [b'1', b'2']}),
+ (";", {}),
+ (";;", {}),
+ (";a=b", {'a': ['b']}),
+ ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}),
+ ("a=1;a=2", {'a': ['1', '2']}),
+ (b";", {}),
+ (b";;", {}),
+ (b";a=b", {b'a': [b'b']}),
+ (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
+ (b"a=1;a=2", {b'a': [b'1', b'2']}),
+]
+
+class UrlParseTestCase(unittest.TestCase):
+
+ def checkRoundtrips(self, url, parsed, split):
+ result = urllib.parse.urlparse(url)
+ self.assertEqual(result, parsed)
+ t = (result.scheme, result.netloc, result.path,
+ result.params, result.query, result.fragment)
+ self.assertEqual(t, parsed)
+ # put it back together and it should be the same
+ result2 = urllib.parse.urlunparse(result)
+ self.assertEqual(result2, url)
+ self.assertEqual(result2, result.geturl())
+
+ # the result of geturl() is a fixpoint; we can always parse it
+ # again to get the same result:
+ result3 = urllib.parse.urlparse(result.geturl())
+ self.assertEqual(result3.geturl(), result.geturl())
+ self.assertEqual(result3, result)
+ self.assertEqual(result3.scheme, result.scheme)
+ self.assertEqual(result3.netloc, result.netloc)
+ self.assertEqual(result3.path, result.path)
+ self.assertEqual(result3.params, result.params)
+ self.assertEqual(result3.query, result.query)
+ self.assertEqual(result3.fragment, result.fragment)
+ self.assertEqual(result3.username, result.username)
+ self.assertEqual(result3.password, result.password)
+ self.assertEqual(result3.hostname, result.hostname)
+ self.assertEqual(result3.port, result.port)
+
+ # check the roundtrip using urlsplit() as well
+ result = urllib.parse.urlsplit(url)
+ self.assertEqual(result, split)
+ t = (result.scheme, result.netloc, result.path,
+ result.query, result.fragment)
+ self.assertEqual(t, split)
+ result2 = urllib.parse.urlunsplit(result)
+ self.assertEqual(result2, url)
+ self.assertEqual(result2, result.geturl())
+
+ # check the fixpoint property of re-parsing the result of geturl()
+ result3 = urllib.parse.urlsplit(result.geturl())
+ self.assertEqual(result3.geturl(), result.geturl())
+ self.assertEqual(result3, result)
+ self.assertEqual(result3.scheme, result.scheme)
+ self.assertEqual(result3.netloc, result.netloc)
+ self.assertEqual(result3.path, result.path)
+ self.assertEqual(result3.query, result.query)
+ self.assertEqual(result3.fragment, result.fragment)
+ self.assertEqual(result3.username, result.username)
+ self.assertEqual(result3.password, result.password)
+ self.assertEqual(result3.hostname, result.hostname)
+ self.assertEqual(result3.port, result.port)
+
+ def test_qsl(self):
+ for orig, expect in parse_qsl_test_cases:
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
+ expect_without_blanks = [v for v in expect if len(v[1])]
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=False)
+ self.assertEqual(result, expect_without_blanks,
+ "Error parsing %r" % orig)
+
+ def test_qs(self):
+ for orig, expect in parse_qs_test_cases:
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
+ expect_without_blanks = {v: expect[v]
+ for v in expect if len(expect[v][0])}
+ result = urllib.parse.parse_qs(orig, keep_blank_values=False)
+ self.assertEqual(result, expect_without_blanks,
+ "Error parsing %r" % orig)
+
+ def test_roundtrips(self):
+ str_cases = [
+ ('file:///tmp/junk.txt',
+ ('file', '', '/tmp/junk.txt', '', '', ''),
+ ('file', '', '/tmp/junk.txt', '', '')),
+ ('imap://mail.python.org/mbox1',
+ ('imap', 'mail.python.org', '/mbox1', '', '', ''),
+ ('imap', 'mail.python.org', '/mbox1', '', '')),
+ ('mms://wms.sys.hinet.net/cts/Drama/09006251100.asf',
+ ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf',
+ '', '', ''),
+ ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf',
+ '', '')),
+ ('nfs://server/path/to/file.txt',
+ ('nfs', 'server', '/path/to/file.txt', '', '', ''),
+ ('nfs', 'server', '/path/to/file.txt', '', '')),
+ ('svn+ssh://svn.zope.org/repos/main/ZConfig/trunk/',
+ ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/',
+ '', '', ''),
+ ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/',
+ '', '')),
+ ('git+ssh://git@github.com/user/project.git',
+ ('git+ssh', 'git@github.com','/user/project.git',
+ '','',''),
+ ('git+ssh', 'git@github.com','/user/project.git',
+ '', '')),
+ ]
+ def _encode(t):
+ return (t[0].encode('ascii'),
+ tuple(x.encode('ascii') for x in t[1]),
+ tuple(x.encode('ascii') for x in t[2]))
+ bytes_cases = [_encode(x) for x in str_cases]
+ for url, parsed, split in str_cases + bytes_cases:
+ self.checkRoundtrips(url, parsed, split)
+
+ def test_http_roundtrips(self):
+ # urllib.parse.urlsplit treats 'http:' as an optimized special case,
+ # so we test both 'http:' and 'https:' in all the following.
+ # Three cheers for white box knowledge!
+ str_cases = [
+ ('://www.python.org',
+ ('www.python.org', '', '', '', ''),
+ ('www.python.org', '', '', '')),
+ ('://www.python.org#abc',
+ ('www.python.org', '', '', '', 'abc'),
+ ('www.python.org', '', '', 'abc')),
+ ('://www.python.org?q=abc',
+ ('www.python.org', '', '', 'q=abc', ''),
+ ('www.python.org', '', 'q=abc', '')),
+ ('://www.python.org/#abc',
+ ('www.python.org', '/', '', '', 'abc'),
+ ('www.python.org', '/', '', 'abc')),
+ ('://a/b/c/d;p?q#f',
+ ('a', '/b/c/d', 'p', 'q', 'f'),
+ ('a', '/b/c/d;p', 'q', 'f')),
+ ]
+ def _encode(t):
+ return (t[0].encode('ascii'),
+ tuple(x.encode('ascii') for x in t[1]),
+ tuple(x.encode('ascii') for x in t[2]))
+ bytes_cases = [_encode(x) for x in str_cases]
+ str_schemes = ('http', 'https')
+ bytes_schemes = (b'http', b'https')
+ str_tests = str_schemes, str_cases
+ bytes_tests = bytes_schemes, bytes_cases
+ for schemes, test_cases in (str_tests, bytes_tests):
+ for scheme in schemes:
+ for url, parsed, split in test_cases:
+ url = scheme + url
+ parsed = (scheme,) + parsed
+ split = (scheme,) + split
+ self.checkRoundtrips(url, parsed, split)
+
+ def checkJoin(self, base, relurl, expected):
+ str_components = (base, relurl, expected)
+ self.assertEqual(urllib.parse.urljoin(base, relurl), expected)
+ bytes_components = baseb, relurlb, expectedb = [
+ x.encode('ascii') for x in str_components]
+ self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb)
+
+ def test_unparse_parse(self):
+ str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]
+ bytes_cases = [x.encode('ascii') for x in str_cases]
+ for u in str_cases + bytes_cases:
+ self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u)
+ self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u)
+
+ def test_RFC1808(self):
+ # "normal" cases from RFC 1808:
+ self.checkJoin(RFC1808_BASE, 'g:h', 'g:h')
+ self.checkJoin(RFC1808_BASE, 'g', 'http://a/b/c/g')
+ self.checkJoin(RFC1808_BASE, './g', 'http://a/b/c/g')
+ self.checkJoin(RFC1808_BASE, 'g/', 'http://a/b/c/g/')
+ self.checkJoin(RFC1808_BASE, '/g', 'http://a/g')
+ self.checkJoin(RFC1808_BASE, '//g', 'http://g')
+ self.checkJoin(RFC1808_BASE, 'g?y', 'http://a/b/c/g?y')
+ self.checkJoin(RFC1808_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x')
+ self.checkJoin(RFC1808_BASE, '#s', 'http://a/b/c/d;p?q#s')
+ self.checkJoin(RFC1808_BASE, 'g#s', 'http://a/b/c/g#s')
+ self.checkJoin(RFC1808_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x')
+ self.checkJoin(RFC1808_BASE, 'g?y#s', 'http://a/b/c/g?y#s')
+ self.checkJoin(RFC1808_BASE, 'g;x', 'http://a/b/c/g;x')
+ self.checkJoin(RFC1808_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s')
+ self.checkJoin(RFC1808_BASE, '.', 'http://a/b/c/')
+ self.checkJoin(RFC1808_BASE, './', 'http://a/b/c/')
+ self.checkJoin(RFC1808_BASE, '..', 'http://a/b/')
+ self.checkJoin(RFC1808_BASE, '../', 'http://a/b/')
+ self.checkJoin(RFC1808_BASE, '../g', 'http://a/b/g')
+ self.checkJoin(RFC1808_BASE, '../..', 'http://a/')
+ self.checkJoin(RFC1808_BASE, '../../', 'http://a/')
+ self.checkJoin(RFC1808_BASE, '../../g', 'http://a/g')
+
+ # "abnormal" cases from RFC 1808:
+ self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f')
+ self.checkJoin(RFC1808_BASE, 'g.', 'http://a/b/c/g.')
+ self.checkJoin(RFC1808_BASE, '.g', 'http://a/b/c/.g')
+ self.checkJoin(RFC1808_BASE, 'g..', 'http://a/b/c/g..')
+ self.checkJoin(RFC1808_BASE, '..g', 'http://a/b/c/..g')
+ self.checkJoin(RFC1808_BASE, './../g', 'http://a/b/g')
+ self.checkJoin(RFC1808_BASE, './g/.', 'http://a/b/c/g/')
+ self.checkJoin(RFC1808_BASE, 'g/./h', 'http://a/b/c/g/h')
+ self.checkJoin(RFC1808_BASE, 'g/../h', 'http://a/b/c/h')
+
+ # RFC 1808 and RFC 1630 disagree on these (according to RFC 1808),
+ # so we'll not actually run these tests (which expect 1808 behavior).
+ #self.checkJoin(RFC1808_BASE, 'http:g', 'http:g')
+ #self.checkJoin(RFC1808_BASE, 'http:', 'http:')
+
+ # XXX: The following tests are no longer compatible with RFC3986
+ # self.checkJoin(RFC1808_BASE, '../../../g', 'http://a/../g')
+ # self.checkJoin(RFC1808_BASE, '../../../../g', 'http://a/../../g')
+ # self.checkJoin(RFC1808_BASE, '/./g', 'http://a/./g')
+ # self.checkJoin(RFC1808_BASE, '/../g', 'http://a/../g')
+
+
+ def test_RFC2368(self):
+ # Issue 11467: path that starts with a number is not parsed correctly
+ self.assertEqual(urllib.parse.urlparse('mailto:1337@example.org'),
+ ('mailto', '', '1337@example.org', '', '', ''))
+
+ def test_RFC2396(self):
+ # cases from RFC 2396
+
+ self.checkJoin(RFC2396_BASE, 'g:h', 'g:h')
+ self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g')
+ self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g')
+ self.checkJoin(RFC2396_BASE, 'g/', 'http://a/b/c/g/')
+ self.checkJoin(RFC2396_BASE, '/g', 'http://a/g')
+ self.checkJoin(RFC2396_BASE, '//g', 'http://g')
+ self.checkJoin(RFC2396_BASE, 'g?y', 'http://a/b/c/g?y')
+ self.checkJoin(RFC2396_BASE, '#s', 'http://a/b/c/d;p?q#s')
+ self.checkJoin(RFC2396_BASE, 'g#s', 'http://a/b/c/g#s')
+ self.checkJoin(RFC2396_BASE, 'g?y#s', 'http://a/b/c/g?y#s')
+ self.checkJoin(RFC2396_BASE, 'g;x', 'http://a/b/c/g;x')
+ self.checkJoin(RFC2396_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s')
+ self.checkJoin(RFC2396_BASE, '.', 'http://a/b/c/')
+ self.checkJoin(RFC2396_BASE, './', 'http://a/b/c/')
+ self.checkJoin(RFC2396_BASE, '..', 'http://a/b/')
+ self.checkJoin(RFC2396_BASE, '../', 'http://a/b/')
+ self.checkJoin(RFC2396_BASE, '../g', 'http://a/b/g')
+ self.checkJoin(RFC2396_BASE, '../..', 'http://a/')
+ self.checkJoin(RFC2396_BASE, '../../', 'http://a/')
+ self.checkJoin(RFC2396_BASE, '../../g', 'http://a/g')
+ self.checkJoin(RFC2396_BASE, '', RFC2396_BASE)
+ self.checkJoin(RFC2396_BASE, 'g.', 'http://a/b/c/g.')
+ self.checkJoin(RFC2396_BASE, '.g', 'http://a/b/c/.g')
+ self.checkJoin(RFC2396_BASE, 'g..', 'http://a/b/c/g..')
+ self.checkJoin(RFC2396_BASE, '..g', 'http://a/b/c/..g')
+ self.checkJoin(RFC2396_BASE, './../g', 'http://a/b/g')
+ self.checkJoin(RFC2396_BASE, './g/.', 'http://a/b/c/g/')
+ self.checkJoin(RFC2396_BASE, 'g/./h', 'http://a/b/c/g/h')
+ self.checkJoin(RFC2396_BASE, 'g/../h', 'http://a/b/c/h')
+ self.checkJoin(RFC2396_BASE, 'g;x=1/./y', 'http://a/b/c/g;x=1/y')
+ self.checkJoin(RFC2396_BASE, 'g;x=1/../y', 'http://a/b/c/y')
+ self.checkJoin(RFC2396_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x')
+ self.checkJoin(RFC2396_BASE, 'g?y/../x', 'http://a/b/c/g?y/../x')
+ self.checkJoin(RFC2396_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x')
+ self.checkJoin(RFC2396_BASE, 'g#s/../x', 'http://a/b/c/g#s/../x')
+
+ # XXX: The following tests are no longer compatible with RFC3986
+ # self.checkJoin(RFC2396_BASE, '../../../g', 'http://a/../g')
+ # self.checkJoin(RFC2396_BASE, '../../../../g', 'http://a/../../g')
+ # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g')
+ # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g')
+
+ def test_RFC3986(self):
+ self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y')
+ self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x')
+ self.checkJoin(RFC3986_BASE, 'g:h','g:h')
+ self.checkJoin(RFC3986_BASE, 'g','http://a/b/c/g')
+ self.checkJoin(RFC3986_BASE, './g','http://a/b/c/g')
+ self.checkJoin(RFC3986_BASE, 'g/','http://a/b/c/g/')
+ self.checkJoin(RFC3986_BASE, '/g','http://a/g')
+ self.checkJoin(RFC3986_BASE, '//g','http://g')
+ self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y')
+ self.checkJoin(RFC3986_BASE, 'g?y','http://a/b/c/g?y')
+ self.checkJoin(RFC3986_BASE, '#s','http://a/b/c/d;p?q#s')
+ self.checkJoin(RFC3986_BASE, 'g#s','http://a/b/c/g#s')
+ self.checkJoin(RFC3986_BASE, 'g?y#s','http://a/b/c/g?y#s')
+ self.checkJoin(RFC3986_BASE, ';x','http://a/b/c/;x')
+ self.checkJoin(RFC3986_BASE, 'g;x','http://a/b/c/g;x')
+ self.checkJoin(RFC3986_BASE, 'g;x?y#s','http://a/b/c/g;x?y#s')
+ self.checkJoin(RFC3986_BASE, '','http://a/b/c/d;p?q')
+ self.checkJoin(RFC3986_BASE, '.','http://a/b/c/')
+ self.checkJoin(RFC3986_BASE, './','http://a/b/c/')
+ self.checkJoin(RFC3986_BASE, '..','http://a/b/')
+ self.checkJoin(RFC3986_BASE, '../','http://a/b/')
+ self.checkJoin(RFC3986_BASE, '../g','http://a/b/g')
+ self.checkJoin(RFC3986_BASE, '../..','http://a/')
+ self.checkJoin(RFC3986_BASE, '../../','http://a/')
+ self.checkJoin(RFC3986_BASE, '../../g','http://a/g')
+ self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g')
+
+ # Abnormal Examples
+
+ # The 'abnormal scenarios' are incompatible with RFC2986 parsing
+ # Tests are here for reference.
+
+ self.checkJoin(RFC3986_BASE, '../../../g','http://a/g')
+ self.checkJoin(RFC3986_BASE, '../../../../g','http://a/g')
+ self.checkJoin(RFC3986_BASE, '/./g','http://a/g')
+ self.checkJoin(RFC3986_BASE, '/../g','http://a/g')
+ self.checkJoin(RFC3986_BASE, 'g.','http://a/b/c/g.')
+ self.checkJoin(RFC3986_BASE, '.g','http://a/b/c/.g')
+ self.checkJoin(RFC3986_BASE, 'g..','http://a/b/c/g..')
+ self.checkJoin(RFC3986_BASE, '..g','http://a/b/c/..g')
+ self.checkJoin(RFC3986_BASE, './../g','http://a/b/g')
+ self.checkJoin(RFC3986_BASE, './g/.','http://a/b/c/g/')
+ self.checkJoin(RFC3986_BASE, 'g/./h','http://a/b/c/g/h')
+ self.checkJoin(RFC3986_BASE, 'g/../h','http://a/b/c/h')
+ self.checkJoin(RFC3986_BASE, 'g;x=1/./y','http://a/b/c/g;x=1/y')
+ self.checkJoin(RFC3986_BASE, 'g;x=1/../y','http://a/b/c/y')
+ self.checkJoin(RFC3986_BASE, 'g?y/./x','http://a/b/c/g?y/./x')
+ self.checkJoin(RFC3986_BASE, 'g?y/../x','http://a/b/c/g?y/../x')
+ self.checkJoin(RFC3986_BASE, 'g#s/./x','http://a/b/c/g#s/./x')
+ self.checkJoin(RFC3986_BASE, 'g#s/../x','http://a/b/c/g#s/../x')
+ #self.checkJoin(RFC3986_BASE, 'http:g','http:g') # strict parser
+ self.checkJoin(RFC3986_BASE, 'http:g','http://a/b/c/g') #relaxed parser
+
+ # Test for issue9721
+ self.checkJoin('http://a/b/c/de', ';x','http://a/b/c/;x')
+
+ def test_urljoins(self):
+ self.checkJoin(SIMPLE_BASE, 'g:h','g:h')
+ self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g')
+ self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d')
+ self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g')
+ self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g')
+ self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/')
+ self.checkJoin(SIMPLE_BASE, '/g','http://a/g')
+ self.checkJoin(SIMPLE_BASE, '//g','http://g')
+ self.checkJoin(SIMPLE_BASE, '?y','http://a/b/c/d?y')
+ self.checkJoin(SIMPLE_BASE, 'g?y','http://a/b/c/g?y')
+ self.checkJoin(SIMPLE_BASE, 'g?y/./x','http://a/b/c/g?y/./x')
+ self.checkJoin(SIMPLE_BASE, '.','http://a/b/c/')
+ self.checkJoin(SIMPLE_BASE, './','http://a/b/c/')
+ self.checkJoin(SIMPLE_BASE, '..','http://a/b/')
+ self.checkJoin(SIMPLE_BASE, '../','http://a/b/')
+ self.checkJoin(SIMPLE_BASE, '../g','http://a/b/g')
+ self.checkJoin(SIMPLE_BASE, '../..','http://a/')
+ self.checkJoin(SIMPLE_BASE, '../../g','http://a/g')
+ self.checkJoin(SIMPLE_BASE, './../g','http://a/b/g')
+ self.checkJoin(SIMPLE_BASE, './g/.','http://a/b/c/g/')
+ self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h')
+ self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h')
+ self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g')
+ self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d')
+ self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y')
+ self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y')
+ self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x')
+ self.checkJoin('http:///', '..','http:///')
+ self.checkJoin('', 'http://a/b/c/g?y/./x','http://a/b/c/g?y/./x')
+ self.checkJoin('', 'http://a/./g', 'http://a/./g')
+ self.checkJoin('svn://pathtorepo/dir1', 'dir2', 'svn://pathtorepo/dir2')
+ self.checkJoin('svn+ssh://pathtorepo/dir1', 'dir2', 'svn+ssh://pathtorepo/dir2')
+ self.checkJoin('ws://a/b','g','ws://a/g')
+ self.checkJoin('wss://a/b','g','wss://a/g')
+
+ # XXX: The following tests are no longer compatible with RFC3986
+ # self.checkJoin(SIMPLE_BASE, '../../../g','http://a/../g')
+ # self.checkJoin(SIMPLE_BASE, '/./g','http://a/./g')
+
+ # test for issue22118 duplicate slashes
+ self.checkJoin(SIMPLE_BASE + '/', 'foo', SIMPLE_BASE + '/foo')
+
+ # Non-RFC-defined tests, covering variations of base and trailing
+ # slashes
+ self.checkJoin('http://a/b/c/d/e/', '../../f/g/', 'http://a/b/c/f/g/')
+ self.checkJoin('http://a/b/c/d/e', '../../f/g/', 'http://a/b/f/g/')
+ self.checkJoin('http://a/b/c/d/e/', '/../../f/g/', 'http://a/f/g/')
+ self.checkJoin('http://a/b/c/d/e', '/../../f/g/', 'http://a/f/g/')
+ self.checkJoin('http://a/b/c/d/e/', '../../f/g', 'http://a/b/c/f/g')
+ self.checkJoin('http://a/b/', '../../f/g/', 'http://a/f/g/')
+
+ # issue 23703: don't duplicate filename
+ self.checkJoin('a', 'b', 'b')
+
+ def test_RFC2732(self):
+ str_cases = [
+ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432),
+ ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432),
+ ('http://[::1]:5432/foo/', '::1', 5432),
+ ('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432),
+ ('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432),
+ ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/',
+ 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432),
+ ('http://[::12.34.56.78]:5432/foo/', '::12.34.56.78', 5432),
+ ('http://[::ffff:12.34.56.78]:5432/foo/',
+ '::ffff:12.34.56.78', 5432),
+ ('http://Test.python.org/foo/', 'test.python.org', None),
+ ('http://12.34.56.78/foo/', '12.34.56.78', None),
+ ('http://[::1]/foo/', '::1', None),
+ ('http://[dead:beef::1]/foo/', 'dead:beef::1', None),
+ ('http://[dead:beef::]/foo/', 'dead:beef::', None),
+ ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/',
+ 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
+ ('http://[::12.34.56.78]/foo/', '::12.34.56.78', None),
+ ('http://[::ffff:12.34.56.78]/foo/',
+ '::ffff:12.34.56.78', None),
+ ('http://Test.python.org:/foo/', 'test.python.org', None),
+ ('http://12.34.56.78:/foo/', '12.34.56.78', None),
+ ('http://[::1]:/foo/', '::1', None),
+ ('http://[dead:beef::1]:/foo/', 'dead:beef::1', None),
+ ('http://[dead:beef::]:/foo/', 'dead:beef::', None),
+ ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:/foo/',
+ 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
+ ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None),
+ ('http://[::ffff:12.34.56.78]:/foo/',
+ '::ffff:12.34.56.78', None),
+ ]
+ def _encode(t):
+ return t[0].encode('ascii'), t[1].encode('ascii'), t[2]
+ bytes_cases = [_encode(x) for x in str_cases]
+ for url, hostname, port in str_cases + bytes_cases:
+ urlparsed = urllib.parse.urlparse(url)
+ self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port))
+
+ str_cases = [
+ 'http://::12.34.56.78]/',
+ 'http://[::1/foo/',
+ 'ftp://[::1/foo/bad]/bad',
+ 'http://[::1/foo/bad]/bad',
+ 'http://[::ffff:12.34.56.78']
+ bytes_cases = [x.encode('ascii') for x in str_cases]
+ for invalid_url in str_cases + bytes_cases:
+ self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url)
+
+ def test_urldefrag(self):
+ str_cases = [
+ ('http://python.org#frag', 'http://python.org', 'frag'),
+ ('http://python.org', 'http://python.org', ''),
+ ('http://python.org/#frag', 'http://python.org/', 'frag'),
+ ('http://python.org/', 'http://python.org/', ''),
+ ('http://python.org/?q#frag', 'http://python.org/?q', 'frag'),
+ ('http://python.org/?q', 'http://python.org/?q', ''),
+ ('http://python.org/p#frag', 'http://python.org/p', 'frag'),
+ ('http://python.org/p?q', 'http://python.org/p?q', ''),
+ (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'),
+ (RFC2396_BASE, 'http://a/b/c/d;p?q', ''),
+ ]
+ def _encode(t):
+ return type(t)(x.encode('ascii') for x in t)
+ bytes_cases = [_encode(x) for x in str_cases]
+ for url, defrag, frag in str_cases + bytes_cases:
+ result = urllib.parse.urldefrag(url)
+ self.assertEqual(result.geturl(), url)
+ self.assertEqual(result, (defrag, frag))
+ self.assertEqual(result.url, defrag)
+ self.assertEqual(result.fragment, frag)
+
+ def test_urlsplit_scoped_IPv6(self):
+ p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234')
+ self.assertEqual(p.hostname, "fe80::822a:a8ff:fe49:470c%tESt")
+ self.assertEqual(p.netloc, '[FE80::822a:a8ff:fe49:470c%tESt]:1234')
+
+ p = urllib.parse.urlsplit(b'http://[FE80::822a:a8ff:fe49:470c%tESt]:1234')
+ self.assertEqual(p.hostname, b"fe80::822a:a8ff:fe49:470c%tESt")
+ self.assertEqual(p.netloc, b'[FE80::822a:a8ff:fe49:470c%tESt]:1234')
+
+ def test_urlsplit_attributes(self):
+ url = "HTTP://WWW.PYTHON.ORG/doc/#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, "http")
+ self.assertEqual(p.netloc, "WWW.PYTHON.ORG")
+ self.assertEqual(p.path, "/doc/")
+ self.assertEqual(p.query, "")
+ self.assertEqual(p.fragment, "frag")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, "www.python.org")
+ self.assertEqual(p.port, None)
+ # geturl() won't return exactly the original URL in this case
+ # since the scheme is always case-normalized
+ # We handle this by ignoring the first 4 characters of the URL
+ self.assertEqual(p.geturl()[4:], url[4:])
+
+ url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, "http")
+ self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
+ self.assertEqual(p.path, "/doc/")
+ self.assertEqual(p.query, "query=yes")
+ self.assertEqual(p.fragment, "frag")
+ self.assertEqual(p.username, "User")
+ self.assertEqual(p.password, "Pass")
+ self.assertEqual(p.hostname, "www.python.org")
+ self.assertEqual(p.port, 80)
+ self.assertEqual(p.geturl(), url)
+
+ # Addressing issue1698, which suggests Username can contain
+ # "@" characters. Though not RFC compliant, many ftp sites allow
+ # and request email addresses as usernames.
+
+ url = "http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, "http")
+ self.assertEqual(p.netloc, "User@example.com:Pass@www.python.org:080")
+ self.assertEqual(p.path, "/doc/")
+ self.assertEqual(p.query, "query=yes")
+ self.assertEqual(p.fragment, "frag")
+ self.assertEqual(p.username, "User@example.com")
+ self.assertEqual(p.password, "Pass")
+ self.assertEqual(p.hostname, "www.python.org")
+ self.assertEqual(p.port, 80)
+ self.assertEqual(p.geturl(), url)
+
+ # And check them all again, only with bytes this time
+ url = b"HTTP://WWW.PYTHON.ORG/doc/#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, b"http")
+ self.assertEqual(p.netloc, b"WWW.PYTHON.ORG")
+ self.assertEqual(p.path, b"/doc/")
+ self.assertEqual(p.query, b"")
+ self.assertEqual(p.fragment, b"frag")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, b"www.python.org")
+ self.assertEqual(p.port, None)
+ self.assertEqual(p.geturl()[4:], url[4:])
+
+ url = b"http://User:Pass@www.python.org:080/doc/?query=yes#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, b"http")
+ self.assertEqual(p.netloc, b"User:Pass@www.python.org:080")
+ self.assertEqual(p.path, b"/doc/")
+ self.assertEqual(p.query, b"query=yes")
+ self.assertEqual(p.fragment, b"frag")
+ self.assertEqual(p.username, b"User")
+ self.assertEqual(p.password, b"Pass")
+ self.assertEqual(p.hostname, b"www.python.org")
+ self.assertEqual(p.port, 80)
+ self.assertEqual(p.geturl(), url)
+
+ url = b"http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag"
+ p = urllib.parse.urlsplit(url)
+ self.assertEqual(p.scheme, b"http")
+ self.assertEqual(p.netloc, b"User@example.com:Pass@www.python.org:080")
+ self.assertEqual(p.path, b"/doc/")
+ self.assertEqual(p.query, b"query=yes")
+ self.assertEqual(p.fragment, b"frag")
+ self.assertEqual(p.username, b"User@example.com")
+ self.assertEqual(p.password, b"Pass")
+ self.assertEqual(p.hostname, b"www.python.org")
+ self.assertEqual(p.port, 80)
+ self.assertEqual(p.geturl(), url)
+
+ # Verify an illegal port raises ValueError
+ url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag"
+ p = urllib.parse.urlsplit(url)
+ with self.assertRaisesRegex(ValueError, "out of range"):
+ p.port
+
+ def test_attributes_bad_port(self):
+ """Check handling of invalid ports."""
+ for bytes in (False, True):
+ for parse in (urllib.parse.urlsplit, urllib.parse.urlparse):
+ for port in ("foo", "1.5", "-1", "0x10"):
+ with self.subTest(bytes=bytes, parse=parse, port=port):
+ netloc = "www.example.net:" + port
+ url = "http://" + netloc
+ if bytes:
+ netloc = netloc.encode("ascii")
+ url = url.encode("ascii")
+ p = parse(url)
+ self.assertEqual(p.netloc, netloc)
+ with self.assertRaises(ValueError):
+ p.port
+
+ def test_attributes_without_netloc(self):
+ # This example is straight from RFC 3261. It looks like it
+ # should allow the username, hostname, and port to be filled
+ # in, but doesn't. Since it's a URI and doesn't use the
+ # scheme://netloc syntax, the netloc and related attributes
+ # should be left empty.
+ uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
+ p = urllib.parse.urlsplit(uri)
+ self.assertEqual(p.netloc, "")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, None)
+ self.assertEqual(p.port, None)
+ self.assertEqual(p.geturl(), uri)
+
+ p = urllib.parse.urlparse(uri)
+ self.assertEqual(p.netloc, "")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, None)
+ self.assertEqual(p.port, None)
+ self.assertEqual(p.geturl(), uri)
+
+ # You guessed it, repeating the test with bytes input
+ uri = b"sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
+ p = urllib.parse.urlsplit(uri)
+ self.assertEqual(p.netloc, b"")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, None)
+ self.assertEqual(p.port, None)
+ self.assertEqual(p.geturl(), uri)
+
+ p = urllib.parse.urlparse(uri)
+ self.assertEqual(p.netloc, b"")
+ self.assertEqual(p.username, None)
+ self.assertEqual(p.password, None)
+ self.assertEqual(p.hostname, None)
+ self.assertEqual(p.port, None)
+ self.assertEqual(p.geturl(), uri)
+
+ def test_noslash(self):
+ # Issue 1637: http://foo.com?query is legal
+ self.assertEqual(urllib.parse.urlparse("http://example.com?blahblah=/foo"),
+ ('http', 'example.com', '', '', 'blahblah=/foo', ''))
+ self.assertEqual(urllib.parse.urlparse(b"http://example.com?blahblah=/foo"),
+ (b'http', b'example.com', b'', b'', b'blahblah=/foo', b''))
+
+ def test_withoutscheme(self):
+ # Test urlparse without scheme
+ # Issue 754016: urlparse goes wrong with IP:port without scheme
+ # RFC 1808 specifies that netloc should start with //, urlparse expects
+ # the same, otherwise it classifies the portion of url as path.
+ self.assertEqual(urllib.parse.urlparse("path"),
+ ('','','path','','',''))
+ self.assertEqual(urllib.parse.urlparse("//www.python.org:80"),
+ ('','www.python.org:80','','','',''))
+ self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
+ ('http','www.python.org:80','','','',''))
+ # Repeat for bytes input
+ self.assertEqual(urllib.parse.urlparse(b"path"),
+ (b'',b'',b'path',b'',b'',b''))
+ self.assertEqual(urllib.parse.urlparse(b"//www.python.org:80"),
+ (b'',b'www.python.org:80',b'',b'',b'',b''))
+ self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
+ (b'http',b'www.python.org:80',b'',b'',b'',b''))
+
+ def test_portseparator(self):
+ # Issue 754016 makes changes for port separator ':' from scheme separator
+ self.assertEqual(urllib.parse.urlparse("path:80"),
+ ('','','path:80','','',''))
+ self.assertEqual(urllib.parse.urlparse("http:"),('http','','','','',''))
+ self.assertEqual(urllib.parse.urlparse("https:"),('https','','','','',''))
+ self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
+ ('http','www.python.org:80','','','',''))
+ # As usual, need to check bytes input as well
+ self.assertEqual(urllib.parse.urlparse(b"path:80"),
+ (b'',b'',b'path:80',b'',b'',b''))
+ self.assertEqual(urllib.parse.urlparse(b"http:"),(b'http',b'',b'',b'',b'',b''))
+ self.assertEqual(urllib.parse.urlparse(b"https:"),(b'https',b'',b'',b'',b'',b''))
+ self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
+ (b'http',b'www.python.org:80',b'',b'',b'',b''))
+
+ def test_usingsys(self):
+ # Issue 3314: sys module is used in the error
+ self.assertRaises(TypeError, urllib.parse.urlencode, "foo")
+
+ def test_anyscheme(self):
+ # Issue 7904: s3://foo.com/stuff has netloc "foo.com".
+ self.assertEqual(urllib.parse.urlparse("s3://foo.com/stuff"),
+ ('s3', 'foo.com', '/stuff', '', '', ''))
+ self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff"),
+ ('x-newscheme', 'foo.com', '/stuff', '', '', ''))
+ self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query#fragment"),
+ ('x-newscheme', 'foo.com', '/stuff', '', 'query', 'fragment'))
+ self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query"),
+ ('x-newscheme', 'foo.com', '/stuff', '', 'query', ''))
+
+ # And for bytes...
+ self.assertEqual(urllib.parse.urlparse(b"s3://foo.com/stuff"),
+ (b's3', b'foo.com', b'/stuff', b'', b'', b''))
+ self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff"),
+ (b'x-newscheme', b'foo.com', b'/stuff', b'', b'', b''))
+ self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query#fragment"),
+ (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'fragment'))
+ self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"),
+ (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b''))
+
+ def test_default_scheme(self):
+ # Exercise the scheme parameter of urlparse() and urlsplit()
+ for func in (urllib.parse.urlparse, urllib.parse.urlsplit):
+ with self.subTest(function=func):
+ result = func("http://example.net/", "ftp")
+ self.assertEqual(result.scheme, "http")
+ result = func(b"http://example.net/", b"ftp")
+ self.assertEqual(result.scheme, b"http")
+ self.assertEqual(func("path", "ftp").scheme, "ftp")
+ self.assertEqual(func("path", scheme="ftp").scheme, "ftp")
+ self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp")
+ self.assertEqual(func("path").scheme, "")
+ self.assertEqual(func(b"path").scheme, b"")
+ self.assertEqual(func(b"path", "").scheme, b"")
+
+ def test_parse_fragments(self):
+ # Exercise the allow_fragments parameter of urlparse() and urlsplit()
+ tests = (
+ ("http:#frag", "path", "frag"),
+ ("//example.net#frag", "path", "frag"),
+ ("index.html#frag", "path", "frag"),
+ (";a=b#frag", "params", "frag"),
+ ("?a=b#frag", "query", "frag"),
+ ("#frag", "path", "frag"),
+ ("abc#@frag", "path", "@frag"),
+ ("//abc#@frag", "path", "@frag"),
+ ("//abc:80#@frag", "path", "@frag"),
+ ("//abc#@frag:80", "path", "@frag:80"),
+ )
+ for url, attr, expected_frag in tests:
+ for func in (urllib.parse.urlparse, urllib.parse.urlsplit):
+ if attr == "params" and func is urllib.parse.urlsplit:
+ attr = "path"
+ with self.subTest(url=url, function=func):
+ result = func(url, allow_fragments=False)
+ self.assertEqual(result.fragment, "")
+ self.assertTrue(
+ getattr(result, attr).endswith("#" + expected_frag))
+ self.assertEqual(func(url, "", False).fragment, "")
+
+ result = func(url, allow_fragments=True)
+ self.assertEqual(result.fragment, expected_frag)
+ self.assertFalse(
+ getattr(result, attr).endswith(expected_frag))
+ self.assertEqual(func(url, "", True).fragment,
+ expected_frag)
+ self.assertEqual(func(url).fragment, expected_frag)
+
+ def test_mixed_types_rejected(self):
+ # Several functions that process either strings or ASCII encoded bytes
+ # accept multiple arguments. Check they reject mixed type input
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlparse("www.python.org", b"http")
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlparse(b"www.python.org", "http")
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlsplit("www.python.org", b"http")
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlsplit(b"www.python.org", "http")
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlunparse(( b"http", "www.python.org","","","",""))
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlunparse(("http", b"www.python.org","","","",""))
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlunsplit((b"http", "www.python.org","","",""))
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urlunsplit(("http", b"www.python.org","","",""))
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urljoin("http://python.org", b"http://python.org")
+ with self.assertRaisesRegex(TypeError, "Cannot mix str"):
+ urllib.parse.urljoin(b"http://python.org", "http://python.org")
+
+ def _check_result_type(self, str_type):
+ num_args = len(str_type._fields)
+ bytes_type = str_type._encoded_counterpart
+ self.assertIs(bytes_type._decoded_counterpart, str_type)
+ str_args = ('',) * num_args
+ bytes_args = (b'',) * num_args
+ str_result = str_type(*str_args)
+ bytes_result = bytes_type(*bytes_args)
+ encoding = 'ascii'
+ errors = 'strict'
+ self.assertEqual(str_result, str_args)
+ self.assertEqual(bytes_result.decode(), str_args)
+ self.assertEqual(bytes_result.decode(), str_result)
+ self.assertEqual(bytes_result.decode(encoding), str_args)
+ self.assertEqual(bytes_result.decode(encoding), str_result)
+ self.assertEqual(bytes_result.decode(encoding, errors), str_args)
+ self.assertEqual(bytes_result.decode(encoding, errors), str_result)
+ self.assertEqual(bytes_result, bytes_args)
+ self.assertEqual(str_result.encode(), bytes_args)
+ self.assertEqual(str_result.encode(), bytes_result)
+ self.assertEqual(str_result.encode(encoding), bytes_args)
+ self.assertEqual(str_result.encode(encoding), bytes_result)
+ self.assertEqual(str_result.encode(encoding, errors), bytes_args)
+ self.assertEqual(str_result.encode(encoding, errors), bytes_result)
+
+ def test_result_pairs(self):
+ # Check encoding and decoding between result pairs
+ result_types = [
+ urllib.parse.DefragResult,
+ urllib.parse.SplitResult,
+ urllib.parse.ParseResult,
+ ]
+ for result_type in result_types:
+ self._check_result_type(result_type)
+
+ def test_parse_qs_encoding(self):
+ result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1")
+ self.assertEqual(result, {'key': ['\u0141\xE9']})
+ result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="utf-8")
+ self.assertEqual(result, {'key': ['\u0141\xE9']})
+ result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="ascii")
+ self.assertEqual(result, {'key': ['\u0141\ufffd\ufffd']})
+ result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii")
+ self.assertEqual(result, {'key': ['\u0141\ufffd-']})
+ result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii",
+ errors="ignore")
+ self.assertEqual(result, {'key': ['\u0141-']})
+
+ def test_parse_qsl_encoding(self):
+ result = urllib.parse.parse_qsl("key=\u0141%E9", encoding="latin-1")
+ self.assertEqual(result, [('key', '\u0141\xE9')])
+ result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="utf-8")
+ self.assertEqual(result, [('key', '\u0141\xE9')])
+ result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="ascii")
+ self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')])
+ result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii")
+ self.assertEqual(result, [('key', '\u0141\ufffd-')])
+ result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii",
+ errors="ignore")
+ self.assertEqual(result, [('key', '\u0141-')])
+
+ def test_parse_qsl_max_num_fields(self):
+ with self.assertRaises(ValueError):
+ urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10)
+ with self.assertRaises(ValueError):
+ urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10)
+ urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10)
+
+ def test_urlencode_sequences(self):
+ # Other tests incidentally urlencode things; test non-covered cases:
+ # Sequence and object values.
+ result = urllib.parse.urlencode({'a': [1, 2], 'b': (3, 4, 5)}, True)
+ # we cannot rely on ordering here
+ assert set(result.split('&')) == {'a=1', 'a=2', 'b=3', 'b=4', 'b=5'}
+
+ class Trivial:
+ def __str__(self):
+ return 'trivial'
+
+ result = urllib.parse.urlencode({'a': Trivial()}, True)
+ self.assertEqual(result, 'a=trivial')
+
+ def test_urlencode_quote_via(self):
+ result = urllib.parse.urlencode({'a': 'some value'})
+ self.assertEqual(result, "a=some+value")
+ result = urllib.parse.urlencode({'a': 'some value/another'},
+ quote_via=urllib.parse.quote)
+ self.assertEqual(result, "a=some%20value%2Fanother")
+ result = urllib.parse.urlencode({'a': 'some value/another'},
+ safe='/', quote_via=urllib.parse.quote)
+ self.assertEqual(result, "a=some%20value/another")
+
+ def test_quote_from_bytes(self):
+ self.assertRaises(TypeError, urllib.parse.quote_from_bytes, 'foo')
+ result = urllib.parse.quote_from_bytes(b'archaeological arcana')
+ self.assertEqual(result, 'archaeological%20arcana')
+ result = urllib.parse.quote_from_bytes(b'')
+ self.assertEqual(result, '')
+
+ def test_unquote_to_bytes(self):
+ result = urllib.parse.unquote_to_bytes('abc%20def')
+ self.assertEqual(result, b'abc def')
+ result = urllib.parse.unquote_to_bytes('')
+ self.assertEqual(result, b'')
+
+ def test_quote_errors(self):
+ self.assertRaises(TypeError, urllib.parse.quote, b'foo',
+ encoding='utf-8')
+ self.assertRaises(TypeError, urllib.parse.quote, b'foo', errors='strict')
+
+ def test_issue14072(self):
+ p1 = urllib.parse.urlsplit('tel:+31-641044153')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '+31-641044153')
+ p2 = urllib.parse.urlsplit('tel:+31641044153')
+ self.assertEqual(p2.scheme, 'tel')
+ self.assertEqual(p2.path, '+31641044153')
+ # assert the behavior for urlparse
+ p1 = urllib.parse.urlparse('tel:+31-641044153')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '+31-641044153')
+ p2 = urllib.parse.urlparse('tel:+31641044153')
+ self.assertEqual(p2.scheme, 'tel')
+ self.assertEqual(p2.path, '+31641044153')
+
+ def test_port_casting_failure_message(self):
+ message = "Port could not be cast to integer value as 'oracle'"
+ p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle')
+ with self.assertRaisesRegex(ValueError, message):
+ p1.port
+
+ p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle')
+ with self.assertRaisesRegex(ValueError, message):
+ p2.port
+
+ def test_telurl_params(self):
+ p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '123-4')
+ self.assertEqual(p1.params, 'phone-context=+1-650-516')
+
+ p1 = urllib.parse.urlparse('tel:+1-201-555-0123')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '+1-201-555-0123')
+ self.assertEqual(p1.params, '')
+
+ p1 = urllib.parse.urlparse('tel:7042;phone-context=example.com')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '7042')
+ self.assertEqual(p1.params, 'phone-context=example.com')
+
+ p1 = urllib.parse.urlparse('tel:863-1234;phone-context=+1-914-555')
+ self.assertEqual(p1.scheme, 'tel')
+ self.assertEqual(p1.path, '863-1234')
+ self.assertEqual(p1.params, 'phone-context=+1-914-555')
+
+ def test_Quoter_repr(self):
+ quoter = urllib.parse.Quoter(urllib.parse._ALWAYS_SAFE)
+ self.assertIn('Quoter', repr(quoter))
+
+ def test_all(self):
+ expected = []
+ undocumented = {
+ 'splitattr', 'splithost', 'splitnport', 'splitpasswd',
+ 'splitport', 'splitquery', 'splittag', 'splittype', 'splituser',
+ 'splitvalue',
+ 'Quoter', 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap',
+ }
+ for name in dir(urllib.parse):
+ if name.startswith('_') or name in undocumented:
+ continue
+ object = getattr(urllib.parse, name)
+ if getattr(object, '__module__', None) == 'urllib.parse':
+ expected.append(name)
+ self.assertCountEqual(urllib.parse.__all__, expected)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_urlsplit_normalization(self):
+ # Certain characters should never occur in the netloc,
+ # including under normalization.
+ # Ensure that ALL of them are detected and cause an error
+ illegal_chars = '/:#?@'
+ hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars}
+ denorm_chars = [
+ c for c in map(chr, range(128, sys.maxunicode))
+ if (hex_chars & set(unicodedata.decomposition(c).split()))
+ and c not in illegal_chars
+ ]
+ # Sanity check that we found at least one such character
+ self.assertIn('\u2100', denorm_chars)
+ self.assertIn('\uFF03', denorm_chars)
+
+ # bpo-36742: Verify port separators are ignored when they
+ # existed prior to decomposition
+ urllib.parse.urlsplit('http://\u30d5\u309a:80')
+ with self.assertRaises(ValueError):
+ urllib.parse.urlsplit('http://\u30d5\u309a\ufe1380')
+
+ for scheme in ["http", "https", "ftp"]:
+ for netloc in ["netloc{}false.netloc", "n{}user@netloc"]:
+ for c in denorm_chars:
+ url = "{}://{}/path".format(scheme, netloc.format(c))
+ with self.subTest(url=url, char='{:04X}'.format(ord(c))):
+ with self.assertRaises(ValueError):
+ urllib.parse.urlsplit(url)
+
+class Utility_Tests(unittest.TestCase):
+ """Testcase to test the various utility functions in the urllib."""
+ # In Python 2 this test class was in test_urllib.
+
+ def test_splittype(self):
+ splittype = urllib.parse._splittype
+ self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring'))
+ self.assertEqual(splittype('opaquestring'), (None, 'opaquestring'))
+ self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring'))
+ self.assertEqual(splittype('type:'), ('type', ''))
+ self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string'))
+
+ def test_splithost(self):
+ splithost = urllib.parse._splithost
+ self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'),
+ ('www.example.org:80', '/foo/bar/baz.html'))
+ self.assertEqual(splithost('//www.example.org:80'),
+ ('www.example.org:80', ''))
+ self.assertEqual(splithost('/foo/bar/baz.html'),
+ (None, '/foo/bar/baz.html'))
+
+ # bpo-30500: # starts a fragment.
+ self.assertEqual(splithost('//127.0.0.1#@host.com'),
+ ('127.0.0.1', '/#@host.com'))
+ self.assertEqual(splithost('//127.0.0.1#@host.com:80'),
+ ('127.0.0.1', '/#@host.com:80'))
+ self.assertEqual(splithost('//127.0.0.1:80#@host.com'),
+ ('127.0.0.1:80', '/#@host.com'))
+
+ # Empty host is returned as empty string.
+ self.assertEqual(splithost("///file"),
+ ('', '/file'))
+
+ # Trailing semicolon, question mark and hash symbol are kept.
+ self.assertEqual(splithost("//example.net/file;"),
+ ('example.net', '/file;'))
+ self.assertEqual(splithost("//example.net/file?"),
+ ('example.net', '/file?'))
+ self.assertEqual(splithost("//example.net/file#"),
+ ('example.net', '/file#'))
+
+ def test_splituser(self):
+ splituser = urllib.parse._splituser
+ self.assertEqual(splituser('User:Pass@www.python.org:080'),
+ ('User:Pass', 'www.python.org:080'))
+ self.assertEqual(splituser('@www.python.org:080'),
+ ('', 'www.python.org:080'))
+ self.assertEqual(splituser('www.python.org:080'),
+ (None, 'www.python.org:080'))
+ self.assertEqual(splituser('User:Pass@'),
+ ('User:Pass', ''))
+ self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'),
+ ('User@example.com:Pass', 'www.python.org:080'))
+
+ def test_splitpasswd(self):
+ # Some of the password examples are not sensible, but it is added to
+ # confirming to RFC2617 and addressing issue4675.
+ splitpasswd = urllib.parse._splitpasswd
+ self.assertEqual(splitpasswd('user:ab'), ('user', 'ab'))
+ self.assertEqual(splitpasswd('user:a\nb'), ('user', 'a\nb'))
+ self.assertEqual(splitpasswd('user:a\tb'), ('user', 'a\tb'))
+ self.assertEqual(splitpasswd('user:a\rb'), ('user', 'a\rb'))
+ self.assertEqual(splitpasswd('user:a\fb'), ('user', 'a\fb'))
+ self.assertEqual(splitpasswd('user:a\vb'), ('user', 'a\vb'))
+ self.assertEqual(splitpasswd('user:a:b'), ('user', 'a:b'))
+ self.assertEqual(splitpasswd('user:a b'), ('user', 'a b'))
+ self.assertEqual(splitpasswd('user 2:ab'), ('user 2', 'ab'))
+ self.assertEqual(splitpasswd('user+1:a+b'), ('user+1', 'a+b'))
+ self.assertEqual(splitpasswd('user:'), ('user', ''))
+ self.assertEqual(splitpasswd('user'), ('user', None))
+ self.assertEqual(splitpasswd(':ab'), ('', 'ab'))
+
+ def test_splitport(self):
+ splitport = urllib.parse._splitport
+ self.assertEqual(splitport('parrot:88'), ('parrot', '88'))
+ self.assertEqual(splitport('parrot'), ('parrot', None))
+ self.assertEqual(splitport('parrot:'), ('parrot', None))
+ self.assertEqual(splitport('127.0.0.1'), ('127.0.0.1', None))
+ self.assertEqual(splitport('parrot:cheese'), ('parrot:cheese', None))
+ self.assertEqual(splitport('[::1]:88'), ('[::1]', '88'))
+ self.assertEqual(splitport('[::1]'), ('[::1]', None))
+ self.assertEqual(splitport(':88'), ('', '88'))
+
+ def test_splitnport(self):
+ splitnport = urllib.parse._splitnport
+ self.assertEqual(splitnport('parrot:88'), ('parrot', 88))
+ self.assertEqual(splitnport('parrot'), ('parrot', -1))
+ self.assertEqual(splitnport('parrot', 55), ('parrot', 55))
+ self.assertEqual(splitnport('parrot:'), ('parrot', -1))
+ self.assertEqual(splitnport('parrot:', 55), ('parrot', 55))
+ self.assertEqual(splitnport('127.0.0.1'), ('127.0.0.1', -1))
+ self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55))
+ self.assertEqual(splitnport('parrot:cheese'), ('parrot', None))
+ self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None))
+
+ def test_splitquery(self):
+ # Normal cases are exercised by other tests; ensure that we also
+ # catch cases with no port specified (testcase ensuring coverage)
+ splitquery = urllib.parse._splitquery
+ self.assertEqual(splitquery('http://python.org/fake?foo=bar'),
+ ('http://python.org/fake', 'foo=bar'))
+ self.assertEqual(splitquery('http://python.org/fake?foo=bar?'),
+ ('http://python.org/fake?foo=bar', ''))
+ self.assertEqual(splitquery('http://python.org/fake'),
+ ('http://python.org/fake', None))
+ self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar'))
+
+ def test_splittag(self):
+ splittag = urllib.parse._splittag
+ self.assertEqual(splittag('http://example.com?foo=bar#baz'),
+ ('http://example.com?foo=bar', 'baz'))
+ self.assertEqual(splittag('http://example.com?foo=bar#'),
+ ('http://example.com?foo=bar', ''))
+ self.assertEqual(splittag('#baz'), ('', 'baz'))
+ self.assertEqual(splittag('http://example.com?foo=bar'),
+ ('http://example.com?foo=bar', None))
+ self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'),
+ ('http://example.com?foo=bar#baz', 'boo'))
+
+ def test_splitattr(self):
+ splitattr = urllib.parse._splitattr
+ self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'),
+ ('/path', ['attr1=value1', 'attr2=value2']))
+ self.assertEqual(splitattr('/path;'), ('/path', ['']))
+ self.assertEqual(splitattr(';attr1=value1;attr2=value2'),
+ ('', ['attr1=value1', 'attr2=value2']))
+ self.assertEqual(splitattr('/path'), ('/path', []))
+
+ def test_splitvalue(self):
+ # Normal cases are exercised by other tests; test pathological cases
+ # with no key/value pairs. (testcase ensuring coverage)
+ splitvalue = urllib.parse._splitvalue
+ self.assertEqual(splitvalue('foo=bar'), ('foo', 'bar'))
+ self.assertEqual(splitvalue('foo='), ('foo', ''))
+ self.assertEqual(splitvalue('=bar'), ('', 'bar'))
+ self.assertEqual(splitvalue('foobar'), ('foobar', None))
+ self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz'))
+
+ def test_to_bytes(self):
+ result = urllib.parse._to_bytes('http://www.python.org')
+ self.assertEqual(result, 'http://www.python.org')
+ self.assertRaises(UnicodeError, urllib.parse._to_bytes,
+ 'http://www.python.org/medi\u00e6val')
+
+ def test_unwrap(self):
+ for wrapped_url in ('', '',
+ 'URL:scheme://host/path', 'scheme://host/path'):
+ url = urllib.parse.unwrap(wrapped_url)
+ self.assertEqual(url, 'scheme://host/path')
+
+
+class DeprecationTest(unittest.TestCase):
+
+ def test_splittype_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splittype('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splittype() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splithost_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splithost('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splithost() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splituser_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splituser('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splituser() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitpasswd_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitpasswd('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitpasswd() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitport_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitport('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitport() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitnport_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitnport('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitnport() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitquery_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitquery('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitquery() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splittag_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splittag('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splittag() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitattr_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitattr('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitattr() is deprecated as of 3.8, '
+ 'use urllib.parse.urlparse() instead')
+
+ def test_splitvalue_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.splitvalue('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.splitvalue() is deprecated as of 3.8, '
+ 'use urllib.parse.parse_qsl() instead')
+
+ def test_to_bytes_deprecation(self):
+ with self.assertWarns(DeprecationWarning) as cm:
+ urllib.parse.to_bytes('')
+ self.assertEqual(str(cm.warning),
+ 'urllib.parse.to_bytes() is deprecated as of 3.8')
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
new file mode 100644
index 000000000..98343b6bc
--- /dev/null
+++ b/Lib/test/test_venv.py
@@ -0,0 +1,544 @@
+"""
+Test harness for the venv module.
+
+Copyright (C) 2011-2012 Vinay Sajip.
+Licensed to the PSF under a contributor agreement.
+"""
+
+# pip isn't working yet
+# import ensurepip
+import os
+import os.path
+import re
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+from test.support import (captured_stdout, captured_stderr, requires_zlib,
+ can_symlink, EnvironmentVarGuard, rmtree,
+ import_module,
+ skip_if_broken_multiprocessing_synchronize)
+import unittest
+import venv
+from unittest.mock import patch
+
+try:
+ import ctypes
+except ImportError:
+ ctypes = None
+
+# Platforms that set sys._base_executable can create venvs from within
+# another venv, so no need to skip tests that require venv.create().
+requireVenvCreate = unittest.skipUnless(
+ sys.prefix == sys.base_prefix
+ or sys._base_executable != sys.executable,
+ 'cannot run venv.create from within a venv on this platform')
+
+def check_output(cmd, encoding=None):
+ p = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding=encoding)
+ out, err = p.communicate()
+ if p.returncode:
+ raise subprocess.CalledProcessError(
+ p.returncode, cmd, out, err)
+ return out, err
+
+class BaseTest(unittest.TestCase):
+ """Base class for venv tests."""
+ maxDiff = 80 * 50
+
+ def setUp(self):
+ self.env_dir = os.path.realpath(tempfile.mkdtemp())
+ if os.name == 'nt':
+ self.bindir = 'Scripts'
+ self.lib = ('Lib',)
+ self.include = 'Include'
+ else:
+ self.bindir = 'bin'
+ self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
+ self.include = 'include'
+ executable = sys._base_executable
+ self.exe = os.path.split(executable)[-1]
+ if (sys.platform == 'win32'
+ and os.path.lexists(executable)
+ and not os.path.exists(executable)):
+ self.cannot_link_exe = True
+ else:
+ self.cannot_link_exe = False
+
+ def tearDown(self):
+ rmtree(self.env_dir)
+
+ def run_with_capture(self, func, *args, **kwargs):
+ with captured_stdout() as output:
+ with captured_stderr() as error:
+ func(*args, **kwargs)
+ return output.getvalue(), error.getvalue()
+
+ def get_env_file(self, *args):
+ return os.path.join(self.env_dir, *args)
+
+ def get_text_file_contents(self, *args, encoding='utf-8'):
+ with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
+ result = f.read()
+ return result
+
+class BasicTest(BaseTest):
+ """Test venv module functionality."""
+
+ def isdir(self, *args):
+ fn = self.get_env_file(*args)
+ self.assertTrue(os.path.isdir(fn))
+
+ def test_defaults(self):
+ """
+ Test the create function with default arguments.
+ """
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ self.isdir(self.bindir)
+ self.isdir(self.include)
+ self.isdir(*self.lib)
+ # Issue 21197
+ p = self.get_env_file('lib64')
+ conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
+ (sys.platform != 'darwin'))
+ if conditions:
+ self.assertTrue(os.path.islink(p))
+ else:
+ self.assertFalse(os.path.exists(p))
+ data = self.get_text_file_contents('pyvenv.cfg')
+ executable = sys._base_executable
+ path = os.path.dirname(executable)
+ self.assertIn('home = %s' % path, data)
+ fn = self.get_env_file(self.bindir, self.exe)
+ if not os.path.exists(fn): # diagnostics for Windows buildbot failures
+ bd = self.get_env_file(self.bindir)
+ print('Contents of %r:' % bd)
+ print(' %r' % os.listdir(bd))
+ self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
+
+ def test_prompt(self):
+ env_name = os.path.split(self.env_dir)[1]
+
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder()
+ self.run_with_capture(builder.create, self.env_dir)
+ context = builder.ensure_directories(self.env_dir)
+ data = self.get_text_file_contents('pyvenv.cfg')
+ self.assertEqual(context.prompt, '(%s) ' % env_name)
+ self.assertNotIn("prompt = ", data)
+
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder(prompt='My prompt')
+ self.run_with_capture(builder.create, self.env_dir)
+ context = builder.ensure_directories(self.env_dir)
+ data = self.get_text_file_contents('pyvenv.cfg')
+ self.assertEqual(context.prompt, '(My prompt) ')
+ self.assertIn("prompt = 'My prompt'\n", data)
+
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder(prompt='.')
+ cwd = os.path.basename(os.getcwd())
+ self.run_with_capture(builder.create, self.env_dir)
+ context = builder.ensure_directories(self.env_dir)
+ data = self.get_text_file_contents('pyvenv.cfg')
+ self.assertEqual(context.prompt, '(%s) ' % cwd)
+ self.assertIn("prompt = '%s'\n" % cwd, data)
+
+ def test_upgrade_dependencies(self):
+ builder = venv.EnvBuilder()
+ bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
+ python_exe = 'python.exe' if sys.platform == 'win32' else 'python'
+ with tempfile.TemporaryDirectory() as fake_env_dir:
+
+ def pip_cmd_checker(cmd):
+ self.assertEqual(
+ cmd,
+ [
+ os.path.join(fake_env_dir, bin_path, python_exe),
+ '-m',
+ 'pip',
+ 'install',
+ '--upgrade',
+ 'pip',
+ 'setuptools'
+ ]
+ )
+
+ fake_context = builder.ensure_directories(fake_env_dir)
+ with patch('venv.subprocess.check_call', pip_cmd_checker):
+ builder.upgrade_dependencies(fake_context)
+
+ @requireVenvCreate
+ def test_prefixes(self):
+ """
+ Test that the prefix values are as expected.
+ """
+ # check a venv's prefixes
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ envpy = os.path.join(self.env_dir, self.bindir, self.exe)
+ cmd = [envpy, '-c', None]
+ for prefix, expected in (
+ ('prefix', self.env_dir),
+ ('exec_prefix', self.env_dir),
+ ('base_prefix', sys.base_prefix),
+ ('base_exec_prefix', sys.base_exec_prefix)):
+ cmd[2] = 'import sys; print(sys.%s)' % prefix
+ out, err = check_output(cmd)
+ self.assertEqual(out.strip(), expected.encode())
+
+ if sys.platform == 'win32':
+ ENV_SUBDIRS = (
+ ('Scripts',),
+ ('Include',),
+ ('Lib',),
+ ('Lib', 'site-packages'),
+ )
+ else:
+ ENV_SUBDIRS = (
+ ('bin',),
+ ('include',),
+ ('lib',),
+ ('lib', 'python%d.%d' % sys.version_info[:2]),
+ ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
+ )
+
+ def create_contents(self, paths, filename):
+ """
+ Create some files in the environment which are unrelated
+ to the virtual environment.
+ """
+ for subdirs in paths:
+ d = os.path.join(self.env_dir, *subdirs)
+ os.mkdir(d)
+ fn = os.path.join(d, filename)
+ with open(fn, 'wb') as f:
+ f.write(b'Still here?')
+
+ def test_overwrite_existing(self):
+ """
+ Test creating environment in an existing directory.
+ """
+ self.create_contents(self.ENV_SUBDIRS, 'foo')
+ venv.create(self.env_dir)
+ for subdirs in self.ENV_SUBDIRS:
+ fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
+ self.assertTrue(os.path.exists(fn))
+ with open(fn, 'rb') as f:
+ self.assertEqual(f.read(), b'Still here?')
+
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(self.env_dir)
+ for subdirs in self.ENV_SUBDIRS:
+ fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
+ self.assertFalse(os.path.exists(fn))
+
+ def clear_directory(self, path):
+ for fn in os.listdir(path):
+ fn = os.path.join(path, fn)
+ if os.path.islink(fn) or os.path.isfile(fn):
+ os.remove(fn)
+ elif os.path.isdir(fn):
+ rmtree(fn)
+
+ def test_unoverwritable_fails(self):
+ #create a file clashing with directories in the env dir
+ for paths in self.ENV_SUBDIRS[:3]:
+ fn = os.path.join(self.env_dir, *paths)
+ with open(fn, 'wb') as f:
+ f.write(b'')
+ self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
+ self.clear_directory(self.env_dir)
+
+ def test_upgrade(self):
+ """
+ Test upgrading an existing environment directory.
+ """
+ # See Issue #21643: the loop needs to run twice to ensure
+ # that everything works on the upgrade (the first run just creates
+ # the venv).
+ for upgrade in (False, True):
+ builder = venv.EnvBuilder(upgrade=upgrade)
+ self.run_with_capture(builder.create, self.env_dir)
+ self.isdir(self.bindir)
+ self.isdir(self.include)
+ self.isdir(*self.lib)
+ fn = self.get_env_file(self.bindir, self.exe)
+ if not os.path.exists(fn):
+ # diagnostics for Windows buildbot failures
+ bd = self.get_env_file(self.bindir)
+ print('Contents of %r:' % bd)
+ print(' %r' % os.listdir(bd))
+ self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
+
+ def test_isolation(self):
+ """
+ Test isolation from system site-packages
+ """
+ for ssp, s in ((True, 'true'), (False, 'false')):
+ builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
+ builder.create(self.env_dir)
+ data = self.get_text_file_contents('pyvenv.cfg')
+ self.assertIn('include-system-site-packages = %s\n' % s, data)
+
+ @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+ def test_symlinking(self):
+ """
+ Test symlinking works as expected
+ """
+ for usl in (False, True):
+ builder = venv.EnvBuilder(clear=True, symlinks=usl)
+ builder.create(self.env_dir)
+ fn = self.get_env_file(self.bindir, self.exe)
+ # Don't test when False, because e.g. 'python' is always
+ # symlinked to 'python3.3' in the env, even when symlinking in
+ # general isn't wanted.
+ if usl:
+ if self.cannot_link_exe:
+ # Symlinking is skipped when our executable is already a
+ # special app symlink
+ self.assertFalse(os.path.islink(fn))
+ else:
+ self.assertTrue(os.path.islink(fn))
+
+ # If a venv is created from a source build and that venv is used to
+ # run the test, the pyvenv.cfg in the venv created in the test will
+ # point to the venv being used to run the test, and we lose the link
+ # to the source build - so Python can't initialise properly.
+ @requireVenvCreate
+ def test_executable(self):
+ """
+ Test that the sys.executable value is as expected.
+ """
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ out, err = check_output([envpy, '-c',
+ 'import sys; print(sys.executable)'])
+ self.assertEqual(out.strip(), envpy.encode())
+
+ @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+ def test_executable_symlinks(self):
+ """
+ Test that the sys.executable value is as expected.
+ """
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder(clear=True, symlinks=True)
+ builder.create(self.env_dir)
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ out, err = check_output([envpy, '-c',
+ 'import sys; print(sys.executable)'])
+ self.assertEqual(out.strip(), envpy.encode())
+
+ @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
+ def test_unicode_in_batch_file(self):
+ """
+ Test handling of Unicode paths
+ """
+ rmtree(self.env_dir)
+ env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(env_dir)
+ activate = os.path.join(env_dir, self.bindir, 'activate.bat')
+ envpy = os.path.join(env_dir, self.bindir, self.exe)
+ out, err = check_output(
+ [activate, '&', self.exe, '-c', 'print(0)'],
+ encoding='oem',
+ )
+ self.assertEqual(out.strip(), '0')
+
+ @requireVenvCreate
+ def test_multiprocessing(self):
+ """
+ Test that the multiprocessing is able to spawn.
+ """
+ # bpo-36342: Instantiation of a Pool object imports the
+ # multiprocessing.synchronize module. Skip the test if this module
+ # cannot be imported.
+ skip_if_broken_multiprocessing_synchronize()
+
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ out, err = check_output([envpy, '-c',
+ 'from multiprocessing import Pool; '
+ 'pool = Pool(1); '
+ 'print(pool.apply_async("Python".lower).get(3)); '
+ 'pool.terminate()'])
+ self.assertEqual(out.strip(), "python".encode())
+
+ @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
+ def test_deactivate_with_strict_bash_opts(self):
+ bash = shutil.which("bash")
+ if bash is None:
+ self.skipTest("bash required for this test")
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder(clear=True)
+ builder.create(self.env_dir)
+ activate = os.path.join(self.env_dir, self.bindir, "activate")
+ test_script = os.path.join(self.env_dir, "test_strict.sh")
+ with open(test_script, "w") as f:
+ f.write("set -euo pipefail\n"
+ f"source {activate}\n"
+ "deactivate\n")
+ out, err = check_output([bash, test_script])
+ self.assertEqual(out, "".encode())
+ self.assertEqual(err, "".encode())
+
+
+ @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
+ def test_macos_env(self):
+ rmtree(self.env_dir)
+ builder = venv.EnvBuilder()
+ builder.create(self.env_dir)
+
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ out, err = check_output([envpy, '-c',
+ 'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
+ self.assertEqual(out.strip(), 'False'.encode())
+
+@requireVenvCreate
+class EnsurePipTest(BaseTest):
+ """Test venv module installation of pip."""
+ def assert_pip_not_installed(self):
+ envpy = os.path.join(os.path.realpath(self.env_dir),
+ self.bindir, self.exe)
+ out, err = check_output([envpy, '-c',
+ 'try:\n import pip\nexcept ImportError:\n print("OK")'])
+ # We force everything to text, so unittest gives the detailed diff
+ # if we get unexpected results
+ err = err.decode("latin-1") # Force to text, prevent decoding errors
+ self.assertEqual(err, "")
+ out = out.decode("latin-1") # Force to text, prevent decoding errors
+ self.assertEqual(out.strip(), "OK")
+
+
+ def test_no_pip_by_default(self):
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir)
+ self.assert_pip_not_installed()
+
+ def test_explicit_no_pip(self):
+ rmtree(self.env_dir)
+ self.run_with_capture(venv.create, self.env_dir, with_pip=False)
+ self.assert_pip_not_installed()
+
+ def test_devnull(self):
+ # Fix for issue #20053 uses os.devnull to force a config file to
+ # appear empty. However http://bugs.python.org/issue20541 means
+ # that doesn't currently work properly on Windows. Once that is
+ # fixed, the "win_location" part of test_with_pip should be restored
+ with open(os.devnull, "rb") as f:
+ self.assertEqual(f.read(), b"")
+
+ self.assertTrue(os.path.exists(os.devnull))
+
+ def do_test_with_pip(self, system_site_packages):
+ rmtree(self.env_dir)
+ with EnvironmentVarGuard() as envvars:
+ # pip's cross-version compatibility may trigger deprecation
+ # warnings in current versions of Python. Ensure related
+ # environment settings don't cause venv to fail.
+ envvars["PYTHONWARNINGS"] = "e"
+ # ensurepip is different enough from a normal pip invocation
+ # that we want to ensure it ignores the normal pip environment
+ # variable settings. We set PIP_NO_INSTALL here specifically
+ # to check that ensurepip (and hence venv) ignores it.
+ # See http://bugs.python.org/issue19734
+ envvars["PIP_NO_INSTALL"] = "1"
+ # Also check that we ignore the pip configuration file
+ # See http://bugs.python.org/issue20053
+ with tempfile.TemporaryDirectory() as home_dir:
+ envvars["HOME"] = home_dir
+ bad_config = "[global]\nno-install=1"
+ # Write to both config file names on all platforms to reduce
+ # cross-platform variation in test code behaviour
+ win_location = ("pip", "pip.ini")
+ posix_location = (".pip", "pip.conf")
+ # Skips win_location due to http://bugs.python.org/issue20541
+ for dirname, fname in (posix_location,):
+ dirpath = os.path.join(home_dir, dirname)
+ os.mkdir(dirpath)
+ fpath = os.path.join(dirpath, fname)
+ with open(fpath, 'w') as f:
+ f.write(bad_config)
+
+ # Actually run the create command with all that unhelpful
+ # config in place to ensure we ignore it
+ try:
+ self.run_with_capture(venv.create, self.env_dir,
+ system_site_packages=system_site_packages,
+ with_pip=True)
+ except subprocess.CalledProcessError as exc:
+ # The output this produces can be a little hard to read,
+ # but at least it has all the details
+ details = exc.output.decode(errors="replace")
+ msg = "{}\n\n**Subprocess Output**\n{}"
+ self.fail(msg.format(exc, details))
+ # Ensure pip is available in the virtual environment
+ envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
+ # Ignore DeprecationWarning since pip code is not part of Python
+ out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I',
+ '-m', 'pip', '--version'])
+ # We force everything to text, so unittest gives the detailed diff
+ # if we get unexpected results
+ err = err.decode("latin-1") # Force to text, prevent decoding errors
+ self.assertEqual(err, "")
+ out = out.decode("latin-1") # Force to text, prevent decoding errors
+ expected_version = "pip {}".format(ensurepip.version())
+ self.assertEqual(out[:len(expected_version)], expected_version)
+ env_dir = os.fsencode(self.env_dir).decode("latin-1")
+ self.assertIn(env_dir, out)
+
+ # http://bugs.python.org/issue19728
+ # Check the private uninstall command provided for the Windows
+ # installers works (at least in a virtual environment)
+ with EnvironmentVarGuard() as envvars:
+ out, err = check_output([envpy,
+ '-W', 'ignore::DeprecationWarning', '-I',
+ '-m', 'ensurepip._uninstall'])
+ # We force everything to text, so unittest gives the detailed diff
+ # if we get unexpected results
+ err = err.decode("latin-1") # Force to text, prevent decoding errors
+ # Ignore the warning:
+ # "The directory '$HOME/.cache/pip/http' or its parent directory
+ # is not owned by the current user and the cache has been disabled.
+ # Please check the permissions and owner of that directory. If
+ # executing pip with sudo, you may want sudo's -H flag."
+ # where $HOME is replaced by the HOME environment variable.
+ err = re.sub("^(WARNING: )?The directory .* or its parent directory "
+ "is not owned or is not writable by the current user.*$", "",
+ err, flags=re.MULTILINE)
+ self.assertEqual(err.rstrip(), "")
+ # Being fairly specific regarding the expected behaviour for the
+ # initial bundling phase in Python 3.4. If the output changes in
+ # future pip versions, this test can likely be relaxed further.
+ out = out.decode("latin-1") # Force to text, prevent decoding errors
+ self.assertIn("Successfully uninstalled pip", out)
+ self.assertIn("Successfully uninstalled setuptools", out)
+ # Check pip is now gone from the virtual environment. This only
+ # applies in the system_site_packages=False case, because in the
+ # other case, pip may still be available in the system site-packages
+ if not system_site_packages:
+ self.assert_pip_not_installed()
+
+ # Issue #26610: pip/pep425tags.py requires ctypes
+ # TODO: RUSTPYTHON
+ @unittest.skipUnless(ctypes, 'pip requires ctypes')
+ @requires_zlib
+ @unittest.expectedFailure
+ def test_with_pip(self):
+ self.do_test_with_pip(False)
+ self.do_test_with_pip(True)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
new file mode 100644
index 000000000..89bbc7020
--- /dev/null
+++ b/Lib/test/test_zipimport.py
@@ -0,0 +1,789 @@
+import sys
+import os
+import marshal
+import importlib
+import importlib.util
+import struct
+import time
+import unittest
+import unittest.mock
+
+from test import support
+
+from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED
+
+import zipimport
+import linecache
+import doctest
+import inspect
+import io
+from traceback import extract_tb, extract_stack, print_tb
+try:
+ import zlib
+except ImportError:
+ zlib = None
+
+test_src = """\
+def get_name():
+ return __name__
+def get_file():
+ return __file__
+"""
+test_co = compile(test_src, "", "exec")
+raise_src = 'def do_raise(): raise TypeError\n'
+
+def make_pyc(co, mtime, size):
+ data = marshal.dumps(co)
+ if type(mtime) is type(0.0):
+ # Mac mtimes need a bit of special casing
+ if mtime < 0x7fffffff:
+ mtime = int(mtime)
+ else:
+ mtime = int(-0x100000000 + int(mtime))
+ pyc = (importlib.util.MAGIC_NUMBER +
+ struct.pack("", "exec"), NOW, len(src))
+ files = {TESTMOD + pyc_ext: (NOW, pyc),
+ "some.data": (NOW, "some data")}
+ self.doTest(pyc_ext, files, TESTMOD)
+
+ def testDefaultOptimizationLevel(self):
+ # zipimport should use the default optimization level (#28131)
+ src = """if 1: # indent hack
+ def test(val):
+ assert(val)
+ return val\n"""
+ files = {TESTMOD + '.py': (NOW, src)}
+ self.makeZip(files)
+ sys.path.insert(0, TEMP_ZIP)
+ mod = importlib.import_module(TESTMOD)
+ self.assertEqual(mod.test(1), 1)
+ self.assertRaises(AssertionError, mod.test, False)
+
+ def testImport_WithStuff(self):
+ # try importing from a zipfile which contains additional
+ # stuff at the beginning of the file
+ files = {TESTMOD + ".py": (NOW, test_src)}
+ self.doTest(".py", files, TESTMOD,
+ stuff=b"Some Stuff"*31)
+
+ def assertModuleSource(self, module):
+ self.assertEqual(inspect.getsource(module), test_src)
+
+ def testGetSource(self):
+ files = {TESTMOD + ".py": (NOW, test_src)}
+ self.doTest(".py", files, TESTMOD, call=self.assertModuleSource)
+
+ def testGetCompiledSource(self):
+ pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src))
+ files = {TESTMOD + ".py": (NOW, test_src),
+ TESTMOD + pyc_ext: (NOW, pyc)}
+ self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource)
+
+ def runDoctest(self, callback):
+ files = {TESTMOD + ".py": (NOW, test_src),
+ "xyz.txt": (NOW, ">>> log.append(True)\n")}
+ self.doTest(".py", files, TESTMOD, call=callback)
+
+ def doDoctestFile(self, module):
+ log = []
+ old_master, doctest.master = doctest.master, None
+ try:
+ doctest.testfile(
+ 'xyz.txt', package=module, module_relative=True,
+ globs=locals()
+ )
+ finally:
+ doctest.master = old_master
+ self.assertEqual(log,[True])
+
+ def testDoctestFile(self):
+ self.runDoctest(self.doDoctestFile)
+
+ def doDoctestSuite(self, module):
+ log = []
+ doctest.DocFileTest(
+ 'xyz.txt', package=module, module_relative=True,
+ globs=locals()
+ ).run()
+ self.assertEqual(log,[True])
+
+ def testDoctestSuite(self):
+ self.runDoctest(self.doDoctestSuite)
+
+ def doTraceback(self, module):
+ try:
+ module.do_raise()
+ except:
+ tb = sys.exc_info()[2].tb_next
+
+ f,lno,n,line = extract_tb(tb, 1)[0]
+ self.assertEqual(line, raise_src.strip())
+
+ f,lno,n,line = extract_stack(tb.tb_frame, 1)[0]
+ self.assertEqual(line, raise_src.strip())
+
+ s = io.StringIO()
+ print_tb(tb, 1, s)
+ self.assertTrue(s.getvalue().endswith(raise_src))
+ else:
+ raise AssertionError("This ought to be impossible")
+
+ def testTraceback(self):
+ files = {TESTMOD + ".py": (NOW, raise_src)}
+ self.doTest(None, files, TESTMOD, call=self.doTraceback)
+
+ @unittest.skipIf(support.TESTFN_UNENCODABLE is None,
+ "need an unencodable filename")
+ def testUnencodable(self):
+ filename = support.TESTFN_UNENCODABLE + ".zip"
+ self.addCleanup(support.unlink, filename)
+ with ZipFile(filename, "w") as z:
+ zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW))
+ zinfo.compress_type = self.compression
+ z.writestr(zinfo, test_src)
+ zipimport.zipimporter(filename).load_module(TESTMOD)
+
+ def testBytesPath(self):
+ filename = support.TESTFN + ".zip"
+ self.addCleanup(support.unlink, filename)
+ with ZipFile(filename, "w") as z:
+ zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW))
+ zinfo.compress_type = self.compression
+ z.writestr(zinfo, test_src)
+
+ zipimport.zipimporter(filename)
+ zipimport.zipimporter(os.fsencode(filename))
+ with self.assertRaises(TypeError):
+ zipimport.zipimporter(bytearray(os.fsencode(filename)))
+ with self.assertRaises(TypeError):
+ zipimport.zipimporter(memoryview(os.fsencode(filename)))
+
+ def testComment(self):
+ files = {TESTMOD + ".py": (NOW, test_src)}
+ self.doTest(".py", files, TESTMOD, comment=b"comment")
+
+ def testBeginningCruftAndComment(self):
+ files = {TESTMOD + ".py": (NOW, test_src)}
+ self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi")
+
+ def testLargestPossibleComment(self):
+ files = {TESTMOD + ".py": (NOW, test_src)}
+ self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1))
+
+
+@support.requires_zlib
+class CompressedZipImportTestCase(UncompressedZipImportTestCase):
+ compression = ZIP_DEFLATED
+
+
+class BadFileZipImportTestCase(unittest.TestCase):
+ def assertZipFailure(self, filename):
+ self.assertRaises(zipimport.ZipImportError,
+ zipimport.zipimporter, filename)
+
+ def testNoFile(self):
+ self.assertZipFailure('AdfjdkFJKDFJjdklfjs')
+
+ def testEmptyFilename(self):
+ self.assertZipFailure('')
+
+ def testBadArgs(self):
+ self.assertRaises(TypeError, zipimport.zipimporter, None)
+ self.assertRaises(TypeError, zipimport.zipimporter, TESTMOD, kwd=None)
+ self.assertRaises(TypeError, zipimport.zipimporter,
+ list(os.fsencode(TESTMOD)))
+
+ def testFilenameTooLong(self):
+ self.assertZipFailure('A' * 33000)
+
+ def testEmptyFile(self):
+ support.unlink(TESTMOD)
+ support.create_empty_file(TESTMOD)
+ self.assertZipFailure(TESTMOD)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def testFileUnreadable(self):
+ support.unlink(TESTMOD)
+ fd = os.open(TESTMOD, os.O_CREAT, 000)
+ try:
+ os.close(fd)
+
+ with self.assertRaises(zipimport.ZipImportError) as cm:
+ zipimport.zipimporter(TESTMOD)
+ finally:
+ # If we leave "the read-only bit" set on Windows, nothing can
+ # delete TESTMOD, and later tests suffer bogus failures.
+ os.chmod(TESTMOD, 0o666)
+ support.unlink(TESTMOD)
+
+ def testNotZipFile(self):
+ support.unlink(TESTMOD)
+ fp = open(TESTMOD, 'w+')
+ fp.write('a' * 22)
+ fp.close()
+ self.assertZipFailure(TESTMOD)
+
+ # XXX: disabled until this works on Big-endian machines
+ def _testBogusZipFile(self):
+ support.unlink(TESTMOD)
+ fp = open(TESTMOD, 'w+')
+ fp.write(struct.pack('=I', 0x06054B50))
+ fp.write('a' * 18)
+ fp.close()
+ z = zipimport.zipimporter(TESTMOD)
+
+ try:
+ self.assertRaises(TypeError, z.find_module, None)
+ self.assertRaises(TypeError, z.load_module, None)
+ self.assertRaises(TypeError, z.is_package, None)
+ self.assertRaises(TypeError, z.get_code, None)
+ self.assertRaises(TypeError, z.get_data, None)
+ self.assertRaises(TypeError, z.get_source, None)
+
+ error = zipimport.ZipImportError
+ self.assertEqual(z.find_module('abc'), None)
+
+ self.assertRaises(error, z.load_module, 'abc')
+ self.assertRaises(error, z.get_code, 'abc')
+ self.assertRaises(OSError, z.get_data, 'abc')
+ self.assertRaises(error, z.get_source, 'abc')
+ self.assertRaises(error, z.is_package, 'abc')
+ finally:
+ zipimport._zip_directory_cache.clear()
+
+
+def test_main():
+ try:
+ support.run_unittest(
+ UncompressedZipImportTestCase,
+ CompressedZipImportTestCase,
+ BadFileZipImportTestCase,
+ )
+ finally:
+ support.unlink(TESTMOD)
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
new file mode 100644
index 000000000..759e4b7f3
--- /dev/null
+++ b/Lib/test/test_zlib.py
@@ -0,0 +1,960 @@
+import unittest
+from test import support
+import binascii
+import copy
+import pickle
+import random
+import sys
+from test.support import bigmemtest, _1G, _4G
+
+zlib = support.import_module('zlib')
+
+requires_Compress_copy = unittest.skipUnless(
+ hasattr(zlib.compressobj(), "copy"),
+ 'requires Compress.copy()')
+requires_Decompress_copy = unittest.skipUnless(
+ hasattr(zlib.decompressobj(), "copy"),
+ 'requires Decompress.copy()')
+
+
+class VersionTestCase(unittest.TestCase):
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_library_version(self):
+ # Test that the major version of the actual library in use matches the
+ # major version that we were compiled against. We can't guarantee that
+ # the minor versions will match (even on the machine on which the module
+ # was compiled), and the API is stable between minor versions, so
+ # testing only the major versions avoids spurious failures.
+ self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
+
+
+class ChecksumTestCase(unittest.TestCase):
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ # checksum test cases
+ def test_crc32start(self):
+ self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0))
+ self.assertTrue(zlib.crc32(b"abc", 0xffffffff))
+
+ def test_crc32empty(self):
+ self.assertEqual(zlib.crc32(b"", 0), 0)
+ self.assertEqual(zlib.crc32(b"", 1), 1)
+ self.assertEqual(zlib.crc32(b"", 432), 432)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_adler32start(self):
+ self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1))
+ self.assertTrue(zlib.adler32(b"abc", 0xffffffff))
+
+ def test_adler32empty(self):
+ self.assertEqual(zlib.adler32(b"", 0), 0)
+ self.assertEqual(zlib.adler32(b"", 1), 1)
+ self.assertEqual(zlib.adler32(b"", 432), 432)
+
+ def test_penguins(self):
+ self.assertEqual(zlib.crc32(b"penguin", 0), 0x0e5c1a120)
+ self.assertEqual(zlib.crc32(b"penguin", 1), 0x43b6aa94)
+ self.assertEqual(zlib.adler32(b"penguin", 0), 0x0bcf02f6)
+ self.assertEqual(zlib.adler32(b"penguin", 1), 0x0bd602f7)
+
+ self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0))
+ self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1))
+
+ def test_crc32_adler32_unsigned(self):
+ foo = b'abcdefghijklmnop'
+ # explicitly test signed behavior
+ self.assertEqual(zlib.crc32(foo), 2486878355)
+ self.assertEqual(zlib.crc32(b'spam'), 1138425661)
+ self.assertEqual(zlib.adler32(foo+foo), 3573550353)
+ self.assertEqual(zlib.adler32(b'spam'), 72286642)
+
+ def test_same_as_binascii_crc32(self):
+ foo = b'abcdefghijklmnop'
+ crc = 2486878355
+ self.assertEqual(binascii.crc32(foo), crc)
+ self.assertEqual(zlib.crc32(foo), crc)
+ self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
+
+
+# Issue #10276 - check that inputs >=4 GiB are handled correctly.
+class ChecksumBigBufferTestCase(unittest.TestCase):
+
+ @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
+ def test_big_buffer(self, size):
+ data = b"nyan" * (_1G + 1)
+ self.assertEqual(zlib.crc32(data), 1044521549)
+ self.assertEqual(zlib.adler32(data), 2256789997)
+
+
+class ExceptionTestCase(unittest.TestCase):
+ # make sure we generate some expected errors
+ def test_badlevel(self):
+ # specifying compression level out of range causes an error
+ # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib
+ # accepts 0 too)
+ self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10)
+
+ def test_badargs(self):
+ self.assertRaises(TypeError, zlib.adler32)
+ self.assertRaises(TypeError, zlib.crc32)
+ self.assertRaises(TypeError, zlib.compress)
+ self.assertRaises(TypeError, zlib.decompress)
+ for arg in (42, None, '', 'abc', (), []):
+ self.assertRaises(TypeError, zlib.adler32, arg)
+ self.assertRaises(TypeError, zlib.crc32, arg)
+ self.assertRaises(TypeError, zlib.compress, arg)
+ self.assertRaises(TypeError, zlib.decompress, arg)
+
+ @unittest.skip('TODO: RUSTPYTHON')
+ def test_badcompressobj(self):
+ # verify failure on building compress object with bad params
+ self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
+ # specifying total bits too large causes an error
+ self.assertRaises(ValueError,
+ zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
+
+ @unittest.skip('TODO: RUSTPYTHON')
+ def test_baddecompressobj(self):
+ # verify failure on building decompress object with bad params
+ self.assertRaises(ValueError, zlib.decompressobj, -1)
+
+ def test_decompressobj_badflush(self):
+ # verify failure on calling decompressobj.flush with bad params
+ self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
+ self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
+
+ @support.cpython_only
+ def test_overflow(self):
+ with self.assertRaisesRegex(OverflowError, 'int too large'):
+ zlib.decompress(b'', 15, sys.maxsize + 1)
+ with self.assertRaisesRegex(OverflowError, 'int too large'):
+ zlib.decompressobj().decompress(b'', sys.maxsize + 1)
+ with self.assertRaisesRegex(OverflowError, 'int too large'):
+ zlib.decompressobj().flush(sys.maxsize + 1)
+
+
+class BaseCompressTestCase(object):
+ def check_big_compress_buffer(self, size, compress_func):
+ _1M = 1024 * 1024
+ # Generate 10 MiB worth of random, and expand it by repeating it.
+ # The assumption is that zlib's memory is not big enough to exploit
+ # such spread out redundancy.
+ data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little')
+ for i in range(10)])
+ data = data * (size // len(data) + 1)
+ try:
+ compress_func(data)
+ finally:
+ # Release memory
+ data = None
+
+ def check_big_decompress_buffer(self, size, decompress_func):
+ data = b'x' * size
+ try:
+ compressed = zlib.compress(data, 1)
+ finally:
+ # Release memory
+ data = None
+ data = decompress_func(compressed)
+ # Sanity check
+ try:
+ self.assertEqual(len(data), size)
+ self.assertEqual(len(data.strip(b'x')), 0)
+ finally:
+ data = None
+
+
+class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
+ # Test compression in one go (whole message compression)
+ def test_speech(self):
+ x = zlib.compress(HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_keywords(self):
+ x = zlib.compress(HAMLET_SCENE, level=3)
+ self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
+ with self.assertRaises(TypeError):
+ zlib.compress(data=HAMLET_SCENE, level=3)
+ self.assertEqual(zlib.decompress(x,
+ wbits=zlib.MAX_WBITS,
+ bufsize=zlib.DEF_BUF_SIZE),
+ HAMLET_SCENE)
+
+ def test_speech128(self):
+ # compress more data
+ data = HAMLET_SCENE * 128
+ x = zlib.compress(data)
+ self.assertEqual(zlib.compress(bytearray(data)), x)
+ for ob in x, bytearray(x):
+ self.assertEqual(zlib.decompress(ob), data)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_incomplete_stream(self):
+ # A useful error message is given
+ x = zlib.compress(HAMLET_SCENE)
+ self.assertRaisesRegex(zlib.error,
+ "Error -5 while decompressing data: incomplete or truncated stream",
+ zlib.decompress, x[:-1])
+
+ # Memory use of the following functions takes into account overallocation
+
+ @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
+ def test_big_compress_buffer(self, size):
+ compress = lambda s: zlib.compress(s, 1)
+ self.check_big_compress_buffer(size, compress)
+
+ @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
+ def test_big_decompress_buffer(self, size):
+ self.check_big_decompress_buffer(size, zlib.decompress)
+
+ @bigmemtest(size=_4G, memuse=1)
+ def test_large_bufsize(self, size):
+ # Test decompress(bufsize) parameter greater than the internal limit
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ self.assertEqual(zlib.decompress(compressed, 15, size), data)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_custom_bufsize(self):
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ self.assertEqual(zlib.decompress(compressed, 15, CustomInt()), data)
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @bigmemtest(size=_4G + 100, memuse=4)
+ def test_64bit_compress(self, size):
+ data = b'x' * size
+ try:
+ comp = zlib.compress(data, 0)
+ self.assertEqual(zlib.decompress(comp), data)
+ finally:
+ comp = data = None
+
+
+class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ # Test compression object
+ def test_pair(self):
+ # straightforward compress/decompress objects
+ datasrc = HAMLET_SCENE * 128
+ datazip = zlib.compress(datasrc)
+ # should compress both bytes and bytearray data
+ for data in (datasrc, bytearray(datasrc)):
+ co = zlib.compressobj()
+ x1 = co.compress(data)
+ x2 = co.flush()
+ self.assertRaises(zlib.error, co.flush) # second flush should not work
+ self.assertEqual(x1 + x2, datazip)
+ for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
+ dco = zlib.decompressobj()
+ y1 = dco.decompress(v1 + v2)
+ y2 = dco.flush()
+ self.assertEqual(data, y1 + y2)
+ self.assertIsInstance(dco.unconsumed_tail, bytes)
+ self.assertIsInstance(dco.unused_data, bytes)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_keywords(self):
+ level = 2
+ method = zlib.DEFLATED
+ wbits = -12
+ memLevel = 9
+ strategy = zlib.Z_FILTERED
+ co = zlib.compressobj(level=level,
+ method=method,
+ wbits=wbits,
+ memLevel=memLevel,
+ strategy=strategy,
+ zdict=b"")
+ do = zlib.decompressobj(wbits=wbits, zdict=b"")
+ with self.assertRaises(TypeError):
+ co.compress(data=HAMLET_SCENE)
+ with self.assertRaises(TypeError):
+ do.decompress(data=zlib.compress(HAMLET_SCENE))
+ x = co.compress(HAMLET_SCENE) + co.flush()
+ y = do.decompress(x, max_length=len(HAMLET_SCENE)) + do.flush()
+ self.assertEqual(HAMLET_SCENE, y)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_compressoptions(self):
+ # specify lots of options to compressobj()
+ level = 2
+ method = zlib.DEFLATED
+ wbits = -12
+ memLevel = 9
+ strategy = zlib.Z_FILTERED
+ co = zlib.compressobj(level, method, wbits, memLevel, strategy)
+ x1 = co.compress(HAMLET_SCENE)
+ x2 = co.flush()
+ dco = zlib.decompressobj(wbits)
+ y1 = dco.decompress(x1 + x2)
+ y2 = dco.flush()
+ self.assertEqual(HAMLET_SCENE, y1 + y2)
+
+ @unittest.skip('TODO: RUSTPYTHON')
+ def test_compressincremental(self):
+ # compress object in steps, decompress object as one-shot
+ data = HAMLET_SCENE * 128
+ co = zlib.compressobj()
+ bufs = []
+ for i in range(0, len(data), 256):
+ bufs.append(co.compress(data[i:i+256]))
+ bufs.append(co.flush())
+ combuf = b''.join(bufs)
+
+ dco = zlib.decompressobj()
+ y1 = dco.decompress(b''.join(bufs))
+ y2 = dco.flush()
+ self.assertEqual(data, y1 + y2)
+
+ def test_decompinc(self, flush=False, source=None, cx=256, dcx=64):
+ # compress object in steps, decompress object in steps
+ source = source or HAMLET_SCENE
+ data = source * 128
+ co = zlib.compressobj()
+ bufs = []
+ for i in range(0, len(data), cx):
+ bufs.append(co.compress(data[i:i+cx]))
+ bufs.append(co.flush())
+ combuf = b''.join(bufs)
+
+ decombuf = zlib.decompress(combuf)
+ # Test type of return value
+ self.assertIsInstance(decombuf, bytes)
+
+ self.assertEqual(data, decombuf)
+
+ dco = zlib.decompressobj()
+ bufs = []
+ for i in range(0, len(combuf), dcx):
+ bufs.append(dco.decompress(combuf[i:i+dcx]))
+ self.assertEqual(b'', dco.unconsumed_tail, ########
+ "(A) uct should be b'': not %d long" %
+ len(dco.unconsumed_tail))
+ self.assertEqual(b'', dco.unused_data)
+ if flush:
+ bufs.append(dco.flush())
+ else:
+ while True:
+ chunk = dco.decompress(b'')
+ if chunk:
+ bufs.append(chunk)
+ else:
+ break
+ self.assertEqual(b'', dco.unconsumed_tail, ########
+ "(B) uct should be b'': not %d long" %
+ len(dco.unconsumed_tail))
+ self.assertEqual(b'', dco.unused_data)
+ self.assertEqual(data, b''.join(bufs))
+ # Failure means: "decompressobj with init options failed"
+
+ def test_decompincflush(self):
+ self.test_decompinc(flush=True)
+
+ def test_decompimax(self, source=None, cx=256, dcx=64):
+ # compress in steps, decompress in length-restricted steps
+ source = source or HAMLET_SCENE
+ # Check a decompression object with max_length specified
+ data = source * 128
+ co = zlib.compressobj()
+ bufs = []
+ for i in range(0, len(data), cx):
+ bufs.append(co.compress(data[i:i+cx]))
+ bufs.append(co.flush())
+ combuf = b''.join(bufs)
+ self.assertEqual(data, zlib.decompress(combuf),
+ 'compressed data failure')
+
+ dco = zlib.decompressobj()
+ bufs = []
+ cb = combuf
+ while cb:
+ #max_length = 1 + len(cb)//10
+ chunk = dco.decompress(cb, dcx)
+ self.assertFalse(len(chunk) > dcx,
+ 'chunk too big (%d>%d)' % (len(chunk), dcx))
+ bufs.append(chunk)
+ cb = dco.unconsumed_tail
+ bufs.append(dco.flush())
+ self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
+
+ def test_decompressmaxlen(self, flush=False):
+ # Check a decompression object with max_length specified
+ data = HAMLET_SCENE * 128
+ co = zlib.compressobj()
+ bufs = []
+ for i in range(0, len(data), 256):
+ bufs.append(co.compress(data[i:i+256]))
+ bufs.append(co.flush())
+ combuf = b''.join(bufs)
+ self.assertEqual(data, zlib.decompress(combuf),
+ 'compressed data failure')
+
+ dco = zlib.decompressobj()
+ bufs = []
+ cb = combuf
+ while cb:
+ max_length = 1 + len(cb)//10
+ chunk = dco.decompress(cb, max_length)
+ self.assertFalse(len(chunk) > max_length,
+ 'chunk too big (%d>%d)' % (len(chunk),max_length))
+ bufs.append(chunk)
+ cb = dco.unconsumed_tail
+ if flush:
+ bufs.append(dco.flush())
+ else:
+ while chunk:
+ chunk = dco.decompress(b'', max_length)
+ self.assertFalse(len(chunk) > max_length,
+ 'chunk too big (%d>%d)' % (len(chunk),max_length))
+ bufs.append(chunk)
+ self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
+
+ def test_decompressmaxlenflush(self):
+ self.test_decompressmaxlen(flush=True)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_maxlenmisc(self):
+ # Misc tests of max_length
+ dco = zlib.decompressobj()
+ self.assertRaises(ValueError, dco.decompress, b"", -1)
+ self.assertEqual(b'', dco.unconsumed_tail)
+
+ def test_maxlen_large(self):
+ # Sizes up to sys.maxsize should be accepted, although zlib is
+ # internally limited to expressing sizes with unsigned int
+ data = HAMLET_SCENE * 10
+ self.assertGreater(len(data), zlib.DEF_BUF_SIZE)
+ compressed = zlib.compress(data, 1)
+ dco = zlib.decompressobj()
+ self.assertEqual(dco.decompress(compressed, sys.maxsize), data)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_maxlen_custom(self):
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ dco = zlib.decompressobj()
+ self.assertEqual(dco.decompress(compressed, CustomInt()), data[:100])
+
+ def test_clear_unconsumed_tail(self):
+ # Issue #12050: calling decompress() without providing max_length
+ # should clear the unconsumed_tail attribute.
+ cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc"
+ dco = zlib.decompressobj()
+ ddata = dco.decompress(cdata, 1)
+ ddata += dco.decompress(dco.unconsumed_tail)
+ self.assertEqual(dco.unconsumed_tail, b"")
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_flushes(self):
+ # Test flush() with the various options, using all the
+ # different levels in order to provide more variations.
+ sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH',
+ 'Z_PARTIAL_FLUSH']
+
+ ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.'))
+ # Z_BLOCK has a known failure prior to 1.2.5.3
+ if ver >= (1, 2, 5, 3):
+ sync_opt.append('Z_BLOCK')
+
+ sync_opt = [getattr(zlib, opt) for opt in sync_opt
+ if hasattr(zlib, opt)]
+ data = HAMLET_SCENE * 8
+
+ for sync in sync_opt:
+ for level in range(10):
+ try:
+ obj = zlib.compressobj( level )
+ a = obj.compress( data[:3000] )
+ b = obj.flush( sync )
+ c = obj.compress( data[3000:] )
+ d = obj.flush()
+ except:
+ print("Error for flush mode={}, level={}"
+ .format(sync, level))
+ raise
+ self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
+ data, ("Decompress failed: flush "
+ "mode=%i, level=%i") % (sync, level))
+ del obj
+
+ @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
+ 'requires zlib.Z_SYNC_FLUSH')
+ def test_odd_flush(self):
+ # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1
+ import random
+ # Testing on 17K of "random" data
+
+ # Create compressor and decompressor objects
+ co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
+ dco = zlib.decompressobj()
+
+ # Try 17K of data
+ # generate random data stream
+ try:
+ # In 2.3 and later, WichmannHill is the RNG of the bug report
+ gen = random.WichmannHill()
+ except AttributeError:
+ try:
+ # 2.2 called it Random
+ gen = random.Random()
+ except AttributeError:
+ # others might simply have a single RNG
+ gen = random
+ gen.seed(1)
+ data = genblock(1, 17 * 1024, generator=gen)
+
+ # compress, sync-flush, and decompress
+ first = co.compress(data)
+ second = co.flush(zlib.Z_SYNC_FLUSH)
+ expanded = dco.decompress(first + second)
+
+ # if decompressed data is different from the input data, choke.
+ self.assertEqual(expanded, data, "17K random source doesn't match")
+
+ def test_empty_flush(self):
+ # Test that calling .flush() on unused objects works.
+ # (Bug #1083110 -- calling .flush() on decompress objects
+ # caused a core dump.)
+
+ co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
+ self.assertTrue(co.flush()) # Returns a zlib header
+ dco = zlib.decompressobj()
+ self.assertEqual(dco.flush(), b"") # Returns nothing
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_dictionary(self):
+ h = HAMLET_SCENE
+ # Build a simulated dictionary out of the words in HAMLET.
+ words = h.split()
+ random.shuffle(words)
+ zdict = b''.join(words)
+ # Use it to compress HAMLET.
+ co = zlib.compressobj(zdict=zdict)
+ cd = co.compress(h) + co.flush()
+ # Verify that it will decompress with the dictionary.
+ dco = zlib.decompressobj(zdict=zdict)
+ self.assertEqual(dco.decompress(cd) + dco.flush(), h)
+ # Verify that it fails when not given the dictionary.
+ dco = zlib.decompressobj()
+ self.assertRaises(zlib.error, dco.decompress, cd)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_dictionary_streaming(self):
+ # This simulates the reuse of a compressor object for compressing
+ # several separate data streams.
+ co = zlib.compressobj(zdict=HAMLET_SCENE)
+ do = zlib.decompressobj(zdict=HAMLET_SCENE)
+ piece = HAMLET_SCENE[1000:1500]
+ d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
+ d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
+ d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
+ self.assertEqual(do.decompress(d0), piece)
+ self.assertEqual(do.decompress(d1), piece[100:])
+ self.assertEqual(do.decompress(d2), piece[:-100])
+
+ def test_decompress_incomplete_stream(self):
+ # This is 'foo', deflated
+ x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
+ # For the record
+ self.assertEqual(zlib.decompress(x), b'foo')
+ self.assertRaises(zlib.error, zlib.decompress, x[:-5])
+ # Omitting the stream end works with decompressor objects
+ # (see issue #8672).
+ dco = zlib.decompressobj()
+ y = dco.decompress(x[:-5])
+ y += dco.flush()
+ self.assertEqual(y, b'foo')
+
+ def test_decompress_eof(self):
+ x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
+ dco = zlib.decompressobj()
+ self.assertFalse(dco.eof)
+ dco.decompress(x[:-5])
+ self.assertFalse(dco.eof)
+ dco.decompress(x[-5:])
+ self.assertTrue(dco.eof)
+ dco.flush()
+ self.assertTrue(dco.eof)
+
+ def test_decompress_eof_incomplete_stream(self):
+ x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
+ dco = zlib.decompressobj()
+ self.assertFalse(dco.eof)
+ dco.decompress(x[:-5])
+ self.assertFalse(dco.eof)
+ dco.flush()
+ self.assertFalse(dco.eof)
+
+ def test_decompress_unused_data(self):
+ # Repeated calls to decompress() after EOF should accumulate data in
+ # dco.unused_data, instead of just storing the arg to the last call.
+ source = b'abcdefghijklmnopqrstuvwxyz'
+ remainder = b'0123456789'
+ y = zlib.compress(source)
+ x = y + remainder
+ for maxlen in 0, 1000:
+ for step in 1, 2, len(y), len(x):
+ dco = zlib.decompressobj()
+ data = b''
+ for i in range(0, len(x), step):
+ if i < len(y):
+ self.assertEqual(dco.unused_data, b'')
+ if maxlen == 0:
+ data += dco.decompress(x[i : i + step])
+ self.assertEqual(dco.unconsumed_tail, b'')
+ else:
+ data += dco.decompress(
+ dco.unconsumed_tail + x[i : i + step], maxlen)
+ data += dco.flush()
+ self.assertTrue(dco.eof)
+ self.assertEqual(data, source)
+ self.assertEqual(dco.unconsumed_tail, b'')
+ self.assertEqual(dco.unused_data, remainder)
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ # issue27164
+ def test_decompress_raw_with_dictionary(self):
+ zdict = b'abcdefghijklmnopqrstuvwxyz'
+ co = zlib.compressobj(wbits=-zlib.MAX_WBITS, zdict=zdict)
+ comp = co.compress(zdict) + co.flush()
+ dco = zlib.decompressobj(wbits=-zlib.MAX_WBITS, zdict=zdict)
+ uncomp = dco.decompress(comp) + dco.flush()
+ self.assertEqual(zdict, uncomp)
+
+ def test_flush_with_freed_input(self):
+ # Issue #16411: decompressor accesses input to last decompress() call
+ # in flush(), even if this object has been freed in the meanwhile.
+ input1 = b'abcdefghijklmnopqrstuvwxyz'
+ input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM'
+ data = zlib.compress(input1)
+ dco = zlib.decompressobj()
+ dco.decompress(data, 1)
+ del data
+ data = zlib.compress(input2)
+ self.assertEqual(dco.flush(), input1[1:])
+
+ @bigmemtest(size=_4G, memuse=1)
+ def test_flush_large_length(self, size):
+ # Test flush(length) parameter greater than internal limit UINT_MAX
+ input = HAMLET_SCENE * 10
+ data = zlib.compress(input, 1)
+ dco = zlib.decompressobj()
+ dco.decompress(data, 1)
+ self.assertEqual(dco.flush(size), input[1:])
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_flush_custom_length(self):
+ input = HAMLET_SCENE * 10
+ data = zlib.compress(input, 1)
+ dco = zlib.decompressobj()
+ dco.decompress(data, 1)
+ self.assertEqual(dco.flush(CustomInt()), input[1:])
+
+ @requires_Compress_copy
+ def test_compresscopy(self):
+ # Test copying a compression object
+ data0 = HAMLET_SCENE
+ data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii")
+ for func in lambda c: c.copy(), copy.copy, copy.deepcopy:
+ c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
+ bufs0 = []
+ bufs0.append(c0.compress(data0))
+
+ c1 = func(c0)
+ bufs1 = bufs0[:]
+
+ bufs0.append(c0.compress(data0))
+ bufs0.append(c0.flush())
+ s0 = b''.join(bufs0)
+
+ bufs1.append(c1.compress(data1))
+ bufs1.append(c1.flush())
+ s1 = b''.join(bufs1)
+
+ self.assertEqual(zlib.decompress(s0),data0+data0)
+ self.assertEqual(zlib.decompress(s1),data0+data1)
+
+ @requires_Compress_copy
+ def test_badcompresscopy(self):
+ # Test copying a compression object in an inconsistent state
+ c = zlib.compressobj()
+ c.compress(HAMLET_SCENE)
+ c.flush()
+ self.assertRaises(ValueError, c.copy)
+ self.assertRaises(ValueError, copy.copy, c)
+ self.assertRaises(ValueError, copy.deepcopy, c)
+
+ @requires_Decompress_copy
+ def test_decompresscopy(self):
+ # Test copying a decompression object
+ data = HAMLET_SCENE
+ comp = zlib.compress(data)
+ # Test type of return value
+ self.assertIsInstance(comp, bytes)
+
+ for func in lambda c: c.copy(), copy.copy, copy.deepcopy:
+ d0 = zlib.decompressobj()
+ bufs0 = []
+ bufs0.append(d0.decompress(comp[:32]))
+
+ d1 = func(d0)
+ bufs1 = bufs0[:]
+
+ bufs0.append(d0.decompress(comp[32:]))
+ s0 = b''.join(bufs0)
+
+ bufs1.append(d1.decompress(comp[32:]))
+ s1 = b''.join(bufs1)
+
+ self.assertEqual(s0,s1)
+ self.assertEqual(s0,data)
+
+ @requires_Decompress_copy
+ def test_baddecompresscopy(self):
+ # Test copying a compression object in an inconsistent state
+ data = zlib.compress(HAMLET_SCENE)
+ d = zlib.decompressobj()
+ d.decompress(data)
+ d.flush()
+ self.assertRaises(ValueError, d.copy)
+ self.assertRaises(ValueError, copy.copy, d)
+ self.assertRaises(ValueError, copy.deepcopy, d)
+
+ def test_compresspickle(self):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ with self.assertRaises((TypeError, pickle.PicklingError)):
+ pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto)
+
+ def test_decompresspickle(self):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ with self.assertRaises((TypeError, pickle.PicklingError)):
+ pickle.dumps(zlib.decompressobj(), proto)
+
+ # Memory use of the following functions takes into account overallocation
+
+ @unittest.skip('TODO: RUSTPYTHON')
+ @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
+ def test_big_compress_buffer(self, size):
+ c = zlib.compressobj(1)
+ compress = lambda s: c.compress(s) + c.flush()
+ self.check_big_compress_buffer(size, compress)
+
+ @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
+ def test_big_decompress_buffer(self, size):
+ d = zlib.decompressobj()
+ decompress = lambda s: d.decompress(s) + d.flush()
+ self.check_big_decompress_buffer(size, decompress)
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @bigmemtest(size=_4G + 100, memuse=4)
+ def test_64bit_compress(self, size):
+ data = b'x' * size
+ co = zlib.compressobj(0)
+ do = zlib.decompressobj()
+ try:
+ comp = co.compress(data) + co.flush()
+ uncomp = do.decompress(comp) + do.flush()
+ self.assertEqual(uncomp, data)
+ finally:
+ comp = uncomp = data = None
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @bigmemtest(size=_4G + 100, memuse=3)
+ def test_large_unused_data(self, size):
+ data = b'abcdefghijklmnop'
+ unused = b'x' * size
+ comp = zlib.compress(data) + unused
+ do = zlib.decompressobj()
+ try:
+ uncomp = do.decompress(comp) + do.flush()
+ self.assertEqual(unused, do.unused_data)
+ self.assertEqual(uncomp, data)
+ finally:
+ unused = comp = do = None
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @bigmemtest(size=_4G + 100, memuse=5)
+ def test_large_unconsumed_tail(self, size):
+ data = b'x' * size
+ do = zlib.decompressobj()
+ try:
+ comp = zlib.compress(data, 0)
+ uncomp = do.decompress(comp, 1) + do.flush()
+ self.assertEqual(uncomp, data)
+ self.assertEqual(do.unconsumed_tail, b'')
+ finally:
+ comp = uncomp = data = None
+
+ # TODO: RUSTPYTHON
+ @unittest.expectedFailure
+ def test_wbits(self):
+ # wbits=0 only supported since zlib v1.2.3.5
+ # Register "1.2.3" as "1.2.3.0"
+ # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux"
+ v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.')
+ if len(v) < 4:
+ v.append('0')
+ elif not v[-1].isnumeric():
+ v[-1] = '0'
+
+ v = tuple(map(int, v))
+ supports_wbits_0 = v >= (1, 2, 3, 5)
+
+ co = zlib.compressobj(level=1, wbits=15)
+ zlib15 = co.compress(HAMLET_SCENE) + co.flush()
+ self.assertEqual(zlib.decompress(zlib15, 15), HAMLET_SCENE)
+ if supports_wbits_0:
+ self.assertEqual(zlib.decompress(zlib15, 0), HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(zlib15, 32 + 15), HAMLET_SCENE)
+ with self.assertRaisesRegex(zlib.error, 'invalid window size'):
+ zlib.decompress(zlib15, 14)
+ dco = zlib.decompressobj(wbits=32 + 15)
+ self.assertEqual(dco.decompress(zlib15), HAMLET_SCENE)
+ dco = zlib.decompressobj(wbits=14)
+ with self.assertRaisesRegex(zlib.error, 'invalid window size'):
+ dco.decompress(zlib15)
+
+ co = zlib.compressobj(level=1, wbits=9)
+ zlib9 = co.compress(HAMLET_SCENE) + co.flush()
+ self.assertEqual(zlib.decompress(zlib9, 9), HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(zlib9, 15), HAMLET_SCENE)
+ if supports_wbits_0:
+ self.assertEqual(zlib.decompress(zlib9, 0), HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(zlib9, 32 + 9), HAMLET_SCENE)
+ dco = zlib.decompressobj(wbits=32 + 9)
+ self.assertEqual(dco.decompress(zlib9), HAMLET_SCENE)
+
+ co = zlib.compressobj(level=1, wbits=-15)
+ deflate15 = co.compress(HAMLET_SCENE) + co.flush()
+ self.assertEqual(zlib.decompress(deflate15, -15), HAMLET_SCENE)
+ dco = zlib.decompressobj(wbits=-15)
+ self.assertEqual(dco.decompress(deflate15), HAMLET_SCENE)
+
+ co = zlib.compressobj(level=1, wbits=-9)
+ deflate9 = co.compress(HAMLET_SCENE) + co.flush()
+ self.assertEqual(zlib.decompress(deflate9, -9), HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(deflate9, -15), HAMLET_SCENE)
+ dco = zlib.decompressobj(wbits=-9)
+ self.assertEqual(dco.decompress(deflate9), HAMLET_SCENE)
+
+ co = zlib.compressobj(level=1, wbits=16 + 15)
+ gzip = co.compress(HAMLET_SCENE) + co.flush()
+ self.assertEqual(zlib.decompress(gzip, 16 + 15), HAMLET_SCENE)
+ self.assertEqual(zlib.decompress(gzip, 32 + 15), HAMLET_SCENE)
+ dco = zlib.decompressobj(32 + 15)
+ self.assertEqual(dco.decompress(gzip), HAMLET_SCENE)
+
+
+def genblock(seed, length, step=1024, generator=random):
+ """length-byte stream of random data from a seed (in step-byte blocks)."""
+ if seed is not None:
+ generator.seed(seed)
+ randint = generator.randint
+ if length < step or step < 2:
+ step = length
+ blocks = bytes()
+ for i in range(0, length, step):
+ blocks += bytes(randint(0, 255) for x in range(step))
+ return blocks
+
+
+
+def choose_lines(source, number, seed=None, generator=random):
+ """Return a list of number lines randomly chosen from the source"""
+ if seed is not None:
+ generator.seed(seed)
+ sources = source.split('\n')
+ return [generator.choice(sources) for n in range(number)]
+
+
+
+HAMLET_SCENE = b"""
+LAERTES
+
+ O, fear me not.
+ I stay too long: but here my father comes.
+
+ Enter POLONIUS
+
+ A double blessing is a double grace,
+ Occasion smiles upon a second leave.
+
+LORD POLONIUS
+
+ Yet here, Laertes! aboard, aboard, for shame!
+ The wind sits in the shoulder of your sail,
+ And you are stay'd for. There; my blessing with thee!
+ And these few precepts in thy memory
+ See thou character. Give thy thoughts no tongue,
+ Nor any unproportioned thought his act.
+ Be thou familiar, but by no means vulgar.
+ Those friends thou hast, and their adoption tried,
+ Grapple them to thy soul with hoops of steel;
+ But do not dull thy palm with entertainment
+ Of each new-hatch'd, unfledged comrade. Beware
+ Of entrance to a quarrel, but being in,
+ Bear't that the opposed may beware of thee.
+ Give every man thy ear, but few thy voice;
+ Take each man's censure, but reserve thy judgment.
+ Costly thy habit as thy purse can buy,
+ But not express'd in fancy; rich, not gaudy;
+ For the apparel oft proclaims the man,
+ And they in France of the best rank and station
+ Are of a most select and generous chief in that.
+ Neither a borrower nor a lender be;
+ For loan oft loses both itself and friend,
+ And borrowing dulls the edge of husbandry.
+ This above all: to thine ownself be true,
+ And it must follow, as the night the day,
+ Thou canst not then be false to any man.
+ Farewell: my blessing season this in thee!
+
+LAERTES
+
+ Most humbly do I take my leave, my lord.
+
+LORD POLONIUS
+
+ The time invites you; go; your servants tend.
+
+LAERTES
+
+ Farewell, Ophelia; and remember well
+ What I have said to you.
+
+OPHELIA
+
+ 'Tis in my memory lock'd,
+ And you yourself shall keep the key of it.
+
+LAERTES
+
+ Farewell.
+"""
+
+
+class CustomInt:
+ def __index__(self):
+ return 100
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py
index c5b675d16..8cd901f13 100644
--- a/Lib/urllib/error.py
+++ b/Lib/urllib/error.py
@@ -16,14 +16,10 @@ import urllib.response
__all__ = ['URLError', 'HTTPError', 'ContentTooShortError']
-# do these error classes make sense?
-# make sure all of the OSError stuff is overridden. we just want to be
-# subtypes.
-
class URLError(OSError):
# URLError is a sub-type of OSError, but it doesn't share any of
# the implementation. need to override __init__ and __str__.
- # It sets self.args for compatibility with other EnvironmentError
+ # It sets self.args for compatibility with other OSError
# subclasses, but args doesn't have the typical format with errno in
# slot 0 and strerror in slot 1. This may be better than nothing.
def __init__(self, reason, filename=None):
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index 958767a08..e2b6f133e 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -30,6 +30,7 @@ test_urlparse.py provides a good indicator of parsing behavior.
import re
import sys
import collections
+import warnings
__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
"urlsplit", "urlunsplit", "urlencode", "parse_qs",
@@ -38,29 +39,37 @@ __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
"DefragResult", "ParseResult", "SplitResult",
"DefragResultBytes", "ParseResultBytes", "SplitResultBytes"]
-# A classification of schemes ('' means apply by default)
-uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap',
+# A classification of schemes.
+# The empty string classifies URLs with no scheme specified,
+# being the default value returned by “urlsplit” and “urlparse”.
+
+uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap',
'wais', 'file', 'https', 'shttp', 'mms',
- 'prospero', 'rtsp', 'rtspu', '', 'sftp',
+ 'prospero', 'rtsp', 'rtspu', 'sftp',
'svn', 'svn+ssh', 'ws', 'wss']
-uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet',
+
+uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet',
'imap', 'wais', 'file', 'mms', 'https', 'shttp',
- 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '',
+ 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync',
'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh',
'ws', 'wss']
-uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap',
+
+uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap',
'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips',
- 'mms', '', 'sftp', 'tel']
+ 'mms', 'sftp', 'tel']
# These are not actually used anymore, but should stay for backwards
# compatibility. (They are undocumented, but have a public-looking name.)
+
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
'telnet', 'wais', 'imap', 'snews', 'sip', 'sips']
-uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms',
- 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', '']
-uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news',
+
+uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms',
+ 'gopher', 'rtsp', 'rtspu', 'sip', 'sips']
+
+uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news',
'nntp', 'wais', 'https', 'shttp', 'snews',
- 'file', 'prospero', '']
+ 'file', 'prospero']
# Characters valid in scheme names
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
@@ -147,16 +156,22 @@ class _NetlocResultMixinBase(object):
def hostname(self):
hostname = self._hostinfo[0]
if not hostname:
- hostname = None
- elif hostname is not None:
- hostname = hostname.lower()
- return hostname
+ return None
+ # Scoped IPv6 address may have zone info, which must not be lowercased
+ # like http://[fe80::822a:a8ff:fe49:470c%tESt]:1234/keys
+ separator = '%' if isinstance(hostname, str) else b'%'
+ hostname, percent, zone = hostname.partition(separator)
+ return hostname.lower() + percent + zone
@property
def port(self):
port = self._hostinfo[1]
if port is not None:
- port = int(port, 10)
+ try:
+ port = int(port, 10)
+ except ValueError:
+ message = f'Port could not be cast to integer value as {port!r}'
+ raise ValueError(message) from None
if not ( 0 <= port <= 65535):
raise ValueError("Port out of range 0-65535")
return port
@@ -274,7 +289,7 @@ by reference to a primary resource and additional identifying information.
"""
_ParseResultBase.__doc__ = """
-ParseResult(scheme, netloc, path, params, query, fragment)
+ParseResult(scheme, netloc, path, params, query, fragment)
A 6-tuple that contains components of a parsed URL.
"""
@@ -381,6 +396,24 @@ def _splitnetloc(url, start=0):
delim = min(delim, wdelim) # use earliest delim position
return url[start:delim], url[delim:] # return (domain, rest)
+def _checknetloc(netloc):
+ if not netloc or netloc.isascii():
+ return
+ # looking for characters like \u2100 that expand to 'a/c'
+ # IDNA uses NFKC equivalence, so normalize for this check
+ import unicodedata
+ n = netloc.replace('@', '') # ignore characters already included
+ n = n.replace(':', '') # but not the surrounding text
+ n = n.replace('#', '')
+ n = n.replace('?', '')
+ netloc2 = unicodedata.normalize('NFKC', n)
+ if n == netloc2:
+ return
+ for c in '/?#@:':
+ if c in netloc2:
+ raise ValueError("netloc '" + netloc + "' contains invalid " +
+ "characters under NFKC normalization")
+
def urlsplit(url, scheme='', allow_fragments=True):
"""Parse a URL into 5 components:
:///?#
@@ -399,7 +432,6 @@ def urlsplit(url, scheme='', allow_fragments=True):
i = url.find(':')
if i > 0:
if url[:i] == 'http': # optimize the common case
- scheme = url[:i].lower()
url = url[i+1:]
if url[:2] == '//':
netloc, url = _splitnetloc(url, 2)
@@ -410,7 +442,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
url, fragment = url.split('#', 1)
if '?' in url:
url, query = url.split('?', 1)
- v = SplitResult(scheme, netloc, url, query, fragment)
+ _checknetloc(netloc)
+ v = SplitResult('http', netloc, url, query, fragment)
_parse_cache[key] = v
return _coerce_result(v)
for c in url[:i]:
@@ -433,6 +466,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
url, fragment = url.split('#', 1)
if '?' in url:
url, query = url.split('?', 1)
+ _checknetloc(netloc)
v = SplitResult(scheme, netloc, url, query, fragment)
_parse_cache[key] = v
return _coerce_result(v)
@@ -574,7 +608,7 @@ def unquote_to_bytes(string):
# if the function is never called
global _hextobyte
if _hextobyte is None:
- _hextobyte = {(a + b).encode(): bytes([int(a + b, 16)])
+ _hextobyte = {(a + b).encode(): bytes.fromhex(a + b)
for a in _hexdig for b in _hexdig}
for item in bits[1:]:
try:
@@ -612,8 +646,9 @@ def unquote(string, encoding='utf-8', errors='replace'):
append(bits[i + 1])
return ''.join(res)
+
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
- encoding='utf-8', errors='replace'):
+ encoding='utf-8', errors='replace', max_num_fields=None):
"""Parse a query given as a string argument.
Arguments:
@@ -633,10 +668,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
encoding and errors: specify how to decode percent-encoded sequences
into Unicode characters, as accepted by the bytes.decode() method.
+
+ max_num_fields: int. If set, then throws a ValueError if there
+ are more than n fields read by parse_qsl().
+
+ Returns a dictionary.
"""
parsed_result = {}
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
- encoding=encoding, errors=errors)
+ encoding=encoding, errors=errors,
+ max_num_fields=max_num_fields)
for name, value in pairs:
if name in parsed_result:
parsed_result[name].append(value)
@@ -644,30 +685,43 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
parsed_result[name] = [value]
return parsed_result
+
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
- encoding='utf-8', errors='replace'):
+ encoding='utf-8', errors='replace', max_num_fields=None):
"""Parse a query given as a string argument.
- Arguments:
+ Arguments:
- qs: percent-encoded query string to be parsed
+ qs: percent-encoded query string to be parsed
- keep_blank_values: flag indicating whether blank values in
- percent-encoded queries should be treated as blank strings. A
- true value indicates that blanks should be retained as blank
- strings. The default false value indicates that blank values
- are to be ignored and treated as if they were not included.
+ keep_blank_values: flag indicating whether blank values in
+ percent-encoded queries should be treated as blank strings.
+ A true value indicates that blanks should be retained as blank
+ strings. The default false value indicates that blank values
+ are to be ignored and treated as if they were not included.
- strict_parsing: flag indicating what to do with parsing errors. If
- false (the default), errors are silently ignored. If true,
- errors raise a ValueError exception.
+ strict_parsing: flag indicating what to do with parsing errors. If
+ false (the default), errors are silently ignored. If true,
+ errors raise a ValueError exception.
- encoding and errors: specify how to decode percent-encoded sequences
- into Unicode characters, as accepted by the bytes.decode() method.
+ encoding and errors: specify how to decode percent-encoded sequences
+ into Unicode characters, as accepted by the bytes.decode() method.
- Returns a list, as G-d intended.
+ max_num_fields: int. If set, then throws a ValueError
+ if there are more than n fields read by parse_qsl().
+
+ Returns a list, as G-d intended.
"""
qs, _coerce_result = _coerce_args(qs)
+
+ # If max_num_fields is defined then check that the number of fields
+ # is less than max_num_fields. This prevents a memory exhaustion DOS
+ # attack via post bodies with many fields.
+ if max_num_fields is not None:
+ num_fields = 1 + qs.count('&') + qs.count(';')
+ if max_num_fields < num_fields:
+ raise ValueError('Max number of fields exceeded')
+
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
r = []
for name_value in pairs:
@@ -704,7 +758,7 @@ def unquote_plus(string, encoding='utf-8', errors='replace'):
_ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
b'abcdefghijklmnopqrstuvwxyz'
b'0123456789'
- b'_.-')
+ b'_.-~')
_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE)
_safe_quoters = {}
@@ -734,22 +788,32 @@ def quote(string, safe='/', encoding=None, errors=None):
"""quote('abc def') -> 'abc%20def'
Each part of a URL, e.g. the path info, the query, etc., has a
- different set of reserved characters that must be quoted.
+ different set of reserved characters that must be quoted. The
+ quote function offers a cautious (not minimal) way to quote a
+ string for most of these parts.
- RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
- the following reserved characters.
+ RFC 3986 Uniform Resource Identifier (URI): Generic Syntax lists
+ the following (un)reserved characters.
- reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
- "$" | ","
+ unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ reserved = gen-delims / sub-delims
+ gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+ sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
+ / "*" / "+" / "," / ";" / "="
- Each of these characters is reserved in some component of a URL,
+ Each of the reserved characters is reserved in some component of a URL,
but not necessarily in all of them.
- By default, the quote function is intended for quoting the path
- section of a URL. Thus, it will not encode '/'. This character
- is reserved, but in typical usage the quote function is being
- called on a path where the existing slash characters are used as
- reserved characters.
+ The quote function %-escapes all characters that are neither in the
+ unreserved chars ("always safe") nor the additional chars set via the
+ safe arg.
+
+ The default for the safe arg is '/'. The character is reserved, but in
+ typical usage the quote function is being called on a path where the
+ existing slash characters are to be preserved.
+
+ Python 3.7 updates from using RFC 2396 to RFC 3986 to quote URL strings.
+ Now, "~" is included in the set of unreserved characters.
string and safe may be either str or bytes objects. encoding and errors
must not be specified if string is a bytes object.
@@ -893,7 +957,14 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None,
l.append(k + '=' + elt)
return '&'.join(l)
+
def to_bytes(url):
+ warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8",
+ DeprecationWarning, stacklevel=2)
+ return _to_bytes(url)
+
+
+def _to_bytes(url):
"""to_bytes(u"URL") --> 'URL'."""
# Most URL schemes require ASCII. If that changes, the conversion
# can be relaxed.
@@ -906,16 +977,29 @@ def to_bytes(url):
" contains non-ASCII characters")
return url
+
def unwrap(url):
- """unwrap('') --> 'type://host/path'."""
+ """Transform a string like '' into 'scheme://host/path'.
+
+ The string is returned unchanged if it's not a wrapped URL.
+ """
url = str(url).strip()
if url[:1] == '<' and url[-1:] == '>':
url = url[1:-1].strip()
- if url[:4] == 'URL:': url = url[4:].strip()
+ if url[:4] == 'URL:':
+ url = url[4:].strip()
return url
-_typeprog = None
+
def splittype(url):
+ warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splittype(url)
+
+
+_typeprog = None
+def _splittype(url):
"""splittype('type:opaquestring') --> 'type', 'opaquestring'."""
global _typeprog
if _typeprog is None:
@@ -927,12 +1011,20 @@ def splittype(url):
return scheme.lower(), data
return None, url
-_hostprog = None
+
def splithost(url):
+ warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splithost(url)
+
+
+_hostprog = None
+def _splithost(url):
"""splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
global _hostprog
if _hostprog is None:
- _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL)
+ _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL)
match = _hostprog.match(url)
if match:
@@ -942,32 +1034,64 @@ def splithost(url):
return host_port, path
return None, url
+
def splituser(host):
+ warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splituser(host)
+
+
+def _splituser(host):
"""splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
user, delim, host = host.rpartition('@')
return (user if delim else None), host
+
def splitpasswd(user):
+ warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitpasswd(user)
+
+
+def _splitpasswd(user):
"""splitpasswd('user:passwd') -> 'user', 'passwd'."""
user, delim, passwd = user.partition(':')
return user, (passwd if delim else None)
+
+def splitport(host):
+ warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitport(host)
+
+
# splittag('/path#tag') --> '/path', 'tag'
_portprog = None
-def splitport(host):
+def _splitport(host):
"""splitport('host:port') --> 'host', 'port'."""
global _portprog
if _portprog is None:
- _portprog = re.compile('(.*):([0-9]*)$', re.DOTALL)
+ _portprog = re.compile('(.*):([0-9]*)', re.DOTALL)
- match = _portprog.match(host)
+ match = _portprog.fullmatch(host)
if match:
host, port = match.groups()
if port:
return host, port
return host, None
+
def splitnport(host, defport=-1):
+ warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitnport(host, defport)
+
+
+def _splitnport(host, defport=-1):
"""Split host and port, returning numeric port.
Return given default port if no ':' found; defaults to -1.
Return numerical port if a valid number are found after ':'.
@@ -983,27 +1107,59 @@ def splitnport(host, defport=-1):
return host, nport
return host, defport
+
def splitquery(url):
+ warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitquery(url)
+
+
+def _splitquery(url):
"""splitquery('/path?query') --> '/path', 'query'."""
path, delim, query = url.rpartition('?')
if delim:
return path, query
return url, None
+
def splittag(url):
+ warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splittag(url)
+
+
+def _splittag(url):
"""splittag('/path#tag') --> '/path', 'tag'."""
path, delim, tag = url.rpartition('#')
if delim:
return path, tag
return url, None
+
def splitattr(url):
+ warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, "
+ "use urllib.parse.urlparse() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitattr(url)
+
+
+def _splitattr(url):
"""splitattr('/path;attr1=value1;attr2=value2;...') ->
'/path', ['attr1=value1', 'attr2=value2', ...]."""
words = url.split(';')
return words[0], words[1:]
+
def splitvalue(attr):
+ warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, "
+ "use urllib.parse.parse_qsl() instead",
+ DeprecationWarning, stacklevel=2)
+ return _splitvalue(attr)
+
+
+def _splitvalue(attr):
"""splitvalue('attr=value') --> 'attr', 'value'."""
attr, delim, value = attr.partition('=')
return attr, (value if delim else None)
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 5f15b74f4..e44073886 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -94,7 +94,6 @@ import socket
import string
import sys
import time
-import collections
import tempfile
import contextlib
import warnings
@@ -103,8 +102,8 @@ import warnings
from urllib.error import URLError, HTTPError, ContentTooShortError
from urllib.parse import (
urlparse, urlsplit, urljoin, unwrap, quote, unquote,
- splittype, splithost, splitport, splituser, splitpasswd,
- splitattr, splitquery, splitvalue, splittag, to_bytes,
+ _splittype, _splithost, _splitport, _splituser, _splitpasswd,
+ _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes,
unquote_to_bytes, urlunparse)
from urllib.response import addinfourl, addclosehook
@@ -199,7 +198,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
global _opener
if cafile or capath or cadefault:
import warnings
- warnings.warn("cafile, cpath and cadefault are deprecated, use a "
+ warnings.warn("cafile, capath and cadefault are deprecated, use a "
"custom context instead.", DeprecationWarning, 2)
if context is not None:
raise ValueError(
@@ -243,7 +242,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None):
Returns a tuple containing the path to the newly created
data file as well as the resulting HTTPMessage object.
"""
- url_type, path = splittype(url)
+ url_type, path = _splittype(url)
with contextlib.closing(urlopen(url, data)) as fp:
headers = fp.info()
@@ -351,7 +350,7 @@ class Request:
def full_url(self, url):
# unwrap('') --> 'type://host/path'
self._full_url = unwrap(url)
- self._full_url, self.fragment = splittag(self._full_url)
+ self._full_url, self.fragment = _splittag(self._full_url)
self._parse()
@full_url.deleter
@@ -379,10 +378,10 @@ class Request:
self.data = None
def _parse(self):
- self.type, rest = splittype(self._full_url)
+ self.type, rest = _splittype(self._full_url)
if self.type is None:
raise ValueError("unknown url type: %r" % self.full_url)
- self.host, self.selector = splithost(rest)
+ self.host, self.selector = _splithost(rest)
if self.host:
self.host = unquote(self.host)
@@ -427,8 +426,7 @@ class Request:
self.unredirected_hdrs.pop(header_name, None)
def header_items(self):
- hdrs = self.unredirected_hdrs.copy()
- hdrs.update(self.headers)
+ hdrs = {**self.unredirected_hdrs, **self.headers}
return list(hdrs.items())
class OpenerDirector:
@@ -523,6 +521,7 @@ class OpenerDirector:
meth = getattr(processor, meth_name)
req = meth(req)
+ sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
response = self._open(req, data)
# post-process response
@@ -684,8 +683,8 @@ class HTTPRedirectHandler(BaseHandler):
newurl = newurl.replace(' ', '%20')
CONTENT_HEADERS = ("content-length", "content-type")
- newheaders = dict((k, v) for k, v in req.headers.items()
- if k.lower() not in CONTENT_HEADERS)
+ newheaders = {k: v for k, v in req.headers.items()
+ if k.lower() not in CONTENT_HEADERS}
return Request(newurl,
headers=newheaders,
origin_req_host=req.origin_req_host,
@@ -769,7 +768,7 @@ def _parse_proxy(proxy):
According to RFC 3986, having an authority component means the URL must
have two slashes after the scheme.
"""
- scheme, r_scheme = splittype(proxy)
+ scheme, r_scheme = _splittype(proxy)
if not r_scheme.startswith("/"):
# authority
scheme = None
@@ -784,9 +783,9 @@ def _parse_proxy(proxy):
if end == -1:
end = None
authority = r_scheme[2:end]
- userinfo, hostport = splituser(authority)
+ userinfo, hostport = _splituser(authority)
if userinfo is not None:
- user, password = splitpasswd(userinfo)
+ user, password = _splitpasswd(userinfo)
else:
user = password = None
return scheme, user, password, hostport
@@ -801,6 +800,7 @@ class ProxyHandler(BaseHandler):
assert hasattr(proxies, 'keys'), "proxies must be a mapping"
self.proxies = proxies
for type, url in proxies.items():
+ type = type.lower()
setattr(self, '%s_open' % type,
lambda r, proxy=url, type=type, meth=self.proxy_open:
meth(r, proxy, type))
@@ -846,7 +846,7 @@ class HTTPPasswordMgr:
self.passwd[realm] = {}
for default_port in True, False:
reduced_uri = tuple(
- [self.reduce_uri(u, default_port) for u in uri])
+ self.reduce_uri(u, default_port) for u in uri)
self.passwd[realm][reduced_uri] = (user, passwd)
def find_user_password(self, realm, authuri):
@@ -873,7 +873,7 @@ class HTTPPasswordMgr:
scheme = None
authority = uri
path = '/'
- host, port = splitport(authority)
+ host, port = _splitport(authority)
if default_port and port is None and scheme is not None:
dport = {"http": 80,
"https": 443,
@@ -945,8 +945,15 @@ class AbstractBasicAuthHandler:
# allow for double- and single-quoted realm values
# (single quotes are a violation of the RFC, but appear in the wild)
- rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
- 'realm=(["\']?)([^"\']*)\\2', re.I)
+ rx = re.compile('(?:^|,)' # start of the string or ','
+ '[ \t]*' # optional whitespaces
+ '([^ \t]+)' # scheme like "Basic"
+ '[ \t]+' # mandatory whitespaces
+ # realm=xxx
+ # realm='xxx'
+ # realm="xxx"
+ 'realm=(["\']?)([^"\']*)\\2',
+ re.I)
# XXX could pre-emptively send auth info already accepted (RFC 2617,
# end of section 2, and section 1.2 immediately after "credentials"
@@ -958,27 +965,51 @@ class AbstractBasicAuthHandler:
self.passwd = password_mgr
self.add_password = self.passwd.add_password
+ def _parse_realm(self, header):
+ # parse WWW-Authenticate header: accept multiple challenges per header
+ found_challenge = False
+ for mo in AbstractBasicAuthHandler.rx.finditer(header):
+ scheme, quote, realm = mo.groups()
+ if quote not in ['"', "'"]:
+ warnings.warn("Basic Auth Realm was unquoted",
+ UserWarning, 3)
+
+ yield (scheme, realm)
+
+ found_challenge = True
+
+ if not found_challenge:
+ if header:
+ scheme = header.split()[0]
+ else:
+ scheme = ''
+ yield (scheme, None)
+
def http_error_auth_reqed(self, authreq, host, req, headers):
# host may be an authority (without userinfo) or a URL with an
# authority
- # XXX could be multiple headers
- authreq = headers.get(authreq, None)
+ headers = headers.get_all(authreq)
+ if not headers:
+ # no header found
+ return
- if authreq:
- scheme = authreq.split()[0]
- if scheme.lower() != 'basic':
- raise ValueError("AbstractBasicAuthHandler does not"
- " support the following scheme: '%s'" %
- scheme)
- else:
- mo = AbstractBasicAuthHandler.rx.search(authreq)
- if mo:
- scheme, quote, realm = mo.groups()
- if quote not in ['"',"'"]:
- warnings.warn("Basic Auth Realm was unquoted",
- UserWarning, 2)
- if scheme.lower() == 'basic':
- return self.retry_http_basic_auth(host, req, realm)
+ unsupported = None
+ for header in headers:
+ for scheme, realm in self._parse_realm(header):
+ if scheme.lower() != 'basic':
+ unsupported = scheme
+ continue
+
+ if realm is not None:
+ # Use the first matching Basic challenge.
+ # Ignore following challenges even if they use the Basic
+ # scheme.
+ return self.retry_http_basic_auth(host, req, realm)
+
+ if unsupported is not None:
+ raise ValueError("AbstractBasicAuthHandler does not "
+ "support the following scheme: %r"
+ % (scheme,))
def retry_http_basic_auth(self, host, req, realm):
user, pw = self.passwd.find_user_password(realm, host)
@@ -1144,7 +1175,11 @@ class AbstractDigestAuthHandler:
A2 = "%s:%s" % (req.get_method(),
# XXX selector: what about proxies and full urls
req.selector)
- if qop == 'auth':
+ # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth`
+ # or `auth-int` to the response back. we use `auth` to send the response back.
+ if qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ elif 'auth' in qop.split(','):
if nonce == self.last_nonce:
self.nonce_count += 1
else:
@@ -1152,10 +1187,8 @@ class AbstractDigestAuthHandler:
self.last_nonce = nonce
ncvalue = '%08x' % self.nonce_count
cnonce = self.get_cnonce(nonce)
- noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+ noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2))
respdig = KD(H(A1), noncebit)
- elif qop is None:
- respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
else:
# XXX handle auth-int.
raise URLError("qop '%s' is not supported." % qop)
@@ -1262,8 +1295,8 @@ class AbstractHTTPHandler(BaseHandler):
sel_host = host
if request.has_proxy():
- scheme, sel = splittype(request.selector)
- sel_host, sel_path = splithost(sel)
+ scheme, sel = _splittype(request.selector)
+ sel_host, sel_path = _splithost(sel)
if not request.has_header('Host'):
request.add_unredirected_header('Host', sel_host)
for name, value in self.parent.addheaders:
@@ -1287,8 +1320,8 @@ class AbstractHTTPHandler(BaseHandler):
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
- headers.update(dict((k, v) for k, v in req.headers.items()
- if k not in headers))
+ headers.update({k: v for k, v in req.headers.items()
+ if k not in headers})
# TODO(jhylton): Should this be redesigned to handle
# persistent connections?
@@ -1300,7 +1333,7 @@ class AbstractHTTPHandler(BaseHandler):
# So make sure the connection gets closed after the (only)
# request.
headers["Connection"] = "close"
- headers = dict((name.title(), val) for name, val in headers.items())
+ headers = {name.title(): val for name, val in headers.items()}
if req._tunnel_host:
tunnel_headers = {}
@@ -1479,7 +1512,7 @@ class FileHandler(BaseHandler):
'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
(mtype or 'text/plain', size, modified))
if host:
- host, port = splitport(host)
+ host, port = _splitport(host)
if not host or \
(not port and _safe_gethostbyname(host) in self.get_names()):
if host:
@@ -1488,7 +1521,6 @@ class FileHandler(BaseHandler):
origurl = 'file://' + filename
return addinfourl(open(localfile, 'rb'), headers, origurl)
except OSError as exp:
- # users shouldn't expect OSErrors coming from urlopen()
raise URLError(exp)
raise URLError('file not on local host')
@@ -1505,16 +1537,16 @@ class FTPHandler(BaseHandler):
host = req.host
if not host:
raise URLError('ftp error: no host given')
- host, port = splitport(host)
+ host, port = _splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
- user, host = splituser(host)
+ user, host = _splituser(host)
if user:
- user, passwd = splitpasswd(user)
+ user, passwd = _splitpasswd(user)
else:
passwd = None
host = unquote(host)
@@ -1525,7 +1557,7 @@ class FTPHandler(BaseHandler):
host = socket.gethostbyname(host)
except OSError as msg:
raise URLError(msg)
- path, attrs = splitattr(req.selector)
+ path, attrs = _splitattr(req.selector)
dirs = path.split('/')
dirs = list(map(unquote, dirs))
dirs, file = dirs[:-1], dirs[-1]
@@ -1535,7 +1567,7 @@ class FTPHandler(BaseHandler):
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
- attr, value = splitvalue(attr)
+ attr, value = _splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
@@ -1658,14 +1690,10 @@ else:
of the 'file' scheme; not recommended for general use."""
return quote(pathname)
-# This really consists of two pieces:
-# (1) a class which handles opening of all sorts of URLs
-# (plus assorted utilities etc.)
-# (2) a set of functions for parsing URLs
-# XXX Should these be separated out into different modules?
-
ftpcache = {}
+
+
class URLopener:
"""Class to open URLs.
This is a class rather than just a subroutine because we may need
@@ -1733,26 +1761,26 @@ class URLopener:
# External interface
def open(self, fullurl, data=None):
"""Use URLopener().open(file) instead of open(file, 'r')."""
- fullurl = unwrap(to_bytes(fullurl))
+ fullurl = unwrap(_to_bytes(fullurl))
fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
if self.tempcache and fullurl in self.tempcache:
filename, headers = self.tempcache[fullurl]
fp = open(filename, 'rb')
return addinfourl(fp, headers, fullurl)
- urltype, url = splittype(fullurl)
+ urltype, url = _splittype(fullurl)
if not urltype:
urltype = 'file'
if urltype in self.proxies:
proxy = self.proxies[urltype]
- urltype, proxyhost = splittype(proxy)
- host, selector = splithost(proxyhost)
+ urltype, proxyhost = _splittype(proxy)
+ host, selector = _splithost(proxyhost)
url = (host, fullurl) # Signal special case to open_*()
else:
proxy = None
name = 'open_' + urltype
self.type = urltype
name = name.replace('-', '_')
- if not hasattr(self, name):
+ if not hasattr(self, name) or name == 'open_local_file':
if proxy:
return self.open_unknown_proxy(proxy, fullurl, data)
else:
@@ -1769,28 +1797,28 @@ class URLopener:
def open_unknown(self, fullurl, data=None):
"""Overridable interface to open unknown URL type."""
- type, url = splittype(fullurl)
+ type, url = _splittype(fullurl)
raise OSError('url error', 'unknown url type', type)
def open_unknown_proxy(self, proxy, fullurl, data=None):
"""Overridable interface to open unknown URL type."""
- type, url = splittype(fullurl)
+ type, url = _splittype(fullurl)
raise OSError('url error', 'invalid proxy for %s' % type, proxy)
# External interface
def retrieve(self, url, filename=None, reporthook=None, data=None):
"""retrieve(url) returns (filename, headers) for a local object
or (tempfilename, headers) for a remote object."""
- url = unwrap(to_bytes(url))
+ url = unwrap(_to_bytes(url))
if self.tempcache and url in self.tempcache:
return self.tempcache[url]
- type, url1 = splittype(url)
+ type, url1 = _splittype(url)
if filename is None and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
fp.close()
- return url2pathname(splithost(url1)[1]), hdrs
+ return url2pathname(_splithost(url1)[1]), hdrs
except OSError as msg:
pass
fp = self.open(url, data)
@@ -1799,11 +1827,10 @@ class URLopener:
if filename:
tfp = open(filename, 'wb')
else:
- import tempfile
- garbage, path = splittype(url)
- garbage, path = splithost(path or "")
- path, garbage = splitquery(path or "")
- path, garbage = splitattr(path or "")
+ garbage, path = _splittype(url)
+ garbage, path = _splithost(path or "")
+ path, garbage = _splitquery(path or "")
+ path, garbage = _splitattr(path or "")
suffix = os.path.splitext(path)[1]
(fd, filename) = tempfile.mkstemp(suffix)
self.__tempfiles.append(filename)
@@ -1860,25 +1887,25 @@ class URLopener:
user_passwd = None
proxy_passwd= None
if isinstance(url, str):
- host, selector = splithost(url)
+ host, selector = _splithost(url)
if host:
- user_passwd, host = splituser(host)
+ user_passwd, host = _splituser(host)
host = unquote(host)
realhost = host
else:
host, selector = url
# check whether the proxy contains authorization information
- proxy_passwd, host = splituser(host)
+ proxy_passwd, host = _splituser(host)
# now we proceed with the url we want to obtain
- urltype, rest = splittype(selector)
+ urltype, rest = _splittype(selector)
url = rest
user_passwd = None
if urltype.lower() != 'http':
realhost = None
else:
- realhost, rest = splithost(rest)
+ realhost, rest = _splithost(rest)
if realhost:
- user_passwd, realhost = splituser(realhost)
+ user_passwd, realhost = _splituser(realhost)
if user_passwd:
selector = "%s://%s%s" % (urltype, realhost, rest)
if proxy_bypass(realhost):
@@ -1984,7 +2011,7 @@ class URLopener:
"""Use local file."""
import email.utils
import mimetypes
- host, file = splithost(url)
+ host, file = _splithost(url)
localname = url2pathname(file)
try:
stats = os.stat(localname)
@@ -2001,7 +2028,7 @@ class URLopener:
if file[:1] == '/':
urlfile = 'file://' + file
return addinfourl(open(localname, 'rb'), headers, urlfile)
- host, port = splitport(host)
+ host, port = _splitport(host)
if (not port
and socket.gethostbyname(host) in ((localhost(),) + thishost())):
urlfile = file
@@ -2017,11 +2044,11 @@ class URLopener:
if not isinstance(url, str):
raise URLError('ftp error: proxy support for ftp protocol currently not implemented')
import mimetypes
- host, path = splithost(url)
+ host, path = _splithost(url)
if not host: raise URLError('ftp error: no host given')
- host, port = splitport(host)
- user, host = splituser(host)
- if user: user, passwd = splitpasswd(user)
+ host, port = _splitport(host)
+ user, host = _splituser(host)
+ if user: user, passwd = _splitpasswd(user)
else: passwd = None
host = unquote(host)
user = unquote(user or '')
@@ -2032,7 +2059,7 @@ class URLopener:
port = ftplib.FTP_PORT
else:
port = int(port)
- path, attrs = splitattr(path)
+ path, attrs = _splitattr(path)
path = unquote(path)
dirs = path.split('/')
dirs, file = dirs[:-1], dirs[-1]
@@ -2054,7 +2081,7 @@ class URLopener:
if not file: type = 'D'
else: type = 'I'
for attr in attrs:
- attr, value = splitvalue(attr)
+ attr, value = _splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
@@ -2237,11 +2264,11 @@ class FancyURLopener(URLopener):
return getattr(self,name)(url, realm, data)
def retry_proxy_http_basic_auth(self, url, realm, data=None):
- host, selector = splithost(url)
+ host, selector = _splithost(url)
newurl = 'http://' + host + selector
proxy = self.proxies['http']
- urltype, proxyhost = splittype(proxy)
- proxyhost, proxyselector = splithost(proxyhost)
+ urltype, proxyhost = _splittype(proxy)
+ proxyhost, proxyselector = _splithost(proxyhost)
i = proxyhost.find('@') + 1
proxyhost = proxyhost[i:]
user, passwd = self.get_user_passwd(proxyhost, realm, i)
@@ -2255,11 +2282,11 @@ class FancyURLopener(URLopener):
return self.open(newurl, data)
def retry_proxy_https_basic_auth(self, url, realm, data=None):
- host, selector = splithost(url)
+ host, selector = _splithost(url)
newurl = 'https://' + host + selector
proxy = self.proxies['https']
- urltype, proxyhost = splittype(proxy)
- proxyhost, proxyselector = splithost(proxyhost)
+ urltype, proxyhost = _splittype(proxy)
+ proxyhost, proxyselector = _splithost(proxyhost)
i = proxyhost.find('@') + 1
proxyhost = proxyhost[i:]
user, passwd = self.get_user_passwd(proxyhost, realm, i)
@@ -2273,7 +2300,7 @@ class FancyURLopener(URLopener):
return self.open(newurl, data)
def retry_http_basic_auth(self, url, realm, data=None):
- host, selector = splithost(url)
+ host, selector = _splithost(url)
i = host.find('@') + 1
host = host[i:]
user, passwd = self.get_user_passwd(host, realm, i)
@@ -2287,7 +2314,7 @@ class FancyURLopener(URLopener):
return self.open(newurl, data)
def retry_https_basic_auth(self, url, realm, data=None):
- host, selector = splithost(url)
+ host, selector = _splithost(url)
i = host.find('@') + 1
host = host[i:]
user, passwd = self.get_user_passwd(host, realm, i)
@@ -2504,23 +2531,26 @@ def proxy_bypass_environment(host, proxies=None):
try:
no_proxy = proxies['no']
except KeyError:
- return 0
+ return False
# '*' is special case for always bypass
if no_proxy == '*':
- return 1
+ return True
+ host = host.lower()
# strip port off host
- hostonly, port = splitport(host)
+ hostonly, port = _splitport(host)
# check if the host ends with any of the DNS suffixes
- no_proxy_list = [proxy.strip() for proxy in no_proxy.split(',')]
- for name in no_proxy_list:
+ for name in no_proxy.split(','):
+ name = name.strip()
if name:
- name = re.escape(name)
- pattern = r'(.+\.)?%s$' % name
- if (re.match(pattern, hostonly, re.I)
- or re.match(pattern, host, re.I)):
- return 1
+ name = name.lstrip('.') # ignore leading dots
+ name = name.lower()
+ if hostonly == name or host == name:
+ return True
+ name = '.' + name
+ if hostonly.endswith(name) or host.endswith(name):
+ return True
# otherwise, don't bypass
- return 0
+ return False
# This code tests an OSX specific data structure but is testable on all
@@ -2539,7 +2569,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings):
"""
from fnmatch import fnmatch
- hostonly, port = splitport(host)
+ hostonly, port = _splitport(host)
def ip2num(ipAddr):
parts = ipAddr.split('.')
@@ -2646,7 +2676,7 @@ elif os.name == 'nt':
for p in proxyServer.split(';'):
protocol, address = p.split('=', 1)
# See if address has a type:// prefix
- if not re.match('^([^/:]+)://', address):
+ if not re.match('(?:[^/:]+)://', address):
address = '%s://%s' % (protocol, address)
proxies[protocol] = address
else:
@@ -2693,7 +2723,7 @@ elif os.name == 'nt':
if not proxyEnable or not proxyOverride:
return 0
# try to make a host list from name and IP address.
- rawHost, port = splitport(host)
+ rawHost, port = _splitport(host)
host = [rawHost]
try:
addr = socket.gethostbyname(rawHost)
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 9dab4c1c3..c58565e39 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -16,6 +16,9 @@ import urllib.request
__all__ = ["RobotFileParser"]
+RequestRate = collections.namedtuple("RequestRate", "requests seconds")
+
+
class RobotFileParser:
""" This class provides a set of methods to read, parse and answer
questions about a single robots.txt file.
@@ -24,6 +27,7 @@ class RobotFileParser:
def __init__(self, url=''):
self.entries = []
+ self.sitemaps = []
self.default_entry = None
self.disallow_all = False
self.allow_all = False
@@ -136,12 +140,14 @@ class RobotFileParser:
# check if all values are sane
if (len(numbers) == 2 and numbers[0].strip().isdigit()
and numbers[1].strip().isdigit()):
- req_rate = collections.namedtuple('req_rate',
- 'requests seconds')
- entry.req_rate = req_rate
- entry.req_rate.requests = int(numbers[0])
- entry.req_rate.seconds = int(numbers[1])
+ entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
state = 2
+ elif line[0] == "sitemap":
+ # According to http://www.sitemaps.org/protocol.html
+ # "This directive is independent of the user-agent line,
+ # so it doesn't matter where you place it in your file."
+ # Therefore we do not change the state of the parser.
+ self.sitemaps.append(line[1])
if state == 2:
self._add_entry(entry)
@@ -180,7 +186,9 @@ class RobotFileParser:
for entry in self.entries:
if entry.applies_to(useragent):
return entry.delay
- return self.default_entry.delay
+ if self.default_entry:
+ return self.default_entry.delay
+ return None
def request_rate(self, useragent):
if not self.mtime():
@@ -188,10 +196,20 @@ class RobotFileParser:
for entry in self.entries:
if entry.applies_to(useragent):
return entry.req_rate
- return self.default_entry.req_rate
+ if self.default_entry:
+ return self.default_entry.req_rate
+ return None
+
+ def site_maps(self):
+ if not self.sitemaps:
+ return None
+ return self.sitemaps
def __str__(self):
- return ''.join([str(entry) + "\n" for entry in self.entries])
+ entries = self.entries
+ if self.default_entry is not None:
+ entries = entries + [self.default_entry]
+ return '\n\n'.join(map(str, entries))
class RuleLine:
@@ -223,10 +241,14 @@ class Entry:
def __str__(self):
ret = []
for agent in self.useragents:
- ret.extend(["User-agent: ", agent, "\n"])
- for line in self.rulelines:
- ret.extend([str(line), "\n"])
- return ''.join(ret)
+ ret.append(f"User-agent: {agent}")
+ if self.delay is not None:
+ ret.append(f"Crawl-delay: {self.delay}")
+ if self.req_rate is not None:
+ rate = self.req_rate
+ ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
+ ret.extend(map(str, self.rulelines))
+ return '\n'.join(ret)
def applies_to(self, useragent):
"""check if this entry applies to the specified agent"""
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
new file mode 100644
index 000000000..d80463762
--- /dev/null
+++ b/Lib/venv/__init__.py
@@ -0,0 +1,503 @@
+"""
+Virtual environment (venv) package for Python. Based on PEP 405.
+
+Copyright (C) 2011-2014 Vinay Sajip.
+Licensed to the PSF under a contributor agreement.
+"""
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import sysconfig
+import types
+
+
+CORE_VENV_DEPS = ('pip', 'setuptools')
+logger = logging.getLogger(__name__)
+
+
+class EnvBuilder:
+ """
+ This class exists to allow virtual environment creation to be
+ customized. The constructor parameters determine the builder's
+ behaviour when called upon to create a virtual environment.
+
+ By default, the builder makes the system (global) site-packages dir
+ *un*available to the created environment.
+
+ If invoked using the Python -m option, the default is to use copying
+ on Windows platforms but symlinks elsewhere. If instantiated some
+ other way, the default is to *not* use symlinks.
+
+ :param system_site_packages: If True, the system (global) site-packages
+ dir is available to created environments.
+ :param clear: If True, delete the contents of the environment directory if
+ it already exists, before environment creation.
+ :param symlinks: If True, attempt to symlink rather than copy files into
+ virtual environment.
+ :param upgrade: If True, upgrade an existing virtual environment.
+ :param with_pip: If True, ensure pip is installed in the virtual
+ environment
+ :param prompt: Alternative terminal prefix for the environment.
+ :param upgrade_deps: Update the base venv modules to the latest on PyPI
+ """
+
+ def __init__(self, system_site_packages=False, clear=False,
+ symlinks=False, upgrade=False, with_pip=False, prompt=None,
+ upgrade_deps=False):
+ self.system_site_packages = system_site_packages
+ self.clear = clear
+ self.symlinks = symlinks
+ self.upgrade = upgrade
+ self.with_pip = with_pip
+ if prompt == '.': # see bpo-38901
+ prompt = os.path.basename(os.getcwd())
+ self.prompt = prompt
+ self.upgrade_deps = upgrade_deps
+
+ def create(self, env_dir):
+ """
+ Create a virtual environment in a directory.
+
+ :param env_dir: The target directory to create an environment in.
+
+ """
+ env_dir = os.path.abspath(env_dir)
+ context = self.ensure_directories(env_dir)
+ # See issue 24875. We need system_site_packages to be False
+ # until after pip is installed.
+ true_system_site_packages = self.system_site_packages
+ self.system_site_packages = False
+ self.create_configuration(context)
+ self.setup_python(context)
+ if self.with_pip:
+ self._setup_pip(context)
+ if not self.upgrade:
+ self.setup_scripts(context)
+ self.post_setup(context)
+ if true_system_site_packages:
+ # We had set it to False before, now
+ # restore it and rewrite the configuration
+ self.system_site_packages = True
+ self.create_configuration(context)
+ if self.upgrade_deps:
+ self.upgrade_dependencies(context)
+
+ def clear_directory(self, path):
+ for fn in os.listdir(path):
+ fn = os.path.join(path, fn)
+ if os.path.islink(fn) or os.path.isfile(fn):
+ os.remove(fn)
+ elif os.path.isdir(fn):
+ shutil.rmtree(fn)
+
+ def ensure_directories(self, env_dir):
+ """
+ Create the directories for the environment.
+
+ Returns a context object which holds paths in the environment,
+ for use by subsequent logic.
+ """
+
+ def create_if_needed(d):
+ if not os.path.exists(d):
+ os.makedirs(d)
+ elif os.path.islink(d) or os.path.isfile(d):
+ raise ValueError('Unable to create directory %r' % d)
+
+ if os.path.exists(env_dir) and self.clear:
+ self.clear_directory(env_dir)
+ context = types.SimpleNamespace()
+ context.env_dir = env_dir
+ context.env_name = os.path.split(env_dir)[1]
+ prompt = self.prompt if self.prompt is not None else context.env_name
+ context.prompt = '(%s) ' % prompt
+ create_if_needed(env_dir)
+ executable = sys._base_executable
+ dirname, exename = os.path.split(os.path.abspath(executable))
+ context.executable = executable
+ context.python_dir = dirname
+ context.python_exe = exename
+ if sys.platform == 'win32':
+ binname = 'Scripts'
+ incpath = 'Include'
+ libpath = os.path.join(env_dir, 'Lib', 'site-packages')
+ else:
+ binname = 'bin'
+ incpath = 'include'
+ libpath = os.path.join(env_dir, 'lib',
+ 'python%d.%d' % sys.version_info[:2],
+ 'site-packages')
+ context.inc_path = path = os.path.join(env_dir, incpath)
+ create_if_needed(path)
+ create_if_needed(libpath)
+ # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX
+ if ((sys.maxsize > 2**32) and (os.name == 'posix') and
+ (sys.platform != 'darwin')):
+ link_path = os.path.join(env_dir, 'lib64')
+ if not os.path.exists(link_path): # Issue #21643
+ os.symlink('lib', link_path)
+ context.bin_path = binpath = os.path.join(env_dir, binname)
+ context.bin_name = binname
+ context.env_exe = os.path.join(binpath, exename)
+ create_if_needed(binpath)
+ return context
+
+ def create_configuration(self, context):
+ """
+ Create a configuration file indicating where the environment's Python
+ was copied from, and whether the system site-packages should be made
+ available in the environment.
+
+ :param context: The information for the environment creation request
+ being processed.
+ """
+ context.cfg_path = path = os.path.join(context.env_dir, 'pyvenv.cfg')
+ with open(path, 'w', encoding='utf-8') as f:
+ f.write('home = %s\n' % context.python_dir)
+ if self.system_site_packages:
+ incl = 'true'
+ else:
+ incl = 'false'
+ f.write('include-system-site-packages = %s\n' % incl)
+ f.write('version = %d.%d.%d\n' % sys.version_info[:3])
+ if self.prompt is not None:
+ f.write(f'prompt = {self.prompt!r}\n')
+
+ if os.name != 'nt':
+ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
+ """
+ Try symlinking a file, and if that fails, fall back to copying.
+ """
+ force_copy = not self.symlinks
+ if not force_copy:
+ try:
+ if not os.path.islink(dst): # can't link to itself!
+ if relative_symlinks_ok:
+ assert os.path.dirname(src) == os.path.dirname(dst)
+ os.symlink(os.path.basename(src), dst)
+ else:
+ os.symlink(src, dst)
+ except Exception: # may need to use a more specific exception
+ logger.warning('Unable to symlink %r to %r', src, dst)
+ force_copy = True
+ if force_copy:
+ shutil.copyfile(src, dst)
+ else:
+ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
+ """
+ Try symlinking a file, and if that fails, fall back to copying.
+ """
+ bad_src = os.path.lexists(src) and not os.path.exists(src)
+ if self.symlinks and not bad_src and not os.path.islink(dst):
+ try:
+ if relative_symlinks_ok:
+ assert os.path.dirname(src) == os.path.dirname(dst)
+ os.symlink(os.path.basename(src), dst)
+ else:
+ os.symlink(src, dst)
+ return
+ except Exception: # may need to use a more specific exception
+ logger.warning('Unable to symlink %r to %r', src, dst)
+
+ # On Windows, we rewrite symlinks to our base python.exe into
+ # copies of venvlauncher.exe
+ basename, ext = os.path.splitext(os.path.basename(src))
+ srcfn = os.path.join(os.path.dirname(__file__),
+ "scripts",
+ "nt",
+ basename + ext)
+ # Builds or venv's from builds need to remap source file
+ # locations, as we do not put them into Lib/venv/scripts
+ if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
+ if basename.endswith('_d'):
+ ext = '_d' + ext
+ basename = basename[:-2]
+ if basename == 'python':
+ basename = 'venvlauncher'
+ elif basename == 'pythonw':
+ basename = 'venvwlauncher'
+ src = os.path.join(os.path.dirname(src), basename + ext)
+ else:
+ src = srcfn
+ if not os.path.exists(src):
+ if not bad_src:
+ logger.warning('Unable to copy %r', src)
+ return
+
+ shutil.copyfile(src, dst)
+
+ def setup_python(self, context):
+ """
+ Set up a Python executable in the environment.
+
+ :param context: The information for the environment creation request
+ being processed.
+ """
+ binpath = context.bin_path
+ path = context.env_exe
+ copier = self.symlink_or_copy
+ dirname = context.python_dir
+ if os.name != 'nt':
+ copier(context.executable, path)
+ if not os.path.islink(path):
+ os.chmod(path, 0o755)
+ for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'):
+ path = os.path.join(binpath, suffix)
+ if not os.path.exists(path):
+ # Issue 18807: make copies if
+ # symlinks are not wanted
+ copier(context.env_exe, path, relative_symlinks_ok=True)
+ if not os.path.islink(path):
+ os.chmod(path, 0o755)
+ else:
+ if self.symlinks:
+ # For symlinking, we need a complete copy of the root directory
+ # If symlinks fail, you'll get unnecessary copies of files, but
+ # we assume that if you've opted into symlinks on Windows then
+ # you know what you're doing.
+ suffixes = [
+ f for f in os.listdir(dirname) if
+ os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll')
+ ]
+ if sysconfig.is_python_build(True):
+ suffixes = [
+ f for f in suffixes if
+ os.path.normcase(f).startswith(('python', 'vcruntime'))
+ ]
+ else:
+ suffixes = ['python.exe', 'python_d.exe', 'pythonw.exe',
+ 'pythonw_d.exe']
+
+ for suffix in suffixes:
+ src = os.path.join(dirname, suffix)
+ if os.path.lexists(src):
+ copier(src, os.path.join(binpath, suffix))
+
+ if sysconfig.is_python_build(True):
+ # copy init.tcl
+ for root, dirs, files in os.walk(context.python_dir):
+ if 'init.tcl' in files:
+ tcldir = os.path.basename(root)
+ tcldir = os.path.join(context.env_dir, 'Lib', tcldir)
+ if not os.path.exists(tcldir):
+ os.makedirs(tcldir)
+ src = os.path.join(root, 'init.tcl')
+ dst = os.path.join(tcldir, 'init.tcl')
+ shutil.copyfile(src, dst)
+ break
+
+ def _setup_pip(self, context):
+ """Installs or upgrades pip in a virtual environment"""
+ # TODO: RustPython
+ msg = ("Pip isn't supported yet. To create a virtual environment"
+ "without pip, call venv with the --without-pip flag.")
+ raise NotImplementedError(msg)
+ # We run ensurepip in isolated mode to avoid side effects from
+ # environment vars, the current directory and anything else
+ # intended for the global Python environment
+ cmd = [context.env_exe, '-Im', 'ensurepip', '--upgrade',
+ '--default-pip']
+ subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+ def setup_scripts(self, context):
+ """
+ Set up scripts into the created environment from a directory.
+
+ This method installs the default scripts into the environment
+ being created. You can prevent the default installation by overriding
+ this method if you really need to, or if you need to specify
+ a different location for the scripts to install. By default, the
+ 'scripts' directory in the venv package is used as the source of
+ scripts to install.
+ """
+ path = os.path.abspath(os.path.dirname(__file__))
+ path = os.path.join(path, 'scripts')
+ self.install_scripts(context, path)
+
+ def post_setup(self, context):
+ """
+ Hook for post-setup modification of the venv. Subclasses may install
+ additional packages or scripts here, add activation shell scripts, etc.
+
+ :param context: The information for the environment creation request
+ being processed.
+ """
+ pass
+
+ def replace_variables(self, text, context):
+ """
+ Replace variable placeholders in script text with context-specific
+ variables.
+
+ Return the text passed in , but with variables replaced.
+
+ :param text: The text in which to replace placeholder variables.
+ :param context: The information for the environment creation request
+ being processed.
+ """
+ text = text.replace('__VENV_DIR__', context.env_dir)
+ text = text.replace('__VENV_NAME__', context.env_name)
+ text = text.replace('__VENV_PROMPT__', context.prompt)
+ text = text.replace('__VENV_BIN_NAME__', context.bin_name)
+ text = text.replace('__VENV_PYTHON__', context.env_exe)
+ return text
+
+ def install_scripts(self, context, path):
+ """
+ Install scripts into the created environment from a directory.
+
+ :param context: The information for the environment creation request
+ being processed.
+ :param path: Absolute pathname of a directory containing script.
+ Scripts in the 'common' subdirectory of this directory,
+ and those in the directory named for the platform
+ being run on, are installed in the created environment.
+ Placeholder variables are replaced with environment-
+ specific values.
+ """
+ binpath = context.bin_path
+ plen = len(path)
+ for root, dirs, files in os.walk(path):
+ if root == path: # at top-level, remove irrelevant dirs
+ for d in dirs[:]:
+ if d not in ('common', os.name):
+ dirs.remove(d)
+ continue # ignore files in top level
+ for f in files:
+ if (os.name == 'nt' and f.startswith('python')
+ and f.endswith(('.exe', '.pdb'))):
+ continue
+ srcfile = os.path.join(root, f)
+ suffix = root[plen:].split(os.sep)[2:]
+ if not suffix:
+ dstdir = binpath
+ else:
+ dstdir = os.path.join(binpath, *suffix)
+ if not os.path.exists(dstdir):
+ os.makedirs(dstdir)
+ dstfile = os.path.join(dstdir, f)
+ with open(srcfile, 'rb') as f:
+ data = f.read()
+ if not srcfile.endswith(('.exe', '.pdb')):
+ try:
+ data = data.decode('utf-8')
+ data = self.replace_variables(data, context)
+ data = data.encode('utf-8')
+ except UnicodeError as e:
+ data = None
+ logger.warning('unable to copy script %r, '
+ 'may be binary: %s', srcfile, e)
+ if data is not None:
+ with open(dstfile, 'wb') as f:
+ f.write(data)
+ shutil.copymode(srcfile, dstfile)
+
+ def upgrade_dependencies(self, context):
+ logger.debug(
+ f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}'
+ )
+ if sys.platform == 'win32':
+ python_exe = os.path.join(context.bin_path, 'python.exe')
+ else:
+ python_exe = os.path.join(context.bin_path, 'python')
+ cmd = [python_exe, '-m', 'pip', 'install', '--upgrade']
+ cmd.extend(CORE_VENV_DEPS)
+ subprocess.check_call(cmd)
+
+
+def create(env_dir, system_site_packages=False, clear=False,
+ symlinks=False, with_pip=False, prompt=None, upgrade_deps=False):
+ """Create a virtual environment in a directory."""
+ builder = EnvBuilder(system_site_packages=system_site_packages,
+ clear=clear, symlinks=symlinks, with_pip=with_pip,
+ prompt=prompt, upgrade_deps=upgrade_deps)
+ builder.create(env_dir)
+
+def main(args=None):
+ compatible = True
+ if sys.version_info < (3, 3):
+ compatible = False
+ elif not hasattr(sys, 'base_prefix'):
+ compatible = False
+ if not compatible:
+ raise ValueError('This script is only for use with Python >= 3.3')
+ else:
+ import argparse
+
+ parser = argparse.ArgumentParser(prog=__name__,
+ description='Creates virtual Python '
+ 'environments in one or '
+ 'more target '
+ 'directories.',
+ epilog='Once an environment has been '
+ 'created, you may wish to '
+ 'activate it, e.g. by '
+ 'sourcing an activate script '
+ 'in its bin directory.')
+ parser.add_argument('dirs', metavar='ENV_DIR', nargs='+',
+ help='A directory to create the environment in.')
+ parser.add_argument('--system-site-packages', default=False,
+ action='store_true', dest='system_site',
+ help='Give the virtual environment access to the '
+ 'system site-packages dir.')
+ if os.name == 'nt':
+ use_symlinks = False
+ else:
+ use_symlinks = True
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--symlinks', default=use_symlinks,
+ action='store_true', dest='symlinks',
+ help='Try to use symlinks rather than copies, '
+ 'when symlinks are not the default for '
+ 'the platform.')
+ group.add_argument('--copies', default=not use_symlinks,
+ action='store_false', dest='symlinks',
+ help='Try to use copies rather than symlinks, '
+ 'even when symlinks are the default for '
+ 'the platform.')
+ parser.add_argument('--clear', default=False, action='store_true',
+ dest='clear', help='Delete the contents of the '
+ 'environment directory if it '
+ 'already exists, before '
+ 'environment creation.')
+ parser.add_argument('--upgrade', default=False, action='store_true',
+ dest='upgrade', help='Upgrade the environment '
+ 'directory to use this version '
+ 'of Python, assuming Python '
+ 'has been upgraded in-place.')
+ parser.add_argument('--without-pip', dest='with_pip',
+ default=True, action='store_false',
+ help='Skips installing or upgrading pip in the '
+ 'virtual environment (pip is bootstrapped '
+ 'by default)')
+ parser.add_argument('--prompt',
+ help='Provides an alternative prompt prefix for '
+ 'this environment.')
+ parser.add_argument('--upgrade-deps', default=False, action='store_true',
+ dest='upgrade_deps',
+ help='Upgrade core dependencies: {} to the latest '
+ 'version in PyPI'.format(
+ ' '.join(CORE_VENV_DEPS)))
+ options = parser.parse_args(args)
+ if options.upgrade and options.clear:
+ raise ValueError('you cannot supply --upgrade and --clear together.')
+ builder = EnvBuilder(system_site_packages=options.system_site,
+ clear=options.clear,
+ symlinks=options.symlinks,
+ upgrade=options.upgrade,
+ with_pip=options.with_pip,
+ prompt=options.prompt,
+ upgrade_deps=options.upgrade_deps)
+ for d in options.dirs:
+ builder.create(d)
+
+if __name__ == '__main__':
+ rc = 1
+ try:
+ main()
+ rc = 0
+ except Exception as e:
+ print('Error: %s' % e, file=sys.stderr)
+ sys.exit(rc)
diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py
new file mode 100644
index 000000000..912423e4a
--- /dev/null
+++ b/Lib/venv/__main__.py
@@ -0,0 +1,10 @@
+import sys
+from . import main
+
+rc = 1
+try:
+ main()
+ rc = 0
+except Exception as e:
+ print('Error: %s' % e, file=sys.stderr)
+sys.exit(rc)
diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1
new file mode 100644
index 000000000..2fb3852c3
--- /dev/null
+++ b/Lib/venv/scripts/common/Activate.ps1
@@ -0,0 +1,241 @@
+<#
+.Synopsis
+Activate a Python virtual environment for the current PowerShell session.
+
+.Description
+Pushes the python executable for a virtual environment to the front of the
+$Env:PATH environment variable and sets the prompt to signify that you are
+in a Python virtual environment. Makes use of the command line switches as
+well as the `pyvenv.cfg` file values present in the virtual environment.
+
+.Parameter VenvDir
+Path to the directory that contains the virtual environment to activate. The
+default value for this is the parent of the directory that the Activate.ps1
+script is located within.
+
+.Parameter Prompt
+The prompt prefix to display when this virtual environment is activated. By
+default, this prompt is the name of the virtual environment folder (VenvDir)
+surrounded by parentheses and followed by a single space (ie. '(.venv) ').
+
+.Example
+Activate.ps1
+Activates the Python virtual environment that contains the Activate.ps1 script.
+
+.Example
+Activate.ps1 -Verbose
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and shows extra information about the activation as it executes.
+
+.Example
+Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
+Activates the Python virtual environment located in the specified location.
+
+.Example
+Activate.ps1 -Prompt "MyPython"
+Activates the Python virtual environment that contains the Activate.ps1 script,
+and prefixes the current prompt with the specified string (surrounded in
+parentheses) while the virtual environment is active.
+
+.Notes
+On Windows, it may be required to enable this Activate.ps1 script by setting the
+execution policy for the user. You can do this by issuing the following PowerShell
+command:
+
+PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
+
+For more information on Execution Policies:
+https://go.microsoft.com/fwlink/?LinkID=135170
+
+#>
+Param(
+ [Parameter(Mandatory = $false)]
+ [String]
+ $VenvDir,
+ [Parameter(Mandatory = $false)]
+ [String]
+ $Prompt
+)
+
+<# Function declarations --------------------------------------------------- #>
+
+<#
+.Synopsis
+Remove all shell session elements added by the Activate script, including the
+addition of the virtual environment's Python executable from the beginning of
+the PATH variable.
+
+.Parameter NonDestructive
+If present, do not remove this function from the global namespace for the
+session.
+
+#>
+function global:deactivate ([switch]$NonDestructive) {
+ # Revert to original values
+
+ # The prior prompt:
+ if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
+ Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
+ Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
+ }
+
+ # The prior PYTHONHOME:
+ if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
+ Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
+ Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
+ }
+
+ # The prior PATH:
+ if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
+ Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
+ Remove-Item -Path Env:_OLD_VIRTUAL_PATH
+ }
+
+ # Just remove the VIRTUAL_ENV altogether:
+ if (Test-Path -Path Env:VIRTUAL_ENV) {
+ Remove-Item -Path env:VIRTUAL_ENV
+ }
+
+ # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
+ if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
+ Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
+ }
+
+ # Leave deactivate function in the global namespace if requested:
+ if (-not $NonDestructive) {
+ Remove-Item -Path function:deactivate
+ }
+}
+
+<#
+.Description
+Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
+given folder, and returns them in a map.
+
+For each line in the pyvenv.cfg file, if that line can be parsed into exactly
+two strings separated by `=` (with any amount of whitespace surrounding the =)
+then it is considered a `key = value` line. The left hand string is the key,
+the right hand is the value.
+
+If the value starts with a `'` or a `"` then the first and last character is
+stripped from the value before being captured.
+
+.Parameter ConfigDir
+Path to the directory that contains the `pyvenv.cfg` file.
+#>
+function Get-PyVenvConfig(
+ [String]
+ $ConfigDir
+) {
+ Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
+
+ # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
+ $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
+
+ # An empty map will be returned if no config file is found.
+ $pyvenvConfig = @{ }
+
+ if ($pyvenvConfigPath) {
+
+ Write-Verbose "File exists, parse `key = value` lines"
+ $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
+
+ $pyvenvConfigContent | ForEach-Object {
+ $keyval = $PSItem -split "\s*=\s*", 2
+ if ($keyval[0] -and $keyval[1]) {
+ $val = $keyval[1]
+
+ # Remove extraneous quotations around a string value.
+ if ("'""".Contains($val.Substring(0, 1))) {
+ $val = $val.Substring(1, $val.Length - 2)
+ }
+
+ $pyvenvConfig[$keyval[0]] = $val
+ Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
+ }
+ }
+ }
+ return $pyvenvConfig
+}
+
+
+<# Begin Activate script --------------------------------------------------- #>
+
+# Determine the containing directory of this script
+$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
+$VenvExecDir = Get-Item -Path $VenvExecPath
+
+Write-Verbose "Activation script is located in path: '$VenvExecPath'"
+Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
+Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
+
+# Set values required in priority: CmdLine, ConfigFile, Default
+# First, get the location of the virtual environment, it might not be
+# VenvExecDir if specified on the command line.
+if ($VenvDir) {
+ Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
+}
+else {
+ Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
+ $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
+ Write-Verbose "VenvDir=$VenvDir"
+}
+
+# Next, read the `pyvenv.cfg` file to determine any required value such
+# as `prompt`.
+$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
+
+# Next, set the prompt from the command line, or the config file, or
+# just use the name of the virtual environment folder.
+if ($Prompt) {
+ Write-Verbose "Prompt specified as argument, using '$Prompt'"
+}
+else {
+ Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
+ if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
+ Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
+ $Prompt = $pyvenvCfg['prompt'];
+ }
+ else {
+ Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)"
+ Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
+ $Prompt = Split-Path -Path $venvDir -Leaf
+ }
+}
+
+Write-Verbose "Prompt = '$Prompt'"
+Write-Verbose "VenvDir='$VenvDir'"
+
+# Deactivate any currently active virtual environment, but leave the
+# deactivate function in place.
+deactivate -nondestructive
+
+# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
+# that there is an activated venv.
+$env:VIRTUAL_ENV = $VenvDir
+
+if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
+
+ Write-Verbose "Setting prompt to '$Prompt'"
+
+ # Set the prompt to include the env name
+ # Make sure _OLD_VIRTUAL_PROMPT is global
+ function global:_OLD_VIRTUAL_PROMPT { "" }
+ Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
+ New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
+
+ function global:prompt {
+ Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
+ _OLD_VIRTUAL_PROMPT
+ }
+}
+
+# Clear PYTHONHOME
+if (Test-Path -Path Env:PYTHONHOME) {
+ Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
+ Remove-Item -Path Env:PYTHONHOME
+}
+
+# Add the venv to the PATH
+Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
+$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"
diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate
new file mode 100644
index 000000000..45af3536a
--- /dev/null
+++ b/Lib/venv/scripts/common/activate
@@ -0,0 +1,66 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+ # reset old environment variables
+ if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
+ PATH="${_OLD_VIRTUAL_PATH:-}"
+ export PATH
+ unset _OLD_VIRTUAL_PATH
+ fi
+ if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
+ PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
+ export PYTHONHOME
+ unset _OLD_VIRTUAL_PYTHONHOME
+ fi
+
+ # This should detect bash and zsh, which have a hash command that must
+ # be called to get it to forget past commands. Without forgetting
+ # past commands the $PATH changes we made may not be respected
+ if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+ hash -r 2> /dev/null
+ fi
+
+ if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
+ PS1="${_OLD_VIRTUAL_PS1:-}"
+ export PS1
+ unset _OLD_VIRTUAL_PS1
+ fi
+
+ unset VIRTUAL_ENV
+ if [ ! "${1:-}" = "nondestructive" ] ; then
+ # Self destruct!
+ unset -f deactivate
+ fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="__VENV_DIR__"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "${PYTHONHOME:-}" ] ; then
+ _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
+ unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
+ _OLD_VIRTUAL_PS1="${PS1:-}"
+ PS1="__VENV_PROMPT__${PS1:-}"
+ export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands. Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
+ hash -r 2> /dev/null
+fi
diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat
new file mode 100644
index 000000000..f61413e23
--- /dev/null
+++ b/Lib/venv/scripts/nt/activate.bat
@@ -0,0 +1,33 @@
+@echo off
+
+rem This file is UTF-8 encoded, so we need to update the current code page while executing it
+for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
+ set _OLD_CODEPAGE=%%a
+)
+if defined _OLD_CODEPAGE (
+ "%SystemRoot%\System32\chcp.com" 65001 > nul
+)
+
+set VIRTUAL_ENV=__VENV_DIR__
+
+if not defined PROMPT set PROMPT=$P$G
+
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
+
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=__VENV_PROMPT__%PROMPT%
+
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
+
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH%
+
+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
+
+:END
+if defined _OLD_CODEPAGE (
+ "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
+ set _OLD_CODEPAGE=
+)
diff --git a/Lib/venv/scripts/nt/deactivate.bat b/Lib/venv/scripts/nt/deactivate.bat
new file mode 100644
index 000000000..313c07911
--- /dev/null
+++ b/Lib/venv/scripts/nt/deactivate.bat
@@ -0,0 +1,21 @@
+@echo off
+
+if defined _OLD_VIRTUAL_PROMPT (
+ set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
+)
+set _OLD_VIRTUAL_PROMPT=
+
+if defined _OLD_VIRTUAL_PYTHONHOME (
+ set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
+ set _OLD_VIRTUAL_PYTHONHOME=
+)
+
+if defined _OLD_VIRTUAL_PATH (
+ set "PATH=%_OLD_VIRTUAL_PATH%"
+)
+
+set _OLD_VIRTUAL_PATH=
+
+set VIRTUAL_ENV=
+
+:END
diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh
new file mode 100644
index 000000000..68a0dc74e
--- /dev/null
+++ b/Lib/venv/scripts/posix/activate.csh
@@ -0,0 +1,25 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+# Created by Davide Di Blasi .
+# Ported to Python 3.3 venv by Andrew Svetlov
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV "__VENV_DIR__"
+
+set _OLD_VIRTUAL_PATH="$PATH"
+setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH"
+
+
+set _OLD_VIRTUAL_PROMPT="$prompt"
+
+if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
+ set prompt = "__VENV_PROMPT__$prompt"
+endif
+
+alias pydoc python -m pydoc
+
+rehash
diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish
new file mode 100644
index 000000000..54b9ea567
--- /dev/null
+++ b/Lib/venv/scripts/posix/activate.fish
@@ -0,0 +1,64 @@
+# This file must be used with "source /bin/activate.fish" *from fish*
+# (https://fishshell.com/); you cannot run it directly.
+
+function deactivate -d "Exit virtual environment and return to normal shell environment"
+ # reset old environment variables
+ if test -n "$_OLD_VIRTUAL_PATH"
+ set -gx PATH $_OLD_VIRTUAL_PATH
+ set -e _OLD_VIRTUAL_PATH
+ end
+ if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+ set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
+ set -e _OLD_VIRTUAL_PYTHONHOME
+ end
+
+ if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+ functions -e fish_prompt
+ set -e _OLD_FISH_PROMPT_OVERRIDE
+ functions -c _old_fish_prompt fish_prompt
+ functions -e _old_fish_prompt
+ end
+
+ set -e VIRTUAL_ENV
+ if test "$argv[1]" != "nondestructive"
+ # Self-destruct!
+ functions -e deactivate
+ end
+end
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV "__VENV_DIR__"
+
+set -gx _OLD_VIRTUAL_PATH $PATH
+set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH
+
+# Unset PYTHONHOME if set.
+if set -q PYTHONHOME
+ set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+ set -e PYTHONHOME
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+ # fish uses a function instead of an env var to generate the prompt.
+
+ # Save the current fish_prompt function as the function _old_fish_prompt.
+ functions -c fish_prompt _old_fish_prompt
+
+ # With the original prompt function renamed, we can override with our own.
+ function fish_prompt
+ # Save the return status of the last command.
+ set -l old_status $status
+
+ # Output the venv prompt; color taken from the blue of the Python logo.
+ printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal)
+
+ # Restore the return status of the previous command.
+ echo "exit $old_status" | .
+ # Output the original/"old" prompt.
+ _old_fish_prompt
+ end
+
+ set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+end
diff --git a/README.md b/README.md
index a50600b4a..f5a45c601 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,8 @@ To test RustPython, do the following:
$ git clone https://github.com/RustPython/RustPython
$ cd RustPython
+ # if you're on windows:
+ $ powershell scripts\symlinks-to-hardlinks.ps1
$ cargo run demo.py
Hello, RustPython!
diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml
index 289f9fc6a..95206a3bf 100644
--- a/bytecode/Cargo.toml
+++ b/bytecode/Cargo.toml
@@ -11,8 +11,8 @@ license = "MIT"
[dependencies]
bincode = "1.1"
bitflags = "1.1"
-lz4-compress = "0.1.1"
-num-bigint = { version = "0.2", features = ["serde"] }
+lz4-compression = "0.7"
+num-bigint = { version = "0.3", features = ["serde"] }
num-complex = { version = "0.2", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
-itertools = "0.8"
+itertools = "0.9"
diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs
index 09d0e03d5..a931e888c 100644
--- a/bytecode/src/bytecode.rs
+++ b/bytecode/src/bytecode.rs
@@ -409,14 +409,15 @@ impl CodeObject {
/// Load a code object from bytes
pub fn from_bytes(data: &[u8]) -> Result> {
- let data = lz4_compress::decompress(data)?;
+ let data = lz4_compression::decompress::decompress(data)
+ .map_err(|e| format!("lz4 error: {:?}", e))?;
bincode::deserialize::(&data).map_err(|e| e.into())
}
/// Serialize this bytecode to bytes.
pub fn to_bytes(&self) -> Vec {
let data = bincode::serialize(&self).expect("Code object must be serializable");
- lz4_compress::compress(&data)
+ lz4_compression::compress::compress(&data)
}
pub fn get_constants(&self) -> impl Iterator- {
diff --git a/common/Cargo.toml b/common/Cargo.toml
new file mode 100644
index 000000000..951660bfb
--- /dev/null
+++ b/common/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "rustpython-common"
+version = "0.0.0"
+authors = ["RustPython Team"]
+edition = "2018"
+
+[features]
+threading = ["parking_lot"]
+
+[dependencies]
+parking_lot = { version = "0.11.0", optional = true }
+num-traits = "0.2"
+num-bigint = "0.3"
+lexical-core = "0.7"
+hexf-parse = "0.1.0"
+cfg-if = "0.1"
+
+[dev-dependencies]
+rand = "0.7.3"
diff --git a/common/src/cell.rs b/common/src/cell.rs
new file mode 100644
index 000000000..4c48326eb
--- /dev/null
+++ b/common/src/cell.rs
@@ -0,0 +1,132 @@
+#[cfg(feature = "threading")]
+use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
+#[cfg(not(feature = "threading"))]
+use std::cell::{Ref, RefCell, RefMut};
+use std::ops::{Deref, DerefMut};
+
+cfg_if::cfg_if! {
+ if #[cfg(feature = "threading")] {
+ type MutexInner = Mutex;
+ type MutexGuardInner<'a, T> = MutexGuard<'a, T>;
+ const fn new_mutex(value: T) -> MutexInner {
+ parking_lot::const_mutex(value)
+ }
+ fn lock_mutex(m: &MutexInner) -> MutexGuardInner {
+ m.lock()
+ }
+ } else {
+ type MutexInner = RefCell;
+ type MutexGuardInner<'a, T> = RefMut<'a, T>;
+ const fn new_mutex(value: T) -> MutexInner {
+ RefCell::new(value)
+ }
+ fn lock_mutex(m: &MutexInner) -> MutexGuardInner {
+ m.borrow_mut()
+ }
+ }
+}
+
+#[derive(Debug, Default)]
+#[repr(transparent)]
+pub struct PyMutex(MutexInner);
+
+impl PyMutex {
+ pub const fn new(value: T) -> Self {
+ Self(new_mutex(value))
+ }
+}
+
+impl PyMutex {
+ pub fn lock(&self) -> PyMutexGuard {
+ PyMutexGuard(lock_mutex(&self.0))
+ }
+}
+
+#[derive(Debug)]
+#[repr(transparent)]
+pub struct PyMutexGuard<'a, T: ?Sized>(MutexGuardInner<'a, T>);
+impl Deref for PyMutexGuard<'_, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.0.deref()
+ }
+}
+impl DerefMut for PyMutexGuard<'_, T> {
+ fn deref_mut(&mut self) -> &mut T {
+ self.0.deref_mut()
+ }
+}
+
+cfg_if::cfg_if! {
+ if #[cfg(feature = "threading")] {
+ type RwLockInner = RwLock;
+ type RwLockReadInner<'a, T> = RwLockReadGuard<'a, T>;
+ type RwLockWriteInner<'a, T> = RwLockWriteGuard<'a, T>;
+ const fn new_rwlock(value: T) -> RwLockInner {
+ parking_lot::const_rwlock(value)
+ }
+ fn read_rwlock(m: &RwLockInner) -> RwLockReadInner {
+ m.read()
+ }
+ fn write_rwlock(m: &RwLockInner) -> RwLockWriteInner {
+ m.write()
+ }
+ } else {
+ type RwLockInner = RefCell;
+ type RwLockReadInner<'a, T> = Ref<'a, T>;
+ type RwLockWriteInner<'a, T> = RefMut<'a, T>;
+ const fn new_rwlock(value: T) -> RwLockInner {
+ RefCell::new(value)
+ }
+ fn read_rwlock(m: &RwLockInner) -> RwLockReadInner {
+ m.borrow()
+ }
+ fn write_rwlock(m: &RwLockInner) -> RwLockWriteInner {
+ m.borrow_mut()
+ }
+ }
+}
+
+#[derive(Debug, Default)]
+#[repr(transparent)]
+pub struct PyRwLock(RwLockInner);
+
+impl PyRwLock {
+ pub const fn new(value: T) -> Self {
+ Self(new_rwlock(value))
+ }
+}
+
+impl PyRwLock {
+ pub fn read(&self) -> PyRwLockReadGuard {
+ PyRwLockReadGuard(read_rwlock(&self.0))
+ }
+ pub fn write(&self) -> PyRwLockWriteGuard {
+ PyRwLockWriteGuard(write_rwlock(&self.0))
+ }
+}
+
+#[derive(Debug)]
+#[repr(transparent)]
+pub struct PyRwLockReadGuard<'a, T: ?Sized>(RwLockReadInner<'a, T>);
+impl Deref for PyRwLockReadGuard<'_, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.0.deref()
+ }
+}
+
+#[derive(Debug)]
+#[repr(transparent)]
+pub struct PyRwLockWriteGuard<'a, T: ?Sized>(RwLockWriteInner<'a, T>);
+impl Deref for PyRwLockWriteGuard<'_, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.0.deref()
+ }
+}
+impl DerefMut for PyRwLockWriteGuard<'_, T> {
+ fn deref_mut(&mut self) -> &mut T {
+ self.0.deref_mut()
+ }
+}
diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs
new file mode 100644
index 000000000..d7ebadee2
--- /dev/null
+++ b/common/src/float_ops.rs
@@ -0,0 +1,194 @@
+use num_bigint::{BigInt, ToBigInt};
+use num_traits::{Float, Signed, ToPrimitive, Zero};
+
+pub fn ufrexp(value: f64) -> (f64, i32) {
+ if 0.0 == value {
+ (0.0, 0i32)
+ } else {
+ let bits = value.to_bits();
+ let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022;
+ let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52);
+ (f64::from_bits(mantissa_bits), exponent)
+ }
+}
+
+pub fn eq_int(value: f64, other: &BigInt) -> bool {
+ if let (Some(self_int), Some(other_float)) = (value.to_bigint(), other.to_f64()) {
+ value == other_float && self_int == *other
+ } else {
+ false
+ }
+}
+
+pub fn lt_int(value: f64, other_int: &BigInt) -> bool {
+ match (value.to_bigint(), other_int.to_f64()) {
+ (Some(self_int), Some(other_float)) => value < other_float || self_int < *other_int,
+ // finite float, other_int too big for float,
+ // the result depends only on other_int’s sign
+ (Some(_), None) => other_int.is_positive(),
+ // infinite float must be bigger or lower than any int, depending on its sign
+ _ if value.is_infinite() => value.is_sign_negative(),
+ // NaN, always false
+ _ => false,
+ }
+}
+
+pub fn gt_int(value: f64, other_int: &BigInt) -> bool {
+ match (value.to_bigint(), other_int.to_f64()) {
+ (Some(self_int), Some(other_float)) => value > other_float || self_int > *other_int,
+ // finite float, other_int too big for float,
+ // the result depends only on other_int’s sign
+ (Some(_), None) => other_int.is_negative(),
+ // infinite float must be bigger or lower than any int, depending on its sign
+ _ if value.is_infinite() => value.is_sign_positive(),
+ // NaN, always false
+ _ => false,
+ }
+}
+
+pub fn parse_str(literal: &str) -> Option {
+ if literal.starts_with('_') || literal.ends_with('_') {
+ return None;
+ }
+
+ let mut buf = String::with_capacity(literal.len());
+ let mut last_tok: Option = None;
+ for c in literal.chars() {
+ if !(c.is_ascii_alphanumeric() || c == '_' || c == '+' || c == '-' || c == '.') {
+ return None;
+ }
+
+ if !c.is_ascii_alphanumeric() {
+ if let Some(l) = last_tok {
+ if !l.is_ascii_alphanumeric() && !(c == '.' && (l == '-' || l == '+')) {
+ return None;
+ }
+ }
+ }
+
+ if c != '_' {
+ buf.push(c);
+ }
+ last_tok = Some(c);
+ }
+
+ if let Ok(f) = lexical_core::parse(buf.as_bytes()) {
+ Some(f)
+ } else {
+ None
+ }
+}
+
+pub fn is_integer(v: f64) -> bool {
+ (v - v.round()).abs() < std::f64::EPSILON
+}
+
+pub fn to_string(value: f64) -> String {
+ let lit = format!("{:e}", value);
+ if let Some(position) = lit.find('e') {
+ let significand = &lit[..position];
+ let exponent = &lit[position + 1..];
+ let exponent = exponent.parse::().unwrap();
+ if exponent < 16 && exponent > -5 {
+ if is_integer(value) {
+ format!("{:.1?}", value)
+ } else {
+ value.to_string()
+ }
+ } else {
+ format!("{}e{:+#03}", significand, exponent)
+ }
+ } else {
+ value.to_string()
+ }
+}
+
+pub fn from_hex(s: &str) -> Option {
+ if let Ok(f) = hexf_parse::parse_hexf64(s, false) {
+ return Some(f);
+ }
+ match s.to_ascii_lowercase().as_str() {
+ "nan" => Some(f64::NAN),
+ "+nan" => Some(f64::NAN),
+ "-nan" => Some(f64::NAN),
+ "inf" => Some(f64::INFINITY),
+ "infinity" => Some(f64::INFINITY),
+ "+inf" => Some(f64::INFINITY),
+ "+infinity" => Some(f64::INFINITY),
+ "-inf" => Some(f64::NEG_INFINITY),
+ "-infinity" => Some(f64::NEG_INFINITY),
+ value => {
+ let mut hex = String::with_capacity(value.len());
+ let has_0x = value.contains("0x");
+ let has_p = value.contains('p');
+ let has_dot = value.contains('.');
+ let mut start = 0;
+
+ if !has_0x && value.starts_with('-') {
+ hex.push_str("-0x");
+ start += 1;
+ } else if !has_0x {
+ hex.push_str("0x");
+ if value.starts_with('+') {
+ start += 1;
+ }
+ }
+
+ for (index, ch) in value.chars().enumerate() {
+ if ch == 'p' && has_dot {
+ hex.push_str("p");
+ } else if ch == 'p' && !has_dot {
+ hex.push_str(".p");
+ } else if index >= start {
+ hex.push(ch);
+ }
+ }
+
+ if !has_p && has_dot {
+ hex.push_str("p0");
+ } else if !has_p && !has_dot {
+ hex.push_str(".p0")
+ }
+
+ hexf_parse::parse_hexf64(hex.as_str(), false).ok()
+ }
+ }
+}
+
+pub fn to_hex(value: f64) -> String {
+ let (mantissa, exponent, sign) = value.integer_decode();
+ let sign_fmt = if sign < 0 { "-" } else { "" };
+ match value {
+ value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt),
+ value if value.is_infinite() => format!("{}inf", sign_fmt),
+ value if value.is_nan() => "nan".to_owned(),
+ _ => {
+ const BITS: i16 = 52;
+ const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff;
+ format!(
+ "{}0x{:x}.{:013x}p{:+}",
+ sign_fmt,
+ mantissa >> BITS,
+ mantissa & FRACT_MASK,
+ exponent + BITS
+ )
+ }
+ }
+}
+
+#[test]
+fn test_to_hex() {
+ use rand::Rng;
+ for _ in 0..20000 {
+ let bytes = rand::thread_rng().gen::<[u64; 1]>();
+ let f = f64::from_bits(bytes[0]);
+ if !f.is_finite() {
+ continue;
+ }
+ let hex = to_hex(f);
+ // println!("{} -> {}", f, hex);
+ let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap();
+ // println!(" -> {}", roundtrip);
+ assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip);
+ }
+}
diff --git a/vm/src/pyhash.rs b/common/src/hash.rs
similarity index 52%
rename from vm/src/pyhash.rs
rename to common/src/hash.rs
index 60a6d132e..58c554f43 100644
--- a/vm/src/pyhash.rs
+++ b/common/src/hash.rs
@@ -1,12 +1,8 @@
use num_bigint::BigInt;
use num_traits::ToPrimitive;
+use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
-use crate::obj::objfloat;
-use crate::pyobject::PyObjectRef;
-use crate::pyobject::PyResult;
-use crate::vm::VirtualMachine;
-
pub type PyHash = i64;
pub type PyUHash = u64;
@@ -18,9 +14,26 @@ pub const MODULUS: PyUHash = (1 << BITS) - 1;
pub const INF: PyHash = 314_159;
pub const NAN: PyHash = 0;
pub const IMAG: PyHash = MULTIPLIER;
+pub const ALGO: &str = "siphasher13";
+pub const HASH_BITS: usize = std::mem::size_of::() * 8;
+// internally DefaultHasher uses 2 u64s as the seed, but
+// that's not guaranteed to be consistent across Rust releases
+// TODO: use something like the siphasher crate as our hash algorithm
+pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8;
// pub const CUTOFF: usize = 7;
+#[inline]
+pub fn mod_int(value: i64) -> PyHash {
+ value % MODULUS as i64
+}
+
+pub fn hash_value(data: &T) -> PyHash {
+ let mut hasher = DefaultHasher::new();
+ data.hash(&mut hasher);
+ mod_int(hasher.finish() as PyHash)
+}
+
pub fn hash_float(value: f64) -> PyHash {
// cpython _Py_HashDouble
if !value.is_finite() {
@@ -35,7 +48,7 @@ pub fn hash_float(value: f64) -> PyHash {
};
}
- let frexp = objfloat::ufrexp(value);
+ let frexp = super::float_ops::ufrexp(value);
// process 28 bits at a time; this should work well both for binary
// and hexadecimal floating point.
@@ -66,68 +79,44 @@ pub fn hash_float(value: f64) -> PyHash {
x as PyHash * value.signum() as PyHash
}
-pub fn hash_value(data: &T) -> PyHash {
- let mut hasher = std::collections::hash_map::DefaultHasher::new();
- data.hash(&mut hasher);
- hasher.finish() as PyHash
-}
-
-pub fn hash_iter<'a, I: std::iter::Iterator
- >(
- iter: I,
- vm: &VirtualMachine,
-) -> PyResult {
- let mut hasher = std::collections::hash_map::DefaultHasher::new();
+pub fn hash_iter<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result
+where
+ I: IntoIterator
- ,
+ F: Fn(&'a T) -> Result,
+{
+ let mut hasher = DefaultHasher::new();
for element in iter {
- let item_hash = vm._hash(&element)?;
+ let item_hash = hashf(element)?;
item_hash.hash(&mut hasher);
}
- Ok(hasher.finish() as PyHash)
+ Ok(mod_int(hasher.finish() as PyHash))
}
-pub fn hash_iter_unordered<'a, I: std::iter::Iterator
- >(
- iter: I,
- vm: &VirtualMachine,
-) -> PyResult {
+pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result
+where
+ I: IntoIterator
- ,
+ F: Fn(&'a T) -> Result,
+{
let mut hash: PyHash = 0;
for element in iter {
- let item_hash = vm._hash(element)?;
+ let item_hash = hashf(element)?;
// xor is commutative and hash should be independent of order
hash ^= item_hash;
}
- Ok(hash)
+ Ok(mod_int(hash))
}
pub fn hash_bigint(value: &BigInt) -> PyHash {
- match value.to_i64() {
- Some(i64_value) => (i64_value % MODULUS as i64),
- None => (value % MODULUS).to_i64().unwrap(),
- }
+ value.to_i64().map_or_else(
+ || {
+ (value % MODULUS).to_i64().unwrap_or_else(||
+ // guaranteed to be safe by mod
+ unsafe { std::hint::unreachable_unchecked() })
+ },
+ mod_int,
+ )
}
-#[pystruct_sequence(name = "sys.hash_info")]
-#[derive(Debug)]
-pub(crate) struct PyHashInfo {
- width: usize,
- modulus: PyUHash,
- inf: PyHash,
- nan: PyHash,
- imag: PyHash,
- algorithm: &'static str,
- hash_bits: usize,
- seed_bits: usize,
-}
-impl PyHashInfo {
- pub const INFO: Self = PyHashInfo {
- width: BITS,
- modulus: MODULUS,
- inf: INF,
- nan: NAN,
- imag: IMAG,
- algorithm: "siphash13",
- hash_bits: std::mem::size_of::() * 8,
- // internally hash_map::DefaultHasher uses 2 u64s as the seed, but
- // that's not guaranteed to be consistent across Rust releases
- // TODO: use something like the siphasher crate as our hash algorithm
- seed_bits: std::mem::size_of::() * 2 * 8,
- };
+pub fn hash_str(value: &str) -> PyHash {
+ hash_value(value)
}
diff --git a/common/src/lib.rs b/common/src/lib.rs
new file mode 100644
index 000000000..7519f9d94
--- /dev/null
+++ b/common/src/lib.rs
@@ -0,0 +1,7 @@
+//! A crate to hold types and functions common to all rustpython components.
+
+pub mod cell;
+pub mod float_ops;
+pub mod hash;
+pub mod rc;
+pub mod str;
diff --git a/common/src/rc.rs b/common/src/rc.rs
new file mode 100644
index 000000000..8021b6112
--- /dev/null
+++ b/common/src/rc.rs
@@ -0,0 +1,14 @@
+#[cfg(not(feature = "threading"))]
+use std::rc::{Rc, Weak};
+#[cfg(feature = "threading")]
+use std::sync::{Arc, Weak};
+
+// type aliases instead of newtypes because you can't do `fn method(self: PyRc)` with a
+// newtype; requires the arbitrary_self_types unstable feature
+
+#[cfg(feature = "threading")]
+pub type PyRc = Arc;
+#[cfg(not(feature = "threading"))]
+pub type PyRc = Rc;
+
+pub type PyWeak = Weak;
diff --git a/common/src/str.rs b/common/src/str.rs
new file mode 100644
index 000000000..a89c25f49
--- /dev/null
+++ b/common/src/str.rs
@@ -0,0 +1,48 @@
+pub fn get_chars(s: &str, range: std::ops::Range) -> &str {
+ let mut chars = s.chars();
+ for _ in 0..range.start {
+ let _ = chars.next();
+ }
+ let start = chars.as_str();
+ for _ in range {
+ let _ = chars.next();
+ }
+ let end = chars.as_str();
+ &start[..start.len() - end.len()]
+}
+
+pub fn zfill(bytes: &[u8], width: usize) -> Vec {
+ if width <= bytes.len() {
+ bytes.to_vec()
+ } else {
+ let (sign, s) = match bytes.first() {
+ Some(_sign @ b'+') | Some(_sign @ b'-') => {
+ (unsafe { bytes.get_unchecked(..1) }, &bytes[1..])
+ }
+ _ => (&b""[..], bytes),
+ };
+ let mut filled = Vec::new();
+ filled.extend_from_slice(sign);
+ filled.extend(std::iter::repeat(b'0').take(width - bytes.len()));
+ filled.extend_from_slice(s);
+ filled
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_get_chars() {
+ let s = "0123456789";
+ assert_eq!(get_chars(s, 3..7), "3456");
+ assert_eq!(get_chars(s, 3..7), &s[3..7]);
+
+ let s = "0유니코드 문자열9";
+ assert_eq!(get_chars(s, 3..7), "코드 문");
+
+ let s = "0😀😃😄😁😆😅😂🤣9";
+ assert_eq!(get_chars(s, 3..7), "😄😁😆😅");
+ }
+}
diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml
index d968cb119..6846c4386 100644
--- a/compiler/Cargo.toml
+++ b/compiler/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
[dependencies]
indexmap = "1.0"
-itertools = "0.8.0"
+itertools = "0.9"
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
rustpython-parser = { path = "../parser", version = "0.1.1" }
num-complex = { version = "0.2", features = ["serde"] }
diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs
index 1b9707dac..d121fa19d 100644
--- a/compiler/src/symboltable.rs
+++ b/compiler/src/symboltable.rs
@@ -97,10 +97,13 @@ pub struct Symbol {
pub name: String,
// pub table: SymbolTableRef,
pub scope: SymbolScope,
+ // TODO: Use bitflags replace
pub is_referenced: bool,
pub is_assigned: bool,
pub is_parameter: bool,
pub is_free: bool,
+ pub is_annotated: bool,
+ pub is_imported: bool,
// indicates if the symbol gets a value assigned by a named expression in a comprehension
// this is required to correct the scope in the analysis.
@@ -121,6 +124,8 @@ impl Symbol {
is_assigned: false,
is_parameter: false,
is_free: false,
+ is_annotated: false,
+ is_imported: false,
is_assign_namedexpr_in_comprehension: false,
is_iter: false,
}
@@ -396,7 +401,10 @@ enum SymbolUsage {
Nonlocal,
Used,
Assigned,
+ Imported,
+ AnnotationAssigned,
Parameter,
+ AnnotationParameter,
AssignedNamedExprInCompr,
Iter,
}
@@ -462,7 +470,12 @@ impl SymbolTableBuilder {
}
fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
- self.register_name(¶meter.arg, SymbolUsage::Parameter)
+ let usage = if parameter.annotation.is_some() {
+ SymbolUsage::AnnotationParameter
+ } else {
+ SymbolUsage::Parameter
+ };
+ self.register_name(¶meter.arg, usage)
}
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -566,12 +579,12 @@ impl SymbolTableBuilder {
for name in names {
if let Some(alias) = &name.alias {
// `import mymodule as myalias`
- self.register_name(alias, SymbolUsage::Assigned)?;
+ self.register_name(alias, SymbolUsage::Imported)?;
} else {
// `import module`
self.register_name(
name.symbol.split('.').next().unwrap(),
- SymbolUsage::Assigned,
+ SymbolUsage::Imported,
)?;
}
}
@@ -603,7 +616,12 @@ impl SymbolTableBuilder {
annotation,
value,
} => {
- self.scan_expression(target, &ExpressionContext::Store)?;
+ // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
+ if let ast::ExpressionType::Identifier { ref name } = target.node {
+ self.register_name(name, SymbolUsage::AnnotationAssigned)?;
+ } else {
+ self.scan_expression(target, &ExpressionContext::Store)?;
+ }
self.scan_expression(annotation, &ExpressionContext::Load)?;
if let Some(value) = value {
self.scan_expression(value, &ExpressionContext::Load)?;
@@ -986,9 +1004,21 @@ impl SymbolTableBuilder {
});
}
}
+ SymbolUsage::Imported => {
+ symbol.is_assigned = true;
+ symbol.is_imported = true;
+ }
SymbolUsage::Parameter => {
symbol.is_parameter = true;
}
+ SymbolUsage::AnnotationParameter => {
+ symbol.is_parameter = true;
+ symbol.is_annotated = true;
+ }
+ SymbolUsage::AnnotationAssigned => {
+ symbol.is_assigned = true;
+ symbol.is_annotated = true;
+ }
SymbolUsage::Assigned => {
symbol.is_assigned = true;
}
diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs
index 3cb925b72..cec3cb605 100644
--- a/derive/src/compile_bytecode.rs
+++ b/derive/src/compile_bytecode.rs
@@ -274,6 +274,8 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result Result ::#crate_name::bytecode::FrozenModule {
+ m.insert(#module_name.into(), ::#crate_name::bytecode::FrozenModule {
code: ::#crate_name::bytecode::CodeObject::from_bytes(
#bytes
).expect("Deserializing CodeObject failed"),
package: #package,
- }
+ });
}
});
let output = quote! {
- ({
- use ::#crate_name::__exports::hashmap;
- hashmap! {
- #(#modules),*
- }
- })
+ {
+ let mut m = ::std::collections::HashMap::with_capacity(#modules_len);
+ #(#modules)*
+ m
+ }
};
Ok(output)
diff --git a/derive/src/lib.rs b/derive/src/lib.rs
index 46547071e..da5a7dd42 100644
--- a/derive/src/lib.rs
+++ b/derive/src/lib.rs
@@ -59,17 +59,7 @@ pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream {
result_to_tokens(pyclass::impl_pystruct_sequence(attr, item))
}
-fn result_to_tokens_expr(result: Result) -> TokenStream {
- let tokens2 = result.unwrap_or_else(ToTokens::into_token_stream);
- let ret = quote::quote! {
- macro_rules! __proc_macro_call {
- () => {{ #tokens2 }}
- }
- };
- ret.into()
-}
-
#[proc_macro]
pub fn py_compile_bytecode(input: TokenStream) -> TokenStream {
- result_to_tokens_expr(compile_bytecode::impl_py_compile_bytecode(input.into()))
+ result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into()))
}
diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs
index 11af6cdcd..7c1d4a3b9 100644
--- a/derive/src/pyclass.rs
+++ b/derive/src/pyclass.rs
@@ -1,5 +1,8 @@
use super::Diagnostic;
-use crate::util::{def_to_name, path_eq, strip_prefix, ItemIdent, ItemMeta};
+use crate::util::{
+ def_to_name, module_class_name, optional_attribute_arg, path_eq, strip_prefix, ItemIdent,
+ ItemMeta,
+};
use proc_macro2::{Span, TokenStream as TokenStream2};
use quote::{quote, quote_spanned, ToTokens};
use std::collections::{HashMap, HashSet};
@@ -478,7 +481,9 @@ pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result Result, vm: &rustpython_vm::VirtualMachine) -> ::rustpython_vm::pyobject::PyResult {
+ let s = if let Some(_guard) = rustpython_vm::vm::ReprGuard::enter(zelf.as_object()) {
+ let field_names = vec![#(stringify!(#field_names)),*];
+ let mut fields = Vec::new();
+ for (value, field_name) in zelf.as_slice().iter().zip(field_names.iter()) {
+ let s = vm.to_repr(value)?;
+ fields.push(format!("{}: {}", field_name, s));
+ }
+
+ if fields.len() == 1 {
+ format!("{}({},)", #py_class_name, fields[0])
+ } else {
+ format!("{}({})", #py_class_name, fields.join(", "))
+ }
+ } else {
+ concat!(#class_name, "(...)").to_string()
+ };
+ Ok(s)
+ }
}
+
impl ::rustpython_vm::pyobject::PyClassImpl for #ty {
fn impl_extend_class(
ctx: &::rustpython_vm::pyobject::PyContext,
class: &::rustpython_vm::obj::objtype::PyClassRef,
) {
#(#properties)*
+ class.set_str_attr("__repr__", ctx.new_method(Self::repr));
}
fn make_class(
diff --git a/derive/src/pymodule.rs b/derive/src/pymodule.rs
index 7a2669647..b69c5fbcc 100644
--- a/derive/src/pymodule.rs
+++ b/derive/src/pymodule.rs
@@ -15,7 +15,7 @@ fn meta_to_vec(meta: Meta) -> Result, Meta> {
#[derive(Default)]
struct Module {
- items: HashSet,
+ items: HashSet<(ModuleItem, Vec)>,
}
#[derive(PartialEq, Eq, Hash)]
@@ -25,13 +25,13 @@ enum ModuleItem {
}
impl Module {
- fn add_item(&mut self, item: ModuleItem, span: Span) -> Result<(), Diagnostic> {
+ fn add_item(&mut self, item: (ModuleItem, Vec), span: Span) -> Result<(), Diagnostic> {
if self.items.insert(item) {
Ok(())
} else {
Err(Diagnostic::span_error(
span,
- "Duplicate #[py*] attribute on pyimpl".to_owned(),
+ "Duplicate #[py*] attribute on pymodule".to_owned(),
))
}
}
@@ -70,12 +70,35 @@ impl Module {
})
}
+ fn extract_struct_sequence(ident: &Ident, meta: Meta) -> Result {
+ let nesteds = meta_to_vec(meta).map_err(|meta| {
+ err_span!(
+ meta,
+ "#[pystruct_sequence = \"...\"] cannot be a name/value, you probably meant \
+ #[pystruct_sequence(name = \"...\")]",
+ )
+ })?;
+
+ let item_meta = ItemMeta::from_nested_meta(
+ "pystruct_sequence",
+ &ident,
+ &nesteds,
+ ItemMeta::STRUCT_SEQUENCE_NAMES,
+ )?;
+ Ok(ModuleItem::Class {
+ item_ident: ident.clone(),
+ py_name: item_meta.simple_name()?,
+ })
+ }
+
fn extract_item_from_syn(
&mut self,
attrs: &mut Vec,
ident: &Ident,
) -> Result<(), Diagnostic> {
let mut attr_idxs = Vec::new();
+ let mut items = Vec::new();
+ let mut cfgs = Vec::new();
for (i, meta) in attrs
.iter()
.filter_map(|attr| attr.parse_meta().ok())
@@ -86,17 +109,28 @@ impl Module {
Some(name) => name,
None => continue,
};
- let item = match name.to_string().as_str() {
+ match name.to_string().as_str() {
"pyfunction" => {
attr_idxs.push(i);
- Self::extract_function(ident, meta)?
+ items.push((Self::extract_function(ident, meta)?, meta_span));
+ }
+ "pyclass" => {
+ items.push((Self::extract_class(ident, meta)?, meta_span));
+ }
+ "pystruct_sequence" => {
+ items.push((Self::extract_struct_sequence(ident, meta)?, meta_span));
+ }
+ "cfg" => {
+ cfgs.push(meta);
+ continue;
}
- "pyclass" => Self::extract_class(ident, meta)?,
_ => {
continue;
}
};
- self.add_item(item, meta_span)?;
+ }
+ for (item, meta) in items {
+ self.add_item((item, cfgs.clone()), meta)?;
}
let mut i = 0;
let mut attr_idxs = &*attr_idxs;
@@ -130,7 +164,7 @@ fn extract_module_items(
);
}
- let functions = module.items.into_iter().map(|item| match item {
+ let functions = module.items.into_iter().map(|(item, cfgs)| match item {
ModuleItem::Function {
item_ident,
py_name,
@@ -140,6 +174,7 @@ fn extract_module_items(
#module_name.to_owned(),
#py_name.to_owned()));
quote! {
+ #( #[ #cfgs ])*
vm.__module_set_attr(&module, #py_name, vm.ctx#new_func).unwrap();
}
}
@@ -192,7 +227,7 @@ pub fn impl_pymodule(attr: AttributeArgs, item: Item) -> Result Result, Diagnostic> {
+ let mut arg_value = None;
+ for attr in attr {
+ if let NestedMeta::Meta(meta) = attr {
+ if let Meta::NameValue(name_value) = meta {
+ if path_eq(&name_value.path, arg_name) {
+ if let Lit::Str(lit) = name_value.lit {
+ arg_value = Some(lit.value());
+ } else {
+ bail_span!(
+ name_value.lit,
+ "#[{}({} = ...)] must be a string",
+ attr_name,
+ arg_name
+ );
+ }
+ }
+ }
+ }
+ }
+ Ok(arg_value)
+}
+
+pub fn module_class_name(mod_name: Option, class_name: &str) -> String {
+ if let Some(mod_name) = mod_name {
+ format!("{}.{}", mod_name, class_name)
+ } else {
+ class_name.into()
+ }
+}
+
pub fn strip_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
if s.starts_with(prefix) {
Some(&s[prefix.len()..])
@@ -53,6 +88,7 @@ pub struct ItemMeta<'a> {
impl<'a> ItemMeta<'a> {
pub const SIMPLE_NAMES: &'static [&'static str] = &["name"];
+ pub const STRUCT_SEQUENCE_NAMES: &'static [&'static str] = &["module", "name"];
pub const ATTRIBUTE_NAMES: &'static [&'static str] = &["name", "magic"];
pub const PROPERTY_NAMES: &'static [&'static str] = &["name", "magic", "setter"];
diff --git a/examples/freeze/Cargo.toml b/examples/freeze/Cargo.toml
deleted file mode 100644
index eafbed069..000000000
--- a/examples/freeze/Cargo.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-[package]
-name = "rustpython_freeze"
-version = "0.1.2"
-authors = ["RustPython Team"]
-license = "MIT"
-description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to frozen executable"
-repository = "https://github.com/RustPython/RustPython/tree/master/freeze"
-edition = "2018"
-
-[[bin]]
-name = "rustpython_freeze"
-path = "src/main.rs"
-
-
-[dependencies]
-rustpython-vm = { path = "../../vm", default-features = false }
diff --git a/examples/freeze/src/main.rs b/examples/freeze/main.rs
similarity index 56%
rename from examples/freeze/src/main.rs
rename to examples/freeze/main.rs
index 0bd9f8f5f..39a335619 100644
--- a/examples/freeze/src/main.rs
+++ b/examples/freeze/main.rs
@@ -7,8 +7,10 @@ fn main() -> vm::pyobject::PyResult<()> {
let scope = vm.new_scope_with_builtins();
+ // the file parameter is relevant to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR:
+ // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates
let modules: HashMap =
- vm::py_compile_bytecode!(file = "freeze.py");
+ vm::py_compile_bytecode!(file = "examples/freeze/freeze.py");
let res = vm.run_code_obj(
vm.ctx
@@ -17,7 +19,7 @@ fn main() -> vm::pyobject::PyResult<()> {
);
if let Err(err) = res {
- vm::exceptions::print_exception(&vm, &err)
+ vm::exceptions::print_exception(&vm, err);
}
Ok(())
diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs
index 92d78eb1d..2ad627f6c 100644
--- a/examples/mini_repl.rs
+++ b/examples/mini_repl.rs
@@ -40,7 +40,7 @@ macro_rules! add_python_function {
// inserts the first function found in the module into the provided scope.
$scope.globals.set_item(
- &def.obj_name,
+ def.obj_name.as_str(),
$vm.context().new_pyfunction(
vm::obj::objcode::PyCode::new(*def.clone()).into_ref(&$vm),
$scope.clone(),
@@ -103,7 +103,7 @@ fn main() -> vm::pyobject::PyResult<()> {
}
}
Err(e) => {
- vm::exceptions::print_exception(&vm, &e);
+ vm::exceptions::print_exception(&vm, e);
}
}
}
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index d4c0c7e7a..76a62c680 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -9,12 +9,12 @@ license = "MIT"
edition = "2018"
[build-dependencies]
-lalrpop="0.17"
+lalrpop="0.19"
[dependencies]
-lalrpop-util="0.17"
-log="0.4.1"
-num-bigint = "0.2"
+lalrpop-util = "0.19"
+log = "0.4.1"
+num-bigint = "0.3"
num-traits = "0.2"
unic-emoji-char = "0.9"
unic-ucd-ident = "0.9"
diff --git a/src/main.rs b/src/main.rs
index 604b891b4..e9f202ae6 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -64,18 +64,18 @@ fn main() {
return;
}
if let Ok(s) = vm.to_str(&arg) {
- println!("{}", s);
+ eprintln!("{}", s);
}
}
}),
_ => {
if let Ok(r) = vm.to_repr(args.as_object()) {
- println!("{}", r);
+ eprintln!("{}", r);
}
}
}
} else {
- print_exception(&vm, &err);
+ print_exception(&vm, err);
}
process::exit(1);
}
@@ -158,6 +158,17 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> {
Arg::with_name("ignore-environment")
.short("E")
.help("Ignore environment variables PYTHON* such as PYTHONPATH"),
+ )
+ .arg(
+ Arg::with_name("isolate")
+ .short("I")
+ .help("isolate Python from the user's environment (implies -E and -s)"),
+ )
+ .arg(
+ Arg::with_name("implementation-option")
+ .short("X")
+ .takes_value(true)
+ .help("set implementation-specific option"),
);
#[cfg(feature = "flame-it")]
let app = app
@@ -179,21 +190,22 @@ fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> {
/// Create settings by examining command line arguments and environment
/// variables.
fn create_settings(matches: &ArgMatches) -> PySettings {
- let ignore_environment = matches.is_present("ignore-environment");
+ let ignore_environment =
+ matches.is_present("ignore-environment") || matches.is_present("isolate");
let mut settings: PySettings = Default::default();
settings.ignore_environment = ignore_environment;
// add the current directory to sys.path
settings.path_list.push("".to_owned());
+ // BUILDTIME_RUSTPYTHONPATH should be set when distributing
if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
settings.path_list.extend(
std::env::split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()),
)
- } else if option_env!("RUSTPYTHONPATH").is_none() {
- settings
- .path_list
- .push(concat!(env!("CARGO_MANIFEST_DIR"), "/Lib").to_owned());
+ } else {
+ #[cfg(all(feature = "pylib", not(feature = "freeze-stdlib")))]
+ settings.path_list.push(pylib::LIB_PATH.to_owned());
}
if !ignore_environment {
@@ -232,6 +244,7 @@ fn create_settings(matches: &ArgMatches) -> PySettings {
settings.no_site = matches.is_present("no-site");
if matches.is_present("no-user-site")
+ || matches.is_present("isolate")
|| (!ignore_environment && env::var_os("PYTHONNOUSERSITE").is_some())
{
settings.no_user_site = true;
diff --git a/src/shell.rs b/src/shell.rs
index 2785c8ae1..c8ca36122 100644
--- a/src/shell.rs
+++ b/src/shell.rs
@@ -131,7 +131,7 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
repl.save_history(&repl_history_path).unwrap();
return Err(exc);
}
- print_exception(vm, &exc);
+ print_exception(vm, exc);
}
}
repl.save_history(&repl_history_path).unwrap();
diff --git a/tests/snippets/builtin_format.py b/tests/snippets/builtin_format.py
index 46b627ad5..a30f300d4 100644
--- a/tests/snippets/builtin_format.py
+++ b/tests/snippets/builtin_format.py
@@ -27,3 +27,4 @@ assert f'{100:_}' == '100'
assert f'{1024:_}' == '1_024'
assert f'{65536:,}' == '65,536'
assert f'{4294967296:,}' == '4,294,967,296'
+assert 'F' == "{0:{base}}".format(15, base="X")
diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py
index 0eb820170..5f312efb8 100644
--- a/tests/snippets/stdlib_os.py
+++ b/tests/snippets/stdlib_os.py
@@ -10,16 +10,17 @@ assert fd > 0
os.close(fd)
assert_raises(OSError, lambda: os.read(fd, 10))
-assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY))
-assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY))
-assert_raises(FileNotFoundError, lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2'))
+assert_raises(FileNotFoundError,
+ lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY))
+assert_raises(FileNotFoundError,
+ lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY))
+assert_raises(FileNotFoundError,
+ lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2'))
try:
- os.open('DOES_NOT_EXIST', 0)
+ os.open('DOES_NOT_EXIST', 0)
except OSError as err:
- assert err.errno == 2
-
-
+ assert err.errno == 2
assert os.O_RDONLY == 0
assert os.O_WRONLY == 1
@@ -28,7 +29,7 @@ assert os.O_RDWR == 2
ENV_KEY = "TEST_ENV_VAR"
ENV_VALUE = "value"
-assert os.getenv(ENV_KEY) == None
+assert os.getenv(ENV_KEY) is None
assert ENV_KEY not in os.environ
assert os.getenv(ENV_KEY, 5) == 5
os.environ[ENV_KEY] = ENV_VALUE
@@ -36,31 +37,32 @@ assert ENV_KEY in os.environ
assert os.getenv(ENV_KEY) == ENV_VALUE
del os.environ[ENV_KEY]
assert ENV_KEY not in os.environ
-assert os.getenv(ENV_KEY) == None
+assert os.getenv(ENV_KEY) is None
if os.name == "posix":
- os.putenv(ENV_KEY, ENV_VALUE)
- os.unsetenv(ENV_KEY)
- assert os.getenv(ENV_KEY) == None
+ os.putenv(ENV_KEY, ENV_VALUE)
+ os.unsetenv(ENV_KEY)
+ assert os.getenv(ENV_KEY) is None
assert os.curdir == "."
assert os.pardir == ".."
assert os.extsep == "."
if os.name == "nt":
- assert os.sep == "\\"
- assert os.linesep == "\r\n"
- assert os.altsep == "/"
- assert os.pathsep == ";"
+ assert os.sep == "\\"
+ assert os.linesep == "\r\n"
+ assert os.altsep == "/"
+ assert os.pathsep == ";"
else:
- assert os.sep == "/"
- assert os.linesep == "\n"
- assert os.altsep == None
- assert os.pathsep == ":"
+ assert os.sep == "/"
+ assert os.linesep == "\n"
+ assert os.altsep is None
+ assert os.pathsep == ":"
assert os.fspath("Testing") == "Testing"
assert os.fspath(b"Testing") == b"Testing"
-assert_raises(TypeError, lambda: os.fspath([1,2,3]))
+assert_raises(TypeError, lambda: os.fspath([1, 2, 3]))
+
class TestWithTempDir():
def __enter__(self):
@@ -69,7 +71,8 @@ class TestWithTempDir():
else:
base_folder = "/tmp"
- name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time())))
+ name = os.path.join(base_folder,
+ "rustpython_test_os_" + str(int(time.time())))
while os.path.isdir(name):
name = name + "_"
@@ -81,6 +84,7 @@ class TestWithTempDir():
def __exit__(self, exc_type, exc_val, exc_tb):
pass
+
class TestWithTempCurrentDir():
def __enter__(self):
self.prev_cwd = os.getcwd()
@@ -100,174 +104,244 @@ CONTENT2 = b"rustpython"
CONTENT3 = b"BOYA"
with TestWithTempDir() as tmpdir:
- fname = os.path.join(tmpdir, FILE_NAME)
- fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
- assert os.write(fd, CONTENT2) == len(CONTENT2)
- os.close(fd)
+ fname = os.path.join(tmpdir, FILE_NAME)
+ fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
+ assert os.write(fd, CONTENT2) == len(CONTENT2)
+ os.close(fd)
- fd = os.open(fname, os.O_WRONLY | os.O_APPEND)
- assert os.write(fd, CONTENT3) == len(CONTENT3)
- os.close(fd)
+ fd = os.open(fname, os.O_WRONLY | os.O_APPEND)
+ assert os.write(fd, CONTENT3) == len(CONTENT3)
+ os.close(fd)
- assert_raises(FileExistsError, lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL))
+ assert_raises(FileExistsError,
+ lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL))
- fd = os.open(fname, os.O_RDONLY)
- assert os.read(fd, len(CONTENT2)) == CONTENT2
- assert os.read(fd, len(CONTENT3)) == CONTENT3
- os.close(fd)
+ fd = os.open(fname, os.O_RDONLY)
+ assert os.read(fd, len(CONTENT2)) == CONTENT2
+ assert os.read(fd, len(CONTENT3)) == CONTENT3
+ os.close(fd)
- fname3 = os.path.join(tmpdir, FILE_NAME3)
- os.rename(fname, fname3)
- assert os.path.exists(fname) == False
- assert os.path.exists(fname3) == True
+ fname3 = os.path.join(tmpdir, FILE_NAME3)
+ os.rename(fname, fname3)
+ assert os.path.exists(fname) is False
+ assert os.path.exists(fname3) is True
- fd = os.open(fname3, 0)
- assert os.read(fd, len(CONTENT2) + len(CONTENT3)) == CONTENT2 + CONTENT3
- os.close(fd)
+ fd = os.open(fname3, 0)
+ assert os.read(fd, len(CONTENT2) + len(CONTENT3)) == CONTENT2 + CONTENT3
+ os.close(fd)
- assert not os.isatty(fd)
+ assert not os.isatty(fd)
- # TODO: get os.lseek working on windows
- if os.name != 'nt':
- fd = os.open(fname3, 0)
- assert os.read(fd, len(CONTENT2)) == CONTENT2
- assert os.read(fd, len(CONTENT3)) == CONTENT3
- os.lseek(fd, len(CONTENT2), os.SEEK_SET)
- assert os.read(fd, len(CONTENT3)) == CONTENT3
- os.close(fd)
+ # TODO: get os.lseek working on windows
+ if os.name != 'nt':
+ fd = os.open(fname3, 0)
+ assert os.read(fd, len(CONTENT2)) == CONTENT2
+ assert os.read(fd, len(CONTENT3)) == CONTENT3
+ os.lseek(fd, len(CONTENT2), os.SEEK_SET)
+ assert os.read(fd, len(CONTENT3)) == CONTENT3
+ os.close(fd)
- os.rename(fname3, fname)
- assert os.path.exists(fname3) == False
- assert os.path.exists(fname) == True
+ os.rename(fname3, fname)
+ assert os.path.exists(fname3) is False
+ assert os.path.exists(fname) is True
- # wait a little bit to ensure that the file times aren't the same
- time.sleep(0.1)
+ # wait a little bit to ensure that the file times aren't the same
+ time.sleep(0.1)
- fname2 = os.path.join(tmpdir, FILE_NAME2)
- with open(fname2, "wb"):
- pass
- folder = os.path.join(tmpdir, FOLDER)
- os.mkdir(folder)
+ fname2 = os.path.join(tmpdir, FILE_NAME2)
+ with open(fname2, "wb"):
+ pass
+ folder = os.path.join(tmpdir, FOLDER)
+ os.mkdir(folder)
- symlink_file = os.path.join(tmpdir, SYMLINK_FILE)
- os.symlink(fname, symlink_file)
- symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER)
- os.symlink(folder, symlink_folder)
+ symlink_file = os.path.join(tmpdir, SYMLINK_FILE)
+ os.symlink(fname, symlink_file)
+ symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER)
+ os.symlink(folder, symlink_folder)
- names = set()
- paths = set()
- dirs = set()
- dirs_no_symlink = set()
- files = set()
- files_no_symlink = set()
- symlinks = set()
- for dir_entry in os.scandir(tmpdir):
- names.add(dir_entry.name)
- paths.add(dir_entry.path)
- if dir_entry.is_dir():
- assert stat.S_ISDIR(dir_entry.stat().st_mode) == True
- dirs.add(dir_entry.name)
- if dir_entry.is_dir(follow_symlinks=False):
- assert stat.S_ISDIR(dir_entry.stat().st_mode) == True
- dirs_no_symlink.add(dir_entry.name)
- if dir_entry.is_file():
- files.add(dir_entry.name)
- assert stat.S_ISREG(dir_entry.stat().st_mode) == True
- if dir_entry.is_file(follow_symlinks=False):
- files_no_symlink.add(dir_entry.name)
- assert stat.S_ISREG(dir_entry.stat().st_mode) == True
- if dir_entry.is_symlink():
- symlinks.add(dir_entry.name)
+ names = set()
+ paths = set()
+ dirs = set()
+ dirs_no_symlink = set()
+ files = set()
+ files_no_symlink = set()
+ symlinks = set()
+ for dir_entry in os.scandir(tmpdir):
+ names.add(dir_entry.name)
+ paths.add(dir_entry.path)
+ if dir_entry.is_dir():
+ assert stat.S_ISDIR(dir_entry.stat().st_mode) is True
+ dirs.add(dir_entry.name)
+ if dir_entry.is_dir(follow_symlinks=False):
+ assert stat.S_ISDIR(dir_entry.stat().st_mode) is True
+ dirs_no_symlink.add(dir_entry.name)
+ if dir_entry.is_file():
+ files.add(dir_entry.name)
+ assert stat.S_ISREG(dir_entry.stat().st_mode) is True
+ if dir_entry.is_file(follow_symlinks=False):
+ files_no_symlink.add(dir_entry.name)
+ assert stat.S_ISREG(dir_entry.stat().st_mode) is True
+ if dir_entry.is_symlink():
+ symlinks.add(dir_entry.name)
- assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER])
- assert paths == set([fname, fname2, folder, symlink_file, symlink_folder])
- assert dirs == set([FOLDER, SYMLINK_FOLDER])
- assert dirs_no_symlink == set([FOLDER])
- assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE])
- assert files_no_symlink == set([FILE_NAME, FILE_NAME2])
- assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER])
+ assert names == set(
+ [FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER])
+ assert paths == set([fname, fname2, folder, symlink_file, symlink_folder])
+ assert dirs == set([FOLDER, SYMLINK_FOLDER])
+ assert dirs_no_symlink == set([FOLDER])
+ assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE])
+ assert files_no_symlink == set([FILE_NAME, FILE_NAME2])
+ assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER])
- # Stat
- stat_res = os.stat(fname)
- print(stat_res.st_mode)
- assert stat.S_ISREG(stat_res.st_mode) == True
- print(stat_res.st_ino)
- print(stat_res.st_dev)
- print(stat_res.st_nlink)
- print(stat_res.st_uid)
- print(stat_res.st_gid)
- print(stat_res.st_size)
- assert stat_res.st_size == len(CONTENT2) + len(CONTENT3)
- print(stat_res.st_atime)
- print(stat_res.st_ctime)
- print(stat_res.st_mtime)
- # test that it all of these times are greater than the 10 May 2019, when this test was written
- assert stat_res.st_atime > 1557500000
- assert stat_res.st_ctime > 1557500000
- assert stat_res.st_mtime > 1557500000
+ # Stat
+ stat_res = os.stat(fname)
+ print(stat_res.st_mode)
+ assert stat.S_ISREG(stat_res.st_mode) is True
+ print(stat_res.st_ino)
+ print(stat_res.st_dev)
+ print(stat_res.st_nlink)
+ print(stat_res.st_uid)
+ print(stat_res.st_gid)
+ print(stat_res.st_size)
+ assert stat_res.st_size == len(CONTENT2) + len(CONTENT3)
+ print(stat_res.st_atime)
+ print(stat_res.st_ctime)
+ print(stat_res.st_mtime)
+ # test that it all of these times are greater than the 10 May 2019,
+ # when this test was written
+ assert stat_res.st_atime > 1557500000
+ assert stat_res.st_ctime > 1557500000
+ assert stat_res.st_mtime > 1557500000
- bytes_stats_res = os.stat(fname.encode())
+ bytes_stats_res = os.stat(fname.encode())
- stat_file2 = os.stat(fname2)
- print(stat_file2.st_ctime)
- assert stat_file2.st_ctime > stat_res.st_ctime
+ stat_file2 = os.stat(fname2)
+ print(stat_file2.st_ctime)
+ assert stat_file2.st_ctime > stat_res.st_ctime
- # wait a little bit to ensures that the access/modify time will change
- time.sleep(0.1)
+ # wait a little bit to ensures that the access/modify time will change
+ time.sleep(0.1)
- old_atime = stat_res.st_atime
- old_mtime = stat_res.st_mtime
+ old_atime = stat_res.st_atime
+ old_mtime = stat_res.st_mtime
- fd = os.open(fname, os.O_RDWR)
- os.write(fd, CONTENT)
- os.fsync(fd)
+ fd = os.open(fname, os.O_RDWR)
+ os.write(fd, CONTENT)
+ os.fsync(fd)
- # wait a little bit to ensures that the access/modify time is different
- time.sleep(0.1)
+ # wait a little bit to ensures that the access/modify time is different
+ time.sleep(0.1)
- os.read(fd, 1)
- os.fsync(fd)
- os.close(fd)
+ os.read(fd, 1)
+ os.fsync(fd)
+ os.close(fd)
- # retrieve update file stats
- stat_res = os.stat(fname)
- print(stat_res.st_atime)
- print(stat_res.st_ctime)
- print(stat_res.st_mtime)
- if os.name != "nt":
- # access time on windows has a resolution ranging from 1 hour to 1 day
- # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime
- assert stat_res.st_atime > old_atime, "Access time should be update"
- assert stat_res.st_atime > stat_res.st_mtime
- assert stat_res.st_mtime > old_mtime, "Modified time should be update"
+ # retrieve update file stats
+ stat_res = os.stat(fname)
+ print(stat_res.st_atime)
+ print(stat_res.st_ctime)
+ print(stat_res.st_mtime)
+ if os.name != "nt":
+ # access time on windows has a resolution ranging from 1 hour to 1 day
+ # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime
+ assert stat_res.st_atime > old_atime, "Access time should be update"
+ assert stat_res.st_atime > stat_res.st_mtime
+ assert stat_res.st_mtime > old_mtime, "Modified time should be update"
- # stat default is follow_symlink=True
- os.stat(fname).st_ino == os.stat(symlink_file).st_ino
- os.stat(fname).st_mode == os.stat(symlink_file).st_mode
+ # stat default is follow_symlink=True
+ os.stat(fname).st_ino == os.stat(symlink_file).st_ino
+ os.stat(fname).st_mode == os.stat(symlink_file).st_mode
- os.stat(fname, follow_symlinks=False).st_ino == os.stat(symlink_file, follow_symlinks=False).st_ino
- os.stat(fname, follow_symlinks=False).st_mode == os.stat(symlink_file, follow_symlinks=False).st_mode
+ os.stat(fname, follow_symlinks=False).st_ino == os.stat(
+ symlink_file, follow_symlinks=False).st_ino
+ os.stat(fname, follow_symlinks=False).st_mode == os.stat(
+ symlink_file, follow_symlinks=False).st_mode
- # os.chmod
- if os.name != "nt":
- os.chmod(fname, 0o666)
- assert oct(os.stat(fname).st_mode) == '0o100666'
+ # os.chmod
+ if os.name != "nt":
+ os.chmod(fname, 0o666)
+ assert oct(os.stat(fname).st_mode) == '0o100666'
- # os.path
- assert os.path.exists(fname) == True
- assert os.path.exists("NO_SUCH_FILE") == False
- assert os.path.isfile(fname) == True
- assert os.path.isdir(folder) == True
- assert os.path.isfile(folder) == False
- assert os.path.isdir(fname) == False
+# os.chown
+ if os.name != "nt":
+ # setup
+ root_in_posix = False
+ if hasattr(os, 'geteuid'):
+ root_in_posix = (os.geteuid() == 0)
+ try:
+ import pwd
+ all_users = [u.pw_uid for u in pwd.getpwall()]
+ except (ImportError, AttributeError):
+ all_users = []
- assert os.path.basename(fname) == FILE_NAME
- assert os.path.dirname(fname) == tmpdir
+ fname1 = os.path.join(tmpdir, FILE_NAME)
+ fname2 = os.path.join(tmpdir, FILE_NAME2)
+ fd = os.open(fname2, os.O_RDONLY)
- with TestWithTempCurrentDir():
- os.chdir(tmpdir)
- assert os.path.realpath(os.getcwd()) == os.path.realpath(tmpdir)
- assert os.path.exists(FILE_NAME)
+ # test chown without root permissions
+ if not root_in_posix and len(all_users) > 1:
+ uid_1, uid_2 = all_users[:2]
+ gid = os.stat(fname1).st_gid
+ assert_raises(PermissionError,
+ lambda: os.chown(fname1, uid_1, gid))
+ assert_raises(PermissionError,
+ lambda: os.chown(fname1, uid_2, gid))
+
+ # test chown with root perm and file name
+ if root_in_posix and len(all_users) > 1:
+ uid_1, uid_2 = all_users[:2]
+ gid = os.stat(fname1).st_gid
+ os.chown(fname1, uid_1, gid)
+ uid = os.stat(fname1).st_uid
+ assert uid == uid_1
+ os.chown(fname1, uid_2, gid)
+ uid = os.stat(fname1).st_uid
+ assert uid == uid_2
+
+ # test chown with root perm and file descriptor
+ if root_in_posix and len(all_users) > 1:
+ uid_1, uid_2 = all_users[:2]
+ gid = os.stat(fd).st_gid
+ os.chown(fd, uid_1, gid)
+ uid = os.stat(fd).st_uid
+ assert uid == uid_1
+ os.chown(fd, uid_2, gid)
+ uid = os.stat(fd).st_uid
+ assert uid == uid_2
+
+ # test gid change
+ if hasattr(os, 'getgroups'):
+ groups = os.getgroups()
+ if len(groups) > 1:
+ gid_1, gid_2 = groups[:2]
+ uid = os.stat(fname1).st_uid
+
+ os.chown(fname1, uid, gid_1)
+ gid = os.stat(fname1).st_gid
+ assert gid == gid_1
+
+ os.chown(fname1, uid, gid_2)
+ gid = os.stat(fname1).st_gid
+ assert gid == gid_2
+
+ # teardown
+ os.close(fd)
+
+ # os.path
+ assert os.path.exists(fname) is True
+ assert os.path.exists("NO_SUCH_FILE") is False
+ assert os.path.isfile(fname) is True
+ assert os.path.isdir(folder) is True
+ assert os.path.isfile(folder) is False
+ assert os.path.isdir(fname) is False
+
+ assert os.path.basename(fname) == FILE_NAME
+ assert os.path.dirname(fname) == tmpdir
+
+ with TestWithTempCurrentDir():
+ os.chdir(tmpdir)
+ assert os.path.realpath(os.getcwd()) == os.path.realpath(tmpdir)
+ assert os.path.exists(FILE_NAME)
# supports
assert isinstance(os.supports_fd, set)
@@ -367,7 +441,6 @@ if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'):
os.close(rfd)
os.close(wfd)
-
with TestWithTempDir() as tmpdir:
for i in range(0, 4):
file_name = os.path.join(tmpdir, 'file' + str(i))
@@ -386,10 +459,13 @@ with TestWithTempDir() as tmpdir:
dir_iter.close()
- expected_files_bytes = [(file.encode(), os.path.join(tmpdir, file).encode()) for file in expected_files]
+ expected_files_bytes = [(file.encode(), os.path.join(tmpdir,
+ file).encode())
+ for file in expected_files]
dir_iter_bytes = os.scandir(tmpdir.encode())
- collected_files_bytes = [(dir_entry.name, dir_entry.path) for dir_entry in dir_iter_bytes]
+ collected_files_bytes = [(dir_entry.name, dir_entry.path)
+ for dir_entry in dir_iter_bytes]
assert set(collected_files_bytes) == set(expected_files_bytes)
@@ -399,7 +475,8 @@ with TestWithTempDir() as tmpdir:
assert set(collected_files) == set(expected_files)
collected_files = os.listdir(tmpdir.encode())
- assert set(collected_files) == set([file.encode() for file in expected_files ])
+ assert set(collected_files) == set(
+ [file.encode() for file in expected_files])
with TestWithTempCurrentDir():
os.chdir(tmpdir)
@@ -415,4 +492,25 @@ if "win" not in sys.platform:
for arg in [None, 1, 1.0, TabError]:
assert_raises(TypeError, os.system, arg)
+if sys.platform.startswith("win"):
+ winver = sys.getwindowsversion()
+
+ # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members).
+ all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \
+ 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000
+
+ # We really can't test if the results are correct, so it just checks for meaningful value
+ assert winver.major > 0
+ assert winver.minor >= 0
+ assert winver.build > 0
+ assert winver.platform == 2
+ assert isinstance(winver.service_pack, str)
+ assert 0 <= winver.suite_mask <= all_masks
+ assert 1 <= winver.product_type <= 3
+
+ # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest
+ assert winver.major == winver.platform_version[0]
+ assert winver.minor == winver.platform_version[1]
+ assert winver.build == winver.platform_version[2]
+
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
index a65bc2408..f7a9ac5a8 100644
--- a/vm/Cargo.toml
+++ b/vm/Cargo.toml
@@ -9,10 +9,12 @@ edition = "2018"
include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"]
[features]
-default = ["rustpython-parser", "rustpython-compiler"]
+default = ["compile-parse", "threading"]
vm-tracing-logging = []
flame-it = ["flame", "flamer"]
freeze-stdlib = ["rustpython-pylib"]
+threading = ["rustpython-common/threading"]
+compile-parse = ["rustpython-parser", "rustpython-compiler"]
ssl = ["openssl", "openssl-sys", "openssl-probe"]
@@ -27,10 +29,10 @@ blake2 = "0.8"
volatile = "0.2"
num-complex = { version = "0.2.2", features = ["serde"] }
-num-bigint = { version = "0.2.4", features = ["serde"] }
+num-bigint = { version = "0.3", features = ["serde"] }
num-traits = "0.2.8"
num-integer = "0.1.41"
-num-rational = "0.2.2"
+num-rational = "0.3"
num-iter = "0.1.39"
rand = { version = "0.7", features = ["wasm-bindgen"] }
rand_core = "0.5"
@@ -50,8 +52,8 @@ statrs = "0.12.0"
caseless = "0.2.1"
chrono = { version = "0.4", features = ["wasmbind"] }
once_cell = "1.3.1"
-lexical = "4"
-itertools = "0.8"
+lexical-core = "0.7"
+itertools = "0.9"
hex = "0.4.0"
hexf-parse = "0.1.0"
indexmap = "1.0.2"
@@ -90,10 +92,9 @@ unic-ucd-age = "0.9"
unic-ucd-ident = "0.9"
flame = { version = "0.2", optional = true }
-flamer = { version = "0.3", optional = true }
+flamer = { version = "0.4", optional = true }
-[target.'cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))'.dependencies]
-pwd = "1"
+rustpython-common = { path = "../common" }
[target.'cfg(unix)'.dependencies]
exitcode = "1.1.2"
@@ -134,4 +135,4 @@ features = [
wasm-bindgen = "0.2"
[build-dependencies]
-itertools = "0.8"
+itertools = "0.9"
diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml
index b49379f27..1fea89844 100644
--- a/vm/pylib-crate/Cargo.toml
+++ b/vm/pylib-crate/Cargo.toml
@@ -1,10 +1,17 @@
[package]
name = "rustpython-pylib"
version = "0.1.0"
-authors = ["Noah <33094578+coolreader18@users.noreply.github.com>"]
+authors = ["RustPython Team"]
+description = "A subset of the Python standard library for use with RustPython"
+repository = "https://github.com/RustPython/RustPython"
+license-file = "Lib/PSF-LICENSE"
edition = "2018"
+include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"]
+
+[features]
+default = ["compiled-bytecode"]
+compiled-bytecode = ["rustpython-derive", "rustpython-bytecode"]
[dependencies]
-rustpython-derive = { version = "0.1.2", path = "../../derive" }
-rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" }
-maplit = "1.0"
+rustpython-derive = { version = "0.1.2", path = "../../derive", optional = true }
+rustpython-bytecode = { version = "0.1.2", path = "../../bytecode", optional = true }
diff --git a/vm/pylib-crate/src/lib.rs b/vm/pylib-crate/src/lib.rs
index 6adec5c74..98059df0b 100644
--- a/vm/pylib-crate/src/lib.rs
+++ b/vm/pylib-crate/src/lib.rs
@@ -4,25 +4,14 @@
extern crate self as rustpython_pylib;
-use rustpython_bytecode::bytecode::{self, FrozenModule};
-use std::collections::HashMap;
-
-use rustpython_derive::py_compile_bytecode as _py_compile_bytecode;
-#[macro_export]
-macro_rules! py_compile_bytecode {
- ($($arg:tt)*) => {{
- #[macro_use]
- mod __m {
- $crate::_py_compile_bytecode!($($arg)*);
- }
- __proc_macro_call!()
- }};
-}
-
-mod __exports {
- pub use maplit::hashmap;
-}
+pub const LIB_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/Lib");
+#[cfg(feature = "compiled-bytecode")]
+use {
+ rustpython_bytecode::bytecode::{self, FrozenModule},
+ std::collections::HashMap,
+};
+#[cfg(feature = "compiled-bytecode")]
pub fn frozen_stdlib() -> HashMap {
- py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib")
+ rustpython_derive::py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib")
}
diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs
index a11db6084..ae2cf4723 100644
--- a/vm/src/builtins.rs
+++ b/vm/src/builtins.rs
@@ -6,9 +6,6 @@ use crate::vm::VirtualMachine;
#[pymodule(name = "builtins")]
mod decl {
- use std::char;
- use std::str;
-
use num_bigint::Sign;
use num_traits::{Signed, ToPrimitive, Zero};
#[cfg(feature = "rustpython-compiler")]
@@ -17,10 +14,10 @@ mod decl {
use rustpython_parser::parser;
use super::to_ascii;
+ use crate::byteslike::PyBytesLike;
use crate::exceptions::PyBaseExceptionRef;
use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs};
use crate::obj::objbool::{self, IntoPyBool};
- use crate::obj::objbyteinner::PyByteInner;
use crate::obj::objbytes::PyBytesRef;
use crate::obj::objcode::PyCodeRef;
use crate::obj::objdict::PyDictRef;
@@ -30,7 +27,6 @@ mod decl {
use crate::obj::objsequence;
use crate::obj::objstr::{PyString, PyStringRef};
use crate::obj::objtype::{self, PyClassRef};
- use crate::pyhash;
use crate::pyobject::{
Either, IdProtocol, ItemProtocol, PyCallable, PyIterable, PyObjectRef, PyResult, PyValue,
TryFromObject, TypeProtocol,
@@ -39,7 +35,9 @@ mod decl {
use crate::scope::Scope;
#[cfg(feature = "rustpython-parser")]
use crate::stdlib::ast;
+ use crate::sysmodule;
use crate::vm::VirtualMachine;
+ use rustpython_common::hash::PyHash;
#[pyfunction]
fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult {
@@ -95,7 +93,7 @@ mod decl {
#[pyfunction]
fn chr(i: u32, vm: &VirtualMachine) -> PyResult {
- match char::from_u32(i) {
+ match std::char::from_u32(i) {
Some(value) => Ok(value.to_string()),
None => Err(vm.new_value_error("chr() arg not in range(0x110000)".to_owned())),
}
@@ -124,7 +122,7 @@ mod decl {
// TODO: compile::compile should probably get bytes
let source = match &args.source {
Either::A(string) => string.as_str(),
- Either::B(bytes) => str::from_utf8(bytes).unwrap(),
+ Either::B(bytes) => std::str::from_utf8(bytes).unwrap(),
};
let mode_str = args.mode.as_str();
@@ -332,7 +330,7 @@ mod decl {
}
#[pyfunction]
- fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
+ fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult {
vm._hash(&obj)
}
@@ -557,18 +555,18 @@ mod decl {
}
#[pyfunction]
- fn ord(string: Either, vm: &VirtualMachine) -> PyResult {
+ fn ord(string: Either, vm: &VirtualMachine) -> PyResult {
match string {
- Either::A(bytes) => {
- let bytes_len = bytes.elements.len();
+ Either::A(bytes) => bytes.with_ref(|bytes| {
+ let bytes_len = bytes.len();
if bytes_len != 1 {
return Err(vm.new_type_error(format!(
"ord() expected a character, but string of length {} found",
bytes_len
)));
}
- Ok(u32::from(bytes.elements[0]))
- }
+ Ok(u32::from(bytes[0]))
+ }),
Either::B(string) => {
let string = string.as_str();
let string_len = string.chars().count();
@@ -650,7 +648,7 @@ mod decl {
pub fn print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> {
let file = match options.file {
Some(f) => f,
- None => vm.get_attribute(vm.sys_module.clone(), "stdout")?,
+ None => sysmodule::get_stdout(vm)?,
};
let write = |obj: PyStringRef| vm.call_method(&file, "write", vec![obj.into_object()]);
diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/bytesinner.rs
similarity index 78%
rename from vm/src/obj/objbyteinner.rs
rename to vm/src/bytesinner.rs
index 730e44e77..02a3dd1f8 100644
--- a/vm/src/obj/objbyteinner.rs
+++ b/vm/src/bytesinner.rs
@@ -1,56 +1,56 @@
use bstr::ByteSlice;
use num_bigint::{BigInt, ToBigInt};
use num_traits::{One, Signed, ToPrimitive, Zero};
-use std::convert::TryFrom;
use std::ops::Range;
-use super::objbytearray::{PyByteArray, PyByteArrayRef};
-use super::objbytes::{PyBytes, PyBytesRef};
-use super::objint::{self, PyInt, PyIntRef};
-use super::objlist::PyList;
-use super::objmemory::PyMemoryView;
-use super::objnone::PyNoneRef;
-use super::objsequence::{PySliceableSequence, SequenceIndex};
-use super::objslice::PySliceRef;
-use super::objstr::{self, PyString, PyStringRef};
-use super::pystr::{self, PyCommonString, PyCommonStringWrapper};
+use crate::byteslike::PyBytesLike;
use crate::function::{OptionalArg, OptionalOption};
-use crate::pyhash;
+use crate::obj::objbytearray::PyByteArray;
+use crate::obj::objbytes::PyBytes;
+use crate::obj::objint::{self, PyInt, PyIntRef};
+use crate::obj::objlist::PyList;
+use crate::obj::objmemory::PyMemoryView;
+use crate::obj::objnone::PyNoneRef;
+use crate::obj::objsequence::{PySliceableSequence, SequenceIndex};
+use crate::obj::objslice::PySliceRef;
+use crate::obj::objstr::{self, PyString, PyStringRef};
use crate::pyobject::{
Either, PyComparisonValue, PyIterable, PyObjectRef, PyResult, TryFromObject, TypeProtocol,
};
+use crate::pystr::{self, PyCommonString, PyCommonStringContainer, PyCommonStringWrapper};
use crate::vm::VirtualMachine;
+use rustpython_common::hash;
#[derive(Debug, Default, Clone)]
-pub struct PyByteInner {
- pub elements: Vec,
+pub struct PyBytesInner {
+ pub(crate) elements: Vec,
}
-impl From> for PyByteInner {
- fn from(elements: Vec) -> PyByteInner {
+impl From> for PyBytesInner {
+ fn from(elements: Vec) -> PyBytesInner {
Self { elements }
}
}
-impl TryFromObject for PyByteInner {
+impl TryFromObject for PyBytesInner {
fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult {
match_class!(match obj {
- i @ PyBytes => Ok(PyByteInner {
+ i @ PyBytes => Ok(PyBytesInner {
elements: i.get_value().to_vec()
}),
- j @ PyByteArray => Ok(PyByteInner {
+ j @ PyByteArray => Ok(PyBytesInner {
elements: j.borrow_value().elements.to_vec()
}),
- k @ PyMemoryView => Ok(PyByteInner {
- elements: k.try_value().unwrap()
+ k @ PyMemoryView => Ok(PyBytesInner {
+ elements: k.try_bytes(|v| v.to_vec()).unwrap()
}),
- l @ PyList => l.get_byte_inner(vm),
+ l @ PyList => l.to_byte_inner(vm),
obj => {
let iter = vm.get_method_or_type_error(obj.clone(), "__iter__", || {
format!("a bytes-like object is required, not {}", obj.class())
})?;
let iter = PyIterable::from_method(iter);
- Ok(PyByteInner {
+ Ok(PyBytesInner {
elements: iter.iter(vm)?.collect::>()?,
})
}
@@ -67,13 +67,13 @@ pub struct ByteInnerNewOptions {
}
impl ByteInnerNewOptions {
- pub fn get_value(self, vm: &VirtualMachine) -> PyResult {
+ pub fn get_value(self, vm: &VirtualMachine) -> PyResult {
// First handle bytes(string, encoding[, errors])
if let OptionalArg::Present(enc) = self.encoding {
if let OptionalArg::Present(eval) = self.val_option {
if let Ok(input) = eval.downcast::() {
let bytes = objstr::encode_string(input, Some(enc), None, vm)?;
- Ok(PyByteInner {
+ Ok(PyBytesInner {
elements: bytes.get_value().to_vec(),
})
} else {
@@ -113,13 +113,13 @@ impl ByteInnerNewOptions {
// TODO: only support this method in the bytes() constructor
if let Some(bytes_method) = vm.get_method(obj.clone(), "__bytes__") {
let bytes = vm.invoke(&bytes_method?, vec![])?;
- return PyByteInner::try_from_object(vm, bytes);
+ return PyBytesInner::try_from_object(vm, bytes);
}
- let elements = vm.extract_elements(&obj).or_else(|_| {
- Err(vm.new_type_error(format!(
+ let elements = vm.extract_elements(&obj).map_err(|_| {
+ vm.new_type_error(format!(
"cannot convert '{}' object to bytes",
obj.class().name
- )))
+ ))
})?;
let mut data_bytes = vec![];
@@ -140,7 +140,7 @@ impl ByteInnerNewOptions {
Ok(vec![])
};
match value {
- Ok(val) => Ok(PyByteInner { elements: val }),
+ Ok(val) => Ok(PyBytesInner { elements: val }),
Err(err) => Err(err),
}
}
@@ -150,7 +150,7 @@ impl ByteInnerNewOptions {
#[derive(FromArgs)]
pub struct ByteInnerFindOptions {
#[pyarg(positional_only, optional = false)]
- sub: Either,
+ sub: Either,
#[pyarg(positional_only, default = "None")]
start: Option,
#[pyarg(positional_only, default = "None")]
@@ -183,15 +183,20 @@ pub struct ByteInnerPaddingOptions {
impl ByteInnerPaddingOptions {
fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> {
let fillchar = if let OptionalArg::Present(v) = self.fillchar {
- match try_as_byte(v.clone()) {
- Some(x) if x.len() == 1 => x[0],
- _ => {
- return Err(vm.new_type_error(format!(
- "{}() argument 2 must be a byte string of length 1, not {}",
- fn_name, &v
- )));
+ try_as_bytes(v.clone(), |bytes| {
+ if bytes.len() == 1 {
+ Some(bytes[0])
+ } else {
+ None
}
- }
+ })
+ .flatten()
+ .ok_or_else(|| {
+ vm.new_type_error(format!(
+ "{}() argument 2 must be a byte string of length 1, not {}",
+ fn_name, &v
+ ))
+ })?
} else {
b' ' // default is space
};
@@ -203,9 +208,9 @@ impl ByteInnerPaddingOptions {
#[derive(FromArgs)]
pub struct ByteInnerTranslateOptions {
#[pyarg(positional_only, optional = false)]
- table: Either,
+ table: Either,
#[pyarg(positional_or_keyword, optional = true)]
- delete: OptionalArg,
+ delete: OptionalArg,
}
impl ByteInnerTranslateOptions {
@@ -230,31 +235,11 @@ impl ByteInnerTranslateOptions {
}
}
-pub type ByteInnerSplitOptions = pystr::SplitArgs;
-
-#[derive(FromArgs)]
-pub struct ByteInnerSplitlinesOptions {
- #[pyarg(positional_or_keyword, optional = true)]
- keepends: OptionalArg,
-}
-
-impl ByteInnerSplitlinesOptions {
- pub fn get_value(self) -> bool {
- match self.keepends.into_option() {
- Some(x) => x,
- None => false,
- }
- // if let OptionalArg::Present(value) = self.keepends {
- // Ok(bool::try_from_object(vm, value)?)
- // } else {
- // Ok(false)
- // }
- }
-}
+pub type ByteInnerSplitOptions<'a> = pystr::SplitArgs<'a, PyBytesInner, [u8], u8>;
#[allow(clippy::len_without_is_empty)]
-impl PyByteInner {
- pub fn repr(&self) -> PyResult {
+impl PyBytesInner {
+ pub fn repr(&self) -> String {
let mut res = String::with_capacity(self.elements.len());
for i in self.elements.iter() {
match i {
@@ -267,7 +252,7 @@ impl PyByteInner {
_ => res.push_str(&format!("\\x{:x}", i)),
}
}
- Ok(res)
+ res
}
pub fn len(&self) -> usize {
@@ -304,21 +289,17 @@ impl PyByteInner {
self.cmp(other, |a, b| a < b, vm)
}
- pub fn hash(&self) -> pyhash::PyHash {
- pyhash::hash_value(&self.elements)
+ pub fn hash(&self) -> hash::PyHash {
+ hash::hash_value(&self.elements)
}
- pub fn add(&self, other: PyByteInner) -> Vec {
- self.elements
- .iter()
- .chain(other.elements.iter())
- .cloned()
- .collect::>()
+ pub fn add(&self, other: PyBytesInner) -> Vec {
+ self.elements.py_add(&other.elements)
}
pub fn contains(
&self,
- needle: Either,
+ needle: Either,
vm: &VirtualMachine,
) -> PyResult {
Ok(match needle {
@@ -377,7 +358,7 @@ impl PyByteInner {
.collect::>>()?)
}
_ => match_class!(match object {
- i @ PyMemoryView => Ok(i.try_value().unwrap()),
+ i @ PyMemoryView => Ok(i.try_bytes(|v| v.to_vec()).unwrap()),
_ => Err(vm.new_index_error(
"can assign only bytes, buffers, or iterables of ints in range(0, 256)"
.to_owned()
@@ -558,31 +539,13 @@ impl PyByteInner {
}
pub fn islower(&self) -> bool {
- // CPython _Py_bytes_islower
- let mut cased = false;
- for b in self.elements.iter() {
- let c = *b as char;
- if c.is_uppercase() {
- return false;
- } else if !cased && c.is_lowercase() {
- cased = true
- }
- }
- cased
+ self.elements
+ .py_iscase(char::is_lowercase, char::is_uppercase)
}
pub fn isupper(&self) -> bool {
- // CPython _Py_bytes_isupper
- let mut cased = false;
- for b in self.elements.iter() {
- let c = *b as char;
- if c.is_lowercase() {
- return false;
- } else if !cased && c.is_uppercase() {
- cased = true
- }
- }
- cased
+ self.elements
+ .py_iscase(char::is_uppercase, char::is_lowercase)
}
pub fn isspace(&self) -> bool {
@@ -694,17 +657,17 @@ impl PyByteInner {
}
#[inline]
- fn pad(
+ fn _pad(
&self,
options: ByteInnerPaddingOptions,
- pad: fn(&[u8], usize, u8) -> Vec,
+ pad: fn(&[u8], usize, u8, usize) -> Vec,
vm: &VirtualMachine,
) -> PyResult> {
let (width, fillchar) = options.get_value("center", vm)?;
Ok(if self.len() as isize >= width {
Vec::from(&self.elements[..])
} else {
- pad(&self.elements, width as usize, fillchar)
+ pad(&self.elements, width as usize, fillchar, self.len())
})
}
@@ -713,7 +676,7 @@ impl PyByteInner {
options: ByteInnerPaddingOptions,
vm: &VirtualMachine,
) -> PyResult> {
- self.pad(options, PyCommonString::::py_center, vm)
+ self._pad(options, PyCommonString::::py_center, vm)
}
pub fn ljust(
@@ -721,7 +684,7 @@ impl PyByteInner {
options: ByteInnerPaddingOptions,
vm: &VirtualMachine,
) -> PyResult> {
- self.pad(options, PyCommonString::::py_ljust, vm)
+ self._pad(options, PyCommonString::::py_ljust, vm)
}
pub fn rjust(
@@ -729,7 +692,7 @@ impl PyByteInner {
options: ByteInnerPaddingOptions,
vm: &VirtualMachine,
) -> PyResult> {
- self.pad(options, PyCommonString::::py_rjust, vm)
+ self._pad(options, PyCommonString::::py_rjust, vm)
}
pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult {
@@ -739,17 +702,13 @@ impl PyByteInner {
.py_count(needle.as_slice(), range, |h, n| h.find_iter(n).count()))
}
- pub fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult> {
- let mut refs = Vec::new();
- for v in iter.iter(vm)? {
- let v = v?;
- if !refs.is_empty() {
- refs.extend(&self.elements);
- }
- refs.extend(v.elements);
- }
-
- Ok(refs)
+ pub fn join(
+ &self,
+ iterable: PyIterable,
+ vm: &VirtualMachine,
+ ) -> PyResult> {
+ let iter = iterable.iter(vm)?;
+ self.elements.py_join(iter)
}
#[inline]
@@ -766,7 +725,7 @@ impl PyByteInner {
Ok(self.elements.py_find(&needle, range, find))
}
- pub fn maketrans(from: PyByteInner, to: PyByteInner, vm: &VirtualMachine) -> PyResult {
+ pub fn maketrans(from: PyBytesInner, to: PyBytesInner, vm: &VirtualMachine) -> PyResult {
let mut res = vec![];
for i in 0..=255 {
@@ -804,7 +763,7 @@ impl PyByteInner {
Ok(res)
}
- pub fn strip(&self, chars: OptionalOption