mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

Merge branch 'main' into copilot/fix-isalnum-function-behavior

Browse Source
This commit is contained in:
Jeong, YunWon authored and GitHub committed 2026-08-18 13:49:51 +09:00
2 parents 9bc27415e4 2ddf8fec67
commit 7aedaf919e
282 files changed
+14458 -4130
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.claude/settings.json
-15
View File
@@ -1,15 +0,0 @@
{
"hooks": {
"SessionStart": [
{
"matcher": "",
"hooks": [
{
"type": "command",
"command": "bash .claude/scripts/setup-env.sh"
}
]
}
]
}
}
.cspell.json
+1
View File
@@ -59,6 +59,7 @@
"alnum",
"csock",
"coro",
"contig",
"Crnl",
"dedentations",
"dedents",
.github/workflows/ci.yaml
+8 -5
View File
@@ -526,7 +526,7 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
- uses: cargo-bins/cargo-binstall@ead08b90bd7b2e6d81963fb9cf0b7239f66d5db4 # v1.21.0
- uses: cargo-bins/cargo-binstall@e00d2c94cc0067b77737821097a62d91c0301baa # v1.21.1
- name: cargo shear
run: |
@@ -554,10 +554,10 @@ jobs:
components: rustfmt
- name: actionlint
uses: reviewdog/action-actionlint@6fb7acc99f4a1008869fa8a0f09cfca740837d9d # v1.72.0
uses: reviewdog/action-actionlint@d63ba7532e0942965320cd8d73cbae4c7b3c5283 # v1.73.1
- name: zizmor
uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7
uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2
- name: restore prek cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -567,7 +567,7 @@ jobs:
- name: install prek
id: prek
uses: j178/prek-action@e98a699c41eb69ab013a45817a0406469a748f8d # v2.0.5
uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0
with:
cache: false
show-verbose-logs: false
@@ -648,7 +648,10 @@ jobs:
env:
# miri-ignore-leaks because the type-object circular reference means that there will always be
# a memory leak, at least until we have proper cyclic gc
MIRIFLAGS: "-Zmiri-ignore-leaks"
# miri-permissive-provenance because function pointer identity checks (slot comparisons)
# cast fn pointers to usize, which strips provenance — this is the standard pattern for
# fn pointer comparison in Rust and not a soundness issue
MIRIFLAGS: "-Zmiri-ignore-leaks -Zmiri-permissive-provenance"
wasm:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
.github/workflows/cron-ci.yaml
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
- uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11
with:
tool: cargo-llvm-cov
.github/workflows/oscca-pr.yml
+70
View File
@@ -0,0 +1,70 @@
name: Manage OSCCA pull requests
on:
pull_request_target:
types: [opened]
permissions: {}
jobs:
label-and-assign:
name: Label and assign OSCCA pull request
runs-on: ubuntu-slim
timeout-minutes: 5
permissions:
pull-requests: write
steps:
- name: Label and assign pull request
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const osccaUsers = new Set(
[
"2jiyong",
"chestnut1717",
"devyubin",
"fregataa",
"hyoinandout",
"HyoJongPark",
"JaceJung-dev",
"jinmay",
"jiwahn",
"kangdora",
"kim-jaedeok",
"kyokuping",
"leehanjeong",
"lms0806",
"lsahn-gh",
"moreal",
"name-of-okja",
"rlaisqls",
"seungje0612",
"shAn-kor",
"sigmaith",
"teddygood",
"widehyo1",
"YangSiJun528",
"zzarbttoo",
].map((login) => login.toLowerCase()),
);
const pullRequest = context.payload.pull_request;
const author = pullRequest.user.login;
if (!osccaUsers.has(author.toLowerCase())) {
core.info(`${author} is not an OSCCA participant; skipping.`);
return;
}
const issue = {
...context.repo,
issue_number: pullRequest.number,
};
await github.rest.issues.addLabels({
...issue,
labels: ["z-ca-2026"],
});
await github.rest.issues.addAssignees({
...issue,
assignees: [author],
});
.github/workflows/update-doc-db.yml
+1 -1
View File
@@ -8,7 +8,7 @@ on:
python-version:
description: Target python version to generate doc db for
type: string
default: "3.14.3"
default: "3.14.7"
base-ref:
description: Base branch to create the update branch from
type: string
.github/workflows/upgrade-pylib.lock.yml
Generated
+5 -5
View File
@@ -58,7 +58,7 @@ jobs:
comment_repo: ""
steps:
- name: Setup Scripts
uses: github/gh-aw/actions/setup@8b820ae1073f301991aaf2f307f7e271f618bb9f # v0.82.14
uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3
with:
destination: /opt/gh-aw/actions
- name: Check workflow file timestamps
@@ -99,7 +99,7 @@ jobs:
secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
steps:
- name: Setup Scripts
uses: github/gh-aw/actions/setup@8b820ae1073f301991aaf2f307f7e271f618bb9f # v0.82.14
uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3
with:
destination: /opt/gh-aw/actions
- name: Checkout repository
@@ -806,7 +806,7 @@ jobs:
total_count: ${{ steps.missing_tool.outputs.total_count }}
steps:
- name: Setup Scripts
uses: github/gh-aw/actions/setup@8b820ae1073f301991aaf2f307f7e271f618bb9f # v0.82.14
uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3
with:
destination: /opt/gh-aw/actions
- name: Download agent output artifact
@@ -927,7 +927,7 @@ jobs:
success: ${{ steps.parse_results.outputs.success }}
steps:
- name: Setup Scripts
uses: github/gh-aw/actions/setup@8b820ae1073f301991aaf2f307f7e271f618bb9f # v0.82.14
uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3
with:
destination: /opt/gh-aw/actions
- name: Download agent artifacts
@@ -1039,7 +1039,7 @@ jobs:
process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
steps:
- name: Setup Scripts
uses: github/gh-aw/actions/setup@8b820ae1073f301991aaf2f307f7e271f618bb9f # v0.82.14
uses: github/gh-aw/actions/setup@53258938b59e0797fefeed05ec0c681514b2a827 # v0.84.3
with:
destination: /opt/gh-aw/actions
- name: Download agent output artifact
.github/workflows/upgrade-pylib.md
+1 -1
View File
@@ -52,7 +52,7 @@ cache:
- cpython-lib-
env:
PYTHON_VERSION: "v3.14.6"
PYTHON_VERSION: "v3.14.7"
ISSUE_ID: "6839"
---
.github/zizmor.yml
+5
View File
@@ -1,4 +1,9 @@
rules:
dangerous-triggers:
ignore:
# pull_request_target is needed to label and assign PRs from forks with issues: write.
# The workflow does not check out or execute pull request code.
- oscca-pr.yml:3
excessive-permissions:
ignore:
# pull_request_target is needed to post PR comments with pull-requests: write.
.pre-commit-config.yaml
+4 -4
View File
@@ -10,7 +10,7 @@ repos:
priority: 0
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.22
rev: v0.16.2
hooks:
- id: ruff-format
priority: 0
@@ -42,7 +42,7 @@ repos:
- id: generate-rs-opcode-metadata
name: generate rust opcode metadata
entry: python tools/opcode_metadata/generate_rs_opcode_metadata.py
entry: python3 tools/opcode_metadata/generate_rs_opcode_metadata.py
files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
pass_filenames: false
language: system
@@ -53,7 +53,7 @@ repos:
- id: generate-py-opcode-metadata
name: generate python opcode metadata
entry: python tools/opcode_metadata/generate_py_opcode_metadata.py
entry: python3 tools/opcode_metadata/generate_py_opcode_metadata.py
files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
pass_filenames: false
language: system
@@ -77,7 +77,7 @@ repos:
priority: 0
- repo: https://github.com/rbubley/mirrors-prettier
rev: v3.9.5
rev: v3.9.6
hooks:
- id: prettier
files: '^wasm/.*$'
.python-version
+1 -1
View File
@@ -1 +1 @@
3.14.6
3.14.7
AGENTS.md
+14 -6
View File
@@ -17,17 +17,22 @@ See the "Code organization" section in [CONTRIBUTING.md](CONTRIBUTING.md#code-or
## AI Agent Rules
**CRITICAL: AI Policy**
- Follow RustPython's [AI Policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md) for every AI-assisted contribution.
- Disclose AI assistance in commit messages with an `Assisted-by: AGENT_NAME:MODEL_VERSION` trailer. Use one trailer per AI tool, and never use `Co-authored-by` for an AI assistant.
**CRITICAL: Git Operations**
- NEVER create pull requests directly without explicit user permission
- NEVER push commits to remote without explicit user permission
- Always ask the user before performing any git operations that affect the remote repository
- Commits can be created locally when requested, but pushing and PR creation require explicit approval
**CRITICAL: Pre-commit Checks**
- Before creating ANY commit, you MUST run `prek run --all-files` (or `pre-commit run --all-files`) AND the full test suite. Both must pass — do not commit if either fails.
- Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test <module>` modules.
- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes, re-run `prek` until it reports a clean pass, then re-run the tests, then commit.
- NEVER bypass these checks with `--no-verify`, `--no-gpg-sign`, or by skipping tests "because the change is small". If a hook or test fails, fix the underlying issue and create a new commit — do not amend or force the failing commit through.
**CRITICAL: Commit Hooks and Validation**
- Install the repository's pre-commit hook with `prek install` (or `pre-commit install`) after cloning the repository.
- Every commit must run the configured pre-commit hook. NEVER bypass it with `--no-verify`. Automated workflows that use a normal `git commit`, such as `scripts/update_lib quick`, should be allowed to create local commits through the hook.
- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes and retry the commit. Do not amend or force a failing commit through.
- Before completing a task, run the tests appropriate for the change. Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi`, then run `cargo test` from `crates/capi`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test <module>` modules.
## Important Development Notes
@@ -113,7 +118,10 @@ rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r
```bash
# Run Rust unit tests
cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher
cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi
# Run C-API tests from their directory so their separate Cargo config applies
(cd crates/capi && cargo test)
# Run Python snippets tests (debug mode recommended for faster compilation)
cargo run -- extra_tests/snippets/builtin_bytes.py
Cargo.lock
Generated
+67 -136
View File
@@ -73,15 +73,6 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anes"
version = "0.1.6"
@@ -402,9 +393,9 @@ dependencies = [
[[package]]
name = "bstr"
version = "1.12.3"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79"
checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530"
dependencies = [
"memchr",
"regex-automata",
@@ -509,19 +500,6 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "chrono"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"js-sys",
"num-traits",
"wasm-bindgen",
"windows-link",
]
[[package]]
name = "ciborium"
version = "0.2.2"
@@ -1567,30 +1545,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"log",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "icu_casemap"
version = "2.2.0"
@@ -1814,29 +1768,59 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jiff"
version = "0.2.31"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634"
checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc"
dependencies = [
"defmt",
"jiff-core",
"jiff-static",
"jiff-tzdb-platform",
"js-sys",
"log",
"portable-atomic",
"portable-atomic-util",
"serde_core",
"wasm-bindgen",
"windows-link",
]
[[package]]
name = "jiff-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
dependencies = [
"defmt",
]
[[package]]
name = "jiff-static"
version = "0.2.31"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2"
checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204"
dependencies = [
"jiff-core",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "jiff-tzdb"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e"
[[package]]
name = "jiff-tzdb-platform"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8"
dependencies = [
"jiff-tzdb",
]
[[package]]
name = "jni"
version = "0.22.4"
@@ -2049,9 +2033,9 @@ dependencies = [
[[package]]
name = "libz-rs-sys"
version = "0.6.6"
version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50474818739ccab820cd57bca432d6b02d090b47f9e85501d963cd05851f82eb"
checksum = "03dcace986b149f29509af6ca70e6182bccce916b644424ecf484faa8ddc899a"
dependencies = [
"zlib-rs",
]
@@ -2323,9 +2307,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]]
name = "num-integer"
version = "0.1.46"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
dependencies = [
"num-traits",
]
@@ -2773,18 +2757,18 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.106"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "psm"
version = "0.1.31"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea"
checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
dependencies = [
"ar_archive_writer",
"cc",
@@ -2806,9 +2790,9 @@ dependencies = [
[[package]]
name = "pyo3"
version = "0.29.0"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
dependencies = [
"libc",
"once_cell",
@@ -2820,18 +2804,18 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
version = "0.29.0"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
dependencies = [
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.29.0"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
dependencies = [
"libc",
"pyo3-build-config",
@@ -2839,9 +2823,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
version = "0.29.0"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@@ -2851,9 +2835,9 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
version = "0.29.0"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
dependencies = [
"heck",
"proc-macro2",
@@ -2863,9 +2847,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.46"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
@@ -3174,9 +3158,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.42"
version = "0.23.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -3576,7 +3560,6 @@ dependencies = [
"base64",
"blake2",
"bzip2",
"chrono",
"constant_time_eq",
"crc32fast",
"crossbeam-utils",
@@ -3592,6 +3575,7 @@ dependencies = [
"indexmap",
"insta",
"itertools 0.15.0",
"jiff",
"libc",
"libsqlite3-sys",
"libz-rs-sys",
@@ -3670,7 +3654,6 @@ dependencies = [
"ascii",
"bitflags 2.13.1",
"bstr",
"chrono",
"constant_time_eq",
"crossbeam-utils",
"exitcode",
@@ -3683,6 +3666,7 @@ dependencies = [
"is-macro",
"itertools 0.15.0",
"itoa",
"jiff",
"libc",
"log",
"malachite-bigint",
@@ -4197,7 +4181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -4226,18 +4210,18 @@ checksum = "b0f7e269b48f0a7dd0146680fa24b50cc67fc0373f086a5b2f99bd084639b482"
[[package]]
name = "thiserror"
version = "2.0.19"
version = "2.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "2.0.19"
version = "2.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
dependencies = [
"proc-macro2",
"quote",
@@ -4714,65 +4698,12 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-interface"
version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-result"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -5151,9 +5082,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.6.6"
version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
[[package]]
name = "zmij"
Cargo.toml
+1 -1
View File
@@ -200,7 +200,6 @@ bitflags = "2.11.0"
bitflagset = "0.0.3"
bstr = { version = "1", default-features = false, features = ["unicode"] }
bzip2 = "0.6"
chrono = { version = "0.4.44", default-features = false, features = ["clock", "std"] }
console_error_panic_hook = "0.1"
constant_time_eq = "0.5"
cranelift = "0.132.0"
@@ -231,6 +230,7 @@ insta = "1.47"
itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] }
itoa = "1"
is-macro = "0.3.7"
jiff = "0.2"
js-sys = "0.3"
junction = "2.0.0"
lexical-parse-float = "1.0.6"
Lib/test/test_ast/test_ast.py
Vendored
-1
View File
@@ -1888,7 +1888,6 @@ Module(
self.assertRaises(ValueError, ast.literal_eval, '+True')
self.assertRaises(ValueError, ast.literal_eval, '2+3')
@unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised
def test_literal_eval_str_int_limit(self):
with support.adjust_int_max_str_digits(4000):
ast.literal_eval('3'*4000) # no error
Lib/test/test_asyncio/test_ssl.py
Vendored
-2
View File
@@ -738,7 +738,6 @@ class TestSSL(test_utils.TestCase):
asyncio.wait_for(client(srv.addr),
timeout=support.SHORT_TIMEOUT))
@unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly
def test_create_connection_memory_leak(self):
HELLO_MSG = b'1' * self.PAYLOAD_SIZE
@@ -1617,7 +1616,6 @@ class TestSSL(test_utils.TestCase):
else:
self.fail('Unexpected ResourceWarning: {}'.format(cm.warning))
@unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly
def test_handshake_timeout_handler_leak(self):
s = socket.socket(socket.AF_INET)
s.bind(('127.0.0.1', 0))
Lib/test/test_bigmem.py
Vendored
-16
View File
@@ -789,14 +789,6 @@ class BytesTest(unittest.TestCase, BaseStrTest):
def test_swapcase(self, size):
self._test_swapcase(size)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_isspace(self):
return super().test_isspace()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_istitle(self):
return super().test_istitle()
class BytearrayTest(unittest.TestCase, BaseStrTest):
@@ -823,14 +815,6 @@ class BytearrayTest(unittest.TestCase, BaseStrTest):
test_hash = None
test_split_large = None
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_isspace(self):
return super().test_isspace()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_istitle(self):
return super().test_istitle()
class TupleTest(unittest.TestCase):
# Tuples have a small, fixed-sized head and an array of pointers to
Lib/test/test_buffer.py
Vendored
-11
View File
@@ -4471,7 +4471,6 @@ class TestBufferProtocol(unittest.TestCase):
class TestPythonBufferProtocol(unittest.TestCase):
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_basic(self):
class MyBuffer:
def __buffer__(self, flags):
@@ -4500,7 +4499,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
self.assertRaises(TypeError, memoryview, WrongArity())
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_buffer(self):
class WhatToRelease:
def __init__(self):
@@ -4523,7 +4521,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
self.assertEqual(mv.tobytes(), b"hello")
self.assertFalse(wr.held)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_same_buffer_returned(self):
class WhatToRelease:
def __init__(self):
@@ -4549,7 +4546,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
self.assertEqual(mv.tobytes(), b"hello")
self.assertFalse(wr.held)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_buffer_flags(self):
class PossiblyMutable:
def __init__(self, data, mutable) -> None:
@@ -4589,7 +4585,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
mv[0] = ord(b'x')
self.assertEqual(mv.tobytes(), b"hello")
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_call_builtins(self):
ba = bytearray(b"hello")
mv = ba.__buffer__(0)
@@ -4651,7 +4646,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
mv = memoryview(a)
self.assertEqual(mv.tobytes(), b"hello")
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_inheritance_releasebuffer(self):
rb_call_count = 0
class B(bytearray):
@@ -4668,7 +4662,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
self.assertEqual(rb_call_count, 0)
self.assertEqual(rb_call_count, 1)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_inherit_but_return_something_else(self):
class A(bytearray):
def __buffer__(self, flags):
@@ -4708,7 +4701,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
with memoryview(c) as mv:
self.assertEqual(mv.tobytes(), b"hello")
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_saves_reference(self):
smuggled_buffer = None
@@ -4736,7 +4728,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
with self.assertRaises(ValueError):
smuggled_buffer.tobytes()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_saves_reference_no_subclassing(self):
ba = bytearray(b"hello")
@@ -4757,7 +4748,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
c.buffer.release()
ba.clear()
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_multiple_inheritance_buffer_last(self):
class A:
def __buffer__(self, flags):
@@ -4817,7 +4807,6 @@ class TestPythonBufferProtocol(unittest.TestCase):
c.clear()
self.assertIs(c.buffer, None)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_release_buffer_with_exception_set(self):
class A:
def __buffer__(self, flags):
Lib/test/test_code_module.py
Vendored
-1
View File
@@ -128,7 +128,6 @@ class TestInteractiveConsole(unittest.TestCase, MockSys):
self.assertIsNone(self.sysmod.last_value.__traceback__)
self.assertIs(self.sysmod.last_exc, self.sysmod.last_value)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: '
def test_unicode_error(self):
self.infunc.side_effect = ["'\ud800'", EOFError('Finished')]
self.console.interact()
Lib/test/test_codeccallbacks.py
Vendored
+1 -2
View File
@@ -1067,8 +1067,7 @@ class CodecCallbackTest(unittest.TestCase):
decoded = input.decode(enc, "test.bug36819")
self.assertEqual(decoded, 'abcdx' * 51)
# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON")
def test_encodehelper_bug36819(self):
handler = RepeatedPosReturn()
codecs.register_error("test.bug36819", handler.handle)
Lib/test/test_collections.py
Vendored
-1
View File
@@ -1956,7 +1956,6 @@ class TestCollectionABCs(ABCTestCase):
# No metaclass conflict
class Z(ByteString, Awaitable): pass
@unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types)
def test_Buffer(self):
for sample in [bytes, bytearray, memoryview]:
self.assertIsInstance(sample(b"x"), Buffer)
Lib/test/test_eof.py
Vendored
-5
View File
@@ -18,7 +18,6 @@ class EOFTestCase(unittest.TestCase):
self.assertEqual(str(cm.exception), expect)
self.assertEqual(cm.exception.offset, 1)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_EOFS(self):
expect = ("unterminated triple-quoted string literal (detected at line 3) (<string>, line 1)")
with self.assertRaises(SyntaxError) as cm:
@@ -45,7 +44,6 @@ class EOFTestCase(unittest.TestCase):
self.assertEqual(cm.exception.text, "ä = '''thîs is ")
self.assertEqual(cm.exception.offset, 5)
@unittest.expectedFailure # TODO: RUSTPYTHON
@force_not_colorized
def test_EOFS_with_file(self):
expect = ("(<string>, line 1)")
@@ -86,7 +84,6 @@ class EOFTestCase(unittest.TestCase):
' ^',
'SyntaxError: unterminated triple-quoted string literal (detected at line 4)'])
@unittest.expectedFailure # TODO: RUSTPYTHON
@warnings_helper.ignore_warnings(category=SyntaxWarning)
def test_eof_with_line_continuation(self):
expect = "unexpected EOF while parsing (<string>, line 1)"
@@ -94,7 +91,6 @@ class EOFTestCase(unittest.TestCase):
compile('"\\Xhh" \\', '<string>', 'exec')
self.assertEqual(str(cm.exception), expect)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_line_continuation_EOF(self):
"""A continuation at the end of input must be an error; bpo2180."""
expect = 'unexpected EOF while parsing (<string>, line 1)'
@@ -127,7 +123,6 @@ class EOFTestCase(unittest.TestCase):
exec('\\')
self.assertEqual(str(cm.exception), expect)
@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.skipIf(not sys.executable, "sys.executable required")
@force_not_colorized
def test_line_continuation_EOF_from_file_bpo2180(self):
Lib/test/test_exceptions.py
Vendored
-1
View File
@@ -2145,7 +2145,6 @@ class AssertionErrorTests(unittest.TestCase):
def tearDown(self):
unlink(TESTFN)
@unittest.expectedFailure # TODO: RUSTPYTHON
@force_not_colorized
def test_assertion_error_location(self):
cases = [
Lib/test/test_format.py
Vendored
-1
View File
@@ -558,7 +558,6 @@ class FormatTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, str_err):
"{a:%ЫйЯЧ}".format(a='a')
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_negative_zero(self):
## default behavior
self.assertEqual(f"{-0.:.1f}", "-0.0")
Lib/test/test_funcattrs.py
Vendored
-1
View File
@@ -432,7 +432,6 @@ def empty_cell(empty=True):
class CellTest(unittest.TestCase):
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_comparison(self):
# These tests are here simply to exercise the comparison code;
# their presence should not be interpreted as providing any
Lib/test/test_itertools.py
Vendored
-3
View File
@@ -1136,7 +1136,6 @@ class TestBasicOps(unittest.TestCase):
self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)")
self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)")
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_map(self):
self.assertEqual(list(map(operator.pow, range(3), range(1,7))),
[0**1, 1**2, 2**3])
@@ -1261,7 +1260,6 @@ class TestBasicOps(unittest.TestCase):
self.assertEqual(list(t), [1, 1, 1])
self.assertRaises(StopIteration, next, t)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_dropwhile(self):
data = [1, 3, 5, 20, 2, 4, 6, 8]
self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8])
@@ -1272,7 +1270,6 @@ class TestBasicOps(unittest.TestCase):
self.assertRaises(TypeError, next, dropwhile(10, [(4,5)]))
self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)]))
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_tee(self):
n = 200
Lib/test/test_marshal.py
Vendored
-10
View File
@@ -319,7 +319,6 @@ class BugsTestCase(unittest.TestCase):
last.append([0])
self.assertRaises(ValueError, marshal.dumps, head)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_reference_loop_list(self):
a = []
a.append(a)
@@ -331,7 +330,6 @@ class BugsTestCase(unittest.TestCase):
self.assertIsInstance(b, list)
self.assertIs(b[0], b)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_reference_loop_dict(self):
a = {}
a[None] = a
@@ -343,7 +341,6 @@ class BugsTestCase(unittest.TestCase):
self.assertIsInstance(b, dict)
self.assertIs(b[None], b)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_reference_loop_tuple(self):
a = ([],)
a[0].append(a)
@@ -356,7 +353,6 @@ class BugsTestCase(unittest.TestCase):
self.assertIsInstance(b[0], list)
self.assertIs(b[0][0], b)
@unittest.skip("TODO: RUSTPYTHON; unexpected payload for constant python value")
def test_reference_loop_code(self):
def f():
return 1234.5
@@ -370,7 +366,6 @@ class BugsTestCase(unittest.TestCase):
for v in range(marshal.version + 1):
self.assertRaises(ValueError, marshal.dumps, code, v)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by dumps
def test_reference_loop_slice(self):
a = slice([], None)
a.start.append(a)
@@ -387,21 +382,18 @@ class BugsTestCase(unittest.TestCase):
for v in range(marshal.version + 1):
self.assertRaises(ValueError, marshal.dumps, a, v)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_loads_reference_loop_list(self):
data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [<R>]
a = marshal.loads(data)
self.assertIsInstance(a, list)
self.assertIs(a[0], a)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_loads_reference_loop_dict(self):
data = b'\xfbNr\x00\x00\x00\x000' # {None: <R>}
a = marshal.loads(data)
self.assertIsInstance(a, dict)
self.assertIs(a[None], a)
@unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data
def test_loads_abnormal_reference_loops(self):
# Indirect self-references of tuples.
data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([<R>],)
@@ -547,7 +539,6 @@ class BugsTestCase(unittest.TestCase):
_, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
self.assertEqual(dump_0, dump_1)
@unittest.skip("TODO: RUSTPYTHON; unexpected payload for constant python value")
def test_unmarshallable(self):
# Check no crash after encountering unmarshallable objects.
# See https://github.com/python/cpython/issues/106287.
@@ -748,7 +739,6 @@ class InterningTestCase(unittest.TestCase, HelperMixin):
strobj = "this is an interned string"
strobj = sys.intern(strobj)
@unittest.expectedFailure # TODO: RUSTPYTHON
def testIntern(self):
s = marshal.loads(marshal.dumps(self.strobj))
self.assertEqual(s, self.strobj)
Lib/test/test_memoryio.py
Vendored
-16
View File
@@ -587,7 +587,6 @@ class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
self.ioclass(initial_bytes=buf)
self.assertRaises(TypeError, self.ioclass, buf, foo=None)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_concurrent_close(self):
class B:
def __buffer__(self, flags):
@@ -601,7 +600,6 @@ class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
# concurrently mutates (e.g., closes or exports) 'memio'.
# See: https://github.com/python/cpython/issues/143378.
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_writelines_concurrent_close(self):
class B:
def __buffer__(self, flags):
@@ -611,7 +609,6 @@ class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
memio = self.ioclass()
self.assertRaises(ValueError, memio.writelines, [B()])
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_concurrent_export(self):
class B:
buf = None
@@ -622,7 +619,6 @@ class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
memio = self.ioclass()
self.assertRaises(BufferError, memio.write, B())
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_writelines_concurrent_export(self):
class B:
buf = None
@@ -633,7 +629,6 @@ class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
memio = self.ioclass()
self.assertRaises(BufferError, memio.writelines, [B()])
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B'
def test_write_mutating_buffer(self):
# Test that buffer is exported only once during write().
# See: https://github.com/python/cpython/issues/143602.
@@ -930,9 +925,6 @@ class CBytesIOTest(PyBytesIOTest):
def test_flags(self):
return super().test_flags()
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write
def test_write(self):
return super().test_write()
class CStringIOTest(PyStringIOTest):
ioclass = io.StringIO
@@ -1004,10 +996,6 @@ class CStringIOTest(PyStringIOTest):
def test_flags(self):
return super().test_flags()
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'?
def test_newlines_property(self):
return super().test_newlines_property()
class CStringIOPickleTest(PyStringIOPickleTest):
UnsupportedOperation = io.UnsupportedOperation
@@ -1017,9 +1005,5 @@ class CStringIOPickleTest(PyStringIOPickleTest):
def __init__(self, *args, **kwargs):
pass
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'?
def test_newlines_property(self):
return super().test_newlines_property()
if __name__ == '__main__':
unittest.main()
Lib/test/test_memoryview.py
Vendored
-1
View File
@@ -797,7 +797,6 @@ class OtherTest(unittest.TestCase):
m[0] = MyBool()
self.assertEqual(ba[:8], b'\0'*8)
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__'
def test_buffer_reference_loop(self):
m = memoryview(b'abc').__buffer__(0)
o = MyObject()
Lib/test/test_modulefinder.py
Vendored
-2
View File
@@ -390,8 +390,6 @@ class ModuleFinderTest(unittest.TestCase):
os.remove(source_path)
self._do_test(bytecode_test)
# TODO: RUSTPYTHON; panics at code.rs with 'called Option::unwrap() on a None value'
@unittest.skip("TODO: RUSTPYTHON; panics in co_filename replacement")
def test_replace_paths(self):
old_path = os.path.join(self.test_dir, 'a', 'module.py')
new_path = os.path.join(self.test_dir, 'a', 'spam.py')
Lib/test/test_named_expressions.py
Vendored
-1
View File
@@ -365,7 +365,6 @@ class NamedExpressionInvalidTest(unittest.TestCase):
with self.assertRaisesRegex(SyntaxError, msg):
exec(f"lambda: {code}", {}) # Function scope
@unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message
def test_named_expression_invalid_mangled_class_variables(self):
code = """class Foo:
def bar(self):
Lib/test/test_reprlib.py
Vendored
-1
View File
@@ -237,7 +237,6 @@ class ReprTests(unittest.TestCase):
eq(r([[[[[[{}]]]]]]), "[[[[[[{}]]]]]]")
eq(r([[[[[[[{}]]]]]]]), "[[[[[[[...]]]]]]]")
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_cell(self):
def get_cell():
x = 42
Lib/test/test_set.py
Vendored
-1
View File
@@ -330,7 +330,6 @@ class TestJointOps:
name = repr(s).partition('(')[0] # strip class name
self.assertEqual(repr(s), '%s({%s(...)})' % (name, name))
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_do_not_rehash_dict_keys(self):
n = 10
d = dict.fromkeys(map(HashCountingInt, range(n)))
Lib/test/test_sqlite3/test_dbapi.py
Vendored
-3
View File
@@ -364,7 +364,6 @@ class ConnectionTests(unittest.TestCase):
with self.cx:
pass
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_exceptions(self):
# Optional DB-API extension.
self.assertEqual(self.cx.Warning, sqlite.Warning)
@@ -401,7 +400,6 @@ class ConnectionTests(unittest.TestCase):
with self.assertRaises(AttributeError):
self.cx.in_transaction = True
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_connection_exceptions(self):
exceptions = [
"DataError",
@@ -527,7 +525,6 @@ class ConnectionTests(unittest.TestCase):
cx.executemany, "insert into t values(?)",
((v,) for v in range(3)))
@unittest.expectedFailure # TODO: RUSTPYTHON; SQLITE_DBCONFIG constants not implemented
def test_connection_config(self):
op = sqlite.SQLITE_DBCONFIG_ENABLE_FKEY
with memory_database() as cx:
Lib/test/test_sqlite3/test_transactions.py
Vendored
-1
View File
@@ -387,7 +387,6 @@ class AutocommitAttribute(unittest.TestCase):
cx.autocommit = mode
self.assertEqual(cx.autocommit, mode)
@unittest.expectedFailure # TODO: RUSTPYTHON; autocommit validation error messages differ
def test_autocommit_setget_invalid(self):
msg = "autocommit must be True, False, or.*LEGACY"
for mode in "a", 12, (), None:
Lib/test/test_sqlite3/test_userfunctions.py
Vendored
-5
View File
@@ -170,7 +170,6 @@ class FunctionTests(unittest.TestCase):
def tearDown(self):
self.con.close()
@unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args
def test_func_error_on_create(self):
with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"):
self.con.create_function("bla", -100, lambda x: 2*x)
@@ -514,7 +513,6 @@ class WindowFunctionTests(unittest.TestCase):
self.cur.execute(self.query % "sumint")
self.assertEqual(self.cur.fetchall(), self.expected)
@unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args
def test_win_error_on_create(self):
with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"):
self.con.create_window_function("shouldfail", -100, WindowSumInt)
@@ -649,7 +647,6 @@ class AggregateTests(unittest.TestCase):
def tearDown(self):
self.con.close()
@unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args
def test_aggr_error_on_create(self):
with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"):
self.con.create_function("bla", -100, AggrSum)
@@ -803,13 +800,11 @@ class AuthorizerTests(unittest.TestCase):
def tearDown(self):
self.con.close()
@unittest.expectedFailure # TODO: RUSTPYTHON; error message differs
def test_table_access(self):
with self.assertRaises(sqlite.DatabaseError) as cm:
self.con.execute("select * from t2")
self.assertIn('prohibited', str(cm.exception))
@unittest.expectedFailure # TODO: RUSTPYTHON; error message differs
def test_column_access(self):
with self.assertRaises(sqlite.DatabaseError) as cm:
self.con.execute("select c2 from t1")
Lib/test/test_ssl.py
Vendored
-3
View File
@@ -1479,7 +1479,6 @@ class ContextTests(unittest.TestCase):
ctx.set_servername_callback(None)
ctx.set_servername_callback(dummycallback)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Expected 'mock' to not have been called. Called 1 times.
def test_sni_callback_on_dead_references(self):
# See https://github.com/python/cpython/issues/146080.
c_ctx = make_test_context()
@@ -4557,7 +4556,6 @@ class ThreadedTests(unittest.TestCase):
s.sendfile(file)
self.assertEqual(s.recv(1024), TEST_DATA)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_session(self):
client_context, server_context, hostname = testing_context()
# TODO: sessions aren't compatible with TLSv1.3 yet
@@ -4615,7 +4613,6 @@ class ThreadedTests(unittest.TestCase):
self.assertEqual(sess_stat['accept'], 4)
self.assertEqual(sess_stat['hits'], 2)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True
def test_session_handling(self):
client_context, server_context, hostname = testing_context()
client_context2, _, _ = testing_context()
Lib/test/test_struct.py
Vendored
-4
View File
@@ -498,12 +498,10 @@ class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase):
with self.assertRaises((IndexError, OverflowError)):
pack_into(writable_buf, -2**1000, test_string)
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
def test_pack_into(self):
s = struct.Struct('21s')
self._test_pack_into(s.pack_into)
@unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object
def test_pack_into_fn(self):
pack_into = lambda *args: struct.pack_into('21s', *args)
self._test_pack_into(pack_into)
@@ -603,7 +601,6 @@ class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase):
'spam and eggs')
self.assertRaises(struct.error, struct.unpack_from, '14s42', store, 0)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '>h' != '>hh'
def test_Struct_reinitialization(self):
# Issue 9422: there was a memory leak when reinitializing a
# Struct instance. This test can be used to detect the leak
@@ -828,7 +825,6 @@ class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase):
test_error_propagation('N')
test_error_propagation('n')
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_struct_subclass_instantiation(self):
# Regression test for https://github.com/python/cpython/issues/112358
class MyStruct(struct.Struct):
Lib/test/test_structseq.py
Vendored
-6
View File
@@ -87,7 +87,6 @@ class StructSeqTest(unittest.TestCase):
self.assertEqual(t.n_unnamed_fields, 0)
self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument dict
def test_constructor(self):
t = time.struct_time
@@ -111,7 +110,6 @@ class StructSeqTest(unittest.TestCase):
s = "123456789"
self.assertEqual("".join(t(s)), s)
@unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
def test_constructor_with_duplicate_fields(self):
t = time.struct_time
@@ -125,7 +123,6 @@ class StructSeqTest(unittest.TestCase):
with self.assertRaisesRegex(TypeError, error_message):
t("1234567890", dict={"error": 0, "tm_zone": "some zone", "tm_mon": 1})
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2
def test_constructor_with_duplicate_unnamed_fields(self):
assert os.stat_result.n_unnamed_fields > 0
n_visible_fields = os.stat_result.n_sequence_fields
@@ -142,7 +139,6 @@ class StructSeqTest(unittest.TestCase):
re.escape("got duplicate or unexpected field name(s)")):
os.stat_result((*range(n_visible_fields), -1.0), {'st_atime': -1.0})
@unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message
def test_constructor_with_unknown_fields(self):
t = time.struct_time
@@ -185,7 +181,6 @@ class StructSeqTest(unittest.TestCase):
self.assertEqual(t2.tm_year, t.tm_year)
self.assertEqual(t2.tm_zone, t.tm_zone)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2
def test_pickling_with_unnamed_fields(self):
assert os.stat_result.n_unnamed_fields > 0
@@ -220,7 +215,6 @@ class StructSeqTest(unittest.TestCase):
self.assertIsNot(t3[0], t[0])
self.assertIsNot(t3.tm_year, t.tm_year)
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2
def test_copying_with_unnamed_fields(self):
assert os.stat_result.n_unnamed_fields > 0
Lib/test/test_support.py
Vendored
-1
View File
@@ -631,7 +631,6 @@ class TestSupport(unittest.TestCase):
else:
self.assertTrue(support.has_strftime_extensions)
@unittest.expectedFailure # TODO: RUSTPYTHON; - _testinternalcapi module not available
def test_get_recursion_depth(self):
# test support.get_recursion_depth()
code = textwrap.dedent("""
Lib/test/test_symtable.py
Vendored
-6
View File
@@ -247,7 +247,6 @@ class SymtableTest(unittest.TestCase):
self.assertEqual(self.top.get_lineno(), 0)
self.assertEqual(self.spam.get_lineno(), 14)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ: [] != ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x']
def test_function_info(self):
func = self.spam
self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"])
@@ -256,7 +255,6 @@ class SymtableTest(unittest.TestCase):
self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"])
self.assertEqual(self.internal.get_frees(), ("x",))
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_globals(self):
self.assertTrue(self.spam.lookup("glob").is_global())
self.assertFalse(self.spam.lookup("glob").is_declared_global())
@@ -275,7 +273,6 @@ class SymtableTest(unittest.TestCase):
expected = ("some_var",)
self.assertEqual(self.other_internal.get_nonlocals(), expected)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_local(self):
self.assertTrue(self.spam.lookup("x").is_local())
self.assertFalse(self.spam.lookup("bar").is_local())
@@ -283,7 +280,6 @@ class SymtableTest(unittest.TestCase):
self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_local())
self.assertTrue(self.top.lookup("some_assigned_global_var").is_local())
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_free(self):
self.assertTrue(self.internal.lookup("x").is_free())
@@ -328,7 +324,6 @@ class SymtableTest(unittest.TestCase):
self.assertTrue(self.Mine.lookup("a_method").is_assigned())
self.assertFalse(self.internal.lookup("x").is_assigned())
@unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: list index out of range
def test_annotated(self):
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
st2 = st1.get_children()[1]
@@ -493,7 +488,6 @@ class SymtableTest(unittest.TestCase):
self.assertEqual(str(self.top), "<SymbolTable for module ?>")
self.assertEqual(str(self.spam), "<Function SymbolTable for spam in ?>")
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AssertionError: "<symbol 'glob': 0, DEF_GLOBAL>" != "<symbol 'glob': GLOBAL_IMPLICIT, USE>"
def test_symbol_repr(self):
self.assertEqual(repr(self.spam.lookup("glob")),
"<symbol 'glob': GLOBAL_IMPLICIT, USE>")
Lib/test/test_tokenize.py
Vendored
-1
View File
@@ -1922,7 +1922,6 @@ class TestTokenize(TestCase):
tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline))
self.assertEqual(tokens, expected_tokens)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n'
def test_invalid_character_in_fstring_middle(self):
# See gh-103824
script = b'''F"""
Lib/test/test_type_annotations.py
Vendored
-1
View File
@@ -843,7 +843,6 @@ class RegressionTests(unittest.TestCase):
lamb = list(genexp)[0]
self.assertEqual(lamb(), 42)
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__annotate__' != 'f.__annotate__'
def test_annotate_qualname(self):
code = """
def f() -> None:
README.md
+1 -1
View File
@@ -41,7 +41,7 @@ needed to prevent stack overflow on Windows):
```bash
$ cd RustPython
$ cargo run --release demo_closures.py
$ cargo run --release -- -c 'print("Hello, RustPython!")'
Hello, RustPython!
```
crates/capi/src/genericaliasobject.rs
+1 -1
View File
@@ -10,6 +10,6 @@ pub unsafe extern "C" fn Py_GenericAlias(
with_vm(|vm| {
let origin = unsafe { &*origin }.to_owned();
let args = unsafe { &*args }.to_owned();
PyGenericAlias::from_args(origin, args, vm).into_pyobject(vm)
PyGenericAlias::from_args(origin, args, vm).map(|alias| alias.into_pyobject(vm))
})
}
crates/capi/src/objimpl.rs
+16 -12
View File
@@ -9,7 +9,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut PyObject) {
with_vm(|_vm| {
let obj = unsafe { &*op };
if !obj.is_gc_tracked() {
unsafe { gc_state::gc_state().track_object(obj.into()) };
unsafe { gc_state::gc_state().track_object(obj.into(), gc_state::current_owner()) };
}
})
}
@@ -36,29 +36,33 @@ pub unsafe extern "C" fn PyObject_GC_IsFinalized(op: *mut PyObject) -> c_int {
#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Collect() -> isize {
let result = gc_state::gc_state().collect(2);
(result.collected + result.uncollectable) as isize
with_vm(|vm| {
let result = vm.state.gc.collect(2);
(result.collected + result.uncollectable) as isize
})
}
#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Enable() -> c_int {
let gc = gc_state::gc_state();
let was_enabled = gc.is_enabled();
gc.enable();
was_enabled.into()
with_vm(|vm| {
let was_enabled: c_int = vm.state.gc.is_enabled().into();
vm.state.gc.enable();
was_enabled
})
}
#[unsafe(no_mangle)]
pub extern "C" fn PyGC_Disable() -> c_int {
let gc = gc_state::gc_state();
let was_enabled = gc.is_enabled();
gc.disable();
was_enabled.into()
with_vm(|vm| {
let was_enabled: c_int = vm.state.gc.is_enabled().into();
vm.state.gc.disable();
was_enabled
})
}
#[unsafe(no_mangle)]
pub extern "C" fn PyGC_IsEnabled() -> c_int {
gc_state::gc_state().is_enabled().into()
with_vm(|vm| -> c_int { vm.state.gc.is_enabled().into() })
}
#[unsafe(no_mangle)]
crates/capi/src/pyerrors.rs
+1 -1
View File
@@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create(
unsafe { slice::from_raw_parts(object.cast::<u8>(), length) }.to_vec()
};
let exc = vm.new_unicode_decode_error_real(
let exc = vm.new_unicode_decode_error(
vm.ctx.new_str(encoding),
vm.ctx.new_bytes(bytes),
start,
crates/capi/src/pystate.rs
+2 -2
View File
@@ -109,8 +109,8 @@ mod tests {
current_vm_is_set(),
"This thread did not have a vm attached"
);
vm.state.stop_the_world.stop_the_world(vm);
vm.state.stop_the_world.start_the_world(vm);
vm.state.stop_the_world.stop_the_world(&vm.state);
vm.state.stop_the_world.start_the_world(&vm.state);
});
});
});
crates/codegen/src/compile.rs
+80 -21
View File
@@ -2175,7 +2175,7 @@ impl<'warnings> Compiler<'warnings> {
/// On success, returns the saved CompileContext to pass to exit_annotation_scope.
fn enter_annotation_scope(
&mut self,
_func_name: &str,
func_name: &str,
loc: TextRange,
) -> CompileResult<Option<CompileContext>> {
if !self.push_annotation_symbol_table() {
@@ -2200,6 +2200,12 @@ impl<'warnings> Compiler<'warnings> {
lineno.to_u32(),
)?;
// enter_scope() qualified the scope by the enclosing scope only; redo it
// now that the annotated function is known. Only signature annotations
// get this treatment - deferred class and module annotations are
// compiled inside the scope they belong to and are already qualified.
self.set_annotation_qualname(func_name);
// Keep the internal ".format" name; exit_annotation_scope()
// renames it to "format" on the final code object.
self.current_code_info()
@@ -2400,7 +2406,13 @@ impl<'warnings> Compiler<'warnings> {
}
if let FBlockDatum::FinallyBody(ref body) = info.fb_datum {
// This is an extra copy of the finally body, emitted for the
// path that leaves the try block early. The try statement
// emits its own copies afterwards, so rewind the symbol table
// cursors and leave the nested scopes for those copies.
let symbol_table_cursors = self.current_symbol_table_cursors();
self.compile_statements(body)?;
self.set_symbol_table_cursors(symbol_table_cursors);
}
if preserve_tos {
@@ -2605,11 +2617,24 @@ impl<'warnings> Compiler<'warnings> {
/// Set the qualified name for the current code object
// = compiler_set_qualname
fn set_qualname(&mut self) -> String {
let qualname = self.make_qualname();
self.set_qualname_for_function(None)
}
/// Set the qualname of an annotation scope, qualified by the function whose
/// signature it annotates. CPython records that name on the annotation
/// block's symbol table entry (`ste_function_name`) and folds it into the
/// qualname, so `f`'s annotation scope is named `f.__annotate__`.
fn set_annotation_qualname(&mut self, function_name: &str) {
self.set_qualname_for_function(Some(function_name));
}
fn set_qualname_for_function(&mut self, function_name: Option<&str>) -> String {
let qualname = self.make_qualname(function_name);
self.current_code_info().metadata.qualname = Some(qualname.clone());
qualname
}
fn make_qualname(&mut self) -> String {
fn make_qualname(&mut self, function_name: Option<&str>) -> String {
let stack_size = self.code_stack.len();
assert!(stack_size >= 1);
@@ -2693,10 +2718,10 @@ impl<'warnings> Compiler<'warnings> {
}
}
// Build the qualified name
if force_global {
// Build the prefix the current name is qualified by, if any
let base = if force_global {
// For global symbols, qualname is just the name
current_obj_name
None
} else {
// Check parent scope type
let parent_obj_name = &parent.metadata.name;
@@ -2709,23 +2734,32 @@ impl<'warnings> Compiler<'warnings> {
)
);
// Use parent's qualname if available, otherwise use parent_obj_name
let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name);
if is_function_parent {
// For functions, append .<locals> to parent qualname
// Use parent's qualname if available, otherwise use parent_obj_name
let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name);
format!("{parent_qualname}.<locals>.{current_obj_name}")
Some(format!("{parent_qualname}.<locals>"))
} else if parent_qualname == "<module>" {
// Module level, nothing to qualify by
None
} else {
// For classes and other scopes, use parent's qualname directly
// Use parent's qualname if available, otherwise use parent_obj_name
let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name);
if parent_qualname == "<module>" {
// Module level, just use the name
current_obj_name
} else {
// Concatenate parent qualname with current name
format!("{parent_qualname}.{current_obj_name}")
}
Some(parent_qualname.clone())
}
};
// An annotation scope is compiled in the scope enclosing the function it
// annotates, so the function itself is missing from the prefix above.
let base = match (base, function_name) {
(Some(base), Some(function_name)) => Some(format!("{base}.{function_name}")),
(None, Some(function_name)) => Some(function_name.to_owned()),
(base, None) => base,
};
match base {
Some(base) => format!("{base}.{current_obj_name}"),
None => current_obj_name,
}
}
@@ -5055,6 +5089,14 @@ impl<'warnings> Compiler<'warnings> {
func_range: TextRange,
) -> CompileResult<bool> {
if !self.next_function_annotation_symbol_table_uses_annotations() {
// CPython creates a hidden AnnotationBlock for every function
// signature under `from __future__ import annotations`, including
// an unannotated one. It still belongs to this function: consume
// it so the next function sees its own block rather than remaining
// pinned to this unused entry.
if self.push_annotation_symbol_table() {
self.pop_annotation_symbol_table();
}
return Ok(false);
}
@@ -10877,9 +10919,7 @@ impl<'warnings> Compiler<'warnings> {
if sym.flags.contains(SymbolFlags::DEF_PARAM) {
continue; // skip .0
}
let is_local = sym
.flags
.intersects(SymbolFlags::DEF_LOCAL | SymbolFlags::ITER)
let is_local = sym.flags.contains(SymbolFlags::DEF_LOCAL)
&& !sym.flags.contains(SymbolFlags::DEF_NONLOCAL);
if is_local {
pushed_locals.push(name.clone());
@@ -31219,6 +31259,25 @@ def f(x: T): pass
);
}
#[test]
fn future_unannotated_function_does_not_hide_next_annotation_block() {
let code = compile_exec(
"\
from __future__ import annotations
def plain(x): pass
def annotated(x: int): pass
",
);
let annotate = find_direct_child_code(&code, "__annotate__")
.expect("second function must retain its annotation closure");
assert!(
annotate.constants.iter().any(
|constant| matches!(constant, ConstantData::Str { value } if value.as_str() == Ok("int"))
),
"annotation closure must belong to the annotated function"
);
}
#[test]
fn deferred_annotation_format_name_does_not_capture_helper_parameter() {
let code = compile_exec(
crates/codegen/src/symboltable.rs
+13 -66
View File
@@ -312,19 +312,8 @@ bitflags! {
Self::DEF_LOCAL.bits()
| Self::DEF_PARAM.bits()
| Self::DEF_IMPORT.bits()
| Self::ITER.bits()
| Self::DEF_TYPE_PARAM.bits()
);
// TODO: Remove these, RustPython specific
// indicates if the symbol gets a value assigned by a named expression in a comprehension
// this is required to correct the scope in the analysis.
const ASSIGNED_IN_COMPREHENSION = 2 << 11;
// indicates that the symbol is used a bound iterator variable. We distinguish this case
// from normal assignment to detect disallowed re-assignment to iterator variables.
const ITER = 2 << 12;
}
}
@@ -1030,7 +1019,6 @@ enum SymbolUsage {
AnnotationAssigned,
Parameter,
AnnotationParameter,
AssignedNamedExprInComprehension,
Iter,
TypeParam,
}
@@ -1048,8 +1036,6 @@ struct SymbolTableBuilder {
varnames_stack: Vec<Vec<String>>,
// Track if we're inside an iterable definition expression (for nested comprehensions)
in_iter_def_exp: bool,
// Track if we're scanning an inner loop iteration target (not the first generator)
in_comp_inner_loop_target: bool,
// yield/yield from inside comprehension scopes is rejected with a
// message that names the comprehension kind.
comprehension_yield_context: Option<&'static str>,
@@ -1084,7 +1070,6 @@ impl SymbolTableBuilder {
current_varnames: Vec::new(),
varnames_stack: Vec::new(),
in_iter_def_exp: false,
in_comp_inner_loop_target: false,
comprehension_yield_context: None,
in_conditional_block: false,
recursion_depth: 0,
@@ -2525,24 +2510,8 @@ impl SymbolTableBuilder {
self.scan_expression(value, ExpressionContext::Load)?;
// special handling for assigned identifier in named expressions
// that are used in comprehensions. This required to correctly
// propagate the scope of the named assigned named and not to
// propagate inner names.
if let Some((id, target_range)) = named_target {
let table = self.tables.last().unwrap();
if table.typ == CompilerScope::Comprehension {
self.register_name(
id,
SymbolUsage::AssignedNamedExprInComprehension,
target_range,
)?;
} else {
// omit one recursion. When the handling of an store changes for
// Identifiers this needs adapted - more forward safe would be
// calling scan_expression directly.
self.register_name(id, SymbolUsage::Assigned, target_range)?;
}
self.register_name(id, SymbolUsage::Assigned, target_range)?;
} else {
self.scan_expression(target, ExpressionContext::Store)?;
}
@@ -2613,9 +2582,7 @@ impl SymbolTableBuilder {
}
for generator in &generators[1..] {
self.in_comp_inner_loop_target = true;
self.scan_expression(&generator.target, ExpressionContext::Iter)?;
self.in_comp_inner_loop_target = false;
let was_in_iter_def_exp = self.in_iter_def_exp;
self.in_iter_def_exp = true;
self.scan_expression(&generator.iter, ExpressionContext::IterDefinitionExp)?;
@@ -3037,11 +3004,15 @@ impl SymbolTableBuilder {
if self.tables[table_idx]
.symbols
.get(mangled.as_str())
.is_some_and(|symbol| symbol.flags.contains(SymbolFlags::ITER))
.is_some_and(|symbol| {
symbol
.flags
.contains(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_COMP_ITER)
})
{
return Err(SymbolTableError {
error: format!(
"assignment expression cannot rebind comprehension iteration variable '{mangled}'"
"assignment expression cannot rebind comprehension iteration variable '{name}'"
),
location,
});
@@ -3151,7 +3122,6 @@ impl SymbolTableBuilder {
| SymbolUsage::AnnotationAssigned
| SymbolUsage::Parameter
| SymbolUsage::AnnotationParameter
| SymbolUsage::AssignedNamedExprInComprehension
| SymbolUsage::Iter
| SymbolUsage::TypeParam
) {
@@ -3179,16 +3149,16 @@ impl SymbolTableBuilder {
let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
let flags = &symbol.flags;
// INNER_LOOP_CONFLICT: comprehension inner loop cannot rebind
// a variable that was used as a named expression target
// Mirrors CPython's INNER_LOOP_CONFLICT check. extend_namedexpr_scope()
// marks named-expression targets as global or nonlocal in the comprehension.
// Example: [i for i in range(5) if (j := 0) for j in range(5)]
// Here 'j' is used in named expr first, then as inner loop iter target
if self.in_comp_inner_loop_target
&& flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION)
if matches!(role, SymbolUsage::Iter)
&& flags.intersects(SymbolFlags::DEF_GLOBAL | SymbolFlags::DEF_NONLOCAL)
{
return Err(SymbolTableError {
error: format!(
"comprehension inner loop cannot rebind assignment expression target '{name}'"
"comprehension inner loop cannot rebind assignment expression target '{original_name}'"
),
location,
});
@@ -3345,9 +3315,6 @@ impl SymbolTableBuilder {
SymbolUsage::Assigned => {
flags.insert(SymbolFlags::DEF_LOCAL);
}
SymbolUsage::AssignedNamedExprInComprehension => {
flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::ASSIGNED_IN_COMPREHENSION);
}
SymbolUsage::Global => {
symbol.scope = SymbolScope::GlobalExplicit;
flags.insert(SymbolFlags::DEF_GLOBAL);
@@ -3356,33 +3323,13 @@ impl SymbolTableBuilder {
flags.insert(SymbolFlags::USE);
}
SymbolUsage::Iter => {
// CPython symtable_add_def_helper() records an inlined
// comprehension target as a local definition as well as a
// comprehension iterator. Keep ITER as the internal
// re-assignment check marker; DEF_LOCAL is part of the public
// ste_symbols flags exposed by _symtable.
flags.insert(
SymbolFlags::DEF_LOCAL | SymbolFlags::ITER | SymbolFlags::DEF_COMP_ITER,
);
flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_COMP_ITER);
}
SymbolUsage::TypeParam => {
flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_TYPE_PARAM);
}
}
// and even more checking
// it is not allowed to assign to iterator variables (by named expressions)
if flags.contains(SymbolFlags::ITER)
&& flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION)
{
return Err(SymbolTableError {
error: format!(
"assignment expression cannot rebind comprehension iteration variable '{}'",
symbol.name
),
location,
});
}
Ok(())
}
}
crates/common/src/borrow.rs
+11
View File
@@ -34,6 +34,17 @@ impl_from!('a, T, BorrowedValue<'a, T>,
);
impl<'a, T: ?Sized> BorrowedValue<'a, T> {
/// Whether reaching the value holds a lock that other threads wait on.
///
/// An immutable object hands out a plain reference and answers `false`;
/// one whose storage can change hands out a guard. A caller about to wait
/// for something unrelated -- a peer, a file, a signal -- can use this to
/// decide whether it may keep the borrow for the duration.
#[must_use]
pub const fn is_locked(&self) -> bool {
!matches!(self, Self::Ref(_))
}
pub fn map<U: ?Sized, F>(s: Self, f: F) -> BorrowedValue<'a, U>
where
F: FnOnce(&T) -> &U,
crates/common/src/format.rs
+221 -30
View File
@@ -229,6 +229,7 @@ pub struct FormatSpec {
align: Option<FormatAlign>,
align_specified: bool,
sign: Option<FormatSign>,
no_neg_0: bool,
alternate_form: bool,
width: Option<usize>,
grouping_option: Option<FormatGrouping>,
@@ -285,6 +286,14 @@ fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) {
}
}
fn parse_no_negative_zero(text: &Wtf8) -> (bool, &Wtf8) {
let mut chars = text.code_points();
match chars.next().and_then(CodePoint::to_char) {
Some('z') => (true, chars.as_wtf8()),
_ => (false, text),
}
}
fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) {
let mut chars = text.code_points();
match chars.next().and_then(CodePoint::to_char) {
@@ -349,6 +358,7 @@ impl FormatSpec {
let (mut fill, mut align, text) = parse_fill_and_align(text);
let align_specified = align.is_some();
let (sign, text) = FormatSign::parse(text);
let (no_neg_0, text) = parse_no_negative_zero(text);
let (alternate_form, text) = parse_alternate_form(text);
let (zero, text) = parse_zero(text);
let (width, text) = parse_number(text)?;
@@ -378,6 +388,7 @@ impl FormatSpec {
align,
align_specified,
sign,
no_neg_0,
alternate_form,
width,
grouping_option,
@@ -505,6 +516,33 @@ impl FormatSpec {
Ok(())
}
fn formatted_magnitude_is_zero(magnitude: &str) -> bool {
let mut saw_digit = false;
for byte in magnitude.bytes() {
if byte.is_ascii_digit() {
saw_digit = true;
if byte != b'0' {
return false;
}
}
}
saw_digit
}
fn is_negative_after_zero_coercion(&self, num: f64, magnitude: &str) -> bool {
num.is_sign_negative()
&& !num.is_nan()
&& !(self.no_neg_0 && Self::formatted_magnitude_is_zero(magnitude))
}
fn validate_complex_padding_and_alignment(&self) -> Result<(), FormatSpecError> {
match &self.fill.unwrap_or_else(|| ' '.into()).to_char() {
Some('0') => Err(FormatSpecError::ZeroPadding),
_ if self.align == Some(FormatAlign::AfterSign) => Err(FormatSpecError::AlignmentFlag),
_ => Ok(()),
}
}
const fn get_separator_interval(&self) -> usize {
match self.format_type {
Some(FormatType::Binary | FormatType::Octal | FormatType::Hex(_)) => 4,
@@ -666,6 +704,9 @@ impl FormatSpec {
Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10),
_ => return self.format_int(num),
}?;
if self.no_neg_0 {
return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"));
}
let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale);
@@ -715,7 +756,7 @@ impl FormatSpec {
let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale);
let format_sign = self.sign.unwrap_or(FormatSign::Minus);
let sign_str = if num.is_sign_negative() && !num.is_nan() {
let sign_str = if self.is_negative_after_zero_coercion(num, &magnitude_str) {
"-"
} else {
match format_sign {
@@ -784,6 +825,7 @@ impl FormatSpec {
// No parentheses for 'n' format (CPython: add_parens=0)
let magnitude_str = format!("{grouped_re}{grouped_im}");
self.validate_complex_padding_and_alignment()?;
Ok(self.format_sign_and_align(&AsciiStr::new(&magnitude_str), "", FormatAlign::Right))
}
@@ -803,11 +845,16 @@ impl FormatSpec {
self.format_float(x as f64)
}
None => {
if self.no_neg_0 {
return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"));
}
let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase();
Ok(first_letter.collect::<String>() + &input.to_string()[1..])
}
Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(*c, "int")),
_ => Err(FormatSpecError::InvalidFormatSpecifier),
Some(format_type) => {
let ch = char::from(format_type);
Err(FormatSpecError::UnknownFormatCode(ch, "bool"))
}
}
}
@@ -916,8 +963,9 @@ impl FormatSpec {
},
},
};
let raw_magnitude_str = raw_magnitude_str?;
let format_sign = self.sign.unwrap_or(FormatSign::Minus);
let sign_str = if num.is_sign_negative() && !num.is_nan() {
let sign_str = if self.is_negative_after_zero_coercion(num, &raw_magnitude_str) {
"-"
} else {
match format_sign {
@@ -926,7 +974,7 @@ impl FormatSpec {
FormatSign::MinusOrSpace => " ",
}
};
let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str);
let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, sign_str);
let magnitude_str = self.add_frac_separators(magnitude_str);
Ok(
self.format_sign_and_align(
@@ -977,15 +1025,24 @@ impl FormatSpec {
Err(FormatSpecError::UnknownFormatCode('N', "int"))
}
Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")),
Some(FormatType::Character) => match (self.precision, self.sign, self.alternate_form) {
(Some(_), _, _) => Err(FormatSpecError::PrecisionNotAllowed),
(_, Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")),
(_, _, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")),
(_, _, _) => match num.to_u32() {
Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()),
Some(_) | None => Err(FormatSpecError::CodeNotInRange),
},
},
Some(FormatType::Character) => {
if self.precision.is_some() {
Err(FormatSpecError::PrecisionNotAllowed)
} else if self.no_neg_0 {
Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"))
} else {
match (self.sign, self.alternate_form) {
(Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")),
(_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")),
_ => match num.to_u32() {
Some(n) if n <= 0x10ffff => {
Ok(core::char::from_u32(n).unwrap().to_string())
}
Some(_) | None => Err(FormatSpecError::CodeNotInRange),
},
}
}
}
Some(
FormatType::GeneralFormat(_)
| FormatType::FixedPoint(_)
@@ -998,6 +1055,9 @@ impl FormatSpec {
Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(c, "int")),
None => self.format_int_radix(magnitude, 10),
}?;
if self.no_neg_0 {
return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"));
}
let format_sign = self.sign.unwrap_or(FormatSign::Minus);
let sign_str = match num.sign() {
Sign::Minus => "-",
@@ -1023,6 +1083,9 @@ impl FormatSpec {
self.validate_format(FormatType::String)?;
match self.format_type {
Some(FormatType::String) | None => {
if self.no_neg_0 {
return Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string"));
}
if self.align == Some(FormatAlign::AfterSign) && self.align_specified {
return Err(FormatSpecError::StringAlignmentFlag);
}
@@ -1057,24 +1120,16 @@ impl FormatSpec {
} else {
format!("{formatted_re}{formatted_im}")
};
if let Some(FormatAlign::AfterSign) = &self.align {
return Err(FormatSpecError::AlignmentFlag);
}
match &self.fill.unwrap_or_else(|| ' '.into()).to_char() {
Some('0') => Err(FormatSpecError::ZeroPadding),
_ => Ok(self.format_sign_and_align(
&AsciiStr::new(&magnitude_str),
"",
FormatAlign::Right,
)),
}
self.validate_complex_padding_and_alignment()?;
Ok(self.format_sign_and_align(&AsciiStr::new(&magnitude_str), "", FormatAlign::Right))
}
fn format_complex_re_im(&self, num: &Complex64) -> Result<(String, String), FormatSpecError> {
// Format real part
let formatted_re =
if num.re != 0.0 || num.re.is_negative_zero() || self.format_type.is_some() {
let sign_re = if num.re.is_sign_negative() && !num.is_nan() {
let re = self.format_complex_float(num.re)?;
let sign_re = if self.is_negative_after_zero_coercion(num.re, &re) {
"-"
} else {
match self.sign.unwrap_or(FormatSign::Minus) {
@@ -1083,21 +1138,24 @@ impl FormatSpec {
FormatSign::MinusOrSpace => " ",
}
};
let re = self.format_complex_float(num.re)?;
format!("{sign_re}{re}")
} else {
String::new()
};
// Format imaginary part
let sign_im = if num.im.is_sign_negative() && !num.im.is_nan() {
let im = self.format_complex_float(num.im)?;
let sign_im = if self.is_negative_after_zero_coercion(num.im, &im) {
"-"
} else if formatted_re.is_empty() {
""
match self.sign.unwrap_or(FormatSign::Minus) {
FormatSign::Plus => "+",
FormatSign::Minus => "",
FormatSign::MinusOrSpace => " ",
}
} else {
"+"
};
let im = self.format_complex_float(num.im)?;
Ok((formatted_re, format!("{sign_im}{im}j")))
}
@@ -1274,6 +1332,7 @@ pub enum FormatSpecError {
CodeNotInRange,
ZeroPadding,
AlignmentFlag,
NegativeZeroCoercionNotAllowed(&'static str),
StringAlignmentFlag,
NotImplemented(char, &'static str),
}
@@ -1625,6 +1684,7 @@ mod tests {
align: None,
align_specified: false,
sign: None,
no_neg_0: false,
alternate_form: false,
width: Some(33),
grouping_option: None,
@@ -1643,6 +1703,7 @@ mod tests {
align: Some(FormatAlign::Right),
align_specified: true,
sign: None,
no_neg_0: false,
alternate_form: false,
width: Some(33),
grouping_option: None,
@@ -1661,6 +1722,7 @@ mod tests {
align: Some(FormatAlign::Right),
align_specified: true,
sign: Some(FormatSign::Minus),
no_neg_0: false,
alternate_form: true,
width: Some(23),
grouping_option: Some(FormatGrouping::Comma),
@@ -1726,6 +1788,26 @@ mod tests {
);
}
#[test]
fn format_complex_rejects_zero_padding_before_after_sign_alignment() {
for text in [
"08.1f", "=08.1f", "0=8.1f", "#08.1f", "0>8.1f", "0<8.1f", "0^8.1f",
] {
let spec = FormatSpec::parse(text).unwrap();
assert_eq!(
spec.format_complex(&Complex64::new(1.0, 2.0)),
Err(FormatSpecError::ZeroPadding),
"{text}"
);
}
let spec = FormatSpec::parse("=8.1f").unwrap();
assert_eq!(
spec.format_complex(&Complex64::new(1.0, 2.0)),
Err(FormatSpecError::AlignmentFlag)
);
}
#[test]
fn format_int_zero_padding_stays_after_sign() {
let spec = FormatSpec::parse("08").unwrap();
@@ -1736,6 +1818,115 @@ mod tests {
);
}
#[test]
fn format_complex_locale_rejects_zero_padding_before_after_sign_alignment() {
let locale = LocaleInfo {
thousands_sep: String::new(),
decimal_point: ".".to_owned(),
grouping: vec![],
};
for text in ["08n", "=08n", "0=8n", "#08n", "0>8n", "0<8n", "0^8n"] {
let spec = FormatSpec::parse(text).unwrap();
assert_eq!(
spec.format_complex_locale(&Complex64::new(1.0, 2.0), &locale),
Err(FormatSpecError::ZeroPadding),
"{text}"
);
}
let spec = FormatSpec::parse("=8n").unwrap();
assert_eq!(
spec.format_complex_locale(&Complex64::new(1.0, 2.0), &locale),
Err(FormatSpecError::AlignmentFlag)
);
}
#[test]
fn format_negative_zero_coercion() {
let int_spec = FormatSpec::parse("z8").unwrap();
assert_eq!(
int_spec.format_int(&BigInt::from(-42)),
Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"))
);
assert_eq!(
FormatSpec::parse("zs")
.unwrap()
.format_int(&BigInt::from(0)),
Err(FormatSpecError::UnknownFormatCode('s', "int"))
);
assert_eq!(
FormatSpec::parse("z.1d")
.unwrap()
.format_int(&BigInt::from(0)),
Err(FormatSpecError::PrecisionNotAllowed)
);
assert_eq!(
FormatSpec::parse("+zc")
.unwrap()
.format_int(&BigInt::from(0)),
Err(FormatSpecError::NegativeZeroCoercionNotAllowed("integer"))
);
let float_spec = FormatSpec::parse("z.2f").unwrap();
assert_eq!(float_spec.format_float(-0.0001), Ok("0.00".to_owned()));
let complex_spec = FormatSpec::parse("z").unwrap();
assert_eq!(
complex_spec.format_complex(&Complex64::new(-0.0, -0.0)),
Ok("(0+0j)".to_owned())
);
let pure_imaginary = Complex64::new(0.0, -0.0);
assert_eq!(
FormatSpec::parse("+z")
.unwrap()
.format_complex(&pure_imaginary),
Ok("+0j".to_owned())
);
assert_eq!(
FormatSpec::parse(" z")
.unwrap()
.format_complex(&pure_imaginary),
Ok(" 0j".to_owned())
);
let string_value = "value".to_owned();
assert_eq!(
FormatSpec::parse("z").unwrap().format_string(&string_value),
Err(FormatSpecError::NegativeZeroCoercionNotAllowed("string"))
);
assert_eq!(
FormatSpec::parse("zd")
.unwrap()
.format_string(&string_value),
Err(FormatSpecError::UnknownFormatCode('d', "str"))
);
assert_eq!(
FormatSpec::parse("zs").unwrap().format_bool(false),
Err(FormatSpecError::UnknownFormatCode('s', "bool"))
);
let locale = LocaleInfo {
thousands_sep: ",".to_owned(),
decimal_point: ".".to_owned(),
grouping: vec![3, 0],
};
let locale_spec = FormatSpec::parse("zn").unwrap();
assert_eq!(
locale_spec.format_float_locale(-0.0, &locale),
Ok("0".to_owned())
);
assert_eq!(
locale_spec.format_complex_locale(&Complex64::new(-0.0, -0.0), &locale),
Ok("0+0j".to_owned())
);
assert_eq!(
FormatSpec::parse("z.1n")
.unwrap()
.format_int_locale(&BigInt::from(0), &locale),
Err(FormatSpecError::PrecisionNotAllowed)
);
}
#[test]
fn format_int() {
assert_eq!(
crates/common/src/lib.rs
+1
View File
@@ -21,6 +21,7 @@ pub mod rc;
pub mod refcount;
pub mod static_cell;
pub mod str;
pub mod wtf8_index;
pub use rustpython_wtf8 as wtf8;
crates/common/src/str.rs
+193 -16
View File
@@ -1,7 +1,8 @@
// spell-checker:ignore uncomputed
use crate::atomic::{PyAtomic, Radium};
use crate::atomic::{OncePtr, PyAtomic, Radium};
use crate::format::CharLen;
use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
use crate::wtf8_index::Wtf8Index;
use ascii::{AsciiChar, AsciiStr, AsciiString};
use core::fmt;
use core::ops::{Bound, RangeBounds};
@@ -112,11 +113,70 @@ pub enum PyKindStr<'a> {
Wtf8(&'a Wtf8),
}
/// How far from an end an index is resolved by walking rather than by building
/// the code point index.
///
/// PyPy spells this `MAX_UNROLL_NEXT_CODEPOINT_POS`, in a guard that also asks
/// the JIT whether the index is a constant, so that the walk unrolls. There is
/// no JIT here to ask, and the walk is short rather than free -- but four steps
/// still beat a pass over the whole buffer, and skipping the build is what
/// keeps `s[0]` and `s[1:-1]` on a long string from paying for a table.
const MAX_WALK_TO_INDEX: usize = 4;
#[derive(Debug, Clone)]
pub struct StrData {
data: Box<Wtf8>,
kind: StrKind,
len: StrLen,
index: Wtf8IndexSlot,
}
/// A [`Wtf8Index`] built on first use.
///
/// The table is a pure function of `data`, so publishing it races benignly: a
/// thread that loses the exchange drops its own copy and reads the winner's.
#[derive(Default)]
struct Wtf8IndexSlot(OncePtr<Wtf8Index>);
impl Wtf8IndexSlot {
#[inline(always)]
fn new() -> Self {
Self(OncePtr::new())
}
#[inline]
fn get_or_build(&self, data: &Wtf8, char_len: usize) -> &Wtf8Index {
let index = self
.0
.get_or_init(|| Box::new(Wtf8Index::new(data, char_len)));
// The slot owns the table, never replaces it, and outlives the borrow.
unsafe { index.as_ref() }
}
}
impl fmt::Debug for Wtf8IndexSlot {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self.0.get() {
Some(_) => f.write_str("<built>"),
None => f.write_str("<unbuilt>"),
}
}
}
impl Clone for Wtf8IndexSlot {
/// A fresh slot: the clone copies the buffer, so it has to index that copy,
/// and the table is rebuilt on demand rather than eagerly here.
fn clone(&self) -> Self {
Self::new()
}
}
impl Drop for Wtf8IndexSlot {
fn drop(&mut self) {
if let Some(index) = self.0.get() {
drop(unsafe { Box::from_raw(index.as_ptr()) });
}
}
}
struct StrLen(PyAtomic<usize>);
@@ -163,6 +223,7 @@ impl Default for StrData {
data: <Box<Wtf8>>::default(),
kind: StrKind::Ascii,
len: StrLen::zero(),
index: Wtf8IndexSlot::new(),
}
}
}
@@ -193,6 +254,7 @@ impl From<Box<AsciiStr>> for StrData {
len: value.len().into(),
data: value.into(),
kind: StrKind::Ascii,
index: Wtf8IndexSlot::new(),
}
}
}
@@ -212,6 +274,7 @@ impl From<char> for StrData {
data: ch.to_string().into(),
kind: StrKind::Utf8,
len: 1.into(),
index: Wtf8IndexSlot::new(),
}
}
}
@@ -226,6 +289,7 @@ impl From<CodePoint> for StrData {
data: Wtf8Buf::from(ch).into(),
kind: StrKind::Wtf8,
len: 1.into(),
index: Wtf8IndexSlot::new(),
}
}
}
@@ -241,7 +305,12 @@ impl StrData {
StrKind::Ascii => data.len().into(),
_ => StrLen::uncomputed(),
};
Self { data, kind, len }
Self {
data,
kind,
len,
index: Wtf8IndexSlot::new(),
}
}
/// # Safety
@@ -253,6 +322,7 @@ impl StrData {
data,
kind,
len: char_len.into(),
index: Wtf8IndexSlot::new(),
}
}
@@ -322,11 +392,117 @@ impl StrData {
len
}
/// The byte offset the `index`-th code point starts at.
///
/// An `index` at or past the end answers the buffer's byte length, so a
/// caller walking to a bound does not have to special-case it.
///
/// O(1), but the first call on a non-ASCII string builds an index over the
/// whole buffer, so a caller that resolves a single index and stops is
/// better served by [`Self::nth_char`].
pub fn char_index_to_byte(&self, index: usize) -> usize {
// For ASCII the two units coincide, and the table would be a Nth entry
// saying N.
if self.kind.is_ascii() {
return index.min(self.data.len());
}
let char_len = self.char_len();
if index >= char_len {
return self.data.len();
}
self.index
.get_or_build(&self.data, char_len)
.byte_offset(&self.data, index)
}
/// The byte offset of code point `index`, for a caller that resolves one
/// index and stops.
///
/// Building the table costs a pass over the whole buffer, so it is worth it
/// only for a caller that comes back; an index within
/// [`MAX_WALK_TO_INDEX`] steps of either end is cheaper to walk to, and
/// walking keeps `s[0]` on a long string from paying for a table it will
/// never use again. Anything further in builds, on the reasoning that a
/// string indexed once in the middle tends to be indexed again.
fn char_index_to_byte_once(&self, index: usize) -> usize {
if index <= MAX_WALK_TO_INDEX {
return self
.data
.code_point_indices()
.nth(index)
.map_or(self.data.len(), |(byte, _)| byte);
}
let from_end = self.char_len() - index;
if from_end <= MAX_WALK_TO_INDEX {
return self
.data
.code_point_indices()
.nth_back(from_end - 1)
.map_or(self.data.len(), |(byte, _)| byte);
}
self.char_index_to_byte(index)
}
/// The byte range spanned by the code points in `range`, whose end must not
/// exceed the string's code point count.
///
/// A range that reaches within [`MAX_WALK_TO_INDEX`] of *both* ends is
/// walked to for the same reason a single index near one end is -- a slice
/// like `s[1:-1]` should not build a table over the whole string.
#[must_use]
pub fn char_range_to_bytes(&self, range: core::ops::Range<usize>) -> core::ops::Range<usize> {
if self.kind.is_ascii() {
return range;
}
let from_end = self.char_len() - range.end;
if range.start <= MAX_WALK_TO_INDEX && from_end <= MAX_WALK_TO_INDEX {
// Two walks over disjoint ends, each of at most MAX_WALK_TO_INDEX
// steps -- one iterator driven from both sides would have them meet
// on a short string.
let start = self
.data
.code_point_indices()
.nth(range.start)
.map_or(self.data.len(), |(byte, _)| byte);
let end = match from_end {
0 => self.data.len(),
n => self
.data
.code_point_indices()
.nth_back(n - 1)
.map_or(self.data.len(), |(byte, _)| byte),
};
return start..end;
}
self.char_index_to_byte(range.start)..self.char_index_to_byte(range.end)
}
/// The character index of the character starting at byte offset `bytepos`,
/// the inverse of [`Self::char_index_to_byte`].
///
/// `bytepos` must be a character boundary at or before the end.
///
/// Logarithmic rather than constant, because the index is keyed the other
/// way -- but a search whose bounds came from `char_index_to_byte` has the
/// table already, and this is what turns a byte offset back into the answer
/// a caller asked for in characters.
pub fn byte_to_char_index(&self, bytepos: usize) -> usize {
if self.kind.is_ascii() {
return bytepos;
}
let char_len = self.char_len();
self.index
.get_or_build(&self.data, char_len)
.char_index_at_byte(&self.data, bytepos, char_len)
}
pub fn nth_char(&self, index: usize) -> CodePoint {
match self.as_str_kind() {
PyKindStr::Ascii(s) => s[index].into(),
PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(),
PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(),
_ => self.data[self.char_index_to_byte_once(index)..]
.code_points()
.next()
.unwrap(),
}
}
}
@@ -416,20 +592,21 @@ pub fn codepoint_range_end(s: &Wtf8, n_chars: usize) -> Option<usize> {
}
#[must_use]
pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> {
/// Returns `None` for a width whose result cannot be allocated.
pub fn zfill(bytes: &[u8], width: usize) -> Option<Vec<u8>> {
if width <= bytes.len() {
bytes.to_vec()
} else {
let (sign, s) = match bytes.first() {
Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]),
_ => (&b""[..], bytes),
};
let mut filled = Vec::new();
filled.extend_from_slice(sign);
filled.extend(core::iter::repeat_n(b'0', width - bytes.len()));
filled.extend_from_slice(s);
filled
return Some(bytes.to_vec());
}
let (sign, s) = match bytes.first() {
Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]),
_ => (&b""[..], bytes),
};
let mut filled = Vec::new();
filled.try_reserve_exact(width).ok()?;
filled.extend_from_slice(sign);
filled.extend(core::iter::repeat_n(b'0', width - bytes.len()));
filled.extend_from_slice(s);
Some(filled)
}
/// Convert a string to ascii compatible, escaping unicode-s into escape
crates/common/src/wtf8_index.rs
+299
View File
@@ -0,0 +1,299 @@
// spell-checker:ignore rpython rlib rutf
//! Random access into a WTF-8 buffer.
//!
//! WTF-8 is variable width, so a buffer's n-th code point can only be found by
//! decoding the n-1 before it: [`Wtf8`]'s iterators are sequential, and
//! resolving an index through them is O(n). Code that indexes the same string
//! repeatedly -- a regex scan restarting at successive positions, say -- then
//! walks the whole buffer once per index, which is quadratic in its length.
//!
//! [`Wtf8Index`] is the side table that makes the lookup O(1): one 24-byte
//! group per 64 code points, so 0.375 bytes per code point. It is a cache, and
//! holds no state of its own beyond the buffer's shape -- building it twice for
//! the same buffer yields the same table.
//!
//! The layout is PyPy's `UTF8_INDEX_STORAGE` (`rpython/rlib/rutf8.py`).
use crate::wtf8::Wtf8;
/// One group of 64 code points.
#[derive(Clone, Copy)]
struct Group {
/// The byte offset the group's first code point starts at.
base: usize,
/// `ofs[i]` is the byte offset of the group's `4 * i + 1`-th code point,
/// relative to `base`. One entry covers four code points, so the widest
/// offset an entry has to hold is that of the 61st code point of a group,
/// at most `61 * 4 = 244` bytes in -- inside a `u8`, which is what buys the
/// table its density.
ofs: [u8; 16],
}
/// A code-point-index to byte-offset table for one WTF-8 buffer.
pub struct Wtf8Index {
groups: Box<[Group]>,
}
impl Wtf8Index {
/// Builds the table for `data`, whose code point count is `char_len`.
///
/// O(`data.len()`), and touches every byte, so it pays for itself only when
/// the caller goes on to index the buffer more than a couple of times.
#[must_use]
pub fn new(data: &Wtf8, char_len: usize) -> Self {
let mut groups = vec![
Group {
base: 0,
ofs: [0; 16],
};
char_len / 64 + 1
];
// Signed: the countdown overshoots the last group -- the loop stops on
// the first negative value rather than at a group boundary.
let mut remaining = char_len as isize;
let mut base = 0;
let mut current = 0;
loop {
groups[current].base = base;
let mut next = base;
let mut group_filled = true;
for i in 0..16 {
// Past the end, step as if one more single-byte code point
// followed, so the entry stays in range and is never read.
next = if remaining == 0 {
next + 1
} else {
next_pos(data, next)
};
groups[current].ofs[i] = (next - base) as u8;
remaining -= 4;
if remaining < 0 {
debug_assert_eq!(current + 1, groups.len());
group_filled = false;
break;
}
next = next_pos(data, next_pos(data, next_pos(data, next)));
}
if !group_filled {
break;
}
current += 1;
base = next;
}
Self {
groups: groups.into_boxed_slice(),
}
}
/// The byte offset of `data`'s `index`-th code point.
///
/// `data` must be the buffer the table was built for, and `index` must be
/// below its code point count.
#[inline]
#[must_use]
pub fn byte_offset(&self, data: &Wtf8, index: usize) -> usize {
let group = &self.groups[index >> 6];
// The entry sits on the 4k+1-th code point of the group, so a lookup is
// one table read plus at most two steps in either direction.
let pos = group.base + group.ofs[(index >> 2) & 0x0F] as usize;
match index & 0x3 {
0 => prev_pos(data, pos),
1 => pos,
2 => next_pos(data, pos),
_ => next_pos(data, next_pos(data, pos)),
}
}
/// The index of the code point starting at byte offset `bytepos`, the
/// inverse of [`Self::byte_offset`].
///
/// `data` must be the buffer the table was built for, `char_len` its code
/// point count, and `bytepos` a code point boundary at or before its end.
///
/// Logarithmic rather than constant: the table is keyed by code point
/// index, so going the other way is a search through it. The bracketing
/// below is what keeps that search short -- a code point occupies one to
/// four bytes, which pins the answer to a narrow band around `bytepos`
/// before the first comparison.
#[must_use]
pub fn char_index_at_byte(&self, data: &Wtf8, bytepos: usize, char_len: usize) -> usize {
let bytes_remaining = data.len() - bytepos;
// At least one byte per remaining code point, and at most four, so the
// group holding the answer lies between these.
let mut group_min =
usize::max(bytepos / 4, char_len.saturating_sub(bytes_remaining + 1)) >> 6;
let mut group_max = usize::min(bytepos, char_len.saturating_sub(bytes_remaining / 4)) >> 6;
while group_min < group_max {
let middle = group_min.midpoint(group_max) + 1;
if bytepos < self.groups[middle].base {
group_max = middle - 1;
} else {
group_min = middle;
}
}
let base = self.groups[group_min].base;
if base == bytepos {
return group_min << 6;
}
// Walk the group's entries to the last one at or before `bytepos`,
// then step the remaining code points, of which there are at most
// three -- an entry covers four.
let entries = if group_min == self.groups.len() - 1 {
((char_len - 1) >> 2) & 0x0F
} else {
16
};
let mut index = group_min << 6;
let mut pos = base;
for entry in 0..entries {
let at = base + self.groups[group_min].ofs[entry] as usize;
if at >= bytepos {
break;
}
pos = at;
index = (group_min << 6) + (entry << 2) + 1;
}
while pos < bytepos {
pos = next_pos(data, pos);
index += 1;
}
index
}
/// The table's heap footprint, in bytes.
#[must_use]
pub fn byte_size(&self) -> usize {
core::mem::size_of_val(&*self.groups)
}
}
/// The byte offset of the code point after the one at `pos`.
///
/// `data` must be well-formed WTF-8 and `pos` a code point boundary before its
/// end -- reading only the lead byte is what makes this branch-light.
#[inline]
fn next_pos(data: &Wtf8, pos: usize) -> usize {
match data.as_bytes()[pos] {
0x00..=0x7F => pos + 1,
0x80..=0xDF => pos + 2,
0xE0..=0xEF => pos + 3,
_ => pos + 4,
}
}
/// The byte offset of the code point before the one at `pos`, which must not be
/// zero.
///
/// A `pos` one past the end reads as the extra code point [`Wtf8Index::new`]
/// steps over there.
#[inline]
fn prev_pos(data: &Wtf8, pos: usize) -> usize {
let data = data.as_bytes();
let mut pos = pos - 1;
if pos >= data.len() || data[pos] <= 0x7F {
return pos;
}
pos -= 1;
if data[pos] >= 0xC0 {
return pos;
}
pos -= 1;
if data[pos] >= 0xC0 {
return pos;
}
pos - 1
}
#[cfg(test)]
mod tests {
use super::*;
use crate::wtf8::{CodePoint, Wtf8Buf};
/// Every index of `s`, both ways, against the offsets its own iterator
/// reports.
fn check(s: &Wtf8) {
let expected: Vec<usize> = s
.code_point_indices()
.map(|(byte_offset, _)| byte_offset)
.collect();
let char_len = expected.len();
let index = Wtf8Index::new(s, char_len);
for (i, &want) in expected.iter().enumerate() {
assert_eq!(
index.byte_offset(s, i),
want,
"index {i} of {s:?} ({char_len} code points)"
);
assert_eq!(
index.char_index_at_byte(s, want, char_len),
i,
"byte {want} of {s:?} ({char_len} code points)"
);
}
// One past the last code point is a boundary too, and the searches that
// use this ask for it as an end bound.
assert_eq!(
index.char_index_at_byte(s, s.len(), char_len),
char_len,
"end of {s:?}"
);
}
fn wtf8(s: &str) -> Wtf8Buf {
Wtf8Buf::from(s)
}
#[test]
fn empty() {
check(wtf8("").as_ref());
}
#[test]
fn widths() {
// One case per encoded width, and the boundaries between them.
check(wtf8("abc").as_ref());
check(wtf8("\u{80}\u{7ff}").as_ref());
check(wtf8("\u{800}\u{ffff}").as_ref());
check(wtf8("\u{10000}\u{10ffff}").as_ref());
check(wtf8("a\u{80}\u{800}\u{10000}").as_ref());
}
#[test]
fn group_boundaries() {
// A group covers 64 code points and an entry four, so the interesting
// lengths are the ones on and around both.
for len in [1, 3, 4, 5, 63, 64, 65, 127, 128, 129, 255, 256, 257] {
for unit in ["a", "\u{80}", "\u{800}", "\u{10000}"] {
check(wtf8(&unit.repeat(len)).as_ref());
}
// Mixed widths, so a group's entries do not share a stride.
check(wtf8(&"a\u{80}\u{800}\u{10000}".repeat(len)).as_ref());
}
}
#[test]
fn lone_surrogates() {
let mut s = wtf8("a");
for cp in [0xD800, 0xDBFF, 0xDC00, 0xDFFF] {
s.push(CodePoint::from_u32(cp).unwrap());
s.push_str("b");
}
check(s.as_ref());
// Surrogates only, spanning more than one group.
let mut s = wtf8("");
for i in 0..200 {
s.push(CodePoint::from_u32(0xD800 + (i % 0x400)).unwrap());
}
check(s.as_ref());
}
#[test]
fn byte_size_is_one_group_per_64_code_points() {
let s = wtf8(&"\u{10000}".repeat(200));
let index = Wtf8Index::new(s.as_ref(), 200);
assert_eq!(index.byte_size(), (200 / 64 + 1) * size_of::<Group>());
assert_eq!(size_of::<Group>(), 24);
}
}
crates/compiler-core/src/marshal.rs
+391 -55
View File
@@ -19,6 +19,14 @@ pub enum MarshalError {
InvalidLocation,
/// Bad type marker
BadType,
/// A type marker no reader knows
UnknownType,
/// A back reference that names nothing
InvalidRef,
/// A marker that stands for no object at all
NullObject,
/// A container length that is negative or does not fit, named by what it counts
BadSize(&'static str),
}
impl core::fmt::Display for MarshalError {
@@ -29,6 +37,10 @@ impl core::fmt::Display for MarshalError {
Self::InvalidUtf8 => f.write_str("invalid utf8"),
Self::InvalidLocation => f.write_str("invalid source location"),
Self::BadType => f.write_str("bad type marker"),
Self::UnknownType => f.write_str("unknown type code"),
Self::InvalidRef => f.write_str("invalid reference"),
Self::NullObject => f.write_str("NULL object in marshal data for object"),
Self::BadSize(what) => write!(f, "{what} size out of range"),
}
}
}
@@ -111,7 +123,7 @@ impl TryFrom<u8> for Type {
b'A' => Self::AsciiInterned,
b'z' => Self::ShortAscii,
b'Z' => Self::ShortAsciiInterned,
_ => return Err(MarshalError::BadType),
_ => return Err(MarshalError::UnknownType),
})
}
}
@@ -146,6 +158,13 @@ pub trait Read {
fn read_u64(&mut self) -> Result<u64> {
Ok(u64::from_le_bytes(*self.read_array()?))
}
/// A length, read the way `r_long` reads one: it is signed, so a value
/// with the top bit set is out of range rather than four billion items.
fn read_len(&mut self, what: &'static str) -> Result<usize> {
let len = self.read_u32()? as i32;
usize::try_from(len).map_err(|_| MarshalError::BadSize(what))
}
}
pub(crate) trait ReadBorrowed<'a>: Read {
@@ -305,7 +324,7 @@ fn reserve_ref_slot<T>(has_flag: bool, refs: &mut Vec<Option<T>>) -> Option<usiz
fn resolve_ref<T: Clone>(idx: usize, refs: &[Option<T>]) -> Result<T> {
refs.get(idx)
.and_then(|v| v.clone())
.ok_or(MarshalError::InvalidBytecode)
.ok_or(MarshalError::InvalidRef)
}
/// Read a marshal bytes object (TYPE_STRING = b's'), resolving TYPE_REF
@@ -408,7 +427,7 @@ fn read_marshal_str_vec<R: Read, Bag: ConstantBag>(
}
let n = match type_byte {
b'(' => rdr.read_u32()? as usize,
b'(' => rdr.read_len("tuple")?,
b')' => rdr.read_u8()? as usize,
_ => return Err(MarshalError::BadType),
};
@@ -471,7 +490,7 @@ fn read_marshal_const_tuple<R: Read, Bag: ConstantBag>(
}
let n = match type_byte {
b'(' => rdr.read_u32()? as usize,
b'(' => rdr.read_len("tuple")?,
b')' => rdr.read_u8()? as usize,
_ => return Err(MarshalError::BadType),
};
@@ -516,7 +535,7 @@ fn read_const_value<R: Read, Bag: ConstantBag>(
let code = deserialize_code_inner(rdr, bag, depth - 1, refs)?;
bag.make_code(code)
} else {
deserialize_value_typed(rdr, bag, depth, refs, typ)?
deserialize_value_typed(rdr, bag, depth, refs, typ, slot)?
};
if let Some(idx) = slot {
refs[idx] = Some(value.clone());
@@ -540,6 +559,10 @@ pub trait MarshalBag: Copy {
fn make_str(&self, value: &Wtf8) -> Self::Value;
fn make_interned_str(&self, value: &Wtf8) -> Self::Value {
self.make_str(value)
}
fn make_bytes(&self, value: &[u8]) -> Self::Value;
fn make_int(&self, value: BigInt) -> Self::Value;
@@ -549,7 +572,19 @@ pub trait MarshalBag: Copy {
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value;
) -> Result<Self::Value>;
/// Construct a runtime code object while retaining the exact values read
/// from ``co_consts``. Compiler bags ignore this second channel; runtime
/// bags use it for marshalable values (lists, dicts, sets, recursive
/// containers) that their compiler constant representation cannot hold.
fn make_code_with_constants(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
_constants: Vec<Self::Value>,
) -> Result<Self::Value> {
self.make_code(code)
}
fn make_stop_iter(&self) -> Result<Self::Value>;
@@ -564,6 +599,55 @@ pub trait MarshalBag: Copy {
it: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value>;
/// Install partially-built containers in the marshal reference table
/// before reading their children, as CPython's `r_object()` does.
/// Runtime bags can opt in; constant bags retain collect-then-construct.
///
/// `len` comes straight from the input and is only bounded by what a
/// length can hold, so a bag that opts in reports the room it cannot get
/// rather than taking it for granted.
fn make_tuple_placeholder(&self, _len: usize) -> Result<Option<Self::Value>> {
Ok(None)
}
fn set_tuple_item(
&self,
_tuple: &Self::Value,
_index: usize,
_value: Self::Value,
) -> Result<()> {
Err(MarshalError::BadType)
}
fn make_list_placeholder(&self, _len: usize) -> Result<Option<Self::Value>> {
Ok(None)
}
fn set_list_item(&self, _list: &Self::Value, _index: usize, _value: Self::Value) -> Result<()> {
Err(MarshalError::BadType)
}
fn make_set_placeholder(&self) -> Option<Self::Value> {
None
}
fn insert_set_item(&self, _set: &Self::Value, _value: Self::Value) -> Result<()> {
Err(MarshalError::BadType)
}
fn make_dict_placeholder(&self) -> Option<Self::Value> {
None
}
fn insert_dict_item(
&self,
_dict: &Self::Value,
_key: Self::Value,
_value: Self::Value,
) -> Result<()> {
Err(MarshalError::BadType)
}
fn make_slice(
&self,
_start: Self::Value,
@@ -581,6 +665,30 @@ pub trait MarshalBag: Copy {
) -> Option<<Self::ConstantBag as ConstantBag>::Constant> {
None
}
/// Convert a runtime constant to the compiler-side shape stored in
/// ``CodeObject``. Runtime implementations may return a semantically
/// unused placeholder when the exact value is carried by
/// `make_code_with_constants` instead.
fn code_constant_from_value(
&self,
value: &Self::Value,
) -> Result<<Self::ConstantBag as ConstantBag>::Constant> {
self.constant_ref_from_value(value)
.ok_or(MarshalError::BadType)
}
fn bytes_from_value(&self, _value: &Self::Value) -> Option<Vec<u8>> {
None
}
fn str_from_value(&self, _value: &Self::Value) -> Option<alloc::string::String> {
None
}
fn tuple_elements_from_value(&self, _value: &Self::Value) -> Option<Vec<Self::Value>> {
None
}
}
impl<Bag: ConstantBag> MarshalBag for Bag {
@@ -640,8 +748,8 @@ impl<Bag: ConstantBag> MarshalBag for Bag {
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value {
self.make_code(code)
) -> Result<Self::Value> {
Ok(self.make_code(code))
}
fn make_stop_iter(&self) -> Result<Self::Value> {
@@ -682,6 +790,27 @@ impl<Bag: ConstantBag> MarshalBag for Bag {
) -> Option<<Self::ConstantBag as ConstantBag>::Constant> {
Some(value.clone())
}
fn bytes_from_value(&self, value: &Self::Value) -> Option<Vec<u8>> {
match value.borrow_constant() {
BorrowedConstant::Bytes { value } => Some(value.to_vec()),
_ => None,
}
}
fn str_from_value(&self, value: &Self::Value) -> Option<alloc::string::String> {
match value.borrow_constant() {
BorrowedConstant::Str { value } => Some(value.to_string_lossy().into_owned()),
_ => None,
}
}
fn tuple_elements_from_value(&self, value: &Self::Value) -> Option<Vec<Self::Value>> {
match value.borrow_constant() {
BorrowedConstant::Tuple { elements } => Some(elements.to_vec()),
_ => None,
}
}
}
pub const MAX_MARSHAL_STACK_DEPTH: usize = 2000;
@@ -724,10 +853,7 @@ fn deserialize_value_after_header<R: Read, Bag: MarshalBag>(
// TYPE_REF: return previously stored object
if type_code == Type::Ref as u8 {
let idx = rdr.read_u32()? as usize;
return refs
.get(idx)
.and_then(|v| v.clone())
.ok_or(MarshalError::InvalidBytecode);
return resolve_ref(idx, refs);
}
// Reserve ref slot before reading (matches write order)
@@ -740,22 +866,10 @@ fn deserialize_value_after_header<R: Read, Bag: MarshalBag>(
};
let typ = Type::try_from(type_code)?;
// CPython's r_object() uses one global ref table: TYPE_CODE reserves its
// slot before reading code fields, and those fields may use later TYPE_REF
// indexes. Keep the same indexes even when Bag::Value and Constant differ.
let value = if matches!(typ, Type::Code) {
let mut inner_refs: Vec<Option<<Bag::ConstantBag as ConstantBag>::Constant>> = refs
.iter()
.map(|value| {
value
.as_ref()
.and_then(|value| bag.constant_ref_from_value(value))
})
.collect();
let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?;
bag.make_code(code)
deserialize_code_value_inner(rdr, bag, depth - 1, refs)?
} else {
deserialize_value_typed(rdr, bag, depth, refs, typ)?
deserialize_value_typed(rdr, bag, depth, refs, typ, slot)?
};
if let Some(idx) = slot {
@@ -764,12 +878,144 @@ fn deserialize_value_after_header<R: Read, Bag: MarshalBag>(
Ok(value)
}
/// Decode a code object through the runtime bag. CPython's marshal reader
/// keeps one reference table for the code fields and `co_consts`; using
/// `Bag::Value` here preserves that index space and lets runtime-only
/// constants survive alongside the compiler representation.
fn deserialize_code_value_inner<R: Read, Bag: MarshalBag>(
rdr: &mut R,
bag: Bag,
depth: usize,
refs: &mut Vec<Option<Bag::Value>>,
) -> Result<Bag::Value> {
if depth == 0 {
return Err(MarshalError::InvalidBytecode);
}
let arg_count = rdr.read_u32()?;
let posonlyarg_count = rdr.read_u32()?;
let kwonlyarg_count = rdr.read_u32()?;
let max_stackdepth = rdr.read_u32()?;
let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?);
let child_depth = depth - 1;
let code_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
let code_bytes = bag
.bytes_from_value(&code_value)
.ok_or(MarshalError::BadType)?;
let consts_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
let constant_values = bag
.tuple_elements_from_value(&consts_value)
.ok_or(MarshalError::BadType)?;
let constants = constant_values
.iter()
.map(|value| bag.code_constant_from_value(value))
.collect::<Result<Vec<_>>>()?
.into_iter()
.collect();
let read_strings =
|rdr: &mut R, refs: &mut Vec<Option<Bag::Value>>| -> Result<Vec<alloc::string::String>> {
let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?;
bag.tuple_elements_from_value(&tuple)
.ok_or(MarshalError::BadType)?
.iter()
.map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType))
.collect()
};
let names_raw = read_strings(rdr, refs)?;
let localsplusnames = read_strings(rdr, refs)?;
let kinds_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
let localspluskinds = bag
.bytes_from_value(&kinds_value)
.ok_or(MarshalError::BadType)?;
let read_string =
|rdr: &mut R, refs: &mut Vec<Option<Bag::Value>>| -> Result<alloc::string::String> {
let value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
bag.str_from_value(&value).ok_or(MarshalError::BadType)
};
let source_path_raw = read_string(rdr, refs)?;
let obj_name_raw = read_string(rdr, refs)?;
let qualname_raw = read_string(rdr, refs)?;
let first_line_raw = rdr.read_u32()? as i32;
let first_line_number = if first_line_raw > 0 {
OneIndexed::new(first_line_raw as usize)
} else {
None
};
let linetable_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
let linetable = bag
.bytes_from_value(&linetable_value)
.ok_or(MarshalError::BadType)?
.into_boxed_slice();
let exceptiontable_value = deserialize_value_depth(rdr, bag, child_depth, refs)?;
let exceptiontable = bag
.bytes_from_value(&exceptiontable_value)
.ok_or(MarshalError::BadType)?
.into_boxed_slice();
let lp = split_localplus(
&localsplusnames
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>(),
&localspluskinds,
arg_count,
kwonlyarg_count,
flags,
)?;
let instructions = CodeUnits::try_from(code_bytes.as_slice())?;
let locations = linetable_to_locations(&linetable, first_line_raw, instructions.len());
let constant_bag = bag.constant_bag();
let code = CodeObject {
instructions,
locations,
flags,
posonlyarg_count,
arg_count,
kwonlyarg_count,
source_path: constant_bag.make_name(&source_path_raw),
first_line_number,
max_stackdepth,
obj_name: constant_bag.make_name(&obj_name_raw),
qualname: constant_bag.make_name(&qualname_raw),
constants,
names: names_raw
.iter()
.map(|name| constant_bag.make_name(name))
.collect(),
varnames: lp
.varnames
.iter()
.map(|name| constant_bag.make_name(name))
.collect(),
cellvars: lp
.cellvars
.iter()
.map(|name| constant_bag.make_name(name))
.collect(),
freevars: lp
.freevars
.iter()
.map(|name| constant_bag.make_name(name))
.collect(),
localspluskinds: localspluskinds.into_boxed_slice(),
linetable,
exceptiontable,
};
bag.make_code_with_constants(code, constant_values)
}
fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
rdr: &mut R,
bag: Bag,
depth: usize,
refs: &mut Vec<Option<Bag::Value>>,
typ: Type,
slot: Option<usize>,
) -> Result<Bag::Value> {
if depth == 0 {
return Err(MarshalError::InvalidBytecode);
@@ -806,71 +1052,141 @@ fn deserialize_value_typed<R: Read, Bag: MarshalBag>(
let value = Complex64 { re, im };
bag.make_complex(value)
}
Type::Ascii | Type::AsciiInterned | Type::Unicode | Type::Interned => {
let len = rdr.read_u32()?;
let value = rdr.read_wtf8(len)?;
Type::Ascii | Type::Unicode => {
let len = rdr.read_len("string")?;
let value = rdr.read_wtf8(len as u32)?;
bag.make_str(value)
}
Type::ShortAscii | Type::ShortAsciiInterned => {
Type::AsciiInterned | Type::Interned => {
let len = rdr.read_len("string")?;
let value = rdr.read_wtf8(len as u32)?;
bag.make_interned_str(value)
}
Type::ShortAscii => {
let len = rdr.read_u8()? as u32;
let value = rdr.read_wtf8(len)?;
bag.make_str(value)
}
Type::ShortAsciiInterned => {
let len = rdr.read_u8()? as u32;
let value = rdr.read_wtf8(len)?;
bag.make_interned_str(value)
}
Type::SmallTuple => {
let len = rdr.read_u8()? as usize;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_tuple(it))?
if let Some(index) = slot
&& let Some(tuple) = bag.make_tuple_placeholder(len)?
{
refs[index] = Some(tuple.clone());
for item_index in 0..len {
let item = deserialize_value_depth(rdr, bag, d, refs)?;
bag.set_tuple_item(&tuple, item_index, item)?;
}
tuple
} else {
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_tuple(it))?
}
}
Type::Null => {
return Err(MarshalError::BadType);
return Err(MarshalError::NullObject);
}
Type::Ref => {
// Handled in deserialize_value_depth before calling this function
return Err(MarshalError::BadType);
}
Type::Tuple => {
let len = rdr.read_u32()?;
let len = rdr.read_len("tuple")?;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_tuple(it))?
if let Some(index) = slot
&& let Some(tuple) = bag.make_tuple_placeholder(len)?
{
refs[index] = Some(tuple.clone());
for item_index in 0..len {
let item = deserialize_value_depth(rdr, bag, d, refs)?;
bag.set_tuple_item(&tuple, item_index, item)?;
}
tuple
} else {
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_tuple(it))?
}
}
Type::List => {
let len = rdr.read_u32()?;
let len = rdr.read_len("list")?;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_list(it))??
if let Some(index) = slot
&& let Some(list) = bag.make_list_placeholder(len)?
{
refs[index] = Some(list.clone());
for item_index in 0..len {
let item = deserialize_value_depth(rdr, bag, d, refs)?;
bag.set_list_item(&list, item_index, item)?;
}
list
} else {
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_list(it))??
}
}
Type::Set => {
let len = rdr.read_u32()?;
let len = rdr.read_len("set")?;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_set(it))??
if let Some(index) = slot
&& let Some(set) = bag.make_set_placeholder()
{
refs[index] = Some(set.clone());
for _ in 0..len {
let item = deserialize_value_depth(rdr, bag, d, refs)?;
bag.insert_set_item(&set, item)?;
}
set
} else {
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_set(it))??
}
}
Type::FrozenSet => {
let len = rdr.read_u32()?;
let len = rdr.read_len("set")?;
let d = depth - 1;
let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs));
itertools::process_results(it, |it| bag.make_frozenset(it))??
}
Type::Dict => {
let d = depth - 1;
let mut pairs = Vec::new();
loop {
let raw = rdr.read_u8()?;
if raw & !FLAG_REF == b'0' {
break;
if let Some(index) = slot
&& let Some(dict) = bag.make_dict_placeholder()
{
refs[index] = Some(dict.clone());
loop {
let raw = rdr.read_u8()?;
if raw & !FLAG_REF == b'0' {
break;
}
let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?;
let value = deserialize_value_depth(rdr, bag, d, refs)?;
bag.insert_dict_item(&dict, key, value)?;
}
let k = deserialize_value_after_header(rdr, bag, d, refs, raw)?;
let v = deserialize_value_depth(rdr, bag, d, refs)?;
pairs.push((k, v));
dict
} else {
let mut pairs = Vec::new();
loop {
let raw = rdr.read_u8()?;
if raw & !FLAG_REF == b'0' {
break;
}
let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?;
let value = deserialize_value_depth(rdr, bag, d, refs)?;
pairs.push((key, value));
}
bag.make_dict(pairs.into_iter())?
}
bag.make_dict(pairs.into_iter())?
}
Type::Bytes => {
// After marshaling, byte arrays are converted into bytes.
let len = rdr.read_u32()?;
let value = rdr.read_slice(len)?;
let len = rdr.read_len("bytes object")?;
let value = rdr.read_slice(len as u32)?;
bag.make_bytes(value)
}
Type::Code => return Err(MarshalError::BadType),
@@ -1102,6 +1418,25 @@ pub fn serialize_value<W: Write, D: Dumpable>(
/// Split varnames/cellvars/freevars are reassembled into
/// co_localsplusnames/co_localspluskinds.
pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) {
serialize_code_with(buf, code, |buf, constant| {
serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {});
Ok::<(), core::convert::Infallible>(())
})
.unwrap_or_else(|x| match x {})
}
/// Serialize a code object, writing each `co_consts` entry through
/// `write_constant`.
///
/// A runtime caller passes its own object writer so that values its constant
/// representation carries but `BorrowedConstant` cannot describe — lists,
/// dicts, sets — reach the stream, and so a constant shared with the enclosing
/// object keeps its entry in that writer's reference table.
pub fn serialize_code_with<W: Write, C: Constant, E>(
buf: &mut W,
code: &CodeObject<C>,
mut write_constant: impl FnMut(&mut W, &C) -> core::result::Result<(), E>,
) -> core::result::Result<(), E> {
// 1–5: scalar fields
buf.write_u32(code.arg_count);
buf.write_u32(code.posonlyarg_count);
@@ -1118,7 +1453,7 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
buf.write_u8(Type::Tuple as u8);
write_len(buf, code.constants.len());
for constant in &*code.constants {
serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {})
write_constant(buf, constant)?;
}
// 8: co_names (tuple of strings)
@@ -1161,6 +1496,7 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
// 16: co_exceptiontable
buf.write_u8(Type::Bytes as u8);
write_vec(buf, &code.exceptiontable);
Ok(())
}
fn write_marshal_str<W: Write>(buf: &mut W, s: &str) {
crates/compiler/src/lib.rs
+116 -102
View File
@@ -1,4 +1,4 @@
pub use ruff_python_ast::token::TokenKind;
pub use ruff_python_ast::token::{TokenKind, Tokens};
use ruff_python_parser::ParseErrorType;
use ruff_source_file::{PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation};
use ruff_text_size::{Ranged, TextSize, TextSlice};
@@ -50,9 +50,13 @@ pub enum CompileError {
impl CompileError {
#[must_use]
pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self {
pub fn from_ruff_parse_error(
error: parser::ParseError,
source_file: &SourceFile,
mode: Mode,
) -> Self {
let raw_location = error.location;
let diagnostic = match cpython_parse_diagnostic_override(&error, source_file) {
let diagnostic = match cpython_parse_diagnostic_override(&error, source_file, mode) {
Some(diagnostic) => diagnostic,
None => default_parse_diagnostic(error, source_file),
};
@@ -129,6 +133,13 @@ fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation
.source_location(offset, PositionEncoding::Utf8)
}
// Call only with UTF-8 character boundaries for Python-facing offsets.
fn source_location_in_code_points(source_file: &SourceFile, offset: TextSize) -> SourceLocation {
source_file
.to_source_code()
.source_location(offset, PositionEncoding::Utf32)
}
fn source_locations(
source_file: &SourceFile,
start: TextSize,
@@ -175,6 +186,21 @@ impl NormalizedParseDiagnostic {
)
}
fn other_in_code_points(
source_file: &SourceFile,
message: String,
start: usize,
end: usize,
) -> Self {
let start = TextSize::new(start as u32);
let end = TextSize::new(end as u32);
Self::new(
parser::ParseErrorType::OtherError(message),
source_location_in_code_points(source_file, start),
source_location_in_code_points(source_file, end),
)
}
const fn with_unclosed_bracket(mut self, is_unclosed_bracket: bool) -> Self {
self.is_unclosed_bracket = is_unclosed_bracket;
self
@@ -184,6 +210,7 @@ impl NormalizedParseDiagnostic {
fn cpython_parse_diagnostic_override(
error: &parser::ParseError,
source_file: &SourceFile,
mode: Mode,
) -> Option<NormalizedParseDiagnostic> {
let source_text = source_file.source_text();
@@ -223,6 +250,18 @@ fn cpython_parse_diagnostic_override(
&error.error,
parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError)
) {
// Only a backslash at the end of the source is an EOF error.
let terminal_backslash = source_text.len().checked_sub(1);
if !matches!(mode, Mode::Eval)
&& terminal_backslash == Some(error.location.start().to_usize())
{
let loc = source_line_end_location(source_file, error.location.start());
return Some(NormalizedParseDiagnostic::new(
parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()),
loc,
loc,
));
}
let loc = source_location(source_file, error.location.start() + TextSize::from(1));
return Some(NormalizedParseDiagnostic::new(
error.error.clone(),
@@ -231,7 +270,15 @@ fn cpython_parse_diagnostic_override(
));
}
source_error!(unterminated_string_error(source_text));
if let Some((message, start, end)) = unterminated_string_error(source_text) {
// The scanner reports quote positions, which are UTF-8 character boundaries.
return Some(NormalizedParseDiagnostic::other_in_code_points(
source_file,
message,
start,
end,
));
}
source_error!(expected_indented_block_error(error, source_text));
if matches!(
@@ -4752,97 +4799,45 @@ fn invalid_legacy_statement_error(source: &str) -> Option<(String, usize, usize)
None
}
fn long_decimal_integer_literal_error(
source: &str,
/// Return the syntax error for a decimal integer literal exceeding the configured limit.
///
/// The parser has already distinguished integer literals from strings, comments, floats, and
/// complex numbers. Inspecting its tokens keeps the limit consistent for every source parsing
/// entry point without reimplementing Python's lexer here.
#[must_use]
pub fn long_decimal_integer_literal_error(
source_file: &SourceFile,
tokens: &Tokens,
max_str_digits: usize,
) -> Option<(String, usize, usize)> {
) -> Option<CompileError> {
if max_str_digits == 0 {
return None;
}
let bytes = source.as_bytes();
let mut index = 0;
while index < bytes.len() {
match bytes[index] {
b'#' => {
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
}
b'\'' | b'"' => {
index = skip_quoted_string(bytes, index);
}
byte if byte >= 0x80 || byte == b'_' || byte.is_ascii_alphabetic() => {
index += 1;
while index < bytes.len()
&& (bytes[index] >= 0x80 || is_ascii_identifier_char(bytes[index]))
{
index += 1;
}
}
b'.' => {
if bytes
.get(index + 1)
.is_some_and(|byte| byte.is_ascii_digit())
{
let (_, end) = number_literal_end(bytes, index)?;
index = end.max(index + 1);
} else {
index += 1;
}
}
b'0'..=b'9' => {
if bytes.get(index) == Some(&b'0')
&& matches!(
bytes.get(index + 1),
Some(b'x' | b'X' | b'o' | b'O' | b'b' | b'B')
)
{
let Some((_, end)) = number_literal_end(bytes, index) else {
index += 1;
continue;
};
index = end.max(index + 1);
continue;
}
let start = index;
let mut digits = 0usize;
while index < bytes.len() {
match bytes[index] {
b'0'..=b'9' => {
digits += 1;
index += 1;
}
b'_' if bytes
.get(index + 1)
.is_some_and(|byte| byte.is_ascii_digit()) =>
{
index += 1;
}
_ => break,
}
}
if matches!(bytes.get(index), Some(b'.' | b'e' | b'E' | b'j' | b'J')) {
let Some((_, end)) = number_literal_end(bytes, start) else {
continue;
};
index = end.max(index + 1);
continue;
}
if digits > max_str_digits {
return Some((
format!(
"Exceeds the limit ({max_str_digits} digits) for integer string conversion: value has {digits} digits; use sys.set_int_max_str_digits() to increase the limit - Consider hexadecimal for huge integer literals to avoid decimal conversion limits."
),
start,
start,
));
}
}
_ => index += 1,
tokens.iter().find_map(|token| {
if token.kind() != TokenKind::Int {
return None;
}
}
None
let literal = source_file.source_text().slice(token.range());
if literal
.as_bytes()
.get(..2)
.is_some_and(|prefix| matches!(prefix, b"0x" | b"0X" | b"0o" | b"0O" | b"0b" | b"0B"))
{
return None;
}
let digits = literal.bytes().filter(u8::is_ascii_digit).count();
(digits > max_str_digits).then(|| {
let start = token.range().start().to_usize();
CompileError::from_source_error(
source_file,
format!(
"Exceeds the limit ({max_str_digits} digits) for integer string conversion: value has {digits} digits; use sys.set_int_max_str_digits() to increase the limit - Consider hexadecimal for huge integer literals to avoid decimal conversion limits."
),
start,
start,
)
})
})
}
fn invalid_parenthesized_import_star_error(source: &str) -> Option<(String, usize, usize)> {
@@ -4951,12 +4946,27 @@ fn invalid_unparenthesized_yield_after_comma_error(source: &str) -> Option<(Stri
None
}
fn post_parse_source_error(source_file: &SourceFile, opts: &CompileOpts) -> Option<CompileError> {
too_many_nested_parentheses_error(source_file.source_text())
.or_else(|| {
long_decimal_integer_literal_error(source_file.source_text(), opts.int_max_str_digits)
})
.or_else(|| invalid_call_argument_error(source_file.source_text()))
fn post_parse_source_error(
source_file: &SourceFile,
tokens: &Tokens,
opts: &CompileOpts,
) -> Option<CompileError> {
if let Some((message, start, end)) =
too_many_nested_parentheses_error(source_file.source_text())
{
return Some(CompileError::from_source_error(
source_file,
message,
start,
end,
));
}
if let Some(error) =
long_decimal_integer_literal_error(source_file, tokens, opts.int_max_str_digits)
{
return Some(error);
}
invalid_call_argument_error(source_file.source_text())
.or_else(|| invalid_match_mapping_rest_wildcard_error(source_file.source_text()))
.or_else(|| invalid_match_as_target_error(source_file.source_text()))
.or_else(|| invalid_unparenthesized_yield_after_comma_error(source_file.source_text()))
@@ -5213,14 +5223,14 @@ fn _compile_with_syntax_warning_handler<'a>(
};
let parser_options = parser::ParseOptions::from(parser_mode);
let parsed = parser::parse(source_file.source_text(), parser_options)
.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?;
.map_err(|err| CompileError::from_ruff_parse_error(err, &source_file, mode))?;
if opts.dont_imply_dedent
&& matches!(mode, Mode::Single)
&& let Some(error) = dont_imply_dedent_source_error(&source_file)
{
return Err(error);
}
if let Some(error) = post_parse_source_error(&source_file, &opts) {
if let Some(error) = post_parse_source_error(&source_file, parsed.tokens(), &opts) {
return Err(error);
}
let ast = parsed.into_syntax();
@@ -5272,8 +5282,10 @@ pub fn _compile_symtable(
let res = match mode {
Mode::Exec | Mode::Single | Mode::BlockExpr => {
let ast = ruff_python_parser::parse_module(source_file.source_text())
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?;
if let Some(error) = post_parse_source_error(&source_file, &CompileOpts::default()) {
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?;
if let Some(error) =
post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default())
{
return Err(error);
}
let ast = ast.into_syntax();
@@ -5289,8 +5301,10 @@ pub fn _compile_symtable(
source_file.source_text(),
parser::Mode::Expression.into(),
)
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?;
if let Some(error) = post_parse_source_error(&source_file, &CompileOpts::default()) {
.map_err(|e| CompileError::from_ruff_parse_error(e, &source_file, mode))?;
if let Some(error) =
post_parse_source_error(&source_file, ast.tokens(), &CompileOpts::default())
{
return Err(error);
}
symboltable::SymbolTable::scan_expr(
crates/derive-impl/src/pyclass.rs
+6 -3
View File
@@ -1162,13 +1162,16 @@ where
let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span());
let slot_name = slot_ident.to_string();
let tokens = {
const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"];
const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"];
const STATIC_GEN_SLOTS: &[&str] = &["as_number"];
if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) {
if slot_name == "as_buffer" {
// bf_releasebuffer is not a separate function in RustPython; the
// exporter's BufferMethods already release. Only its presence is
// observable, and AsBuffer declares that.
quote_spanned! { span =>
slots.#slot_ident = Some(Self::#ident as _);
slots.#slot_ident.store(Some(Self::#ident as _));
slots.has_release_buffer.store(Self::RELEASE_BUFFER);
}
} else if POINTER_SLOTS.contains(&slot_name.as_str()) {
quote_spanned! { span =>
crates/host_env/src/fileutils.rs
+10 -20
View File
@@ -1,32 +1,25 @@
// Python/fileutils.c in CPython
#![allow(non_snake_case)]
use alloc::ffi::CString;
#[cfg(not(windows))]
pub use libc::stat as StatStruct;
pub use rustix::fs::Stat as StatStruct;
#[cfg(windows)]
pub use windows::{StatStruct, fstat};
#[cfg(not(windows))]
pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result<StatStruct> {
let mut stat = core::mem::MaybeUninit::uninit();
unsafe {
let ret = libc::fstat(fd.as_raw(), stat.as_mut_ptr());
if ret == -1 {
Err(crate::os::errno_io_error())
} else {
Ok(stat.assume_init())
}
}
rustix::fs::fstat(fd).map_err(Into::into)
}
#[cfg(windows)]
pub mod windows {
use crate::crt_fd;
use crate::windows::ToWideString;
use alloc::ffi::CString;
use libc::{S_IFCHR, S_IFDIR, S_IFMT};
use std::ffi::{OsStr, OsString};
use std::ffi::OsStr;
use std::os::windows::io::AsRawHandle;
use std::sync::OnceLock;
use windows_sys::Win32::Foundation::{
@@ -41,6 +34,7 @@ pub mod windows {
use windows_sys::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW};
use windows_sys::Win32::System::SystemServices::IO_REPARSE_TAG_SYMLINK;
use windows_sys::core::PCWSTR;
use windows_sys::w;
pub const S_IFIFO: libc::c_int = 0o010000;
pub const S_IFLNK: libc::c_int = 0o120000;
@@ -310,16 +304,13 @@ pub mod windows {
let GetFileInformationByName = GET_FILE_INFORMATION_BY_NAME
.get_or_init(|| {
let library_name =
OsString::from("api-ms-win-core-file-l2-1-4.dll").to_wide_with_nul();
let module = unsafe { LoadLibraryW(library_name.as_ptr()) };
let library_name = w!("api-ms-win-core-file-l2-1-4.dll");
let module = unsafe { LoadLibraryW(library_name) };
if module.is_null() {
return None;
}
let name = CString::new("GetFileInformationByName").unwrap();
if let Some(proc) =
unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) }
{
let name = c"GetFileInformationByName";
if let Some(proc) = unsafe { GetProcAddress(module, name.as_ptr().cast()) } {
Some(unsafe {
core::mem::transmute::<
unsafe extern "system" fn() -> isize,
@@ -466,7 +457,6 @@ pub unsafe fn fclose(fp: *mut CFile) -> core::ffi::c_int {
reason = "false positive: core::io::ErrorKind is unstable (core_io)"
)]
pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut CFile> {
use alloc::ffi::CString;
use std::fs::File;
// Currently only supports read mode
crates/host_env/src/io.rs
+28 -2
View File
@@ -2,6 +2,9 @@
use core::ffi::CStr;
use std::io;
#[cfg(any(unix, target_os = "wasi"))]
use rustix::{fs::FileType, io::Errno};
#[cfg(any(unix, target_os = "wasi"))]
use crate::fileutils;
use crate::{crt_fd, os};
@@ -148,8 +151,8 @@ pub struct FileTargetInfo {
#[cfg(any(unix, target_os = "wasi"))]
pub fn inspect_file_target(fd: crt_fd::Borrowed<'_>) -> io::Result<FileTargetInfo> {
let status = fileutils::fstat(fd)?;
if (status.st_mode & libc::S_IFMT) == libc::S_IFDIR {
return Err(io::Error::from_raw_os_error(libc::EISDIR));
if FileType::from_raw_mode(status.st_mode).is_dir() {
return Err(io::Error::from(Errno::ISDIR));
}
#[allow(clippy::useless_conversion, reason = "needed for 32-bit platforms")]
let blksize = (status.st_blksize > 1).then(|| i64::from(status.st_blksize));
@@ -196,6 +199,29 @@ pub fn is_seekable(fd: crt_fd::Borrowed<'_>) -> bool {
os::seek_fd(fd, 0, libc::SEEK_CUR).is_ok()
}
/// Whether a read from `fd` answers from data the file already holds, rather
/// than waiting for whoever writes the other end.
///
/// Seeking answers this everywhere but Windows, where a pipe seeks too --
/// `lseek` on one succeeds and reports a position, so a reader that took
/// seekability for an answer would wait on a peer while holding whatever it
/// holds for the length of the call.
#[cfg(not(windows))]
pub fn reads_without_waiting(fd: crt_fd::Borrowed<'_>) -> bool {
is_seekable(fd)
}
#[cfg(windows)]
pub fn reads_without_waiting(fd: crt_fd::Borrowed<'_>) -> bool {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Storage::FileSystem::{FILE_TYPE_DISK, GetFileType};
let Ok(handle) = crt_fd::as_handle(fd) else {
return false;
};
unsafe { GetFileType(handle.as_raw_handle() as _) == FILE_TYPE_DISK }
}
pub fn validate_whence(whence: i32) -> bool {
let standard = (0..=2).contains(&whence);
#[cfg(any(target_os = "dragonfly", target_os = "freebsd", target_os = "linux"))]
crates/host_env/src/io_unsupported.rs
+4
View File
@@ -176,6 +176,10 @@ pub fn is_seekable(_fd: crt_fd::Borrowed<'_>) -> bool {
false
}
pub fn reads_without_waiting(_fd: crt_fd::Borrowed<'_>) -> bool {
false
}
pub fn validate_whence(whence: i32) -> bool {
(0..=2).contains(&whence)
}
crates/host_env/src/nt.rs
+19 -18
View File
@@ -22,19 +22,22 @@ use crate::{
windows::{CheckWin32Bool, CheckWin32Handle, CheckWin32Sentinel, HandleToOwned, ToWideString},
};
use libc::intptr_t;
use windows_sys::Win32::{
Foundation::{
CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH,
use windows_sys::{
Win32::{
Foundation::{
CloseHandle, ERROR_INVALID_HANDLE, GetLastError, HANDLE, INVALID_HANDLE_VALUE, MAX_PATH,
},
Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte},
Storage::FileSystem::{
CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT,
FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW,
GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW,
INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle,
WIN32_FIND_DATAW,
},
System::{Console, Threading},
},
Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte},
Storage::FileSystem::{
CreateFileW, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT,
FILE_READ_ATTRIBUTES, FILE_TYPE_UNKNOWN, FileBasicInfo, FindClose, FindFirstFileW,
GetFileAttributesW, GetFileInformationByHandleEx, GetFileType, GetFullPathNameW,
INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, SetFileAttributesW, SetFileInformationByHandle,
WIN32_FIND_DATAW,
},
System::{Console, Threading},
w,
};
pub type Handle = HANDLE;
@@ -1172,12 +1175,10 @@ pub fn mkdir(path: &widestring::WideCStr, mode: i32) -> io::Result<()> {
lpSecurityDescriptor: core::ptr::null_mut(),
bInheritHandle: 0,
};
let sddl: Vec<u16> = "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)\0"
.encode_utf16()
.collect();
let sddl = w!("D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)");
unsafe {
ConvertStringSecurityDescriptorToSecurityDescriptorW(
sddl.as_ptr(),
sddl,
SDDL_REVISION_1,
&mut sec_attr.lpSecurityDescriptor,
core::ptr::null_mut(),
@@ -1699,10 +1700,10 @@ pub fn get_terminal_size_handle(h: HANDLE) -> io::Result<(usize, usize)> {
if err != windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED {
return Err(io::Error::last_os_error());
}
let conout: Vec<u16> = "CONOUT$\0".encode_utf16().collect();
let conout = w!("CONOUT$");
let console_handle = unsafe {
CreateFileW(
conout.as_ptr(),
conout,
windows_sys::Win32::Foundation::GENERIC_READ
| windows_sys::Win32::Foundation::GENERIC_WRITE,
windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ
crates/host_env/src/os.rs
+4 -3
View File
@@ -519,13 +519,14 @@ pub fn set_errno(value: i32) {
#[cfg(not(any(unix, windows, target_os = "wasi")))]
pub fn set_errno(_value: i32) {}
#[cfg(unix)]
// WASIp1, like Unix, provides byte-preserving OsStr conversions.
#[cfg(any(unix, all(target_os = "wasi", not(target_env = "p2"))))]
pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> {
use std::os::unix::ffi::OsStrExt;
use self::ffi::OsStrExt;
Ok(std::ffi::OsStr::from_bytes(b))
}
#[cfg(not(unix))]
#[cfg(not(any(unix, all(target_os = "wasi", not(target_env = "p2")))))]
pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> {
Ok(core::str::from_utf8(b)?.as_ref())
}
crates/host_env/src/posix.rs
+21 -51
View File
@@ -1,5 +1,4 @@
use alloc::ffi::CString;
#[cfg(all(unix, not(target_os = "redox")))]
use alloc::vec::Vec;
use core::ffi::CStr;
#[cfg(all(unix, not(target_os = "redox")))]
@@ -22,6 +21,12 @@ pub struct UnameInfo {
pub machine: String,
}
#[derive(Debug)]
pub struct UnameDecodeError {
pub bytes: Vec<u8>,
pub error: core::str::Utf8Error,
}
#[cfg(all(unix, not(target_os = "redox")))]
#[derive(Clone, Copy, Debug)]
pub struct StatVfsInfo {
@@ -304,50 +309,6 @@ pub fn fchown(fd: BorrowedFd<'_>, uid: Option<u32>, gid: Option<u32>) -> std::io
.map_err(std::io::Error::from)
}
#[cfg(not(windows))]
#[expect(
clippy::std_instead_of_core,
reason = "false positive: core::io::ErrorKind is unstable (core_io)"
)]
pub fn stat_path(
path: &OsStr,
dir_fd: Option<i32>,
follow_symlinks: bool,
) -> std::io::Result<Option<crate::fileutils::StatStruct>> {
use crate::os::ffi::OsStrExt;
let path = match CString::new(path.as_bytes()) {
Ok(path) => path,
Err(_) => return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput)),
};
let mut stat = core::mem::MaybeUninit::uninit();
#[cfg(not(target_os = "redox"))]
if let Some(dir_fd) = dir_fd {
let flags = if follow_symlinks {
0
} else {
libc::AT_SYMLINK_NOFOLLOW
};
let ret = unsafe { libc::fstatat(dir_fd, path.as_ptr(), stat.as_mut_ptr(), flags) };
if ret < 0 {
return Err(std::io::Error::last_os_error());
}
return Ok(Some(unsafe { stat.assume_init() }));
}
let ret = if follow_symlinks {
unsafe { libc::stat(path.as_ptr(), stat.as_mut_ptr()) }
} else {
unsafe { libc::lstat(path.as_ptr(), stat.as_mut_ptr()) }
};
if ret < 0 {
Err(std::io::Error::last_os_error())
} else {
Ok(Some(unsafe { stat.assume_init() }))
}
}
#[cfg(not(windows))]
pub fn stat_fd(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result<crate::fileutils::StatStruct> {
crate::fileutils::fstat(fd)
@@ -398,14 +359,23 @@ pub fn fchownat(
.map_err(std::io::Error::from)
}
pub fn uname_info() -> Result<UnameInfo, core::str::Utf8Error> {
pub fn uname_info() -> Result<UnameInfo, UnameDecodeError> {
fn decode(value: &CStr) -> Result<String, UnameDecodeError> {
core::str::from_utf8(value.to_bytes())
.map(str::to_owned)
.map_err(|error| UnameDecodeError {
bytes: value.to_bytes().to_vec(),
error,
})
}
let info = rustix::system::uname();
Ok(UnameInfo {
sysname: info.sysname().to_str()?.into(),
nodename: info.nodename().to_str()?.into(),
release: info.release().to_str()?.into(),
version: info.version().to_str()?.into(),
machine: info.machine().to_str()?.into(),
sysname: decode(info.sysname())?,
nodename: decode(info.nodename())?,
release: decode(info.release())?,
version: decode(info.version())?,
machine: decode(info.machine())?,
})
}
crates/host_env/src/posix_unix_like.rs
+21 -2
View File
@@ -2,11 +2,14 @@
use std::{io, path::Path};
use rustix::{fd::AsFd, fs};
use rustix::{
fd::AsFd,
fs::{self, AtFlags},
};
pub use rustix::fs::RawMode;
use crate::crt_fd;
use crate::{crt_fd, fileutils::StatStruct};
/// https://pubs.opengroup.org/onlinepubs/9799919799/functions/mkdir.html
pub fn make_dir(
@@ -45,3 +48,19 @@ pub fn replace(
) -> io::Result<()> {
rename(from, from_fd, to, to_fd)
}
pub fn stat_path(
path: impl AsRef<Path>,
dir_fd: Option<crt_fd::Borrowed<'_>>,
follow_symlinks: bool,
) -> io::Result<Option<StatStruct>> {
let flags = if follow_symlinks {
AtFlags::empty()
} else {
AtFlags::SYMLINK_NOFOLLOW
};
let dir_fd = dir_fd.as_ref().map_or(fs::CWD, AsFd::as_fd);
fs::statat(dir_fd, path.as_ref(), flags)
.map(Option::Some)
.map_err(Into::into)
}
crates/host_env/src/posix_wasi.rs
-33
View File
@@ -12,39 +12,6 @@ pub fn remove_dir_at(dir_fd: i32, path: &CStr) -> io::Result<()> {
Ok(())
}
pub fn stat_path(
path: &OsStr,
dir_fd: Option<i32>,
follow_symlinks: bool,
) -> io::Result<Option<crate::fileutils::StatStruct>> {
use crate::os::ffi::OsStrExt;
let path = match CString::new(path.as_bytes()) {
Ok(path) => path,
Err(_) => return Err(io::Error::from(io::ErrorKind::InvalidInput)),
};
let mut stat = core::mem::MaybeUninit::uninit();
if let Some(dir_fd) = dir_fd {
let flags = if follow_symlinks {
0
} else {
libc::AT_SYMLINK_NOFOLLOW
};
unsafe { libc::fstatat(dir_fd, path.as_ptr(), stat.as_mut_ptr(), flags) }
.check_libc_neg()?;
return Ok(Some(unsafe { stat.assume_init() }));
}
let ret = if follow_symlinks {
unsafe { libc::stat(path.as_ptr(), stat.as_mut_ptr()) }
} else {
unsafe { libc::lstat(path.as_ptr(), stat.as_mut_ptr()) }
};
ret.check_libc_neg()?;
Ok(Some(unsafe { stat.assume_init() }))
}
pub fn stat_fd(fd: crate::crt_fd::Borrowed<'_>) -> io::Result<crate::fileutils::StatStruct> {
crate::fileutils::fstat(fd)
}
crates/host_env/src/socket.rs
+8 -2
View File
@@ -44,9 +44,15 @@ pub use libc::{AF_ALG, AF_CAN};
#[cfg(target_os = "linux")]
pub use libc::{sockaddr_alg, sockaddr_can};
/// Set the system's hostname from its filesystem-encoded bytes.
///
/// `socketmodule.c socket_sethostname` reads the argument as a buffer and
/// passes `buf.buf`/`buf.len` straight to the syscall, so a name is not
/// required to be UTF-8; taking `&[u8]` keeps that true here as well.
#[cfg(all(unix, not(target_os = "redox")))]
pub fn sethostname(hostname: &str) -> io::Result<()> {
nix::unistd::sethostname(hostname).map_err(io::Error::from)
pub fn sethostname(hostname: &[u8]) -> io::Result<()> {
use std::os::unix::ffi::OsStrExt;
nix::unistd::sethostname(std::ffi::OsStr::from_bytes(hostname)).map_err(io::Error::from)
}
#[cfg(unix)]
crates/host_env/src/winapi.rs
+3 -2
View File
@@ -65,6 +65,7 @@ pub use windows_sys::Win32::{
},
UI::WindowsAndMessaging::SW_HIDE,
};
use windows_sys::w;
pub type Handle = HANDLE;
pub type StdHandle = windows_sys::Win32::System::Console::STD_HANDLE;
@@ -1093,14 +1094,14 @@ where
return Err(MimeRegistryReadError::Os(err));
}
let content_type_key: Vec<u16> = "Content Type\0".encode_utf16().collect();
let content_type_key = w!("Content Type");
let mut type_buf = [0u16; 256];
let mut cb_type = (type_buf.len() * 2) as u32;
let mut reg_type = 0;
let err = unsafe {
RegQueryValueExW(
subkey,
content_type_key.as_ptr(),
content_type_key,
core::ptr::null_mut(),
&mut reg_type,
type_buf.as_mut_ptr().cast(),
crates/host_env/src/windows.rs
+23 -20
View File
@@ -4,24 +4,27 @@ use std::{
io,
os::windows::ffi::{OsStrExt, OsStringExt},
};
use windows_sys::Win32::{
Foundation::{
E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS, ERROR_NO_UNICODE_TRANSLATION,
MAX_PATH, S_OK,
},
Networking::WinSock::WSAStartup,
Storage::FileSystem::{
GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW,
},
System::{
Diagnostics::Debug::{
FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM,
FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW,
use windows_sys::{
Win32::{
Foundation::{
E_POINTER, ERROR_INSUFFICIENT_BUFFER, ERROR_INVALID_FLAGS,
ERROR_NO_UNICODE_TRANSLATION, MAX_PATH, S_OK,
},
Networking::WinSock::WSAStartup,
Storage::FileSystem::{
GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW,
},
System::{
Diagnostics::Debug::{
FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM,
FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW,
},
LibraryLoader::{GetModuleFileNameW, GetModuleHandleW},
SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW},
Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee},
},
LibraryLoader::{GetModuleFileNameW, GetModuleHandleW},
SystemInformation::{GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW},
Threading::{GetCurrentThreadStackLimits, SetThreadStackGuarantee},
},
w,
};
/// _MAX_ENV from Windows CRT stdlib.h - maximum environment variable size
@@ -154,8 +157,8 @@ pub struct WindowsVersionInfo {
fn get_kernel32_version() -> io::Result<(u32, u32, u32)> {
unsafe {
let module_name: Vec<u16> = OsStr::new("kernel32.dll").to_wide_with_nul();
let h_kernel32 = GetModuleHandleW(module_name.as_ptr()).check_nonnull()?;
let module_name = w!("kernel32.dll");
let h_kernel32 = GetModuleHandleW(module_name).check_nonnull()?;
let mut kernel32_path = [0u16; MAX_PATH as usize];
let len = GetModuleFileNameW(
@@ -181,13 +184,13 @@ fn get_kernel32_version() -> io::Result<(u32, u32, u32)> {
)
.check_win32_bool()?;
let sub_block: Vec<u16> = OsStr::new("").to_wide_with_nul();
let sub_block = w!("");
let mut ffi_ptr: *mut VS_FIXEDFILEINFO = core::ptr::null_mut();
let mut ffi_len: u32 = 0;
VerQueryValueW(
ver_block.as_ptr() as *const _,
sub_block.as_ptr(),
sub_block,
&mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _,
&mut ffi_len as *mut u32,
)
crates/host_env/src/wmi.rs
+12 -12
View File
@@ -7,15 +7,17 @@
use core::ffi::c_void;
use core::ptr::{NonNull, null, null_mut};
use widestring::WideCString;
use windows_sys::Win32::Foundation::{
CloseHandle, ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, HANDLE,
WAIT_OBJECT_0, WAIT_TIMEOUT,
CloseHandle, ERROR_BROKEN_PIPE, ERROR_INVALID_NAME, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY,
GetLastError, HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT,
};
use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile};
use windows_sys::Win32::System::Pipes::CreatePipe;
use windows_sys::Win32::System::Threading::{
CreateEventW, CreateThread, GetExitCodeThread, SetEvent, WaitForSingleObject,
};
use windows_sys::w;
use crate::ctypes::wcslen;
@@ -256,10 +258,6 @@ const fn failed(hr: HRESULT) -> bool {
hr < 0
}
fn wide_str(s: &str) -> Vec<u16> {
s.encode_utf16().chain(core::iter::once(0)).collect()
}
unsafe fn wait_event(event: HANDLE, timeout: u32) -> u32 {
match unsafe { WaitForSingleObject(event, timeout) } {
WAIT_OBJECT_0 => 0,
@@ -346,8 +344,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 {
}
if succeeded(hr) {
let root_cimv2 = wide_str("ROOT\\CIMV2");
let bstr_root = unsafe { SysAllocString(root_cimv2.as_ptr()) };
let root_cimv2 = w!("ROOT\\CIMV2");
let bstr_root = unsafe { SysAllocString(root_cimv2) };
hr = unsafe {
locator_connect_server(
locator,
@@ -384,8 +382,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 {
};
}
if succeeded(hr) {
let wql = wide_str("WQL");
let bstr_wql = unsafe { SysAllocString(wql.as_ptr()) };
let wql = w!("WQL");
let bstr_wql = unsafe { SysAllocString(wql) };
hr = unsafe {
services_exec_query(
services,
@@ -557,7 +555,9 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 {
}
pub fn exec_query(query_str: &str) -> Result<String, ExecQueryError> {
let query_wide = wide_str(query_str);
let query = WideCString::from_str(query_str)
.map_err(|_| ExecQueryError::Code(ERROR_INVALID_NAME))?
.into();
let mut h_thread: HANDLE = null_mut();
let mut err: u32 = 0;
@@ -579,7 +579,7 @@ pub fn exec_query(query_str: &str) -> Result<String, ExecQueryError> {
err = GetLastError();
} else {
let thread_data = Box::new(QueryThreadData {
query: query_wide,
query,
write_pipe,
init_event,
connect_event,
crates/jit/src/instructions.rs
+37 -20
View File
@@ -39,6 +39,7 @@ impl JitValue {
JitType::Int => Self::Int(val),
JitType::Float => Self::Float(val),
JitType::Bool => Self::Bool(val),
JitType::None => unreachable!("None cannot be used as an argument type"),
}
}
@@ -47,7 +48,8 @@ impl JitValue {
Self::Int(_) => Some(JitType::Int),
Self::Float(_) => Some(JitType::Float),
Self::Bool(_) => Some(JitType::Bool),
Self::None | Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None,
Self::None => Some(JitType::None),
Self::Null | Self::Tuple(_) | Self::FuncRef(_) => None,
}
}
@@ -112,8 +114,9 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
#[expect(clippy::mut_mut, reason = "This seems like a false positive")]
let builder = &mut self.builder;
let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?;
let cranelift_ty = ty.to_cranelift().ok_or(JitCompileError::NotSupported)?;
let local = self.variables[idx].get_or_insert_with(|| {
let var = builder.declare_var(ty.to_cranelift());
let var = builder.declare_var(cranelift_ty);
Local {
var,
ty: ty.clone(),
@@ -328,27 +331,27 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
}
fn return_value(&mut self, val: JitValue) -> Result<(), JitCompileError> {
if let Some(ref ty) = self.sig.ret {
// If the signature has a return type, enforce it
if val.to_jit_type().as_ref() != Some(ty) {
let val_type = val.to_jit_type().ok_or(JitCompileError::NotSupported)?;
if let Some(ref ret_type) = self.sig.ret {
if ret_type != &val_type {
return Err(JitCompileError::NotSupported);
}
} else {
// First time we see a return, define it in the signature
let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?;
self.sig.ret = Some(ty.clone());
self.builder
.func
.signature
.returns
.push(AbiParam::new(ty.to_cranelift()));
self.sig.ret = Some(val_type.clone());
if let Some(val_type) = val_type.to_cranelift() {
self.builder
.func
.signature
.returns
.push(AbiParam::new(val_type));
}
}
// If this is e.g. an Int, Float, or Bool we have a Cranelift `Value`.
// If we have JitValue::None or .Tuple(...) but can't handle that, error out (or handle differently).
let cr_val = val.into_value().ok_or(JitCompileError::NotSupported)?;
self.builder.ins().return_(&[cr_val]);
if let Some(cr_val) = val.into_value() {
self.builder.ins().return_(&[cr_val]);
} else {
self.builder.ins().return_(&[]);
}
Ok(())
}
@@ -545,8 +548,22 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> {
match self.stack.pop().ok_or(JitCompileError::BadBytecode)? {
JitValue::FuncRef(reference) => {
let call = self.builder.ins().call(reference, &args);
let returns = self.builder.inst_results(call);
self.stack.push(JitValue::Int(returns[0]));
// The only callable reachable here is this function itself,
// so the result carries the declared return type - it is not
// always an Int. A function whose return type is still
// unknown has no return slot in the signature it was
// declared with, and there is nothing to type the result as.
let ret = match *self.builder.inst_results(call) {
[] => None,
[val] => Some(val),
_ => return Err(JitCompileError::NotSupported),
};
let val = match (self.sig.ret.clone(), ret) {
(Some(JitType::None), None) => JitValue::None,
(Some(ty), Some(val)) => JitValue::from_type_and_value(ty, val),
_ => return Err(JitCompileError::NotSupported),
};
self.stack.push(val);
Ok(())
}
crates/jit/src/lib.rs
+16 -16
View File
@@ -61,19 +61,12 @@ impl Jit {
ret: Option<JitType>,
) -> Result<(FuncId, JitSig), JitCompileError> {
for arg in args {
self.ctx
.func
.signature
.params
.push(AbiParam::new(arg.to_cranelift()));
let arg = arg.to_cranelift().ok_or(JitCompileError::NotSupported)?;
self.ctx.func.signature.params.push(AbiParam::new(arg));
}
if ret.is_some() {
self.ctx
.func
.signature
.returns
.push(AbiParam::new(ret.clone().unwrap().to_cranelift()));
if let Some(ret) = ret.as_ref().and_then(JitType::to_cranelift) {
self.ctx.func.signature.returns.push(AbiParam::new(ret));
}
let id = self.module.declare_function(
@@ -167,7 +160,10 @@ impl CompiledCode {
libffi::middle::CodePtr::from_ptr(self.code as *const _),
cif_args,
);
self.sig.ret.as_ref().map(|ty| value.to_typed(ty))
match self.sig.ret.as_ref() {
Some(JitType::None) | None => None,
Some(ty) => Some(value.to_typed(ty)),
}
}
}
}
@@ -193,14 +189,16 @@ pub enum JitType {
Int,
Float,
Bool,
None,
}
impl JitType {
fn to_cranelift(&self) -> types::Type {
fn to_cranelift(&self) -> Option<types::Type> {
match self {
Self::Int => types::I64,
Self::Float => types::F64,
Self::Bool => types::I8,
Self::Int => Some(types::I64),
Self::Float => Some(types::F64),
Self::Bool => Some(types::I8),
Self::None => None,
}
}
@@ -209,6 +207,7 @@ impl JitType {
Self::Int => libffi::middle::Type::i64(),
Self::Float => libffi::middle::Type::f64(),
Self::Bool => libffi::middle::Type::u8(),
Self::None => libffi::middle::Type::void(),
}
}
}
@@ -306,6 +305,7 @@ impl UnTypedAbiValue {
JitType::Int => AbiValue::Int(self.int),
JitType::Float => AbiValue::Float(self.float),
JitType::Bool => AbiValue::Bool(self.boolean != 0),
JitType::None => unreachable!("None has no ABI value"),
}
}
}
crates/jit/tests/bool_tests.rs
+14
View File
@@ -202,4 +202,18 @@ mod tests {
assert_eq!(lte(false, 1), Ok(1));
assert_eq!(lte(true, 0), Ok(0));
}
#[test]
fn recursive_bool() {
let recursive_bool = jit_function! { recursive_bool(n: i64) -> bool => r##"
def recursive_bool(n: int) -> bool:
if n == 0:
return True
return not recursive_bool(n - 1)
"## };
assert_eq!(recursive_bool(0), Ok(true));
assert_eq!(recursive_bool(1), Ok(false));
assert_eq!(recursive_bool(4), Ok(true));
}
}
crates/jit/tests/float_tests.rs
+14
View File
@@ -379,4 +379,18 @@ mod tests {
assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false));
assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false));
}
#[test]
fn recursive_float() {
let recursive_float = jit_function! { recursive_float(n: i64) -> f64 => r##"
def recursive_float(n: int) -> float:
if n == 0:
return 1.0
return recursive_float(n - 1) / 2.0
"## };
assert_eq!(recursive_float(0), Ok(1.0));
assert_eq!(recursive_float(1), Ok(0.5));
assert_eq!(recursive_float(4), Ok(0.0625));
}
}
crates/jit/tests/misc_tests.rs
+9 -10
View File
@@ -2,16 +2,15 @@
mod tests {
use rustpython_jit::{AbiValue, JitArgumentError};
// TODO currently broken
// #[test]
// fn test_no_return_value() {
// let func = jit_function! { func() => r##"
// def func():
// pass
// "## };
//
// assert_eq!(func(), Ok(()));
// }
#[test]
fn no_return_value() {
let func = jit_function! { func() => r##"
def func():
pass
"## };
assert_eq!(func(), Ok(()));
}
#[test]
fn invoke() {
crates/sre_engine/src/engine.rs
+10 -6
View File
@@ -581,7 +581,11 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
..ctx
};
for _ in group_start..group_end {
// Walk the group itself rather than counting to its
// width: `g_ctx` is already stepping over exactly
// the characters being compared, so its own cursor
// is the loop bound.
while g_ctx.cursor.position < group_end {
#[allow(clippy::redundant_closure_call)]
if ctx.at_end(req)
|| $f(ctx.peek_char::<S>()) != $f(g_ctx.peek_char::<S>())
@@ -998,12 +1002,12 @@ fn search_info_literal<const LITERAL: bool, S: StrDrive>(
return true;
}
// `state.cursor` is `req.start + skip`, the position the
// tail match resumes from; advancing past the prefix
// instead would resume at `req.start + len` and only
// agree when the prefix ends at the skip boundary.
let mut next_ctx = ctx;
if skip != 0 {
next_ctx.advance_char::<S>();
} else {
next_ctx.cursor = state.cursor;
}
next_ctx.cursor = state.cursor;
if _match(req, state, next_ctx) {
return true;
crates/sre_engine/src/string.rs
+35
View File
@@ -1,5 +1,12 @@
use rustpython_wtf8::Wtf8;
/// A position in the subject, paired with the byte pointer it resolves to.
///
/// `position` is a **character index**, never a byte offset. The engine does
/// arithmetic on it directly — it subtracts two positions to get a character
/// count, adds a repeat count to get a bound, and compares one against a
/// lookbehind width — so the unit is part of the [`StrDrive`] contract rather
/// than a detail each implementation may pick.
#[derive(Debug, Clone, Copy)]
pub struct StringCursor {
pub(crate) ptr: *const u8,
@@ -15,15 +22,43 @@ impl Default for StringCursor {
}
}
/// Random access over the subject being matched.
///
/// An implementation chooses how a character is spelled in memory — one byte
/// for `&[u8]`, one code point for `&str` and `&Wtf8` — but **not** how
/// positions are counted. Every position this trait produces or consumes is a
/// character index: `count` is the subject's length in characters, and
/// `skip(n)` advances a cursor's `position` by exactly `n`.
///
/// That is load-bearing, not incidental. The engine reads position arithmetic
/// as character arithmetic in several places — `_count` bounds a repeat with
/// `position + max_count` and reports the repeat's length as a difference of
/// positions, `ASSERT` tests `position < back` against a lookbehind width, and
/// `search_info` recovers a match start as `position - (len - 1)`. A drive
/// that stored byte offsets here would leave all of those type-correct and
/// silently wrong, and would index a lookbehind out of bounds.
///
/// So a drive over a variable-width encoding pays for the mapping: `count`
/// and `create_cursor` have to resolve character indices, and cannot simply
/// hand back byte lengths and byte offsets.
pub trait StrDrive: Copy {
/// The subject's length, in characters.
fn count(&self) -> usize;
/// A cursor at character index `n`.
fn create_cursor(&self, n: usize) -> StringCursor;
/// Move `cursor` to character index `n`, from wherever it is now.
fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize);
/// Consume one character, returning it; `position` grows by one.
fn advance(cursor: &mut StringCursor) -> u32;
/// The character at `cursor`, without moving it.
fn peek(cursor: &StringCursor) -> u32;
/// Skip `n` characters, so `position` grows by exactly `n`.
fn skip(cursor: &mut StringCursor, n: usize);
/// Step back over one character, returning it; `position` shrinks by one.
fn back_advance(cursor: &mut StringCursor) -> u32;
/// The character before `cursor`, without moving it.
fn back_peek(cursor: &StringCursor) -> u32;
/// Step back `n` characters, so `position` shrinks by exactly `n`.
fn back_skip(cursor: &mut StringCursor, n: usize);
}
crates/sre_engine/tests/tests.rs
+15
View File
@@ -252,4 +252,19 @@ mod tests {
#[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] };
// END GENERATED
}
#[test]
fn search_literal_prefix_longer_than_skip() {
// The INFO block carries prefix_len=4 and prefix_skip=2, so the tail
// match has to resume at the skip boundary rather than past the whole
// prefix.
// pattern p = re.compile('ab(cd)')
// START GENERATED by generate_tests.py
#[rustfmt::skip] let p = Pattern { pattern: "ab(cd)", code: &[14, 14, 1, 4, 4, 4, 2, 97, 98, 99, 100, 0, 0, 0, 0, 16, 97, 16, 98, 17, 0, 16, 99, 16, 100, 17, 1, 1] };
// END GENERATED
let (req, mut state) = p.state("xabcdcd");
assert!(state.search(req));
assert_eq!(state.start, 1);
assert_eq!(state.cursor.position, 5);
}
}
crates/stdlib/Cargo.toml
+1 -1
View File
@@ -86,10 +86,10 @@ libz-rs-sys = { workspace = true }
bzip2 = { workspace = true }
# tkinter
jiff = { workspace = true }
tk-sys = { workspace = true, optional = true }
tcl-sys = { workspace = true, optional = true }
widestring = { workspace = true, optional = true }
chrono.workspace = true
# uuid
[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies]
crates/stdlib/src/_asyncio.rs
+88 -59
View File
@@ -12,8 +12,8 @@ pub(crate) mod _asyncio {
vm::{
AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{
PyBaseException, PyBaseExceptionRef, PyDict, PyDictRef, PyGenericAlias, PyList,
PyListRef, PyModule, PySet, PyTuple, PyType, PyTypeRef,
PyBaseException, PyBaseExceptionRef, PyDict, PyGenericAlias, PyList, PyListRef,
PyModule, PySet, PyTuple, PyType, PyTypeRef,
},
extend_module,
function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue},
@@ -724,47 +724,56 @@ pub(crate) mod _asyncio {
/// Add waiter to fut_awaited_by with single-object optimization
fn awaited_by_add(&self, waiter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
let mut awaited_by = self.fut_awaited_by.write();
if awaited_by.is_none() {
// First waiter - store directly
*awaited_by = Some(waiter);
return Ok(());
}
// Storing a waiter in the set runs its __hash__ and __eq__, which can
// come back to this future, so the field is locked only while it is
// read or written.
let existing = {
let mut awaited_by = self.fut_awaited_by.write();
match awaited_by.as_ref() {
// First waiter - store directly
None => {
*awaited_by = Some(waiter);
return Ok(());
}
Some(existing) => existing.clone(),
}
};
if self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Already a Set - add to it
let set = awaited_by.as_ref().unwrap();
vm.call_method(set, "add", (waiter,))?;
} else {
// Single object - convert to Set
let existing = awaited_by.take().unwrap();
let new_set = PySet::default().into_ref(&vm.ctx);
new_set.add(existing, vm)?;
new_set.add(waiter, vm)?;
*awaited_by = Some(new_set.into());
self.fut_awaited_by_is_set.store(true, Ordering::Relaxed);
return vm.call_method(&existing, "add", (waiter,)).map(drop);
}
// Single object - convert to Set
let new_set = PySet::default().into_ref(&vm.ctx);
new_set.add(existing, vm)?;
new_set.add(waiter, vm)?;
*self.fut_awaited_by.write() = Some(new_set.into());
self.fut_awaited_by_is_set.store(true, Ordering::Relaxed);
Ok(())
}
/// Discard waiter from fut_awaited_by with single-object optimization
fn awaited_by_discard(&self, waiter: &PyObject, vm: &VirtualMachine) -> PyResult<()> {
let mut awaited_by = self.fut_awaited_by.write();
if awaited_by.is_none() {
return Ok(());
}
let obj = awaited_by.as_ref().unwrap();
if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Single object - check if it matches
if obj.is(waiter) {
*awaited_by = None;
// As in awaited_by_add, discarding from the set runs Python.
let set = {
let mut awaited_by = self.fut_awaited_by.write();
let Some(obj) = awaited_by.as_ref() else {
return Ok(());
};
if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) {
// Single object - check if it matches
if obj.is(waiter) {
*awaited_by = None;
}
return Ok(());
}
} else {
// It's a Set - use discard
vm.call_method(obj, "discard", (waiter.to_owned(),))?;
}
Ok(())
obj.clone()
};
// It's a Set - use discard
vm.call_method(&set, "discard", (waiter.to_owned(),))
.map(drop)
}
#[pymethod]
@@ -779,7 +788,7 @@ pub(crate) mod _asyncio {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
}
@@ -1036,7 +1045,7 @@ pub(crate) mod _asyncio {
)));
}
let exc = if exc_type.fast_isinstance(vm.ctx.types.type_type) {
let exc: PyBaseExceptionRef = if exc_type.fast_isinstance(vm.ctx.types.type_type) {
// exc_type is a class
let exc_class: PyTypeRef = exc_type.clone().downcast().unwrap();
// Must be a subclass of BaseException
@@ -1047,12 +1056,23 @@ pub(crate) mod _asyncio {
}
let val = exc_val.unwrap_or_none(vm);
if vm.is_none(&val) {
let exc = if vm.is_none(&val) {
exc_type.call((), vm)?
} else if val.fast_isinstance(&exc_class) {
val
} else {
exc_type.call((val,), vm)?
};
match exc.downcast() {
Ok(exc) => exc,
Err(obj) => {
let exc_class_repr = exc_class.as_object().repr(vm)?;
vm.new_type_error(format!(
"calling {} should have returned an instance of BaseException, not {}",
exc_class_repr.as_wtf8(),
obj.class()
))
}
}
} else if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) {
// exc_type is an exception instance
@@ -1063,7 +1083,7 @@ pub(crate) mod _asyncio {
vm.new_type_error("instance exception may not have a separate value")
);
}
exc_type
exc_type.downcast().unwrap()
} else {
// exc_type is neither a class nor an exception instance
return Err(vm.new_type_error(format!(
@@ -1075,10 +1095,11 @@ pub(crate) mod _asyncio {
if let OptionalArg::Present(tb) = exc_tb
&& !vm.is_none(&tb)
{
exc.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?;
exc.as_object()
.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?;
}
Err(exc.downcast().unwrap())
Err(exc)
}
#[pymethod]
@@ -1840,7 +1861,7 @@ pub(crate) mod _asyncio {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
}
@@ -2405,7 +2426,9 @@ pub(crate) mod _asyncio {
// Slow path: look up in the module-level dict for cross-thread queries
let current_tasks = get_current_tasks_dict(vm)?;
let dict: PyDictRef = current_tasks.downcast().unwrap();
let Ok(dict) = current_tasks.downcast::<PyDict>() else {
return Ok(vm.ctx.none());
};
match dict.get_item(&*loop_obj, vm) {
Ok(task) => Ok(task),
@@ -2485,15 +2508,17 @@ pub(crate) mod _asyncio {
#[pyfunction]
fn _enter_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
// Per-thread check, matching CPython's ts->asyncio_running_task
{
let running_task = vm.asyncio_running_task.borrow();
if running_task.is_some() {
return Err(vm.new_runtime_error(format!(
"Cannot enter into task {:?} while another task {:?} is being executed.",
task,
running_task.as_ref().unwrap()
)));
}
let running_task = vm.asyncio_running_task.borrow().clone();
if let Some(running_task) = running_task {
let task_repr = task.repr(vm)?;
let running_task_repr = running_task.repr(vm)?;
return Err(vm.new_runtime_error(wtf8_concat!(
"Cannot enter into task ",
task_repr.as_wtf8(),
" while another task ",
running_task_repr.as_wtf8(),
" is being executed."
)));
}
*vm.asyncio_running_task.borrow_mut() = Some(task.clone());
@@ -2729,16 +2754,20 @@ pub(crate) mod _asyncio {
}
}
fn get_invalid_state_error_type(vm: &VirtualMachine) -> PyResult<PyTypeRef> {
let module = vm.import("asyncio.exceptions", 0)?;
let exc_type = vm
.get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError"))?
.ok_or_else(|| vm.new_attribute_error("InvalidStateError not found"))?;
exc_type
.downcast()
.map_err(|_| vm.new_type_error("InvalidStateError is not a type"))
}
fn new_invalid_state_error(vm: &VirtualMachine, msg: &str) -> PyBaseExceptionRef {
match vm.import("asyncio.exceptions", 0) {
Ok(module) => {
match vm.get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError")) {
Ok(Some(exc_type)) => match exc_type.call((msg,), vm) {
Ok(exc) => exc.downcast().unwrap(),
Err(_) => vm.new_runtime_error(msg.to_string()),
},
_ => vm.new_runtime_error(msg.to_string()),
}
match get_invalid_state_error_type(vm) {
Ok(invalid_state_error) => {
vm.new_exception_msg(invalid_state_error, msg.to_string().into())
}
Err(_) => vm.new_runtime_error(msg.to_string()),
}
crates/stdlib/src/_queue.rs
+25 -10
View File
@@ -74,9 +74,20 @@ mod _queue {
}
}
fn release(&self) {
/// Take `mutex`, detaching first so that blocking on it cannot stall a
/// stop-the-world request.
///
/// A waiter holds this mutex across its `allow_threads` wait, so it can
/// still hold it when it is stopped. An attached thread blocking on it
/// would then never reach a safepoint, the stop would never complete,
/// and the holder would never be resumed to release it.
fn lock_count(&self, vm: &VirtualMachine) -> parking_lot::MutexGuard<'_, usize> {
vm.allow_threads(|| self.mutex.lock())
}
fn release(&self, vm: &VirtualMachine) {
{
let mut count = self.mutex.lock();
let mut count = self.lock_count(vm);
*count += 1;
} // lock dropped. now we can notify a waiting thread
@@ -95,7 +106,7 @@ mod _queue {
// Guard must be dropped before check_signals() below, since a
// signal handler may call back into this same queue.
{
let mut count = self.mutex.lock();
let mut count = self.lock_count(vm);
if *count > 0 {
*count -= 1;
@@ -151,11 +162,15 @@ mod _queue {
}
impl PySimpleQueue {
fn push(&self, item: PyObjectRef) {
#[cfg_attr(
not(feature = "threading"),
expect(unused_variables, reason = "only the semaphore needs the vm")
)]
fn push(&self, item: PyObjectRef, vm: &VirtualMachine) {
self.buf.lock().push_back(item);
#[cfg(feature = "threading")]
self.sem.release();
self.sem.release(vm);
}
/// Returns a strong reference from the head of the buffer.
@@ -221,14 +236,14 @@ mod _queue {
}
#[pymethod]
fn put(&self, args: PutArgs) {
fn put(&self, args: PutArgs, vm: &VirtualMachine) {
let PutArgs { item, .. } = args;
self.push(item);
self.push(item, vm);
}
#[pymethod]
fn put_nowait(&self, item: PyObjectRef) {
self.push(item);
fn put_nowait(&self, item: PyObjectRef, vm: &VirtualMachine) {
self.push(item, vm);
}
#[pymethod]
@@ -282,7 +297,7 @@ mod _queue {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
}
crates/stdlib/src/_sqlite3.rs
+159 -16
View File
@@ -31,11 +31,11 @@ mod _sqlite3 {
sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text,
sqlite3_column_type, sqlite3_complete, sqlite3_context, sqlite3_context_db_handle,
sqlite3_create_collation_v2, sqlite3_create_function_v2, sqlite3_create_window_function,
sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec,
sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit,
sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit,
sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset,
sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error,
sqlite3_data_count, sqlite3_db_config, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg,
sqlite3_exec, sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize,
sqlite3_get_autocommit, sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion,
sqlite3_limit, sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler,
sqlite3_reset, sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error,
sqlite3_result_error_nomem, sqlite3_result_error_toobig, sqlite3_result_int64,
sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep,
sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe,
@@ -161,7 +161,17 @@ mod _sqlite3 {
SQLITE_ALTER_TABLE, SQLITE_ANALYZE, SQLITE_ATTACH, SQLITE_CREATE_INDEX,
SQLITE_CREATE_TABLE, SQLITE_CREATE_TEMP_INDEX, SQLITE_CREATE_TEMP_TABLE,
SQLITE_CREATE_TEMP_TRIGGER, SQLITE_CREATE_TEMP_VIEW, SQLITE_CREATE_TRIGGER,
SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH,
SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DBCONFIG_DEFENSIVE,
SQLITE_DBCONFIG_DQS_DDL, SQLITE_DBCONFIG_DQS_DML, SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE,
SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, SQLITE_DBCONFIG_ENABLE_COMMENTS,
SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,
SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_DBCONFIG_ENABLE_QPSG,
SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_DBCONFIG_ENABLE_VIEW,
SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_DBCONFIG_LEGACY_FILE_FORMAT,
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_DBCONFIG_RESET_DATABASE,
SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_DBCONFIG_STMT_SCANSTATUS,
SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_DBCONFIG_TRUSTED_SCHEMA,
SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH,
SQLITE_DROP_INDEX, SQLITE_DROP_TABLE, SQLITE_DROP_TEMP_INDEX, SQLITE_DROP_TEMP_TABLE,
SQLITE_DROP_TEMP_TRIGGER, SQLITE_DROP_TEMP_VIEW, SQLITE_DROP_TRIGGER, SQLITE_DROP_VIEW,
SQLITE_DROP_VTABLE, SQLITE_FUNCTION, SQLITE_IGNORE, SQLITE_INSERT, SQLITE_LIMIT_ATTACHED,
@@ -322,7 +332,7 @@ mod _sqlite3 {
)))
}
} else {
Err(vm.new_type_error(format!(
Err(vm.new_value_error(format!(
"autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {}",
obj.class().name()
)))
@@ -587,10 +597,10 @@ mod _sqlite3 {
) -> c_int {
let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() };
let f = || -> PyResult<c_int> {
let arg1 = ptr_to_str(arg1, vm)?;
let arg2 = ptr_to_str(arg2, vm)?;
let db_name = ptr_to_str(db_name, vm)?;
let access = ptr_to_str(access, vm)?;
let arg1 = ptr_to_str_or_none(arg1, vm)?;
let arg2 = ptr_to_str_or_none(arg2, vm)?;
let db_name = ptr_to_str_or_none(db_name, vm)?;
let access = ptr_to_str_or_none(access, vm)?;
let val = callable.call((action, arg1, arg2, db_name, access), vm)?;
let Some(val) = val.downcast_ref::<PyInt>() else {
@@ -1279,6 +1289,7 @@ mod _sqlite3 {
SQLITE_UTF8
};
let db = self.db_lock(vm)?;
check_num_params(&db, args.narg, "narg", vm)?;
let Some(data) = CallbackData::new(args.func, vm) else {
return db.create_function(
name.as_ptr(),
@@ -1310,6 +1321,7 @@ mod _sqlite3 {
fn create_aggregate(&self, args: CreateAggregateArgs, vm: &VirtualMachine) -> PyResult<()> {
let name = args.name.to_cstring(vm)?;
let db = self.db_lock(vm)?;
check_num_params(&db, args.narg, "n_arg", vm)?;
let Some(data) = CallbackData::new(args.aggregate_class, vm) else {
return db.create_function(
name.as_ptr(),
@@ -1392,6 +1404,7 @@ mod _sqlite3 {
) -> PyResult<()> {
let name = name.to_cstring(vm)?;
let db = self.db_lock(vm)?;
check_num_params(&db, narg, "num_params", vm)?;
let Some(data) = CallbackData::new(aggregate_class, vm) else {
unsafe {
sqlite3_create_window_function(
@@ -1516,6 +1529,39 @@ mod _sqlite3 {
self.db_lock(vm)?.limit(category, limit, vm)
}
#[pymethod]
fn setconfig(
&self,
op: c_int,
enable: OptionalArg<bool>,
vm: &VirtualMachine,
) -> PyResult<()> {
let db = self.db_lock(vm)?;
if !is_int_dbconfig(op) {
return Err(vm.new_value_error(format!("unknown config 'op': {op}")));
}
let enable = enable.unwrap_or(true) as c_int;
let mut actual: c_int = 0;
let rc = unsafe { sqlite3_db_config(db.db, op, enable, &mut actual) };
db.check(rc, vm)?;
if enable != actual {
return Err(new_operational_error(vm, "Unable to set config".to_owned()));
}
Ok(())
}
#[pymethod]
fn getconfig(&self, op: c_int, vm: &VirtualMachine) -> PyResult<bool> {
let db = self.db_lock(vm)?;
if !is_int_dbconfig(op) {
return Err(vm.new_value_error(format!("unknown config 'op': {op}")));
}
let mut current: c_int = 0;
let rc = unsafe { sqlite3_db_config(db.db, op, -1, &mut current) };
db.check(rc, vm)?;
Ok(current != 0)
}
#[pymethod]
fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> {
zelf
@@ -1642,6 +1688,47 @@ mod _sqlite3 {
fn total_changes(&self, vm: &VirtualMachine) -> PyResult<c_int> {
self._db_lock(vm).map(|x| x.total_changes())
}
#[pygetset(name = "Warning")]
fn exc_warning(&self) -> PyTypeRef {
warning_type().to_owned()
}
#[pygetset(name = "Error")]
fn exc_error(&self) -> PyTypeRef {
error_type().to_owned()
}
#[pygetset(name = "InterfaceError")]
fn exc_interface_error(&self) -> PyTypeRef {
interface_error_type().to_owned()
}
#[pygetset(name = "DatabaseError")]
fn exc_database_error(&self) -> PyTypeRef {
database_error_type().to_owned()
}
#[pygetset(name = "DataError")]
fn exc_data_error(&self) -> PyTypeRef {
data_error_type().to_owned()
}
#[pygetset(name = "OperationalError")]
fn exc_operational_error(&self) -> PyTypeRef {
operational_error_type().to_owned()
}
#[pygetset(name = "IntegrityError")]
fn exc_integrity_error(&self) -> PyTypeRef {
integrity_error_type().to_owned()
}
#[pygetset(name = "InternalError")]
fn exc_internal_error(&self) -> PyTypeRef {
internal_error_type().to_owned()
}
#[pygetset(name = "ProgrammingError")]
fn exc_programming_error(&self) -> PyTypeRef {
programming_error_type().to_owned()
}
#[pygetset(name = "NotSupportedError")]
fn exc_not_supported_error(&self) -> PyTypeRef {
not_supported_error_type().to_owned()
}
}
#[pyattr]
@@ -2798,12 +2885,14 @@ mod _sqlite3 {
}
let sql_cstr = sql.to_cstring(vm)?;
let db = connection.db_lock(vm)?;
db.sql_limit(sql.byte_len(), vm)?;
let raw = {
let db = connection.db_lock(vm)?;
db.sql_limit(sql.byte_len(), vm)?;
**db
};
let mut tail = null();
let st = db.prepare(sql_cstr.as_ptr(), &mut tail, vm)?;
let st = raw.prepare(sql_cstr.as_ptr(), &mut tail, vm)?;
let Some(st) = st else {
return Ok(None);
@@ -3475,12 +3564,66 @@ mod _sqlite3 {
Ok(obj)
}
fn check_num_params(
db: &Sqlite,
n: c_int,
param_name: &str,
vm: &VirtualMachine,
) -> PyResult<()> {
let limit = unsafe { sqlite3_limit(db.db, SQLITE_LIMIT_FUNCTION_ARG, -1) };
if n < -1 || n > limit {
return Err(new_programming_error(
vm,
format!("'{param_name}' must be between -1 and {limit}, not {n}"),
));
}
Ok(())
}
fn is_int_dbconfig(op: c_int) -> bool {
use libsqlite3_sys::*;
matches!(
op,
SQLITE_DBCONFIG_ENABLE_FKEY
| SQLITE_DBCONFIG_ENABLE_TRIGGER
| SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
| SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
| SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
| SQLITE_DBCONFIG_ENABLE_QPSG
| SQLITE_DBCONFIG_TRIGGER_EQP
| SQLITE_DBCONFIG_RESET_DATABASE
| SQLITE_DBCONFIG_DEFENSIVE
| SQLITE_DBCONFIG_WRITABLE_SCHEMA
| SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
| SQLITE_DBCONFIG_DQS_DDL
| SQLITE_DBCONFIG_DQS_DML
| SQLITE_DBCONFIG_ENABLE_VIEW
| SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
| SQLITE_DBCONFIG_TRUSTED_SCHEMA
| SQLITE_DBCONFIG_STMT_SCANSTATUS
| SQLITE_DBCONFIG_REVERSE_SCANORDER
| SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE
| SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE
| SQLITE_DBCONFIG_ENABLE_COMMENTS
)
}
fn ptr_to_str<'a>(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<&'a str> {
if p.is_null() {
return Err(vm.new_memory_error("string pointer is null"));
}
unsafe { CStr::from_ptr(p).to_str() }
.map_err(|_| vm.new_value_error("Invalid UIF-8 codepoint"))
.map_err(|_| vm.new_value_error("Invalid UTF-8 codepoint"))
}
fn ptr_to_str_or_none(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
if p.is_null() {
return Ok(vm.ctx.none());
}
let s = unsafe { CStr::from_ptr(p) }
.to_str()
.map_err(|_| vm.new_value_error("Invalid UTF-8 codepoint".to_owned()))?;
Ok(vm.ctx.new_str(s).into())
}
fn ptr_to_string(
crates/stdlib/src/array.rs
+118 -31
View File
@@ -27,8 +27,8 @@ pub mod array {
ArgBytesLike, ArgIntoFloat, ArgIterable, KwArgs, OptionalArg, PyComparisonValue,
},
protocol::{
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods, PySequenceMethods,
BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer,
PyIterReturn, PyMappingMethods, PySequenceMethods,
},
sequence::{OptionalRangeArgs, SequenceExt, SequenceMutExt},
sliceable::{
@@ -55,6 +55,11 @@ pub mod array {
$($n(Vec<$t>),)*
}
/// One item, already converted to the array's element type.
enum ArrayItem {
$($n($t),)*
}
impl ArrayContentType {
fn from_char(c: char) -> Result<Self, String> {
match c {
@@ -303,17 +308,31 @@ pub mod array {
}
}
fn setitem_by_index(
&mut self,
i: isize,
/// Convert an object to the element type of the array with
/// this typecode. This runs the object's conversion methods,
/// which can reach the array, so it takes the typecode by
/// value and holds no lock on it.
fn item_from_object(
typecode: char,
value: PyObjectRef,
vm: &VirtualMachine
) -> PyResult<ArrayItem> {
match typecode {
$($c => Ok(ArrayItem::$n(<$t>::try_into_from_object(vm, value)?)),)*
_ => unreachable!("array has a typecode"),
}
}
fn setitem_by_item(
&mut self,
i: isize,
item: ArrayItem,
vm: &VirtualMachine
) -> PyResult<()> {
match self {
$(ArrayContentType::$n(v) => {
let value = <$t>::try_into_from_object(vm, value)?;
v.setitem_by_index(vm, i, value)
})*
match (self, item) {
$((ArrayContentType::$n(v), ArrayItem::$n(value)) =>
v.setitem_by_index(vm, i, value),)*
_ => unreachable!("item was converted for this array"),
}
}
@@ -638,7 +657,7 @@ pub mod array {
impl ToPyResult for WideChar {
fn to_pyresult(self, vm: &VirtualMachine) -> PyResult {
Ok(CodePoint::try_from(self)
.map_err(|e| vm.new_unicode_encode_error(e))?
.map_err(|e| vm.new_value_error(e))?
.to_pyobject(vm))
}
}
@@ -732,12 +751,12 @@ pub mod array {
}
} else if init.downcastable::<PyBytes>() || init.downcastable::<PyByteArray>() {
init.try_bytes_like(vm, |x| array.frombytes(x))?;
} else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) {
} else {
// Everything else is taken item by item, buffer or not.
let iter = ArgIterable::try_from_object(vm, init)?;
for obj in iter.iter(vm)? {
array.push(obj?, vm)?;
}
} else {
init.try_bytes_like(vm, |x| array.frombytes(x))?;
}
}
@@ -906,6 +925,11 @@ pub mod array {
#[pymethod]
fn frombytes(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> {
// The source is read as bytes, so items of any other width would
// be reinterpreted rather than appended.
if b.itemsize() != 1 {
return Err(vm.new_type_error("a bytes-like object is required"));
}
let b = b.borrow_buf();
let itemsize = self.read().itemsize();
self._from_bytes(&b, itemsize, vm)
@@ -1047,7 +1071,11 @@ pub mod array {
vm: &VirtualMachine,
) -> PyResult<()> {
match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? {
SequenceIndex::Int(i) => zelf.write().setitem_by_index(i, value, vm),
SequenceIndex::Int(i) => {
let typecode = zelf.read().typecode();
let item = ArrayContentType::item_from_object(typecode, value, vm)?;
zelf.write().setitem_by_item(i, item, vm)
}
SequenceIndex::Slice(slice) => {
let cloned;
let guard;
@@ -1234,7 +1262,7 @@ pub mod array {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
}
@@ -1291,20 +1319,42 @@ pub mod array {
}
}
impl PyArray {
fn buffer_desc(&self) -> BufferDescriptor {
let array = self.read();
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
)
}
}
impl AsBuffer for PyArray {
const RELEASE_BUFFER: bool = true;
// array_buffer_getbuf, which reports the type code only when the request
// asked for a format.
fn slot_as_buffer(
zelf: &PyObject,
flags: BufferFlags,
vm: &VirtualMachine,
) -> PyResult<PyBuffer> {
let zelf = zelf
.downcast_ref::<Self>()
.ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?;
let desc = zelf.buffer_desc().projected(flags);
flags.check_writable(desc.readonly, "Object is not writable.", vm)?;
Ok(PyBuffer::new(zelf.to_owned().into(), desc, &BUFFER_METHODS))
}
fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let array = zelf.read();
let buf = PyBuffer::new(
Ok(PyBuffer::new(
zelf.to_owned().into(),
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
),
zelf.buffer_desc(),
&BUFFER_METHODS,
);
Ok(buf)
))
}
}
@@ -1386,7 +1436,9 @@ pub mod array {
ass_item: atomic_func!(|seq, i, value, vm| {
let zelf = PyArray::sequence_downcast(seq);
if let Some(value) = value {
zelf.write().setitem_by_index(i, value, vm)
let typecode = zelf.read().typecode();
let item = ArrayContentType::item_from_object(typecode, value, vm)?;
zelf.write().setitem_by_item(i, item, vm)
} else {
zelf.write().delitem_by_index(i, vm)
}
@@ -1421,8 +1473,15 @@ pub mod array {
type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>;
fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> {
let w = self.write();
(self.exports.load(atomic::Ordering::SeqCst) == 0).then_some(w)
// An export is a borrow someone else still holds, so it is
// answered before the lock rather than by waiting on it.
(self.exports.load(atomic::Ordering::SeqCst) == 0).then(|| self.write())
}
fn try_resizable(&self, vm: &VirtualMachine) -> PyResult<Self::Resizable<'_>> {
self.try_resizable_opt().ok_or_else(|| {
vm.new_buffer_error("cannot resize an array that is exporting buffers")
})
}
}
@@ -1696,8 +1755,17 @@ pub mod array {
})?,
MachineFormatCode::Utf16 { big_endian } => {
let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect();
let s = String::from_utf16(&utf16)
.map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?;
let s = String::from_utf16(&utf16).map_err(|_| {
let (index, reason) = invalid_utf16(&utf16).unwrap();
vm.new_unicode_decode_error(
vm.ctx
.new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }),
args.items.clone(),
index * 2,
index * 2 + 2,
vm.ctx.new_str(reason),
)
})?;
let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize());
array.frombytes_move(bytes);
}
@@ -1713,6 +1781,25 @@ pub mod array {
PyArray::from(array).into_ref_with_type(vm, cls)
}
fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> {
let mut index = 0;
while index < units.len() {
let unit = units[index];
if (0xd800..=0xdbff).contains(&unit) {
match units.get(index + 1) {
Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2,
Some(_) => return Some((index, "illegal UTF-16 surrogate")),
None => return Some((index, "unexpected end of data")),
}
} else if (0xdc00..=0xdfff).contains(&unit) {
return Some((index, "illegal encoding"));
} else {
index += 1;
}
}
None
}
// Register array.array as collections.abc.MutableSequence
pub(crate) fn module_exec(
vm: &VirtualMachine,
crates/stdlib/src/blake2.rs
+8 -1
View File
@@ -5,7 +5,7 @@ pub(crate) use _blake2::module_def;
#[pymodule]
mod _blake2 {
use crate::hashlib::_hashlib::{BlakeHashArgs, local_blake2b, local_blake2s};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyattr(name = "_GIL_MINSIZE")]
const GIL_MINSIZE: u16 = 2048;
@@ -43,4 +43,11 @@ mod _blake2 {
fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_blake2s(args, vm)?.into_pyobject(vm))
}
#[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")]
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
crates/stdlib/src/contextvars.rs
+79 -74
View File
@@ -15,16 +15,16 @@ mod _contextvars {
AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func,
builtins::{PyGenericAlias, PyList, PyStrRef, PyType, PyTypeRef},
class::StaticType,
common::{hash::PyHash, lock::LazyLock, wtf8::Wtf8Buf},
common::{
hash::PyHash,
lock::{LazyLock, PyMutex},
wtf8::Wtf8Buf,
},
function::{ArgCallable, FuncArgs, OptionalArg},
protocol::{PyMappingMethods, PySequenceMethods},
types::{AsMapping, AsSequence, Constructor, Hashable, Iterable, Representable},
};
use core::{
cell::{Cell, RefCell, UnsafeCell},
sync::atomic::Ordering,
};
use crossbeam_utils::atomic::AtomicCell;
use core::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
use indexmap::IndexMap;
// TODO: Real hamt implementation
@@ -33,7 +33,7 @@ mod _contextvars {
#[pyclass(no_attr, name = "Hamt", module = "contextvars")]
#[derive(Debug, PyPayload)]
pub(crate) struct HamtObject {
hamt: RefCell<Hamt>,
hamt: PyMutex<Hamt>,
}
#[pyclass]
@@ -42,23 +42,19 @@ mod _contextvars {
impl Default for HamtObject {
fn default() -> Self {
Self {
hamt: RefCell::new(Hamt::default()),
hamt: PyMutex::new(Hamt::default()),
}
}
}
unsafe impl Sync for HamtObject {}
#[derive(Debug)]
struct ContextInner {
idx: Cell<usize>,
idx: AtomicUsize,
vars: PyRef<HamtObject>,
// PyObject *ctx_weakreflist;
entered: Cell<bool>,
entered: AtomicBool,
}
unsafe impl Sync for ContextInner {}
#[pyattr]
#[pyclass(name = "Context")]
#[derive(Debug, PyPayload)]
@@ -71,23 +67,30 @@ mod _contextvars {
fn empty(vm: &VirtualMachine) -> Self {
Self {
inner: ContextInner {
idx: Cell::new(usize::MAX),
idx: AtomicUsize::new(usize::MAX),
vars: HamtObject::default().into_ref(&vm.ctx),
entered: Cell::new(false),
entered: AtomicBool::new(false),
},
}
}
fn borrow_vars(&self) -> impl core::ops::Deref<Target = Hamt> + '_ {
self.inner.vars.hamt.borrow()
fn borrow_vars(&self) -> impl core::ops::DerefMut<Target = Hamt> + '_ {
self.inner.vars.hamt.lock()
}
fn borrow_vars_mut(&self) -> impl core::ops::DerefMut<Target = Hamt> + '_ {
self.inner.vars.hamt.borrow_mut()
self.inner.vars.hamt.lock()
}
fn enter(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> {
if zelf.inner.entered.get() {
// A context is entered by one thread at a time, so the check and the
// claim have to be a single step.
if zelf
.inner
.entered
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return Err(vm.new_runtime_error(format!(
"cannot enter context: {} is already entered",
zelf.as_object().repr(vm)?
@@ -95,16 +98,15 @@ mod _contextvars {
}
super::CONTEXTS.with_borrow_mut(|ctxs| {
zelf.inner.idx.set(ctxs.len());
zelf.inner.idx.store(ctxs.len(), Ordering::Relaxed);
ctxs.push(zelf.to_owned());
});
zelf.inner.entered.set(true);
Ok(())
}
fn exit(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> {
if !zelf.inner.entered.get() {
if !zelf.inner.entered.load(Ordering::Acquire) {
return Err(vm.new_runtime_error(format!(
"cannot exit context: {} is not entered",
zelf.as_object().repr(vm)?
@@ -120,7 +122,7 @@ mod _contextvars {
)
})
})?;
zelf.inner.entered.set(false);
zelf.inner.entered.store(false, Ordering::Release);
Ok(())
}
@@ -131,8 +133,8 @@ mod _contextvars {
ctx.clone()
} else {
let ctx = Self::empty(vm);
ctx.inner.idx.set(0);
ctx.inner.entered.set(true);
ctx.inner.idx.store(0, Ordering::Relaxed);
ctx.inner.entered.store(true, Ordering::Release);
let ctx = ctx.into_ref(&vm.ctx);
ctxs.push(ctx);
ctxs[0].clone()
@@ -170,13 +172,13 @@ mod _contextvars {
fn copy(&self, vm: &VirtualMachine) -> Self {
// Deep copy the vars - clone the underlying Hamt data, not just the PyRef
let vars_copy = HamtObject {
hamt: RefCell::new(self.inner.vars.hamt.borrow().clone()),
hamt: PyMutex::new(self.inner.vars.hamt.lock().clone()),
};
Self {
inner: ContextInner {
idx: Cell::new(usize::MAX),
idx: AtomicUsize::new(usize::MAX),
vars: vars_copy.into_ref(&vm.ctx),
entered: Cell::new(false),
entered: AtomicBool::new(false),
},
}
}
@@ -186,11 +188,8 @@ mod _contextvars {
var: PyRef<ContextVar>,
vm: &VirtualMachine,
) -> PyResult<PyObjectRef> {
let vars = self.borrow_vars();
let item = vars
.get(&*var)
.ok_or_else(|| vm.new_key_error(var.into()))?;
Ok(item.to_owned())
let item = self.borrow_vars().get(&*var).map(|item| item.to_owned());
item.ok_or_else(|| vm.new_key_error(var.into()))
}
fn __len__(&self) -> usize {
@@ -290,11 +289,11 @@ mod _contextvars {
name: String,
default: Option<PyObjectRef>,
#[pytraverse(skip)]
cached: AtomicCell<Option<ContextVarCache>>,
cached: PyMutex<Option<ContextVarCache>>,
#[pytraverse(skip)]
cached_id: core::sync::atomic::AtomicUsize, // cached_tsid in CPython
cached_id: AtomicUsize, // cached_tsid in CPython
#[pytraverse(skip)]
hash: UnsafeCell<PyHash>,
hash: AtomicI64,
}
impl core::fmt::Debug for ContextVar {
@@ -303,8 +302,6 @@ mod _contextvars {
}
}
unsafe impl Sync for ContextVar {}
impl PartialEq for ContextVar {
fn eq(&self, other: &Self) -> bool {
core::ptr::eq(self, other)
@@ -320,12 +317,15 @@ mod _contextvars {
impl ContextVar {
fn delete(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> {
zelf.cached.store(None);
let cached = zelf.cached.lock().take();
drop(cached);
let ctx = PyContext::current(vm);
let mut vars = ctx.borrow_vars_mut();
if vars.swap_remove(zelf).is_none() {
let removed = ctx.borrow_vars_mut().swap_remove(zelf);
let existed = removed.is_some();
drop(removed);
if !existed {
// TODO:
// PyErr_SetObject(PyExc_LookupError, (PyObject *)var);
return Err(vm.new_lookup_error(zelf.as_object().repr(vm)?.as_wtf8().to_owned()));
@@ -338,16 +338,17 @@ mod _contextvars {
fn set_inner(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) {
let ctx = PyContext::current(vm);
let mut vars = ctx.borrow_vars_mut();
vars.insert(zelf.to_owned(), value.clone());
let replaced = ctx.borrow_vars_mut().insert(zelf.to_owned(), value.clone());
drop(replaced);
zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst);
let cache = ContextVarCache {
object: value,
idx: ctx.inner.idx.get(),
idx: ctx.inner.idx.load(Ordering::Relaxed),
};
zelf.cached.store(Some(cache));
let replaced = zelf.cached.lock().replace(cache);
drop(replaced);
}
fn generate_hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyHash {
@@ -370,28 +371,32 @@ mod _contextvars {
default: OptionalArg<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<Option<PyObjectRef>> {
let found = super::CONTEXTS.with_borrow(|ctxs| {
let ctx = ctxs.last()?;
let cached_ptr = zelf.cached.as_ptr();
debug_assert!(!cached_ptr.is_null());
if let Some(cached) = unsafe { &*cached_ptr }
// The replaced cache entry comes back out so that dropping it, which
// can run a __del__ that calls back in, happens with no lock held.
let (found, replaced) = super::CONTEXTS.with_borrow(|ctxs| {
let Some(ctx) = ctxs.last() else {
return (None, None);
};
let mut cached = zelf.cached.lock();
if let Some(cached) = &*cached
&& zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id()
&& cached.idx + 1 == ctxs.len()
{
return Some(cached.object.clone());
return (Some(cached.object.clone()), None);
}
let vars = ctx.borrow_vars();
let obj = vars.get(zelf)?;
let Some(obj) = ctx.borrow_vars().get(zelf).map(|obj| obj.to_owned()) else {
return (None, None);
};
zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst);
// TODO: ensure cached is not changed
let _removed = zelf.cached.swap(Some(ContextVarCache {
let replaced = cached.replace(ContextVarCache {
object: obj.clone(),
idx: ctxs.len() - 1,
}));
});
Some(obj.clone())
(Some(obj), replaced)
});
drop(replaced);
let value = if let Some(value) = found {
value
@@ -425,7 +430,7 @@ mod _contextvars {
#[pymethod]
fn reset(zelf: &Py<Self>, token: PyRef<ContextToken>, vm: &VirtualMachine) -> PyResult<()> {
if token.used.get() {
if token.used.load(Ordering::Acquire) {
return Err(vm.new_runtime_error(format!(
"{} has already been used once",
token.as_object().repr(vm)?
@@ -447,7 +452,7 @@ mod _contextvars {
)));
}
token.used.set(true);
token.used.store(true, Ordering::Release);
if let Some(old_value) = &token.old_value {
Self::set_inner(zelf, old_value.clone(), vm);
@@ -462,7 +467,7 @@ mod _contextvars {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
}
@@ -484,15 +489,13 @@ mod _contextvars {
name: args.name.to_string(),
default: args.default.into_option(),
cached_id: 0.into(),
cached: AtomicCell::new(None),
hash: UnsafeCell::new(0),
cached: PyMutex::new(None),
hash: AtomicI64::new(0),
};
let py_var = var.into_ref_with_type(vm, cls)?;
unsafe {
// SAFETY: py_var is not exposed to python memory model yet
*py_var.hash.get() = Self::generate_hash(&py_var, vm)
};
let hash = Self::generate_hash(&py_var, vm);
py_var.hash.store(hash, Ordering::Relaxed);
Ok(py_var.into())
}
@@ -504,14 +507,14 @@ mod _contextvars {
impl core::hash::Hash for ContextVar {
#[inline]
fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
unsafe { *self.hash.get() }.hash(state)
self.hash.load(Ordering::Relaxed).hash(state)
}
}
impl Hashable for ContextVar {
#[inline]
fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> {
Ok(unsafe { *zelf.hash.get() })
Ok(zelf.hash.load(Ordering::Relaxed))
}
}
@@ -537,11 +540,9 @@ mod _contextvars {
ctx: PyRef<PyContext>, // tok_ctx in CPython
var: PyRef<ContextVar>, // tok_var in CPython
old_value: Option<PyObjectRef>, // tok_oldval in CPython
used: Cell<bool>,
used: AtomicBool,
}
unsafe impl Sync for ContextToken {}
#[pyclass(with(Constructor, Representable))]
impl ContextToken {
#[pygetset]
@@ -562,7 +563,7 @@ mod _contextvars {
cls: PyTypeRef,
args: PyObjectRef,
vm: &VirtualMachine,
) -> PyGenericAlias {
) -> PyResult<PyGenericAlias> {
PyGenericAlias::from_args(cls, args, vm)
}
@@ -598,7 +599,11 @@ mod _contextvars {
impl Representable for ContextToken {
#[inline]
fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> {
let used = if zelf.used.get() { " used" } else { "" };
let used = if zelf.used.load(Ordering::Acquire) {
" used"
} else {
""
};
let var = Representable::repr_wtf8(&zelf.var, vm)?;
let ptr = zelf.as_object().get_id() as *const u8;
let mut result = Wtf8Buf::from(format!("<Token{used} var="));
crates/stdlib/src/csv.rs
+11 -6
View File
@@ -64,7 +64,7 @@ mod _csv {
bytes: &[u8],
err: core::str::Utf8Error,
) -> PyBaseExceptionRef {
vm.new_unicode_decode_error_real(
vm.new_unicode_decode_error(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
err.valid_up_to(),
@@ -779,11 +779,16 @@ mod _csv {
// TODO: Maybe need to update the obj from HashMap
}
DialectItem::Obj(o) => Ok(self.update_py_dialect(o.clone())),
DialectItem::None => {
let g = GLOBAL_HASHMAP.lock();
let res = g.get("excel").unwrap().clone();
Ok(self.update_py_dialect(res))
}
DialectItem::None => Ok(self.update_py_dialect(PyDialect {
delimiter: b',',
quotechar: Some(b'"'),
escapechar: None,
doublequote: true,
skipinitialspace: false,
lineterminator: "\r\n".to_owned(),
quoting: QuoteStyle::Minimal,
strict: false,
})),
}
}
crates/stdlib/src/faulthandler.rs
+2 -2
View File
@@ -260,8 +260,8 @@ mod decl {
use core::sync::atomic::Ordering;
let current_tid = rustpython_vm::stdlib::_thread::get_ident();
{
vm.state.stop_the_world.stop_the_world(vm);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); }
vm.state.stop_the_world.stop_the_world(&vm.state);
scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); }
let registry = vm.state.thread_frames.lock();
#[expect(
clippy::iter_over_hash_type,
crates/stdlib/src/hashlib.rs
+3 -3
View File
@@ -847,15 +847,15 @@ pub(crate) mod _hashlib {
if len < 1 {
return Err(vm.new_value_error("key length must be greater than 0."));
}
usize::try_from(len)
.map_err(|_| vm.new_overflow_error("key length is too great."))?
i32::try_from(len).map_err(|_| vm.new_overflow_error("key length is too great."))?
as usize
}
None => hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))?,
};
let password_buf = args.password.borrow_buf();
let salt_buf = args.salt.borrow_buf();
let mut dk = vec![0u8; dklen];
let mut dk = vm.new_zeroed_bytes(dklen)?;
macro_rules! do_pbkdf2 {
($hash_ty:ty) => {{
crates/stdlib/src/lzma.rs
+18 -17
View File
@@ -337,40 +337,43 @@ mod _lzma {
}
fn parse_filter_chain_spec(
filter_specs: Vec<PyObjectRef>,
filter_specs: PyObjectRef,
vm: &VirtualMachine,
) -> PyResult<Filters> {
const LZMA_FILTERS_MAX: usize = 4;
if filter_specs.len() > LZMA_FILTERS_MAX {
let filter_specs_len = filter_specs.length(vm)?;
if filter_specs_len > LZMA_FILTERS_MAX {
return Err(new_lzma_error(
format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"),
vm,
));
}
let filter_specs = filter_specs.try_sequence(vm)?;
let mut filters = Filters::new();
for spec in &filter_specs {
let filter_id = get_dict_opt_u64(spec, "id", vm)?
for i in 0..filter_specs_len {
let spec = filter_specs.get_item(i as isize, vm)?;
let filter_id = get_dict_opt_u64(&spec, "id", vm)?
.ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?;
match filter_id {
FILTER_LZMA1 => {
let opts = parse_filter_spec_lzma(spec, vm)?;
let opts = parse_filter_spec_lzma(&spec, vm)?;
filters.lzma1(&opts);
}
FILTER_LZMA2 => {
let opts = parse_filter_spec_lzma(spec, vm)?;
let opts = parse_filter_spec_lzma(&spec, vm)?;
filters.lzma2(&opts);
}
FILTER_DELTA => {
let dist = parse_filter_spec_delta(spec, vm)?;
let dist = parse_filter_spec_delta(&spec, vm)?;
filters
.delta_properties(&[(dist - 1) as u8])
.map_err(|e| catch_lzma_error(e, vm))?;
}
FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB
| FILTER_SPARC => {
let start_offset = parse_filter_spec_bcj(spec, vm)?;
let start_offset = parse_filter_spec_bcj(&spec, vm)?;
add_bcj_filter(&mut filters, filter_id, start_offset)
.map_err(|e| catch_lzma_error(e, vm))?;
}
@@ -570,7 +573,7 @@ mod _lzma {
#[pyarg(any, optional)]
memlimit: Option<u64>,
#[pyarg(any, optional)]
filters: Option<Vec<PyObjectRef>>,
filters: Option<PyObjectRef>,
}
impl Constructor for LZMADecompressor {
@@ -735,7 +738,7 @@ mod _lzma {
fn init_xz(
check: i32,
preset: u32,
filters: Option<Vec<PyObjectRef>>,
filters: Option<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<Stream> {
let real_check =
@@ -751,10 +754,11 @@ mod _lzma {
fn init_alone(
preset: u32,
filter_specs: Option<Vec<PyObjectRef>>,
filter_specs: Option<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<Stream> {
if let Some(_filter_specs) = filter_specs {
if let Some(filter_specs) = filter_specs {
filter_specs.length(vm)?;
// TODO: validate single LZMA1 filter and use its options
let options = LzmaOptions::new_preset(preset).map_err(|_| {
new_lzma_error(format!("Invalid compression preset: {preset}"), vm)
@@ -768,10 +772,7 @@ mod _lzma {
}
}
fn init_raw(
filter_specs: Option<Vec<PyObjectRef>>,
vm: &VirtualMachine,
) -> PyResult<Stream> {
fn init_raw(filter_specs: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<Stream> {
let filter_specs = filter_specs
.ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?;
let filters = parse_filter_chain_spec(filter_specs, vm)?;
@@ -788,7 +789,7 @@ mod _lzma {
#[pyarg(any, optional)]
preset: Option<PyObjectRef>,
#[pyarg(any, optional)]
filters: Option<Vec<PyObjectRef>>,
filters: Option<PyObjectRef>,
}
impl Constructor for LZMACompressor {
crates/stdlib/src/math.rs
+11 -16
View File
@@ -727,25 +727,20 @@ mod math {
}
// Generic Python path
let (p_i, q_i) = (p_i.unwrap(), q_i.unwrap());
// Collect current + remaining elements
let p_remaining: Result<Vec<PyObjectRef>, _> =
core::iter::once(Ok(p_i)).chain(p_iter).collect();
let q_remaining: Result<Vec<PyObjectRef>, _> =
core::iter::once(Ok(q_i)).chain(q_iter).collect();
let (p_vec, q_vec) = (p_remaining?, q_remaining?);
if p_vec.len() != q_vec.len() {
return Err(vm.new_value_error("Inputs are not the same length"));
}
let (mut p_i, mut q_i) = (p_i.unwrap(), q_i.unwrap());
let mut total = obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into());
for (p_item, q_item) in p_vec.into_iter().zip(q_vec) {
let prod = vm._mul(&p_item, &q_item)?;
loop {
let prod = vm._mul(&p_i, &q_i)?;
total = vm._add(&total, &prod)?;
let next_p = p_iter.next().transpose()?;
let next_q = q_iter.next().transpose()?;
match (next_p, next_q) {
(Some(next_p), Some(next_q)) => (p_i, q_i) = (next_p, next_q),
(None, None) => return Ok(total),
_ => return Err(vm.new_value_error("Inputs are not the same length")),
}
}
return Ok(total);
}
Ok(obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into()))
crates/stdlib/src/md5.rs
+8 -1
View File
@@ -3,10 +3,17 @@ pub(crate) use _md5::module_def;
#[pymodule]
mod _md5 {
use crate::hashlib::_hashlib::{HashArgs, local_md5};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyfunction]
fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_md5(args, vm)?.into_pyobject(vm))
}
#[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")]
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
crates/stdlib/src/mmap.rs
+25 -9
View File
@@ -611,6 +611,8 @@ mod mmap {
};
impl AsBuffer for PyMmap {
const RELEASE_BUFFER: bool = true;
fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let readonly = matches!(zelf.access, AccessMode::Read);
let buf = PyBuffer::new(
@@ -777,7 +779,10 @@ mod mmap {
let start = options
.start
.map_or_else(|| self.pos(), |start| start.saturated_at(size));
let end = options.end.map_or(size, |end| end.saturated_at(size));
let end = options
.end
.map_or(size, |end| end.saturated_at(size))
.max(start);
(start, end)
}
@@ -886,7 +891,7 @@ mod mmap {
let dest = dest.try_to_primitive(vm).ok()?;
let src = src.try_to_primitive(vm).ok()?;
let cnt = cnt.try_to_primitive(vm).ok()?;
if size - dest < cnt || size - src < cnt {
if dest > size || src > size || size - dest < cnt || size - src < cnt {
return None;
}
Some((dest, src, cnt))
@@ -1147,24 +1152,35 @@ mod mmap {
}
#[pymethod]
fn write(&self, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyIntRef> {
let pos = self.pos();
let size = self.__len__();
fn write(zelf: &Py<Self>, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyIntRef> {
let self_ = &**zelf;
let pos = self_.pos();
let size = self_.__len__();
let data = bytes.borrow_buf();
// Writing locks the map, and reading a source that views this same
// map locks it too, so such a source is copied out first.
let copied;
let borrowed;
let data: &[u8] = if bytes.source_object().is(zelf.as_object()) {
copied = bytes.borrow_buf().to_vec();
&copied
} else {
borrowed = bytes.borrow_buf();
&borrowed
};
if pos > size || size - pos < data.len() {
return Err(vm.new_value_error("data out of range"));
}
let len = self.try_writable(vm, |mmap| {
let len = self_.try_writable(vm, |mmap| {
(&mut mmap[pos..(pos + data.len())])
.write(&data)
.write(data)
.map_err(|err| err.to_pyexception(vm))?;
Ok(data.len())
})??;
self.advance_pos(len);
self_.advance_pos(len);
Ok(PyInt::from(len).into_ref(&vm.ctx))
}
crates/stdlib/src/overlapped.rs
+19 -7
View File
@@ -12,7 +12,7 @@ mod _overlapped {
builtins::{PyBaseExceptionRef, PyBytesRef, PyModule, PyStrRef, PyTupleRef, PyType},
common::lock::PyMutex,
convert::{ToPyException, ToPyObject},
function::OptionalArg,
function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg},
object::{Traverse, TraverseFn},
protocol::PyBuffer,
types::{Constructor, Destructor},
@@ -428,12 +428,14 @@ mod _overlapped {
fn ReadFileInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.ReadFileInto takes
buf: ArgMemoryBuffer,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
@@ -530,13 +532,15 @@ mod _overlapped {
fn WSARecvInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.WSARecvInto takes
buf: ArgMemoryBuffer,
flags: u32,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
@@ -583,10 +587,12 @@ mod _overlapped {
fn WriteFile(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WriteFile takes
buf: ArgBytesLike,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
@@ -629,11 +635,13 @@ mod _overlapped {
fn WSASend(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WSASend takes
buf: ArgBytesLike,
flags: u32,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
@@ -870,12 +878,14 @@ mod _overlapped {
fn WSASendTo(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// y*, as _overlapped.Overlapped.WSASendTo takes
buf: ArgBytesLike,
flags: u32,
address: PyTupleRef,
vm: &VirtualMachine,
) -> PyResult {
use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
@@ -1001,7 +1011,8 @@ mod _overlapped {
fn WSARecvFromInto(
zelf: &Py<Self>,
handle: isize,
buf: PyBuffer,
// w*, as _overlapped.Overlapped.WSARecvFromInto takes
buf: ArgMemoryBuffer,
size: u32,
flags: OptionalArg<u32>,
vm: &VirtualMachine,
@@ -1009,6 +1020,7 @@ mod _overlapped {
use host_winapi::{
ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS,
};
let buf: PyBuffer = buf.into();
let mut inner = zelf.inner.lock();
if !matches!(inner.data, OverlappedData::None) {
crates/stdlib/src/pystruct.rs
+72 -26
View File
@@ -10,13 +10,14 @@ pub(crate) use _struct::module_def;
#[pymodule]
pub(crate) mod _struct {
use crate::vm::{
AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine,
AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
buffer::{FormatSpec, new_struct_error, struct_error_type},
builtins::{PyBytes, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef},
function::{ArgBytesLike, ArgMemoryBuffer, PosArgs},
common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard},
function::{ArgBytesLike, ArgMemoryBuffer, FuncArgs, PosArgs},
match_class,
protocol::PyIterReturn,
types::{Constructor, IterNext, Iterable, Representable, SelfIter},
types::{Constructor, Initializer, IterNext, Iterable, Representable, SelfIter},
};
use crossbeam_utils::atomic::AtomicCell;
use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat};
@@ -251,41 +252,76 @@ pub(crate) mod _struct {
Ok(fmt.format_spec(vm)?.size)
}
/// What a `Struct` is once a format has been read into it. Held apart
/// from the object because `__new__` hands out a `Struct` that `__init__`
/// has not filled in yet, and `__init__` may be called again on one that
/// already holds a format.
#[derive(Debug)]
struct StructSpec {
spec: FormatSpec,
format: PyStrRef,
}
#[pyattr]
#[pyclass(name = "Struct", traverse)]
#[derive(Debug, PyPayload)]
struct PyStruct {
#[pytraverse(skip)]
spec: FormatSpec,
format: PyStrRef,
inner: PyRwLock<Option<StructSpec>>,
}
impl Constructor for PyStruct {
type Args = IntoStructFormatBytes;
type Args = FuncArgs;
fn py_new(_cls: &Py<PyType>, fmt: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
let spec = fmt.format_spec(vm)?;
let format = fmt.0;
Ok(Self { spec, format })
fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> {
Ok(Self {
inner: PyRwLock::new(None),
})
}
}
#[pyclass(with(Constructor, Representable))]
impl Initializer for PyStruct {
type Args = IntoStructFormatBytes;
fn init(zelf: PyRef<Self>, fmt: Self::Args, vm: &VirtualMachine) -> PyResult<()> {
// The format is read before anything is replaced, so a format that
// cannot be read leaves the object as it was.
let spec = fmt.format_spec(vm)?;
*zelf.inner.write() = Some(StructSpec {
spec,
format: fmt.0,
});
Ok(())
}
}
#[pyclass(with(Constructor, Initializer, Representable), flags(BASETYPE))]
impl PyStruct {
#[pygetset]
fn format(&self) -> PyStrRef {
self.format.clone()
/// The format this was initialized with, or an error if `__init__`
/// never ran.
fn ready(&self, vm: &VirtualMachine) -> PyResult<PyMappedRwLockReadGuard<'_, StructSpec>> {
PyRwLockReadGuard::try_map(self.inner.read(), Option::as_ref)
.map_err(|_| vm.new_runtime_error("Struct object is not initialized"))
}
#[pygetset]
#[inline]
const fn size(&self) -> usize {
self.spec.size
fn format(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> {
Ok(self.ready(vm)?.format.clone())
}
/// The size an uninitialized `Struct` reports, which no format has
/// yet given a value.
#[pygetset]
fn size(&self) -> isize {
self.inner
.read()
.as_ref()
.map_or(-1, |inner| inner.spec.size as isize)
}
#[pymethod]
fn pack(&self, args: PosArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
self.spec.pack(args.into_vec(), vm)
self.ready(vm)?.spec.pack(args.into_vec(), vm)
}
#[pymethod]
@@ -296,23 +332,28 @@ pub(crate) mod _struct {
args: PosArgs,
vm: &VirtualMachine,
) -> PyResult<()> {
let offset = get_buffer_offset(buffer.len(), offset, self.size(), true, vm)?;
let inner = self.ready(vm)?;
let offset = get_buffer_offset(buffer.len(), offset, inner.spec.size, true, vm)?;
buffer.with_ref(|data| {
self.spec
inner
.spec
.pack_into(&mut data[offset..], args.into_vec(), vm)
})
}
#[pymethod]
fn unpack(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyTupleRef> {
data.with_ref(|buf| self.spec.unpack(buf, vm))
let inner = self.ready(vm)?;
data.with_ref(|buf| inner.spec.unpack(buf, vm))
}
#[pymethod]
fn unpack_from(&self, args: UpdateFromArgs, vm: &VirtualMachine) -> PyResult<PyTupleRef> {
let offset = get_buffer_offset(args.buffer.len(), args.offset, self.size(), false, vm)?;
let inner = self.ready(vm)?;
let size = inner.spec.size;
let offset = get_buffer_offset(args.buffer.len(), args.offset, size, false, vm)?;
args.buffer
.with_ref(|buf| self.spec.unpack(&buf[offset..][..self.size()], vm))
.with_ref(|buf| inner.spec.unpack(&buf[offset..][..size], vm))
}
#[pymethod]
@@ -321,14 +362,19 @@ pub(crate) mod _struct {
buffer: ArgBytesLike,
vm: &VirtualMachine,
) -> PyResult<UnpackIterator> {
UnpackIterator::with_buffer(vm, self.spec.clone(), buffer)
let spec = self.ready(vm)?.spec.clone();
UnpackIterator::with_buffer(vm, spec, buffer)
}
}
impl Representable for PyStruct {
#[inline]
fn repr_wtf8(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<Wtf8Buf> {
Ok(wtf8_concat!("Struct('", zelf.format.as_wtf8(), "')"))
fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> {
Ok(wtf8_concat!(
"Struct('",
zelf.ready(vm)?.format.as_wtf8(),
"')"
))
}
}
crates/stdlib/src/select.rs
+23 -10
View File
@@ -79,16 +79,26 @@ mod decl {
}
let deadline = timeout.map(|s| time::time(vm).unwrap() + s);
let seq2set = |list: &PyObject| -> PyResult<(Vec<Selectable>, FdSet)> {
let v: Vec<Selectable> = list.try_to_value(vm)?;
let max_fds: usize = cfg_select! {
windows => FD_SETSIZE as usize,
_ => FD_SETSIZE,
};
let too_many_fds = cfg_select! {
windows => v.len() > FD_SETSIZE as usize,
_ => v.len() > FD_SETSIZE,
};
if too_many_fds {
return Err(vm.new_value_error("too many file descriptors in select()"));
}
let seq2set = |list: &PyObject| -> PyResult<(Vec<Selectable>, FdSet)> {
// The limit is answered while the sequence is walked rather than
// from the length of the result. fileno() runs Python and can
// append to the very list being walked, and a walk that re-reads
// the list each step -- which is what `seq2set` does -- then never
// reaches a length to check.
let seen = core::cell::Cell::new(0usize);
let v: Vec<Selectable> = vm.extract_elements_with(list, |obj| {
let selectable = Selectable::try_from_object(vm, obj)?;
seen.set(seen.get() + 1);
if seen.get() > max_fds {
return Err(vm.new_value_error("too many file descriptors in select()"));
}
Ok(selectable)
})?;
let mut fds = FdSet::new();
for fd in &v {
@@ -304,7 +314,10 @@ mod decl {
timeout: OptionalArg<TimeoutArg<true>>,
vm: &VirtualMachine,
) -> PyResult<Vec<PyObjectRef>> {
let mut fds = self.fds.lock();
// Poll a copy: the wait releases the GIL-equivalent and runs
// signal handlers, which can register or unregister on the same
// object, and a held lock would deadlock them.
let mut fds = self.fds.lock().clone();
let TimeoutArg(timeout) = timeout.unwrap_or_default();
let timeout_ms = match timeout {
Some(d) => i32::try_from(d.as_millis())
crates/stdlib/src/sha1.rs
+8 -1
View File
@@ -3,10 +3,17 @@ pub(crate) use _sha1::module_def;
#[pymodule]
mod _sha1 {
use crate::hashlib::_hashlib::{HashArgs, local_sha1};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyfunction]
fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_sha1(args, vm)?.into_pyobject(vm))
}
#[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")]
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
crates/stdlib/src/sha3.rs
+8 -1
View File
@@ -6,7 +6,7 @@ mod _sha3 {
HashArgs, local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128,
local_shake_256,
};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyfunction]
fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult {
@@ -37,4 +37,11 @@ mod _sha3 {
fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_shake_256(args, vm)?.into_pyobject(vm))
}
#[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")]
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
crates/stdlib/src/socket.rs
+51 -28
View File
@@ -40,7 +40,6 @@ mod _socket {
}
use core::{
mem::MaybeUninit,
net::{Ipv4Addr, Ipv6Addr, SocketAddr},
time::Duration,
};
@@ -1589,7 +1588,10 @@ mod _socket {
vm: &VirtualMachine,
) -> Result<Vec<u8>, IoOrPyException> {
let flags = flags.unwrap_or(0);
let mut buffer = Vec::with_capacity(bufsize);
let mut buffer = Vec::new();
buffer
.try_reserve_exact(bufsize)
.map_err(|_| vm.new_memory_error(""))?;
let sock = self.sock()?;
let n = self.sock_op(vm, SockWaitKind::Read, || {
sock.recv_with_flags(buffer.spare_capacity_mut(), flags)
@@ -1608,8 +1610,6 @@ mod _socket {
) -> Result<usize, IoOrPyException> {
let flags = flags.unwrap_or(0);
let sock = self.sock()?;
let mut buf = buf.borrow_buf_mut();
let buf = &mut *buf;
// Handle nbytes parameter
let read_len = if let OptionalArg::Present(nbytes) = nbytes {
@@ -1621,10 +1621,13 @@ mod _socket {
buf.len()
};
let buf = &mut buf[..read_len];
self.sock_op(vm, SockWaitKind::Read, || {
sock.recv_with_flags(unsafe { slice_as_uninit(buf) }, flags)
})
let mut scratch = alloc_recv_scratch(read_len, vm)?;
let n = self.sock_op(vm, SockWaitKind::Read, || {
sock.recv_with_flags(&mut scratch.spare_capacity_mut()[..read_len], flags)
})?;
unsafe { scratch.set_len(n) };
buf.borrow_buf_mut()[..n].copy_from_slice(&scratch);
Ok(n)
}
#[pymethod]
@@ -1638,7 +1641,10 @@ mod _socket {
let bufsize = bufsize
.to_usize()
.ok_or_else(|| vm.new_value_error("negative buffersize in recvfrom"))?;
let mut buffer = Vec::with_capacity(bufsize);
let mut buffer = Vec::new();
buffer
.try_reserve_exact(bufsize)
.map_err(|_| vm.new_memory_error(""))?;
let (n, addr) = self.sock_op(vm, SockWaitKind::Read, || {
self.sock()?
.recv_from_with_flags(buffer.spare_capacity_mut(), flags)
@@ -1655,24 +1661,28 @@ mod _socket {
flags: OptionalArg<i32>,
vm: &VirtualMachine,
) -> Result<(usize, PyObjectRef), IoOrPyException> {
let mut buf = buf.borrow_buf_mut();
let buf = &mut *buf;
let buf = match nbytes {
let read_len = match nbytes {
OptionalArg::Present(i) => {
let i = i.to_usize().ok_or_else(|| {
vm.new_value_error("negative buffersize in recvfrom_into")
})?;
buf.get_mut(..i).ok_or_else(|| {
vm.new_value_error("nbytes is greater than the length of the buffer")
})?
if i > buf.len() {
return Err(vm
.new_value_error("nbytes is greater than the length of the buffer")
.into());
}
i
}
OptionalArg::Missing => buf,
OptionalArg::Missing => buf.len(),
};
let flags = flags.unwrap_or(0);
let sock = self.sock()?;
let mut scratch = alloc_recv_scratch(read_len, vm)?;
let (n, addr) = self.sock_op(vm, SockWaitKind::Read, || {
sock.recv_from_with_flags(unsafe { slice_as_uninit(buf) }, flags)
sock.recv_from_with_flags(&mut scratch.spare_capacity_mut()[..read_len], flags)
})?;
unsafe { scratch.set_len(n) };
buf.borrow_buf_mut()[..n].copy_from_slice(&scratch);
Ok((n, get_addr_tuple(&addr, vm)))
}
@@ -1684,7 +1694,7 @@ mod _socket {
vm: &VirtualMachine,
) -> Result<usize, IoOrPyException> {
let flags = flags.unwrap_or(0);
let buf = bytes.borrow_buf();
let buf = bytes.borrow_buf_unlocked(vm)?;
let buf = &*buf;
self.sock_op(vm, SockWaitKind::Write, || {
self.sock()?.send_with_flags(buf, flags)
@@ -1704,7 +1714,7 @@ mod _socket {
let deadline = timeout.map(Deadline::new);
let buf = bytes.borrow_buf();
let buf = bytes.borrow_buf_unlocked(vm)?;
let buf = &*buf;
let mut buf_offset = 0;
// now we have like 3 layers of interrupt loop :)
@@ -1741,7 +1751,7 @@ mod _socket {
OptionalArg::Missing => (0, arg2),
};
let addr = self.extract_address(address, "sendto", vm)?;
let buf = bytes.borrow_buf();
let buf = bytes.borrow_buf_unlocked(vm)?;
let buf = &*buf;
self.sock_op(vm, SockWaitKind::Write, || {
self.sock()?.send_to_with_flags(buf, &addr, flags)
@@ -1771,8 +1781,8 @@ mod _socket {
let buffers = buffers
.iter()
.map(|buf| buf.borrow_buf())
.collect::<Vec<_>>();
.map(|buf| buf.borrow_buf_unlocked(vm))
.collect::<PyResult<Vec<_>>>()?;
let buffers = buffers
.iter()
.map(|buf| io::IoSlice::new(buf))
@@ -2313,8 +2323,8 @@ mod _socket {
#[cfg(all(unix, not(any(target_os = "redox", target_os = "android"))))]
#[pyfunction]
fn sethostname(hostname: PyUtf8StrRef) -> std::io::Result<()> {
host_socket::sethostname(hostname.as_str())
fn sethostname(hostname: FsPath) -> std::io::Result<()> {
host_socket::sethostname(hostname.as_bytes())
}
#[pyfunction]
@@ -2380,8 +2390,21 @@ mod _socket {
Ok(s.to_string_lossy().into_owned())
}
unsafe fn slice_as_uninit<T>(v: &mut [T]) -> &mut [MaybeUninit<T>] {
unsafe { &mut *(v as *mut [T] as *mut [MaybeUninit<T>]) }
/// Room to receive into that belongs to no Python object.
///
/// A peer may never send, so the wait for it is unbounded. The export of
/// the caller's buffer is held for the whole call, which is what keeps it
/// from being resized, but the borrow that reaches its bytes is a lock
/// every other thread touching that object waits on, and a thread waiting
/// on a lock never reaches a safepoint — holding it across the wait stops
/// the world from being stopped at all. The bytes are copied over once
/// they have arrived.
fn alloc_recv_scratch(len: usize, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let mut scratch = Vec::new();
scratch
.try_reserve_exact(len)
.map_err(|_| vm.new_memory_error(""))?;
Ok(scratch)
}
enum IoOrPyException {
@@ -2601,7 +2624,7 @@ mod _socket {
Some(ArgStrOrBytesLike::Buf(b)) => {
let bytes = b.borrow_buf();
let host_str = core::str::from_utf8(&bytes).map_err(|e| {
vm.new_unicode_decode_error_real(
vm.new_unicode_decode_error(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
e.valid_up_to(),
@@ -2643,7 +2666,7 @@ mod _socket {
let bytes = b.borrow_buf();
core::str::from_utf8(&bytes)
.map_err(|e| {
vm.new_unicode_decode_error_real(
vm.new_unicode_decode_error(
vm.ctx.new_str("utf-8"),
vm.ctx.new_bytes(bytes.to_vec()),
e.valid_up_to(),
crates/stdlib/src/ssl.rs
+304 -171
View File
@@ -48,7 +48,7 @@ mod _ssl {
VirtualMachine,
builtins::{
PyBaseExceptionRef, PyByteArray, PyBytesRef, PyListRef, PyStrRef, PyType,
PyTypeRef, PyUtf8StrRef,
PyTypeRef, PyUtf8StrRef, PyWeak,
},
convert::IntoPyException,
function::{
@@ -137,6 +137,8 @@ mod _ssl {
#[pyattr]
const PROTOCOL_TLSv1_3: i32 = 6;
static NEXT_SSL_SESSION_NONCE: AtomicUsize = AtomicUsize::new(1);
// Protocol version constants for TLSVersion enum
#[pyattr]
const PROTO_SSLv3: i32 = 0x0300;
@@ -321,15 +323,17 @@ mod _ssl {
#[pyattr]
const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120;
// Version info - reporting as OpenSSL 3.3.0 for compatibility
// `ssl.py` still requires OpenSSL-shaped numeric compatibility fields even
// for non-OpenSSL TLS providers. Keep them in the supported 3.x ABI range,
// but report the actual rustls/AWS-LC backend in the human-readable string.
#[pyattr]
const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; // OpenSSL 3.3.0 (808452096)
const OPENSSL_VERSION_NUMBER: i32 = 0x30300000;
#[pyattr]
const OPENSSL_VERSION: &str = "OpenSSL 3.3.0 (rustls/0.23)";
const OPENSSL_VERSION: &str = "OpenSSL 3.3.0-compatible (AWS-LC/rustls 0.23)";
#[pyattr]
const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release
const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15);
#[pyattr]
const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release
const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15);
// Default cipher list for rustls - using modern secure ciphers
#[pyattr]
@@ -439,6 +443,31 @@ mod _ssl {
lifetime: u64,
}
impl SessionData {
// NOTE: This is NOT the actual TLS session ID, just a unique identifier.
fn new(server_name: &str, lifetime: u64) -> Self {
let creation_time = SystemTime::now();
let nonce = NEXT_SSL_SESSION_NONCE.fetch_add(1, Ordering::Relaxed);
let mut hasher = Sha256::new();
hasher.update(server_name.as_bytes());
hasher.update(
creation_time
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.to_le_bytes(),
);
hasher.update(nonce.to_le_bytes());
Self {
_server_name: server_name.to_owned(),
session_id: hasher.finalize()[..16].to_vec(),
creation_time,
lifetime,
}
}
}
// Type alias to simplify complex session cache type
type SessionCache = Arc<ParkingRwLock<HashMap<Vec<u8>, Arc<ParkingMutex<SessionData>>>>>;
@@ -466,20 +495,6 @@ mod _ssl {
// ✓ session_reused - tracked via handshake_kind()
// ✗ Actual TLS session ID/ticket data - NOT ACCESSIBLE
// Generate a synthetic session ID from server name and timestamp
// NOTE: This is NOT the actual TLS session ID, just a unique identifier
fn generate_session_id_from_metadata(server_name: &str, time: SystemTime) -> Vec<u8> {
let mut hasher = Sha256::new();
hasher.update(server_name.as_bytes());
hasher.update(
time.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs()
.to_le_bytes(),
);
hasher.finalize()[..16].to_vec()
}
// Custom ClientSessionStore that tracks session metadata for Python access
// NOTE: This wraps ClientSessionMemoryCache and records metadata when sessions are stored
#[derive(Debug)]
@@ -488,6 +503,39 @@ mod _ssl {
session_cache: SessionCache,
}
impl PythonClientSessionStore {
fn new(session_cache: SessionCache) -> Self {
Self {
inner: Arc::new(ClientSessionMemoryCache::new(SSL_SESSION_CACHE_SIZE)),
session_cache,
}
}
fn transfer_session(
&self,
target: &Self,
server_name: &ServerName<'static>,
kind: ClientSessionKind,
) {
if let Some(group) = self.kx_hint(server_name) {
target.set_kx_hint(server_name.clone(), group);
}
match kind {
ClientSessionKind::Tls12 => {
if let Some(session) = self.tls12_session(server_name) {
target.set_tls12_session(server_name.clone(), session);
}
}
ClientSessionKind::Tls13 => {
if let Some(ticket) = self.take_tls13_ticket(server_name) {
target.insert_tls13_ticket(server_name.clone(), ticket);
}
}
}
}
}
impl ClientSessionStore for PythonClientSessionStore {
fn set_kx_hint(&self, server_name: ServerName<'static>, group: rustls::NamedGroup) {
self.inner.set_kx_hint(server_name, group);
@@ -508,17 +556,8 @@ mod _ssl {
// Record metadata in Python-accessible cache
// NOTE: We can't access value.session_id or value.ticket (private fields)
// So we generate a synthetic ID from metadata
let creation_time = SystemTime::now();
let server_name_str = server_name.to_str();
let session_data = SessionData {
_server_name: server_name_str.as_ref().to_string(),
session_id: generate_session_id_from_metadata(
server_name_str.as_ref(),
creation_time,
),
creation_time,
lifetime: 7200, // TLS 1.2 default session lifetime
};
let session_data = SessionData::new(server_name_str.as_ref(), 7200);
let key = server_name_str.as_bytes().to_vec();
self.session_cache
@@ -552,17 +591,8 @@ mod _ssl {
// Record metadata in Python-accessible cache
// NOTE: We can't access value.ticket or value.lifetime_secs (private fields)
// So we use default values
let creation_time = SystemTime::now();
let server_name_str = server_name.to_str();
let session_data = SessionData {
_server_name: server_name_str.to_string(),
session_id: generate_session_id_from_metadata(
server_name_str.as_ref(),
creation_time,
),
creation_time,
lifetime: 7200, // Default TLS 1.3 ticket lifetime (Rustls uses this)
};
let session_data = SessionData::new(server_name_str.as_ref(), 7200);
let key = server_name_str.as_bytes().to_vec();
self.session_cache
@@ -745,6 +775,8 @@ mod _ssl {
#[pyclass(name = "_SSLContext", module = "ssl", traverse)]
#[derive(Debug, PyPayload)]
struct PySSLContext {
#[pytraverse(skip)]
context_identity: Arc<()>,
#[pytraverse(skip)]
protocol: i32,
#[pytraverse(skip)]
@@ -807,9 +839,6 @@ mod _ssl {
// Session management
#[pytraverse(skip)]
client_session_cache: SessionCache,
// Rustls session store for actual TLS session resumption
#[pytraverse(skip)]
rustls_session_store: Arc<PythonClientSessionStore>,
// Rustls server session store for server-side session resumption
#[pytraverse(skip)]
rustls_server_session_store: Arc<rustls::server::ServerSessionMemoryCache>,
@@ -1129,19 +1158,19 @@ mod _ssl {
let pwd_result = callable.call((), vm)?;
// Convert callable result to string
let password_from_callable = if let Ok(pwd_str) =
PyUtf8StrRef::try_from_object(vm, pwd_result.clone())
{
pwd_str.as_str().to_owned()
} else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, pwd_result) {
String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| {
vm.new_type_error("password callback returned invalid UTF-8 bytes")
})?
} else {
return Err(
vm.new_type_error("password callback must return a string or bytes")
);
};
let password_from_callable =
if let Ok(pwd_str) = PyUtf8StrRef::try_from_object(vm, pwd_result.clone()) {
pwd_str.as_str().to_owned()
} else if pwd_result.check_buffer() {
let pwd_bytes_like = ArgBytesLike::try_from_object(vm, pwd_result)?;
String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| {
vm.new_type_error("password callback returned invalid UTF-8 bytes")
})?
} else {
return Err(
vm.new_type_error("password callback must return a string or bytes")
);
};
// Validate callable password length
if password_from_callable.len() > PEM_BUFSIZE {
@@ -1779,7 +1808,8 @@ mod _ssl {
// Validate filepath is str or bytes
let path_str = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, filepath.clone()) {
s.as_str().to_owned()
} else if let Ok(b) = ArgBytesLike::try_from_object(vm, filepath) {
} else if filepath.check_buffer() {
let b = ArgBytesLike::try_from_object(vm, filepath)?;
String::from_utf8(b.borrow_buf().to_vec())
.map_err(|_| vm.new_value_error("Invalid path encoding"))?
} else {
@@ -1834,7 +1864,8 @@ mod _ssl {
// Validate name is str or bytes
let curve_name = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, name.clone()) {
s.as_str().to_owned()
} else if let Ok(b) = ArgBytesLike::try_from_object(vm, name) {
} else if name.check_buffer() {
let b = ArgBytesLike::try_from_object(vm, name)?;
String::from_utf8(b.borrow_buf().to_vec())
.map_err(|_| vm.new_value_error("Invalid curve name encoding"))?
} else {
@@ -1920,9 +1951,15 @@ mod _ssl {
connection: PyMutex::new(None),
handshake_done: PyMutex::new(false),
session_was_reused: PyMutex::new(false),
owner: PyRwLock::new(args.owner.into_option()),
// Filter out Python None objects - only store actual SSLSession objects
session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))),
owner: PyRwLock::new(
args.owner
.into_option()
.map(|o| o.downgrade(None, vm))
.transpose()?,
),
session: PyRwLock::new(None),
client_config: PyRwLock::new(None),
client_session_store: PyRwLock::new(None),
incoming_bio: None,
outgoing_bio: None,
sni_state: PyRwLock::new(None),
@@ -1940,6 +1977,12 @@ mod _ssl {
.into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket"))
.map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?;
if let Some(session) = args.session.into_option()
&& !vm.is_none(&session)
{
ssl_socket_ref.set_session(session, vm)?;
}
Ok(ssl_socket_ref)
}
@@ -1997,9 +2040,15 @@ mod _ssl {
connection: PyMutex::new(None),
handshake_done: PyMutex::new(false),
session_was_reused: PyMutex::new(false),
owner: PyRwLock::new(args.owner.into_option()),
// Filter out Python None objects - only store actual SSLSession objects
session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))),
owner: PyRwLock::new(
args.owner
.into_option()
.map(|o| o.downgrade(None, vm))
.transpose()?,
),
session: PyRwLock::new(None),
client_config: PyRwLock::new(None),
client_session_store: PyRwLock::new(None),
incoming_bio: Some(args.incoming),
outgoing_bio: Some(args.outgoing),
sni_state: PyRwLock::new(None),
@@ -2016,6 +2065,12 @@ mod _ssl {
.into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket"))
.map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?;
if let Some(session) = args.session.into_option()
&& !vm.is_none(&session)
{
ssl_socket_ref.set_session(session, vm)?;
}
Ok(ssl_socket_ref)
}
@@ -2053,8 +2108,8 @@ mod _ssl {
Ok((Some(pwd_str.as_str().to_owned()), None))
}
// Try bytes-like
else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, p.clone())
{
else if p.check_buffer() {
let pwd_bytes_like = ArgBytesLike::try_from_object(vm, p.clone())?;
let pwd = String::from_utf8(pwd_bytes_like.borrow_buf().to_vec())
.map_err(|_| vm.new_type_error("password bytes must be valid UTF-8"))?;
Ok((Some(pwd), None))
@@ -2294,19 +2349,11 @@ mod _ssl {
_ => (PROTO_MINIMUM_SUPPORTED, PROTO_MAXIMUM_SUPPORTED), // Auto-negotiate
};
// IMPORTANT: Create shared session cache BEFORE PySSLContext
// Both client_session_cache and PythonClientSessionStore.session_cache
// MUST point to the same HashMap to ensure Python-level and Rustls-level
// sessions are synchronized
// Session metadata is scoped to the SSLContext. Each per-connection
// rustls session store records into this shared cache.
let shared_session_cache = Arc::new(ParkingRwLock::new(HashMap::new()));
let rustls_client_store = Arc::new(PythonClientSessionStore {
inner: Arc::new(rustls::client::ClientSessionMemoryCache::new(
SSL_SESSION_CACHE_SIZE,
)),
session_cache: shared_session_cache.clone(),
});
Ok(Self {
context_identity: Arc::new(()),
protocol,
check_hostname: PyRwLock::new(protocol == PROTOCOL_TLS_CLIENT),
verify_mode: PyRwLock::new(default_verify_mode),
@@ -2330,7 +2377,6 @@ mod _ssl {
x509_cert_count: PyRwLock::new(0),
// Use the shared cache created above
client_session_cache: shared_session_cache,
rustls_session_store: rustls_client_store,
rustls_server_session_store: rustls::server::ServerSessionMemoryCache::new(
SSL_SESSION_CACHE_SIZE,
),
@@ -2377,9 +2423,16 @@ mod _ssl {
#[pytraverse(skip)]
session_was_reused: PyMutex<bool>,
// Owner (SSLSocket instance that owns this _SSLSocket)
owner: PyRwLock<Option<PyObjectRef>>,
owner: PyRwLock<Option<PyRef<PyWeak>>>,
// Session for resumption
session: PyRwLock<Option<PyObjectRef>>,
// Client configuration used by this connection. Retained so the resulting
// SSLSession can reuse the same verifier and client credentials.
#[pytraverse(skip)]
client_config: PyRwLock<Option<Arc<rustls::ClientConfig>>>,
// Per-connection store containing the session selected by this connection.
#[pytraverse(skip)]
client_session_store: PyRwLock<Option<Arc<PythonClientSessionStore>>>,
// MemoryBIO mode (optional)
incoming_bio: Option<PyRef<PyMemoryBIO>>,
outgoing_bio: Option<PyRef<PyMemoryBIO>>,
@@ -2477,31 +2530,27 @@ mod _ssl {
}
// Create and store a session object after successful handshake
fn create_session_after_handshake(&self, vm: &VirtualMachine) {
fn create_session_after_handshake(&self, was_resumed: bool, vm: &VirtualMachine) {
// Only create session for client-side connections
if self.server_side {
return;
}
// Check if session already exists
let session_opt = self.session.read().clone();
if let Some(ref s) = session_opt {
if vm.is_none(s) {
} else {
return;
}
}
// Get server hostname
let server_name = self.server_hostname.read().clone();
let previous_session = self.session.read().clone();
// Try to get session data from context's session cache
// IMPORTANT: Acquire and release locks quickly to avoid deadlock
let context = self.context.read();
let session_cache_arc = context.client_session_cache.clone();
drop(context); // Release context lock ASAP
let (context_identity, session_cache_arc) = {
let context = self.context.read();
(
context.context_identity.clone(),
context.client_session_cache.clone(),
)
};
let (session_id, creation_time, lifetime) = if let Some(ref name) = server_name {
let cached_session_data = if let Some(ref name) = server_name {
let key = name.as_bytes().to_vec();
// Clone the data we need while holding the lock, then immediately release
@@ -2511,29 +2560,57 @@ mod _ssl {
}; // Lock released here
if let Some(session_data_arc) = session_data_opt {
let data = session_data_arc.lock();
let result = (data.session_id.clone(), data.creation_time, data.lifetime);
drop(data); // Explicit unlock
result
session_data_arc.lock().clone()
} else {
// Create new session ID if not in cache
let time = std::time::SystemTime::now();
(generate_session_id_from_metadata(name, time), time, 7200)
SessionData::new(name, 7200)
}
} else {
// No server name, use defaults
let time = std::time::SystemTime::now();
(vec![0; 16], time, 7200)
SessionData::new("", 7200)
};
let session_data = if was_resumed {
previous_session
.as_ref()
.and_then(|session| session.downcast_ref::<PySSLSession>())
.map_or(cached_session_data, |session| SessionData {
_server_name: server_name.clone().unwrap_or_default(),
session_id: session.session_id.clone(),
creation_time: session.creation_time,
lifetime: session.lifetime,
})
} else {
cached_session_data
};
let rustls_server_name = server_name.and_then(|name| ServerName::try_from(name).ok());
let protocol_version = self
.connection
.lock()
.as_ref()
.and_then(|connection| connection.protocol_version());
let session_kind = match protocol_version {
Some(rustls::ProtocolVersion::TLSv1_2) => ClientSessionKind::Tls12,
Some(rustls::ProtocolVersion::TLSv1_3) => ClientSessionKind::Tls13,
_ => return,
};
let Some(client_config) = self.client_config.write().take() else {
return;
};
let Some(session_store) = self.client_session_store.write().take() else {
return;
};
// Create a new SSLSession object with real metadata
let session = PySSLSession {
// Use dummy session data to indicate we have a ticket
// TLS 1.2+ always uses session tickets/resumption
session_data: vec![1], // Non-empty to indicate has_ticket=True
session_id,
creation_time,
lifetime,
context_identity,
client_config,
session_store,
server_name: rustls_server_name,
kind: session_kind,
session_id: session_data.session_id,
creation_time: session_data.creation_time,
lifetime: session_data.lifetime,
has_ticket: true,
};
let py_session = session.into_pyobject(vm);
@@ -2628,7 +2705,7 @@ mod _ssl {
let _ = self.track_used_ca_from_capath();
}
self.create_session_after_handshake(vm);
self.create_session_after_handshake(was_resumed, vm);
}
// Internal implementation with timeout control
@@ -2734,7 +2811,19 @@ mod _ssl {
return Ok(());
};
let ssl_sock = self.owner.read().clone().unwrap_or_else(|| vm.ctx.none());
let ssl_sock = self
.owner
.read()
.as_ref()
.and_then(|owner| owner.upgrade())
.ok_or_else(|| {
super::compat::SslError::create_ssl_error_with_reason(
vm,
Some("SSL"),
"PARSE_TLSEXT",
"[SSL: PARSE_TLSEXT] SNI callback owner is no longer available",
)
})?;
let server_name_py: PyObjectRef = match sni_name {
Some(name) => vm.ctx.new_str(name.to_string()).into(),
None => vm.ctx.none(),
@@ -3464,9 +3553,9 @@ mod _ssl {
let check_hostname = *ctx.check_hostname.read();
let verify_flags = *ctx.verify_flags.read();
let context_identity = ctx.context_identity.clone();
// Get session store before dropping ctx
let session_store = ctx.rustls_session_store.clone();
let session_cache = ctx.client_session_cache.clone();
// Get CRLs for revocation checking
let crls_clone = ctx.crls.read().clone();
@@ -3474,31 +3563,6 @@ mod _ssl {
// Drop ctx early to avoid borrow conflicts
drop(ctx);
// Build client config using compat helper
let config_options = ClientConfigOptions {
protocol_settings,
root_store: if verify_mode != CERT_NONE {
Some(root_store_clone)
} else {
None
},
ca_certs_der: ca_certs_der_clone,
cert_chain: if !cert_chain_clone.is_empty() {
Some(cert_chain_clone)
} else {
None
},
private_key: private_key_opt,
verify_server_cert: verify_mode != CERT_NONE,
check_hostname,
verify_flags,
session_store: Some(session_store),
crls: crls_clone,
};
let config =
create_client_config(config_options).map_err(|e| vm.new_value_error(e))?;
// Parse server name for SNI
// Convert to ServerName
use rustls::pki_types::ServerName;
@@ -3517,10 +3581,61 @@ mod _ssl {
)
};
let conn = ClientConnection::new(Arc::new(config), server_name.clone())
.map_err(|e| {
vm.new_value_error(format!("Failed to create client connection: {e}"))
})?;
let explicit_session = self.session.read().clone();
let session_store = Arc::new(PythonClientSessionStore::new(session_cache));
let config = if let Some(session) = explicit_session {
let session = session
.downcast_ref::<PySSLSession>()
.ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?;
if !Arc::ptr_eq(&session.context_identity, &context_identity) {
return Err(
vm.new_value_error("Session refers to a different SSLContext.")
);
}
if session.server_name.as_ref() == Some(&server_name) {
session.session_store.transfer_session(
&session_store,
&server_name,
session.kind,
);
}
let mut config = (*session.client_config).clone();
config.resumption =
rustls::client::Resumption::store(session_store.clone());
Arc::new(config)
} else {
let config_options = ClientConfigOptions {
protocol_settings,
root_store: if verify_mode != CERT_NONE {
Some(root_store_clone)
} else {
None
},
ca_certs_der: ca_certs_der_clone,
cert_chain: if !cert_chain_clone.is_empty() {
Some(cert_chain_clone)
} else {
None
},
private_key: private_key_opt,
verify_server_cert: verify_mode != CERT_NONE,
check_hostname,
verify_flags,
session_store: Some(session_store.clone()),
crls: crls_clone,
};
Arc::new(
create_client_config(config_options)
.map_err(|e| vm.new_value_error(e))?,
)
};
*self.client_config.write() = Some(config.clone());
*self.client_session_store.write() = Some(session_store);
let conn = ClientConnection::new(config, server_name).map_err(|e| {
vm.new_value_error(format!("Failed to create client connection: {e}"))
})?;
*conn_guard = Some(Connection::Client(conn));
}
@@ -3955,12 +4070,13 @@ mod _ssl {
#[pygetset]
fn owner(&self) -> Option<PyObjectRef> {
self.owner.read().clone()
self.owner.read().as_ref().and_then(|owner| owner.upgrade())
}
#[pygetset(setter)]
fn set_owner(&self, owner: PyObjectRef, _vm: &VirtualMachine) {
*self.owner.write() = Some(owner);
fn set_owner(&self, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
*self.owner.write() = Some(owner.downgrade(None, vm)?);
Ok(())
}
#[pygetset]
@@ -4023,12 +4139,15 @@ mod _ssl {
#[pygetset(setter)]
fn set_session(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
// Validate that value is an SSLSession
if !value.is(vm.ctx.types.none_type) {
// Try to downcast to SSLSession to validate
let _ = value
.downcast_ref::<PySSLSession>()
.ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?;
let session = value
.downcast_ref::<PySSLSession>()
.ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?;
if !Arc::ptr_eq(
&session.context_identity,
&self.context.read().context_identity,
) {
return Err(vm.new_value_error("Session refers to a different SSLContext."));
}
// Check if this is a client socket
@@ -4042,11 +4161,7 @@ mod _ssl {
}
// Store the session for potential use during handshake
*self.session.write() = if value.is(vm.ctx.types.none_type) {
None
} else {
Some(value)
};
*self.session.write() = Some(value);
Ok(())
}
@@ -4728,22 +4843,31 @@ mod _ssl {
// SSLSession - represents a cached SSL session
// NOTE: This is an EMULATION - actual session data is managed by Rustls internally
#[derive(Debug, Clone, Copy)]
enum ClientSessionKind {
Tls12,
Tls13,
}
#[pyattr]
#[pyclass(name = "SSLSession", module = "ssl")]
#[derive(Debug, PyPayload)]
struct PySSLSession {
// Session data - serialized rustls session (EMULATED - kept empty)
session_data: Vec<u8>,
context_identity: Arc<()>,
client_config: Arc<rustls::ClientConfig>,
session_store: Arc<PythonClientSessionStore>,
server_name: Option<ServerName<'static>>,
kind: ClientSessionKind,
// Session ID - synthetic ID generated from metadata (NOT actual TLS session ID)
#[allow(dead_code)]
session_id: Vec<u8>,
// Session metadata
creation_time: std::time::SystemTime,
// Lifetime in seconds (default 7200 = 2 hours)
lifetime: u64,
has_ticket: bool,
}
#[pyclass(flags(BASETYPE))]
#[pyclass(flags(BASETYPE), with(Comparable))]
impl PySSLSession {
#[pygetset]
fn time(&self) -> i64 {
@@ -4768,20 +4892,29 @@ mod _ssl {
#[pygetset]
fn id(&self, vm: &VirtualMachine) -> PyBytesRef {
// Return session ID (hash of session data for uniqueness)
let mut hasher = DefaultHasher::new();
self.session_data.hash(&mut hasher);
let hash = hasher.finish();
// Convert hash to bytes
vm.ctx.new_bytes(hash.to_be_bytes().to_vec())
vm.ctx.new_bytes(self.session_id.clone())
}
#[pygetset]
fn has_ticket(&self) -> bool {
// For rustls, if we have session data, we have a ticket
!self.session_data.is_empty()
self.has_ticket
}
}
impl Comparable for PySSLSession {
fn cmp(
zelf: &Py<Self>,
other: &PyObject,
op: PyComparisonOp,
_vm: &VirtualMachine,
) -> PyResult<PyComparisonValue> {
op.eq_only(|| {
if let Some(other_session) = other.downcast_ref::<Self>() {
Ok((zelf.session_id == other_session.session_id).into())
} else {
Ok(PyComparisonValue::NotImplemented)
}
})
}
}
crates/stdlib/src/ssl/cert.rs
+33 -21
View File
@@ -10,7 +10,7 @@
//! - Loading certificates from files, directories, and bytes
use alloc::sync::Arc;
use chrono::{DateTime, Utc};
use jiff::{Timestamp, Zoned, tz::TimeZone};
use parking_lot::RwLock as ParkingRwLock;
use rustls::{
DigitallySignedStruct, RootCertStore, SignatureScheme,
@@ -201,10 +201,10 @@ fn format_ip_address(ip: &[u8]) -> String {
/// Formats certificate validity dates in the format:
/// "Mon DD HH:MM:SS YYYY GMT"
fn format_asn1_time(time: &x509_parser::time::ASN1Time) -> String {
let timestamp = time.timestamp();
DateTime::<Utc>::from_timestamp(timestamp, 0)
.expect("ASN1Time must be valid timestamp")
.format("%b %e %H:%M:%S %Y GMT")
let timestamp =
Timestamp::from_second(time.timestamp()).expect("ASN1Time must be valid timestamp");
Zoned::new(timestamp, TimeZone::UTC)
.strftime("%b %e %H:%M:%S %Y GMT")
.to_string()
}
@@ -287,9 +287,11 @@ pub(super) fn is_ca_certificate(cert_der: &[u8]) -> bool {
return ext.value.ca;
}
// No Basic Constraints extension -> NOT a CA certificate
// (matches OpenSSL X509_check_ca() behavior)
false
// X509_check_ca() also retains OpenSSL's legacy trust-anchor rule: a
// self-issued X.509v1 certificate has no extensions at all, but is still
// classified as a CA. CPython's test CA at capath/4e1295a3.0 exercises
// precisely this case.
cert.version().0 == 0 && cert.subject() == cert.issuer()
}
/// Convert an X509Name to Python nested tuple format for SSL certificate dicts
@@ -341,7 +343,7 @@ pub(super) fn cert_to_dict(
let serial = format_serial_number(&cert.serial);
dict.set_item("serialNumber", vm.ctx.new_str(serial).into(), vm)?;
// Validity dates - format with GMT using chrono
// Validity dates - format with GMT using jiff
dict.set_item(
"notBefore",
vm.ctx
@@ -414,7 +416,7 @@ pub(super) fn cert_der_to_dict_helper(
// CPython ordering: issuer, notAfter, notBefore, serialNumber, subject, version
dict.set_item("issuer", name_to_tuple(cert.issuer())?, vm)?;
// Validity - format with GMT using chrono
// Validity - format with GMT using jiff
dict.set_item(
"notAfter",
vm.ctx
@@ -867,26 +869,36 @@ impl ServerCertVerifier for NoVerifier {
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &DigitallySignedStruct,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
// Accept all signatures without verification
Ok(HandshakeSignatureValid::assertion())
rustls::crypto::verify_tls12_signature(
message,
cert,
dss,
&CryptoExt::get_provider().signature_verification_algorithms,
)
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &DigitallySignedStruct,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
// Accept all signatures without verification
Ok(HandshakeSignatureValid::assertion())
rustls::crypto::verify_tls13_signature(
message,
cert,
dss,
&CryptoExt::get_provider().signature_verification_algorithms,
)
}
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
ALL_SIGNATURE_SCHEMES.to_vec()
CryptoExt::get_provider()
.signature_verification_algorithms
.supported_schemes()
}
}
Loaded 100 of 282 files, more files were not shown because too many files have changed in this diff. Show more