Merge remote-tracking branch 'origin/main' into copilot/fix-infinite-loop-macro-expansion

Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com>

.claude/settings.json

@@ -1,15 +0,0 @@
-{
-  "hooks": {
-    "SessionStart": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "bash .claude/scripts/setup-env.sh"
-          }
-        ]
-      }
-    ]
-  }
-}

.cspell.dict/cpython.txt

@@ -4,6 +4,7 @@ argdefs
 argtypes
 asdl
 asname
+ASNATIVEBYTES
 atopen
 atext
 attro
@@ -34,8 +35,12 @@ ceval
 cfield
 cfws
 CFWS
+chans
+cidx
+cjkcodecs
 CLASSDEREF
 classdict
+cleandoc
 cmpop
 CNOTAB
 codedepth
@@ -57,11 +62,13 @@ dictbytype
 DICTFLAG
 dictoffset
 distpoint
+drvsize
 dynload
 elts
 eooh
 eofs
 EOOH
+esize
 evalloop
 excepthandler
 exceptiontable
@@ -80,6 +87,7 @@ finalizers
 firsttraceable
 flowgraph
 formatfloat
+framesize
 freelist
 freevar
 freevars
@@ -88,6 +96,7 @@ fsrc
 getdict
 getfunc
 getiter
+getlasti
 getsets
 getslice
 globalgetvar
@@ -134,11 +143,19 @@ Lshift
 lslpp
 lsprof
 MAXBLOCKS
+MAXFSTRINGLEVEL
 maxdepth
+mbiencoder
+mbstreamreader
+mbstreamwriter
 metavars
+mimalloc
 miscompiles
 mult
 multibytecodec
+mycmp
+mystricmp
+mystrnicmp
 nameobj
 nameop
 nargsf
@@ -162,13 +179,21 @@ Nondescriptor
 noninteger
 nops
 noraise
+npools
 nseen
 NSIGNALS
 numer
+numdupes
+numkwonly
+numposorkw
 nvars
+obmalloc
+odict
+ODICT
 opname
 opnames
 orelse
+osmodule
 outparam
 outparm
 paramfunc
@@ -176,6 +201,7 @@ parg
 pathconfig
 patma
 peepholer
+pegen
 phcount
 platstdlib
 ploc
@@ -186,18 +212,24 @@ preds
 preinitialized
 pybuilddir
 pycore
+PYGC
 pyinner
 pydecimal
 pyerrors
+pyframe
 Pyfunc
 pylifecycle
 pymain
+pymem
 pyrepl
 pystate
+pystrcmp
 PYTHONTRACEMALLOC
 PYTHONUTF8
 pythonw
+pytime
 PYTHREAD_NAME
+recvd
 releasebuffer
 repr
 resinfo
@@ -233,11 +265,14 @@ sval
 swappedbytes
 swaptimize
 sysdict
+tbexc
 tbstderr
 templatelib
 testconsole
 threadstate
 ticketer
+timespec
+timeval
 tmptype
 tok_oldval
 tstate
@@ -248,6 +283,7 @@ Typeparam
 typeparams
 typeslots
 unaryop
+unboundop
 uncollectable
 Unhandle
 unparse
@@ -269,5 +305,8 @@ winconsoleio
 withitem
 withs
 worklist
+XSETREF
+xibufferview
 xstat
 XXPRIME
+ENOTCAPABLE

.cspell.dict/rust-more.txt

@@ -50,6 +50,7 @@ modpow
 msvc
 muldiv
 nanos
+noalias
 nonoverlapping
 objclass
 peekable
@@ -59,6 +60,7 @@ powf
 powi
 prepended
 punct
+qself
 replacen
 retag
 rmatch

.cspell.dict/rustpython.txt

@@ -1,4 +1,9 @@
+atfork
 cfgs
+csascii
+csisolatin
+genshi
+mimalloc
 miri
 py
 pyarg
@@ -19,6 +24,7 @@ pyname
 pyobj
 pyobject
 pypayload
+pyperformance
 pyref
 pyslot
 pystaticmethod
@@ -26,7 +32,10 @@ pystone
 pystr
 pystruct
 pystructseq
+pyperformance
 pytype
+qsbr
 rustix
 struc
+unscanned
 zelf

.cspell.json

@@ -52,39 +52,74 @@
     "Lib/**",
     "crates/host_env/**"
   ],
+  "overrides": [
+    {
+      "filename": "crates/common/src/encodings/cjk/**",
+      "words": [
+        "CHOSEONG",
+        "EUCKR",
+        "famille",
+        "GBCOMMON",
+        "jong",
+        "JONGSEONG",
+        "JUNGSEONG",
+        "PHINT",
+        "Xabc"
+      ]
+    }
+  ],
   // words - list of words to be always considered correct
   // (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords)
   "words": [
+    "AESGCM",
     "aiterable",
     "alnum",
+    "COMPLEMENTOFALL",
+    "COMPLEMENTOFDEFAULT",
     "csock",
     "coro",
+    "contig",
+    "Crnl",
     "dedentations",
     "dedents",
     "deduped",
     "deoptimized",
     "deoptimize",
+    "EECDH",
     "emscripten",
     "excs",
+    "fdigits",
+    "feff",
+    "flufl",
     "fnfe",
+    "fsdefault",
     "ifexp",
+    "implicits",
+    "inity",
     "interps",
     "jitted",
     "jitting",
     "kwonly",
     "lossily",
     "mcache",
+    "MLKEM",
     "oparg",
     "opargs",
+    "pointee",
     "pyc",
+    "qmark",
     "reborrow",
+    "reborrows",
+    "reparenting",
     "reraises",
     "reraising",
     "significand",
+    "SUITEB",
     "summands",
     "TESTFN",
     "TZPATH",
     "unraisable",
+    "ustr",
     "wasi",
     "weaked",
     // unix

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:bullseye
+FROM rust:bookworm
 
 # Install clang
 RUN apt-get update \

.gitattributes

@@ -1,10 +1,14 @@
 Lib/**      linguist-vendored
 Cargo.lock  linguist-generated
 *.snap      linguist-generated -merge
-vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
-crates/rustpython_doc_db/src/*.inc.rs   linguist-generated=true
+crates/vm/src/stdlib/ast/gen.rs    linguist-generated -merge
+crates/doc/src/data.inc.rs                              generated
+crates/compiler-core/src/bytecode/opcode_metadata.rs    generated
+crates/common/src/encodings/cjk/mappings_*.rs           generated
+
+.github/workflows/*.lock.yml linguist-generated=true merge=ours
 
 # Binary data types
 *.aif binary
@@ -45,10 +49,11 @@ Lib/venv/scripts/posix/* text eol=lf
 # Language aware diff headers
 # https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more
 # https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81
-*.css   diff=css
-*.html  diff=html
-*.py    diff=python
-*.md    diff=markdown
+*.css       diff=css
+*.html      diff=html
+*.py        diff=python
+*.md        diff=markdown
+*.inc.rs    diff=rust
 
 # Generated files
 # https://github.com/github/linguist/blob/master/docs/overrides.md
@@ -66,6 +71,3 @@ Lib/test/certdata/*.0                                   generated
 Lib/test/levenshtein_examples.json                      generated
 Lib/test/test_stable_abi_ctypes.py                      generated
 Lib/token.py                                            generated
-crates/compiler-core/src/bytecode/opcode_metadata.rs    generated
-
-.github/workflows/*.lock.yml linguist-generated=true merge=ours

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,7 +2,10 @@
 Thanks for your contribution!
 -->
 
-- [ ] Closes #xxxx <!-- Replace xxxx with the GitHub issue number -->
+- [ ] Closes #xxxx <!-- Replace xxxx with the GitHub issue number if exists -->
+
+One of checkbox below must be checked.
+- [ ] I did not use AI to write the code of this patch.
 - [ ] This PR follows our [AI policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md)
 
 ## Summary

.github/actions/install-linux-deps/action.yml

@@ -29,6 +29,10 @@ inputs:
     description: Install gcc-aarch64-linux-gnu (gcc-aarch64-linux-gnu)
     required: false
     default: "false"
+  gcc-mingw-w64-x86-64:
+    description: Install gcc-mingw-w64-x86-64 (gcc-mingw-w64-x86-64)
+    required: false
+    default: "false"
   clang:
     description: Install clang (clang)
     required: false
@@ -39,11 +43,52 @@ runs:
     - name: Install Linux dependencies
       shell: bash
       if: ${{ runner.os == 'Linux' }}
-      run: >
-        sudo apt-get update
+      env:
+        GCC_MULTILIB: ${{ inputs.gcc-multilib }}
+        MUSL_TOOLS: ${{ inputs.musl-tools }}
+        CLANG: ${{ inputs.clang }}
+        GCC_AARCH64_LINUX_GNU: ${{ inputs.gcc-aarch64-linux-gnu }}
+        GCC_MINGW_W64_X86_64: ${{ inputs.gcc-mingw-w64-x86-64 }}
+      run: |
+        # `apt-get update` has no deadline of its own, so a source that takes
+        # the connection and then stops answering holds the job rather than
+        # failing it, and the retry below never runs. Bound each attempt and
+        # give the transports a timeout to fail on.
+        apt_update() {
+          sudo timeout 300 apt-get \
+            -o Acquire::Retries=3 \
+            -o Acquire::http::Timeout=20 \
+            -o Acquire::https::Timeout=20 \
+            update
+        }
 
-        sudo apt-get install --no-install-recommends
-        ${{ fromJSON(inputs.gcc-multilib) && 'gcc-multilib' || '' }}
-        ${{ fromJSON(inputs.musl-tools) && 'musl-tools' || '' }}
-        ${{ fromJSON(inputs.clang) && 'clang' || '' }}
-        ${{ fromJSON(inputs.gcc-aarch64-linux-gnu) && 'gcc-aarch64-linux-gnu linux-libc-dev-arm64-cross libc6-dev-arm64-cross' || '' }}
+        if ! apt_update; then
+          echo "::warning::apt-get update did not finish; disabling nonessential Microsoft apt sources and retrying"
+          for source in /etc/apt/sources.list.d/*microsoft* /etc/apt/sources.list.d/*azure-cli*; do
+            if [ -e "$source" ]; then
+              sudo mv "$source" "$source.disabled"
+            fi
+          done
+          apt_update
+        fi
+
+        packages=()
+        if [[ "$GCC_MULTILIB" == "true" ]]; then
+          packages+=(gcc-multilib)
+        fi
+        if [[ "$MUSL_TOOLS" == "true" ]]; then
+          packages+=(musl-tools)
+        fi
+        if [[ "$CLANG" == "true" ]]; then
+          packages+=(clang)
+        fi
+        if [[ "$GCC_AARCH64_LINUX_GNU" == "true" ]]; then
+          packages+=(gcc-aarch64-linux-gnu linux-libc-dev-arm64-cross libc6-dev-arm64-cross)
+        fi
+        if [[ "$GCC_MINGW_W64_X86_64" == "true" ]]; then
+          packages+=(gcc-mingw-w64-x86-64)
+        fi
+
+        if ((${#packages[@]})); then
+          sudo apt-get install --no-install-recommends "${packages[@]}"
+        fi

.github/workflows/ci.yaml

@@ -19,13 +19,13 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env
-  CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,doc
+  CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env,doc
+  CARGO_ARGS_MOST_FEATS: --features capi,encodings,jit,sqlite,threading
   # Crates excluded from workspace builds:
   # - rustpython_wasm: requires wasm target
-  # - rustpython-compiler-source: deprecated
   # - rustpython-venvlauncher: Windows-only
-  WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher
+  WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-venvlauncher
   X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD
   X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include
   CARGO_INCREMENTAL: 0
@@ -34,6 +34,7 @@ env:
   CARGO_PROFILE_RELEASE_DEBUG: 0
   CARGO_TERM_COLOR: always
   CI: true
+  FORCE_COLOR: 1
 
 jobs:
   determine_changes:
@@ -42,8 +43,13 @@ jobs:
     outputs:
       # Flag that is raised when any rust code is changed.
       rust_code: ${{ steps.check_rust_code.outputs.changed }}
+      # WASM demo / notebook / example_projects, independent of rust_code.
+      wasm_code: ${{ steps.check_wasm_code.outputs.changed }}
+      # Cross-target cargo check is skipped unless the diff can change
+      # platform-specific compilation. Force with the run:cross-check label.
+      cross_target: ${{ steps.check_cross_target.outputs.changed }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
@@ -66,6 +72,11 @@ jobs:
             ':.cargo/config.toml' \
             ':crates/**' \
             ':src/**' \
+            ':build.rs' \
+            ':tests/**' \
+            ':examples/**' \
+            ':benches/**' \
+            ':example_projects/**' \
             ':.github/workflows/ci.yaml' \
           ; then
               echo "changed=false" >> "$GITHUB_OUTPUT"
@@ -75,11 +86,132 @@ jobs:
         env:
           MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
 
+      - name: Check if WASM package or demo sources changed
+        id: check_wasm_code
+        run: |
+          if git diff --quiet "${MERGE_BASE}...HEAD" -- \
+            ':wasm/**' \
+            ':crates/wasm/**' \
+            ':example_projects/wasm32_without_js/**' \
+          ; then
+              echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
+
+      - name: Check if cross-target compilation can change
+        id: check_cross_target
+        run: |
+          if [ "${FORCE_CROSS_TARGET}" = "true" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if ! git diff --quiet "${MERGE_BASE}...HEAD" -- \
+            ':rust-toolchain.toml' \
+            ':.cargo/**' \
+            ':.github/workflows/ci.yaml' \
+            ':crates/host_env/**' \
+            ':crates/stdlib/src/openssl/**' \
+            ':crates/stdlib/src/ssl/**' \
+            ':**/_signal.rs' \
+            ':**/_testconsole.rs' \
+            ':**/_thread.rs' \
+            ':**/_winapi.rs' \
+            ':**/_wmi.rs' \
+            ':**/cert_store.rs' \
+            ':**/crt_fd*.rs' \
+            ':**/errno.rs' \
+            ':**/faulthandler.rs' \
+            ':**/fcntl.rs' \
+            ':**/fileutils.rs' \
+            ':**/grp.rs' \
+            ':**/locale.rs' \
+            ':**/mmap.rs' \
+            ':**/msvcrt.rs' \
+            ':**/multiprocessing.rs' \
+            ':**/native_certs.rs' \
+            ':**/nt.rs' \
+            ':**/openssl.rs' \
+            ':**/os.rs' \
+            ':**/overlapped.rs' \
+            ':**/posix*.rs' \
+            ':**/pwd.rs' \
+            ':**/resource.rs' \
+            ':**/scproxy.rs' \
+            ':**/select.rs' \
+            ':**/shm.rs' \
+            ':**/signal.rs' \
+            ':**/socket.rs' \
+            ':**/ssl.rs' \
+            ':**/syslog.rs' \
+            ':**/thread.rs' \
+            ':**/time.rs' \
+            ':**/termios.rs' \
+            ':**/testconsole.rs' \
+            ':**/tkinter.rs' \
+            ':**/uuid.rs' \
+            ':**/winapi.rs' \
+            ':**/windows.rs' \
+            ':**/winreg.rs' \
+            ':**/winsound.rs' \
+            ':**/wmi.rs'
+          then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Added or bumped packages bring new code to every target. Removed
+          # packages only matter while cfg-gated code still references them.
+          lock_diff=$(git diff -U0 "${MERGE_BASE}...HEAD" -- Cargo.lock)
+          if grep -Eq '^\+[^+]' <<< "${lock_diff}"; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          while IFS= read -r crate; do
+            crate=${crate//-/_}
+            if git grep -qE "(^|[^A-Za-z0-9_])${crate}::|(use|crate) ${crate}([^A-Za-z0-9_]|$)" -- '*.rs'; then
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+          done < <(sed -n 's/^-name = "\(.*\)"$/\1/p' <<< "${lock_diff}")
+
+          # Platform cfg, or FFI crates that only exist on some targets.
+          platform_re='target_os|target_arch|target_family|target_env|target_vendor|target_pointer_width|target_endian|cfg\(windows\)|cfg\(unix\)|cfg\(wasi|not\(windows|not\(unix|\[target\.|\blibc::|\bnix::|windows_sys|winapi::|rustix::|std::os::'
+
+          if ! git diff -G "${platform_re}" --quiet "${MERGE_BASE}...HEAD"; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          while IFS= read -r file; do
+            [ -f "${file}" ] || continue
+            if grep -Eq "${platform_re}" "${file}"; then
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+          done < <(git diff --name-only "${MERGE_BASE}...HEAD" -- '*.rs')
+
+          echo "changed=false" >> "$GITHUB_OUTPUT"
+        env:
+          MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
+          FORCE_CROSS_TARGET: ${{ contains(github.event.pull_request.labels.*.name, 'run:cross-check') }}
+
   rust_tests:
+    # Required check: keep the job so a Lib-only PR still gets a green
+    # "Run rust tests (*)" status. Skip the build when rust sources did not change.
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    needs:
+      - determine_changes
     env:
       RUST_BACKTRACE: full
-    name: Run rust tests
+      RUN_RUST_TESTS: ${{ needs.determine_changes.outputs.rust_code == 'true' || github.ref == 'refs/heads/main' }}
+    # Named after the matrix entry rather than left to be named for it: a
+    # generated name lists every value in the entry, so adding or removing one
+    # renames the check and drops it from the required list.
+    name: Run rust tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     strategy:
@@ -87,20 +219,37 @@ jobs:
         os: [macos-latest, ubuntu-latest, windows-2025]
       fail-fast: false
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Skip rust tests when sources did not change
+        if: env.RUN_RUST_TESTS != 'true'
+        run: echo "No rust source changes; required check is green without rebuilding."
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: env.RUN_RUST_TESTS == 'true'
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
+        if: env.RUN_RUST_TESTS == 'true'
+
+      - name: Use a short Cargo home on Windows
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV
+          git config --global core.longpaths true
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        if: env.RUN_RUST_TESTS == 'true'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
+            ${{ runner.temp }}/c/registry/index/
+            ${{ runner.temp }}/c/registry/cache/
+            ${{ runner.temp }}/c/git/db/
             target/
           key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
           restore-keys: |
@@ -110,38 +259,62 @@ jobs:
         continue-on-error: true
 
       - name: Install macOS dependencies
+        if: env.RUN_RUST_TESTS == 'true'
         uses: ./.github/actions/install-macos-deps
 
-      - name: run rust tests
-        run: cargo test --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }}
+      - name: Build rust tests
+        if: env.RUN_RUST_TESTS == 'true'
+        run: cargo test --no-run --locked --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }}
         env:
           INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
 
-      - name: run c-api tests
+      - name: run rust tests
+        if: env.RUN_RUST_TESTS == 'true'
+        run: cargo test --locked --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }} --lib --bins --tests
+        env:
+          INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
+
+      - name: Run rust doctests
+        if: env.RUN_RUST_TESTS == 'true'
+        run: cargo test --doc --locked --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }}
+        env:
+          INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
+
+      - name: Build c-api tests
+        if: env.RUN_RUST_TESTS == 'true' && runner.os != 'Windows'  # Requires pyo3 0.29+ on Windows
         working-directory: crates/capi
-        run: cargo test
-        if: runner.os != 'Windows'  # Requires pyo3 0.29+ on Windows
+        run: cargo test --no-run --locked
+
+      - name: run c-api tests
+        if: env.RUN_RUST_TESTS == 'true' && runner.os != 'Windows'  # Requires pyo3 0.29+ on Windows
+        working-directory: crates/capi
+        run: cargo test --locked --lib --bins --tests
+
+      - name: Run c-api doctests
+        if: env.RUN_RUST_TESTS == 'true' && runner.os != 'Windows'  # Requires pyo3 0.29+ on Windows
+        working-directory: crates/capi
+        run: cargo test --doc --locked
 
       - name: check compilation without host_env (sandbox mode)
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
         run: |
-          cargo check -p rustpython-vm --no-default-features --features compiler
-          cargo check -p rustpython-stdlib --no-default-features --features compiler
-          cargo build --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib
-        if: runner.os == 'Linux'
+          cargo check --locked -p rustpython-vm --no-default-features --features compiler
+          cargo check --locked -p rustpython-stdlib --no-default-features --features compiler
+          cargo build --locked --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib
 
       - name: sandbox smoke test
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
         run: |
           target/debug/rustpython extra_tests/snippets/sandbox_smoke.py
           target/debug/rustpython extra_tests/snippets/stdlib_re.py
-        if: runner.os == 'Linux'
 
       - name: Test openssl build
-        run: cargo build --no-default-features --features ssl-openssl
-        if: runner.os == 'Linux'
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
+        run: cargo build --locked --no-default-features --features ssl-openssl
 
       - name: Test vendored OpenSSL build
-        run: cargo build --no-default-features --features ssl-openssl-vendor
-        if: runner.os == 'Linux'
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
+        run: cargo build --locked --no-default-features --features ssl-openssl-vendor
 
       # - name: Install tk-dev for tkinter build
       #   run: sudo apt-get update && sudo apt-get install -y tk-dev
@@ -151,27 +324,40 @@ jobs:
       #   run: cargo build --features tkinter
       #   if: runner.os == 'Linux'
 
-      - name: Test example projects
+      - name: Build example projects
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
         run: |
-          cargo run --manifest-path example_projects/barebone/Cargo.toml
-          cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml
-        if: runner.os == 'Linux'
+          cargo build --manifest-path example_projects/barebone/Cargo.toml
+          cargo build --manifest-path example_projects/frozen_stdlib/Cargo.toml
+
+      - name: Run example projects
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
+        run: |
+          example_projects/barebone/target/debug/example-barebone
+          example_projects/frozen_stdlib/target/debug/example_frozen_stdlib
+
+      - name: Build rustpython for update_lib tests
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
+        run: cargo build --locked
 
       - name: run update_lib tests
-        run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v
+        if: env.RUN_RUST_TESTS == 'true' && runner.os == 'Linux'
+        run: target/debug/rustpython -m unittest discover -s scripts/update_lib/tests -v
         env:
           PYTHONPATH: scripts
-        if: runner.os == 'Linux'
 
   cargo_check:
-    name: cargo check
+    # Named after the matrix entry rather than left to be named for it: a
+    # generated name lists every value in the entry, so adding or removing one
+    # renames the check and drops it from the required list.
+    name: cargo check (${{ matrix.os }}, ${{ matrix.target }})
     runs-on: ${{ matrix.os }}
     needs:
       - determine_changes
     if: |
       (
         !contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
-        needs.determine_changes.outputs.rust_code == 'true'
+        needs.determine_changes.outputs.cross_target == 'true'
       ) || github.ref == 'refs/heads/main'
     strategy:
       matrix:
@@ -197,13 +383,17 @@ jobs:
           target: aarch64-unknown-linux-gnu
           dependencies:
             gcc-aarch64-linux-gnu: true
+        - os: ubuntu-latest
+          target: x86_64-pc-windows-gnu
+          dependencies:
+            gcc-mingw-w64-x86-64: true
         - os: macos-latest
           target: aarch64-apple-ios
         - os: macos-latest
           target: x86_64-apple-darwin
       fail-fast: false
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -215,9 +405,10 @@ jobs:
           gcc-multilib: ${{ matrix.dependencies.gcc-multilib || false }}
           musl-tools: ${{ matrix.dependencies.musl-tools || false }}
           gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }}
+          gcc-mingw-w64-x86-64: ${{ matrix.dependencies.gcc-mingw-w64-x86-64 || false }}
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
@@ -268,25 +459,20 @@ jobs:
       #     args: --ignore-rust-version
 
       - name: Check compilation with threading
-        run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} --features threading
+        run: cargo check --locked --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} --features threading
 
       - name: Check compilation with ssl
         if: ${{ !matrix.skip_ssl }}
-        run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS }}
+        run: cargo check --locked --target "${{ matrix.target }}" ${{ env.CARGO_ARGS }}
 
   snippets_cpython:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     env:
       RUST_BACKTRACE: full
-      # Tests that can be flaky when running with multiple processes `-j 2`. We will use `-j 1` for these.
-      FLAKY_MP_TESTS: >-
-        test_class
-        test_concurrent_futures
-        test_eintr
-        test_multiprocessing_fork
-        test_multiprocessing_forkserver
-        test_multiprocessing_spawn
-    name: Run snippets and cpython tests
+    # Named after the matrix entry rather than left to be named for it: a
+    # generated name lists every value in the entry, so adding or removing one
+    # renames the check and drops it from the required list.
+    name: Run snippets and cpython tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -294,39 +480,53 @@ jobs:
         - os: macos-latest
           extra_test_args:
             - '-u all'
-          env_polluting_tests:
-            - test_set
+            - '--timeout 600'
+            - '--dont-add-python-opts'
+          env_polluting_tests: []
           skips: []
           timeout: 50
         - os: ubuntu-latest
           extra_test_args:
             - '-u all'
-          env_polluting_tests:
-            - test_set
+            - '--timeout 600'
+            - '--dont-add-python-opts'
+          env_polluting_tests: []
           skips: []
           timeout: 60
         - os: windows-2025
-          extra_test_args: [] # TODO: Enable '-u all'
-          env_polluting_tests:
-            - test_set
+          extra_test_args:
+            - '-u all'
+            - '--timeout 600'
+            - '--dont-add-python-opts'
+          env_polluting_tests: []
           skips: []
           timeout: 50
       fail-fast: false
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
 
+      - name: Use a short Cargo home on Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV
+          git config --global core.longpaths true
+
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
+            ${{ runner.temp }}/c/registry/index/
+            ${{ runner.temp }}/c/registry/cache/
+            ${{ runner.temp }}/c/git/db/
             target/
           key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
           restore-keys: |
@@ -335,15 +535,16 @@ jobs:
         # Windows runners randomly crashes, https://github.com/actions/cache/issues/1754
         continue-on-error: true
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - name: Install macOS dependencies
         uses: ./.github/actions/install-macos-deps
         with:
           openssl: true
 
+      # Keep features in sync with update-caches.yml CARGO_ARGS.
       - name: build rustpython
-        run: cargo build --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }}
+        run: cargo build --locked --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }}
 
       - name: run snippets
         run: python -m pip install -r requirements.txt && pytest -v
@@ -358,33 +559,14 @@ jobs:
 
       - name: Run CPython tests
         run: |
-          target/release/rustpython -m test -j ${{ steps.cores.outputs.cores }} ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v -x ${{ env.FLAKY_MP_TESTS }} ${{ join(matrix.skips, ' ') }}
-        timeout-minutes: ${{ matrix.timeout }}
-        env:
-          RUSTPYTHON_SKIP_ENV_POLLUTERS: true
-
-      - name: Run flaky MP CPython tests
-        run: |
-          for attempt in $(seq 1 5); do
-            echo "::group::Attempt ${attempt}"
-
-            set +e
-            target/release/rustpython -m test -j 1 ${{ join(matrix.extra_test_args, ' ') }} --slowest --fail-env-changed --timeout 600 -v ${{ env.FLAKY_MP_TESTS }}
-            status=$?
-            set -e
-
-            echo "::endgroup::"
-
-            if [ $status -eq 0 ]; then
-              exit 0
-            fi
-          done
-
-          exit 1
+          target/release/rustpython -u -m test --slow-ci -j "$CORES" $EXTRA_TEST_ARGS $SKIPS
         timeout-minutes: ${{ matrix.timeout }}
         shell: bash
         env:
           RUSTPYTHON_SKIP_ENV_POLLUTERS: true
+          CORES: ${{ steps.cores.outputs.cores }}
+          EXTRA_TEST_ARGS: ${{ join(matrix.extra_test_args, ' ') }}
+          SKIPS: ${{ join(matrix.skips, ' ') }}
 
       - name: run cpython tests to check if env polluters have stopped polluting
         shell: bash
@@ -394,7 +576,7 @@ jobs:
           for thing in "${target_array[@]}"; do
             for i in $(seq 1 10); do
               set +e
-              target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v "${thing}"
+              target/release/rustpython -u -m test --slow-ci -u all -j 1 --timeout 600 --dont-add-python-opts "${thing}"
               exit_code=$?
               set -e
               if [ "${exit_code}" -eq 3 ]; then
@@ -429,6 +611,17 @@ jobs:
           target/release/rustpython -m ensurepip
           target/release/rustpython -c "import pip"
 
+      - if: runner.os == 'Windows'
+        name: Check pip HTTPS with the Windows trust store
+        run: >-
+          target/release/rustpython -m pip download
+          --disable-pip-version-check
+          --no-cache-dir
+          --no-deps
+          --only-binary=:all:
+          --dest "$env:RUNNER_TEMP\rustpython-pip-smoke"
+          six
+
       - if: runner.os != 'Windows'
         name: Check if pip inside venv is functional
         run: |
@@ -437,10 +630,13 @@ jobs:
 
       - name: Check whats_left is not broken
         shell: bash
-        run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit
+        run: python -I scripts/whats_left.py --rustpython target/release/rustpython
 
   clippy:
-    name: clippy
+    # Named after the matrix entry rather than left to be named for it: a
+    # generated name lists every value in the entry, so adding or removing one
+    # renames the check and drops it from the required list.
+    name: clippy (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     needs:
       - determine_changes
@@ -453,11 +649,9 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - macos-latest
           - ubuntu-latest
-          - windows-latest
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    steps: &clippy_steps
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -465,14 +659,24 @@ jobs:
         with:
           components: clippy
 
+      - name: Use a short Cargo home on Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV
+          git config --global core.longpaths true
+
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
+            ${{ runner.temp }}/c/registry/index/
+            ${{ runner.temp }}/c/registry/cache/
+            ${{ runner.temp }}/c/git/db/
             target/
           key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
           restore-keys: |
@@ -482,7 +686,27 @@ jobs:
         continue-on-error: true
 
       - name: Clippy
-        run: cargo clippy --keep-going ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings
+        run: cargo clippy --locked --keep-going ${{ env.CARGO_ARGS_MOST_FEATS }} --workspace --all-targets --bins --examples --tests ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings
+
+  # macOS/Windows clippy uses the same OS-surface filter as cargo check.
+  # Force with the run:cross-check label. Check names stay clippy (<os>).
+  clippy_cross:
+    name: clippy (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs:
+      - determine_changes
+    permissions:
+      contents: read
+    if: |
+      github.ref == 'refs/heads/main' ||
+      needs.determine_changes.outputs.cross_target == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - windows-latest
+    steps: *clippy_steps
 
   cargo_shear:
     name: cargo shear
@@ -495,18 +719,19 @@ jobs:
       needs.determine_changes.outputs.rust_code == 'true' ||
       github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: cargo-bins/cargo-binstall@aaa84a43aec4955a42c5ffc65d258961e39f276e # v1.19.1
+      - uses: taiki-e/install-action@94c31af3204a9f15ab40b35ad084410b905bbc73 # v2.87.17
+        with:
+          tool: cargo-shear
+          fallback: none
 
       - name: cargo shear
-        run: |
-          cargo binstall --no-confirm cargo-shear
-          cargo shear
+        run: cargo shear --locked
 
   lint:
     name: Lint
@@ -518,31 +743,31 @@ jobs:
       pull-requests: write
       security-events: write # for zizmor
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
 
       - name: actionlint
-        uses: reviewdog/action-actionlint@6fb7acc99f4a1008869fa8a0f09cfca740837d9d # v1.72.0
+        uses: reviewdog/action-actionlint@320fcdd9c860767cf17fab3b20e22e739d5d02b8 # v1.76.0
 
       - name: zizmor
-        uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
+        uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
 
       - name: restore prek cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
           path: ~/.cache/prek
 
       - name: install prek
         id: prek
-        uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4
+        uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0
         with:
           cache: false
           show-verbose-logs: false
@@ -558,7 +783,7 @@ jobs:
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
       - name: Clone CPython
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           repository: python/cpython
           path: cpython
@@ -572,7 +797,7 @@ jobs:
 
       - name: save prek cache
         if: ${{ github.ref == 'refs/heads/main' }} # only save on main
-        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           key: prek-${{ hashFiles('.pre-commit-config.yaml') }}
           path: ~/.cache/prek
@@ -583,20 +808,27 @@ jobs:
 
       - name: reviewdog
         if: ${{ !cancelled() }}
-        uses: reviewdog/action-suggester@aa38384ceb608d00f84b4690cacc83a5aba307ff # v1.24.0
+        uses: reviewdog/action-suggester@c387862ad6765a9729f2b40ee4d8b5769f2145c7 # v1.26.1
         with:
           level: warning
           fail_level: error
 
   miri:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    needs:
+      - determine_changes
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
+      (
+        needs.determine_changes.outputs.rust_code == 'true' ||
+        github.ref == 'refs/heads/main'
+      )
     name: Run tests under miri
     runs-on: ubuntu-latest
     timeout-minutes: 30
     env:
       NIGHTLY_CHANNEL: nightly
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -606,7 +838,7 @@ jobs:
           components: miri
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
@@ -618,20 +850,36 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-
 
+      - name: Build tests under miri
+        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test --no-run --locked -p rustpython-vm -- miri_test
+        env:
+          MIRIFLAGS: "-Zmiri-ignore-leaks -Zmiri-permissive-provenance"
+
       - name: Run tests under miri
-        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test
+        run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test --locked -p rustpython-vm -- miri_test
         env:
           # miri-ignore-leaks because the type-object circular reference means that there will always be
           # a memory leak, at least until we have proper cyclic gc
-          MIRIFLAGS: "-Zmiri-ignore-leaks"
+          # miri-permissive-provenance because function pointer identity checks (slot comparisons)
+          # cast fn pointers to usize, which strips provenance — this is the standard pattern for
+          # fn pointer comparison in Rust and not a soundness issue
+          MIRIFLAGS: "-Zmiri-ignore-leaks -Zmiri-permissive-provenance"
 
   wasm:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    needs:
+      - determine_changes
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
+      (
+        needs.determine_changes.outputs.rust_code == 'true' ||
+        needs.determine_changes.outputs.wasm_code == 'true' ||
+        github.ref == 'refs/heads/main'
+      )
     name: Check the WASM package and demo
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -640,7 +888,7 @@ jobs:
           components: clippy
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
@@ -655,7 +903,7 @@ jobs:
             ${{ runner.os }}-
 
       - name: cargo clippy
-        run: cargo clippy --keep-going --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings
+        run: cargo --locked clippy --keep-going --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings
 
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
@@ -665,12 +913,12 @@ jobs:
           mkdir geckodriver
           tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - run: python -m pip install -r requirements.txt
         working-directory: ./wasm/tests
 
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           package-manager-cache: false
 
@@ -680,7 +928,7 @@ jobs:
         run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT"
 
       - name: Restore npm cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         # don't restore on main or release
         if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release'
         with:
@@ -703,15 +951,20 @@ jobs:
         with:
           wabt-version: "1.0.36"
 
-      - name: check wasm32-unknown without js
+      - name: Build wasm32-unknown without js
+        working-directory: example_projects/wasm32_without_js/rustpython-without-js
+        run: cargo build
+
+      - name: Build wasm-runtime
+        run: cargo build --release --manifest-path example_projects/wasm32_without_js/wasm-runtime/Cargo.toml
+
+      - name: Check wasm32-unknown without js
+        working-directory: example_projects/wasm32_without_js
         run: |
-          cd example_projects/wasm32_without_js/rustpython-without-js
-          cargo build
-          cd ..
           if wasm-objdump -xj Import rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm; then
             echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2
           fi
-          cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm
+          wasm-runtime/target/release/wasm-runtime rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm
 
       - name: build notebook demo
         if: github.ref == 'refs/heads/release'
@@ -735,27 +988,59 @@ jobs:
       - name: Save npm cache
         # Save only on main or release
         if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'
-        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: node-${{ runner.os }}-wasm-demo-${{ hashFiles('wasm/demo/package-lock.json') }}
 
   wasm-wasi:
+    # Required check: keep the job so a Lib-only PR still gets a green
+    # "Run snippets and cpython tests on wasm-wasi" status.
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    needs:
+      - determine_changes
     name: Run snippets and cpython tests on wasm-wasi
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+      RUN_WASM_WASI: ${{ needs.determine_changes.outputs.rust_code == 'true' || github.ref == 'refs/heads/main' }}
+      # Passing snippets under wasm32-wasip1.
+      WASM_WASI_SNIPPETS: >-
+        stdlib_random
+        stdlib_time
+      # Passing Lib/test files under wasm32-wasip1
+      WASM_WASI_TEST_FILES: >-
+        test_int
+        test_dict
+        test_exceptions
+        test_copy
+        test_isinstance
+        test_dynamic
+        test_dictviews
+        test_glob
+        test_class
+        test_genericpath
+        test_grammar
+        test_fstring
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Skip wasm-wasi when rust sources did not change
+        if: env.RUN_WASM_WASI != 'true'
+        run: echo "No rust source changes; required check is green without rebuilding."
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: env.RUN_WASM_WASI == 'true'
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
+        if: env.RUN_WASM_WASI == 'true'
         with:
           target: wasm32-wasip1
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        if: env.RUN_WASM_WASI == 'true'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
@@ -771,19 +1056,34 @@ jobs:
             ${{ runner.os }}-stable--
 
       - name: Setup Wasmer
+        if: env.RUN_WASM_WASI == 'true'
         uses: wasmerio/setup-wasmer@24b15c95293d23f89c68bd40dac76338f773e924 # v3.1
 
       - name: Install clang
+        if: env.RUN_WASM_WASI == 'true'
         uses: ./.github/actions/install-linux-deps
         with:
           clang: true
 
       - name: build rustpython
-        run: cargo build --profile wasm-release --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib,stdio,importlib,host_env --verbose
+        if: env.RUN_WASM_WASI == 'true'
+        run: cargo build --locked --profile wasm-release --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib,stdio,importlib,host_env,doc --verbose
       - name: run snippets
-        run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_random.py"
+        if: env.RUN_WASM_WASI == 'true'
+        run: |
+          for name in $WASM_WASI_SNIPPETS; do
+            echo "::group::extra_tests/snippets/${name}.py"
+            wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/${name}.py"
+            echo "::endgroup::"
+          done
       - name: run cpython unittest
-        run: wasmer run --dir "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/Lib/test/test_int.py"
+        if: env.RUN_WASM_WASI == 'true'
+        run: |
+          for name in $WASM_WASI_TEST_FILES; do
+            echo "::group::Lib/test/${name}.py"
+            wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/Lib/test/${name}.py"
+            echo "::endgroup::"
+          done
 
   cargo_doc:
     needs:
@@ -798,14 +1098,14 @@ jobs:
     name: cargo doc
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
 
       - name: Restore cache
-        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
@@ -820,4 +1120,3 @@ jobs:
 
       - name: cargo doc
         run: cargo doc --locked
-

.github/workflows/codspeed.yaml

@@ -0,0 +1,114 @@
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [unlabeled, opened, synchronize, reopened]
+  merge_group:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+name: CodSpeed
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_TERM_COLOR: always
+  CI: true
+
+jobs:
+  benchmarks:
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      id-token: write # for OpenID Connect authentication with CodSpeed
+    env:
+      CACHE_PATHS: |
+        # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+        ~/.cargo/.crates2.json
+        ~/.cargo/.crates.toml
+        ~/.cargo/bin/
+        ~/.cargo/registry/index/
+        ~/.cargo/registry/cache/
+        ~/.cargo/git/db/
+        target/
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      # The benchmarks are linked against libpython through the pyo3 dev-dependency.
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+
+      - name: Restore the machine main last measured on
+        # A recording that cannot be fetched is read as "none yet", which the step
+        # below treats as nothing to disagree with. Failing here instead would
+        # turn a cache outage into a red benchmark job.
+        continue-on-error: true
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: build/codspeed-environment-reference.json
+          key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}
+          restore-keys: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Name the machine this run drew
+        id: environment
+        shell: bash
+        run: |
+          reference=build/codspeed-environment-reference.json
+          if [ "$GITHUB_REF" = refs/heads/main ]; then
+            args=(--record "$reference")
+          else
+            args=(--record build/codspeed-environment.json --reference "$reference")
+          fi
+          python3 scripts/codspeed-environment.py "${args[@]}" --github-output
+
+      - name: Restore cache
+        if: ${{ steps.environment.outputs.match != 'false' }}
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.CACHE_PATHS }}
+          key: codspeed-v2-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: codspeed-v2-
+        continue-on-error: true
+
+      - name: Install cargo-codspeed
+        if: ${{ steps.environment.outputs.match != 'false' }}
+        run: cargo install cargo-codspeed --locked --version ^5
+
+      - name: Build the benchmark targets
+        if: ${{ steps.environment.outputs.match != 'false' }}
+        run: cargo codspeed build --locked --measurement-mode simulation -p rustpython -p rustpython-sre_engine
+
+      - name: Save cache
+        if: ${{ github.ref == 'refs/heads/main' }} # only save on main
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.CACHE_PATHS }}
+          key: codspeed-v2-${{ hashFiles('**/Cargo.lock') }}
+        continue-on-error: true
+
+      - name: Run the benchmarks
+        if: ${{ steps.environment.outputs.match != 'false' }}
+        uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1
+        with:
+          mode: simulation
+          run: cargo codspeed run -p rustpython -p rustpython-sre_engine
+
+      - name: Record the machine this baseline was measured on
+        if: ${{ success() && github.ref == 'refs/heads/main' }}
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: build/codspeed-environment-reference.json
+          key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}

.github/workflows/cron-ci.yaml

@@ -14,7 +14,7 @@ on:
       - .github/workflows/cron-ci.yaml
 
 env:
-  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env
+  CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env,doc
 
 jobs:
   # codecov collects code coverage data from the rust tests, python snippets and python test suite.
@@ -27,22 +27,22 @@ jobs:
     env:
       INSTA_WORKSPACE_ROOT: ${{ github.workspace }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: taiki-e/install-action@e49978b799e49ff429d162b7a30601a569ab6538 # v2.81.1
+      - uses: taiki-e/install-action@94c31af3204a9f15ab40b35ad084410b905bbc73 # v2.87.17
         with:
           tool: cargo-llvm-cov
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - run: sudo apt-get update && sudo apt-get -y install lcov
 
       - name: Run cargo-llvm-cov with Rust tests.
-        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env
+        run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env,doc
 
       - name: Run cargo-llvm-cov with Python snippets.
         run: python scripts/cargo-llvm-cov.py
@@ -57,7 +57,7 @@ jobs:
 
       - name: Upload to Codecov
         if: ${{ github.event_name != 'pull_request' }}
-        uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           files: ./codecov.lcov
 
@@ -67,7 +67,7 @@ jobs:
     # Disable this scheduled job when running on a fork.
     if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: true
 
@@ -77,7 +77,7 @@ jobs:
         run: cargo build --release --verbose
 
       - name: collect tests data
-        run: cargo run --release extra_tests/jsontests.py
+        run: target/release/rustpython extra_tests/jsontests.py
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
 
@@ -85,7 +85,6 @@ jobs:
         if: ${{ github.event_name != 'pull_request' }}
         env:
           SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
-          GITHUB_ACTOR: ${{ github.actor }}
         run: |
           echo "$SSHKEY" >~/github_key
           chmod 600 ~/github_key
@@ -95,7 +94,7 @@ jobs:
           cd website
           cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json
           git add ./_data/regrtests_results.json
-          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then
+          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results"; then
             git push
           fi
 
@@ -105,21 +104,19 @@ jobs:
     # Disable this scheduled job when running on a fork.
     if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: true
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - name: build rustpython
-        run: cargo build --release --verbose
+        run: cargo build --release --verbose --features sqlite
 
       - name: Collect what is left data
-        run: |
-          chmod +x ./scripts/whats_left.py
-          ./scripts/whats_left.py --features "ssl,sqlite" > whats_left.temp
+        run: python -I scripts/whats_left.py --rustpython target/release/rustpython > whats_left.temp
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
 
@@ -127,7 +124,6 @@ jobs:
         if: ${{ github.event_name != 'pull_request' }}
         env:
           SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
-          GITHUB_ACTOR: ${{ github.actor }}
         run: |
           echo "$SSHKEY" >~/github_key
           chmod 600 ~/github_key
@@ -158,7 +154,7 @@ jobs:
           }
           EOF
           git add -A
-          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then
+          if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results"; then
             git push
           fi
 
@@ -168,13 +164,13 @@ jobs:
     # Disable this scheduled job when running on a fork.
     if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: true
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - run: cargo install cargo-criterion
 
@@ -204,6 +200,8 @@ jobs:
         if: ${{ github.event_name != 'pull_request' }}
         env:
           SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }}
+          COMMIT_SHA: ${{ github.sha }}
+          REF_NAME: ${{ github.ref_name }}
         run: |
           echo "$SSHKEY" >~/github_key
           chmod 600 ~/github_key
@@ -215,8 +213,8 @@ jobs:
           cp -r ../target/criterion ./assets/criterion
           printf '{\n  "generated_at": "%s",\n  "rustpython_commit": "%s",\n  "rustpython_ref": "%s"\n}\n' \
             "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            "${{ github.sha }}" \
-            "${{ github.ref_name }}" > ./_data/criterion-metadata.json
+            "$COMMIT_SHA" \
+            "$REF_NAME" > ./_data/criterion-metadata.json
           git add ./assets/criterion ./_data/criterion-metadata.json
           if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
             git push

.github/workflows/lib-deps-check.yaml

@@ -6,6 +6,8 @@ on:
     paths:
       - "Lib/**"
 
+permissions: {}
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
@@ -18,7 +20,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout base branch
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           # Use base branch for scripts (security: don't run PR code with elevated permissions)
           ref: ${{ github.event.pull_request.base.ref }}
@@ -26,13 +28,17 @@ jobs:
           persist-credentials: false
 
       - name: Fetch PR head
+        env:
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          git fetch origin ${{ github.event.pull_request.head.sha }}
+          git fetch origin "$PR_HEAD_SHA"
 
       - name: Checkout PR Lib files
+        env:
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           # Checkout only Lib/ directory from PR head for accurate comparison
-          git checkout ${{ github.event.pull_request.head.sha }} -- Lib/
+          git checkout "$PR_HEAD_SHA" -- Lib/
 
       - name: Get target CPython version
         id: cpython-version
@@ -41,7 +47,7 @@ jobs:
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
       - name: Checkout CPython
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           repository: python/cpython
           path: cpython
@@ -51,14 +57,17 @@ jobs:
 
       - name: Get changed Lib files
         id: all-changed-files
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           # Get the list of changed files under Lib/
           {
             echo 'changed<<EOF'
 
-            git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py'
+            git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA" -- 'Lib/*.py' 'Lib/**/*.py'
 
-            echo 'EOF' 
+            echo 'EOF'
           } >> "$GITHUB_OUTPUT"
 
       - name: Parse changed files
@@ -98,7 +107,7 @@ jobs:
 
       - name: Setup Python
         if: steps.changed-files.outputs.modules != ''
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
 
       - name: Run deps check
         if: steps.changed-files.outputs.modules != ''
@@ -114,7 +123,7 @@ jobs:
 
       - name: Post comment
         if: steps.deps-check.outputs.deps_output != ''
-        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
+        uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
         with:
           header: lib-deps-check
           number: ${{ github.event.pull_request.number }}
@@ -131,7 +140,7 @@ jobs:
 
       - name: Remove comment if no Lib changes
         if: steps.changed-files.outputs.modules == ''
-        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
+        uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
         with:
           header: lib-deps-check
           number: ${{ github.event.pull_request.number }}

.github/workflows/oscca-pr.yml

@@ -0,0 +1,70 @@
+name: Manage OSCCA pull requests
+
+on:
+  pull_request_target:
+    types: [opened]
+
+permissions: {}
+
+jobs:
+  label-and-assign:
+    name: Label and assign OSCCA pull request
+    runs-on: ubuntu-slim
+    timeout-minutes: 5
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Label and assign pull request
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const osccaUsers = new Set(
+              [
+                "2jiyong",
+                "chestnut1717",
+                "devyubin",
+                "fregataa",
+                "hyoinandout",
+                "HyoJongPark",
+                "JaceJung-dev",
+                "jinmay",
+                "jiwahn",
+                "kangdora",
+                "kim-jaedeok",
+                "kyokuping",
+                "leehanjeong",
+                "lms0806",
+                "lsahn-gh",
+                "moreal",
+                "name-of-okja",
+                "rlaisqls",
+                "seungje0612",
+                "shAn-kor",
+                "sigmaith",
+                "teddygood",
+                "widehyo1",
+                "YangSiJun528",
+                "zzarbttoo",
+              ].map((login) => login.toLowerCase()),
+            );
+            const pullRequest = context.payload.pull_request;
+            const author = pullRequest.user.login;
+
+            if (!osccaUsers.has(author.toLowerCase())) {
+              core.info(`${author} is not an OSCCA participant; skipping.`);
+              return;
+            }
+
+            const issue = {
+              ...context.repo,
+              issue_number: pullRequest.number,
+            };
+
+            await github.rest.issues.addLabels({
+              ...issue,
+              labels: ["z-ca-2026"],
+            });
+            await github.rest.issues.addAssignees({
+              ...issue,
+              assignees: [author],
+            });

.github/workflows/pyperformance-comment.yaml

@@ -0,0 +1,81 @@
+on:
+  workflow_run:
+    workflows: [pyperformance]
+    types: [completed]
+
+name: pyperformance comment
+
+# Same split as `codspeed-comment.yaml`: a `pull_request` run from a fork gets
+# a read-only token however it is configured, so the comparison is computed
+# there and handed over as an artifact, and this workflow -- which
+# `workflow_run` always runs in the base repo's context -- is the one trusted
+# with a token that may write the comment.
+permissions:
+  contents: read
+  actions: read # needed by download-artifact to fetch another run's artifact
+  pull-requests: write
+
+jobs:
+  comment:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download the comparison produced by the pyperformance workflow
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: pyperformance-diff
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+        continue-on-error: true # absent when the job was skipped or dispatched
+
+      - name: Post or update the PR comment
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const fs = require('fs');
+            if (!fs.existsSync('diff.md')) {
+              core.info('No diff.md in this run; nothing to comment.');
+              return;
+            }
+
+            const marker = '<!-- pyperformance-local-diff -->';
+            const run = context.payload.workflow_run;
+            let prNumber = (run.pull_requests || []).map((pr) => pr.number)[0];
+            if (!prNumber) {
+              const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                commit_sha: run.head_sha,
+              });
+              prNumber = data.find((pr) => pr.state === 'open')?.number;
+            }
+            if (!prNumber) {
+              core.info(`No open pull request for ${run.head_sha}; nothing to comment.`);
+              return;
+            }
+
+            const raw = fs.readFileSync('diff.md', 'utf8').slice(0, 60000);
+            const body = `${marker}\n\n${raw.replace(/<!--/g, '<!-​-')}`;
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+            });
+            const existing = comments.find((c) => c.body && c.body.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+            }

.github/workflows/pyperformance.yaml

@@ -0,0 +1,220 @@
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_dispatch:
+
+name: pyperformance
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_TERM_COLOR: always
+  CI: true
+  # Pure-Python benchmarks only: anything whose workload needs a real C
+  # extension cannot install under RustPython at all, and the async/SSL ones
+  # are far too slow here to be worth a runner's time. Keep this list short --
+  # every entry is paid for three times (CPython, base, head) in one job.
+  BENCHMARKS: >-
+    chaos,comprehensions,coroutines,deepcopy,deltablue,fannkuch,float,
+    generators,go,hexiom,json_dumps,json_loads,logging,mdp,nbody,nqueens,
+    pickle_pure_python,raytrace,regex_compile,regex_v8,richards,scimark,
+    spectral_norm,telco,typing_runtime_protocols,unpack_sequence,
+    unpickle_pure_python,xml_etree
+  # A benchmark that needs longer than this on the base commit is one this
+  # comparison cannot afford to wait for; it shows up as a failure in the
+  # table, which is itself the interesting signal.
+  BENCH_TIMEOUT: 300
+
+jobs:
+  compare:
+    # `skip:ci` mirrors the other workflows; `run:pyperformance` opts a PR in,
+    # because the job costs well over an hour and most PRs do not need it.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (!contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
+       contains(github.event.pull_request.labels.*.name, 'run:pyperformance'))
+    name: base vs. head vs. CPython
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    permissions:
+      contents: read
+      # Honoured for a pull request opened from this repo, which is then
+      # commented on directly by the last step; a fork's run is handed
+      # read-only whatever is asked for here and goes the artifact route.
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      # Both the third interpreter under test and the host pyperformance
+      # itself runs on (it only runs under a real CPython; RustPython is just
+      # the `--python` target it drives).
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        id: cpython
+        with:
+          python-version: "3.14"
+
+      - name: Restore cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: pyperformance-
+        continue-on-error: true
+
+      # The base commit may predate these scripts, and `git checkout <base>`
+      # below would then delete them; run them from a copy outside the tree.
+      - name: Copy the benchmark scripts out of the checkout
+        run: cp -r scripts/pyperformance "${RUNNER_TEMP}/pyperformance-scripts"
+
+      # Each build is copied aside with the stdlib of its own commit, so the
+      # second does not overwrite the first and a PR that touches `Lib/` is
+      # still compared against the base commit's `Lib/`. A binary moved away
+      # from its checkout finds the stdlib only through RUSTPYTHONPATH, which
+      # `run_all.py` forwards through pyperf into the venv it builds.
+      - name: Build head
+        run: |
+          cargo build --release --locked
+          mkdir -p "${RUNNER_TEMP}/head"
+          cp target/release/rustpython "${RUNNER_TEMP}/head/rustpython"
+          cp -r Lib "${RUNNER_TEMP}/head/Lib"
+
+      - name: Build base
+        continue-on-error: true # a base that cannot build leaves head vs. CPython
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git checkout --quiet "${BASE_SHA}" --
+          cargo build --release --locked
+          mkdir -p "${RUNNER_TEMP}/base"
+          cp target/release/rustpython "${RUNNER_TEMP}/base/rustpython"
+          cp -r Lib "${RUNNER_TEMP}/base/Lib"
+
+      - name: Restore head checkout
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: git checkout --quiet "${HEAD_SHA}" --
+
+      - name: Save cache
+        if: always()
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
+        continue-on-error: true
+
+      # The three runs are deliberately adjacent, with no build between them,
+      # so nothing but the interpreter differs across the comparison.
+      - name: Measure CPython
+        env:
+          CPYTHON: ${{ steps.cpython.outputs.python-path }}
+        run: |
+          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
+            --python "${CPYTHON}" \
+            --label cpython --no-psutil-stub \
+            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
+            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
+
+      - name: Measure base
+        continue-on-error: true # as above: the table falls back to head alone
+        env:
+          RUSTPYTHONPATH: ${{ runner.temp }}/base/Lib
+        run: |
+          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
+            --python "${RUNNER_TEMP}/base/rustpython" \
+            --label base \
+            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
+            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
+
+      - name: Measure head
+        env:
+          RUSTPYTHONPATH: ${{ runner.temp }}/head/Lib
+        run: |
+          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
+            --python "${RUNNER_TEMP}/head/rustpython" \
+            --label head \
+            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
+            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
+
+      - name: Render the comparison
+        env:
+          CPYTHON_VERSION: ${{ steps.cpython.outputs.python-version }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          python3 "${RUNNER_TEMP}/pyperformance-scripts/pr_diff.py" \
+            --results-dir "${RUNNER_TEMP}/pyperf-results" \
+            --cpython-version "CPython ${CPYTHON_VERSION}" \
+            --md diff.md --json diff.json
+          echo "${PR_NUMBER}" > pr_number.txt
+
+      # A `pull_request` run triggered from a fork carries a read-only token no
+      # matter what this workflow grants, so the comment is posted from
+      # `pyperformance-comment.yaml`, which `workflow_run` runs in the base
+      # repo's context. This artifact is the hand-off.
+      - name: Upload the comparison
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: pyperformance-diff
+          path: |
+            diff.md
+            diff.json
+            pr_number.txt
+          retention-days: 14
+          if-no-files-found: ignore
+
+      # A pull request opened from this repo can be commented on from here,
+      # which puts the table on the PR in the same run instead of waiting for
+      # the companion workflow. Forks fall through to that workflow instead.
+      - name: Post or update the PR comment
+        if: >-
+          github.event_name == 'pull_request' &&
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('diff.md', 'utf8');
+            const marker = '<!-- pyperformance-local-diff -->';
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.pull_request.number,
+            });
+            const existing = comments.find((c) => c.body && c.body.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.payload.pull_request.number,
+                body,
+              });
+            }

.github/workflows/release.yml

@@ -52,7 +52,7 @@ jobs:
           #   target: aarch64-pc-windows-msvc
       fail-fast: false
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -60,6 +60,13 @@ jobs:
         with:
           target: ${{ matrix.target }}
 
+      - name: Use a short Cargo home on Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV
+          git config --global core.longpaths true
+
       - name: Install macOS dependencies
         uses: ./.github/actions/install-macos-deps
         with:
@@ -68,7 +75,7 @@ jobs:
           libtool: true
 
       - name: Build RustPython
-        run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls-aws-lc,threading,jit
+        run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls-aws-lc,threading,jit,doc
 
       - name: Rename Binary
         run: cp target/${{ matrix.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.target }}
@@ -91,7 +98,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
@@ -100,7 +107,7 @@ jobs:
           targets: wasm32-wasip1
 
       - name: Build RustPython
-        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release
+        run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib,doc --release
 
       - name: Rename Binary
         run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm
@@ -114,7 +121,7 @@ jobs:
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
 
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           package-manager-cache: false
 
@@ -156,7 +163,7 @@ jobs:
     permissions:
       contents: write # for creating a release
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 

.github/workflows/update-caches.yml

@@ -19,7 +19,8 @@ env:
   CARGO_PROFILE_TEST_DEBUG: 0
   CARGO_PROFILE_DEV_DEBUG: 0
   CARGO_PROFILE_RELEASE_DEBUG: 0
-  CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher
+  # Keep feature list in sync with CI's release build in .github/workflows/ci.yaml.
+  CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit,doc --exclude rustpython_wasm --exclude rustpython-venvlauncher
 
 jobs:
   build-caches:
@@ -39,39 +40,49 @@ jobs:
             target: ""
     steps:
       - name: Checkout RustPython main branch
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           repository: RustPython/RustPython
           ref: main
           persist-credentials: false
 
       - name: Setup Rust
-        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
+        uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ matrix.toolchain }}
           target: ${{ matrix.target }}
 
+      - name: Use a short Cargo home on Windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV
+          git config --global core.longpaths true
+
       - name: Install macos dependencies
         uses: ./.github/actions/install-macos-deps
         with:
           openssl: true
 
       - name: Build dev cache # dev profile used by check & doc
-        run: cargo build --profile dev ${{ env.CARGO_ARGS }}
+        run: cargo build --locked --profile dev ${{ env.CARGO_ARGS }}
 
       - name: Build test cache
-        run: cargo build --profile test ${{ env.CARGO_ARGS }}
+        run: cargo build --locked --profile test ${{ env.CARGO_ARGS }}
 
       - name: Build release cache
-        run: cargo build --profile release ${{ env.CARGO_ARGS }}
+        run: cargo build --locked --profile release ${{ env.CARGO_ARGS }}
 
       - name: Save cache
-        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
+            ${{ runner.temp }}/c/registry/index/
+            ${{ runner.temp }}/c/registry/cache/
+            ${{ runner.temp }}/c/git/db/
             target/
           key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }}

.github/workflows/update-doc-db.yml

@@ -8,7 +8,7 @@ on:
       python-version:
         description: Target python version to generate doc db for
         type: string
-        default: "3.14.3"
+        default: "3.14.7"
       base-ref:
         description: Base branch to create the update branch from
         type: string
@@ -30,13 +30,13 @@ jobs:
           - windows-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           sparse-checkout: |
             crates/doc
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ inputs.python-version }}
 
@@ -58,7 +58,7 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: true
           ref: ${{ inputs.base-ref }}
@@ -82,8 +82,8 @@ jobs:
           # Merge all artifacts
           jq -s "add" --sort-keys crates/doc/generated/*.json > crates/doc/generated/merged.json
 
-          # Format merged json for the phf macro
-          jq -r 'to_entries[] | "    \(.key | @json) => \(.value | @json),"' crates/doc/generated/merged.json > crates/doc/generated/raw_entries.txt
+          # Format merged json as a sorted const slice
+          jq -r 'to_entries[] | "    (\(.key | @json), \(.value | @json)),"' crates/doc/generated/merged.json > crates/doc/generated/raw_entries.txt
 
           OUTPUT_FILE='crates/doc/src/data.inc.rs'
 
@@ -93,9 +93,10 @@ jobs:
             echo "// CPython version: ${PYTHON_VERSION}"
             echo '// spell-checker: disable'
             echo ''
-            echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {"
+            echo '#[rustfmt::skip]'
+            echo 'pub const DB: &[(&str, &str)] = &['
             cat crates/doc/generated/raw_entries.txt
-            echo '};'
+            echo '];'
           } > "$OUTPUT_FILE"
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/update-libs-status.yaml

@@ -21,7 +21,7 @@ jobs:
     if: ${{ github.repository == 'RustPython/RustPython' }}
     steps:
       - name: Clone RustPython
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           path: rustpython
           persist-credentials: false
@@ -37,7 +37,7 @@ jobs:
           echo "version=${version}" >> "$GITHUB_OUTPUT"
 
       - name: Clone CPython
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           repository: python/cpython
           path: cpython

.github/workflows/upgrade-pylib.lock.yml

@@ -58,7 +58,7 @@ jobs:
       comment_repo: ""
     steps:
       - name: Setup Scripts
-        uses: github/gh-aw/actions/setup@f990bbb7eb83981a203d4b5eccdc24f677e950c7 # v0.77.5
+        uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17
         with:
           destination: /opt/gh-aw/actions
       - name: Check workflow file timestamps
@@ -99,22 +99,22 @@ jobs:
       secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
     steps:
       - name: Setup Scripts
-        uses: github/gh-aw/actions/setup@f990bbb7eb83981a203d4b5eccdc24f677e950c7 # v0.77.5
+        uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17
         with:
           destination: /opt/gh-aw/actions
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
       - name: Setup Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.14'
       - name: Create gh-aw temp directory
         run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh
       # Cache configuration from frontmatter processed below
       - name: Cache (cpython-lib-${{ env.PYTHON_VERSION }})
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           key: cpython-lib-${{ env.PYTHON_VERSION }}
           path: cpython
@@ -806,7 +806,7 @@ jobs:
       total_count: ${{ steps.missing_tool.outputs.total_count }}
     steps:
       - name: Setup Scripts
-        uses: github/gh-aw/actions/setup@f990bbb7eb83981a203d4b5eccdc24f677e950c7 # v0.77.5
+        uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17
         with:
           destination: /opt/gh-aw/actions
       - name: Download agent output artifact
@@ -927,7 +927,7 @@ jobs:
       success: ${{ steps.parse_results.outputs.success }}
     steps:
       - name: Setup Scripts
-        uses: github/gh-aw/actions/setup@f990bbb7eb83981a203d4b5eccdc24f677e950c7 # v0.77.5
+        uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17
         with:
           destination: /opt/gh-aw/actions
       - name: Download agent artifacts
@@ -1039,7 +1039,7 @@ jobs:
       process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
     steps:
       - name: Setup Scripts
-        uses: github/gh-aw/actions/setup@f990bbb7eb83981a203d4b5eccdc24f677e950c7 # v0.77.5
+        uses: github/gh-aw/actions/setup@00457477720387bcc7d7baaf841dedb22ad06617 # v0.89.17
         with:
           destination: /opt/gh-aw/actions
       - name: Download agent output artifact
@@ -1061,7 +1061,7 @@ jobs:
           path: /tmp/gh-aw/
       - name: Checkout repository
         if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request'))
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           token: ${{ github.token }}
           persist-credentials: false

.github/workflows/upgrade-pylib.md

@@ -52,7 +52,7 @@ cache:
     - cpython-lib-
 
 env:
-  PYTHON_VERSION: "v3.14.6"
+  PYTHON_VERSION: "v3.14.7"
   ISSUE_ID: "6839"
 ---
 

.github/zizmor.yml

@@ -1,4 +1,19 @@
 rules:
+  dangerous-triggers:
+    ignore:
+      # pull_request_target is needed to label and assign PRs from forks with issues: write.
+      # The workflow does not check out or execute pull request code.
+      - oscca-pr.yml:3
+      # workflow_run is needed to post a PR comment from a fork's run, which is
+      # handed a read-only token however the triggering workflow is configured.
+      # The workflow does not check out or execute pull request code: it reads
+      # one artifact and writes a comment.
+      - pyperformance-comment.yaml:1
+  excessive-permissions:
+    ignore:
+      # pull_request_target is needed to post PR comments with pull-requests: write.
+      # Workflow-level permissions: {} restricts defaults; only the job has write access.
+      - lib-deps-check.yaml:3
   unpinned-uses:
     config:
       policies:

.gitignore

@@ -8,12 +8,15 @@ __pycache__/
 .repl_history.txt
 .vscode/
 wasm-pack.log
-.idea/
+.idea/*
+!.idea/icon.svg
+!.idea/vcs.xml
 .envrc
 
 flame-graph.html
 flame.txt
 flamescope.json
+/build/codspeed-environment*.json
 /wapm.lock
 /wapm_packages
 /.cargo/config
@@ -27,4 +30,9 @@ Lib/site-packages/*
 Lib/test/data/*
 !Lib/test/data/README
 cpython/
-.claude/scheduled_tasks.lock
+.claude/
+docs/superpowers/
+
+scripts/pyperformance/results/
+scripts/pyperformance/stub_psutil/build/
+scripts/pyperformance/stub_psutil/src/psutil.egg-info/

.idea/icon.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512">
+  <image href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAAAXNSR0IArs4c6QAAAJ5lWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAAEsAAAAAQAAASwAAAABAASShgAHAAAAGQAAAISgAQADAAAAAQABAACgAgAEAAAAAQAAAICgAwAEAAAAAQAAAIAAAAAAQVNDSUkAAABDcmVhdGVkIHdpdGggR0lNUADcIp6GAAAACXBIWXMAAC4jAAAuIwF4pT92AAABaWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PkNyZWF0ZWQgd2l0aCBHSU1QPC9leGlmOlVzZXJDb21tZW50PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KLa3NoAAAHSZJREFUeAHtXQtgVMXVPnPv3VcSQhDkJaAIiDZAAmilgBosAiLa2haK9rf8WgsERdu/Pn7tX01/sb9FW62Wd/vbap9Sq/IIlFZFAa0PIIBR8VF5+EAeEkIem929d/qdu9llEzZmc+/d7G6yA5O9j5m5M+ecOXPmzJkzRNmQhUAWAlkIZCHQSSEgOmm7U9rso1+lggJJ/WuDdCivnA6ksjJKKj/e2b4tiYT/MprfRaetAYMqNIW2+y+nFbVXUN9UwSLLAdoJ8nI6nRJsoEUuopk6vhkyiFRAX8MDf4Ce9dbQFLGRQu1UnehntOhV9sI2BD4oIW+ffDpdSBomJZ2JAs+UgnwkSfP7aahXoXMbgH0X+K5HBeKZEoL0Fuhgm+2PWywgywEsAo6zoVergQYqFAZdYAi6CIgejscD3ArliGaQNcD/g+j1eF6Py3/i9QbdoBd9ObRdrKR6G9WwlTXLAdoIvs8mUtcuHjoPCJ/kr6cSdOYRLpU8XAwQStypG/AbG5gWmCAURLyqBetfj/F/uVhNVbHpUnHNdcuGBCBQN5XO11T6Nlj7FCDzTGbjkV6NHm0GHtM5MqI58Hs9HDlJHWKuhnccQSSHcb9DVWgd0q11r6bdeBwpCq/aJzRWtX0+lolfOT6Jerq8tECRNAtId7PwxkhlTDHw0JNNpHPvD0o6gkfv48UuyAEV+N2DLu8Hgo8j2VFS6Tq3oNs5LwiJgHwUAFmggTZCCLwkFUJglgCAg5ZCzVQqdqv0OBA/LADezpyde7cLkVk6BLoQft4GLp/Fqw26QhU5q+gTPGMcxw310+hacInvgyjOQFn8b2dQp5ty19LWuBmS/BB1zYZ4EKi+jMb6FFqJHt6XpXUGFEvuIIQ6IP+1kKRyvPu7qxpS/EbyxyujpWcfYLbQy0c9fD6i9+rp0JB11NBS2mQ/zxJAHAjXAfmQ5J8Gok9lgY57PLq0BNt+HGzgAU857YqTLSMfZQmgGdqglRvphlAG9tyHke8JC3tHoaG50bea/tAsecbfZgkgBoVyIg3QffQcxuhBPOazpI+uv08EaIbYQK/EJO0wl51GDzB4/sOerg0er6KFfEFD8SqKSzN0luHCgSfylYcfLuslqgfVobsrGO+hxat9yXXmnXNPv/1A8cD9Qzil1BSMDkZ3SPk9pCHdoBMzoCCWEY/hd3fF0tI94afp/zcKgPSvauI1LJy3KE9TPUMUPTSKDFEsFTkUPbkPSsgHCiF6CTeugWJTtotI7KJA6rmRG/5akJRgtaLValKPdBSGF197BFMITwVigzRYUKjCn81CqCt1I7R+59LSg7FJ0u26WQvSrXqJ12fY9Q/1Ut3eEiHFNCDmi0DCQEXjpRZhSm8E5DA/D/+PRfOJb7DipnmIKHWaPDfTxUnMiUAUiqLxZ8AT9D0GGf+9Y/HcP/NtOoaMJgBm610M93iSytWS5GShaKcJIEAaOpAOhLO2pd0DCAB0J3WMI1wXKRtIhq6rWDIvLQXIjCSA0bOXDQBTngncXoVYrKhhgJtIb3eEx3wQCDeCATq063nqNng0efJPDQ8TUj+iG7IEw8EbManT4jIytqVFZVqrxIj5K/opujFfJzFLUbVejHDuaUYo0FrWdnnPvUkaIao98D516X8OBIUwN1I0d3dFBm7F61ntUpE2fCQjOAALdS6plQpFfA9svi8jPeW9vUUgQ7vLiDeFjcYhSCggDONTkMfoHUtLP2oxa/wXp+PxeYjPIh6Nn8T6U57ppnUYNW/p19zCtRnj6kJUtC/39vRFPoMSqG8uf+AexNtLy8k/KwbY5mwC9zwDjY0QXKOBucYOxJWIY6JPHbxI2yFg1LxFgzFb+wnAOR3QSxs2bxX2PDP4dPv6W5D/q4hsAzgU0YsYK6kyRw4ibkbMQ7wacSPizxCZAzge0pIARs5dMtMQ6v2KovaT5vjOU7gMDxgWGqoOTW1sBVsA7UVkG4HYYZiJgYniuzHP2WqYl5mTEmI/npQPtKXQ0bMf6KGrXe4VimICgKdzmRhQf57+oW/HdG5wsZr9b13/3uoHt6BNxxFZQRSPsllBdQbiWMRZiCWIHO5CvMe8cvBP2hDAiNnLzlc1sVSoWrERYi4YAzwHG9weRfmrDpLmzSXNBy7ORACCIMP4JCe3Z/FLD3ytrZrBYtSZOcdziP90uv5MbSkPI+cs/YZwq38SQhkodUZ+JgdJe/7xGzB2hfJ6n8nSP9YVXMB/aO3rD3/7MQst4yFgE+KHFvK2miXlMkDx3CXfEqr6K3R4L8+hMz8IGjDhGoJaGoLrifYohngyHduWUgIoLl02HdOjFZD0gfx4w2E6gqz1OnnyuvFk0GT/UALxDGZDUNXXt56z/VOkTAYowvxeESpYosztSMg3UQj2r6gq2D8Tgb460KD+Z+Wvr/+s/dHb+hdTwgFGli6+AmPkb1G9dkI+tHO8rCdUU0sXDyxs/cPWvrGBrYHYzPtzA4S8qGKKBb6w+veoocsXsDK5XNHpH0B+2go2rTXvc9tu5eXI0uUXEeztAKmCZE/zoDYG4lmJFIRth2Qh6l189yNw52NSyKghJ/BM02u3zehOdQPY7Bs4hDZG+Nd4C3+/z93zoNcInNDOSaEiL1uD1xlSBpB0P6jkoNSlrphlai4p9bcyxSikXQlgVOniEYbQyoGU08zlUisU1FoeYM+UusNTye1o4NNSKP9wi8DuVxfd2KJCRU6i66CQ/XV0Zs6QkdDIVdMNYhPtbO2zmfq+3QigcPYvBrg133pg55ykTPUiiNeDVdCtPQNu/NtQz25bKstmJLRUyMwbNvv3+VS6jTdw8miAzZy82aMKQ/lPPAFaDLvA2kxFdEv1bhcCGHHLY7lqbd0qobkvdn7pNmqAcQzat0dVnZZtXT7n7ZYa3NpzEME9MPr7n8YtXaYMwMahIIQK7A76JRQnz2BPH2/r6hChXQiguHTJYkXzlDqNfNMuj+dbUv7ZUGjBjkWzK53ACpw4zMa2rfsx0OebW7hRKIjC3BUEYXE/bjfAKHQ9Nohu9X5MH4qt5gJOq5+uv5wGQNXRHXsDQ9Ua7Tl1lakSbjVfMhMknQCK5yyah56/KDzVayZm22iZOb/Wg/8Shrht+9LZjitZApfTGIwqCzFNuoBXJHhrNwdzcyeIgWd4eFaN4WYvSPAVpF2PrWGv5jxjEkg4cePf6ql0lkeje8FBvowdRV2QVoKTvA+ucmfOGnqqSeJ2vkkqAYy8YemXpFQ24CN50amSzQZCXYzRmhdbjD8aqrht5yPfZek+KeHdS8nT30WzgLjvA/FnM7ACQHzEeJRnljxN5MjPgNwj+NmB2xdgAvASCOIz1aD/AMKvwh7D3nALYy4NcGWZoyCPH0QwzbcmOUu9iQAFdU1OKPzOr05xu/XnSVVHOCXxmyyfZDUULLdWLJmzPDk1P7nUI5dSfp5G0/DmGsTxQB6v1UeQHiUIJgTmEEC4yTEgR+helVTmHixT8LuIlTEuSWByCQcTP/KsoQVcXioCOFxyguYOlYH1j3Bq3GfkQ8g7igF/5o4lczYkp9bxS+2+DqyezG1hf/CDnUOrMwGzjLFQLgzH8yFAskkQrEMwezkemggGc4jIEHwPDvEOhgvmEm4QSbUrRHtRzna8SlngejkeoOmbSIqrHGwa5rpors0QFvZktdCD39y27Ia00amzi5iGWhoE2pyAVjJRfBFNPcMDhSM3mZsOpJu9n7kDBMeXsBX8BznrnF/WtQpixwmg+HuPFlCDf5NQXcOcYP2syUOoEdK4etviuautNrQ98kkMFfUqFYLNjwHyR+Kbg0AKAyHw9eEhgFXLIIgaAP2X9XX0QP6zybP0SbS9zhPA3CX3Ki7PnY6wfgh7sLBt0EP6tTuXzf1joo1Kp3SY+l2E3v934N7FsgDLACwAQvh7B15F5qVSAGQ4md3LKYDBsOML6LE3Gk4YdWCQBPIxihg/yFTkM1yxpfwF9PpZ4AiHvJC4GPk8NLg0OgvywPlOwd5qOU4KgQLKmLtU1ZUfNuS0WqVwPraiBRf5PyyqLLJXUupz56ymP/qvoNdg4vhNIH80uEA32LruPJJDD6a6do4RAAS/C2H79vWwPZ+9ZoWVPIEnCnp/ere9ktInt3cVvYfa3Js+NQrXxBkCKCLine truncated
+</svg>

.idea/vcs.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="IssueNavigationConfiguration">
+    <option name="links">
+      <list>
+        <IssueNavigationLink>
+          <option name="issueRegexp" value="#(\d+)" />
+          <option name="linkRegexp" value="https://github.com/RustPython/RustPython/pull/$1" />
+        </IssueNavigationLink>
+      </list>
+    </option>
+  </component>
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
         priority: 0
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.15
+    rev: v0.16.8
     hooks:
       - id: ruff-format
         priority: 0
@@ -42,7 +42,7 @@ repos:
 
       - id: generate-rs-opcode-metadata
         name: generate rust opcode metadata
-        entry: python tools/opcode_metadata/generate_rs_opcode_metadata.py
+        entry: python3 tools/opcode_metadata/generate_rs_opcode_metadata.py
         files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
         pass_filenames: false
         language: system
@@ -53,7 +53,7 @@ repos:
 
       - id: generate-py-opcode-metadata
         name: generate python opcode metadata
-        entry: python tools/opcode_metadata/generate_py_opcode_metadata.py
+        entry: python3 tools/opcode_metadata/generate_py_opcode_metadata.py
         files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$'
         pass_filenames: false
         language: system
@@ -63,7 +63,7 @@ repos:
           - manual
 
   - repo: https://github.com/streetsidesoftware/cspell-cli
-    rev: v10.0.0
+    rev: v10.2.0
     hooks:
       - id: cspell
         types: [rust]
@@ -77,7 +77,7 @@ repos:
         priority: 0
 
   - repo: https://github.com/rbubley/mirrors-prettier
-    rev: v3.8.3
+    rev: v3.9.8
     hooks:
       - id: prettier
         files: '^wasm/.*$'

.python-version

@@ -1 +1 @@
-3.14.6
+3.14.7

AGENTS.md

@@ -1,6 +1,6 @@
-# GitHub Copilot Instructions for RustPython
+# AI Agent Instructions for RustPython
 
-This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project.
+This document provides guidelines for AI coding agents (GitHub Copilot, Claude Code, Gemini, etc.) contributing to the RustPython project.
 
 ## Project Overview
 
@@ -13,36 +13,26 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0
 
 ## Repository Structure
 
-- `src/` - Top-level code for the RustPython binary
-- `vm/` - The Python virtual machine implementation
-  - `builtins/` - Python built-in types and functions
-  - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core
-- `compiler/` - Python compiler components
-  - `parser/` - Parser for converting Python source to AST
-  - `core/` - Bytecode representation in Rust structures
-  - `codegen/` - AST to bytecode compiler
-- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython.
-- `derive/` - Rust macros for RustPython
-- `common/` - Common utilities
-- `extra_tests/` - Integration tests and snippets
-- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality)
-- `wasm/` - WebAssembly support
-- `jit/` - Experimental JIT compiler implementation
-- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically)
+See the "Code organization" section in [CONTRIBUTING.md](CONTRIBUTING.md#code-organization) for the current directory layout.
 
 ## AI Agent Rules
 
+**CRITICAL: AI Policy**
+
+- Follow RustPython's [AI Policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md) for every AI-assisted contribution.
+- Disclose AI assistance in commit messages with an `Assisted-by: AGENT_NAME:MODEL_VERSION` trailer. Use one trailer per AI tool, and never use `Co-authored-by` for an AI assistant.
+
 **CRITICAL: Git Operations**
 - NEVER create pull requests directly without explicit user permission
 - NEVER push commits to remote without explicit user permission
 - Always ask the user before performing any git operations that affect the remote repository
 - Commits can be created locally when requested, but pushing and PR creation require explicit approval
 
-**CRITICAL: Pre-commit Checks**
-- Before creating ANY commit, you MUST run `prek run --all-files` (or `pre-commit run --all-files`) AND the full test suite. Both must pass — do not commit if either fails.
-- Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test <module>` modules.
-- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes, re-run `prek` until it reports a clean pass, then re-run the tests, then commit.
-- NEVER bypass these checks with `--no-verify`, `--no-gpg-sign`, or by skipping tests "because the change is small". If a hook or test fails, fix the underlying issue and create a new commit — do not amend or force the failing commit through.
+**CRITICAL: Commit Hooks and Validation**
+- Install the repository's pre-commit hook with `prek install` (or `pre-commit install`) after cloning the repository.
+- Every commit must run the configured pre-commit hook. NEVER bypass it with `--no-verify`. Automated workflows that use a normal `git commit`, such as `scripts/update_lib quick`, should be allowed to create local commits through the hook.
+- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes and retry the commit. Do not amend or force a failing commit through.
+- Before completing a task, run the tests appropriate for the change. Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi`, then run `cargo test` from `crates/capi`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test <module>` modules.
 
 ## Important Development Notes
 
@@ -128,7 +118,14 @@ rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r
 
 ```bash
 # Run Rust unit tests
-cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher
+cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi
+
+# Run C-API tests from their directory so their separate Cargo config applies
+(cd crates/capi && cargo test)
+
+# Run Clippy with the same C-API separation
+cargo clippy --workspace --all-targets --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi
+(cd crates/capi && cargo clippy --all-targets)
 
 # Run Python snippets tests (debug mode recommended for faster compilation)
 cargo run -- extra_tests/snippets/builtin_bytes.py
@@ -164,7 +161,7 @@ Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is h
 
 - Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code.
 - Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value.
-- Do not put `///` doc comments on items annotated with `#[pyattr]`, `#[pyclass]`, or `#[pyfunction]`. The derive macros pull authoritative docstrings from CPython via the `rustpython-doc` crate; a Rust doc comment overrides that source, and on `#[pyattr]` it is silently dropped.
+- A `///` doc comment on a Python-exposed item (`#[pyattr]`, `#[pyclass]`, `#[pyfunction]`, `#[pymethod]`, `#[pyclassmethod]`, `#[pystaticmethod]`, `#[pygetset]`, `#[pymember]`, `#[pyslot]`, `#[pyexception]`, `#[pymodule]`, `#[pystruct_sequence]`) is that item's docstring and overrides the stored docs in `rustpython-doc` (`crates/doc`). Those stored docs are used only when the item has no `///` comment. Developer notes on these items must be `//` comments.
 
 #### Avoid Duplicate Code in Branches
 
@@ -265,12 +262,10 @@ cargo run --features jit
 
 ### Linux Build and Debug on macOS
 
-See the "Testing on Linux from macOS" section in [DEVELOPMENT.md](DEVELOPMENT.md#testing-on-linux-from-macos).
+See the "Testing on Linux from macOS" section in [CONTRIBUTING.md](CONTRIBUTING.md#testing-on-linux-from-macos).
 
 ### Building venvlauncher (Windows)
 
-See DEVELOPMENT.md "CPython Version Upgrade Checklist" section.
-
 **IMPORTANT**: All 4 venvlauncher binaries use the same source code. Do NOT add multiple `[[bin]]` entries to Cargo.toml. Build once and copy with different names.
 
 ## Test Code Modification Rules
@@ -301,7 +296,7 @@ If you modify any file under `.github/workflows/`, the change must pass a [zizmo
 ## Documentation
 
 - Check the [architecture document](/architecture/architecture.md) for a high-level overview
-- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions
+- Read the [development guide](/CONTRIBUTING.md) for detailed setup instructions
 - Generate documentation with `cargo doc --no-deps --all`
 - Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/)
 - [How to update test files](https://github.com/RustPython/RustPython/wiki/How-to-update-test-files#checkout-cpython-source-code-initial-setup) — guide for syncing test cases from upstream CPython into the `Lib/` directory

CONTRIBUTING.md

@@ -89,7 +89,15 @@ $ pytest -v
 Rust unit tests can be run with `cargo`:
 
 ```shell
-$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher
+$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi
+```
+
+`rustpython-capi` needs to be tested from inside its own directory, since it has a
+separate `cargo` config that only applies there:
+
+```shell
+$ cd crates/capi
+$ cargo test
 ```
 
 Python unit tests can be run by compiling RustPython and running the test module:

Cargo.toml

@@ -11,7 +11,9 @@ license.workspace = true
 
 [features]
 capi = ["dep:rustpython-capi", "threading"]
-default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls-aws-lc", "host_env"]
+default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls-aws-lc", "host_env", "mimalloc", "doc"]
+doc = ["rustpython-vm/doc", "rustpython-stdlib?/doc"]
+mimalloc = ["dep:mimalloc"]
 host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
@@ -20,9 +22,10 @@ stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"]
 flame-it = ["rustpython-vm/flame-it", "rustpython-stdlib/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
+opcode-histogram = ["rustpython-vm/opcode-histogram"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
 sqlite = ["rustpython-stdlib/sqlite"]
-ssl = ["host_env"]
+ssl = ["host_env", "rustpython-stdlib?/ssl"]
 ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"]
 ssl-rustls-aws-lc = ["ssl-rustls", "dep:rustls", "rustls/aws_lc_rs"]
 ssl-rustls-aws-lc-fips = ["ssl-rustls-aws-lc", "rustls/fips"]
@@ -37,7 +40,7 @@ winresource = "0.1"
 rustpython-capi = { workspace = true, optional = true }
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
-rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
+rustpython-stdlib = { workspace = true, optional = true, default-features = false, features = ["compiler"] }
 rustpython-vm = { workspace = true, features = ["compiler", "gc"] }
 
 log = { workspace = true }
@@ -49,17 +52,18 @@ env_logger = "0.11"
 flamescope = { version = "0.1.2", optional = true }
 
 rustls = { workspace = true, optional = true }
-rustls-graviola = { workspace = true, optional = true }
 
 [target.'cfg(windows)'.dependencies]
 libc = { workspace = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 rustyline = { workspace = true }
+mimalloc = { version = "0.1", optional = true, default-features = false }
 
 [dev-dependencies]
 criterion = { workspace = true }
 pyo3 = { workspace = true, features = ["auto-initialize"] }
+rustls-graviola = { workspace = true }
 rustpython-stdlib = { workspace = true }
 ruff_python_parser = { workspace = true }
 
@@ -79,7 +83,6 @@ path = "src/main.rs"
 name = "custom_tls_providers"
 path = "examples/custom_tls_providers.rs"
 required-features = [
-    "rustls-graviola",
     "rustls/ring",
     "rustpython-pylib/freeze-stdlib",
     "rustpython-stdlib/ssl-rustls",
@@ -158,73 +161,61 @@ members = [
 exclude = ["pymath"]
 
 [workspace.package]
-version = "0.5.0"
+version = "0.6.1"
 authors = ["RustPython Team"]
 edition = "2024"
-rust-version = "1.95.0"
+rust-version = "1.99.0"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
-rustpython-capi = { path = "crates/capi", version = "0.5.0" }
-rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" }
-rustpython-compiler = { path = "crates/compiler", version = "0.5.0" }
-rustpython-codegen = { path = "crates/codegen", version = "0.5.0" }
-rustpython-common = { path = "crates/common", version = "0.5.0" }
-rustpython-host_env = { path = "crates/host_env", version = "0.5.0" }
-rustpython-derive = { path = "crates/derive", version = "0.5.0" }
-rustpython-derive-impl = { path = "crates/derive-impl", version = "0.5.0" }
-rustpython-jit = { path = "crates/jit", version = "0.5.0" }
-rustpython-literal = { path = "crates/literal", version = "0.5.0" }
-rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0" }
-rustpython-pylib = { path = "crates/pylib", version = "0.5.0" }
-rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" }
-rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" }
-rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" }
-rustpython-doc = { path = "crates/doc", version = "0.5.0" }
+rustpython-capi = { path = "crates/capi", version = "0.6.1" }
+rustpython-compiler-core = { path = "crates/compiler-core", version = "0.6.1" }
+rustpython-compiler = { path = "crates/compiler", version = "0.6.1" }
+rustpython-codegen = { path = "crates/codegen", version = "0.6.1" }
+rustpython-common = { path = "crates/common", version = "0.6.1" }
+rustpython-host_env = { path = "crates/host_env", version = "0.6.1" }
+rustpython-derive = { path = "crates/derive", version = "0.6.1" }
+rustpython-derive-impl = { path = "crates/derive-impl", version = "0.6.1" }
+rustpython-jit = { path = "crates/jit", version = "0.6.1" }
+rustpython-literal = { path = "crates/literal", version = "0.6.1" }
+rustpython-vm = { path = "crates/vm", default-features = false, version = "0.6.1" }
+rustpython-pylib = { path = "crates/pylib", version = "0.6.1" }
+rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.6.1" }
+rustpython-sre_engine = { path = "crates/sre_engine", version = "0.6.1" }
+rustpython-unicode = { path = "crates/unicode", version = "0.6.1" }
+rustpython-wtf8 = { path = "crates/wtf8", version = "0.6.1" }
+rustpython-doc = { path = "crates/doc", version = "0.6.1" }
 
-# Use RustPython-packaged Ruff crates from the published fork while keeping
-# existing crate names in the codebase.
-ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.15.8" }
-ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.15.8" }
-ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.15.8" }
-ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.15.8" }
-# To update ruff crates, comment out the above lines and uncomment the following lines to pull directly from the Ruff repository at the specified commit hash.
-# Ruff tag 0.15.8 is based on commit c2a8815842f9dc5d24ec19385eae0f1a7188b0d9
-# at the time of this capture. We use the commit hash to ensure reproducible builds.
-# ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
-# ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
-# ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
-# ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "c2a8815842f9dc5d24ec19385eae0f1a7188b0d9" }
+# Use the RustPython Ruff fork for RustPython public `_ast` metadata.
+ruff_python_parser = { package = "rustpython-ruff_python_parser", version = "0.16.5" }
+ruff_python_ast = { package = "rustpython-ruff_python_ast", version = "0.16.5" }
+ruff_text_size = { package = "rustpython-ruff_text_size", version = "0.16.5" }
+ruff_source_file = { package = "rustpython-ruff_source_file", version = "0.16.5" }
 
 der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] }
-phf = { version = "0.13.1", default-features = false, features = ["macros"]}
+phf = { version = "0.14.0", default-features = false, features = ["macros"]}
 adler32 = "1.2.0"
 approx = "0.5.1"
-ascii = "1.1"
-base64 = "0.22"
-blake2 = "0.11.0-rc.6"
+ascii = { version = "1.1", default-features = false }
+base64 = "0.23"
+blake2b_simd = "1.0.4"
+blake2s_simd = "1.0.4"
 bitflags = "2.11.0"
-bitflagset = "0.0.3"
-bstr = "1"
+bitflagset = "0.0.4"
+bstr = { version = "1", default-features = false, features = ["unicode"] }
 bzip2 = "0.6"
-chrono = { version = "0.4.44", default-features = false, features = ["clock", "std"] }
 console_error_panic_hook = "0.1"
-constant_time_eq = "0.5"
-cranelift = "0.132.0"
-cranelift-jit = "0.132.0"
-cranelift-module = "0.132.0"
+constant_time_eq = "0.6"
+cranelift = "0.135.0"
+cranelift-jit = "0.135.0"
+cranelift-module = "0.135.0"
 crc32fast = "1.3.2"
-criterion = { version = "0.8", features = ["html_reports"] }
+criterion = { package = "codspeed-criterion-compat", version = "5.0.1", features = ["html_reports"] }
 crossbeam-utils = "0.8.21"
-csv-core = "0.1.11"
-digest = "0.11"
-dns-lookup = "3.0"
-dyn-clone = "1.0.10"
-exitcode = "1.1.2"
+dns-lookup = "4.0"
 flame = "0.2.2"
 flamer = "0.5"
-flate2 = { version = "1.1.9", default-features = false }
 # Bump only when the openssl crate bumps it
 foreign-types-shared = "0.1"
 gethostname = "1.0.2"
@@ -233,11 +224,12 @@ glob = "0.3"
 half = "2"
 hex = "0.4.3"
 hexf-parse = "0.2.1"
-hmac = "0.13"
 indexmap = { version = "2.14.0", features = ["std"] }
 insta = "1.47"
-itertools = "0.14.0"
+itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] }
+itoa = "1"
 is-macro = "0.3.7"
+jiff = "0.2"
 js-sys = "0.3"
 junction = "2.0.0"
 lexical-parse-float = "1.0.6"
@@ -248,16 +240,17 @@ xz = "0.4.6-rc.0"
 xz-sys = "0.4.6-rc.0"
 libsqlite3-sys = "0.38"
 libz-rs-sys = "0.6"
+zlib-rs = { version = "=0.6.8", default-features = false, features = ["rust-allocator", "__internal-api"] }
 lock_api = "0.4"
 log = "0.4.30"
-lz4_flex = "0.13"
+lz4_flex = "0.14"
 nix = { version = "0.31", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] }
 mac_address = "1.1.3"
-malachite-bigint = "0.9.1"
-malachite-q = "0.9.1"
-malachite-base = "0.9.1"
+malachite-bigint = "0.12.0"
+malachite-q = "0.12.0"
+malachite-base = "0.12.0"
 md-5 = "0.11"
-memchr = "2.8.1"
+memchr = { version = "2.8.1", default-features = false, features = ["alloc"] }
 memmap2 = "0.9.10"
 mt19937 = "3.3"
 num-complex = "0.4.6"
@@ -265,35 +258,33 @@ num-integer = "0.1.46"
 num-traits = "0.2"
 num_cpus = "1.17.0"
 num_enum = { version = "0.7", default-features = false }
-oid-registry = "0.8"
 openssl = "0.10.80"
 openssl-sys = "0.9.110"
-openssl-probe = "0.2.1"
 optional = "0.5"
 parking_lot = "0.12.3"
 paste = "1.0.15"
-pbkdf2 = "0.13"
 pem-rfc7468 = "1.0"
 pkcs8 = "0.11"
 proc-macro2 = "1.0.105"
 psm = "0.1"
 pymath = { version = "0.2.0", features = ["mul_add", "malachite-bigint", "complex"] }
-pyo3 = "0.28"
+pyo3 = "0.29"
 quote = "1.0.45"
 radium = "1.1.1"
 rand = "0.10"
 rapidhash = "4.4.1"
 result-like = "0.5.0"
-rustix = { version = "1.1", features = ["event", "param", "system"] }
+rustix = { version = "1.1", features = ["event", "fs", "param", "system"] }
 rustls = { version = "0.23.39", default-features = false }
-rustls-graviola = "0.3"
+rustls-graviola = "0.4"
 rustls-native-certs = "0.8"
 rustls-pemfile = "2.2"
-rustls-platform-verifier = "0.7"
+rustls-webpki = { version = "0.103", default-features = false, features = ["std"] }
 rustyline = "18"
 serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] }
 schannel = "0.1.29"
 scopeguard = "1"
+scrypt = { version = "0.12", default-features = false }
 serde-wasm-bindgen = "0.6.5"
 sha1 = "0.11"
 sha2 = "0.11"
@@ -306,21 +297,19 @@ strum = "0.28"
 strum_macros = "0.28"
 syn = "2"
 syn-ext = "0.5.0"
-system-configuration = "0.7.0"
+system-configuration = "0.8.0"
 tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" }
 textwrap = { version = "0.16.2", default-features = false }
 termios = "0.3.3"
 thiserror = "2.0"
-timsort = "0.1.2"
+thin-vec = "0.2.14"
 tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" }
 icu_casemap = "2"
 icu_locale = "2"
 icu_properties = "2"
 icu_normalizer = "2"
 uuid = "1.23.2"
-ucd = "0.1.1"
-unic-ucd-age = "0.9.0"
-unicode_names2 = "2.0.0"
+unicode_names2 = { version = "3", default-features = false, features = ["no_std"] }
 widestring = "1.2.0"
 windows-sys = "0.61.2"
 wasm-bindgen = "0.2.106"
@@ -328,7 +317,6 @@ wasm-bindgen-futures = "0.4"
 web-sys = "0.3"
 webpki-roots = "1.0"
 which = "8"
-x509-cert = "0.2.5"
 x509-parser = "0.18"
 xml = "1.3"
 writeable = "0.6"
@@ -359,12 +347,14 @@ similar_names = "allow"
 # restriction lints
 alloc_instead_of_core = "warn"
 cfg_not_test = "warn"
+iter_over_hash_type = "warn"
 redundant_test_prefix = "warn"
 std_instead_of_alloc = "warn"
 std_instead_of_core = "warn"
 tests_outside_test_module = "warn"
 
 # nursery lints to enforce gradually
+collection_is_never_read = "warn"
 debug_assert_with_mut_call = "warn"
 derive_partial_eq_without_eq = "warn"
 imprecise_flops = "warn"
@@ -375,41 +365,71 @@ needless_collect = "warn"
 or_fun_call = "warn"
 redundant_clone = "warn"
 search_is_some = "warn"
+significant_drop_in_scrutinee = "warn"
 single_option_map = "warn"
 trait_duplication_in_bounds = "warn"
+tuple_array_conversions = "warn"
+type_repetition_in_bounds = "warn"
+unnecessary_struct_initialization = "warn"
 unused_peekable = "warn"
 unused_rounding = "warn"
 use_self = "warn"
 useless_let_if_seq = "warn"
+while_float = "warn"
 
 # pedantic lints to enforce gradually
+assigning_clones = "warn"
+bool_to_int_with_if = "warn"
 checked_conversions = "warn"
 cloned_instead_of_copied = "warn"
 collapsible_else_if = "warn"
 comparison_chain = "warn"
+copy_iterator = "warn"
+doc_link_with_quotes = "warn"
 duration_suboptimal_units = "warn"
+elidable_lifetime_names = "warn"
+enum_glob_use = "warn"
+explicit_deref_methods = "warn"
 explicit_into_iter_loop = "warn"
 explicit_iter_loop = "warn"
 filter_map_next = "warn"
 flat_map_option = "warn"
 format_collect = "warn"
-from_iter_instead_of_collect = "warn"
 inconsistent_struct_constructor = "warn"
 index_refutable_slice = "warn"
 inefficient_to_string = "warn"
+ip_constant = "warn"
 iter_filter_is_ok = "warn"
 iter_filter_is_some = "warn"
+large_futures = "warn"
+large_types_passed_by_value = "warn"
+manual_assert = "warn"
+manual_instant_elapsed = "warn"
 manual_is_variant_and = "warn"
 map_unwrap_or = "warn"
 match_bool = "warn"
+mismatching_type_param_order = "warn"
 must_use_candidate = "warn"
 mut_mut = "warn"
 needless_bitwise_bool = "warn"
 needless_for_each = "warn"
+non_std_lazy_statics = "warn"
 option_as_ref_cloned = "warn"
 ptr_offset_by_literal = "warn"
+range_minus_one = "warn"
+range_plus_one = "warn"
 redundant_else = "warn"
+ref_option = "warn"
 return_self_not_must_use = "warn"
+same_functions_in_if_condition = "warn"
+single_char_pattern = "warn"
+trivially_copy_pass_by_ref = "warn"
+unchecked_time_subtraction = "warn"
 uninlined_format_args = "warn"
+unnecessary_box_returns = "warn"
+unnecessary_join = "warn"
 unnecessary_wraps = "warn"
 unnested_or_patterns = "warn"
+
+[lints]
+workspace = true

Lib/_pycodecs.py

@@ -479,6 +479,24 @@ def STORECHAR32(ch, byteorder):
         return [b3, b2, b1, b0]
 
 
+def _encoded_replacement(replacement, store, unit_bytes, encoding, s, pos):
+    """The bytes an error handler's replacement adds to the output.
+
+    A str is encoded the way the rest of the input is. A bytes object is taken
+    as it stands, and only when it fills whole code units.
+    """
+    if isinstance(replacement, bytes):
+        if len(replacement) % unit_bytes:
+            raise UnicodeEncodeError(
+                encoding, s, pos, pos + 1, "surrogates not allowed"
+            )
+        return list(replacement)
+    encoded = []
+    for c in replacement:
+        encoded += store(ord(c))
+    return encoded
+
+
 def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"):
     """Encode a Unicode string to UTF-32."""
     p = []
@@ -494,21 +512,23 @@ def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"):
     elif byteorder == "big":
         bom = "big"
 
+    # Only the mode that writes a BOM leaves the order out of its name.
+    encoding = "utf-32" if byteorder == "native" else "utf-32-" + bom[0] + "e"
+
+    def store(cp):
+        return STORECHAR32(cp, bom)
+
     pos = 0
     while pos < len(s):
         ch = ord(s[pos])
-        if 0xD800 <= ch <= 0xDFFF:
-            if errors == "surrogatepass":
-                p += STORECHAR32(ch, bom)
-                pos += 1
-            else:
-                res, pos = unicode_call_errorhandler(
-                    errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False
-                )
-                for c in res:
-                    p += STORECHAR32(ord(c), bom)
+        if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass":
+            start = pos
+            res, pos = unicode_call_errorhandler(
+                errors, encoding, "surrogates not allowed", s, pos, pos + 1, False
+            )
+            p += _encoded_replacement(res, store, 4, encoding, s, start)
         else:
-            p += STORECHAR32(ch, bom)
+            p += store(ch)
             pos += 1
 
     return p
@@ -541,6 +561,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
     result = []
     pos = 0
     aligned_size = (size // 4) * 4
+    encoding = "utf-32-le" if byteorder == "little" else "utf-32-be"
 
     while pos + 3 < aligned_size:
         if byteorder == "little":
@@ -560,18 +581,16 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
 
         # Validate code point
         if ch > 0x10FFFF:
-            if errors == "strict":
-                raise UnicodeDecodeError(
-                    "utf-32",
-                    bytes(data),
-                    pos,
-                    pos + 4,
-                    "codepoint not in range(0x110000)",
-                )
-            elif errors == "replace":
-                result.append("\ufffd")
-            # 'ignore' - skip this character
-            pos += 4
+            res, pos = unicode_call_errorhandler(
+                errors,
+                encoding,
+                "code point not in range(0x110000)",
+                data,
+                pos,
+                pos + 4,
+                True,
+            )
+            result.append(res)
         elif 0xD800 <= ch <= 0xDFFF:
             if errors == "surrogatepass":
                 result.append(chr(ch))
@@ -579,7 +598,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
             else:
                 msg = "code point in surrogate code point range(0xd800, 0xe000)"
                 res, pos = unicode_call_errorhandler(
-                    errors, "utf-32", msg, data, pos, pos + 4, True
+                    errors, encoding, msg, data, pos, pos + 4, True
                 )
                 result.append(res)
         else:
@@ -590,7 +609,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
     if pos < size:
         if final:
             res, pos = unicode_call_errorhandler(
-                errors, "utf-32", "truncated data", data, pos, size, True
+                errors, encoding, "truncated data", data, pos, size, True
             )
             if res:
                 result.append(res)
@@ -915,7 +934,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
                         i += 1
                         errmsg = "partial character in shift sequence"
                         out, i = unicode_call_errorhandler(
-                            errors, "utf-7", errmsg, s, startinpos, i
+                            errors, "utf7", errmsg, s, startinpos, i
                         )
                         p.append(out)
                         continue
@@ -924,7 +943,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
                             i += 1
                             errmsg = "non-zero padding bits in shift sequence"
                             out, i = unicode_call_errorhandler(
-                                errors, "utf-7", errmsg, s, startinpos, i
+                                errors, "utf7", errmsg, s, startinpos, i
                             )
                             p.append(out)
                             continue
@@ -943,7 +962,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
                 i += 1
                 errmsg = "ill-formed sequence"
                 out, i = unicode_call_errorhandler(
-                    errors, "utf-7", errmsg, s, startinpos, i
+                    errors, "utf7", errmsg, s, startinpos, i
                 )
                 p.append(out)
             else:
@@ -960,7 +979,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
             i += 1
             errmsg = "unexpected special character"
             out, i = unicode_call_errorhandler(
-                errors, "utf-7", errmsg, s, startinpos, i
+                errors, "utf7", errmsg, s, startinpos, i
             )
             p.append(out)
 
@@ -971,7 +990,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
         if surrogate or base64bits >= 6 or (base64bits > 0 and base64buffer != 0):
             errmsg = "unterminated shift sequence"
             out, i = unicode_call_errorhandler(
-                errors, "utf-7", errmsg, s, startinpos, size
+                errors, "utf7", errmsg, s, startinpos, size
             )
             p.append(out)
 
@@ -1102,20 +1121,6 @@ def unicodeescape_string(s, size, quotes):
             pos += 1
             continue
         # endif
-        # /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */
-        elif ord(ch) >= 0xD800 and ord(ch) < 0xDC00:
-            pos += 1
-            ch2 = s[pos]
-
-            if ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF:
-                ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000
-                p.append(b"\\U%08x" % ucs)
-                pos += 1
-                continue
-
-            # /* Fall through: isolated surrogates are copied as-is */
-            pos -= 1
-
         # /* Map 16-bit characters to '\uxxxx' */
         if ord(ch) >= 256:
             p.append(b"\\u%04x" % ord(ch))
@@ -1233,13 +1238,16 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
         ihi = 0
         ilo = 1
 
+    # The order the data turned out to be in is the one an error names.
+    encoding = "utf-16-be" if ihi == 0 else "utf-16-le"
+
     while q < len(s):
         # /* remaining bytes at the end? (size should be even) */
         if len(s) - q < 2:
             if not final:
                 break
             res, q = unicode_call_errorhandler(
-                errors, "utf-16", "truncated data", s, q, len(s), True
+                errors, encoding, "truncated data", s, q, len(s), True
             )
             p.append(res)
             break
@@ -1267,7 +1275,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
                         q += 2
                         continue
                     res, q = unicode_call_errorhandler(
-                        errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True
+                        errors, encoding, "illegal UTF-16 surrogate", s, q, q + 2, True
                     )
                     p.append(res)
             else:
@@ -1279,7 +1287,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
                     q += 2
                     continue
                 res, q = unicode_call_errorhandler(
-                    errors, "utf-16", "unexpected end of data", s, q, len(s), True
+                    errors, encoding, "unexpected end of data", s, q, len(s), True
                 )
                 p.append(res)
         else:
@@ -1289,7 +1297,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
                 q += 2
                 continue
             res, q = unicode_call_errorhandler(
-                errors, "utf-16", "illegal encoding", s, q, q + 2, True
+                errors, encoding, "illegal encoding", s, q, q + 2, True
             )
             p.append(res)
 
@@ -1323,34 +1331,28 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"):
     elif byteorder == "big":
         bom = "big"
 
+    # Only the mode that writes a BOM leaves the order out of its name.
+    encoding = "utf-16" if byteorder == "native" else "utf-16-" + bom[0] + "e"
+
+    def store(cp):
+        if cp >= 0x10000:
+            cp -= 0x10000
+            return STORECHAR(0xD800 | (cp >> 10), bom) + STORECHAR(
+                0xDC00 | (cp & 0x3FF), bom
+            )
+        return STORECHAR(cp, bom)
+
     pos = 0
     while pos < len(s):
         ch = ord(s[pos])
-        if 0xD800 <= ch <= 0xDFFF:
-            if errors == "surrogatepass":
-                p += STORECHAR(ch, bom)
-                pos += 1
-            else:
-                res, pos = unicode_call_errorhandler(
-                    errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False
-                )
-                for c in res:
-                    cp = ord(c)
-                    cp2 = 0
-                    if cp >= 0x10000:
-                        cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF)
-                        cp = 0xD800 | ((cp - 0x10000) >> 10)
-                    p += STORECHAR(cp, bom)
-                    if cp2:
-                        p += STORECHAR(cp2, bom)
+        if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass":
+            start = pos
+            res, pos = unicode_call_errorhandler(
+                errors, encoding, "surrogates not allowed", s, pos, pos + 1, False
+            )
+            p += _encoded_replacement(res, store, 2, encoding, s, start)
         else:
-            ch2 = 0
-            if ch >= 0x10000:
-                ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF)
-                ch = 0xD800 | ((ch - 0x10000) >> 10)
-            p += STORECHAR(ch, bom)
-            if ch2:
-                p += STORECHAR(ch2, bom)
+            p += store(ch)
             pos += 1
 
     return p
@@ -1685,6 +1687,39 @@ def charmapencode_output(c, mapping):
         raise TypeError("character mapping must return integer, None or str")
 
 
+def charmap_encoding_error(p, size, inpos, mapping, errors, res):
+    # /* find all unencodable characters */
+    collendpos = inpos + 1
+    while collendpos < size:
+        try:
+            charmapencode_output(ord(p[collendpos]), mapping)
+        except KeyError:
+            collendpos += 1
+        else:
+            break
+
+    replacement, newpos = unicode_call_errorhandler(
+        errors,
+        "charmap",
+        "character maps to <undefined>",
+        p,
+        inpos,
+        collendpos,
+        False,
+    )
+    if isinstance(replacement, bytes):
+        res += list(replacement)
+    else:
+        for y in replacement:
+            try:
+                res += charmapencode_output(ord(y), mapping)
+            except KeyError:
+                raise UnicodeEncodeError(
+                    "charmap", p, inpos, collendpos, "character maps to <undefined>"
+                )
+    return newpos
+
+
 def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"):
     ##    /* the following variable is used for caching string comparisons
     ##     * -1=not initialized, 0=unknown, 1=strict, 2=replace,
@@ -1701,29 +1736,11 @@ def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"):
         # /* try to encode it */
         try:
             x = charmapencode_output(ord(p[inpos]), mapping)
-            res += x
         except KeyError:
-            x = unicode_call_errorhandler(
-                errors,
-                "charmap",
-                "character maps to <undefined>",
-                p,
-                inpos,
-                inpos + 1,
-                False,
-            )
-            replacement = x[0]
-            if isinstance(replacement, bytes):
-                res += list(replacement)
-            else:
-                try:
-                    for y in replacement:
-                        res += charmapencode_output(ord(y), mapping)
-                except KeyError:
-                    raise UnicodeEncodeError(
-                        "charmap", p, inpos, inpos + 1, "character maps to <undefined>"
-                    )
-        inpos += 1
+            inpos = charmap_encoding_error(p, size, inpos, mapping, errors, res)
+        else:
+            res += x
+            inpos += 1
     return res
 
 

Lib/_pydatetime.py

@@ -55,7 +55,7 @@ def _days_before_year(year):
 
 def _days_in_month(year, month):
     "year, month -> number of days in that month in that year."
-    assert 1 <= month <= 12, month
+    assert 1 <= month <= 12, f"month must be in 1..12, not {month}"
     if month == 2 and _is_leap(year):
         return 29
     return _DAYS_IN_MONTH[month]
@@ -272,12 +272,12 @@ def _wrap_strftime(object, format, timetuple):
                     newformat.append(Zreplace)
                 # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so
                 # year 1000 for %G can go on the fast path.
-                elif ((ch in 'YG' or ch in 'FC') and
-                        object.year < 1000 and _need_normalize_century()):
+                elif (ch in 'YGFC' and timetuple[0] < 1000 and
+                        _need_normalize_century()):
                     if ch == 'G':
                         year = int(_time.strftime("%G", timetuple))
                     else:
-                        year = object.year
+                        year = timetuple[0]
                     if ch == 'C':
                         push('{:02}'.format(year // 100))
                     else:
@@ -358,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr):
 def _parse_isoformat_date(dtstr):
     # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10,
     # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator
-    assert len(dtstr) in (7, 8, 10)
+    if len(dtstr) not in (7, 8, 10):
+        raise ValueError("Invalid isoformat string")
     year = int(dtstr[0:4])
     has_sep = dtstr[4] == '-'
 
@@ -1947,7 +1948,7 @@ class datetime(date):
                 if became_next_day:
                     year, month, day = date_components
                     # Only wrap day/month when it was previously valid
-                    if month <= 12 and day <= (days_in_month := _days_in_month(year, month)):
+                    if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)):
                         # Calculate midnight of the next day
                         day += 1
                         if day > days_in_month:

Lib/_pyio.py

@@ -1904,7 +1904,7 @@ class TextIOBase(IOBase):
         """Truncate size to pos, where pos is an int."""
         self._unsupported("truncate")
 
-    def readline(self):
+    def readline(self, size=-1, /):
         """Read until newline or EOF.
 
         Returns an empty string if EOF is hit immediately.

Lib/argparse.py

@@ -1829,6 +1829,9 @@ def _prog_name(prog=None):
     if modspec is None:
         # simple script
         return _os.path.basename(arg0)
+    if modspec.name != '__main__' and arg0 != modspec.origin:
+        # named module executed as main without altering sys.argv[0]
+        return _os.path.basename(arg0)
     py = _os.path.basename(_sys.executable)
     if modspec.name != '__main__':
         # imported module or package

Lib/base64.py

@@ -1,4 +1,4 @@
-"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
+"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings"""
 
 # Modified 04-Oct-1995 by Jack Jansen to use binascii module
 # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
@@ -147,7 +147,7 @@ the input is incorrectly padded or if there are non-alphabet
 characters present in the input.
 '''
 _B32_DECODE_MAP01_DOCSTRING = '''
-RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+RFC 4648 allows for optional mapping of the digit 0 (zero) to the
 letter O (oh), and for optional mapping of the digit 1 (one) to
 either the letter I (eye) or letter L (el).  The optional argument
 map01 when not None, specifies which letter the digit 1 should be
@@ -261,7 +261,7 @@ b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
                                                     extra_args='')
 
 
-# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
+# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns
 # lowercase.  The RFC also recommends against accepting input case
 # insensitively.
 def b16encode(s):

Lib/collections/__init__.py

@@ -57,8 +57,7 @@ except ImportError:
 try:
     from _collections import defaultdict
 except ImportError:
-    # TODO: RUSTPYTHON - implement defaultdict in Rust
-    from ._defaultdict import defaultdict
+    pass
 
 heapq = None  # Lazily imported
 
@@ -554,6 +553,9 @@ class Counter(dict):
     or multiset.  Elements are stored as dictionary keys and their counts
     are stored as dictionary values.
 
+    When constructed from a Mapping or Counter, the original object's
+    values will be used as the initial counts.
+
     >>> c = Counter('abcdeabcdabcaba')  # count elements from a string
 
     >>> c.most_common(3)                # three most common elements

Lib/collections/_defaultdict.py

@@ -1,62 +0,0 @@
-from reprlib import recursive_repr as _recursive_repr
-
-class defaultdict(dict):
-    def __init__(self, *args, **kwargs):
-        if len(args) >= 1:
-            default_factory = args[0]
-            if default_factory is not None and not callable(default_factory):
-                raise TypeError("first argument must be callable or None")
-            args = args[1:]
-        else:
-            default_factory = None
-        super().__init__(*args, **kwargs)
-        self.default_factory = default_factory
-
-    def __missing__(self, key):
-        if self.default_factory is not None:
-            val = self.default_factory()
-        else:
-            raise KeyError(key)
-        # CPython parity: a recursive __missing__ via factory() may have
-        # already populated key; preserve that value instead of overwriting.
-        if key in self:
-            return self[key]
-        self[key] = val
-        return val
-
-    @_recursive_repr()
-    def __repr_factory(factory):
-        return repr(factory)
-
-    def __repr__(self):
-        return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})"
-
-    def copy(self):
-        return type(self)(self.default_factory, self)
-
-    __copy__ = copy
-
-    def __reduce__(self):
-        if self.default_factory is not None:
-            args = self.default_factory,
-        else:
-            args = ()
-        return type(self), args, None, None, iter(self.items())
-
-    def __or__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-
-        new = defaultdict(self.default_factory, self)
-        new.update(other)
-        return new
-
-    def __ror__(self, other):
-        if not isinstance(other, dict):
-            return NotImplemented
-
-        new = defaultdict(self.default_factory, other)
-        new.update(self)
-        return new
-
-defaultdict.__module__ = 'collections'

Lib/compression/zstd/_zstdfile.py

@@ -34,7 +34,7 @@ class ZstdFile(_streams.BaseStream):
                  level=None, options=None, zstd_dict=None):
         """Open a Zstandard compressed file in binary mode.
 
-        *file* can be either an file-like object, or a file name to open.
+        *file* can be either a file-like object, or a file name to open.
 
         *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x'
         for creating exclusively, or 'a' for appending.  These can
@@ -246,6 +246,7 @@ class ZstdFile(_streams.BaseStream):
         return self._buffer.peek(size)
 
     def __next__(self):
+        self._check_can_read()
         if ret := self._buffer.readline():
             return ret
         raise StopIteration

Lib/concurrent/futures/__init__.py

@@ -17,7 +17,7 @@ from concurrent.futures._base import (FIRST_COMPLETED,
                                       wait,
                                       as_completed)
 
-__all__ = (
+__all__ = [
     'FIRST_COMPLETED',
     'FIRST_EXCEPTION',
     'ALL_COMPLETED',
@@ -31,24 +31,35 @@ __all__ = (
     'as_completed',
     'ProcessPoolExecutor',
     'ThreadPoolExecutor',
-)
+]
+
+
+try:
+    import _interpreters
+except ImportError:
+    _interpreters = None
+
+if _interpreters:
+    __all__.append('InterpreterPoolExecutor')
 
 
 def __dir__():
-    return __all__ + ('__author__', '__doc__')
+    return __all__ + ['__author__', '__doc__']
 
 
 def __getattr__(name):
-    global ProcessPoolExecutor, ThreadPoolExecutor
+    global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor
 
     if name == 'ProcessPoolExecutor':
-        from .process import ProcessPoolExecutor as pe
-        ProcessPoolExecutor = pe
-        return pe
+        from .process import ProcessPoolExecutor
+        return ProcessPoolExecutor
 
     if name == 'ThreadPoolExecutor':
-        from .thread import ThreadPoolExecutor as te
-        ThreadPoolExecutor = te
-        return te
+        from .thread import ThreadPoolExecutor
+        return ThreadPoolExecutor
+
+    if _interpreters and name == 'InterpreterPoolExecutor':
+        from .interpreter import InterpreterPoolExecutor
+        return InterpreterPoolExecutor
 
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

Lib/concurrent/futures/_base.py

@@ -8,6 +8,8 @@ import logging
 import threading
 import time
 import types
+import weakref
+from itertools import islice
 
 FIRST_COMPLETED = 'FIRST_COMPLETED'
 FIRST_EXCEPTION = 'FIRST_EXCEPTION'
@@ -23,14 +25,6 @@ CANCELLED = 'CANCELLED'
 CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
 FINISHED = 'FINISHED'
 
-_FUTURE_STATES = [
-    PENDING,
-    RUNNING,
-    CANCELLED,
-    CANCELLED_AND_NOTIFIED,
-    FINISHED
-]
-
 _STATE_TO_DESCRIPTION_MAP = {
     PENDING: "pending",
     RUNNING: "running",
@@ -200,15 +194,15 @@ def as_completed(fs, timeout=None):
     """An iterator over the given futures that yields each as it completes.
 
     Args:
-        fs: The sequence of Futures (possibly created by different Executors) to
-            iterate over.
-        timeout: The maximum number of seconds to wait. If None, then there
-            is no limit on the wait time.
+        fs: The sequence of Futures (possibly created by different
+            Executors) to iterate over.
+        timeout: The maximum number of seconds to wait.  If None, then
+            there is no limit on the wait time.
 
     Returns:
-        An iterator that yields the given Futures as they complete (finished or
-        cancelled). If any given Futures are duplicated, they will be returned
-        once.
+        An iterator that yields the given Futures as they complete
+        (finished or cancelled).  If any given Futures are duplicated,
+        they will be returned once.
 
     Raises:
         TimeoutError: If the entire result iterator could not be generated
@@ -264,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
     """Wait for the futures in the given sequence to complete.
 
     Args:
-        fs: The sequence of Futures (possibly created by different Executors) to
-            wait upon.
-        timeout: The maximum number of seconds to wait. If None, then there
-            is no limit on the wait time.
-        return_when: Indicates when this function should return. The options
-            are:
+        fs: The sequence of Futures (possibly created by different
+            Executors) to wait upon.
+        timeout: The maximum number of seconds to wait.  If None, then
+            there is no limit on the wait time.
+        return_when: Indicates when this function should return.
+            The options are:
 
             FIRST_COMPLETED - Return when any future finishes or is
                               cancelled.
             FIRST_EXCEPTION - Return when any future finishes by raising an
-                              exception. If no future raises an exception
+                              exception.  If no future raises an exception
                               then it is equivalent to ALL_COMPLETED.
-            ALL_COMPLETED -   Return when all futures finish or are cancelled.
+            ALL_COMPLETED -   Return when all futures finish or are
+                              cancelled.
 
     Returns:
         A named 2-tuple of sets. The first set, named 'done', contains the
@@ -410,11 +405,12 @@ class Future(object):
 
         Args:
             fn: A callable that will be called with this future as its only
-                argument when the future completes or is cancelled. The callable
-                will always be called by a thread in the same process in which
-                it was added. If the future has already completed or been
-                cancelled then the callable will be called immediately. These
-                callables are called in the order that they were added.
+                argument when the future completes or is cancelled.  The
+                callable will always be called by a thread in the same
+                process in which it was added.  If the future has already
+                completed or been cancelled then the callable will be
+                called immediately.  These callables are called in the
+                order that they were added.
         """
         with self._condition:
             if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
@@ -429,17 +425,19 @@ class Future(object):
         """Return the result of the call that the future represents.
 
         Args:
-            timeout: The number of seconds to wait for the result if the future
-                isn't done. If None, then there is no limit on the wait time.
+            timeout: The number of seconds to wait for the result if the
+                future isn't done.  If None, then there is no limit on the
+                wait time.
 
         Returns:
             The result of the call that the future represents.
 
         Raises:
             CancelledError: If the future was cancelled.
-            TimeoutError: If the future didn't finish executing before the given
-                timeout.
-            Exception: If the call raised then that exception will be raised.
+            TimeoutError: If the future didn't finish executing before the
+                given timeout.
+            Exception: If the call raised then that exception will be
+                raised.
         """
         try:
             with self._condition:
@@ -465,17 +463,17 @@ class Future(object):
 
         Args:
             timeout: The number of seconds to wait for the exception if the
-                future isn't done. If None, then there is no limit on the wait
-                time.
+                future isn't done.  If None, then there is no limit on the
+                wait time.
 
         Returns:
-            The exception raised by the call that the future represents or None
-            if the call completed without raising.
+            The exception raised by the call that the future represents or
+            None if the call completed without raising.
 
         Raises:
             CancelledError: If the future was cancelled.
-            TimeoutError: If the future didn't finish executing before the given
-                timeout.
+            TimeoutError: If the future didn't finish executing before the
+                given timeout.
         """
 
         with self._condition:
@@ -500,22 +498,23 @@ class Future(object):
         Should only be used by Executor implementations and unit tests.
 
         If the future has been cancelled (cancel() was called and returned
-        True) then any threads waiting on the future completing (though calls
-        to as_completed() or wait()) are notified and False is returned.
+        True) then any threads waiting on the future completing (though
+        calls to as_completed() or wait()) are notified and False is
+        returned.
 
         If the future was not cancelled then it is put in the running state
         (future calls to running() will return True) and True is returned.
 
         This method should be called by Executor implementations before
-        executing the work associated with this future. If this method returns
-        False then the work should not be executed.
+        executing the work associated with this future. If this method
+        returns False then the work should not be executed.
 
         Returns:
             False if the Future was cancelled, True otherwise.
 
         Raises:
-            RuntimeError: if this method was already called or if set_result()
-                or set_exception() was called.
+            RuntimeError: if this method was already called or if
+                set_result() or set_exception() was called.
         """
         with self._condition:
             if self._state == CANCELLED:
@@ -572,40 +571,61 @@ class Executor(object):
     def submit(self, fn, /, *args, **kwargs):
         """Submits a callable to be executed with the given arguments.
 
-        Schedules the callable to be executed as fn(*args, **kwargs) and returns
-        a Future instance representing the execution of the callable.
+        Schedules the callable to be executed as fn(*args, **kwargs) and
+        returns a Future instance representing the execution of the
+        callable.
 
         Returns:
             A Future representing the given call.
         """
         raise NotImplementedError()
 
-    def map(self, fn, *iterables, timeout=None, chunksize=1):
+    def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
         """Returns an iterator equivalent to map(fn, iter).
 
         Args:
             fn: A callable that will take as many arguments as there are
                 passed iterables.
-            timeout: The maximum number of seconds to wait. If None, then there
-                is no limit on the wait time.
-            chunksize: The size of the chunks the iterable will be broken into
-                before being passed to a child process. This argument is only
-                used by ProcessPoolExecutor; it is ignored by
+            timeout: The maximum number of seconds to wait. If None, then
+                there is no limit on the wait time.
+            chunksize: The size of the chunks the iterable will be broken
+                into before being passed to a child process.  This argument
+                is only used by ProcessPoolExecutor; it is ignored by
                 ThreadPoolExecutor.
+            buffersize: The number of submitted tasks whose results have not
+                yet been yielded.  If the buffer is full, iteration over the
+                iterables pauses until a result is yielded from the buffer.
+                If None, all input elements are eagerly collected, and
+                a task is submitted for each.
 
         Returns:
-            An iterator equivalent to: map(func, *iterables) but the calls may
-            be evaluated out-of-order.
+            An iterator equivalent to: map(func, *iterables) but the calls
+            may be evaluated out-of-order.
 
         Raises:
-            TimeoutError: If the entire result iterator could not be generated
-                before the given timeout.
+            TimeoutError: If the entire result iterator could not be
+                generated before the given timeout.
             Exception: If fn(*args) raises for any values.
         """
+        if buffersize is not None and not isinstance(buffersize, int):
+            raise TypeError("buffersize must be an integer or None")
+        if buffersize is not None and buffersize < 1:
+            raise ValueError("buffersize must be None or > 0")
+
         if timeout is not None:
             end_time = timeout + time.monotonic()
 
-        fs = [self.submit(fn, *args) for args in zip(*iterables)]
+        zipped_iterables = zip(*iterables)
+        if buffersize:
+            fs = collections.deque(
+                self.submit(fn, *args) for args in islice(zipped_iterables, buffersize)
+            )
+        else:
+            fs = [self.submit(fn, *args) for args in zipped_iterables]
+
+        # Use a weak reference to ensure that the executor can be garbage
+        # collected independently of the result_iterator closure.
+        executor_weakref = weakref.ref(self)
 
         # Yield must be hidden in closure so that the futures are submitted
         # before the first iterator value is required.
@@ -614,6 +634,12 @@ class Executor(object):
                 # reverse to keep finishing order
                 fs.reverse()
                 while fs:
+                    if (
+                        buffersize
+                        and (executor := executor_weakref())
+                        and (args := next(zipped_iterables, None))
+                    ):
+                        fs.appendleft(executor.submit(fn, *args))
                     # Careful not to keep a reference to the popped future
                     if timeout is None:
                         yield _result_or_cancel(fs.pop())
@@ -632,8 +658,8 @@ class Executor(object):
 
         Args:
             wait: If True then shutdown will not return until all running
-                futures have finished executing and the resources used by the
-                executor have been reclaimed.
+                futures have finished executing and the resources used by
+                the executor have been reclaimed.
             cancel_futures: If True then shutdown will cancel all pending
                 futures. Futures that are completed or running will not be
                 cancelled.

Lib/concurrent/futures/interpreter.py

@@ -0,0 +1,123 @@
+"""Implements InterpreterPoolExecutor."""
+
+from concurrent import interpreters
+import sys
+from . import thread as _thread
+import traceback
+
+
+def do_call(results, func, args, kwargs):
+    try:
+        return func(*args, **kwargs)
+    except BaseException as exc:
+        # Send the captured exception out on the results queue,
+        # but still leave it unhandled for the interpreter to handle.
+        try:
+            results.put(exc)
+        except interpreters.NotShareableError:
+            # The exception is not shareable.
+            print('exception is not shareable:', file=sys.stderr)
+            traceback.print_exception(exc)
+            results.put(None)
+        raise  # re-raise
+
+
+class WorkerContext(_thread.WorkerContext):
+
+    @classmethod
+    def prepare(cls, initializer, initargs):
+        def resolve_task(fn, args, kwargs):
+            if isinstance(fn, str):
+                # XXX Circle back to this later.
+                raise TypeError('scripts not supported')
+            else:
+                task = (fn, args, kwargs)
+            return task
+
+        if initializer is not None:
+            try:
+                initdata = resolve_task(initializer, initargs, {})
+            except ValueError:
+                if isinstance(initializer, str) and initargs:
+                    raise ValueError(f'an initializer script does not take args, got {initargs!r}')
+                raise  # re-raise
+        else:
+            initdata = None
+        def create_context():
+            return cls(initdata)
+        return create_context, resolve_task
+
+    def __init__(self, initdata):
+        self.initdata = initdata
+        self.interp = None
+        self.results = None
+
+    def __del__(self):
+        if self.interp is not None:
+            self.finalize()
+
+    def initialize(self):
+        assert self.interp is None, self.interp
+        self.interp = interpreters.create()
+        try:
+            maxsize = 0
+            self.results = interpreters.create_queue(maxsize)
+
+            if self.initdata:
+                self.run(self.initdata)
+        except BaseException:
+            self.finalize()
+            raise  # re-raise
+
+    def finalize(self):
+        interp = self.interp
+        results = self.results
+        self.results = None
+        self.interp = None
+        if results is not None:
+            del results
+        if interp is not None:
+            interp.close()
+
+    def run(self, task):
+        try:
+            return self.interp.call(do_call, self.results, *task)
+        except interpreters.ExecutionFailed as wrapper:
+            # Wait for the exception data to show up.
+            exc = self.results.get()
+            if exc is None:
+                # The exception must have been not shareable.
+                raise  # re-raise
+            raise exc from wrapper
+
+
+class BrokenInterpreterPool(_thread.BrokenThreadPool):
+    """
+    Raised when a worker thread in an InterpreterPoolExecutor failed initializing.
+    """
+
+
+class InterpreterPoolExecutor(_thread.ThreadPoolExecutor):
+
+    BROKEN = BrokenInterpreterPool
+
+    @classmethod
+    def prepare_context(cls, initializer, initargs):
+        return WorkerContext.prepare(initializer, initargs)
+
+    def __init__(self, max_workers=None, thread_name_prefix='',
+                 initializer=None, initargs=()):
+        """Initializes a new InterpreterPoolExecutor instance.
+
+        Args:
+            max_workers: The maximum number of interpreters that can be used
+                to execute the given calls.
+            thread_name_prefix: An optional name prefix to give our threads.
+            initializer: A callable or script used to initialize
+                each worker interpreter.
+            initargs: A tuple of arguments to pass to the initializer.
+        """
+        thread_name_prefix = (thread_name_prefix or
+                              (f"InterpreterPoolExecutor-{self._counter()}"))
+        super().__init__(max_workers, thread_name_prefix,
+                         initializer, initargs)

Lib/concurrent/futures/process.py

@@ -583,7 +583,7 @@ def _check_system_limits():
             raise NotImplementedError(_system_limited)
     _system_limits_checked = True
     try:
-        import multiprocessing.synchronize
+        import multiprocessing.synchronize  # noqa: F401
     except ImportError:
         _system_limited = (
             "This Python build lacks multiprocessing.synchronize, usually due "
@@ -626,6 +626,14 @@ class BrokenProcessPool(_base.BrokenExecutor):
     while a future was in the running state.
     """
 
+_TERMINATE = "terminate"
+_KILL = "kill"
+
+_SHUTDOWN_CALLBACK_OPERATION = {
+    _TERMINATE,
+    _KILL
+}
+
 
 class ProcessPoolExecutor(_base.Executor):
     def __init__(self, max_workers=None, mp_context=None,
@@ -634,19 +642,21 @@ class ProcessPoolExecutor(_base.Executor):
 
         Args:
             max_workers: The maximum number of processes that can be used to
-                execute the given calls. If None or not given then as many
-                worker processes will be created as the machine has processors.
-            mp_context: A multiprocessing context to launch the workers created
-                using the multiprocessing.get_context('start method') API. This
-                object should provide SimpleQueue, Queue and Process.
+                execute the given calls.  If None or not given then as many
+                worker processes will be created as the machine has
+                processors.
+            mp_context: A multiprocessing context to launch the workers
+                created using the multiprocessing.get_context('start method')
+                API.  This object should provide SimpleQueue, Queue and
+                Process.
             initializer: A callable used to initialize worker processes.
             initargs: A tuple of arguments to pass to the initializer.
-            max_tasks_per_child: The maximum number of tasks a worker process
-                can complete before it will exit and be replaced with a fresh
-                worker process. The default of None means worker process will
-                live as long as the executor. Requires a non-'fork' mp_context
-                start method. When given, we default to using 'spawn' if no
-                mp_context is supplied.
+            max_tasks_per_child: The maximum number of tasks a worker
+                process can complete before it will exit and be replaced
+                with a fresh worker process.  The default of None means
+                worker process will live as long as the executor.  Requires
+                a non-'fork' mp_context start method.  When given, we
+                default to using 'spawn' if no mp_context is supplied.
         """
         _check_system_limits()
 
@@ -810,25 +820,31 @@ class ProcessPoolExecutor(_base.Executor):
             return f
     submit.__doc__ = _base.Executor.submit.__doc__
 
-    def map(self, fn, *iterables, timeout=None, chunksize=1):
+    def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
         """Returns an iterator equivalent to map(fn, iter).
 
         Args:
             fn: A callable that will take as many arguments as there are
                 passed iterables.
-            timeout: The maximum number of seconds to wait. If None, then there
-                is no limit on the wait time.
-            chunksize: If greater than one, the iterables will be chopped into
-                chunks of size chunksize and submitted to the process pool.
-                If set to one, the items in the list will be sent one at a time.
+            timeout: The maximum number of seconds to wait.  If None, then
+                there is no limit on the wait time.
+            chunksize: If greater than one, the iterables will be chopped
+                into chunks of size chunksize and submitted to the process
+                pool.  If set to one, the items in the list will be sent
+                one at a time.
+            buffersize: The number of submitted tasks whose results have not
+                yet been yielded.  If the buffer is full, iteration over the
+                iterables pauses until a result is yielded from the buffer.
+                If None, all input elements are eagerly collected, and
+                a task is submitted for each.
 
         Returns:
-            An iterator equivalent to: map(func, *iterables) but the calls may
-            be evaluated out-of-order.
+            An iterator equivalent to: map(func, *iterables) but the calls
+            may be evaluated out-of-order.
 
         Raises:
-            TimeoutError: If the entire result iterator could not be generated
-                before the given timeout.
+            TimeoutError: If the entire result iterator could not be
+                generated before the given timeout.
             Exception: If fn(*args) raises for any values.
         """
         if chunksize < 1:
@@ -836,7 +852,8 @@ class ProcessPoolExecutor(_base.Executor):
 
         results = super().map(partial(_process_chunk, fn),
                               itertools.batched(zip(*iterables), chunksize),
-                              timeout=timeout)
+                              timeout=timeout,
+                              buffersize=buffersize)
         return _chain_from_iterable_of_lists(results)
 
     def shutdown(self, wait=True, *, cancel_futures=False):
@@ -860,3 +877,66 @@ class ProcessPoolExecutor(_base.Executor):
         self._executor_manager_thread_wakeup = None
 
     shutdown.__doc__ = _base.Executor.shutdown.__doc__
+
+    def _force_shutdown(self, operation):
+        """Attempts to terminate or kill the executor's workers based off the
+        given operation. Iterates through all of the current processes and
+        performs the relevant task if the process is still alive.
+
+        After terminating workers, the pool will be in a broken state
+        and no longer usable (for instance, new tasks should not be
+        submitted).
+        """
+        if operation not in _SHUTDOWN_CALLBACK_OPERATION:
+            raise ValueError(f"Unsupported operation: {operation!r}")
+
+        processes = {}
+        if self._processes:
+            processes = self._processes.copy()
+
+        # shutdown will invalidate ._processes, so we copy it right before
+        # calling. If we waited here, we would deadlock if a process decides not
+        # to exit.
+        self.shutdown(wait=False, cancel_futures=True)
+
+        if not processes:
+            return
+
+        for proc in processes.values():
+            try:
+                if not proc.is_alive():
+                    continue
+            except ValueError:
+                # The process is already exited/closed out.
+                continue
+
+            try:
+                if operation == _TERMINATE:
+                    proc.terminate()
+                elif operation == _KILL:
+                    proc.kill()
+            except ProcessLookupError:
+                # The process just ended before our signal
+                continue
+
+    def terminate_workers(self):
+        """Attempts to terminate the executor's workers.
+        Iterates through all of the current worker processes and terminates
+        each one that is still alive.
+
+        After terminating workers, the pool will be in a broken state
+        and no longer usable (for instance, new tasks should not be
+        submitted).
+        """
+        return self._force_shutdown(operation=_TERMINATE)
+
+    def kill_workers(self):
+        """Attempts to kill the executor's workers.
+        Iterates through all of the current worker processes and kills
+        each one that is still alive.
+
+        After killing workers, the pool will be in a broken state
+        and no longer usable (for instance, new tasks should not be
+        submitted).
+        """
+        return self._force_shutdown(operation=_KILL)

Lib/concurrent/futures/thread.py

@@ -44,19 +44,46 @@ if hasattr(os, 'register_at_fork'):
     os.register_at_fork(after_in_child=_threads_queues.clear)
 
 
-class _WorkItem:
-    def __init__(self, future, fn, args, kwargs):
-        self.future = future
-        self.fn = fn
-        self.args = args
-        self.kwargs = kwargs
+class WorkerContext:
 
-    def run(self):
+    @classmethod
+    def prepare(cls, initializer, initargs):
+        if initializer is not None:
+            if not callable(initializer):
+                raise TypeError("initializer must be a callable")
+        def create_context():
+            return cls(initializer, initargs)
+        def resolve_task(fn, args, kwargs):
+            return (fn, args, kwargs)
+        return create_context, resolve_task
+
+    def __init__(self, initializer, initargs):
+        self.initializer = initializer
+        self.initargs = initargs
+
+    def initialize(self):
+        if self.initializer is not None:
+            self.initializer(*self.initargs)
+
+    def finalize(self):
+        pass
+
+    def run(self, task):
+        fn, args, kwargs = task
+        return fn(*args, **kwargs)
+
+
+class _WorkItem:
+    def __init__(self, future, task):
+        self.future = future
+        self.task = task
+
+    def run(self, ctx):
         if not self.future.set_running_or_notify_cancel():
             return
 
         try:
-            result = self.fn(*self.args, **self.kwargs)
+            result = ctx.run(self.task)
         except BaseException as exc:
             self.future.set_exception(exc)
             # Break a reference cycle with the exception 'exc'
@@ -67,16 +94,15 @@ class _WorkItem:
     __class_getitem__ = classmethod(types.GenericAlias)
 
 
-def _worker(executor_reference, work_queue, initializer, initargs):
-    if initializer is not None:
-        try:
-            initializer(*initargs)
-        except BaseException:
-            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
-            executor = executor_reference()
-            if executor is not None:
-                executor._initializer_failed()
-            return
+def _worker(executor_reference, ctx, work_queue):
+    try:
+        ctx.initialize()
+    except BaseException:
+        _base.LOGGER.critical('Exception in initializer:', exc_info=True)
+        executor = executor_reference()
+        if executor is not None:
+            executor._initializer_failed()
+        return
     try:
         while True:
             try:
@@ -90,7 +116,7 @@ def _worker(executor_reference, work_queue, initializer, initargs):
                 work_item = work_queue.get(block=True)
 
             if work_item is not None:
-                work_item.run()
+                work_item.run(ctx)
                 # Delete references to object. See GH-60488
                 del work_item
                 continue
@@ -111,6 +137,8 @@ def _worker(executor_reference, work_queue, initializer, initargs):
             del executor
     except BaseException:
         _base.LOGGER.critical('Exception in worker', exc_info=True)
+    finally:
+        ctx.finalize()
 
 
 class BrokenThreadPool(_base.BrokenExecutor):
@@ -121,11 +149,17 @@ class BrokenThreadPool(_base.BrokenExecutor):
 
 class ThreadPoolExecutor(_base.Executor):
 
+    BROKEN = BrokenThreadPool
+
     # Used to assign unique thread names when thread_name_prefix is not supplied.
     _counter = itertools.count().__next__
 
+    @classmethod
+    def prepare_context(cls, initializer, initargs):
+        return WorkerContext.prepare(initializer, initargs)
+
     def __init__(self, max_workers=None, thread_name_prefix='',
-                 initializer=None, initargs=()):
+                 initializer=None, initargs=(), **ctxkwargs):
         """Initializes a new ThreadPoolExecutor instance.
 
         Args:
@@ -134,6 +168,7 @@ class ThreadPoolExecutor(_base.Executor):
             thread_name_prefix: An optional name prefix to give our threads.
             initializer: A callable used to initialize worker threads.
             initargs: A tuple of arguments to pass to the initializer.
+            ctxkwargs: Additional arguments to cls.prepare_context().
         """
         if max_workers is None:
             # ThreadPoolExecutor is often used to:
@@ -147,8 +182,9 @@ class ThreadPoolExecutor(_base.Executor):
         if max_workers <= 0:
             raise ValueError("max_workers must be greater than 0")
 
-        if initializer is not None and not callable(initializer):
-            raise TypeError("initializer must be a callable")
+        (self._create_worker_context,
+         self._resolve_work_item_task,
+         ) = type(self).prepare_context(initializer, initargs, **ctxkwargs)
 
         self._max_workers = max_workers
         self._work_queue = queue.SimpleQueue()
@@ -159,13 +195,11 @@ class ThreadPoolExecutor(_base.Executor):
         self._shutdown_lock = threading.Lock()
         self._thread_name_prefix = (thread_name_prefix or
                                     ("ThreadPoolExecutor-%d" % self._counter()))
-        self._initializer = initializer
-        self._initargs = initargs
 
     def submit(self, fn, /, *args, **kwargs):
         with self._shutdown_lock, _global_shutdown_lock:
             if self._broken:
-                raise BrokenThreadPool(self._broken)
+                raise self.BROKEN(self._broken)
 
             if self._shutdown:
                 raise RuntimeError('cannot schedule new futures after shutdown')
@@ -174,7 +208,8 @@ class ThreadPoolExecutor(_base.Executor):
                                    'interpreter shutdown')
 
             f = _base.Future()
-            w = _WorkItem(f, fn, args, kwargs)
+            task = self._resolve_work_item_task(fn, args, kwargs)
+            w = _WorkItem(f, task)
 
             self._work_queue.put(w)
             self._adjust_thread_count()
@@ -197,9 +232,8 @@ class ThreadPoolExecutor(_base.Executor):
                                      num_threads)
             t = threading.Thread(name=thread_name, target=_worker,
                                  args=(weakref.ref(self, weakref_cb),
-                                       self._work_queue,
-                                       self._initializer,
-                                       self._initargs))
+                                       self._create_worker_context(),
+                                       self._work_queue))
             t.start()
             self._threads.add(t)
             _threads_queues[t] = self._work_queue
@@ -215,7 +249,7 @@ class ThreadPoolExecutor(_base.Executor):
                 except queue.Empty:
                     break
                 if work_item is not None:
-                    work_item.future.set_exception(BrokenThreadPool(self._broken))
+                    work_item.future.set_exception(self.BROKEN(self._broken))
 
     def shutdown(self, wait=True, *, cancel_futures=False):
         with self._shutdown_lock:

Lib/concurrent/interpreters/__init__.py

@@ -0,0 +1,247 @@
+"""Subinterpreters High Level Module."""
+
+import threading
+import weakref
+import _interpreters
+
+# aliases:
+from _interpreters import (
+    InterpreterError, InterpreterNotFoundError, NotShareableError,
+    is_shareable,
+)
+from ._queues import (
+    create as create_queue,
+    Queue, QueueEmpty, QueueFull,
+)
+
+
+__all__ = [
+    'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
+    'Interpreter',
+    'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed',
+    'NotShareableError',
+    'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
+]
+
+
+_EXEC_FAILURE_STR = """
+{superstr}
+
+Uncaught in the interpreter:
+
+{formatted}
+""".strip()
+
+class ExecutionFailed(InterpreterError):
+    """An unhandled exception happened during execution.
+
+    This is raised from Interpreter.exec() and Interpreter.call().
+    """
+
+    def __init__(self, excinfo):
+        msg = excinfo.formatted
+        if not msg:
+            if excinfo.type and excinfo.msg:
+                msg = f'{excinfo.type.__name__}: {excinfo.msg}'
+            else:
+                msg = excinfo.type.__name__ or excinfo.msg
+        super().__init__(msg)
+        self.excinfo = excinfo
+
+    def __str__(self):
+        try:
+            formatted = self.excinfo.errdisplay
+        except Exception:
+            return super().__str__()
+        else:
+            return _EXEC_FAILURE_STR.format(
+                superstr=super().__str__(),
+                formatted=formatted,
+            )
+
+
+def create():
+    """Return a new (idle) Python interpreter."""
+    id = _interpreters.create(reqrefs=True)
+    return Interpreter(id, _ownsref=True)
+
+
+def list_all():
+    """Return all existing interpreters."""
+    return [Interpreter(id, _whence=whence)
+            for id, whence in _interpreters.list_all(require_ready=True)]
+
+
+def get_current():
+    """Return the currently running interpreter."""
+    id, whence = _interpreters.get_current()
+    return Interpreter(id, _whence=whence)
+
+
+def get_main():
+    """Return the main interpreter."""
+    id, whence = _interpreters.get_main()
+    assert whence == _interpreters.WHENCE_RUNTIME, repr(whence)
+    return Interpreter(id, _whence=whence)
+
+
+_known = weakref.WeakValueDictionary()
+
+class Interpreter:
+    """A single Python interpreter.
+
+    Attributes:
+
+    "id" - the unique process-global ID number for the interpreter
+    "whence" - indicates where the interpreter was created
+
+    If the interpreter wasn't created by this module
+    then any method that modifies the interpreter will fail,
+    i.e. .close(), .prepare_main(), .exec(), and .call()
+    """
+
+    _WHENCE_TO_STR = {
+       _interpreters.WHENCE_UNKNOWN: 'unknown',
+       _interpreters.WHENCE_RUNTIME: 'runtime init',
+       _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
+       _interpreters.WHENCE_CAPI: 'C-API',
+       _interpreters.WHENCE_XI: 'cross-interpreter C-API',
+       _interpreters.WHENCE_STDLIB: '_interpreters module',
+    }
+
+    def __new__(cls, id, /, _whence=None, _ownsref=None):
+        # There is only one instance for any given ID.
+        if not isinstance(id, int):
+            raise TypeError(f'id must be an int, got {id!r}')
+        id = int(id)
+        if _whence is None:
+            if _ownsref:
+                _whence = _interpreters.WHENCE_STDLIB
+            else:
+                _whence = _interpreters.whence(id)
+        assert _whence in cls._WHENCE_TO_STR, repr(_whence)
+        if _ownsref is None:
+            _ownsref = (_whence == _interpreters.WHENCE_STDLIB)
+        try:
+            self = _known[id]
+            assert hasattr(self, '_ownsref')
+        except KeyError:
+            self = super().__new__(cls)
+            _known[id] = self
+            self._id = id
+            self._whence = _whence
+            self._ownsref = _ownsref
+            if _ownsref:
+                # This may raise InterpreterNotFoundError:
+                _interpreters.incref(id)
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __del__(self):
+        self._decref()
+
+    # for pickling:
+    def __reduce__(self):
+        return (type(self), (self._id,))
+
+    # gh-135729: Globals might be destroyed by the time this is called, so we
+    # need to keep references ourself
+    def _decref(self, *,
+                InterpreterNotFoundError=InterpreterNotFoundError,
+                _interp_decref=_interpreters.decref,
+                ):
+        if not self._ownsref:
+            return
+        self._ownsref = False
+        try:
+            _interp_decref(self._id)
+        except InterpreterNotFoundError:
+            pass
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def whence(self):
+        return self._WHENCE_TO_STR[self._whence]
+
+    def is_running(self):
+        """Return whether or not the identified interpreter is running."""
+        return _interpreters.is_running(self._id)
+
+    # Everything past here is available only to interpreters created by
+    # interpreters.create().
+
+    def close(self):
+        """Finalize and destroy the interpreter.
+
+        Attempting to destroy the current interpreter results
+        in an InterpreterError.
+        """
+        return _interpreters.destroy(self._id, restrict=True)
+
+    def prepare_main(self, ns=None, /, **kwargs):
+        """Bind the given values into the interpreter's __main__.
+
+        The values must be shareable.
+        """
+        ns = dict(ns, **kwargs) if ns is not None else kwargs
+        _interpreters.set___main___attrs(self._id, ns, restrict=True)
+
+    def exec(self, code, /):
+        """Run the given source code in the interpreter.
+
+        This is essentially the same as calling the builtin "exec"
+        with this interpreter, using the __dict__ of its __main__
+        module as both globals and locals.
+
+        There is no return value.
+
+        If the code raises an unhandled exception then an ExecutionFailed
+        exception is raised, which summarizes the unhandled exception.
+        The actual exception is discarded because objects cannot be
+        shared between interpreters.
+
+        This blocks the current Python thread until done.  During
+        that time, the previous interpreter is allowed to run
+        in other threads.
+        """
+        excinfo = _interpreters.exec(self._id, code, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+
+    def _call(self, callable, args, kwargs):
+        res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True)
+        if excinfo is not None:
+            raise ExecutionFailed(excinfo)
+        return res
+
+    def call(self, callable, /, *args, **kwargs):
+        """Call the object in the interpreter with given args/kwargs.
+
+        Nearly all callables, args, kwargs, and return values are
+        supported.  All "shareable" objects are supported, as are
+        "stateless" functions (meaning non-closures that do not use
+        any globals).  This method will fall back to pickle.
+
+        If the callable raises an exception then the error display
+        (including full traceback) is sent back between the interpreters
+        and an ExecutionFailed exception is raised, much like what
+        happens with Interpreter.exec().
+        """
+        return self._call(callable, args, kwargs)
+
+    def call_in_thread(self, callable, /, *args, **kwargs):
+        """Return a new thread that calls the object in the interpreter.
+
+        The return value and any raised exception are discarded.
+        """
+        t = threading.Thread(target=self._call, args=(callable, args, kwargs))
+        t.start()
+        return t

Lib/concurrent/interpreters/_crossinterp.py

@@ -0,0 +1,107 @@
+"""Common code between queues and channels."""
+
+
+class ItemInterpreterDestroyed(Exception):
+    """Raised when trying to get an item whose interpreter was destroyed."""
+
+
+class classonly:
+    """A non-data descriptor that makes a value only visible on the class.
+
+    This is like the "classmethod" builtin, but does not show up on
+    instances of the class.  It may be used as a decorator.
+    """
+
+    def __init__(self, value):
+        self.value = value
+        self.getter = classmethod(value).__get__
+        self.name = None
+
+    def __set_name__(self, cls, name):
+        if self.name is not None:
+            raise TypeError('already used')
+        self.name = name
+
+    def __get__(self, obj, cls):
+        if obj is not None:
+            raise AttributeError(self.name)
+        # called on the class
+        return self.getter(None, cls)
+
+
+class UnboundItem:
+    """Represents a cross-interpreter item no longer bound to an interpreter.
+
+    An item is unbound when the interpreter that added it to the
+    cross-interpreter container is destroyed.
+    """
+
+    __slots__ = ()
+
+    @classonly
+    def singleton(cls, kind, module, name='UNBOUND'):
+        doc = cls.__doc__
+        if doc:
+            doc = doc.replace(
+                'cross-interpreter container', kind,
+            ).replace(
+                'cross-interpreter', kind,
+            )
+        subclass = type(
+            f'Unbound{kind.capitalize()}Item',
+            (cls,),
+            {
+                "_MODULE": module,
+                "_NAME": name,
+                "__doc__": doc,
+            },
+        )
+        return object.__new__(subclass)
+
+    _MODULE = __name__
+    _NAME = 'UNBOUND'
+
+    def __new__(cls):
+        raise Exception(f'use {cls._MODULE}.{cls._NAME}')
+
+    def __repr__(self):
+        return f'{self._MODULE}.{self._NAME}'
+#        return f'interpreters._queues.UNBOUND'
+
+
+UNBOUND = object.__new__(UnboundItem)
+UNBOUND_ERROR = object()
+UNBOUND_REMOVE = object()
+
+_UNBOUND_CONSTANT_TO_FLAG = {
+    UNBOUND_REMOVE: 1,
+    UNBOUND_ERROR: 2,
+    UNBOUND: 3,
+}
+_UNBOUND_FLAG_TO_CONSTANT = {v: k
+                             for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}
+
+
+def serialize_unbound(unbound):
+    op = unbound
+    try:
+        flag = _UNBOUND_CONSTANT_TO_FLAG[op]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
+    return flag,
+
+
+def resolve_unbound(flag, exctype_destroyed):
+    try:
+        op = _UNBOUND_FLAG_TO_CONSTANT[flag]
+    except KeyError:
+        raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
+    if op is UNBOUND_REMOVE:
+        # "remove" not possible here
+        raise NotImplementedError
+    elif op is UNBOUND_ERROR:
+        raise exctype_destroyed("item's original interpreter destroyed")
+    elif op is UNBOUND:
+        return UNBOUND
+    else:
+        raise NotImplementedError(repr(op))

Lib/concurrent/interpreters/_queues.py

@@ -0,0 +1,290 @@
+"""Cross-interpreter Queues High Level Module."""
+
+import pickle
+import queue
+import time
+import weakref
+import _interpqueues as _queues
+from . import _crossinterp
+
+# aliases:
+from _interpqueues import (
+    QueueError, QueueNotFoundError,
+)
+from ._crossinterp import (
+    UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+__all__ = [
+    'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+    'create', 'list_all',
+    'Queue',
+    'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
+    'ItemInterpreterDestroyed',
+]
+
+
+class QueueEmpty(QueueError, queue.Empty):
+    """Raised from get_nowait() when the queue is empty.
+
+    It is also raised from get() if it times out.
+    """
+
+
+class QueueFull(QueueError, queue.Full):
+    """Raised from put_nowait() when the queue is full.
+
+    It is also raised from put() if it times out.
+    """
+
+
+class ItemInterpreterDestroyed(QueueError,
+                               _crossinterp.ItemInterpreterDestroyed):
+    """Raised from get() and get_nowait()."""
+
+
+_SHARED_ONLY = 0
+_PICKLED = 1
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+    if unbound is UNBOUND:
+        unbound = _crossinterp.UNBOUND
+    return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+    resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+    if resolved is _crossinterp.UNBOUND:
+        resolved = UNBOUND
+    return resolved
+
+
+def create(maxsize=0, *, unbounditems=UNBOUND):
+    """Return a new cross-interpreter queue.
+
+    The queue may be used to pass data safely between interpreters.
+
+    "unbounditems" sets the default for Queue.put(); see that method for
+    supported values.  The default value is UNBOUND, which replaces
+    the unbound item.
+    """
+    unbound = _serialize_unbound(unbounditems)
+    unboundop, = unbound
+    qid = _queues.create(maxsize, unboundop, -1)
+    self = Queue(qid)
+    self._set_unbound(unboundop, unbounditems)
+    return self
+
+
+def list_all():
+    """Return a list of all open queues."""
+    queues = []
+    for qid, unboundop, _ in _queues.list_all():
+        self = Queue(qid)
+        if not hasattr(self, '_unbound'):
+            self._set_unbound(unboundop)
+        else:
+            assert self._unbound[0] == unboundop
+        queues.append(self)
+    return queues
+
+
+_known_queues = weakref.WeakValueDictionary()
+
+class Queue:
+    """A cross-interpreter queue."""
+
+    def __new__(cls, id, /):
+        # There is only one instance for any given ID.
+        if isinstance(id, int):
+            id = int(id)
+        else:
+            raise TypeError(f'id must be an int, got {id!r}')
+        try:
+            self = _known_queues[id]
+        except KeyError:
+            self = super().__new__(cls)
+            self._id = id
+            _known_queues[id] = self
+            _queues.bind(id)
+        return self
+
+    def __del__(self):
+        try:
+            _queues.release(self._id)
+        except QueueNotFoundError:
+            pass
+        try:
+            del _known_queues[self._id]
+        except KeyError:
+            pass
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    # for pickling:
+    def __reduce__(self):
+        return (type(self), (self._id,))
+
+    def _set_unbound(self, op, items=None):
+        assert not hasattr(self, '_unbound')
+        if items is None:
+            items = _resolve_unbound(op)
+        unbound = (op, items)
+        self._unbound = unbound
+        return unbound
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def unbounditems(self):
+        try:
+            _, items = self._unbound
+        except AttributeError:
+            op, _ = _queues.get_queue_defaults(self._id)
+            _, items = self._set_unbound(op)
+        return items
+
+    @property
+    def maxsize(self):
+        try:
+            return self._maxsize
+        except AttributeError:
+            self._maxsize = _queues.get_maxsize(self._id)
+            return self._maxsize
+
+    def empty(self):
+        return self.qsize() == 0
+
+    def full(self):
+        return _queues.is_full(self._id)
+
+    def qsize(self):
+        return _queues.get_count(self._id)
+
+    def put(self, obj, block=True, timeout=None, *,
+            unbounditems=None,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Add the object to the queue.
+
+        If "block" is true, this blocks while the queue is full.
+
+        For most objects, the object received through Queue.get() will
+        be a new one, equivalent to the original and not sharing any
+        actual underlying data.  The notable exceptions include
+        cross-interpreter types (like Queue) and memoryview, where the
+        underlying data is actually shared.  Furthermore, some types
+        can be sent through a queue more efficiently than others.  This
+        group includes various immutable types like int, str, bytes, and
+        tuple (if the items are likewise efficiently shareable).
+        See interpreters.is_shareable().
+
+        "unbounditems" controls the behavior of Queue.get() for the given
+        object if the current interpreter (calling put()) is later
+        destroyed.
+
+        If "unbounditems" is None (the default) then it uses the
+        queue's default, set with create_queue(),
+        which is usually UNBOUND.
+
+        If "unbounditems" is UNBOUND_ERROR then get() will raise an
+        ItemInterpreterDestroyed exception if the original interpreter
+        has been destroyed.  This does not otherwise affect the queue;
+        the next call to put() will work like normal, returning the next
+        item in the queue.
+
+        If "unbounditems" is UNBOUND_REMOVE then the item will be removed
+        from the queue as soon as the original interpreter is destroyed.
+        Be aware that this will introduce an imbalance between put()
+        and get() calls.
+
+        If "unbounditems" is UNBOUND then it is returned by get() in place
+        of the unbound item.
+        """
+        if not block:
+            return self.put_nowait(obj, unbounditems=unbounditems)
+        if unbounditems is None:
+            unboundop = -1
+        else:
+            unboundop, = _serialize_unbound(unbounditems)
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        while True:
+            try:
+                _queues.put(self._id, obj, unboundop)
+            except QueueFull as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+
+    def put_nowait(self, obj, *, unbounditems=None):
+        if unbounditems is None:
+            unboundop = -1
+        else:
+            unboundop, = _serialize_unbound(unbounditems)
+        _queues.put(self._id, obj, unboundop)
+
+    def get(self, block=True, timeout=None, *,
+            _delay=10 / 1000,  # 10 milliseconds
+            ):
+        """Return the next object from the queue.
+
+        If "block" is true, this blocks while the queue is empty.
+
+        If the next item's original interpreter has been destroyed
+        then the "next object" is determined by the value of the
+        "unbounditems" argument to put().
+        """
+        if not block:
+            return self.get_nowait()
+        if timeout is not None:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise ValueError(f'timeout value must be non-negative')
+            end = time.time() + timeout
+        while True:
+            try:
+                obj, unboundop = _queues.get(self._id)
+            except QueueEmpty as exc:
+                if timeout is not None and time.time() >= end:
+                    raise  # re-raise
+                time.sleep(_delay)
+            else:
+                break
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+    def get_nowait(self):
+        """Return the next object from the channel.
+
+        If the queue is empty then raise QueueEmpty.  Otherwise this
+        is the same as get().
+        """
+        try:
+            obj, unboundop = _queues.get(self._id)
+        except QueueEmpty as exc:
+            raise  # re-raise
+        if unboundop is not None:
+            assert obj is None, repr(obj)
+            return _resolve_unbound(unboundop)
+        return obj
+
+
+_queues._register_heap_types(Queue, QueueEmpty, QueueFull)

Lib/configparser.py

@@ -992,7 +992,9 @@ class RawConfigParser(MutableMapping):
             value = self._interpolation.before_write(self, section_name, key,
                                                      value)
             if value is not None or not self._allow_no_value:
-                value = delimiter + str(value).replace('\n', '\n\t')
+                # Convert all possible line-endings into '\n\t'
+                value = (delimiter + str(value).replace('\r\n', '\n')
+                         .replace('\r', '\n').replace('\n', '\n\t'))
             else:
                 value = ""
             fp.write("{}{}\n".format(key, value))

Lib/csv.py

@@ -243,8 +243,14 @@ class Sniffer:
     def sniff(self, sample, delimiters=None):
         """
         Returns a dialect (or None) corresponding to the sample
+
+        If several delimiters fit the sample equally well, the
+        delimiters listed in the preferred attribute are preferred, in
+        that order, no matter how many times each of them occurs.
         """
 
+        sample = sample.replace('\r\n', '\n').replace('\r', '\n')
+
         quotechar, doublequote, delimiter, skipinitialspace = \
                    self._guess_quote_and_delimiter(sample, delimiters)
         if not delimiter:
@@ -282,12 +288,16 @@ class Sniffer:
         """
         import re
 
+        # The body of a quoted field ends at the first quote which is
+        # not doubled, as it does for a reader.  A lazy ".*?" scans to
+        # the end of the sample instead, from every start: quadratically.
+        body = r'(?:(?P=quote){2}|(?!(?P=quote)).)*+'
         matches = []
-        for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
-                      r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',   #  ".*?",
-                      r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',   # ,".*?"
-                      r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
-            regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
+        for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)',   # ,"...",
+                      r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',     #  "...",
+                      r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\n)',  # ,"..."
+                      r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\n)'):                           #  "..." (no delim, no space)
+            regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
             matches = regexp.findall(data)
             if matches:
                 break
@@ -330,18 +340,22 @@ class Sniffer:
             delim = ''
             skipinitialspace = 0
 
-        # if we see an extra quote between delimiters, we've got a
-        # double quoted format
-        dq_regexp = re.compile(
-                               r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \
-                               {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE)
-
-
-
-        if dq_regexp.search(data):
-            doublequote = True
-        else:
-            doublequote = False
+        # A doubled quote character inside a quoted field means
+        # a double quoted format.  Match whole fields, so that a match
+        # cannot slide across field boundaries.
+        doublequote = False
+        if delim:
+            dq_regexp = re.compile(
+                    r"(?:(?<=%(delim)s)|^)%(space)s%(quote)s"     # ,"
+                    r"((?:%(quote)s%(quote)s|[^%(quote)s]++)*+)"  # the body
+                    r"%(quote)s(?:%(delim)s|$)"                   # ",
+                    % {'delim': re.escape(delim), 'quote': quotechar,
+                       # Skipping spaces after a space rescans them.
+                       'space': ' *+' if delim != ' ' else ''},
+                    re.MULTILINE)
+            dquotechar = quotechar * 2
+            doublequote = any(dquotechar in m[1]
+                              for m in dq_regexp.finditer(data))
 
         return (quotechar, doublequote, delim, skipinitialspace)
 

Lib/dataclasses.py

@@ -1247,6 +1247,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
 # classes with slots.  These could be slightly more performant if we generated
 # the code instead of iterating over fields.  But that can be a project for
 # another day, if performance becomes an issue.
+
 def _dataclass_getstate(self):
     return [getattr(self, f.name) for f in fields(self)]
 

Lib/difflib.py

@@ -1977,6 +1977,8 @@ class HtmlDiff(object):
 
         # change tabs to spaces before it gets more difficult after we insert
         # markup
+        # it also removes trailing newlines, causing some diffs to be missed
+        # see: gh-71896
         fromlines,tolines = self._tab_newline_replace(fromlines,tolines)
 
         # create diffs iterator which generates side by side from/to data

Lib/email/utils.py

@@ -317,10 +317,13 @@ def parsedate_to_datetime(data):
     if parsed_date_tz is None:
         raise ValueError('Invalid date value or format "%s"' % str(data))
     *dtuple, tz = parsed_date_tz
-    if tz is None:
-        return datetime.datetime(*dtuple[:6])
-    return datetime.datetime(*dtuple[:6],
-            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+    try:
+        if tz is None:
+            return datetime.datetime(*dtuple[:6])
+        return datetime.datetime(*dtuple[:6],
+                tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+    except OverflowError as exc:
+        raise ValueError('Invalid date value or format "%s"' % str(data)) from exc
 
 
 def parseaddr(addr, *, strict=True):

Lib/ensurepip/__init__.py

@@ -10,7 +10,7 @@ from shutil import copy2
 
 
 __all__ = ["version", "bootstrap"]
-_PIP_VERSION = "26.1.1"
+_PIP_VERSION = "26.1.2"
 
 # Directory of system wheel packages. Some Linux distribution packaging
 # policies recommend against bundling dependencies. For example, Fedora

Lib/glob.py

@@ -34,8 +34,8 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
     If `dir_fd` is not None, it should be a file descriptor referring to a
     directory, and paths will then be relative to that directory.
 
-    If `include_hidden` is true, the patterns '*', '?', '**'  will match
-    hidden directories.
+    If `include_hidden` is true, wildcards can match path segments beginning
+    with a dot ('.').
 
     If `recursive` is true, the pattern '**' will match any files and
     zero or more directories and subdirectories.
@@ -64,8 +64,8 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
     If `dir_fd` is not None, it should be a file descriptor referring to a
     directory, and paths will then be relative to that directory.
 
-    If `include_hidden` is true, the patterns '*', '?', '**'  will match
-    hidden directories.
+    If `include_hidden` is true, wildcards can match path segments beginning
+    with a dot ('.').
 
     If `recursive` is true, the pattern '**' will match any files and
     zero or more directories and subdirectories.

Lib/html/parser.py

@@ -157,6 +157,9 @@ class HTMLParser(_markupbase.ParserBase):
         self.cdata_elem = None
         self._support_cdata = True
         self._escapable = True
+        self._pending = []
+        self._pending_len = 0
+        self._parse_threshold = 1
         super().reset()
 
     def feed(self, data):
@@ -165,11 +168,36 @@ class HTMLParser(_markupbase.ParserBase):
         Call this as often as you want, with as little or as much text
         as you want (may include '\n').
         """
-        self.rawdata = self.rawdata + data
-        self.goahead(0)
+        # Accumulate new data in a list and only join and parse it once
+        # enough has piled up.  Rescanning an unparsed buffer (e.g. an
+        # unterminated tag) and concatenating onto it on every call would
+        # both be quadratic in the input size.
+        self._pending_len += len(data)
+        if self._pending_len < self._parse_threshold:
+            self._pending.append(data)
+        else:
+            if not self._pending:
+                self.rawdata += data
+            else:
+                self._pending.append(data)
+                self.rawdata += ''.join(self._pending)
+                self._pending.clear()
+            self._pending_len = 0
+            n = len(self.rawdata)
+            self.goahead(0)
+            if len(self.rawdata) < n:
+                # Some data was parsed; resume on the next call.
+                self._parse_threshold = 1
+            else:
+                # Nothing was parsed; wait until the buffer doubles.
+                self._parse_threshold = len(self.rawdata)
 
     def close(self):
         """Handle any buffered data."""
+        if self._pending:
+            self.rawdata += ''.join(self._pending)
+            self._pending.clear()
+            self._pending_len = 0
         self.goahead(1)
 
     __starttag_text = None
@@ -387,9 +415,11 @@ class HTMLParser(_markupbase.ParserBase):
     def parse_comment(self, i, report=True):
         rawdata = self.rawdata
         assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()'
-        match = commentclose.search(rawdata, i+4)
+        # An empty comment is abruptly closed by the first ">" or "->",
+        # taking priority over a later "-->" or "--!>" close.
+        match = commentabruptclose.match(rawdata, i+4)
         if not match:
-            match = commentabruptclose.match(rawdata, i+4)
+            match = commentclose.search(rawdata, i+4)
             if not match:
                 return -1
         if report:

Lib/idlelib/CREDITS.txt

@@ -0,0 +1,48 @@
+Guido van Rossum, as well as being the creator of the Python language, is the
+original creator of IDLE.  Other contributors prior to Version 0.8 include
+Mark Hammond, Jeremy Hylton, Tim Peters, and Moshe Zadka.
+
+Until Python 2.3, IDLE's development was carried out in the SF IDLEfork project. The
+objective was to develop a version of IDLE which had an execution environment
+which could be initialized prior to each run of user code.
+IDLefork was merged into the Python code base in 2003.
+
+The IDLEfork project was initiated by David Scherer, with some help from Peter
+Schneider-Kamp and Nicholas Riley.  David wrote the first version of the RPC
+code and designed a fast turn-around environment for VPython.  Guido developed
+the RPC code and Remote Debugger currently integrated in IDLE.  Bruce Sherwood
+contributed considerable time testing and suggesting improvements.
+
+Besides David and Guido, the main developers who were active on IDLEfork
+are Stephen M. Gava, who implemented the configuration GUI, the new
+configuration system, and the About dialog, and Kurt B. Kaiser, who completed
+the integration of the RPC and remote debugger, implemented the threaded
+subprocess, and made a number of usability enhancements.
+
+Other contributors include Raymond Hettinger, Tony Lownds (Mac integration),
+Neal Norwitz (code check and clean-up), Ronald Oussoren (Mac integration),
+Noam Raphael (Code Context, Call Tips, many other patches), and Chui Tey (RPC
+integration, debugger integration and persistent breakpoints).
+
+Scott David Daniels, Tal Einat, Hernan Foffani, Christos Georgiou,
+Jim Jewett, Martin v. Löwis, Jason Orendorff, Guilherme Polo, Josh Robb,
+Nigel Rowe, Bruce Sherwood, Jeff Shute, and Weeble have submitted useful
+patches.  Thanks, guys!
+
+Major contributors since 2005:
+
+- 2005: Tal Einat
+- 2010: Terry Jan Reedy (current maintainer)
+- 2013: Roger Serwy
+- 2014: Saimadhav Heblikar
+- 2015: Mark Roseman
+- 2017: Louie Lu, Cheryl Sabella, and Serhiy Storchaka
+- 2025: Stan Ulbrych
+
+For additional details refer to NEWS.txt and Changelog.
+
+If we missed you, feel free to submit a PR with a summary of
+contributions (for instance, at least 5 merged PRs).
+
+
+

Lib/idlelib/HISTORY.txt

@@ -0,0 +1,296 @@
+IDLE History from Python 1.5.2 to 2.1
+=====================================
+
+This file contains the release messages for previous IDLE releases.
+As you read on you go back to the dark ages of IDLE's history.
+
+
+What's New in IDLEfork 0.8.1?
+=============================
+
+*Release date: 22-Jul-2001*
+
+- New tarball released as a result of the 'revitalisation' of the IDLEfork
+  project.
+
+- This release requires python 2.1 or better. Compatibility with earlier
+  versions of python (especially ancient ones like 1.5x) is no longer a
+  priority in IDLEfork development.
+
+- This release is based on a merging of the earlier IDLE fork work with current
+  cvs IDLE (post IDLE version 0.8), with some minor additional coding by Kurt
+  B. Kaiser and Stephen M. Gava.
+
+- This release is basically functional but also contains some known breakages,
+  particularly with running things from the shell window. Also the debugger is
+  not working, but I believe this was the case with the previous IDLE fork
+  release (0.7.1) as well.
+
+- This release is being made now to mark the point at which IDLEfork is
+  launching into a new stage of development.
+
+- IDLEfork CVS will now be branched to enable further development and
+  exploration of the two "execution in a remote process" patches submitted by
+  David Scherer (David's is currently in IDLEfork) and GvR, while stabilisation
+  and development of less heavyweight improvements (like user customisation)
+  can continue on the trunk.
+
+
+What's New in IDLEfork 0.7.1?
+==============================
+
+*Release date: 15-Aug-2000*
+
+- First project tarball released.
+
+- This was the first release of IDLE fork, which at this stage was a
+  combination of IDLE 0.5 and the VPython idle fork, with additional changes
+  coded by David Scherer, Peter Schneider-Kamp and Nicholas Riley.
+
+
+
+IDLEfork 0.7.1 - 29 May 2000
+-----------------------------
+
+   David Scherer  <dscherer@cmu.edu>
+
+- This is a modification of the CVS version of IDLE 0.5, updated as of
+  2000-03-09.  It is alpha software and might be unstable.  If it breaks, you
+  get to keep both pieces.
+
+- If you have problems or suggestions, you should either contact me or post to
+  the idle-dev mailing list (making it clear that you are using this modified
+  version of IDLE).
+
+- Changes:
+
+  - The ExecBinding module, a replacement for ScriptBinding, executes programs
+    in a separate process, piping standard I/O through an RPC mechanism to an
+    OnDemandOutputWindow in IDLE.  It supports executing unnamed programs
+    (through a temporary file).  It does not yet support debugging.
+
+  - When running programs with ExecBinding, tracebacks will be clipped to
+    exclude system modules.  If, however, a system module calls back into the
+    user program, that part of the traceback will be shown.
+
+  - The OnDemandOutputWindow class has been improved.  In particular, it now
+    supports a readline() function used to implement user input, and a
+    scroll_clear() operation which is used to hide the output of a previous run
+    by scrolling it out of the window.
+
+  - Startup behavior has been changed.  By default IDLE starts up with just a
+    blank editor window, rather than an interactive window.  Opening a file in
+    such a blank window replaces the (nonexistent) contents of that window
+    instead of creating another window.  Because of the need to have a
+    well-known port for the ExecBinding protocol, only one copy of IDLE can be
+    running.  Additional invocations use the RPC mechanism to report their
+    command line arguments to the copy already running.
+
+  - The menus have been reorganized.  In particular, the excessively large
+    'edit' menu has been split up into 'edit', 'format', and 'run'.
+
+  - 'Python Documentation' now works on Windows, if the win32api module is
+    present.
+
+  - A few key bindings have been changed: F1 now loads Python Documentation
+    instead of the IDLE help; shift-TAB is now a synonym for unindent.
+
+- New modules:
+
+  ExecBinding.py         Executes program through loader
+  loader.py              Bootstraps user program
+  protocol.py            RPC protocol
+  Remote.py              User-process interpreter
+  spawn.py               OS-specific code to start programs
+
+- Files modified:
+
+  autoindent.py          ( bindings tweaked )
+  bindings.py            ( menus reorganized )
+  config.txt             ( execbinding enabled )
+  editorwindow.py        ( new menus, fixed 'Python Documentation' )
+  filelist.py            ( hook for "open in same window" )
+  formatparagraph.py     ( bindings tweaked )
+  idle.bat               ( removed absolute pathname )
+  idle.pyw               ( weird bug due to import with same name? )
+  iobinding.py           ( open in same window, EOL convention )
+  keydefs.py             ( bindings tweaked )
+  outputwindow.py        ( readline, scroll_clear, etc )
+  pyshell.py             ( changed startup behavior )
+  readme.txt             ( <Recursion on file with id=1234567> )
+
+
+
+IDLE 0.5 - February 2000 - Release Notes
+----------------------------------------
+
+This is an early release of IDLE, my own attempt at a Tkinter-based
+IDE for Python.
+
+(For a more detailed change log, see the file ChangeLog.)
+
+FEATURES
+
+IDLE has the following features:
+
+- coded in 100% pure Python, using the Tkinter GUI toolkit (i.e. Tcl/Tk)
+
+- cross-platform: works on Windows and Unix (on the Mac, there are
+currently problems with Tcl/Tk)
+
+- multi-window text editor with multiple undo, Python colorizing
+and many other features, e.g. smart indent and call tips
+
+- Python shell window (a.k.a. interactive interpreter)
+
+- debugger (not complete, but you can set breakpoints, view  and step)
+
+USAGE
+
+The main program is in the file "idle.py"; on Unix, you should be able
+to run it by typing "./idle.py" to your shell.  On Windows, you can
+run it by double-clicking it; you can use idle.pyw to avoid popping up
+a DOS console.  If you want to pass command line arguments on Windows,
+use the batch file idle.bat.
+
+Command line arguments: files passed on the command line are executed,
+not opened for editing, unless you give the -e command line option.
+Try "./idle.py -h" to see other command line options.
+
+IDLE requires Python 1.5.2, so it is currently only usable with a
+Python 1.5.2 distribution.  (An older version of IDLE is distributed
+with Python 1.5.2; you can drop this version on top of it.)
+
+COPYRIGHT
+
+IDLE is covered by the standard Python copyright notice
+(https://www.python.org/doc/copyright).
+
+
+New in IDLE 0.5 (2/15/2000)
+---------------------------
+
+Tons of stuff, much of it contributed by Tim Peters and Mark Hammond:
+
+- Status bar, displaying current line/column (Moshe Zadka).
+
+- Better stack viewer, using tree widget.  (XXX Only used by Stack
+Viewer menu, not by the debugger.)
+
+- Format paragraph now recognizes Python block comments and reformats
+them correctly (MH)
+
+- New version of pyclbr.py parses top-level functions and understands
+much more of Python's syntax; this is reflected in the class and path
+browsers (TP)
+
+- Much better auto-indent; knows how to indent the insides of
+multi-line statements (TP)
+
+- Call tip window pops up when you type the name of a known function
+followed by an open parenthesis.  Hit ESC or click elsewhere in the
+window to close the tip window (MH)
+
+- Comment out region now inserts ## to make it stand out more (TP)
+
+- New path and class browsers based on a tree widget that looks
+familiar to Windows users
+
+- Reworked script running commands to be more intuitive: I/O now
+always goes to the *Python Shell* window, and raw_input() works
+correctly.  You use F5 to import/reload a module: this adds the module
+name to the __main__ namespace.  You use Control-F5 to run a script:
+this runs the script *in* the __main__ namespace.  The latter also
+sets sys.argv[] to the script name
+
+
+New in IDLE 0.4 (4/7/99)
+------------------------
+
+Most important change: a new menu entry "File -> Path browser", shows
+a 4-column hierarchical browser which lets you browse sys.path,
+directories, modules, and classes.  Yes, it's a superset of the Class
+browser menu entry.  There's also a new internal module,
+MultiScrolledLists.py, which provides the framework for this dialog.
+
+
+New in IDLE 0.3 (2/17/99)
+-------------------------
+
+Most important changes:
+
+- Enabled support for running a module, with or without the debugger.
+Output goes to a new window.  Pressing F5 in a module is effectively a
+reload of that module; Control-F5 loads it under the debugger.
+
+- Re-enable tearing off the Windows menu, and make a torn-off Windows
+menu update itself whenever a window is opened or closed.
+
+- Menu items can now be have a checkbox (when the menu label starts
+with "!"); use this for the Debugger and "Auto-open stack viewer"
+(was: JIT stack viewer) menu items.
+
+- Added a Quit button to the Debugger API.
+
+- The current directory is explicitly inserted into sys.path.
+
+- Fix the debugger (when using Python 1.5.2b2) to use canonical
+filenames for breakpoints, so these actually work.  (There's still a
+lot of work to be done to the management of breakpoints in the
+debugger though.)
+
+- Closing a window that is still colorizing now actually works.
+
+- Allow dragging of the separator between the two list boxes in the
+class browser.
+
+- Bind ESC to "close window" of the debugger, stack viewer and class
+browser.  It removes the selection highlighting in regular text
+windows.  (These are standard Windows conventions.)
+
+
+New in IDLE 0.2 (1/8/99)
+------------------------
+
+Lots of changes; here are the highlights:
+
+General:
+
+- You can now write and configure your own IDLE extension modules; see
+extend.txt.
+
+
+File menu:
+
+The command to open the Python shell window is now in the File menu.
+
+
+Edit menu:
+
+New Find dialog with more options; replace dialog; find in files dialog.
+
+Commands to tabify or untabify a region.
+
+Command to format a paragraph.
+
+
+Debug menu:
+
+JIT (Just-In-Time) stack viewer toggle -- if set, the stack viewer
+automatically pops up when you get a traceback.
+
+Windows menu:
+
+Zoom height -- make the window full height.
+
+
+Help menu:
+
+The help text now show up in a regular window so you can search and
+even edit it if you like.
+
+
+
+IDLE 0.1 was distributed with the Python 1.5.2b1 release on 12/22/98.
+
+======================================================================

Lib/idlelib/Icons/README.txt

@@ -0,0 +1,51 @@
+IDLE-PYTHON LOGOS
+
+These are sent to tk on Windows, *NIX, and non-Aqua macOS
+in pyshell following "# set application icon".
+
+
+2006?: Andrew Clover made variously sized python icons for win23.
+https://www.doxdesk.com/software/py/pyicons.html
+
+2006: 16, 32, and 48 bit .png versions were copied to CPython
+as Python application icons, maybe in PC/icons/py.ico.
+https://github.com/python/cpython/issues/43372
+
+2014: They were copied (perhaps a bit revised) to idlelib/Icons.
+https://github.com/python/cpython/issues/64605
+.gif versions were also added.
+
+2020: Add Clover's 256-bit image.
+https://github.com/python/cpython/issues/82620
+Other fixups were done.
+
+The idle.ico file used for Windows was created with ImageMagick:
+  $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico
+** This needs redoing whenever files are changed.
+?? Do Start, Desktop, and Taskbar use idlelib/Icons files?
+
+Issue added Windows Store PC/icons/idlex44.png and .../idlex150.png.
+https://github.com/python/cpython/pull/22817
+?? Should these be updated with major changes?
+
+2022: Optimize .png images in CPython repository with external program.
+https://github.com/python/cpython/pull/21348
+idle.ico (and idlex##) were not updated.
+
+The idlexx.gif files are only needed for *nix running tcl/tk 8.5.
+As of 2022, this was known true for 1 'major' Linux distribution.
+(Same would be true for any non-Aqua macOS with 8.5, but now none?)
+Can be deleted when we require 8.6 or it is known always used.
+
+Future: Derivatives of Python logo should be submitted for approval.
+PSF Trademark Working Group / Committee psf-trademarks@python.org
+https://www.python.org/community/logos/  # Original files
+https://www.python.org/psf/trademarks-faq/
+https://www.python.org/psf/trademarks/  # Usage.
+
+
+OTHER GIFS: These are used by browsers using idlelib.tree.
+At least some will not be used when tree is replaced by ttk.Treeview.
+
+
+Edited 2024 August 26 by TJR.

Lib/idlelib/NEWS2x.txt

@@ -0,0 +1,660 @@
+What's New in IDLE 2.7? (Merged into 3.1 before 2.7 release.)
+=======================
+*Release date: 03-Jul-2010*
+
+- idle.py modified and simplified to better support developing experimental
+  versions of IDLE which are not installed in the standard location.
+
+- OutputWindow/PyShell right click menu "Go to file/line" wasn't working with
+  file paths containing spaces.  Bug 5559.
+
+- Windows: Version string for the .chm help file changed, file not being
+  accessed  Patch 5783 Guilherme Polo
+
+- Allow multiple IDLE GUI/subprocess pairs to exist simultaneously. Thanks to
+  David Scherer for suggesting the use of an ephemeral port for the GUI.
+  Patch 1529142 Weeble.
+
+- Remove port spec from run.py and fix bug where subprocess fails to
+  extract port from command line when warnings are present.
+
+- Tk 8.5 Text widget requires 'wordprocessor' tabstyle attr to handle
+  mixed space/tab properly. Issue 5129, patch by Guilherme Polo.
+
+- Issue #3549: On MacOS the preferences menu was not present
+
+- IDLE would print a "Unhandled server exception!" message when internal
+  debugging is enabled.
+
+- Issue #4455: IDLE failed to display the windows list when two windows have
+  the same title.
+
+- Issue #4383: When IDLE cannot make the connection to its subprocess, it would
+  fail to properly display the error message.
+
+- help() was not paging to the shell.  Issue1650.
+
+- CodeContext was not importing.
+
+- Corrected two 3.0 compatibility errors reported by Mark Summerfield:
+  http://mail.python.org/pipermail/python-3000/2007-December/011491.html
+
+- Shell was not colorizing due to bug introduced at r57998,  Bug 1586.
+
+- Issue #1585: IDLE uses non-existent xrange() function.
+
+- Windows EOL sequence not converted correctly, encoding error.
+  Caused file save to fail. Bug 1130.
+
+- IDLE converted to Python 3000 syntax.
+
+- Strings became Unicode.
+
+- CallTips module now uses the inspect module to produce the argspec.
+
+- IDLE modules now use absolute import instead of implied relative import.
+
+- atexit call replaces sys.exitfunc.  The functionality of delete-exitfunc flag
+  in config-main.cfg remains unchanged: if set, registered exit functions will
+  be cleared before IDLE exits.
+
+
+What's New in IDLE 2.6
+======================
+*Release date: 01-Oct-2008*, merged into 3.0 releases detailed above (3.0rc2)
+
+- Issue #2665: On Windows, an IDLE installation upgraded from an old version
+  would not start if a custom theme was defined.
+
+- Home / Control-A toggles between left margin and end of leading white
+  space.  issue1196903, patch by Jeff Shute.
+
+- Improved AutoCompleteWindow logic.  issue2062, patch by Tal Einat.
+
+- Autocompletion of filenames now support alternate separators, e.g. the
+  '/' char on Windows.  issue2061 Patch by Tal Einat.
+
+- Configured selection highlighting colors were ignored; updating highlighting
+  in the config dialog would cause non-Python files to be colored as if they
+  were Python source; improve use of ColorDelagator.  Patch 1334. Tal Einat.
+
+- ScriptBinding event handlers weren't returning 'break'. Patch 2050, Tal Einat
+
+- There was an error on exit if no sys.exitfunc was defined. Issue 1647.
+
+- Could not open files in .idlerc directory if latter was hidden on Windows.
+  Issue 1743, Issue 1862.
+
+- Configure Dialog: improved layout for keybinding.  Patch 1457 Tal Einat.
+
+- tabpage.py updated: tabbedPages.py now supports multiple dynamic rows
+  of tabs.  Patch 1612746 Tal Einat.
+
+- Add confirmation dialog before printing.  Patch 1717170 Tal Einat.
+
+- Show paste position if > 80 col.  Patch 1659326 Tal Einat.
+
+- Update cursor color without restarting.  Patch 1725576 Tal Einat.
+
+- Allow keyboard interrupt only when user code is executing in subprocess.
+  Patch 1225 Tal Einat (reworked from IDLE-Spoon).
+
+- configDialog cleanup. Patch 1730217 Tal Einat.
+
+- textView cleanup. Patch 1718043 Tal Einat.
+
+- Clean up EditorWindow close.
+
+- Patch 1693258: Fix for duplicate "preferences" menu-OS X. Backport of r56204.
+
+- OSX: Avoid crash for those versions of Tcl/Tk which don't have a console
+
+- Bug in idlelib.MultiCall: Options dialog was crashing IDLE if there was an
+  option in config-extensions w/o a value. Patch #1672481, Tal Einat
+
+- Corrected some bugs in AutoComplete.  Also, Page Up/Down in ACW implemented;
+  mouse and cursor selection in ACWindow implemented; double Tab inserts
+  current selection and closes ACW (similar to double-click and Return); scroll
+  wheel now works in ACW.  Added AutoComplete instructions to IDLE Help.
+
+- AutoCompleteWindow moved below input line, will move above if there
+  isn't enough space.  Patch 1621265 Tal Einat
+
+- Calltips now 'handle' tuples in the argument list (display '<tuple>' :)
+  Suggested solution by Christos Georgiou, Bug 791968.
+
+- Add 'raw' support to configHandler. Patch 1650174 Tal Einat.
+
+- Avoid hang when encountering a duplicate in a completion list. Bug 1571112.
+
+- Patch #1362975: Rework CodeContext indentation algorithm to
+  avoid hard-coding pixel widths.
+
+- Bug #813342: Start the IDLE subprocess with -Qnew if the parent
+  is started with that option.
+
+- Honor the "Cancel" action in the save dialog (Debian bug #299092)
+
+- Some syntax errors were being caught by tokenize during the tabnanny
+  check, resulting in obscure error messages.  Do the syntax check
+  first.  Bug 1562716, 1562719
+
+- IDLE's version number takes a big jump to match the version number of
+  the Python release of which it's a part.
+
+
+What's New in IDLE 1.2?
+=======================
+*Release date: 19-SEP-2006*
+
+- File menu hotkeys: there were three 'p' assignments.  Reassign the
+  'Save Copy As' and 'Print' hotkeys to 'y' and 't'.  Change the
+  Shell hotkey from 's' to 'l'.
+
+- IDLE honors new quit() and exit() commands from site.py Quitter() object.
+  Patch 1540892, Jim Jewett
+
+- The 'with' statement is now a Code Context block opener.
+  Patch 1540851, Jim Jewett
+
+- Retrieval of previous shell command was not always preserving indentation
+  (since 1.2a1) Patch 1528468 Tal Einat.
+
+- Changing tokenize (39046) to detect dedent broke tabnanny check (since 1.2a1)
+
+- ToggleTab dialog was setting indent to 8 even if cancelled (since 1.2a1).
+
+- When used w/o subprocess, all exceptions were preceded by an error
+  message claiming they were IDLE internal errors (since 1.2a1).
+
+- Bug #1525817: Don't truncate short lines in IDLE's tool tips.
+
+- Bug #1517990: IDLE keybindings on MacOS X now work correctly
+
+- Bug #1517996: IDLE now longer shows the default Tk menu when a
+  path browser, class browser or debugger is the frontmost window on MacOS X
+
+- EditorWindow.test() was failing.  Bug 1417598
+
+- EditorWindow failed when used stand-alone if sys.ps1 not set.
+  Bug 1010370 Dave Florek
+
+- Tooltips failed on new-syle class __init__ args.  Bug 1027566 Loren Guthrie
+
+- Avoid occasional failure to detect closing paren properly.
+  Patch 1407280 Tal Einat
+
+- Rebinding Tab key was inserting 'tab' instead of 'Tab'.  Bug 1179168.
+
+- Colorizer now handles #<builtin> correctly, also unicode strings and
+  'as' keyword in comment directly following import command. Closes 1325071.
+  Patch 1479219 Tal Einat
+
+- Patch #1162825: Support non-ASCII characters in IDLE window titles.
+
+- Source file f.flush() after writing; trying to avoid lossage if user
+  kills GUI.
+
+- Options / Keys / Advanced dialog made functional.  Also, allow binding
+  of 'movement' keys.
+
+- 'syntax' patch adds improved calltips and a new class attribute listbox.
+  MultiCall module allows binding multiple actions to an event.
+  Patch 906702 Noam Raphael
+
+- Better indentation after first line of string continuation.
+  IDLEfork Patch 681992, Noam Raphael
+
+- Fixed CodeContext alignment problem, following suggestion from Tal Einat.
+
+- Increased performance in CodeContext extension  Patch 936169 Noam Raphael
+
+- Mac line endings were incorrect when pasting code from some browsers
+  when using X11 and the Fink distribution.  Python Bug 1263656.
+
+- <Enter> when cursor is on a previous command retrieves that command.  Instead
+  of replacing the input line, the previous command is now appended to the
+  input line. Indentation is preserved, and undo is enabled.
+  Patch 1196917  Jeff Shute
+
+- Clarify "tab/space" Error Dialog and "Tab Width" Dialog associated with
+  the Untabify command.
+
+- Corrected "tab/space" Error Dialog to show correct menu for Untabify.
+  Patch 1196980 Jeff Shute
+
+- New files are colorized by default, and colorizing is removed when
+  saving as non-Python files. Patch 1196895 Jeff Shute
+  Closes Python Bugs 775012 and 800432, partial fix IDLEfork 763524
+
+- Improve subprocess link error notification.
+
+- run.py: use Queue's blocking feature instead of sleeping in the main
+  loop.  Patch # 1190163 Michiel de Hoon
+
+- Add config-main option to make the 'history' feature non-cyclic.
+  Default remains cyclic.  Python Patch 914546 Noam Raphael.
+
+- Removed ability to configure tabs indent from Options dialog.  This 'feature'
+  has never worked and no one has complained.  It is still possible to set a
+  default tabs (v. spaces) indent 'manually' via config-main.def (or to turn on
+  tabs for the current EditorWindow via the Format menu) but IDLE will
+  encourage indentation via spaces.
+
+- Enable setting the indentation width using the Options dialog.
+  Bug # 783877
+
+- Add keybindings for del-word-left and del-word-right.
+
+- Discourage using an indent width other than 8 when using tabs to indent
+  Python code.
+
+- Restore use of EditorWindow.set_indentation_params(), was dead code since
+  Autoindent was merged into EditorWindow.  This allows IDLE to conform to the
+  indentation width of a loaded file.  (But it still will not switch to tabs
+  even if the file uses tabs.)  Any change in indent width is local to that
+  window.
+
+- Add Tabnanny check before Run/F5, not just when Checking module.
+
+- If an extension can't be loaded, print warning and skip it instead of
+  erroring out.
+
+- Improve error handling when .idlerc can't be created (warn and exit).
+
+- The GUI was hanging if the shell window was closed while a raw_input()
+  was pending.  Restored the quit() of the readline() mainloop().
+  http://mail.python.org/pipermail/idle-dev/2004-December/002307.html
+
+- The remote procedure call module rpc.py can now access data attributes of
+  remote registered objects.  Changes to these attributes are local, however.
+
+
+What's New in IDLE 1.1?
+=======================
+*Release date: 30-NOV-2004*
+
+- On OpenBSD, terminating IDLE with ctrl-c from the command line caused a
+  stuck subprocess MainThread because only the SocketThread was exiting.
+
+- Saving a Keyset w/o making changes (by using the "Save as New Custom Key Set"
+  button) caused IDLE to fail on restart (no new keyset was created in
+  config-keys.cfg).  Also true for Theme/highlights.  Python Bug 1064535.
+
+- A change to the linecache.py API caused IDLE to exit when an exception was
+  raised while running without the subprocess (-n switch).  Python Bug 1063840.
+
+- When paragraph reformat width was made configurable, a bug was
+  introduced that caused reformatting of comment blocks to ignore how
+  far the block was indented, effectively adding the indentation width
+  to the reformat width.  This has been repaired, and the reformat
+  width is again a bound on the total width of reformatted lines.
+
+- Improve keyboard focus binding, especially in Windows menu.  Improve
+  window raising, especially in the Windows menu and in the debugger.
+  IDLEfork 763524.
+
+- If user passes a non-existent filename on the commandline, just
+  open a new file, don't raise a dialog.  IDLEfork 854928.
+
+- EditorWindow.py was not finding the .chm help file on Windows.  Typo
+  at Rev 1.54.  Python Bug 990954
+
+- checking sys.platform for substring 'win' was breaking IDLE docs on Mac
+  (darwin).  Also, Mac Safari browser requires full file:// URIs.  SF 900580.
+
+- Redirect the warning stream to the shell during the ScriptBinding check of
+  user code and format the warning similarly to an exception for both that
+  check and for runtime warnings raised in the subprocess.
+
+- CodeContext hint pane visibility state is now persistent across sessions.
+  The pane no longer appears in the shell window.  Added capability to limit
+  extensions to shell window or editor windows.  Noam Raphael addition
+  to Patch 936169.
+
+- Paragraph reformat width is now a configurable parameter in the
+  Options GUI.
+
+- New Extension: CodeContext.  Provides block structuring hints for code
+  which has scrolled above an edit window. Patch 936169 Noam Raphael.
+
+- If nulls somehow got into the strings in recent-files.lst
+  EditorWindow.update_recent_files_list() was failing.  Python Bug 931336.
+
+- If the normal background is changed via Configure/Highlighting, it will
+  update immediately, thanks to the previously mentioned patch by Nigel Rowe.
+
+- Add a highlight theme for builtin keywords.  Python Patch 805830 Nigel Rowe
+  This also fixed IDLEfork bug [ 693418 ] Normal text background color not
+  refreshed and Python bug [897872 ] Unknown color name on HP-UX
+
+- rpc.py:SocketIO - Large modules were generating large pickles when downloaded
+  to the execution server.  The return of the OK response from the subprocess
+  initialization was interfering and causing the sending socket to be not
+  ready.  Add an IO ready test to fix this.  Moved the polling IO ready test
+  into pollpacket().
+
+- Fix typo in rpc.py, s/b "pickle.PicklingError" not "pickle.UnpicklingError".
+
+- Added a Tk error dialog to run.py inform the user if the subprocess can't
+  connect to the user GUI process.  Added a timeout to the GUI's listening
+  socket.  Added Tk error dialogs to PyShell.py to announce a failure to bind
+  the port or connect to the subprocess.  Clean up error handling during
+  connection initiation phase.  This is an update of Python Patch 778323.
+
+- Print correct exception even if source file changed since shell was
+  restarted.  IDLEfork Patch 869012 Noam Raphael
+
+- Keybindings with the Shift modifier now work correctly.  So do bindings which
+  use the Space key.  Limit unmodified user keybindings to the function keys.
+  Python Bug 775353, IDLEfork Bugs 755647, 761557
+
+- After an exception, run.py was not setting the exception vector. Noam
+  Raphael suggested correcting this so pdb's postmortem pm() would work.
+  IDLEfork Patch 844675
+
+- IDLE now does not fail to save the file anymore if the Tk buffer is not a
+  Unicode string, yet eol_convention is.  Python Bugs 774680, 788378
+
+- IDLE didn't start correctly when Python was installed in "Program Files" on
+  W2K and XP.  Python Bugs 780451, 784183
+
+- config-main.def documentation incorrectly referred to idle- instead of
+  config-  filenames.  SF 782759  Also added note about .idlerc location.
+
+
+What's New in IDLE 1.0?
+=======================
+*Release date: 29-Jul-2003*
+
+- Added a banner to the shell discussing warnings possibly raised by personal
+  firewall software.  Added same comment to README.txt.
+
+- Calltip error when docstring was None  Python Bug 775541
+
+- Updated extend.txt, help.txt, and config-extensions.def to correctly
+  reflect the current status of the configuration system.  Python Bug 768469
+
+- Fixed: Call Tip Trimming May Loop Forever. Python Patch 769142 (Daniels)
+
+- Replaced apply(f, args, kwds) with f(*args, **kwargs) to improve performance
+  Python Patch 768187
+
+- Break or continue statements outside a loop were causing IDLE crash
+  Python Bug 767794
+
+- Convert Unicode strings from readline to IOBinding.encoding.  Also set
+  sys.std{in|out|err}.encoding, for both the local and the subprocess case.
+  SF IDLEfork patch 682347.
+
+- Extend AboutDialog.ViewFile() to support file encodings.  Make the CREDITS
+  file Latin-1.
+
+- Updated the About dialog to reflect re-integration into Python.  Provide
+  buttons to display Python's NEWS, License, and Credits, plus additional
+  buttons for IDLE's README and NEWS.
+
+- TextViewer() now has a third parameter which allows inserting text into the
+  viewer instead of reading from a file.
+
+- (Created the .../Lib/idlelib directory in the Python CVS, which is a clone of
+  IDLEfork modified to install in the Python environment.  The code in the
+  interrupt module has been moved to thread.interrupt_main(). )
+
+- Printing the Shell window was failing if it was not saved first SF 748975
+
+- When using the Search in Files dialog, if the user had a selection
+  highlighted in his Editor window, insert it into the dialog search field.
+
+- The Python Shell entry was disappearing from the Windows menu.
+
+- Update the Windows file list when a file name change occurs
+
+- Change to File / Open Module: always pop up the dialog, using the current
+  selection as the default value.  This is easier to use habitually.
+
+- Avoided a problem with starting the subprocess when 'localhost' doesn't
+  resolve to the user's loopback interface.  SF 747772
+
+- Fixed an issue with highlighted errors never de-colorizing.  SF 747677.  Also
+  improved notification of Tabnanny Token Error.
+
+- File / New will by default save in the directory of the Edit window from
+  which it was initiated.  SF 748973 Guido van Rossum patch.
+
+
+What's New in IDLEfork 0.9b1?
+=============================
+*Release date: 02-Jun-2003*
+
+- The current working directory of the execution environment (and shell
+  following completion of execution) is now that of the module being run.
+
+- Added the delete-exitfunc option to config-main.def.  (This option is not
+  included in the Options dialog.)  Setting this to True (the default) will
+  cause IDLE to not run sys.exitfunc/atexit when the subprocess exits.
+
+- IDLE now preserves the line ending codes when editing a file produced on
+  a different platform. SF 661759,  SF 538584
+
+- Reduced default editor font size to 10 point and increased window height
+  to provide a better initial impression on Windows.
+
+- Options / Fonts/Tabs / Set Base Editor Font: List box was not highlighting
+  the default font when first installed on Windows.  SF 661676
+
+- Added Autosave feature: when user runs code from edit window, if the file
+  has been modified IDLE will silently save it if Autosave is enabled.  The
+  option is set in the Options dialog, and the default is to prompt the
+  user to save the file.   SF 661318 Bruce Sherwood patch.
+
+- Improved the RESTART annotation in the shell window when the user restarts
+  the shell while it is generating output.  Also improved annotation when user
+  repeatedly hammers the Ctrl-F6 restart.
+
+- Allow IDLE to run when not installed and cwd is not the IDLE directory
+  SF Patch 686254 "Run IDLEfork from any directory without set-up" - Raphael
+
+- When a module is run from an EditorWindow: if its directory is not in
+  sys.path, prepend it.  This allows the module to import other modules in
+  the same directory.  Do the same for a script run from the command line.
+
+- Correctly restart the subprocess if it is running user code and the user
+  attempts to run some other module or restarts the shell.  Do the same if
+  the link is broken and it is possible to restart the subprocess and re-
+  connect to the GUI.   SF RFE 661321.
+
+- Improved exception reporting when running commands or scripts from the
+  command line.
+
+- Added a -n command line switch to start IDLE without the subprocess.
+  Removed the Shell menu when running in that mode.  Updated help messages.
+
+- Added a comment to the shell startup header to indicate when IDLE is not
+  using the subprocess.
+
+- Restore the ability to run without the subprocess.  This can be important for
+  some platforms or configurations.  (Running without the subprocess allows the
+  debugger to trace through parts of IDLE itself, which may or may not be
+  desirable, depending on your point of view.  In addition, the traditional
+  reload/import tricks must be use if user source code is changed.)  This is
+  helpful for developing IDLE using IDLE, because one instance can be used to
+  edit the code and a separate instance run to test changes.  (Multiple
+  concurrent IDLE instances with subprocesses is a future feature)
+
+- Improve the error message a user gets when saving a file with non-ASCII
+  characters and no source encoding is specified.  Done by adding a dialog
+  'EncodingMessage', which contains the line to add in a fixed-font entry
+  widget, and which has a button to add that line to the file automatically.
+  Also, add a configuration option 'EditorWindow/encoding', which has three
+  possible values: none, utf-8, and locale. None is the default: IDLE will show
+  this dialog when non-ASCII characters are encountered. utf-8 means that files
+  with non-ASCII characters are saved as utf-8-with-bom. locale means that
+  files are saved in the locale's encoding; the dialog is only displayed if the
+  source contains characters outside the locale's charset.  SF 710733 - Loewis
+
+- Improved I/O response by tweaking the wait parameter in various
+  calls to signal.signal().
+
+- Implemented a threaded subprocess which allows interrupting a pass
+  loop in user code using the 'interrupt' extension.  User code runs
+  in MainThread, while the RPCServer is handled by SockThread.  This is
+  necessary because Windows doesn't support signals.
+
+- Implemented the 'interrupt' extension module, which allows a subthread
+  to raise a KeyboardInterrupt in the main thread.
+
+- Attempting to save the shell raised an error related to saving
+  breakpoints, which are not implemented in the shell
+
+- Provide a correct message when 'exit' or 'quit' are entered at the
+  IDLE command prompt  SF 695861
+
+- Eliminate extra blank line in shell output caused by not flushing
+  stdout when user code ends with an unterminated print. SF 695861
+
+- Moved responsibility for exception formatting (i.e. pruning IDLE internal
+  calls) out of rpc.py into the client and server.
+
+- Exit IDLE cleanly even when doing subprocess I/O
+
+- Handle subprocess interrupt with an RPC message.
+
+- Restart the subprocess if it terminates itself. (VPython programs do that)
+
+- Support subclassing of exceptions, including in the shell, by moving the
+  exception formatting to the subprocess.
+
+
+What's New in IDLEfork 0.9 Alpha 2?
+===================================
+*Release date: 27-Jan-2003*
+
+- Updated INSTALL.txt to claify use of the python2 rpm.
+
+- Improved formatting in IDLE Help.
+
+- Run menu: Replace "Run Script" with "Run Module".
+
+- Code encountering an unhandled exception under the debugger now shows
+  the correct traceback, with IDLE internal levels pruned out.
+
+- If an exception occurs entirely in IDLE, don't prune the IDLE internal
+  modules from the traceback displayed.
+
+- Class Browser and Path Browser now use Alt-Key-2 for vertical zoom.
+
+- IDLE icons will now install correctly even when setup.py is run from the
+  build directory
+
+- Class Browser now compatible with Python2.3 version of pyclbr.py
+
+- Left cursor move in presence of selected text now moves from left end
+  of the selection.
+
+- Add Meta keybindings to "IDLE Classic Windows" to handle reversed
+  Alt/Meta on some Linux distros.
+
+- Change default: IDLE now starts with Python Shell.
+
+- Removed the File Path from the Additional Help Sources scrolled list.
+
+- Add capability to access Additional Help Sources on the web if the
+  Help File Path begins with //http or www.  (Otherwise local path is
+  validated, as before.)
+
+- Additional Help Sources were not being posted on the Help menu in the
+  order entered.  Implement sorting the list by [HelpFiles] 'option'
+  number.
+
+- Add Browse button to New Help Source dialog.  Arrange to start in
+  Python/Doc if platform is Windows, otherwise start in current directory.
+
+- Put the Additional Help Sources directly on the Help menu instead of in
+  an Extra Help cascade menu.  Rearrange the Help menu so the Additional
+  Help Sources come last.  Update help.txt appropriately.
+
+- Fix Tk root pop-ups in configSectionNameDialog.py  and configDialog.py
+
+- Uniform capitalization in General tab of ConfigDialog, update the doc string.
+
+- Fix bug in ConfigDialog where SaveAllChangedConfig() was unexpectedly
+  deleting Additional Help Sources from the user's config file.
+
+- Make configHelpSourceEdit OK button the default and bind <Return>
+
+- Fix Tk root pop-ups in configHelpSourceEdit: error dialogs not attached
+  to parents.
+
+- Use os.startfile() to open both Additional Help and Python Help on the
+  Windows platform.  The application associated with the file type will act as
+  the viewer.  Windows help files (.chm) are now supported via the
+  Settings/General/Additional Help facility.
+
+- If Python Help files are installed locally on Linux, use them instead of
+  accessing python.org.
+
+- Make the methods for finding the Python help docs more robust, and make
+  them work in the installed configuration, also.
+
+- On the Save Before Run dialog, make the OK button the default.  One
+  less mouse action!
+
+- Add a method: EditorWindow.get_geometry() for future use in implementing
+  window location persistence.
+
+- Removed the "Help/Advice" menu entry.  Thanks, David!  We'll remember!
+
+- Change the "Classic Windows" theme's paste key to be <ctrl-v>.
+
+- Rearrange the Shell menu to put Stack Viewer entries adjacent.
+
+- Add the ability to restart the subprocess interpreter from the shell window;
+  add an associated menu entry "Shell/Restart" with binding Control-F6.  Update
+  IDLE help.
+
+- Upon a restart, annotate the shell window with a "restart boundary".  Add a
+  shell window menu "Shell/View Restart" with binding F6 to jump to the most
+  recent restart boundary.
+
+- Add Shell menu to Python Shell; change "Settings" to "Options".
+
+- Remove incorrect comment in setup.py: IDLEfork is now installed as a package.
+
+- Add INSTALL.txt, HISTORY.txt, NEWS.txt to installed configuration.
+
+- In installer text, fix reference to Visual Python, should be VPython.
+  Properly credit David Scherer.
+
+- Modified idle, idle.py, idle.pyw to improve exception handling.
+
+
+What's New in IDLEfork 0.9 Alpha 1?
+===================================
+*Release date: 31-Dec-2002*
+
+- First release of major new functionality.  For further details refer to
+  Idle-dev and/or the Sourceforge CVS.
+
+- Adapted to the Mac platform.
+
+- Overhauled the IDLE startup options and revised the idle -h help message,
+  which provides details of command line usage.
+
+- Multiple bug fixes and usability enhancements.
+
+- Introduced the new RPC implementation, which includes a debugger.  The output
+  of user code is to the shell, and the shell may be used to inspect the
+  environment after the run has finished.  (In version 0.8.1 the shell
+  environment was separate from the environment of the user code.)
+
+- Introduced the configuration GUI and a new About dialog.
+
+- Removed David Scherer's Remote Procedure Call code and replaced with Guido
+  van Rossum's.  GvR code has support for the IDLE debugger and uses the shell
+  to inspect the environment of code Run from an Edit window.  Files removed:
+  ExecBinding.py, loader.py, protocol.py, Remote.py, spawn.py
+
+--------------------------------------------------------------------
+Refer to HISTORY.txt for additional information on earlier releases.
+--------------------------------------------------------------------

Lib/idlelib/README.txt

@@ -0,0 +1,290 @@
+README.txt: an index to idlelib files and the IDLE menu.
+
+IDLE is Python's Integrated Development and Learning
+Environment.  The user documentation is part of the Library Reference and
+is available in IDLE by selecting Help => IDLE Help.  This README documents
+idlelib for IDLE developers and curious users.
+
+IDLELIB FILES lists files alphabetically by category,
+with a short description of each.
+
+IDLE MENU show the menu tree, annotated with the module
+or module object that implements the corresponding function.
+
+This file is descriptive, not prescriptive, and may have errors
+and omissions and lag behind changes in idlelib.
+
+
+IDLELIB FILES
+=============
+
+Implementation files not in IDLE MENU are marked (nim).
+
+Startup
+-------
+__init__.py  # import, does nothing
+__main__.py  # -m, starts IDLE
+idle.bat
+idle.py
+idle.pyw
+
+Implementation
+--------------
+autocomplete.py   # Complete attribute names or filenames.
+autocomplete_w.py # Display completions.
+autoexpand.py     # Expand word with previous word in file.
+browser.py        # Create module browser window.
+calltip.py        # Create calltip text.
+calltip_w.py      # Display calltip.
+codecontext.py    # Show compound statement headers otherwise not visible.
+colorizer.py      # Colorize text (nim).
+config.py         # Load, fetch, and save configuration (nim).
+configdialog.py   # Display user configuration dialogs.
+config_key.py     # Change keybindings.
+debugger.py       # Debug code run from shell or editor; show window.
+debugger_r.py     # Debug code run in remote process.
+debugobj.py       # Define class used in stackviewer.
+debugobj_r.py     # Communicate objects between processes with rpc (nim).
+delegator.py      # Define base class for delegators (nim).
+dynoption.py      # Define mutable OptionMenu widget (nim)
+editor.py         # Define most of editor and utility functions.
+filelist.py       # Open files and manage list of open windows (nim).
+format.py         # Define format menu options.
+grep.py           # Find all occurrences of pattern in multiple files.
+help.py           # Display IDLE's html doc.
+help_about.py     # Display About IDLE dialog.
+history.py        # Get previous or next user input in shell (nim)
+hyperparser.py    # Parse code around a given index.
+iomenu.py         # Open, read, and write files
+macosx.py         # Help IDLE run on Macs (nim).
+mainmenu.py       # Define most of IDLE menu.
+multicall.py      # Wrap tk widget to allow multiple calls per event (nim).
+outwin.py         # Create window for grep output.
+parenmatch.py     # Match fenceposts: (), [], and {}.
+pathbrowser.py    # Create path browser window.
+percolator.py     # Manage delegator stack (nim).
+pyparse.py        # Give information on code indentation
+pyshell.py        # Start IDLE, manage shell, complete editor window
+query.py          # Query user for information
+redirector.py     # Intercept widget subcommands (for percolator) (nim).
+replace.py        # Search and replace pattern in text.
+rpc.py            # Communicate between idle and user processes (nim).
+run.py            # Manage user code execution subprocess.
+runscript.py      # Check and run user code.
+scrolledlist.py   # Define scrolledlist widget for IDLE (nim).
+search.py         # Search for pattern in text.
+searchbase.py     # Define base for search, replace, and grep dialogs.
+searchengine.py   # Define engine for all 3 search dialogs.
+sidebar.py        # Define line number and shell prompt sidebars.
+squeezer.py       # Squeeze long shell output (nim).
+stackviewer.py    # View stack after exception.
+statusbar.py      # Define status bar for windows (nim).
+tabbedpages.py    # Define tabbed pages widget (nim).
+textview.py       # Define read-only text widget (nim).
+tooltip.py        # Define popups for calltips, squeezer (nim).
+tree.py           # Define tree widget, used in browsers (nim).
+undo.py           # Manage undo stack.
+util.py           # Define common objects imported elsewhere (nim).
+windows.py        # Manage window list and define listed top level.
+zoomheight.py     # Zoom window to full height of screen.
+zzdummy.py        # Example extension.
+
+Configuration
+-------------
+config-extensions.def # Defaults for extensions
+config-highlight.def  # Defaults for colorizing
+config-keys.def       # Defaults for key bindings
+config-main.def       # Defaults for font and general tabs
+
+Text
+----
+CREDITS.txt  # not maintained, displayed by About IDLE
+HISTORY.txt  # NEWS up to July 2001
+NEWS.txt     # commits, displayed by About IDLE
+NEWS2.txt    # commits to Python2
+README.txt   # this file, displayed by About IDLE
+TODO.txt     # needs review
+extend.txt   # about writing extensions
+help.html    # copy of idle.html in docs, displayed by IDLE Help
+
+Subdirectories
+--------------
+Icons        # small image files
+idle_test    # files for human test and automated unit tests
+
+
+IDLE MENUS
+==========
+
+Top level items and most submenu items are defined in mainmenu.
+Extensions add submenu items when active.  The names given are
+found, quoted, in one of these modules, paired with a '<<pseudoevent>>'.
+Each pseudoevent is bound to an event handler.  Some event handlers
+call another function that does the actual work.  The annotations below
+are intended to at least give the module where the actual work is done.
+'eEW' = editor.EditorWindow
+
+File
+  New File         # eEW.new_callback
+  Open...          # iomenu.open
+  Open Module      # eEw.open_module
+  Recent Files
+  Class Browser    # eEW.open_class_browser, browser.ClassBrowser
+  Path Browser     # eEW.open_path_browser, pathbrowser
+  ---
+  Save             # iomenu.save
+  Save As...       # iomenu.save_as
+  Save Copy As...  # iomenu.save_a_copy
+  ---
+  Print Window     # iomenu.print_window
+  ---
+  Close            # eEW.close_event
+  Exit             # flist.close_all_callback (bound in eEW)
+
+Edit
+  Undo             # undodelegator
+  Redo             # undodelegator
+  ---              # eEW.right_menu_event
+  Cut              # eEW.cut
+  Copy             # eEW.copy
+  Paste            # eEW.past
+  Select All       # eEW.select_all (+ see eEW.remove_selection)
+  ---              # Next 5 items use searchengine; dialogs use searchbase
+  Find             # eEW.find_event, search.SearchDialog.find
+  Find Again       # eEW.find_again_event, sSD.find_again
+  Find Selection   # eEW.find_selection_event, sSD.find_selection
+  Find in Files... # eEW.find_in_files_event, grep
+  Replace...       # eEW.replace_event, replace.ReplaceDialog.replace
+  Go to Line       # eEW.goto_line_event
+  Show Completions # autocomplete extension and autocompleteWidow (&HP)
+  Expand Word      # autoexpand extension
+  Show call tip    # Calltips extension and CalltipWindow (& Hyperparser)
+  Show surrounding parens  # parenmatch (& Hyperparser)
+
+Format (Editor only) [fFR = format.FormatRegion]
+  Format Paragraph # format.FormatParagraph.format_paragraph_event
+  Indent Region    # fFR.indent_region_event
+  Dedent Region    # fFR.dedent_region_event
+  Comment Out Reg. # fFR.comment_region_event
+  Uncomment Region # fFR.uncomment_region_event
+  Tabify Region    # fFR.tabify_region_event
+  Untabify Region  # fFR.untabify_region_event
+  Toggle Tabs      # format.Indents.toggle_tabs_event
+  New Indent Width # format.Indents.change_indentwidth_event
+  Strip tailing whitespace  # format.rstrip
+  Zin              # zzdummy
+  Zout             # zzdummy
+
+Run (Editor only)
+  Run Module         # runscript.ScriptBinding.run_module_event
+  Run... Customized  # runscript.ScriptBinding.run_custom_event
+  Check Module       # runscript.ScriptBinding.check_module_event
+  Python Shell       # pyshell.Pyshell, pyshell.ModifiedInterpreter
+
+Shell  # pyshell
+  View Last Restart    # pyshell.PyShell.view_restart_mark
+  Restart Shell        # pyshell.PyShell.restart_shell
+  Previous History     # history.History.history_prev
+  Next History         # history.History.history_next
+  Interrupt Execution  # pyshell.PyShell.cancel_callback
+
+Debug (Shell only)
+  Go to File/Line  # outwin.OutputWindow.goto_file_line
+  debugger         # debugger, debugger_r, PyShell.toggle_debugger
+  Stack Viewer     # stackviewer, PyShell.open_stack_viewer
+  Auto-open Stack Viewer  # stackviewer
+
+Options
+  Configure IDLE   # eEW.config_dialog, config, configdialog (cd)
+    (Parts of the dialog)
+    Buttons        # cd.ConfigDialog
+    Font tab       # cd.FontPage, config-main.def
+    Highlight tab  # cd.HighPage, query, config-highlight.def
+    Keys tab       # cd.KeysPage, query, config_key, config_keys.def
+    Windows tab    # cd.WinPage, config_main.def
+    Shell/Ed tab   # cd.ShedPage, config-main.def
+    Extensions tab # config-extensions.def, corresponding .py files
+  ---
+  ... Code Context # codecontext
+  ... Line Numbers # sidebar
+  Zoomheight       # zoomheight
+
+Window
+  <open windows>   # windows
+
+Help
+  About IDLE       # eEW.about_dialog, help_about.AboutDialog
+  ---
+  IDLE Help        # eEW.help_dialog, help.show_idlehelp
+  Python Docs      # eEW.python_docs
+  Turtle Demo      # eEW.open_turtle_demo
+  ---
+  <other help sources>
+
+<Context Menu> (right click)
+  Defined in editor, PyShell.pyshell
+    Cut
+    Copy
+    Paste
+    ---
+    Go to file/line (shell and output only)
+    Set Breakpoint (editor only)
+    Clear Breakpoint (editor only)
+  Defined in debugger
+    Go to source line
+    Show stack frame
+
+<No menu>
+Center Insert      # eEW.center_insert_event
+
+
+OTHER TOPICS
+============
+
+Generally use PEP 8.
+
+import statements
+-----------------
+Put imports at the top, unless there is a good reason otherwise.
+PEP 8 says to group stdlib, 3rd-party dependencies, and package imports.
+For idlelib, the groups are general stdlib, tkinter, and idlelib.
+Sort modules within each group, except that tkinter.ttk follows tkinter.
+Sort 'from idlelib import mod1' and 'from idlelib.mod2 import object'
+together by module, ignoring within module objects.
+Put 'import __main__' after other idlelib imports.
+
+Imports only needed for testing are put not at the top but in an
+htest function def or "if __name__ == '__main__'" clause.
+
+Within module imports like "from idlelib.mod import class" may cause
+circular imports to deadlock.  Even without this, circular imports may
+require at least one of the imports to be delayed until a function call.
+
+What's New entries
+------------------
+
+Repository directory Doc/whatsnew/ has a file 3.n.rst for each 3.n
+Python version.  For the first entry in each file, add subsection
+'IDLE and idlelib', in alphabetical position, to the 'Improved Modules'
+section.  For the rest of cpython, entries to 3.(n+1).rst begin with
+the release of 3.n.0b1.  For IDLE, entries for features backported from
+'main' to '3.n' during its beta period do not got in 3.(n+1).rst.  The
+latter usually gets its first entry during the 3.n.0 candidate period
+or after the 3.n.0 release.
+
+When, as per PEP 434, feature changes are backported, entries are placed
+in the 3.n.rst file *in the main branch* for each Python version n that
+gets the backport. (Note: the format of entries have varied between
+versions.)  Add a line "New in 3.n maintenance releases." before the
+first back-ported feature after 3.n.0 is released. Since each older
+version file gets a different number of backports, it is easiest to
+make a separate PR for each file and label it with the backports
+needed.
+
+Github repository and issues
+----------------------------
+
+The CPython repository is https://github.com/python/cpython.  The
+IDLE Issues listing is https://github.com/orgs/python/projects/31.
+The main classification is by Topic, based on the IDLE menu.  View the
+topics list by clicking the [<]] button in the upper right.

Lib/idlelib/TODO.txt

@@ -0,0 +1,210 @@
+Original IDLE todo, much of it now outdated:
+============================================
+TO DO:
+
+- improve debugger:
+    - manage breakpoints globally, allow bp deletion, tbreak, cbreak etc.
+    - real object browser
+    - help on how to use it (a simple help button will do wonders)
+    - performance?  (updates of large sets of locals are slow)
+    - better integration of "debug module"
+    - debugger should be global resource (attached to flist, not to shell)
+    - fix the stupid bug where you need to step twice
+    - display class name in stack viewer entries for methods
+    - suppress tracing through IDLE internals (e.g. print) DONE
+    - add a button to suppress through a specific module or class or method
+    - more object inspection to stack viewer, e.g. to view all array items
+- insert the initial current directory into sys.path DONE
+- default directory attribute for each window instead of only for windows
+  that have an associated filename
+- command expansion from keywords, module contents, other buffers, etc.
+- "Recent documents" menu item DONE
+- Filter region command
+- Optional horizontal scroll bar
+- more Emacsisms:
+    - ^K should cut to buffer
+    - M-[, M-] to move by paragraphs
+    - incremental search?
+- search should indicate wrap-around in some way
+- restructure state sensitive code to avoid testing flags all the time
+- persistent user state (e.g. window and cursor positions, bindings)
+- make backups when saving
+- check file mtimes at various points
+- Pluggable interface with RCS/CVS/Perforce/Clearcase
+- better help?
+- don't open second class browser on same module (nor second path browser)
+- unify class and path browsers
+- Need to define a standard way whereby one can determine one is running
+  inside IDLE (needed for Tk mainloop, also handy for $PYTHONSTARTUP)
+- Add more utility methods for use by extensions (a la get_selection)
+- Way to run command in totally separate interpreter (fork+os.system?) DONE
+- Way to find definition of fully-qualified name:
+  In other words, select "UserDict.UserDict", hit some magic key and
+  it loads up UserDict.py and finds the first def or class for UserDict.
+- need a way to force colorization on/off
+- need a way to force auto-indent on/off
+
+Details:
+
+- ^O (on Unix -- open-line) should honor autoindent
+- after paste, show end of pasted text
+- on Windows, should turn short filename to long filename (not only in argv!)
+  (shouldn't this be done -- or undone -- by ntpath.normpath?)
+- new autoindent after colon even indents when the colon is in a comment!
+- sometimes forward slashes in pathname remain
+- sometimes star in window name remains in Windows menu
+- With unix bindings, ESC by itself is ignored
+- Sometimes for no apparent reason a selection from the cursor to the
+  end of the command buffer appears, which is hard to get rid of
+  because it stays when you are typing!
+- The Line/Col in the status bar can be wrong initially in PyShell DONE
+
+Structural problems:
+
+- too much knowledge in FileList about EditorWindow (for example)
+- should add some primitives for accessing the selection etc.
+  to repeat cumbersome code over and over
+
+======================================================================
+
+Jeff Bauer suggests:
+
+- Open Module doesn't appear to handle hierarchical packages.
+- Class browser should also allow hierarchical packages.
+- Open and Open Module could benefit from a history, DONE
+  either command line style, or Microsoft recent-file
+  style.
+- Add a Smalltalk-style inspector  (i.e. Tkinspect)
+
+The last suggestion is already a reality, but not yet
+integrated into IDLE.  I use a module called inspector.py,
+that used to be available from python.org(?)  It no longer
+appears to be in the contributed section, and the source
+has no author attribution.
+
+In any case, the code is useful for visually navigating
+an object's attributes, including its container hierarchy.
+
+    >>> from inspector import Tkinspect
+    >>> Tkinspect(None, myObject)
+
+Tkinspect could probably be extended and refined to
+integrate better into IDLE.
+
+======================================================================
+
+Comparison to PTUI
+------------------
+
++ PTUI's help is better (HTML!)
+
++ PTUI can attach a shell to any module
+
++ PTUI has some more I/O commands:
+  open multiple
+  append
+  examine (what's that?)
+
+======================================================================
+
+Notes after trying to run Grail
+-------------------------------
+
+- Grail does stuff to sys.path based on sys.argv[0]; you must set
+sys.argv[0] to something decent first (it is normally set to the path of
+the idle script).
+
+- Grail must be exec'ed in __main__ because that's imported by some
+other parts of Grail.
+
+- Grail uses a module called History and so does idle :-(
+
+======================================================================
+
+Robin Friedrich's items:
+
+Things I'd like to see:
+    - I'd like support for shift-click extending the selection. There's a
+      bug now that it doesn't work the first time you try it.
+    - Printing is needed. How hard can that be on Windows? FIRST CUT DONE
+    - The python-mode trick of autoindenting a line with <tab> is neat and
+      very handy.
+    - (someday) a spellchecker for docstrings and comments.
+    - a pagedown/up command key which moves to next class/def statement (top
+      level)
+    - split window capability
+    - DnD text relocation/copying
+
+Things I don't want to see.
+    - line numbers...  will probably slow things down way too much.
+    - Please use another icon for the tree browser leaf. The small snake
+      isn't cutting it.
+
+----------------------------------------------------------------------
+
+- Customizable views (multi-window or multi-pane).  (Markus Gritsch)
+
+- Being able to double click (maybe double right click) on a callable
+object in the editor which shows the source of the object, if
+possible.  (Gerrit Holl)
+
+- Hooks into the guts, like in Emacs.  (Mike Romberg)
+
+- Sharing the editor with a remote tutor.  (Martijn Faassen)
+
+- Multiple views on the same file.  (Tony J Ibbs)
+
+- Store breakpoints in a global (per-project) database (GvR); Dirk
+Heise adds: save some space-trimmed context and search around when
+reopening a file that might have been edited by someone else.
+
+- Capture menu events in extensions without changing the IDLE source.
+(Matthias Barmeier)
+
+- Use overlapping panels (a "notebook" in MFC terms I think) for info
+that doesn't need to be accessible simultaneously (e.g. HTML source
+and output).  Use multi-pane windows for info that does need to be
+shown together (e.g. class browser and source).  (Albert Brandl)
+
+- A project should invisibly track all symbols, for instant search,
+replace and cross-ref.  Projects should be allowed to span multiple
+directories, hosts, etc.  Project management files are placed in a
+directory you specify.  A global mapping between project names and
+project directories should exist [not so sure --GvR].  (Tim Peters)
+
+- Merge attr-tips and auto-expand.  (Mark Hammond, Tim Peters)
+
+- Python Shell should behave more like a "shell window" as users know
+it -- i.e. you can only edit the current command, and the cursor can't
+escape from the command area.  (Albert Brandl)
+
+- Set X11 class to "idle/Idle", set icon and title to something
+beginning with "idle" -- for window managers.  (Randall Hopper)
+
+- Config files editable through a preferences dialog.  (me) DONE
+
+- Config files still editable outside the preferences dialog.
+(Randall Hopper) DONE
+
+- When you're editing a command in PyShell, and there are only blank
+lines below the cursor, hitting Return should ignore or delete those
+blank lines rather than deciding you're not on the last line.  (me)
+
+- Run command (F5 c.s.) should be more like Pythonwin's Run -- a
+dialog with options to give command line arguments, run the debugger,
+etc.  (me)
+
+- Shouldn't be able to delete part of the prompt (or any text before
+it) in the PyShell.  (Martijn Faassen)   DONE
+
+- Emacs style auto-fill (also smart about comments and strings).
+(Jeremy Hylton)
+
+- Output of Run Script should go to a separate output window, not to
+the shell window.  Output of separate runs should all go to the same
+window but clearly delimited.  (David Scherer) REJECT FIRST, LATTER DONE
+
+- GUI form designer to kick VB's butt.  (Robert Geiger) THAT'S NOT IDLE
+
+- Printing!  Possibly via generation of PDF files which the user must
+then send to the printer separately.  (Dinu Gherman)  FIRST CUT

Lib/idlelib/__init__.py

@@ -0,0 +1,10 @@
+"""The idlelib package implements the Idle application.
+
+Idle includes an interactive shell and editor.
+Starting with Python 3.6, IDLE requires tcl/tk 8.5 or later.
+Use the files named idle.* to start Idle.
+
+The other files are private implementations.  Their details are subject to
+change.  See PEP 434 for more.  Import them at your own risk.
+"""
+testing = False  # Set True by test.test_idle.

Lib/idlelib/__main__.py

@@ -0,0 +1,7 @@
+"""
+IDLE main entry point
+
+Run IDLE as python -m idlelib
+"""
+import idlelib.pyshell
+idlelib.pyshell.main()

Lib/idlelib/autocomplete.py

@@ -0,0 +1,228 @@
+"""Complete either attribute names or file names.
+
+Either on demand or after a user-selected delay after a key character,
+pop up a list of candidates.
+"""
+import __main__
+import keyword
+import os
+import string
+import sys
+
+# Modified keyword list is used in fetch_completions.
+completion_kwds = [s for s in keyword.kwlist
+                     if s not in {'True', 'False', 'None'}]  # In builtins.
+completion_kwds.extend(('match', 'case'))  # Context keywords.
+completion_kwds.sort()
+
+# Two types of completions; defined here for autocomplete_w import below.
+ATTRS, FILES = 0, 1
+from idlelib import autocomplete_w
+from idlelib.config import idleConf
+from idlelib.hyperparser import HyperParser
+
+# Tuples passed to open_completions.
+#       EvalFunc, Complete, WantWin, Mode
+FORCE = True,     False,    True,    None   # Control-Space.
+TAB   = False,    True,     True,    None   # Tab.
+TRY_A = False,    False,    False,   ATTRS  # '.' for attributes.
+TRY_F = False,    False,    False,   FILES  # '/' in quotes for file name.
+
+# This string includes all chars that may be in an identifier.
+# TODO Update this here and elsewhere.
+ID_CHARS = string.ascii_letters + string.digits + "_"
+
+SEPS = f"{os.sep}{os.altsep if os.altsep else ''}"
+TRIGGERS = f".{SEPS}"
+
+class AutoComplete:
+
+    def __init__(self, editwin=None, tags=None):
+        self.editwin = editwin
+        if editwin is not None:   # not in subprocess or no-gui test
+            self.text = editwin.text
+        self.tags = tags
+        self.autocompletewindow = None
+        # id of delayed call, and the index of the text insert when
+        # the delayed call was issued. If _delayed_completion_id is
+        # None, there is no delayed call.
+        self._delayed_completion_id = None
+        self._delayed_completion_index = None
+
+    @classmethod
+    def reload(cls):
+        cls.popupwait = idleConf.GetOption(
+            "extensions", "AutoComplete", "popupwait", type="int", default=0)
+
+    def _make_autocomplete_window(self):  # Makes mocking easier.
+        return autocomplete_w.AutoCompleteWindow(self.text, tags=self.tags)
+
+    def _remove_autocomplete_window(self, event=None):
+        if self.autocompletewindow:
+            self.autocompletewindow.hide_window()
+            self.autocompletewindow = None
+
+    def force_open_completions_event(self, event):
+        "(^space) Open completion list, even if a function call is needed."
+        self.open_completions(FORCE)
+        return "break"
+
+    def autocomplete_event(self, event):
+        "(tab) Complete word or open list if multiple options."
+        if hasattr(event, "mc_state") and event.mc_state or\
+                not self.text.get("insert linestart", "insert").strip():
+            # A modifier was pressed along with the tab or
+            # there is only previous whitespace on this line, so tab.
+            return None
+        if self.autocompletewindow and self.autocompletewindow.is_active():
+            self.autocompletewindow.complete()
+            return "break"
+        else:
+            opened = self.open_completions(TAB)
+            return "break" if opened else None
+
+    def try_open_completions_event(self, event=None):
+        "(./) Open completion list after pause with no movement."
+        lastchar = self.text.get("insert-1c")
+        if lastchar in TRIGGERS:
+            args = TRY_A if lastchar == "." else TRY_F
+            self._delayed_completion_index = self.text.index("insert")
+            if self._delayed_completion_id is not None:
+                self.text.after_cancel(self._delayed_completion_id)
+            self._delayed_completion_id = self.text.after(
+                self.popupwait, self._delayed_open_completions, args)
+
+    def _delayed_open_completions(self, args):
+        "Call open_completions if index unchanged."
+        self._delayed_completion_id = None
+        if self.text.index("insert") == self._delayed_completion_index:
+            self.open_completions(args)
+
+    def open_completions(self, args):
+        """Find the completions and create the AutoCompleteWindow.
+        Return True if successful (no syntax error or so found).
+        If complete is True, then if there's nothing to complete and no
+        start of completion, won't open completions and return False.
+        If mode is given, will open a completion list only in this mode.
+        """
+        evalfuncs, complete, wantwin, mode = args
+        # Cancel another delayed call, if it exists.
+        if self._delayed_completion_id is not None:
+            self.text.after_cancel(self._delayed_completion_id)
+            self._delayed_completion_id = None
+
+        hp = HyperParser(self.editwin, "insert")
+        curline = self.text.get("insert linestart", "insert")
+        i = j = len(curline)
+        if hp.is_in_string() and (not mode or mode==FILES):
+            # Find the beginning of the string.
+            # fetch_completions will look at the file system to determine
+            # whether the string value constitutes an actual file name
+            # XXX could consider raw strings here and unescape the string
+            # value if it's not raw.
+            self._remove_autocomplete_window()
+            mode = FILES
+            # Find last separator or string start
+            while i and curline[i-1] not in "'\"" + SEPS:
+                i -= 1
+            comp_start = curline[i:j]
+            j = i
+            # Find string start
+            while i and curline[i-1] not in "'\"":
+                i -= 1
+            comp_what = curline[i:j]
+        elif hp.is_in_code() and (not mode or mode==ATTRS):
+            self._remove_autocomplete_window()
+            mode = ATTRS
+            while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127):
+                i -= 1
+            comp_start = curline[i:j]
+            if i and curline[i-1] == '.':  # Need object with attributes.
+                hp.set_index("insert-%dc" % (len(curline)-(i-1)))
+                comp_what = hp.get_expression()
+                if (not comp_what or
+                   (not evalfuncs and comp_what.find('(') != -1)):
+                    return None
+            else:
+                comp_what = ""
+        else:
+            return None
+
+        if complete and not comp_what and not comp_start:
+            return None
+        comp_lists = self.fetch_completions(comp_what, mode)
+        if not comp_lists[0]:
+            return None
+        self.autocompletewindow = self._make_autocomplete_window()
+        return not self.autocompletewindow.show_window(
+                comp_lists, "insert-%dc" % len(comp_start),
+                complete, mode, wantwin)
+
+    def fetch_completions(self, what, mode):
+        """Return a pair of lists of completions for something. The first list
+        is a sublist of the second. Both are sorted.
+
+        If there is a Python subprocess, get the comp. list there.  Otherwise,
+        either fetch_completions() is running in the subprocess itself or it
+        was called in an IDLE EditorWindow before any script had been run.
+
+        The subprocess environment is that of the most recently run script.  If
+        two unrelated modules are being edited some calltips in the current
+        module may be inoperative if the module was not the last to run.
+        """
+        try:
+            rpcclt = self.editwin.flist.pyshell.interp.rpcclt
+        except:
+            rpcclt = None
+        if rpcclt:
+            return rpcclt.remotecall("exec", "get_the_completion_list",
+                                     (what, mode), {})
+        else:
+            if mode == ATTRS:
+                if what == "":  # Main module names.
+                    namespace = {**__main__.__builtins__.__dict__,
+                                 **__main__.__dict__}
+                    bigl = eval("dir()", namespace)
+                    bigl.extend(completion_kwds)
+                    bigl.sort()
+                    if "__all__" in bigl:
+                        smalll = sorted(eval("__all__", namespace))
+                    else:
+                        smalll = [s for s in bigl if s[:1] != '_']
+                else:
+                    try:
+                        entity = self.get_entity(what)
+                        bigl = dir(entity)
+                        bigl.sort()
+                        if "__all__" in bigl:
+                            smalll = sorted(entity.__all__)
+                        else:
+                            smalll = [s for s in bigl if s[:1] != '_']
+                    except:
+                        return [], []
+
+            elif mode == FILES:
+                if what == "":
+                    what = "."
+                try:
+                    expandedpath = os.path.expanduser(what)
+                    bigl = os.listdir(expandedpath)
+                    bigl.sort()
+                    smalll = [s for s in bigl if s[:1] != '.']
+                except OSError:
+                    return [], []
+
+            if not smalll:
+                smalll = bigl
+            return smalll, bigl
+
+    def get_entity(self, name):
+        "Lookup name in a namespace spanning sys.modules and __main.dict__."
+        return eval(name, {**sys.modules, **__main__.__dict__})
+
+
+AutoComplete.reload()
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_autocomplete', verbosity=2)

Lib/idlelib/autocomplete_w.py

@@ -0,0 +1,498 @@
+"""
+An auto-completion window for IDLE, used by the autocomplete extension
+"""
+import platform
+
+from tkinter import *
+from tkinter.ttk import Scrollbar
+
+from idlelib.autocomplete import FILES, ATTRS
+from idlelib.multicall import MC_SHIFT
+
+HIDE_VIRTUAL_EVENT_NAME = "<<autocompletewindow-hide>>"
+HIDE_FOCUS_OUT_SEQUENCE = "<FocusOut>"
+HIDE_SEQUENCES = (HIDE_FOCUS_OUT_SEQUENCE, "<ButtonPress>")
+KEYPRESS_VIRTUAL_EVENT_NAME = "<<autocompletewindow-keypress>>"
+# We need to bind event beyond <Key> so that the function will be called
+# before the default specific IDLE function
+KEYPRESS_SEQUENCES = ("<Key>", "<Key-BackSpace>", "<Key-Return>", "<Key-Tab>",
+                      "<Key-Up>", "<Key-Down>", "<Key-Home>", "<Key-End>",
+                      "<Key-Prior>", "<Key-Next>", "<Key-Escape>")
+KEYRELEASE_VIRTUAL_EVENT_NAME = "<<autocompletewindow-keyrelease>>"
+KEYRELEASE_SEQUENCE = "<KeyRelease>"
+LISTUPDATE_SEQUENCE = "<B1-ButtonRelease>"
+WINCONFIG_SEQUENCE = "<Configure>"
+DOUBLECLICK_SEQUENCE = "<B1-Double-ButtonRelease>"
+
+class AutoCompleteWindow:
+
+    def __init__(self, widget, tags):
+        # The widget (Text) on which we place the AutoCompleteWindow
+        self.widget = widget
+        # Tags to mark inserted text with
+        self.tags = tags
+        # The widgets we create
+        self.autocompletewindow = self.listbox = self.scrollbar = None
+        # The default foreground and background of a selection. Saved because
+        # they are changed to the regular colors of list items when the
+        # completion start is not a prefix of the selected completion
+        self.origselforeground = self.origselbackground = None
+        # The list of completions
+        self.completions = None
+        # A list with more completions, or None
+        self.morecompletions = None
+        # The completion mode, either autocomplete.ATTRS or .FILES.
+        self.mode = None
+        # The current completion start, on the text box (a string)
+        self.start = None
+        # The index of the start of the completion
+        self.startindex = None
+        # The last typed start, used so that when the selection changes,
+        # the new start will be as close as possible to the last typed one.
+        self.lasttypedstart = None
+        # Do we have an indication that the user wants the completion window
+        # (for example, he clicked the list)
+        self.userwantswindow = None
+        # event ids
+        self.hideid = self.keypressid = self.listupdateid = \
+            self.winconfigid = self.keyreleaseid = self.doubleclickid = None
+        # Flag set if last keypress was a tab
+        self.lastkey_was_tab = False
+        # Flag set to avoid recursive <Configure> callback invocations.
+        self.is_configuring = False
+
+    def _change_start(self, newstart):
+        min_len = min(len(self.start), len(newstart))
+        i = 0
+        while i < min_len and self.start[i] == newstart[i]:
+            i += 1
+        if i < len(self.start):
+            self.widget.delete("%s+%dc" % (self.startindex, i),
+                               "%s+%dc" % (self.startindex, len(self.start)))
+        if i < len(newstart):
+            self.widget.insert("%s+%dc" % (self.startindex, i),
+                               newstart[i:],
+                               self.tags)
+        self.start = newstart
+
+    def _binary_search(self, s):
+        """Find the first index in self.completions where completions[i] is
+        greater or equal to s, or the last index if there is no such.
+        """
+        i = 0; j = len(self.completions)
+        while j > i:
+            m = (i + j) // 2
+            if self.completions[m] >= s:
+                j = m
+            else:
+                i = m + 1
+        return min(i, len(self.completions)-1)
+
+    def _complete_string(self, s):
+        """Assuming that s is the prefix of a string in self.completions,
+        return the longest string which is a prefix of all the strings which
+        s is a prefix of them. If s is not a prefix of a string, return s.
+        """
+        first = self._binary_search(s)
+        if self.completions[first][:len(s)] != s:
+            # There is not even one completion which s is a prefix of.
+            return s
+        # Find the end of the range of completions where s is a prefix of.
+        i = first + 1
+        j = len(self.completions)
+        while j > i:
+            m = (i + j) // 2
+            if self.completions[m][:len(s)] != s:
+                j = m
+            else:
+                i = m + 1
+        last = i-1
+
+        if first == last: # only one possible completion
+            return self.completions[first]
+
+        # We should return the maximum prefix of first and last
+        first_comp = self.completions[first]
+        last_comp = self.completions[last]
+        min_len = min(len(first_comp), len(last_comp))
+        i = len(s)
+        while i < min_len and first_comp[i] == last_comp[i]:
+            i += 1
+        return first_comp[:i]
+
+    def _selection_changed(self):
+        """Call when the selection of the Listbox has changed.
+
+        Updates the Listbox display and calls _change_start.
+        """
+        cursel = int(self.listbox.curselection()[0])
+
+        self.listbox.see(cursel)
+
+        lts = self.lasttypedstart
+        selstart = self.completions[cursel]
+        if self._binary_search(lts) == cursel:
+            newstart = lts
+        else:
+            min_len = min(len(lts), len(selstart))
+            i = 0
+            while i < min_len and lts[i] == selstart[i]:
+                i += 1
+            newstart = selstart[:i]
+        self._change_start(newstart)
+
+        if self.completions[cursel][:len(self.start)] == self.start:
+            # start is a prefix of the selected completion
+            self.listbox.configure(selectbackground=self.origselbackground,
+                                   selectforeground=self.origselforeground)
+        else:
+            self.listbox.configure(selectbackground=self.listbox.cget("bg"),
+                                   selectforeground=self.listbox.cget("fg"))
+            # If there are more completions, show them, and call me again.
+            if self.morecompletions:
+                self.completions = self.morecompletions
+                self.morecompletions = None
+                self.listbox.delete(0, END)
+                for item in self.completions:
+                    self.listbox.insert(END, item)
+                self.listbox.select_set(self._binary_search(self.start))
+                self._selection_changed()
+
+    def show_window(self, comp_lists, index, complete, mode, userWantsWin):
+        """Show the autocomplete list, bind events.
+
+        If complete is True, complete the text, and if there is exactly
+        one matching completion, don't open a list.
+        """
+        # Handle the start we already have
+        self.completions, self.morecompletions = comp_lists
+        self.mode = mode
+        self.startindex = self.widget.index(index)
+        self.start = self.widget.get(self.startindex, "insert")
+        if complete:
+            completed = self._complete_string(self.start)
+            start = self.start
+            self._change_start(completed)
+            i = self._binary_search(completed)
+            if self.completions[i] == completed and \
+               (i == len(self.completions)-1 or
+                self.completions[i+1][:len(completed)] != completed):
+                # There is exactly one matching completion
+                return completed == start
+        self.userwantswindow = userWantsWin
+        self.lasttypedstart = self.start
+
+        self.autocompletewindow = acw = Toplevel(self.widget)
+        acw.withdraw()
+        acw.wm_overrideredirect(1)
+        try:
+            # Prevent grabbing focus on macOS.
+            acw.tk.call("::tk::unsupported::MacWindowStyle", "style", acw._w,
+                        "help", "noActivates")
+        except TclError:
+            pass
+        self.scrollbar = scrollbar = Scrollbar(acw, orient=VERTICAL)
+        self.listbox = listbox = Listbox(acw, yscrollcommand=scrollbar.set,
+                                         exportselection=False)
+        for item in self.completions:
+            listbox.insert(END, item)
+        self.origselforeground = listbox.cget("selectforeground")
+        self.origselbackground = listbox.cget("selectbackground")
+        scrollbar.config(command=listbox.yview)
+        scrollbar.pack(side=RIGHT, fill=Y)
+        listbox.pack(side=LEFT, fill=BOTH, expand=True)
+        #acw.update_idletasks() # Need for tk8.6.8 on macOS: #40128.
+        acw.lift()  # work around bug in Tk 8.5.18+ (issue #24570)
+
+        # Initialize the listbox selection
+        self.listbox.select_set(self._binary_search(self.start))
+        self._selection_changed()
+
+        # bind events
+        self.hideaid = acw.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event)
+        self.hidewid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event)
+        acw.event_add(HIDE_VIRTUAL_EVENT_NAME, HIDE_FOCUS_OUT_SEQUENCE)
+        for seq in HIDE_SEQUENCES:
+            self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq)
+
+        self.keypressid = self.widget.bind(KEYPRESS_VIRTUAL_EVENT_NAME,
+                                           self.keypress_event)
+        for seq in KEYPRESS_SEQUENCES:
+            self.widget.event_add(KEYPRESS_VIRTUAL_EVENT_NAME, seq)
+        self.keyreleaseid = self.widget.bind(KEYRELEASE_VIRTUAL_EVENT_NAME,
+                                             self.keyrelease_event)
+        self.widget.event_add(KEYRELEASE_VIRTUAL_EVENT_NAME,KEYRELEASE_SEQUENCE)
+        self.listupdateid = listbox.bind(LISTUPDATE_SEQUENCE,
+                                         self.listselect_event)
+        self.is_configuring = False
+        self.winconfigid = acw.bind(WINCONFIG_SEQUENCE, self.winconfig_event)
+        self.doubleclickid = listbox.bind(DOUBLECLICK_SEQUENCE,
+                                          self.doubleclick_event)
+        return None
+
+    def winconfig_event(self, event):
+        if self.is_configuring:
+            # Avoid running on recursive <Configure> callback invocations.
+            return
+
+        self.is_configuring = True
+        if not self.is_active():
+            return
+
+        # Since the <Configure> event may occur after the completion window is gone,
+        # catch potential TclError exceptions when accessing acw.  See: bpo-41611.
+        try:
+            # Position the completion list window
+            text = self.widget
+            text.see(self.startindex)
+            x, y, cx, cy = text.bbox(self.startindex)
+            acw = self.autocompletewindow
+            if platform.system().startswith('Windows'):
+                # On Windows an update() call is needed for the completion
+                # list window to be created, so that we can fetch its width
+                # and height.  However, this is not needed on other platforms
+                # (tested on Ubuntu and macOS) but at one point began
+                # causing freezes on macOS.  See issues 37849 and 41611.
+                acw.update()
+            acw_width, acw_height = acw.winfo_width(), acw.winfo_height()
+            text_width, text_height = text.winfo_width(), text.winfo_height()
+            new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width))
+            new_y = text.winfo_rooty() + y
+            if (text_height - (y + cy) >= acw_height # enough height below
+                or y < acw_height): # not enough height above
+                # place acw below current line
+                new_y += cy
+            else:
+                # place acw above current line
+                new_y -= acw_height
+            acw.wm_geometry("+%d+%d" % (new_x, new_y))
+            acw.deiconify()
+            acw.update_idletasks()
+        except TclError:
+            pass
+
+        if platform.system().startswith('Windows'):
+            # See issue 15786.  When on Windows platform, Tk will misbehave
+            # to call winconfig_event multiple times, we need to prevent this,
+            # otherwise mouse button double click will not be able to used.
+            try:
+                acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid)
+            except TclError:
+                pass
+            self.winconfigid = None
+
+        self.is_configuring = False
+
+    def _hide_event_check(self):
+        if not self.autocompletewindow:
+            return
+
+        try:
+            if not self.autocompletewindow.focus_get():
+                self.hide_window()
+        except KeyError:
+            # See issue 734176, when user click on menu, acw.focus_get()
+            # will get KeyError.
+            self.hide_window()
+
+    def hide_event(self, event):
+        # Hide autocomplete list if it exists and does not have focus or
+        # mouse click on widget / text area.
+        if self.is_active():
+            if event.type == EventType.FocusOut:
+                # On Windows platform, it will need to delay the check for
+                # acw.focus_get() when click on acw, otherwise it will return
+                # None and close the window
+                self.widget.after(1, self._hide_event_check)
+            elif event.type == EventType.ButtonPress:
+                # ButtonPress event only bind to self.widget
+                self.hide_window()
+
+    def listselect_event(self, event):
+        if self.is_active():
+            self.userwantswindow = True
+            cursel = int(self.listbox.curselection()[0])
+            self._change_start(self.completions[cursel])
+
+    def doubleclick_event(self, event):
+        # Put the selected completion in the text, and close the list
+        cursel = int(self.listbox.curselection()[0])
+        self._change_start(self.completions[cursel])
+        self.hide_window()
+
+    def keypress_event(self, event):
+        if not self.is_active():
+            return None
+        keysym = event.keysym
+        if hasattr(event, "mc_state"):
+            state = event.mc_state
+        else:
+            state = 0
+        if keysym != "Tab":
+            self.lastkey_was_tab = False
+        if (len(keysym) == 1 or keysym in ("underscore", "BackSpace")
+            or (self.mode == FILES and keysym in
+                ("period", "minus"))) \
+           and not (state & ~MC_SHIFT):
+            # Normal editing of text
+            if len(keysym) == 1:
+                self._change_start(self.start + keysym)
+            elif keysym == "underscore":
+                self._change_start(self.start + '_')
+            elif keysym == "period":
+                self._change_start(self.start + '.')
+            elif keysym == "minus":
+                self._change_start(self.start + '-')
+            else:
+                # keysym == "BackSpace"
+                if len(self.start) == 0:
+                    self.hide_window()
+                    return None
+                self._change_start(self.start[:-1])
+            self.lasttypedstart = self.start
+            self.listbox.select_clear(0, int(self.listbox.curselection()[0]))
+            self.listbox.select_set(self._binary_search(self.start))
+            self._selection_changed()
+            return "break"
+
+        elif keysym == "Return":
+            self.complete()
+            self.hide_window()
+            return 'break'
+
+        elif (self.mode == ATTRS and keysym in
+              ("period", "space", "parenleft", "parenright", "bracketleft",
+               "bracketright")) or \
+             (self.mode == FILES and keysym in
+              ("slash", "backslash", "quotedbl", "apostrophe")) \
+             and not (state & ~MC_SHIFT):
+            # If start is a prefix of the selection, but is not '' when
+            # completing file names, put the whole
+            # selected completion. Anyway, close the list.
+            cursel = int(self.listbox.curselection()[0])
+            if self.completions[cursel][:len(self.start)] == self.start \
+               and (self.mode == ATTRS or self.start):
+                self._change_start(self.completions[cursel])
+            self.hide_window()
+            return None
+
+        elif keysym in ("Home", "End", "Prior", "Next", "Up", "Down") and \
+             not state:
+            # Move the selection in the listbox
+            self.userwantswindow = True
+            cursel = int(self.listbox.curselection()[0])
+            if keysym == "Home":
+                newsel = 0
+            elif keysym == "End":
+                newsel = len(self.completions)-1
+            elif keysym in ("Prior", "Next"):
+                jump = self.listbox.nearest(self.listbox.winfo_height()) - \
+                       self.listbox.nearest(0)
+                if keysym == "Prior":
+                    newsel = max(0, cursel-jump)
+                else:
+                    assert keysym == "Next"
+                    newsel = min(len(self.completions)-1, cursel+jump)
+            elif keysym == "Up":
+                newsel = max(0, cursel-1)
+            else:
+                assert keysym == "Down"
+                newsel = min(len(self.completions)-1, cursel+1)
+            self.listbox.select_clear(cursel)
+            self.listbox.select_set(newsel)
+            self._selection_changed()
+            self._change_start(self.completions[newsel])
+            return "break"
+
+        elif (keysym == "Tab" and not state):
+            if self.lastkey_was_tab:
+                # two tabs in a row; insert current selection and close acw
+                cursel = int(self.listbox.curselection()[0])
+                self._change_start(self.completions[cursel])
+                self.hide_window()
+                return "break"
+            else:
+                # first tab; let AutoComplete handle the completion
+                self.userwantswindow = True
+                self.lastkey_was_tab = True
+                return None
+
+        elif any(s in keysym for s in ("Shift", "Control", "Alt",
+                                       "Meta", "Command", "Option")):
+            # A modifier key, so ignore
+            return None
+
+        elif event.char and event.char >= ' ':
+            # Regular character with a non-length-1 keycode
+            self._change_start(self.start + event.char)
+            self.lasttypedstart = self.start
+            self.listbox.select_clear(0, int(self.listbox.curselection()[0]))
+            self.listbox.select_set(self._binary_search(self.start))
+            self._selection_changed()
+            return "break"
+
+        else:
+            # Unknown event, close the window and let it through.
+            self.hide_window()
+            return None
+
+    def keyrelease_event(self, event):
+        if not self.is_active():
+            return
+        if self.widget.index("insert") != \
+           self.widget.index("%s+%dc" % (self.startindex, len(self.start))):
+            # If we didn't catch an event which moved the insert, close window
+            self.hide_window()
+
+    def is_active(self):
+        return self.autocompletewindow is not None
+
+    def complete(self):
+        self._change_start(self._complete_string(self.start))
+        # The selection doesn't change.
+
+    def hide_window(self):
+        if not self.is_active():
+            return
+
+        # unbind events
+        self.autocompletewindow.event_delete(HIDE_VIRTUAL_EVENT_NAME,
+                                             HIDE_FOCUS_OUT_SEQUENCE)
+        for seq in HIDE_SEQUENCES:
+            self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq)
+
+        self.autocompletewindow.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideaid)
+        self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hidewid)
+        self.hideaid = None
+        self.hidewid = None
+        for seq in KEYPRESS_SEQUENCES:
+            self.widget.event_delete(KEYPRESS_VIRTUAL_EVENT_NAME, seq)
+        self.widget.unbind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypressid)
+        self.keypressid = None
+        self.widget.event_delete(KEYRELEASE_VIRTUAL_EVENT_NAME,
+                                 KEYRELEASE_SEQUENCE)
+        self.widget.unbind(KEYRELEASE_VIRTUAL_EVENT_NAME, self.keyreleaseid)
+        self.keyreleaseid = None
+        self.listbox.unbind(LISTUPDATE_SEQUENCE, self.listupdateid)
+        self.listupdateid = None
+        if self.winconfigid:
+            self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid)
+            self.winconfigid = None
+
+        # Re-focusOn frame.text (See issue #15786)
+        self.widget.focus_set()
+
+        # destroy widgets
+        self.scrollbar.destroy()
+        self.scrollbar = None
+        self.listbox.destroy()
+        self.listbox = None
+        self.autocompletewindow.destroy()
+        self.autocompletewindow = None
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_autocomplete_w', verbosity=2, exit=False)
+
+# TODO: autocomplete/w htest here

Lib/idlelib/autoexpand.py

@@ -0,0 +1,96 @@
+'''Complete the current word before the cursor with words in the editor.
+
+Each menu selection or shortcut key selection replaces the word with a
+different word with the same prefix. The search for matches begins
+before the target and moves toward the top of the editor. It then starts
+after the cursor and moves down. It then returns to the original word and
+the cycle starts again.
+
+Changing the current text line or leaving the cursor in a different
+place before requesting the next selection causes AutoExpand to reset
+its state.
+
+There is only one instance of Autoexpand.
+'''
+import re
+import string
+
+
+class AutoExpand:
+    wordchars = string.ascii_letters + string.digits + "_"
+
+    def __init__(self, editwin):
+        self.text = editwin.text
+        self.bell = self.text.bell
+        self.state = None
+
+    def expand_word_event(self, event):
+        "Replace the current word with the next expansion."
+        curinsert = self.text.index("insert")
+        curline = self.text.get("insert linestart", "insert lineend")
+        if not self.state:
+            words = self.getwords()
+            index = 0
+        else:
+            words, index, insert, line = self.state
+            if insert != curinsert or line != curline:
+                words = self.getwords()
+                index = 0
+        if not words:
+            self.bell()
+            return "break"
+        word = self.getprevword()
+        self.text.delete("insert - %d chars" % len(word), "insert")
+        newword = words[index]
+        index = (index + 1) % len(words)
+        if index == 0:
+            self.bell()            # Warn we cycled around
+        self.text.insert("insert", newword)
+        curinsert = self.text.index("insert")
+        curline = self.text.get("insert linestart", "insert lineend")
+        self.state = words, index, curinsert, curline
+        return "break"
+
+    def getwords(self):
+        "Return a list of words that match the prefix before the cursor."
+        word = self.getprevword()
+        if not word:
+            return []
+        before = self.text.get("1.0", "insert wordstart")
+        wbefore = re.findall(r"\b" + word + r"\w+\b", before)
+        del before
+        after = self.text.get("insert wordend", "end")
+        wafter = re.findall(r"\b" + word + r"\w+\b", after)
+        del after
+        if not wbefore and not wafter:
+            return []
+        words = []
+        dict = {}
+        # search backwards through words before
+        wbefore.reverse()
+        for w in wbefore:
+            if dict.get(w):
+                continue
+            words.append(w)
+            dict[w] = w
+        # search onwards through words after
+        for w in wafter:
+            if dict.get(w):
+                continue
+            words.append(w)
+            dict[w] = w
+        words.append(word)
+        return words
+
+    def getprevword(self):
+        "Return the word prefix before the cursor."
+        line = self.text.get("insert linestart", "insert")
+        i = len(line)
+        while i > 0 and line[i-1] in self.wordchars:
+            i = i-1
+        return line[i:]
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_autoexpand', verbosity=2)

Lib/idlelib/browser.py

@@ -0,0 +1,260 @@
+"""Module browser.
+
+XXX TO DO:
+
+- reparse when source changed (maybe just a button would be OK?)
+    (or recheck on window popup)
+- add popup menu with more options (e.g. doc strings, base classes, imports)
+- add base classes to class browser tree
+"""
+
+import os
+import pyclbr
+import sys
+
+from idlelib.config import idleConf
+from idlelib import pyshell
+from idlelib.tree import TreeNode, TreeItem, ScrolledCanvas
+from idlelib.util import py_extensions
+from idlelib.window import ListedToplevel
+
+
+file_open = None  # Method...Item and Class...Item use this.
+# Normally pyshell.flist.open, but there is no pyshell.flist for htest.
+
+# The browser depends on pyclbr and importlib which do not support .pyi files.
+browseable_extension_blocklist = ('.pyi',)
+
+
+def is_browseable_extension(path):
+    _, ext = os.path.splitext(path)
+    ext = os.path.normcase(ext)
+    return ext in py_extensions and ext not in browseable_extension_blocklist
+
+
+def transform_children(child_dict, modname=None):
+    """Transform a child dictionary to an ordered sequence of objects.
+
+    The dictionary maps names to pyclbr information objects.
+    Filter out imported objects.
+    Augment class names with bases.
+    The insertion order of the dictionary is assumed to have been in line
+    number order, so sorting is not necessary.
+
+    The current tree only calls this once per child_dict as it saves
+    TreeItems once created.  A future tree and tests might violate this,
+    so a check prevents multiple in-place augmentations.
+    """
+    obs = []  # Use list since values should already be sorted.
+    for key, obj in child_dict.items():
+        if modname is None or obj.module == modname:
+            if hasattr(obj, 'super') and obj.super and obj.name == key:
+                # If obj.name != key, it has already been suffixed.
+                supers = []
+                for sup in obj.super:
+                    if isinstance(sup, str):
+                        sname = sup
+                    else:
+                        sname = sup.name
+                        if sup.module != obj.module:
+                            sname = f'{sup.module}.{sname}'
+                    supers.append(sname)
+                obj.name += '({})'.format(', '.join(supers))
+            obs.append(obj)
+    return obs
+
+
+class ModuleBrowser:
+    """Browse module classes and functions in IDLE.
+    """
+    # This class is also the base class for pathbrowser.PathBrowser.
+    # Init and close are inherited, other methods are overridden.
+    # PathBrowser.__init__ does not call __init__ below.
+
+    def __init__(self, master, path, *, _htest=False, _utest=False):
+        """Create a window for browsing a module's structure.
+
+        Args:
+            master: parent for widgets.
+            path: full path of file to browse.
+            _htest - bool; change box location when running htest.
+            -utest - bool; suppress contents when running unittest.
+
+        Global variables:
+            file_open: Function used for opening a file.
+
+        Instance variables:
+            name: Module name.
+            file: Full path and module with supported extension.
+                Used in creating ModuleBrowserTreeItem as the rootnode for
+                the tree and subsequently in the children.
+        """
+        self.master = master
+        self.path = path
+        self._htest = _htest
+        self._utest = _utest
+        self.init()
+
+    def close(self, event=None):
+        "Dismiss the window and the tree nodes."
+        self.top.destroy()
+        self.node.destroy()
+
+    def init(self):
+        "Create browser tkinter widgets, including the tree."
+        global file_open
+        root = self.master
+        flist = (pyshell.flist if not (self._htest or self._utest)
+                 else pyshell.PyShellFileList(root))
+        file_open = flist.open
+        pyclbr._modules.clear()
+
+        # create top
+        self.top = top = ListedToplevel(root)
+        top.protocol("WM_DELETE_WINDOW", self.close)
+        top.bind("<Escape>", self.close)
+        if self._htest: # place dialog below parent if running htest
+            top.geometry("+%d+%d" %
+                (root.winfo_rootx(), root.winfo_rooty() + 200))
+        self.settitle()
+        top.focus_set()
+
+        # create scrolled canvas
+        theme = idleConf.CurrentTheme()
+        background = idleConf.GetHighlight(theme, 'normal')['background']
+        sc = ScrolledCanvas(top, bg=background, highlightthickness=0,
+                            takefocus=1)
+        sc.frame.pack(expand=1, fill="both")
+        item = self.rootnode()
+        self.node = node = TreeNode(sc.canvas, None, item)
+        if not self._utest:
+            node.update()
+            node.expand()
+
+    def settitle(self):
+        "Set the window title."
+        self.top.wm_title("Module Browser - " + os.path.basename(self.path))
+        self.top.wm_iconname("Module Browser")
+
+    def rootnode(self):
+        "Return a ModuleBrowserTreeItem as the root of the tree."
+        return ModuleBrowserTreeItem(self.path)
+
+
+class ModuleBrowserTreeItem(TreeItem):
+    """Browser tree for Python module.
+
+    Uses TreeItem as the basis for the structure of the tree.
+    Used by both browsers.
+    """
+
+    def __init__(self, file):
+        """Create a TreeItem for the file.
+
+        Args:
+            file: Full path and module name.
+        """
+        self.file = file
+
+    def GetText(self):
+        "Return the module name as the text string to display."
+        return os.path.basename(self.file)
+
+    def GetIconName(self):
+        "Return the name of the icon to display."
+        return "python"
+
+    def GetSubList(self):
+        "Return ChildBrowserTreeItems for children."
+        return [ChildBrowserTreeItem(obj) for obj in self.listchildren()]
+
+    def OnDoubleClick(self):
+        "Open a module in an editor window when double clicked."
+        if not is_browseable_extension(self.file):
+            return
+        if not os.path.exists(self.file):
+            return
+        file_open(self.file)
+
+    def IsExpandable(self):
+        "Return True if Python file."
+        return is_browseable_extension(self.file)
+
+    def listchildren(self):
+        "Return sequenced classes and functions in the module."
+        if not is_browseable_extension(self.file):
+            return []
+        dir, base = os.path.split(self.file)
+        name, _ = os.path.splitext(base)
+        try:
+            tree = pyclbr.readmodule_ex(name, [dir] + sys.path)
+        except ImportError:
+            return []
+        return transform_children(tree, name)
+
+
+class ChildBrowserTreeItem(TreeItem):
+    """Browser tree for child nodes within the module.
+
+    Uses TreeItem as the basis for the structure of the tree.
+    """
+
+    def __init__(self, obj):
+        "Create a TreeItem for a pyclbr class/function object."
+        self.obj = obj
+        self.name = obj.name
+        self.isfunction = isinstance(obj, pyclbr.Function)
+
+    def GetText(self):
+        "Return the name of the function/class to display."
+        name = self.name
+        if self.isfunction:
+            return "def " + name + "(...)"
+        else:
+            return "class " + name
+
+    def GetIconName(self):
+        "Return the name of the icon to display."
+        if self.isfunction:
+            return "python"
+        else:
+            return "folder"
+
+    def IsExpandable(self):
+        "Return True if self.obj has nested objects."
+        return self.obj.children != {}
+
+    def GetSubList(self):
+        "Return ChildBrowserTreeItems for children."
+        return [ChildBrowserTreeItem(obj)
+                for obj in transform_children(self.obj.children)]
+
+    def OnDoubleClick(self):
+        "Open module with file_open and position to lineno."
+        try:
+            edit = file_open(self.obj.file)
+            edit.gotoline(self.obj.lineno)
+        except (OSError, AttributeError):
+            pass
+
+
+def _module_browser(parent): # htest #
+    if len(sys.argv) > 1:  # If pass file on command line.
+        file = sys.argv[1]
+    else:
+        file = __file__
+        # Add nested objects for htest.
+        class Nested_in_func(TreeNode):
+            def nested_in_class(): pass
+        def closure():
+            class Nested_in_closure: pass
+    ModuleBrowser(parent, file, _htest=True)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:  # If pass file on command line, unittest fails.
+        from unittest import main
+        main('idlelib.idle_test.test_browser', verbosity=2, exit=False)
+
+    from idlelib.idle_test.htest import run
+    run(_module_browser)

Lib/idlelib/calltip.py

@@ -0,0 +1,205 @@
+"""Pop up a reminder of how to call a function.
+
+Call Tips are floating windows which display function, class, and method
+parameter and docstring information when you type an opening parenthesis, and
+which disappear when you type a closing parenthesis.
+"""
+import __main__
+import inspect
+import re
+import sys
+import textwrap
+import types
+
+from idlelib import calltip_w
+from idlelib.hyperparser import HyperParser
+
+
+class Calltip:
+
+    def __init__(self, editwin=None):
+        if editwin is None:  # subprocess and test
+            self.editwin = None
+        else:
+            self.editwin = editwin
+            self.text = editwin.text
+            self.active_calltip = None
+            self._calltip_window = self._make_tk_calltip_window
+
+    def close(self):
+        self._calltip_window = None
+
+    def _make_tk_calltip_window(self):
+        # See __init__ for usage
+        return calltip_w.CalltipWindow(self.text)
+
+    def remove_calltip_window(self, event=None):
+        if self.active_calltip:
+            self.active_calltip.hidetip()
+            self.active_calltip = None
+
+    def force_open_calltip_event(self, event):
+        "The user selected the menu entry or hotkey, open the tip."
+        self.open_calltip(True)
+        return "break"
+
+    def try_open_calltip_event(self, event):
+        """Happens when it would be nice to open a calltip, but not really
+        necessary, for example after an opening bracket, so function calls
+        won't be made.
+        """
+        self.open_calltip(False)
+
+    def refresh_calltip_event(self, event):
+        if self.active_calltip and self.active_calltip.tipwindow:
+            self.open_calltip(False)
+
+    def open_calltip(self, evalfuncs):
+        """Maybe close an existing calltip and maybe open a new calltip.
+
+        Called from (force_open|try_open|refresh)_calltip_event functions.
+        """
+        hp = HyperParser(self.editwin, "insert")
+        sur_paren = hp.get_surrounding_brackets('(')
+
+        # If not inside parentheses, no calltip.
+        if not sur_paren:
+            self.remove_calltip_window()
+            return
+
+        # If a calltip is shown for the current parentheses, do
+        # nothing.
+        if self.active_calltip:
+            opener_line, opener_col = map(int, sur_paren[0].split('.'))
+            if (
+                (opener_line, opener_col) ==
+                (self.active_calltip.parenline, self.active_calltip.parencol)
+            ):
+                return
+
+        hp.set_index(sur_paren[0])
+        try:
+            expression = hp.get_expression()
+        except ValueError:
+            expression = None
+        if not expression:
+            # No expression before the opening parenthesis, e.g.
+            # because it's in a string or the opener for a tuple:
+            # Do nothing.
+            return
+
+        # At this point, the current index is after an opening
+        # parenthesis, in a section of code, preceded by a valid
+        # expression. If there is a calltip shown, it's not for the
+        # same index and should be closed.
+        self.remove_calltip_window()
+
+        # Simple, fast heuristic: If the preceding expression includes
+        # an opening parenthesis, it likely includes a function call.
+        if not evalfuncs and (expression.find('(') != -1):
+            return
+
+        argspec = self.fetch_tip(expression)
+        if not argspec:
+            return
+        self.active_calltip = self._calltip_window()
+        self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1])
+
+    def fetch_tip(self, expression):
+        """Return the argument list and docstring of a function or class.
+
+        If there is a Python subprocess, get the calltip there.  Otherwise,
+        either this fetch_tip() is running in the subprocess or it was
+        called in an IDLE running without the subprocess.
+
+        The subprocess environment is that of the most recently run script.  If
+        two unrelated modules are being edited some calltips in the current
+        module may be inoperative if the module was not the last to run.
+
+        To find methods, fetch_tip must be fed a fully qualified name.
+
+        """
+        try:
+            rpcclt = self.editwin.flist.pyshell.interp.rpcclt
+        except AttributeError:
+            rpcclt = None
+        if rpcclt:
+            return rpcclt.remotecall("exec", "get_the_calltip",
+                                     (expression,), {})
+        else:
+            return get_argspec(get_entity(expression))
+
+
+def get_entity(expression):
+    """Return the object corresponding to expression evaluated
+    in a namespace spanning sys.modules and __main.dict__.
+    """
+    if expression:
+        namespace = {**sys.modules, **__main__.__dict__}
+        try:
+            return eval(expression, namespace)  # Only protect user code.
+        except BaseException:
+            # An uncaught exception closes idle, and eval can raise any
+            # exception, especially if user classes are involved.
+            return None
+
+# The following are used in get_argspec and some in tests
+_MAX_COLS = 85
+_MAX_LINES = 5  # enough for bytes
+_INDENT = ' '*4  # for wrapped signatures
+_first_param = re.compile(r'(?<=\()\w*\,?\s*')
+_default_callable_argspec = "See source or doc"
+_invalid_method = "invalid method signature"
+
+def get_argspec(ob):
+    '''Return a string describing the signature of a callable object, or ''.
+
+    For Python-coded functions and methods, the first line is introspected.
+    Delete 'self' parameter for classes (.__init__) and bound methods.
+    The next lines are the first lines of the doc string up to the first
+    empty line or _MAX_LINES.    For builtins, this typically includes
+    the arguments in addition to the return value.
+    '''
+    # Determine function object fob to inspect.
+    try:
+        ob_call = ob.__call__
+    except BaseException:  # Buggy user object could raise anything.
+        return ''  # No popup for non-callables.
+    # For Get_argspecTest.test_buggy_getattr_class, CallA() & CallB().
+    fob = ob_call if isinstance(ob_call, types.MethodType) else ob
+
+    # Initialize argspec and wrap it to get lines.
+    try:
+        argspec = str(inspect.signature(fob))
+    except Exception as err:
+        msg = str(err)
+        if msg.startswith(_invalid_method):
+            return _invalid_method
+        else:
+            argspec = ''
+
+    if isinstance(fob, type) and argspec == '()':
+        # If fob has no argument, use default callable argspec.
+        argspec = _default_callable_argspec
+
+    lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT)
+             if len(argspec) > _MAX_COLS else [argspec] if argspec else [])
+
+    # Augment lines from docstring, if any, and join to get argspec.
+    doc = inspect.getdoc(ob)
+    if doc:
+        for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]:
+            line = line.strip()
+            if not line:
+                break
+            if len(line) > _MAX_COLS:
+                line = line[: _MAX_COLS - 3] + '...'
+            lines.append(line)
+    argspec = '\n'.join(lines)
+
+    return argspec or _default_callable_argspec
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_calltip', verbosity=2)

Lib/idlelib/calltip_w.py

@@ -0,0 +1,202 @@
+"""A call-tip window class for Tkinter/IDLE.
+
+After tooltip.py, which uses ideas gleaned from PySol.
+Used by calltip.py.
+"""
+from tkinter import Label, LEFT, SOLID, TclError
+
+from idlelib.tooltip import TooltipBase
+
+HIDE_EVENT = "<<calltipwindow-hide>>"
+HIDE_SEQUENCES = ("<Key-Escape>", "<FocusOut>")
+CHECKHIDE_EVENT = "<<calltipwindow-checkhide>>"
+CHECKHIDE_SEQUENCES = ("<KeyRelease>", "<ButtonRelease>")
+CHECKHIDE_TIME = 100  # milliseconds
+
+MARK_RIGHT = "calltipwindowregion_right"
+
+
+class CalltipWindow(TooltipBase):
+    """A call-tip widget for tkinter text widgets."""
+
+    def __init__(self, text_widget):
+        """Create a call-tip; shown by showtip().
+
+        text_widget: a Text widget with code for which call-tips are desired
+        """
+        # Note: The Text widget will be accessible as self.anchor_widget
+        super().__init__(text_widget)
+
+        self.label = self.text = None
+        self.parenline = self.parencol = self.lastline = None
+        self.hideid = self.checkhideid = None
+        self.checkhide_after_id = None
+
+    def get_position(self):
+        """Choose the position of the call-tip."""
+        curline = int(self.anchor_widget.index("insert").split('.')[0])
+        if curline == self.parenline:
+            anchor_index = (self.parenline, self.parencol)
+        else:
+            anchor_index = (curline, 0)
+        box = self.anchor_widget.bbox("%d.%d" % anchor_index)
+        if not box:
+            box = list(self.anchor_widget.bbox("insert"))
+            # align to left of window
+            box[0] = 0
+            box[2] = 0
+        return box[0] + 2, box[1] + box[3]
+
+    def position_window(self):
+        "Reposition the window if needed."
+        curline = int(self.anchor_widget.index("insert").split('.')[0])
+        if curline == self.lastline:
+            return
+        self.lastline = curline
+        self.anchor_widget.see("insert")
+        super().position_window()
+
+    def showtip(self, text, parenleft, parenright):
+        """Show the call-tip, bind events which will close it and reposition it.
+
+        text: the text to display in the call-tip
+        parenleft: index of the opening parenthesis in the text widget
+        parenright: index of the closing parenthesis in the text widget,
+                    or the end of the line if there is no closing parenthesis
+        """
+        # Only called in calltip.Calltip, where lines are truncated
+        self.text = text
+        if self.tipwindow or not self.text:
+            return
+
+        self.anchor_widget.mark_set(MARK_RIGHT, parenright)
+        self.parenline, self.parencol = map(
+            int, self.anchor_widget.index(parenleft).split("."))
+
+        super().showtip()
+
+        self._bind_events()
+
+    def showcontents(self):
+        """Create the call-tip widget."""
+        self.label = Label(self.tipwindow, text=self.text, justify=LEFT,
+                           background="#ffffd0", foreground="black",
+                           relief=SOLID, borderwidth=1,
+                           font=self.anchor_widget['font'])
+        self.label.pack()
+
+    def checkhide_event(self, event=None):
+        """Handle CHECK_HIDE_EVENT: call hidetip or reschedule."""
+        if not self.tipwindow:
+            # If the event was triggered by the same event that unbound
+            # this function, the function will be called nevertheless,
+            # so do nothing in this case.
+            return None
+
+        # Hide the call-tip if the insertion cursor moves outside of the
+        # parenthesis.
+        curline, curcol = map(int, self.anchor_widget.index("insert").split('.'))
+        if curline < self.parenline or \
+           (curline == self.parenline and curcol <= self.parencol) or \
+           self.anchor_widget.compare("insert", ">", MARK_RIGHT):
+            self.hidetip()
+            return "break"
+
+        # Not hiding the call-tip.
+
+        self.position_window()
+        # Re-schedule this function to be called again in a short while.
+        if self.checkhide_after_id is not None:
+            self.anchor_widget.after_cancel(self.checkhide_after_id)
+        self.checkhide_after_id = \
+            self.anchor_widget.after(CHECKHIDE_TIME, self.checkhide_event)
+        return None
+
+    def hide_event(self, event):
+        """Handle HIDE_EVENT by calling hidetip."""
+        if not self.tipwindow:
+            # See the explanation in checkhide_event.
+            return None
+        self.hidetip()
+        return "break"
+
+    def hidetip(self):
+        """Hide the call-tip."""
+        if not self.tipwindow:
+            return
+
+        try:
+            self.label.destroy()
+        except TclError:
+            pass
+        self.label = None
+
+        self.parenline = self.parencol = self.lastline = None
+        try:
+            self.anchor_widget.mark_unset(MARK_RIGHT)
+        except TclError:
+            pass
+
+        try:
+            self._unbind_events()
+        except (TclError, ValueError):
+            # ValueError may be raised by MultiCall
+            pass
+
+        super().hidetip()
+
+    def _bind_events(self):
+        """Bind event handlers."""
+        self.checkhideid = self.anchor_widget.bind(CHECKHIDE_EVENT,
+                                                   self.checkhide_event)
+        for seq in CHECKHIDE_SEQUENCES:
+            self.anchor_widget.event_add(CHECKHIDE_EVENT, seq)
+        self.anchor_widget.after(CHECKHIDE_TIME, self.checkhide_event)
+        self.hideid = self.anchor_widget.bind(HIDE_EVENT,
+                                              self.hide_event)
+        for seq in HIDE_SEQUENCES:
+            self.anchor_widget.event_add(HIDE_EVENT, seq)
+
+    def _unbind_events(self):
+        """Unbind event handlers."""
+        for seq in CHECKHIDE_SEQUENCES:
+            self.anchor_widget.event_delete(CHECKHIDE_EVENT, seq)
+        self.anchor_widget.unbind(CHECKHIDE_EVENT, self.checkhideid)
+        self.checkhideid = None
+        for seq in HIDE_SEQUENCES:
+            self.anchor_widget.event_delete(HIDE_EVENT, seq)
+        self.anchor_widget.unbind(HIDE_EVENT, self.hideid)
+        self.hideid = None
+
+
+def _calltip_window(parent):  # htest #
+    from tkinter import Toplevel, Text, LEFT, BOTH
+
+    top = Toplevel(parent)
+    top.title("Test call-tips")
+    x, y = map(int, parent.geometry().split('+')[1:])
+    top.geometry("250x100+%d+%d" % (x + 175, y + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+
+    calltip = CalltipWindow(text)
+    def calltip_show(event):
+        calltip.showtip("(s='Hello world')", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<<calltip-show>>", "(")
+    text.event_add("<<calltip-hide>>", ")")
+    text.bind("<<calltip-show>>", calltip_show)
+    text.bind("<<calltip-hide>>", calltip_hide)
+
+    text.focus_set()
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_calltip_w', verbosity=2, exit=False)
+
+    from idlelib.idle_test.htest import run
+    run(_calltip_window)

Lib/idlelib/codecontext.py

@@ -0,0 +1,270 @@
+"""codecontext - display the block context above the edit window
+
+Once code has scrolled off the top of a window, it can be difficult to
+determine which block you are in.  This extension implements a pane at the top
+of each IDLE edit window which provides block structure hints.  These hints are
+the lines which contain the block opening keywords, e.g. 'if', for the
+enclosing block.  The number of hint lines is determined by the maxlines
+variable in the codecontext section of config-extensions.def. Lines which do
+not open blocks are not shown in the context hints pane.
+
+For EditorWindows, <<toggle-code-context>> is bound to CodeContext(self).
+toggle_code_context_event.
+"""
+import re
+from sys import maxsize as INFINITY
+
+from tkinter import Frame, Text, TclError
+from tkinter.constants import NSEW, SUNKEN
+
+from idlelib.config import idleConf
+
+BLOCKOPENERS = {'class', 'def', 'if', 'elif', 'else', 'while', 'for',
+                 'try', 'except', 'finally', 'with', 'async'}
+
+
+def get_spaces_firstword(codeline, c=re.compile(r"^(\s*)(\w*)")):
+    "Extract the beginning whitespace and first word from codeline."
+    return c.match(codeline).groups()
+
+
+def get_line_info(codeline):
+    """Return tuple of (line indent value, codeline, block start keyword).
+
+    The indentation of empty lines (or comment lines) is INFINITY.
+    If the line does not start a block, the keyword value is False.
+    """
+    spaces, firstword = get_spaces_firstword(codeline)
+    indent = len(spaces)
+    if len(codeline) == indent or codeline[indent] == '#':
+        indent = INFINITY
+    opener = firstword in BLOCKOPENERS and firstword
+    return indent, codeline, opener
+
+
+class CodeContext:
+    "Display block context above the edit window."
+    UPDATEINTERVAL = 100  # millisec
+
+    def __init__(self, editwin):
+        """Initialize settings for context block.
+
+        editwin is the Editor window for the context block.
+        self.text is the editor window text widget.
+
+        self.context displays the code context text above the editor text.
+          Initially None, it is toggled via <<toggle-code-context>>.
+        self.topvisible is the number of the top text line displayed.
+        self.info is a list of (line number, indent level, line text,
+          block keyword) tuples for the block structure above topvisible.
+          self.info[0] is initialized with a 'dummy' line which
+          starts the toplevel 'block' of the module.
+
+        self.t1 and self.t2 are two timer events on the editor text widget to
+          monitor for changes to the context text or editor font.
+        """
+        self.editwin = editwin
+        self.text = editwin.text
+        self._reset()
+
+    def _reset(self):
+        self.context = None
+        self.cell00 = None
+        self.t1 = None
+        self.topvisible = 1
+        self.info = [(0, -1, "", False)]
+
+    @classmethod
+    def reload(cls):
+        "Load class variables from config."
+        cls.context_depth = idleConf.GetOption("extensions", "CodeContext",
+                                               "maxlines", type="int",
+                                               default=15)
+
+    def __del__(self):
+        "Cancel scheduled events."
+        if self.t1 is not None:
+            try:
+                self.text.after_cancel(self.t1)
+            except TclError:  # pragma: no cover
+                pass
+            self.t1 = None
+
+    def toggle_code_context_event(self, event=None):
+        """Toggle code context display.
+
+        If self.context doesn't exist, create it to match the size of the editor
+        window text (toggle on).  If it does exist, destroy it (toggle off).
+        Return 'break' to complete the processing of the binding.
+        """
+        if self.context is None:
+            # Calculate the border width and horizontal padding required to
+            # align the context with the text in the main Text widget.
+            #
+            # All values are passed through getint(), since some
+            # values may be pixel objects, which can't simply be added to ints.
+            widgets = self.editwin.text, self.editwin.text_frame
+            # Calculate the required horizontal padding and border width.
+            padx = 0
+            border = 0
+            for widget in widgets:
+                info = (widget.grid_info()
+                        if widget is self.editwin.text
+                        else widget.pack_info())
+                padx += widget.tk.getint(info['padx'])
+                padx += widget.tk.getint(widget.cget('padx'))
+                border += widget.tk.getint(widget.cget('border'))
+            context = self.context = Text(
+                self.editwin.text_frame,
+                height=1,
+                width=1,  # Don't request more than we get.
+                highlightthickness=0,
+                padx=padx, border=border, relief=SUNKEN, state='disabled')
+            self.update_font()
+            self.update_highlight_colors()
+            context.bind('<ButtonRelease-1>', self.jumptoline)
+            # Get the current context and initiate the recurring update event.
+            self.timer_event()
+            # Grid the context widget above the text widget.
+            context.grid(row=0, column=1, sticky=NSEW)
+
+            line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(),
+                                                       'linenumber')
+            self.cell00 = Frame(self.editwin.text_frame,
+                                        bg=line_number_colors['background'])
+            self.cell00.grid(row=0, column=0, sticky=NSEW)
+            menu_status = 'Hide'
+        else:
+            self.context.destroy()
+            self.context = None
+            self.cell00.destroy()
+            self.cell00 = None
+            self.text.after_cancel(self.t1)
+            self._reset()
+            menu_status = 'Show'
+        self.editwin.update_menu_label(menu='options', index='*ode*ontext',
+                                       label=f'{menu_status} Code Context')
+        return "break"
+
+    def get_context(self, new_topvisible, stopline=1, stopindent=0):
+        """Return a list of block line tuples and the 'last' indent.
+
+        The tuple fields are (linenum, indent, text, opener).
+        The list represents header lines from new_topvisible back to
+        stopline with successively shorter indents > stopindent.
+        The list is returned ordered by line number.
+        Last indent returned is the smallest indent observed.
+        """
+        assert stopline > 0
+        lines = []
+        # The indentation level we are currently in.
+        lastindent = INFINITY
+        # For a line to be interesting, it must begin with a block opening
+        # keyword, and have less indentation than lastindent.
+        for linenum in range(new_topvisible, stopline-1, -1):
+            codeline = self.text.get(f'{linenum}.0', f'{linenum}.end')
+            indent, text, opener = get_line_info(codeline)
+            if indent < lastindent:
+                lastindent = indent
+                if opener in ("else", "elif"):
+                    # Also show the if statement.
+                    lastindent += 1
+                if opener and linenum < new_topvisible and indent >= stopindent:
+                    lines.append((linenum, indent, text, opener))
+                if lastindent <= stopindent:
+                    break
+        lines.reverse()
+        return lines, lastindent
+
+    def update_code_context(self):
+        """Update context information and lines visible in the context pane.
+
+        No update is done if the text hasn't been scrolled.  If the text
+        was scrolled, the lines that should be shown in the context will
+        be retrieved and the context area will be updated with the code,
+        up to the number of maxlines.
+        """
+        new_topvisible = self.editwin.getlineno("@0,0")
+        if self.topvisible == new_topvisible:      # Haven't scrolled.
+            return
+        if self.topvisible < new_topvisible:       # Scroll down.
+            lines, lastindent = self.get_context(new_topvisible,
+                                                 self.topvisible)
+            # Retain only context info applicable to the region
+            # between topvisible and new_topvisible.
+            while self.info[-1][1] >= lastindent:
+                del self.info[-1]
+        else:  # self.topvisible > new_topvisible: # Scroll up.
+            stopindent = self.info[-1][1] + 1
+            # Retain only context info associated
+            # with lines above new_topvisible.
+            while self.info[-1][0] >= new_topvisible:
+                stopindent = self.info[-1][1]
+                del self.info[-1]
+            lines, lastindent = self.get_context(new_topvisible,
+                                                 self.info[-1][0]+1,
+                                                 stopindent)
+        self.info.extend(lines)
+        self.topvisible = new_topvisible
+        # Last context_depth context lines.
+        context_strings = [x[2] for x in self.info[-self.context_depth:]]
+        showfirst = 0 if context_strings[0] else 1
+        # Update widget.
+        self.context['height'] = len(context_strings) - showfirst
+        self.context['state'] = 'normal'
+        self.context.delete('1.0', 'end')
+        self.context.insert('end', '\n'.join(context_strings[showfirst:]))
+        self.context['state'] = 'disabled'
+
+    def jumptoline(self, event=None):
+        """ Show clicked context line at top of editor.
+
+        If a selection was made, don't jump; allow copying.
+        If no visible context, show the top line of the file.
+        """
+        try:
+            self.context.index("sel.first")
+        except TclError:
+            lines = len(self.info)
+            if lines == 1:  # No context lines are showing.
+                newtop = 1
+            else:
+                # Line number clicked.
+                contextline = int(float(self.context.index('insert')))
+                # Lines not displayed due to maxlines.
+                offset = max(1, lines - self.context_depth) - 1
+                newtop = self.info[offset + contextline][0]
+            self.text.yview(f'{newtop}.0')
+            self.update_code_context()
+
+    def timer_event(self):
+        "Event on editor text widget triggered every UPDATEINTERVAL ms."
+        if self.context is not None:
+            self.update_code_context()
+            self.t1 = self.text.after(self.UPDATEINTERVAL, self.timer_event)
+
+    def update_font(self):
+        if self.context is not None:
+            font = idleConf.GetFont(self.text, 'main', 'EditorWindow')
+            self.context['font'] = font
+
+    def update_highlight_colors(self):
+        if self.context is not None:
+            colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'context')
+            self.context['background'] = colors['background']
+            self.context['foreground'] = colors['foreground']
+
+        if self.cell00 is not None:
+            line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(),
+                                                       'linenumber')
+            self.cell00.config(bg=line_number_colors['background'])
+
+
+CodeContext.reload()
+
+
+if __name__ == "__main__":
+    from unittest import main
+    main('idlelib.idle_test.test_codecontext', verbosity=2, exit=False)
+
+    # Add htest.

Lib/idlelib/colorizer.py

@@ -0,0 +1,384 @@
+import builtins
+import keyword
+import re
+import time
+
+from idlelib.config import idleConf
+from idlelib.delegator import Delegator
+
+DEBUG = False
+
+
+def any(name, alternates):
+    "Return a named group pattern matching list of alternates."
+    return "(?P<%s>" % name + "|".join(alternates) + ")"
+
+
+def make_pat():
+    kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
+    match_softkw = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<MATCH_SOFTKW>match)\b" +
+        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
+            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
+                                 # pattern-matching statement
+            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
+        ]) +
+        r"))"
+    )
+    case_default = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<CASE_SOFTKW>case)" +
+        r"[ \t]+(?P<CASE_DEFAULT_UNDERSCORE>_\b)"
+    )
+    case_softkw_and_pattern = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<CASE_SOFTKW2>case)\b" +
+        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
+            r"_\b",  # a lone underscore
+            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
+                                 # pattern-matching case
+            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
+        ]) +
+        r"))"
+    )
+    builtinlist = [str(name) for name in dir(builtins)
+                   if not name.startswith('_') and
+                   name not in keyword.kwlist]
+    builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
+    comment = any("COMMENT", [r"#[^\n]*"])
+    stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb|t|rt|tr)?"
+    sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
+    dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
+    sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
+    dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
+    string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
+    prog = re.compile("|".join([
+                                builtin, comment, string, kw,
+                                match_softkw, case_default,
+                                case_softkw_and_pattern,
+                                any("SYNC", [r"\n"]),
+                               ]),
+                      re.DOTALL | re.MULTILINE)
+    return prog
+
+
+prog = make_pat()
+idprog = re.compile(r"\s+(\w+)")
+prog_group_name_to_tag = {
+    "MATCH_SOFTKW": "KEYWORD",
+    "CASE_SOFTKW": "KEYWORD",
+    "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
+    "CASE_SOFTKW2": "KEYWORD",
+}
+
+
+def matched_named_groups(re_match):
+    "Get only the non-empty named groups from an re.Match object."
+    return ((k, v) for (k, v) in re_match.groupdict().items() if v)
+
+
+def color_config(text):
+    """Set color options of Text widget.
+
+    If ColorDelegator is used, this should be called first.
+    """
+    # Called from htest, TextFrame, Editor, and Turtledemo.
+    # Not automatic because ColorDelegator does not know 'text'.
+    theme = idleConf.CurrentTheme()
+    normal_colors = idleConf.GetHighlight(theme, 'normal')
+    cursor_color = idleConf.GetHighlight(theme, 'cursor')['foreground']
+    select_colors = idleConf.GetHighlight(theme, 'hilite')
+    text.config(
+        foreground=normal_colors['foreground'],
+        background=normal_colors['background'],
+        insertbackground=cursor_color,
+        selectforeground=select_colors['foreground'],
+        selectbackground=select_colors['background'],
+        inactiveselectbackground=select_colors['background'],  # new in 8.5
+        )
+
+
+class ColorDelegator(Delegator):
+    """Delegator for syntax highlighting (text coloring).
+
+    Instance variables:
+        delegate: Delegator below this one in the stack, meaning the
+                one this one delegates to.
+
+        Used to track state:
+        after_id: Identifier for scheduled after event, which is a
+                timer for colorizing the text.
+        allow_colorizing: Boolean toggle for applying colorizing.
+        colorizing: Boolean flag when colorizing is in process.
+        stop_colorizing: Boolean flag to end an active colorizing
+                process.
+    """
+
+    def __init__(self):
+        Delegator.__init__(self)
+        self.init_state()
+        self.prog = prog
+        self.idprog = idprog
+        self.LoadTagDefs()
+
+    def init_state(self):
+        "Initialize variables that track colorizing state."
+        self.after_id = None
+        self.allow_colorizing = True
+        self.stop_colorizing = False
+        self.colorizing = False
+
+    def setdelegate(self, delegate):
+        """Set the delegate for this instance.
+
+        A delegate is an instance of a Delegator class and each
+        delegate points to the next delegator in the stack.  This
+        allows multiple delegators to be chained together for a
+        widget.  The bottom delegate for a colorizer is a Text
+        widget.
+
+        If there is a delegate, also start the colorizing process.
+        """
+        if self.delegate is not None:
+            self.unbind("<<toggle-auto-coloring>>")
+        Delegator.setdelegate(self, delegate)
+        if delegate is not None:
+            self.config_colors()
+            self.bind("<<toggle-auto-coloring>>", self.toggle_colorize_event)
+            self.notify_range("1.0", "end")
+        else:
+            # No delegate - stop any colorizing.
+            self.stop_colorizing = True
+            self.allow_colorizing = False
+
+    def config_colors(self):
+        "Configure text widget tags with colors from tagdefs."
+        for tag, cnf in self.tagdefs.items():
+            self.tag_configure(tag, **cnf)
+        self.tag_raise('sel')
+
+    def LoadTagDefs(self):
+        "Create dictionary of tag names to text colors."
+        theme = idleConf.CurrentTheme()
+        self.tagdefs = {
+            "COMMENT": idleConf.GetHighlight(theme, "comment"),
+            "KEYWORD": idleConf.GetHighlight(theme, "keyword"),
+            "BUILTIN": idleConf.GetHighlight(theme, "builtin"),
+            "STRING": idleConf.GetHighlight(theme, "string"),
+            "DEFINITION": idleConf.GetHighlight(theme, "definition"),
+            "SYNC": {'background': None, 'foreground': None},
+            "TODO": {'background': None, 'foreground': None},
+            "ERROR": idleConf.GetHighlight(theme, "error"),
+            # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but
+            # that currently isn't technically possible. This should be moved elsewhere in the future
+            # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a
+            # non-modal alternative.
+            "hit": idleConf.GetHighlight(theme, "hit"),
+            }
+        if DEBUG: print('tagdefs', self.tagdefs)
+
+    def insert(self, index, chars, tags=None):
+        "Insert chars into widget at index and mark for colorizing."
+        index = self.index(index)
+        self.delegate.insert(index, chars, tags)
+        self.notify_range(index, index + "+%dc" % len(chars))
+
+    def delete(self, index1, index2=None):
+        "Delete chars between indexes and mark for colorizing."
+        index1 = self.index(index1)
+        self.delegate.delete(index1, index2)
+        self.notify_range(index1)
+
+    def notify_range(self, index1, index2=None):
+        "Mark text changes for processing and restart colorizing, if active."
+        self.tag_add("TODO", index1, index2)
+        if self.after_id:
+            if DEBUG: print("colorizing already scheduled")
+            return
+        if self.colorizing:
+            self.stop_colorizing = True
+            if DEBUG: print("stop colorizing")
+        if self.allow_colorizing:
+            if DEBUG: print("schedule colorizing")
+            self.after_id = self.after(1, self.recolorize)
+        return
+
+    def close(self):
+        if self.after_id:
+            after_id = self.after_id
+            self.after_id = None
+            if DEBUG: print("cancel scheduled recolorizer")
+            self.after_cancel(after_id)
+        self.allow_colorizing = False
+        self.stop_colorizing = True
+
+    def toggle_colorize_event(self, event=None):
+        """Toggle colorizing on and off.
+
+        When toggling off, if colorizing is scheduled or is in
+        process, it will be cancelled and/or stopped.
+
+        When toggling on, colorizing will be scheduled.
+        """
+        if self.after_id:
+            after_id = self.after_id
+            self.after_id = None
+            if DEBUG: print("cancel scheduled recolorizer")
+            self.after_cancel(after_id)
+        if self.allow_colorizing and self.colorizing:
+            if DEBUG: print("stop colorizing")
+            self.stop_colorizing = True
+        self.allow_colorizing = not self.allow_colorizing
+        if self.allow_colorizing and not self.colorizing:
+            self.after_id = self.after(1, self.recolorize)
+        if DEBUG:
+            print("auto colorizing turned",
+                  "on" if self.allow_colorizing else "off")
+        return "break"
+
+    def recolorize(self):
+        """Timer event (every 1ms) to colorize text.
+
+        Colorizing is only attempted when the text widget exists,
+        when colorizing is toggled on, and when the colorizing
+        process is not already running.
+
+        After colorizing is complete, some cleanup is done to
+        make sure that all the text has been colorized.
+        """
+        self.after_id = None
+        if not self.delegate:
+            if DEBUG: print("no delegate")
+            return
+        if not self.allow_colorizing:
+            if DEBUG: print("auto colorizing is off")
+            return
+        if self.colorizing:
+            if DEBUG: print("already colorizing")
+            return
+        try:
+            self.stop_colorizing = False
+            self.colorizing = True
+            if DEBUG: print("colorizing...")
+            t0 = time.perf_counter()
+            self.recolorize_main()
+            t1 = time.perf_counter()
+            if DEBUG: print("%.3f seconds" % (t1-t0))
+        finally:
+            self.colorizing = False
+        if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
+            if DEBUG: print("reschedule colorizing")
+            self.after_id = self.after(1, self.recolorize)
+
+    def recolorize_main(self):
+        "Evaluate text and apply colorizing tags."
+        next = "1.0"
+        while todo_tag_range := self.tag_nextrange("TODO", next):
+            self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1])
+            sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0])
+            head = sync_tag_range[1] if sync_tag_range else "1.0"
+
+            chars = ""
+            next = head
+            lines_to_get = 1
+            ok = False
+            while not ok:
+                mark = next
+                next = self.index(mark + "+%d lines linestart" %
+                                         lines_to_get)
+                lines_to_get = min(lines_to_get * 2, 100)
+                ok = "SYNC" in self.tag_names(next + "-1c")
+                line = self.get(mark, next)
+                ##print head, "get", mark, next, "->", repr(line)
+                if not line:
+                    return
+                for tag in self.tagdefs:
+                    self.tag_remove(tag, mark, next)
+                chars += line
+                self._add_tags_in_section(chars, head)
+                if "SYNC" in self.tag_names(next + "-1c"):
+                    head = next
+                    chars = ""
+                else:
+                    ok = False
+                if not ok:
+                    # We're in an inconsistent state, and the call to
+                    # update may tell us to stop.  It may also change
+                    # the correct value for "next" (since this is a
+                    # line.col string, not a true mark).  So leave a
+                    # crumb telling the next invocation to resume here
+                    # in case update tells us to leave.
+                    self.tag_add("TODO", next)
+                self.update_idletasks()
+                if self.stop_colorizing:
+                    if DEBUG: print("colorizing stopped")
+                    return
+
+    def _add_tag(self, start, end, head, matched_group_name):
+        """Add a tag to a given range in the text widget.
+
+        This is a utility function, receiving the range as `start` and
+        `end` positions, each of which is a number of characters
+        relative to the given `head` index in the text widget.
+
+        The tag to add is determined by `matched_group_name`, which is
+        the name of a regular expression "named group" as matched by
+        by the relevant highlighting regexps.
+        """
+        tag = prog_group_name_to_tag.get(matched_group_name,
+                                         matched_group_name)
+        self.tag_add(tag,
+                     f"{head}+{start:d}c",
+                     f"{head}+{end:d}c")
+
+    def _add_tags_in_section(self, chars, head):
+        """Parse and add highlighting tags to a given part of the text.
+
+        `chars` is a string with the text to parse and to which
+        highlighting is to be applied.
+
+            `head` is the index in the text widget where the text is found.
+        """
+        for m in self.prog.finditer(chars):
+            for name, matched_text in matched_named_groups(m):
+                a, b = m.span(name)
+                self._add_tag(a, b, head, name)
+                if matched_text in ("def", "class"):
+                    if m1 := self.idprog.match(chars, b):
+                        a, b = m1.span(1)
+                        self._add_tag(a, b, head, "DEFINITION")
+
+    def removecolors(self):
+        "Remove all colorizing tags."
+        for tag in self.tagdefs:
+            self.tag_remove(tag, "1.0", "end")
+
+
+def _color_delegator(parent):  # htest #
+    from tkinter import Toplevel, Text
+    from idlelib.idle_test.test_colorizer import source
+    from idlelib.percolator import Percolator
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    x, y = map(int, parent.geometry().split('+')[1:])
+    top.geometry("700x550+%d+%d" % (x + 20, y + 175))
+
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
+    text.insert("insert", source)
+    text.focus_set()
+
+    color_config(text)
+    p = Percolator(text)
+    d = ColorDelegator()
+    p.insertfilter(d)
+
+
+if __name__ == "__main__":
+    from unittest import main
+    main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False)
+
+    from idlelib.idle_test.htest import run
+    run(_color_delegator)

Lib/idlelib/config-extensions.def

@@ -0,0 +1,62 @@
+# config-extensions.def
+#
+# The following sections are for features that are no longer extensions.
+# Their options values are left here for back-compatibility.
+
+[AutoComplete]
+popupwait= 2000
+
+[CodeContext]
+maxlines= 15
+
+[FormatParagraph]
+max-width= 72
+
+[ParenMatch]
+style= expression
+flash-delay= 500
+bell= True
+
+# IDLE reads several config files to determine user preferences.  This
+# file is the default configuration file for IDLE extensions settings.
+#
+# Each extension must have at least one section, named after the
+# extension module. This section must contain an 'enable' item (=True to
+# enable the extension, =False to disable it), it may contain
+# 'enable_editor' or 'enable_shell' items, to apply it only to editor ir
+# shell windows, and may also contain any other general configuration
+# items for the extension.  Other True/False values will also be
+# recognized as boolean by the Extension Configuration dialog.
+#
+# Each extension must define at least one section named
+# ExtensionName_bindings or ExtensionName_cfgBindings. If present,
+# ExtensionName_bindings defines virtual event bindings for the
+# extension that are not user re-configurable. If present,
+# ExtensionName_cfgBindings defines virtual event bindings for the
+# extension that may be sensibly re-configured.
+#
+# If there are no keybindings for a menus' virtual events, include lines
+# like <<toggle-code-context>>=.
+#
+# Currently it is necessary to manually modify this file to change
+# extension key bindings and default values. To customize, create
+# ~/.idlerc/config-extensions.cfg and append the appropriate customized
+# section(s).  Those sections will override the defaults in this file.
+#
+# Note: If a keybinding is already in use when the extension is loaded,
+# the extension's virtual event's keybinding will be set to ''.
+#
+# See config-keys.def for notes on specifying keys and extend.txt for
+# information on creating IDLE extensions.
+
+# A fake extension for testing and example purposes.  When enabled and
+# invoked, inserts or deletes z-text at beginning of every line.
+[ZzDummy]
+enable= False
+enable_shell = False
+enable_editor = True
+z-text= Z
+[ZzDummy_cfgBindings]
+z-in= <Control-Shift-KeyRelease-Insert>
+[ZzDummy_bindings]
+z-out= <Control-Shift-KeyRelease-Delete>

Lib/idlelib/config-highlight.def

@@ -0,0 +1,105 @@
+# IDLE reads several config files to determine user preferences.  This
+# file is the default config file for idle highlight theme settings.
+
+[IDLE Classic]
+normal-foreground= #000000
+normal-background= #ffffff
+keyword-foreground= #ff7700
+keyword-background= #ffffff
+builtin-foreground= #900090
+builtin-background= #ffffff
+comment-foreground= #dd0000
+comment-background= #ffffff
+string-foreground= #00aa00
+string-background= #ffffff
+definition-foreground= #0000ff
+definition-background= #ffffff
+hilite-foreground= #000000
+hilite-background= gray
+break-foreground= black
+break-background= #ffff55
+hit-foreground= #ffffff
+hit-background= #000000
+error-foreground= #000000
+error-background= #ff7777
+context-foreground= #000000
+context-background= lightgray
+linenumber-foreground= gray
+linenumber-background= #ffffff
+#cursor (only foreground can be set, restart IDLE)
+cursor-foreground= black
+#shell window
+stdout-foreground= blue
+stdout-background= #ffffff
+stderr-foreground= red
+stderr-background= #ffffff
+console-foreground= #770000
+console-background= #ffffff
+
+[IDLE New]
+normal-foreground= #000000
+normal-background= #ffffff
+keyword-foreground= #ff7700
+keyword-background= #ffffff
+builtin-foreground= #900090
+builtin-background= #ffffff
+comment-foreground= #dd0000
+comment-background= #ffffff
+string-foreground= #00aa00
+string-background= #ffffff
+definition-foreground= #0000ff
+definition-background= #ffffff
+hilite-foreground= #000000
+hilite-background= gray
+break-foreground= black
+break-background= #ffff55
+hit-foreground= #ffffff
+hit-background= #000000
+error-foreground= #000000
+error-background= #ff7777
+context-foreground= #000000
+context-background= lightgray
+linenumber-foreground= gray
+linenumber-background= #ffffff
+#cursor (only foreground can be set, restart IDLE)
+cursor-foreground= black
+#shell window
+stdout-foreground= blue
+stdout-background= #ffffff
+stderr-foreground= red
+stderr-background= #ffffff
+console-foreground= #770000
+console-background= #ffffff
+
+[IDLE Dark]
+comment-foreground = #dd0000
+console-foreground = #ff4d4d
+error-foreground = #FFFFFF
+hilite-background = #7e7e7e
+string-foreground = #02ff02
+stderr-background = #002240
+stderr-foreground = #ffb3b3
+console-background = #002240
+hit-background = #fbfbfb
+string-background = #002240
+normal-background = #002240
+hilite-foreground = #FFFFFF
+keyword-foreground = #ff8000
+error-background = #c86464
+keyword-background = #002240
+builtin-background = #002240
+break-background = #808000
+builtin-foreground = #ff00ff
+definition-foreground = #5e5eff
+stdout-foreground = #c2d1fa
+definition-background = #002240
+normal-foreground = #FFFFFF
+cursor-foreground = #ffffff
+stdout-background = #002240
+hit-foreground = #002240
+comment-background = #002240
+break-foreground = #FFFFFF
+context-foreground= #ffffff
+context-background= #454545
+linenumber-foreground= gray
+linenumber-background= #002240

Lib/idlelib/config-keys.def

@@ -0,0 +1,309 @@
+# IDLE reads several config files to determine user preferences.  This
+# file is the default config file for idle key binding settings.
+# Where multiple keys are specified for an action: if they are separated
+# by a space (eg. action=<key1> <key2>) then the keys are alternatives, if
+# there is no space (eg. action=<key1><key2>) then the keys comprise a
+# single 'emacs style' multi-keystoke binding. The tk event specifier 'Key'
+# is used in all cases, for consistency in auto key conflict checking in the
+# configuration gui.
+
+[IDLE Classic Windows]
+copy=<Control-Key-c> <Control-Key-C>
+cut=<Control-Key-x> <Control-Key-X>
+paste=<Control-Key-v> <Control-Key-V>
+beginning-of-line= <Key-Home>
+center-insert=<Control-Key-l> <Control-Key-L>
+close-all-windows=<Control-Key-q> <Control-Key-Q>
+close-window=<Alt-Key-F4> <Meta-Key-F4>
+do-nothing=<Control-Key-F12>
+end-of-file=<Control-Key-d> <Control-Key-D>
+python-docs=<Key-F1>
+python-context-help=<Shift-Key-F1>
+history-next=<Alt-Key-n> <Meta-Key-n> <Alt-Key-N> <Meta-Key-N>
+history-previous=<Alt-Key-p> <Meta-Key-p> <Alt-Key-P> <Meta-Key-P>
+interrupt-execution=<Control-Key-c> <Control-Key-C>
+view-restart=<Key-F6>
+restart-shell=<Control-Key-F6>
+open-class-browser=<Alt-Key-c> <Meta-Key-c> <Alt-Key-C> <Meta-Key-C>
+open-module=<Alt-Key-m> <Meta-Key-m> <Alt-Key-M> <Meta-Key-M>
+open-new-window=<Control-Key-n> <Control-Key-N>
+open-window-from-file=<Control-Key-o> <Control-Key-O>
+plain-newline-and-indent=<Control-Key-j> <Control-Key-J>
+print-window=<Control-Key-p> <Control-Key-P>
+redo=<Control-Shift-Key-Z> <Control-Shift-Key-z>
+remove-selection=<Key-Escape>
+save-copy-of-window-as-file=<Alt-Shift-Key-S> <Alt-Shift-Key-s>
+save-window-as-file=<Control-Shift-Key-S> <Control-Shift-Key-s>
+save-window=<Control-Key-s> <Control-Key-S>
+select-all=<Control-Key-a> <Control-Key-A>
+toggle-auto-coloring=<Control-Key-slash>
+undo=<Control-Key-z> <Control-Key-Z>
+find=<Control-Key-f> <Control-Key-F>
+find-again=<Control-Key-g> <Key-F3> <Control-Key-G>
+find-in-files=<Alt-Key-F3> <Meta-Key-F3>
+find-selection=<Control-Key-F3>
+replace=<Control-Key-h> <Control-Key-H>
+goto-line=<Alt-Key-g> <Meta-Key-g> <Alt-Key-G> <Meta-Key-G>
+smart-backspace=<Key-BackSpace>
+newline-and-indent=<Key-Return> <Key-KP_Enter>
+smart-indent=<Key-Tab>
+indent-region=<Control-Key-bracketright>
+dedent-region=<Control-Key-bracketleft>
+comment-region=<Alt-Key-3> <Meta-Key-3>
+uncomment-region=<Alt-Key-4> <Meta-Key-4>
+tabify-region=<Alt-Key-5> <Meta-Key-5>
+untabify-region=<Alt-Key-6> <Meta-Key-6>
+toggle-tabs=<Alt-Key-t> <Meta-Key-t> <Alt-Key-T> <Meta-Key-T>
+change-indentwidth=<Alt-Key-u> <Meta-Key-u> <Alt-Key-U> <Meta-Key-U>
+del-word-left=<Control-Key-BackSpace>
+del-word-right=<Control-Key-Delete>
+force-open-completions= <Control-Key-space>
+expand-word= <Alt-Key-slash>
+force-open-calltip= <Control-Key-backslash>
+format-paragraph= <Alt-Key-q>
+flash-paren= <Control-Key-0>
+run-module= <Key-F5>
+run-custom= <Shift-Key-F5>
+check-module= <Alt-Key-x>
+zoom-height= <Alt-Key-2>
+
+[IDLE Classic Unix]
+copy=<Alt-Key-w> <Meta-Key-w>
+cut=<Control-Key-w>
+paste=<Control-Key-y>
+beginning-of-line=<Control-Key-a> <Key-Home>
+center-insert=<Control-Key-l>
+close-all-windows=<Control-Key-x><Control-Key-c>
+close-window=<Control-Key-x><Control-Key-0>
+do-nothing=<Control-Key-x>
+end-of-file=<Control-Key-d>
+history-next=<Alt-Key-n> <Meta-Key-n>
+history-previous=<Alt-Key-p> <Meta-Key-p>
+interrupt-execution=<Control-Key-c>
+view-restart=<Key-F6>
+restart-shell=<Control-Key-F6>
+open-class-browser=<Control-Key-x><Control-Key-b>
+open-module=<Control-Key-x><Control-Key-m>
+open-new-window=<Control-Key-x><Control-Key-n>
+open-window-from-file=<Control-Key-x><Control-Key-f>
+plain-newline-and-indent=<Control-Key-j>
+print-window=<Control-x><Control-Key-p>
+python-docs=<Control-Key-h>
+python-context-help=<Control-Shift-Key-H>
+redo=<Alt-Key-z> <Meta-Key-z>
+remove-selection=<Key-Escape>
+save-copy-of-window-as-file=<Control-Key-x><Control-Key-y>
+save-window-as-file=<Control-Key-x><Control-Key-w>
+save-window=<Control-Key-x><Control-Key-s>
+select-all=<Alt-Key-a> <Meta-Key-a>
+toggle-auto-coloring=<Control-Key-slash>
+undo=<Control-Key-z>
+find=<Control-Key-u><Control-Key-u><Control-Key-s>
+find-again=<Control-Key-u><Control-Key-s>
+find-in-files=<Alt-Key-s> <Meta-Key-s>
+find-selection=<Control-Key-s>
+replace=<Control-Key-r>
+goto-line=<Alt-Key-g> <Meta-Key-g>
+smart-backspace=<Key-BackSpace>
+newline-and-indent=<Key-Return> <Key-KP_Enter>
+smart-indent=<Key-Tab>
+indent-region=<Control-Key-bracketright>
+dedent-region=<Control-Key-bracketleft>
+comment-region=<Alt-Key-3>
+uncomment-region=<Alt-Key-4>
+tabify-region=<Alt-Key-5>
+untabify-region=<Alt-Key-6>
+toggle-tabs=<Alt-Key-t>
+change-indentwidth=<Alt-Key-u>
+del-word-left=<Alt-Key-BackSpace>
+del-word-right=<Alt-Key-d>
+force-open-completions= <Control-Key-space>
+expand-word= <Alt-Key-slash>
+force-open-calltip= <Control-Key-backslash>
+format-paragraph= <Alt-Key-q>
+flash-paren= <Control-Key-0>
+run-module= <Key-F5>
+run-custom= <Shift-Key-F5>
+check-module= <Alt-Key-x>
+zoom-height= <Alt-Key-2>
+
+[IDLE Modern Unix]
+copy = <Control-Shift-Key-C> <Control-Key-Insert>
+cut = <Control-Key-x> <Shift-Key-Delete>
+paste = <Control-Key-v> <Shift-Key-Insert>
+beginning-of-line = <Key-Home>
+center-insert = <Control-Key-l>
+close-all-windows = <Control-Key-q>
+close-window = <Control-Key-w> <Control-Shift-Key-W>
+do-nothing = <Control-Key-F12>
+end-of-file = <Control-Key-d>
+history-next = <Alt-Key-n> <Meta-Key-n>
+history-previous = <Alt-Key-p> <Meta-Key-p>
+interrupt-execution = <Control-Key-c>
+view-restart = <Key-F6>
+restart-shell = <Control-Key-F6>
+open-class-browser = <Control-Key-b>
+open-module = <Control-Key-m>
+open-new-window = <Control-Key-n>
+open-window-from-file = <Control-Key-o>
+plain-newline-and-indent = <Control-Key-j>
+print-window = <Control-Key-p>
+python-context-help = <Shift-Key-F1>
+python-docs = <Key-F1>
+redo = <Control-Shift-Key-Z>
+remove-selection = <Key-Escape>
+save-copy-of-window-as-file = <Alt-Shift-Key-S>
+save-window-as-file = <Control-Shift-Key-S>
+save-window = <Control-Key-s>
+select-all = <Control-Key-a>
+toggle-auto-coloring = <Control-Key-slash>
+undo = <Control-Key-z>
+find = <Control-Key-f>
+find-again = <Key-F3>
+find-in-files = <Control-Shift-Key-f>
+find-selection = <Control-Key-h>
+replace = <Control-Key-r>
+goto-line = <Control-Key-g>
+smart-backspace = <Key-BackSpace>
+newline-and-indent = <Key-Return> <Key-KP_Enter>
+smart-indent = <Key-Tab>
+indent-region = <Control-Key-bracketright>
+dedent-region = <Control-Key-bracketleft>
+comment-region = <Control-Key-d>
+uncomment-region = <Control-Shift-Key-D>
+tabify-region = <Alt-Key-5>
+untabify-region = <Alt-Key-6>
+toggle-tabs = <Control-Key-T>
+change-indentwidth = <Alt-Key-u>
+del-word-left = <Control-Key-BackSpace>
+del-word-right = <Control-Key-Delete>
+force-open-completions= <Control-Key-space>
+expand-word= <Alt-Key-slash>
+force-open-calltip= <Control-Key-backslash>
+format-paragraph= <Alt-Key-q>
+flash-paren= <Control-Key-0>
+run-module= <Key-F5>
+run-custom= <Shift-Key-F5>
+check-module= <Alt-Key-x>
+zoom-height= <Alt-Key-2>
+
+[IDLE Classic Mac]
+copy=<Command-Key-c>
+cut=<Command-Key-x>
+paste=<Command-Key-v>
+beginning-of-line= <Key-Home>
+center-insert=<Control-Key-l>
+close-all-windows=<Command-Key-q>
+close-window=<Command-Key-w>
+do-nothing=<Control-Key-F12>
+end-of-file=<Control-Key-d>
+python-docs=<Key-F1>
+python-context-help=<Shift-Key-F1>
+history-next=<Control-Key-n>
+history-previous=<Control-Key-p>
+interrupt-execution=<Control-Key-c>
+view-restart=<Key-F6>
+restart-shell=<Control-Key-F6>
+open-class-browser=<Command-Key-b>
+open-module=<Command-Key-m>
+open-new-window=<Command-Key-n>
+open-window-from-file=<Command-Key-o>
+plain-newline-and-indent=<Control-Key-j>
+print-window=<Command-Key-p>
+redo=<Shift-Command-Key-Z>
+remove-selection=<Key-Escape>
+save-window-as-file=<Shift-Command-Key-S>
+save-window=<Command-Key-s>
+save-copy-of-window-as-file=<Option-Command-Key-s>
+select-all=<Command-Key-a>
+toggle-auto-coloring=<Control-Key-slash>
+undo=<Command-Key-z>
+find=<Command-Key-f>
+find-again=<Command-Key-g> <Key-F3>
+find-in-files=<Command-Key-F3>
+find-selection=<Shift-Command-Key-F3>
+replace=<Command-Key-r>
+goto-line=<Command-Key-j>
+smart-backspace=<Key-BackSpace>
+newline-and-indent=<Key-Return> <Key-KP_Enter>
+smart-indent=<Key-Tab>
+indent-region=<Command-Key-bracketright>
+dedent-region=<Command-Key-bracketleft>
+comment-region=<Control-Key-3>
+uncomment-region=<Control-Key-4>
+tabify-region=<Control-Key-5>
+untabify-region=<Control-Key-6>
+toggle-tabs=<Control-Key-t>
+change-indentwidth=<Control-Key-u>
+del-word-left=<Control-Key-BackSpace>
+del-word-right=<Control-Key-Delete>
+force-open-completions= <Control-Key-space>
+expand-word= <Option-Key-slash>
+force-open-calltip= <Control-Key-backslash>
+format-paragraph= <Option-Key-q>
+flash-paren= <Control-Key-0>
+run-module= <Key-F5>
+run-custom= <Shift-Key-F5>
+check-module= <Option-Key-x>
+zoom-height= <Option-Key-0>
+
+[IDLE Classic OSX]
+toggle-tabs = <Control-Key-t>
+interrupt-execution = <Control-Key-c>
+untabify-region = <Control-Key-6>
+remove-selection = <Key-Escape>
+print-window = <Command-Key-p>
+replace = <Command-Key-r>
+goto-line = <Command-Key-j>
+plain-newline-and-indent = <Control-Key-j>
+history-previous = <Control-Key-p>
+beginning-of-line = <Control-Key-Left>
+end-of-line = <Control-Key-Right>
+comment-region = <Control-Key-3>
+redo = <Shift-Command-Key-Z>
+close-window = <Command-Key-w>
+restart-shell = <Control-Key-F6>
+save-window-as-file = <Shift-Command-Key-S>
+close-all-windows = <Command-Key-q>
+view-restart = <Key-F6>
+tabify-region = <Control-Key-5>
+find-again = <Command-Key-g> <Key-F3>
+find = <Command-Key-f>
+toggle-auto-coloring = <Control-Key-slash>
+select-all = <Command-Key-a>
+smart-backspace = <Key-BackSpace>
+change-indentwidth = <Control-Key-u>
+do-nothing = <Control-Key-F12>
+smart-indent = <Key-Tab>
+center-insert = <Control-Key-l>
+history-next = <Control-Key-n>
+del-word-right = <Option-Key-Delete>
+undo = <Command-Key-z>
+save-window = <Command-Key-s>
+uncomment-region = <Control-Key-4>
+cut = <Command-Key-x>
+find-in-files = <Command-Key-F3>
+dedent-region = <Command-Key-bracketleft>
+copy = <Command-Key-c>
+paste = <Command-Key-v>
+indent-region = <Command-Key-bracketright>
+del-word-left = <Option-Key-BackSpace> <Option-Command-Key-BackSpace>
+newline-and-indent = <Key-Return> <Key-KP_Enter>
+end-of-file = <Control-Key-d>
+open-class-browser = <Command-Key-b>
+open-new-window = <Command-Key-n>
+open-module = <Command-Key-m>
+find-selection = <Shift-Command-Key-F3>
+python-context-help = <Shift-Key-F1>
+save-copy-of-window-as-file = <Option-Command-Key-s>
+open-window-from-file = <Command-Key-o>
+python-docs = <Key-F1>
+force-open-completions= <Control-Key-space>
+expand-word= <Option-Key-slash>
+force-open-calltip= <Control-Key-backslash>
+format-paragraph= <Option-Key-q>
+flash-paren= <Control-Key-0>
+run-module= <Key-F5>
+run-custom= <Shift-Key-F5>
+check-module= <Option-Key-x>
+zoom-height= <Option-Key-0>

Lib/idlelib/config-main.def

@@ -0,0 +1,92 @@
+# IDLE reads several config files to determine user preferences.  This
+# file is the default config file for general idle settings.
+#
+# When IDLE starts, it will look in
+# the following two sets of files, in order:
+#
+#     default configuration files in idlelib
+#     --------------------------------------
+#     config-main.def         default general config file
+#     config-extensions.def   default extension config file
+#     config-highlight.def    default highlighting config file
+#     config-keys.def         default keybinding config file
+#
+#     user configuration files in ~/.idlerc
+#     -------------------------------------
+#     config-main.cfg         user general config file
+#     config-extensions.cfg   user extension config file
+#     config-highlight.cfg    user highlighting config file
+#     config-keys.cfg         user keybinding config file
+#
+# On Windows, the default location of the home directory ('~' above)
+# depends on the version.  For Windows 10, it is C:\Users\<username>.
+#
+# Any options the user saves through the config dialog will be saved to
+# the relevant user config file. Reverting any general or extension
+# setting to the default causes that entry to be wiped from the user
+# file and re-read from the default file.  This rule applies to each
+# item, except that the three editor font items are saved as a group.
+#
+# User highlighting themes and keybinding sets must have (section) names
+# distinct from the default names.  All items are added and saved as a
+# group. They are retained unless specifically deleted within the config
+# dialog. Choosing one of the default themes or keysets just applies the
+# relevant settings from the default file.
+#
+# Additional help sources are listed in the [HelpFiles] section below
+# and should be viewable by a web browser. These sources will be listed
+# on the Help menu.  The pattern, and two examples, are:
+#
+# <sequence_number = menu item;/path/to/help/source>
+# 1 = IDLE;C:/Programs/Python36/Lib/idlelib/help.html
+# 2 = Pillow;https://pillow.readthedocs.io/en/latest/
+#
+# You can't use a semi-colon in a menu item or path.  The path will be
+# platform specific because of path separators, drive specs etc.
+#
+# The default files should not be edited except to add new sections to
+# config-extensions.def for added extensions.  The user files should be
+# modified through the Settings dialog.
+
+[General]
+editor-on-startup= 0
+autosave= 0
+print-command-posix=lpr %%s
+print-command-win=start /min notepad /p %%s
+delete-exitfunc= 1
+
+[EditorWindow]
+width= 80
+height= 40
+cursor-blink= 1
+font= TkFixedFont
+# For TkFixedFont, the actual size and boldness are obtained from tk
+# and override 10 and 0.  See idlelib.config.IdleConf.GetFont
+font-size= 10
+font-bold= 0
+encoding= none
+line-numbers-default= 0
+
+[PyShell]
+auto-squeeze-min-lines= 50
+
+[Indent]
+use-spaces= 1
+num-spaces= 4
+
+[Theme]
+default= 1
+name= IDLE Classic
+name2=
+# name2 set in user config-main.cfg for themes added after 2015 Oct 1
+
+[Keys]
+default= 1
+name=
+name2=
+# name2 set in user config-main.cfg for keys added after 2016 July 1
+
+[History]
+cyclic=1
+
+[HelpFiles]

Lib/idlelib/config.py

@@ -0,0 +1,917 @@
+"""idlelib.config -- Manage IDLE configuration information.
+
+The comments at the beginning of config-main.def describe the
+configuration files and the design implemented to update user
+configuration information.  In particular, user configuration choices
+which duplicate the defaults will be removed from the user's
+configuration files, and if a user file becomes empty, it will be
+deleted.
+
+The configuration database maps options to values.  Conceptually, the
+database keys are tuples (config-type, section, item).  As implemented,
+there are  separate dicts for default and user values.  Each has
+config-type keys 'main', 'extensions', 'highlight', and 'keys'.  The
+value for each key is a ConfigParser instance that maps section and item
+to values.  For 'main' and 'extensions', user values override
+default values.  For 'highlight' and 'keys', user sections augment the
+default sections (and must, therefore, have distinct names).
+
+Throughout this module there is an emphasis on returning usable defaults
+when a problem occurs in returning a requested configuration value back to
+idle. This is to allow IDLE to continue to function in spite of errors in
+the retrieval of config information. When a default is returned instead of
+a requested config value, a message is printed to stderr to aid in
+configuration problem notification and resolution.
+"""
+# TODOs added Oct 2014, tjr
+
+from configparser import ConfigParser
+import os
+import sys
+
+from tkinter.font import Font
+import idlelib
+
+class InvalidConfigType(Exception): pass
+class InvalidConfigSet(Exception): pass
+class InvalidTheme(Exception): pass
+
+class IdleConfParser(ConfigParser):
+    """
+    A ConfigParser specialised for idle configuration file handling
+    """
+    def __init__(self, cfgFile, cfgDefaults=None):
+        """
+        cfgFile - string, fully specified configuration file name
+        """
+        self.file = cfgFile  # This is currently '' when testing.
+        ConfigParser.__init__(self, defaults=cfgDefaults, strict=False)
+
+    def Get(self, section, option, type=None, default=None, raw=False):
+        """
+        Get an option value for given section/option or return default.
+        If type is specified, return as type.
+        """
+        # TODO Use default as fallback, at least if not None
+        # Should also print Warning(file, section, option).
+        # Currently may raise ValueError
+        if not self.has_option(section, option):
+            return default
+        if type == 'bool':
+            return self.getboolean(section, option)
+        elif type == 'int':
+            return self.getint(section, option)
+        else:
+            return self.get(section, option, raw=raw)
+
+    def GetOptionList(self, section):
+        "Return a list of options for given section, else []."
+        if self.has_section(section):
+            return self.options(section)
+        else:  #return a default value
+            return []
+
+    def Load(self):
+        "Load the configuration file from disk."
+        if self.file:
+            self.read(self.file)
+
+class IdleUserConfParser(IdleConfParser):
+    """
+    IdleConfigParser specialised for user configuration handling.
+    """
+
+    def SetOption(self, section, option, value):
+        """Return True if option is added or changed to value, else False.
+
+        Add section if required.  False means option already had value.
+        """
+        if self.has_option(section, option):
+            if self.get(section, option) == value:
+                return False
+            else:
+                self.set(section, option, value)
+                return True
+        else:
+            if not self.has_section(section):
+                self.add_section(section)
+            self.set(section, option, value)
+            return True
+
+    def RemoveOption(self, section, option):
+        """Return True if option is removed from section, else False.
+
+        False if either section does not exist or did not have option.
+        """
+        if self.has_section(section):
+            return self.remove_option(section, option)
+        return False
+
+    def AddSection(self, section):
+        "If section doesn't exist, add it."
+        if not self.has_section(section):
+            self.add_section(section)
+
+    def RemoveEmptySections(self):
+        "Remove any sections that have no options."
+        for section in self.sections():
+            if not self.GetOptionList(section):
+                self.remove_section(section)
+
+    def IsEmpty(self):
+        "Return True if no sections after removing empty sections."
+        self.RemoveEmptySections()
+        return not self.sections()
+
+    def Save(self):
+        """Update user configuration file.
+
+        If self not empty after removing empty sections, write the file
+        to disk. Otherwise, remove the file from disk if it exists.
+        """
+        fname = self.file
+        if fname and fname[0] != '#':
+            if not self.IsEmpty():
+                try:
+                    cfgFile = open(fname, 'w')
+                except OSError:
+                    os.unlink(fname)
+                    cfgFile = open(fname, 'w')
+                with cfgFile:
+                    self.write(cfgFile)
+            elif os.path.exists(self.file):
+                os.remove(self.file)
+
+class IdleConf:
+    """Hold config parsers for all idle config files in singleton instance.
+
+    Default config files, self.defaultCfg --
+        for config_type in self.config_types:
+            (idle install dir)/config-{config-type}.def
+
+    User config files, self.userCfg --
+        for config_type in self.config_types:
+        (user home dir)/.idlerc/config-{config-type}.cfg
+    """
+    def __init__(self, _utest=False):
+        self.config_types = ('main', 'highlight', 'keys', 'extensions')
+        self.defaultCfg = {}
+        self.userCfg = {}
+        self.cfg = {}  # TODO use to select userCfg vs defaultCfg
+
+        # See https://bugs.python.org/issue4630#msg356516 for following.
+        # self.blink_off_time = <first editor text>['insertofftime']
+
+        if not _utest:
+            self.CreateConfigHandlers()
+            self.LoadCfgFiles()
+
+    def CreateConfigHandlers(self):
+        "Populate default and user config parser dictionaries."
+        idledir = os.path.dirname(__file__)
+        self.userdir = userdir = '' if idlelib.testing else self.GetUserCfgDir()
+        for cfg_type in self.config_types:
+            self.defaultCfg[cfg_type] = IdleConfParser(
+                os.path.join(idledir, f'config-{cfg_type}.def'))
+            self.userCfg[cfg_type] = IdleUserConfParser(
+                os.path.join(userdir or '#', f'config-{cfg_type}.cfg'))
+
+    def GetUserCfgDir(self):
+        """Return a filesystem directory for storing user config files.
+
+        Creates it if required.
+        """
+        cfgDir = '.idlerc'
+        userDir = os.path.expanduser('~')
+        if userDir != '~': # expanduser() found user home dir
+            if not os.path.exists(userDir):
+                if not idlelib.testing:
+                    warn = ('\n Warning: os.path.expanduser("~") points to\n ' +
+                            userDir + ',\n but the path does not exist.')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
+                userDir = '~'
+        if userDir == "~": # still no path to home!
+            # traditionally IDLE has defaulted to os.getcwd(), is this adequate?
+            userDir = os.getcwd()
+        userDir = os.path.join(userDir, cfgDir)
+        if not os.path.exists(userDir):
+            try:
+                os.mkdir(userDir)
+            except OSError:
+                if not idlelib.testing:
+                    warn = ('\n Warning: unable to create user config directory\n' +
+                            userDir + '\n Check path and permissions.\n Exiting!\n')
+                    try:
+                        print(warn, file=sys.stderr)
+                    except OSError:
+                        pass
+                raise SystemExit
+        # TODO continue without userDIr instead of exit
+        return userDir
+
+    def GetOption(self, configType, section, option, default=None, type=None,
+                  warn_on_default=True, raw=False):
+        """Return a value for configType section option, or default.
+
+        If type is not None, return a value of that type.  Also pass raw
+        to the config parser.  First try to return a valid value
+        (including type) from a user configuration. If that fails, try
+        the default configuration. If that fails, return default, with a
+        default of None.
+
+        Warn if either user or default configurations have an invalid value.
+        Warn if default is returned and warn_on_default is True.
+        """
+        try:
+            if self.userCfg[configType].has_option(section, option):
+                return self.userCfg[configType].Get(section, option,
+                                                    type=type, raw=raw)
+        except ValueError:
+            warning = ('\n Warning: config.py - IdleConf.GetOption -\n'
+                       ' invalid %r value for configuration option %r\n'
+                       ' from section %r: %r' %
+                       (type, option, section,
+                       self.userCfg[configType].Get(section, option, raw=raw)))
+            _warn(warning, configType, section, option)
+        try:
+            if self.defaultCfg[configType].has_option(section,option):
+                return self.defaultCfg[configType].Get(
+                        section, option, type=type, raw=raw)
+        except ValueError:
+            pass
+        #returning default, print warning
+        if warn_on_default:
+            warning = ('\n Warning: config.py - IdleConf.GetOption -\n'
+                       ' problem retrieving configuration option %r\n'
+                       ' from section %r.\n'
+                       ' returning default value: %r' %
+                       (option, section, default))
+            _warn(warning, configType, section, option)
+        return default
+
+    def SetOption(self, configType, section, option, value):
+        """Set section option to value in user config file."""
+        self.userCfg[configType].SetOption(section, option, value)
+
+    def GetSectionList(self, configSet, configType):
+        """Return sections for configSet configType configuration.
+
+        configSet must be either 'user' or 'default'
+        configType must be in self.config_types.
+        """
+        if not (configType in self.config_types):
+            raise InvalidConfigType('Invalid configType specified')
+        if configSet == 'user':
+            cfgParser = self.userCfg[configType]
+        elif configSet == 'default':
+            cfgParser=self.defaultCfg[configType]
+        else:
+            raise InvalidConfigSet('Invalid configSet specified')
+        return cfgParser.sections()
+
+    def GetHighlight(self, theme, element):
+        """Return dict of theme element highlight colors.
+
+        The keys are 'foreground' and 'background'.  The values are
+        tkinter color strings for configuring backgrounds and tags.
+        """
+        cfg = ('default' if self.defaultCfg['highlight'].has_section(theme)
+               else 'user')
+        theme_dict = self.GetThemeDict(cfg, theme)
+        fore = theme_dict[element + '-foreground']
+        if element == 'cursor':
+            element = 'normal'
+        back = theme_dict[element + '-background']
+        return {"foreground": fore, "background": back}
+
+    def GetThemeDict(self, type, themeName):
+        """Return {option:value} dict for elements in themeName.
+
+        type - string, 'default' or 'user' theme type
+        themeName - string, theme name
+        Values are loaded over ultimate fallback defaults to guarantee
+        that all theme elements are present in a newly created theme.
+        """
+        if type == 'user':
+            cfgParser = self.userCfg['highlight']
+        elif type == 'default':
+            cfgParser = self.defaultCfg['highlight']
+        else:
+            raise InvalidTheme('Invalid theme type specified')
+        # Provide foreground and background colors for each theme
+        # element (other than cursor) even though some values are not
+        # yet used by idle, to allow for their use in the future.
+        # Default values are generally black and white.
+        # TODO copy theme from a class attribute.
+        theme ={'normal-foreground':'#000000',
+                'normal-background':'#ffffff',
+                'keyword-foreground':'#000000',
+                'keyword-background':'#ffffff',
+                'builtin-foreground':'#000000',
+                'builtin-background':'#ffffff',
+                'comment-foreground':'#000000',
+                'comment-background':'#ffffff',
+                'string-foreground':'#000000',
+                'string-background':'#ffffff',
+                'definition-foreground':'#000000',
+                'definition-background':'#ffffff',
+                'hilite-foreground':'#000000',
+                'hilite-background':'gray',
+                'break-foreground':'#ffffff',
+                'break-background':'#000000',
+                'hit-foreground':'#ffffff',
+                'hit-background':'#000000',
+                'error-foreground':'#ffffff',
+                'error-background':'#000000',
+                'context-foreground':'#000000',
+                'context-background':'#ffffff',
+                'linenumber-foreground':'#000000',
+                'linenumber-background':'#ffffff',
+                #cursor (only foreground can be set)
+                'cursor-foreground':'#000000',
+                #shell window
+                'stdout-foreground':'#000000',
+                'stdout-background':'#ffffff',
+                'stderr-foreground':'#000000',
+                'stderr-background':'#ffffff',
+                'console-foreground':'#000000',
+                'console-background':'#ffffff',
+                }
+        for element in theme:
+            if not (cfgParser.has_option(themeName, element) or
+                    # Skip warning for new elements.
+                    element.startswith(('context-', 'linenumber-'))):
+                # Print warning that will return a default color
+                warning = ('\n Warning: config.IdleConf.GetThemeDict'
+                           ' -\n problem retrieving theme element %r'
+                           '\n from theme %r.\n'
+                           ' returning default color: %r' %
+                           (element, themeName, theme[element]))
+                _warn(warning, 'highlight', themeName, element)
+            theme[element] = cfgParser.Get(
+                    themeName, element, default=theme[element])
+        return theme
+
+    def CurrentTheme(self):
+        "Return the name of the currently active text color theme."
+        return self.current_colors_and_keys('Theme')
+
+    def CurrentKeys(self):
+        """Return the name of the currently active key set."""
+        return self.current_colors_and_keys('Keys')
+
+    def current_colors_and_keys(self, section):
+        """Return the currently active name for Theme or Keys section.
+
+        idlelib.config-main.def ('default') includes these sections
+
+        [Theme]
+        default= 1
+        name= IDLE Classic
+        name2=
+
+        [Keys]
+        default= 1
+        name=
+        name2=
+
+        Item 'name2', is used for built-in ('default') themes and keys
+        added after 2015 Oct 1 and 2016 July 1.  This kludge is needed
+        because setting 'name' to a builtin not defined in older IDLEs
+        to display multiple error messages or quit.
+        See https://bugs.python.org/issue25313.
+        When default = True, 'name2' takes precedence over 'name',
+        while older IDLEs will just use name.  When default = False,
+        'name2' may still be set, but it is ignored.
+        """
+        cfgname = 'highlight' if section == 'Theme' else 'keys'
+        default = self.GetOption('main', section, 'default',
+                                 type='bool', default=True)
+        name = ''
+        if default:
+            name = self.GetOption('main', section, 'name2', default='')
+        if not name:
+            name = self.GetOption('main', section, 'name', default='')
+        if name:
+            source = self.defaultCfg if default else self.userCfg
+            if source[cfgname].has_section(name):
+                return name
+        return "IDLE Classic" if section == 'Theme' else self.default_keys()
+
+    @staticmethod
+    def default_keys():
+        if sys.platform[:3] == 'win':
+            return 'IDLE Classic Windows'
+        elif sys.platform == 'darwin':
+            return 'IDLE Classic OSX'
+        else:
+            return 'IDLE Modern Unix'
+
+    def GetExtensions(self, active_only=True,
+                      editor_only=False, shell_only=False):
+        """Return extensions in default and user config-extensions files.
+
+        If active_only True, only return active (enabled) extensions
+        and optionally only editor or shell extensions.
+        If active_only False, return all extensions.
+        """
+        extns = self.RemoveKeyBindNames(
+                self.GetSectionList('default', 'extensions'))
+        userExtns = self.RemoveKeyBindNames(
+                self.GetSectionList('user', 'extensions'))
+        for extn in userExtns:
+            if extn not in extns: #user has added own extension
+                extns.append(extn)
+        for extn in ('AutoComplete','CodeContext',
+                     'FormatParagraph','ParenMatch'):
+            extns.remove(extn)
+            # specific exclusions because we are storing config for mainlined old
+            # extensions in config-extensions.def for backward compatibility
+        if active_only:
+            activeExtns = []
+            for extn in extns:
+                if self.GetOption('extensions', extn, 'enable', default=True,
+                                  type='bool'):
+                    #the extension is enabled
+                    if editor_only or shell_only:  # TODO both True contradict
+                        if editor_only:
+                            option = "enable_editor"
+                        else:
+                            option = "enable_shell"
+                        if self.GetOption('extensions', extn,option,
+                                          default=True, type='bool',
+                                          warn_on_default=False):
+                            activeExtns.append(extn)
+                    else:
+                        activeExtns.append(extn)
+            return activeExtns
+        else:
+            return extns
+
+    def RemoveKeyBindNames(self, extnNameList):
+        "Return extnNameList with keybinding section names removed."
+        return [n for n in extnNameList if not n.endswith(('_bindings', '_cfgBindings'))]
+
+    def GetExtnNameForEvent(self, virtualEvent):
+        """Return the name of the extension binding virtualEvent, or None.
+
+        virtualEvent - string, name of the virtual event to test for,
+                       without the enclosing '<< >>'
+        """
+        extName = None
+        vEvent = '<<' + virtualEvent + '>>'
+        for extn in self.GetExtensions(active_only=0):
+            for event in self.GetExtensionKeys(extn):
+                if event == vEvent:
+                    extName = extn  # TODO return here?
+        return extName
+
+    def GetExtensionKeys(self, extensionName):
+        """Return dict: {configurable extensionName event : active keybinding}.
+
+        Events come from default config extension_cfgBindings section.
+        Keybindings come from GetCurrentKeySet() active key dict,
+        where previously used bindings are disabled.
+        """
+        keysName = extensionName + '_cfgBindings'
+        activeKeys = self.GetCurrentKeySet()
+        extKeys = {}
+        if self.defaultCfg['extensions'].has_section(keysName):
+            eventNames = self.defaultCfg['extensions'].GetOptionList(keysName)
+            for eventName in eventNames:
+                event = '<<' + eventName + '>>'
+                binding = activeKeys[event]
+                extKeys[event] = binding
+        return extKeys
+
+    def __GetRawExtensionKeys(self,extensionName):
+        """Return dict {configurable extensionName event : keybinding list}.
+
+        Events come from default config extension_cfgBindings section.
+        Keybindings list come from the splitting of GetOption, which
+        tries user config before default config.
+        """
+        keysName = extensionName+'_cfgBindings'
+        extKeys = {}
+        if self.defaultCfg['extensions'].has_section(keysName):
+            eventNames = self.defaultCfg['extensions'].GetOptionList(keysName)
+            for eventName in eventNames:
+                binding = self.GetOption(
+                        'extensions', keysName, eventName, default='').split()
+                event = '<<' + eventName + '>>'
+                extKeys[event] = binding
+        return extKeys
+
+    def GetExtensionBindings(self, extensionName):
+        """Return dict {extensionName event : active or defined keybinding}.
+
+        Augment self.GetExtensionKeys(extensionName) with mapping of non-
+        configurable events (from default config) to GetOption splits,
+        as in self.__GetRawExtensionKeys.
+        """
+        bindsName = extensionName + '_bindings'
+        extBinds = self.GetExtensionKeys(extensionName)
+        #add the non-configurable bindings
+        if self.defaultCfg['extensions'].has_section(bindsName):
+            eventNames = self.defaultCfg['extensions'].GetOptionList(bindsName)
+            for eventName in eventNames:
+                binding = self.GetOption(
+                        'extensions', bindsName, eventName, default='').split()
+                event = '<<' + eventName + '>>'
+                extBinds[event] = binding
+
+        return extBinds
+
+    def GetKeyBinding(self, keySetName, eventStr):
+        """Return the keybinding list for keySetName eventStr.
+
+        keySetName - name of key binding set (config-keys section).
+        eventStr - virtual event, including brackets, as in '<<event>>'.
+        """
+        eventName = eventStr[2:-2] #trim off the angle brackets
+        binding = self.GetOption('keys', keySetName, eventName, default='',
+                                 warn_on_default=False).split()
+        return binding
+
+    def GetCurrentKeySet(self):
+        "Return CurrentKeys with 'darwin' modifications."
+        result = self.GetKeySet(self.CurrentKeys())
+
+        if sys.platform == "darwin":
+            # macOS (OS X) Tk variants do not support the "Alt"
+            # keyboard modifier.  Replace it with "Option".
+            # TODO (Ned?): the "Option" modifier does not work properly
+            #     for Cocoa Tk and XQuartz Tk so we should not use it
+            #     in the default 'OSX' keyset.
+            for k, v in result.items():
+                v2 = [ x.replace('<Alt-', '<Option-') for x in v ]
+                if v != v2:
+                    result[k] = v2
+
+        return result
+
+    def GetKeySet(self, keySetName):
+        """Return event-key dict for keySetName core plus active extensions.
+
+        If a binding defined in an extension is already in use, the
+        extension binding is disabled by being set to ''
+        """
+        keySet = self.GetCoreKeys(keySetName)
+        activeExtns = self.GetExtensions(active_only=1)
+        for extn in activeExtns:
+            extKeys = self.__GetRawExtensionKeys(extn)
+            if extKeys: #the extension defines keybindings
+                for event in extKeys:
+                    if extKeys[event] in keySet.values():
+                        #the binding is already in use
+                        extKeys[event] = '' #disable this binding
+                    keySet[event] = extKeys[event] #add binding
+        return keySet
+
+    def IsCoreBinding(self, virtualEvent):
+        """Return True if the virtual event is one of the core idle key events.
+
+        virtualEvent - string, name of the virtual event to test for,
+                       without the enclosing '<< >>'
+        """
+        return ('<<'+virtualEvent+'>>') in self.GetCoreKeys()
+
+# TODO make keyBindings a file or class attribute used for test above
+# and copied in function below.
+
+    former_extension_events = {  #  Those with user-configurable keys.
+        '<<force-open-completions>>', '<<expand-word>>',
+        '<<force-open-calltip>>', '<<flash-paren>>', '<<format-paragraph>>',
+         '<<run-module>>', '<<check-module>>', '<<zoom-height>>',
+         '<<run-custom>>',
+         }
+
+    def GetCoreKeys(self, keySetName=None):
+        """Return dict of core virtual-key keybindings for keySetName.
+
+        The default keySetName None corresponds to the keyBindings base
+        dict. If keySetName is not None, bindings from the config
+        file(s) are loaded _over_ these defaults, so if there is a
+        problem getting any core binding there will be an 'ultimate last
+        resort fallback' to the CUA-ish bindings defined here.
+        """
+        # TODO: = dict(sorted([(v-event, keys), ...]))?
+        keyBindings={
+            # virtual-event: list of key events.
+            '<<copy>>': ['<Control-c>', '<Control-C>'],
+            '<<cut>>': ['<Control-x>', '<Control-X>'],
+            '<<paste>>': ['<Control-v>', '<Control-V>'],
+            '<<beginning-of-line>>': ['<Control-a>', '<Home>'],
+            '<<center-insert>>': ['<Control-l>'],
+            '<<close-all-windows>>': ['<Control-q>'],
+            '<<close-window>>': ['<Alt-F4>'],
+            '<<do-nothing>>': ['<Control-x>'],
+            '<<end-of-file>>': ['<Control-d>'],
+            '<<python-docs>>': ['<F1>'],
+            '<<python-context-help>>': ['<Shift-F1>'],
+            '<<history-next>>': ['<Alt-n>'],
+            '<<history-previous>>': ['<Alt-p>'],
+            '<<interrupt-execution>>': ['<Control-c>'],
+            '<<view-restart>>': ['<F6>'],
+            '<<restart-shell>>': ['<Control-F6>'],
+            '<<open-class-browser>>': ['<Alt-c>'],
+            '<<open-module>>': ['<Alt-m>'],
+            '<<open-new-window>>': ['<Control-n>'],
+            '<<open-window-from-file>>': ['<Control-o>'],
+            '<<plain-newline-and-indent>>': ['<Control-j>'],
+            '<<print-window>>': ['<Control-p>'],
+            '<<redo>>': ['<Control-y>'],
+            '<<remove-selection>>': ['<Escape>'],
+            '<<save-copy-of-window-as-file>>': ['<Alt-Shift-S>'],
+            '<<save-window-as-file>>': ['<Alt-s>'],
+            '<<save-window>>': ['<Control-s>'],
+            '<<select-all>>': ['<Alt-a>'],
+            '<<toggle-auto-coloring>>': ['<Control-slash>'],
+            '<<undo>>': ['<Control-z>'],
+            '<<find-again>>': ['<Control-g>', '<F3>'],
+            '<<find-in-files>>': ['<Alt-F3>'],
+            '<<find-selection>>': ['<Control-F3>'],
+            '<<find>>': ['<Control-f>'],
+            '<<replace>>': ['<Control-h>'],
+            '<<goto-line>>': ['<Alt-g>'],
+            '<<smart-backspace>>': ['<Key-BackSpace>'],
+            '<<newline-and-indent>>': ['<Key-Return>', '<Key-KP_Enter>'],
+            '<<smart-indent>>': ['<Key-Tab>'],
+            '<<indent-region>>': ['<Control-Key-bracketright>'],
+            '<<dedent-region>>': ['<Control-Key-bracketleft>'],
+            '<<comment-region>>': ['<Alt-Key-3>'],
+            '<<uncomment-region>>': ['<Alt-Key-4>'],
+            '<<tabify-region>>': ['<Alt-Key-5>'],
+            '<<untabify-region>>': ['<Alt-Key-6>'],
+            '<<toggle-tabs>>': ['<Alt-Key-t>'],
+            '<<change-indentwidth>>': ['<Alt-Key-u>'],
+            '<<del-word-left>>': ['<Control-Key-BackSpace>'],
+            '<<del-word-right>>': ['<Control-Key-Delete>'],
+            '<<force-open-completions>>': ['<Control-Key-space>'],
+            '<<expand-word>>': ['<Alt-Key-slash>'],
+            '<<force-open-calltip>>': ['<Control-Key-backslash>'],
+            '<<flash-paren>>': ['<Control-Key-0>'],
+            '<<format-paragraph>>': ['<Alt-Key-q>'],
+            '<<run-module>>': ['<Key-F5>'],
+            '<<run-custom>>': ['<Shift-Key-F5>'],
+            '<<check-module>>': ['<Alt-Key-x>'],
+            '<<zoom-height>>': ['<Alt-Key-2>'],
+            }
+
+        if keySetName:
+            if not (self.userCfg['keys'].has_section(keySetName) or
+                    self.defaultCfg['keys'].has_section(keySetName)):
+                warning = (
+                    '\n Warning: config.py - IdleConf.GetCoreKeys -\n'
+                    ' key set %r is not defined, using default bindings.' %
+                    (keySetName,)
+                )
+                _warn(warning, 'keys', keySetName)
+            else:
+                for event in keyBindings:
+                    binding = self.GetKeyBinding(keySetName, event)
+                    if binding:
+                        keyBindings[event] = binding
+                    # Otherwise return default in keyBindings.
+                    elif event not in self.former_extension_events:
+                        warning = (
+                            '\n Warning: config.py - IdleConf.GetCoreKeys -\n'
+                            ' problem retrieving key binding for event %r\n'
+                            ' from key set %r.\n'
+                            ' returning default value: %r' %
+                            (event, keySetName, keyBindings[event])
+                        )
+                        _warn(warning, 'keys', keySetName, event)
+        return keyBindings
+
+    def GetExtraHelpSourceList(self, configSet):
+        """Return list of extra help sources from a given configSet.
+
+        Valid configSets are 'user' or 'default'.  Return a list of tuples of
+        the form (menu_item , path_to_help_file , option), or return the empty
+        list.  'option' is the sequence number of the help resource.  'option'
+        values determine the position of the menu items on the Help menu,
+        therefore the returned list must be sorted by 'option'.
+
+        """
+        helpSources = []
+        if configSet == 'user':
+            cfgParser = self.userCfg['main']
+        elif configSet == 'default':
+            cfgParser = self.defaultCfg['main']
+        else:
+            raise InvalidConfigSet('Invalid configSet specified')
+        options=cfgParser.GetOptionList('HelpFiles')
+        for option in options:
+            value=cfgParser.Get('HelpFiles', option, default=';')
+            if value.find(';') == -1: #malformed config entry with no ';'
+                menuItem = '' #make these empty
+                helpPath = '' #so value won't be added to list
+            else: #config entry contains ';' as expected
+                value=value.split(';')
+                menuItem=value[0].strip()
+                helpPath=value[1].strip()
+            if menuItem and helpPath: #neither are empty strings
+                helpSources.append( (menuItem,helpPath,option) )
+        helpSources.sort(key=lambda x: x[2])
+        return helpSources
+
+    def GetAllExtraHelpSourcesList(self):
+        """Return a list of the details of all additional help sources.
+
+        Tuples in the list are those of GetExtraHelpSourceList.
+        """
+        allHelpSources = (self.GetExtraHelpSourceList('default') +
+                self.GetExtraHelpSourceList('user') )
+        return allHelpSources
+
+    def GetFont(self, root, configType, section):
+        """Retrieve a font from configuration (font, font-size, font-bold)
+        Intercept the special value 'TkFixedFont' and substitute
+        the actual font, factoring in some tweaks if needed for
+        appearance sakes.
+
+        The 'root' parameter can normally be any valid Tkinter widget.
+
+        Return a tuple (family, size, weight) suitable for passing
+        to tkinter.Font
+        """
+        family = self.GetOption(configType, section, 'font', default='courier')
+        size = self.GetOption(configType, section, 'font-size', type='int',
+                              default='10')
+        bold = self.GetOption(configType, section, 'font-bold', default=0,
+                              type='bool')
+        if (family == 'TkFixedFont'):
+            f = Font(name='TkFixedFont', exists=True, root=root)
+            actualFont = Font.actual(f)
+            family = actualFont['family']
+            size = actualFont['size']
+            if size <= 0:
+                size = 10  # if font in pixels, ignore actual size
+            bold = actualFont['weight'] == 'bold'
+        return (family, size, 'bold' if bold else 'normal')
+
+    def LoadCfgFiles(self):
+        "Load all configuration files."
+        for key in self.defaultCfg:
+            self.defaultCfg[key].Load()
+            self.userCfg[key].Load() #same keys
+
+    def SaveUserCfgFiles(self):
+        "Write all loaded user configuration files to disk."
+        for key in self.userCfg:
+            self.userCfg[key].Save()
+
+
+idleConf = IdleConf()
+
+_warned = set()
+def _warn(msg, *key):
+    key = (msg,) + key
+    if key not in _warned:
+        try:
+            print(msg, file=sys.stderr)
+        except OSError:
+            pass
+        _warned.add(key)
+
+
+class ConfigChanges(dict):
+    """Manage a user's proposed configuration option changes.
+
+    Names used across multiple methods:
+        page -- one of the 4 top-level dicts representing a
+                .idlerc/config-x.cfg file.
+        config_type -- name of a page.
+        section -- a section within a page/file.
+        option -- name of an option within a section.
+        value -- value for the option.
+
+    Methods
+        add_option: Add option and value to changes.
+        save_option: Save option and value to config parser.
+        save_all: Save all the changes to the config parser and file.
+        delete_section: If section exists,
+                        delete from changes, userCfg, and file.
+        clear: Clear all changes by clearing each page.
+    """
+    def __init__(self):
+        "Create a page for each configuration file"
+        self.pages = []  # List of unhashable dicts.
+        for config_type in idleConf.config_types:
+            self[config_type] = {}
+            self.pages.append(self[config_type])
+
+    def add_option(self, config_type, section, item, value):
+        "Add item/value pair for config_type and section."
+        page = self[config_type]
+        value = str(value)  # Make sure we use a string.
+        if section not in page:
+            page[section] = {}
+        page[section][item] = value
+
+    @staticmethod
+    def save_option(config_type, section, item, value):
+        """Return True if the configuration value was added or changed.
+
+        Helper for save_all.
+        """
+        if idleConf.defaultCfg[config_type].has_option(section, item):
+            if idleConf.defaultCfg[config_type].Get(section, item) == value:
+                # The setting equals a default setting, remove it from user cfg.
+                return idleConf.userCfg[config_type].RemoveOption(section, item)
+        # If we got here, set the option.
+        return idleConf.userCfg[config_type].SetOption(section, item, value)
+
+    def save_all(self):
+        """Save configuration changes to the user config file.
+
+        Clear self in preparation for additional changes.
+        Return changed for testing.
+        """
+        idleConf.userCfg['main'].Save()
+
+        changed = False
+        for config_type in self:
+            cfg_type_changed = False
+            page = self[config_type]
+            for section in page:
+                if section == 'HelpFiles':  # Remove it for replacement.
+                    idleConf.userCfg['main'].remove_section('HelpFiles')
+                    cfg_type_changed = True
+                for item, value in page[section].items():
+                    if self.save_option(config_type, section, item, value):
+                        cfg_type_changed = True
+            if cfg_type_changed:
+                idleConf.userCfg[config_type].Save()
+                changed = True
+        for config_type in ['keys', 'highlight']:
+            # Save these even if unchanged!
+            idleConf.userCfg[config_type].Save()
+        self.clear()
+        # ConfigDialog caller must add the following call
+        # self.save_all_changed_extensions()  # Uses a different mechanism.
+        return changed
+
+    def delete_section(self, config_type, section):
+        """Delete a section from self, userCfg, and file.
+
+        Used to delete custom themes and keysets.
+        """
+        if section in self[config_type]:
+            del self[config_type][section]
+        configpage = idleConf.userCfg[config_type]
+        configpage.remove_section(section)
+        configpage.Save()
+
+    def clear(self):
+        """Clear all 4 pages.
+
+        Called in save_all after saving to idleConf.
+        XXX Mark window *title* when there are changes; unmark here.
+        """
+        for page in self.pages:
+            page.clear()
+
+
+# TODO Revise test output, write expanded unittest
+def _dump():  # htest # (not really, but ignore in coverage)
+    from zlib import crc32
+    line, crc = 0, 0
+
+    def sprint(obj):
+        nonlocal line, crc
+        txt = str(obj)
+        line += 1
+        crc = crc32(txt.encode(encoding='utf-8'), crc)
+        print(txt)
+        #print('***', line, crc, '***')  # Uncomment for diagnosis.
+
+    def dumpCfg(cfg):
+        print('\n', cfg, '\n')  # Cfg has variable '0xnnnnnnnn' address.
+        for key in sorted(cfg):
+            sections = cfg[key].sections()
+            sprint(key)
+            sprint(sections)
+            for section in sections:
+                options = cfg[key].options(section)
+                sprint(section)
+                sprint(options)
+                for option in options:
+                    sprint(option + ' = ' + cfg[key].Get(section, option))
+
+    dumpCfg(idleConf.defaultCfg)
+    dumpCfg(idleConf.userCfg)
+    print('\nlines = ', line, ', crc = ', crc, sep='')
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_config', verbosity=2, exit=False)
+
+    _dump()
+    # Run revised _dump() (700+ lines) as htest?  More sorting.
+    # Perhaps as window with tabs for textviews, making it config viewer.

Lib/idlelib/config_key.py

@@ -0,0 +1,354 @@
+"""
+Dialog for building Tkinter accelerator key bindings
+"""
+from tkinter import Toplevel, Listbox, StringVar, TclError
+from tkinter.ttk import Frame, Button, Checkbutton, Entry, Label, Scrollbar
+from tkinter import messagebox
+from tkinter.simpledialog import _setup_dialog
+import string
+import sys
+
+
+FUNCTION_KEYS = ('F1', 'F2' ,'F3' ,'F4' ,'F5' ,'F6',
+                 'F7', 'F8' ,'F9' ,'F10' ,'F11' ,'F12')
+ALPHANUM_KEYS = tuple(string.ascii_lowercase + string.digits)
+PUNCTUATION_KEYS = tuple('~!@#%^&*()_-+={}[]|;:,.<>/?')
+WHITESPACE_KEYS = ('Tab', 'Space', 'Return')
+EDIT_KEYS = ('BackSpace', 'Delete', 'Insert')
+MOVE_KEYS = ('Home', 'End', 'Page Up', 'Page Down', 'Left Arrow',
+             'Right Arrow', 'Up Arrow', 'Down Arrow')
+AVAILABLE_KEYS = (ALPHANUM_KEYS + PUNCTUATION_KEYS + FUNCTION_KEYS +
+                  WHITESPACE_KEYS + EDIT_KEYS + MOVE_KEYS)
+
+
+def translate_key(key, modifiers):
+    "Translate from keycap symbol to the Tkinter keysym."
+    mapping = {'Space':'space',
+            '~':'asciitilde', '!':'exclam', '@':'at', '#':'numbersign',
+            '%':'percent', '^':'asciicircum', '&':'ampersand',
+            '*':'asterisk', '(':'parenleft', ')':'parenright',
+            '_':'underscore', '-':'minus', '+':'plus', '=':'equal',
+            '{':'braceleft', '}':'braceright',
+            '[':'bracketleft', ']':'bracketright', '|':'bar',
+            ';':'semicolon', ':':'colon', ',':'comma', '.':'period',
+            '<':'less', '>':'greater', '/':'slash', '?':'question',
+            'Page Up':'Prior', 'Page Down':'Next',
+            'Left Arrow':'Left', 'Right Arrow':'Right',
+            'Up Arrow':'Up', 'Down Arrow': 'Down', 'Tab':'Tab'}
+    key = mapping.get(key, key)
+    if 'Shift' in modifiers and key in string.ascii_lowercase:
+        key = key.upper()
+    return f'Key-{key}'
+
+
+class GetKeysFrame(Frame):
+
+    # Dialog title for invalid key sequence
+    keyerror_title = 'Key Sequence Error'
+
+    def __init__(self, parent, action, current_key_sequences):
+        """
+        parent - parent of this dialog
+        action - the name of the virtual event these keys will be
+                 mapped to
+        current_key_sequences - a list of all key sequence lists
+                 currently mapped to virtual events, for overlap checking
+        """
+        super().__init__(parent)
+        self['borderwidth'] = 2
+        self['relief'] = 'sunken'
+        self.parent = parent
+        self.action = action
+        self.current_key_sequences = current_key_sequences
+        self.result = ''
+        self.key_string = StringVar(self)
+        self.key_string.set('')
+        # Set self.modifiers, self.modifier_label.
+        self.set_modifiers_for_platform()
+        self.modifier_vars = []
+        for modifier in self.modifiers:
+            variable = StringVar(self)
+            variable.set('')
+            self.modifier_vars.append(variable)
+        self.advanced = False
+        self.create_widgets()
+
+    def showerror(self, *args, **kwargs):
+        # Make testing easier.  Replace in #30751.
+        messagebox.showerror(*args, **kwargs)
+
+    def create_widgets(self):
+        # Basic entry key sequence.
+        self.frame_keyseq_basic = Frame(self, name='keyseq_basic')
+        self.frame_keyseq_basic.grid(row=0, column=0, sticky='nsew',
+                                      padx=5, pady=5)
+        basic_title = Label(self.frame_keyseq_basic,
+                            text=f"New keys for '{self.action}' :")
+        basic_title.pack(anchor='w')
+
+        basic_keys = Label(self.frame_keyseq_basic, justify='left',
+                           textvariable=self.key_string, relief='groove',
+                           borderwidth=2)
+        basic_keys.pack(ipadx=5, ipady=5, fill='x')
+
+        # Basic entry controls.
+        self.frame_controls_basic = Frame(self)
+        self.frame_controls_basic.grid(row=1, column=0, sticky='nsew', padx=5)
+
+        # Basic entry modifiers.
+        self.modifier_checkbuttons = {}
+        column = 0
+        for modifier, variable in zip(self.modifiers, self.modifier_vars):
+            label = self.modifier_label.get(modifier, modifier)
+            check = Checkbutton(self.frame_controls_basic,
+                                command=self.build_key_string, text=label,
+                                variable=variable, onvalue=modifier, offvalue='')
+            check.grid(row=0, column=column, padx=2, sticky='w')
+            self.modifier_checkbuttons[modifier] = check
+            column += 1
+
+        # Basic entry help text.
+        help_basic = Label(self.frame_controls_basic, justify='left',
+                           text="Select the desired modifier keys\n"+
+                                "above, and the final key from the\n"+
+                                "list on the right.\n\n" +
+                                "Use upper case Symbols when using\n" +
+                                "the Shift modifier.  (Letters will be\n" +
+                                "converted automatically.)")
+        help_basic.grid(row=1, column=0, columnspan=4, padx=2, sticky='w')
+
+        # Basic entry key list.
+        self.list_keys_final = Listbox(self.frame_controls_basic, width=15,
+                                       height=10, selectmode='single')
+        self.list_keys_final.insert('end', *AVAILABLE_KEYS)
+        self.list_keys_final.bind('<ButtonRelease-1>', self.final_key_selected)
+        self.list_keys_final.grid(row=0, column=4, rowspan=4, sticky='ns')
+        scroll_keys_final = Scrollbar(self.frame_controls_basic,
+                                      orient='vertical',
+                                      command=self.list_keys_final.yview)
+        self.list_keys_final.config(yscrollcommand=scroll_keys_final.set)
+        scroll_keys_final.grid(row=0, column=5, rowspan=4, sticky='ns')
+        self.button_clear = Button(self.frame_controls_basic,
+                                   text='Clear Keys',
+                                   command=self.clear_key_seq)
+        self.button_clear.grid(row=2, column=0, columnspan=4)
+
+        # Advanced entry key sequence.
+        self.frame_keyseq_advanced = Frame(self, name='keyseq_advanced')
+        self.frame_keyseq_advanced.grid(row=0, column=0, sticky='nsew',
+                                         padx=5, pady=5)
+        advanced_title = Label(self.frame_keyseq_advanced, justify='left',
+                               text=f"Enter new binding(s) for '{self.action}' :\n" +
+                                     "(These bindings will not be checked for validity!)")
+        advanced_title.pack(anchor='w')
+        self.advanced_keys = Entry(self.frame_keyseq_advanced,
+                                   textvariable=self.key_string)
+        self.advanced_keys.pack(fill='x')
+
+        # Advanced entry help text.
+        self.frame_help_advanced = Frame(self)
+        self.frame_help_advanced.grid(row=1, column=0, sticky='nsew', padx=5)
+        help_advanced = Label(self.frame_help_advanced, justify='left',
+            text="Key bindings are specified using Tkinter keysyms as\n"+
+                 "in these samples: <Control-f>, <Shift-F2>, <F12>,\n"
+                 "<Control-space>, <Meta-less>, <Control-Alt-Shift-X>.\n"
+                 "Upper case is used when the Shift modifier is present!\n\n" +
+                 "'Emacs style' multi-keystroke bindings are specified as\n" +
+                 "follows: <Control-x><Control-y>, where the first key\n" +
+                 "is the 'do-nothing' keybinding.\n\n" +
+                 "Multiple separate bindings for one action should be\n"+
+                 "separated by a space, eg., <Alt-v> <Meta-v>." )
+        help_advanced.grid(row=0, column=0, sticky='nsew')
+
+        # Switch between basic and advanced.
+        self.button_level = Button(self, command=self.toggle_level,
+                                  text='<< Basic Key Binding Entry')
+        self.button_level.grid(row=2, column=0, stick='ew', padx=5, pady=5)
+        self.toggle_level()
+
+    def set_modifiers_for_platform(self):
+        """Determine list of names of key modifiers for this platform.
+
+        The names are used to build Tk bindings -- it doesn't matter if the
+        keyboard has these keys; it matters if Tk understands them.  The
+        order is also important: key binding equality depends on it, so
+        config-keys.def must use the same ordering.
+        """
+        if sys.platform == "darwin":
+            self.modifiers = ['Shift', 'Control', 'Option', 'Command']
+        else:
+            self.modifiers = ['Control', 'Alt', 'Shift']
+        self.modifier_label = {'Control': 'Ctrl'}  # Short name.
+
+    def toggle_level(self):
+        "Toggle between basic and advanced keys."
+        if  self.button_level.cget('text').startswith('Advanced'):
+            self.clear_key_seq()
+            self.button_level.config(text='<< Basic Key Binding Entry')
+            self.frame_keyseq_advanced.lift()
+            self.frame_help_advanced.lift()
+            self.advanced_keys.focus_set()
+            self.advanced = True
+        else:
+            self.clear_key_seq()
+            self.button_level.config(text='Advanced Key Binding Entry >>')
+            self.frame_keyseq_basic.lift()
+            self.frame_controls_basic.lift()
+            self.advanced = False
+
+    def final_key_selected(self, event=None):
+        "Handler for clicking on key in basic settings list."
+        self.build_key_string()
+
+    def build_key_string(self):
+        "Create formatted string of modifiers plus the key."
+        keylist = modifiers = self.get_modifiers()
+        final_key = self.list_keys_final.get('anchor')
+        if final_key:
+            final_key = translate_key(final_key, modifiers)
+            keylist.append(final_key)
+        self.key_string.set(f"<{'-'.join(keylist)}>")
+
+    def get_modifiers(self):
+        "Return ordered list of modifiers that have been selected."
+        mod_list = [variable.get() for variable in self.modifier_vars]
+        return [mod for mod in mod_list if mod]
+
+    def clear_key_seq(self):
+        "Clear modifiers and keys selection."
+        self.list_keys_final.select_clear(0, 'end')
+        self.list_keys_final.yview('moveto', '0.0')
+        for variable in self.modifier_vars:
+            variable.set('')
+        self.key_string.set('')
+
+    def ok(self):
+        self.result = ''
+        keys = self.key_string.get().strip()
+        if not keys:
+            self.showerror(title=self.keyerror_title, parent=self,
+                           message="No key specified.")
+            return
+        if (self.advanced or self.keys_ok(keys)) and self.bind_ok(keys):
+            self.result = keys
+        return
+
+    def keys_ok(self, keys):
+        """Validity check on user's 'basic' keybinding selection.
+
+        Doesn't check the string produced by the advanced dialog because
+        'modifiers' isn't set.
+        """
+        final_key = self.list_keys_final.get('anchor')
+        modifiers = self.get_modifiers()
+        title = self.keyerror_title
+        key_sequences = [key for keylist in self.current_key_sequences
+                             for key in keylist]
+        if not keys.endswith('>'):
+            self.showerror(title, parent=self,
+                           message='Missing the final Key')
+        elif (not modifiers
+              and final_key not in FUNCTION_KEYS + MOVE_KEYS):
+            self.showerror(title=title, parent=self,
+                           message='No modifier key(s) specified.')
+        elif (modifiers == ['Shift']) \
+                 and (final_key not in
+                      FUNCTION_KEYS + MOVE_KEYS + ('Tab', 'Space')):
+            msg = 'The shift modifier by itself may not be used with'\
+                  ' this key symbol.'
+            self.showerror(title=title, parent=self, message=msg)
+        elif keys in key_sequences:
+            msg = 'This key combination is already in use.'
+            self.showerror(title=title, parent=self, message=msg)
+        else:
+            return True
+        return False
+
+    def bind_ok(self, keys):
+        "Return True if Tcl accepts the new keys else show message."
+        try:
+            binding = self.bind(keys, lambda: None)
+        except TclError as err:
+            self.showerror(
+                    title=self.keyerror_title, parent=self,
+                    message=(f'The entered key sequence is not accepted.\n\n'
+                             f'Error: {err}'))
+            return False
+        else:
+            self.unbind(keys, binding)
+            return True
+
+
+class GetKeysWindow(Toplevel):
+
+    def __init__(self, parent, title, action, current_key_sequences,
+                 *, _htest=False, _utest=False):
+        """
+        parent - parent of this dialog
+        title - string which is the title of the popup dialog
+        action - string, the name of the virtual event these keys will be
+                 mapped to
+        current_key_sequences - list, a list of all key sequence lists
+                 currently mapped to virtual events, for overlap checking
+        _htest - bool, change box location when running htest
+        _utest - bool, do not wait when running unittest
+        """
+        super().__init__(parent)
+        self.withdraw()  # Hide while setting geometry.
+        self['borderwidth'] = 5
+        self.resizable(height=False, width=False)
+        # Needed for winfo_reqwidth().
+        self.update_idletasks()
+        # Center dialog over parent (or below htest box).
+        x = (parent.winfo_rootx() +
+             (parent.winfo_width()//2 - self.winfo_reqwidth()//2))
+        y = (parent.winfo_rooty() +
+             ((parent.winfo_height()//2 - self.winfo_reqheight()//2)
+              if not _htest else 150))
+        self.geometry(f"+{x}+{y}")
+
+        self.title(title)
+        self.frame = frame = GetKeysFrame(self, action, current_key_sequences)
+        self.protocol("WM_DELETE_WINDOW", self.cancel)
+        frame_buttons = Frame(self)
+        self.button_ok = Button(frame_buttons, text='OK',
+                                width=8, command=self.ok)
+        self.button_cancel = Button(frame_buttons, text='Cancel',
+                                   width=8, command=self.cancel)
+        self.button_ok.grid(row=0, column=0, padx=5, pady=5)
+        self.button_cancel.grid(row=0, column=1, padx=5, pady=5)
+        frame.pack(side='top', expand=True, fill='both')
+        frame_buttons.pack(side='bottom', fill='x')
+
+        self.transient(parent)
+        _setup_dialog(self)
+        self.grab_set()
+        if not _utest:
+            self.deiconify()  # Geometry set, unhide.
+            self.wait_window()
+
+    @property
+    def result(self):
+        return self.frame.result
+
+    @result.setter
+    def result(self, value):
+        self.frame.result = value
+
+    def ok(self, event=None):
+        self.frame.ok()
+        self.grab_release()
+        self.destroy()
+
+    def cancel(self, event=None):
+        self.result = ''
+        self.grab_release()
+        self.destroy()
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_config_key', verbosity=2, exit=False)
+
+    from idlelib.idle_test.htest import run
+    run(GetKeysWindow)

Lib/idlelib/debugger.py

@@ -0,0 +1,602 @@
+"""Debug user code with a GUI interface to a subclass of bdb.Bdb.
+
+The Idb instance 'idb' and Debugger instance 'gui' need references to each
+other or to an rpc proxy for each other.
+
+If IDLE is started with '-n', so that user code and idb both run in the
+IDLE process, Debugger is called without an idb.  Debugger.__init__
+calls Idb with its incomplete self.  Idb.__init__ stores gui and gui
+then stores idb.
+
+If IDLE is started normally, so that user code executes in a separate
+process, debugger_r.start_remote_debugger is called, executing in the
+IDLE process.  It calls 'start the debugger' in the remote process,
+which calls Idb with a gui proxy.  Then Debugger is called in the IDLE
+for more.
+"""
+
+import bdb
+import os
+
+from tkinter import *
+from tkinter.ttk import Frame, Scrollbar
+
+from idlelib import macosx
+from idlelib.scrolledlist import ScrolledList
+from idlelib.window import ListedToplevel
+
+
+class Idb(bdb.Bdb):
+    "Supply user_line and user_exception functions for Bdb."
+
+    def __init__(self, gui):
+        self.gui = gui  # An instance of Debugger or proxy thereof.
+        super().__init__()
+
+    def user_line(self, frame):
+        """Handle a user stopping or breaking at a line.
+
+        Convert frame to a string and send it to gui.
+        """
+        if _in_rpc_code(frame):
+            self.set_step()
+            return
+        message = _frame2message(frame)
+        try:
+            self.gui.interaction(message, frame)
+        except TclError:  # When closing debugger window with [x] in 3.x
+            pass
+
+    def user_exception(self, frame, exc_info):
+        """Handle an the occurrence of an exception."""
+        if _in_rpc_code(frame):
+            self.set_step()
+            return
+        message = _frame2message(frame)
+        self.gui.interaction(message, frame, exc_info)
+
+def _in_rpc_code(frame):
+    "Determine if debugger is within RPC code."
+    if frame.f_code.co_filename.count('rpc.py'):
+        return True  # Skip this frame.
+    else:
+        prev_frame = frame.f_back
+        if prev_frame is None:
+            return False
+        prev_name = prev_frame.f_code.co_filename
+        if 'idlelib' in prev_name and 'debugger' in prev_name:
+            # catch both idlelib/debugger.py and idlelib/debugger_r.py
+            # on both Posix and Windows
+            return False
+        return _in_rpc_code(prev_frame)
+
+def _frame2message(frame):
+    """Return a message string for frame."""
+    code = frame.f_code
+    filename = code.co_filename
+    lineno = frame.f_lineno
+    basename = os.path.basename(filename)
+    message = f"{basename}:{lineno}"
+    if code.co_name != "?":
+        message = f"{message}: {code.co_name}()"
+    return message
+
+
+class Debugger:
+    """The debugger interface.
+
+    This class handles the drawing of the debugger window and
+    the interactions with the underlying debugger session.
+    """
+    vstack = None
+    vsource = None
+    vlocals = None
+    vglobals = None
+    stackviewer = None
+    localsviewer = None
+    globalsviewer = None
+
+    def __init__(self, pyshell, idb=None):
+        """Instantiate and draw a debugger window.
+
+        :param pyshell: An instance of the PyShell Window
+        :type  pyshell: :class:`idlelib.pyshell.PyShell`
+
+        :param idb: An instance of the IDLE debugger (optional)
+        :type  idb: :class:`idlelib.debugger.Idb`
+        """
+        if idb is None:
+            idb = Idb(self)
+        self.pyshell = pyshell
+        self.idb = idb  # If passed, a proxy of remote instance.
+        self.frame = None
+        self.make_gui()
+        self.interacting = False
+        self.nesting_level = 0
+
+    def run(self, *args):
+        """Run the debugger."""
+        # Deal with the scenario where we've already got a program running
+        # in the debugger and we want to start another. If that is the case,
+        # our second 'run' was invoked from an event dispatched not from
+        # the main event loop, but from the nested event loop in 'interaction'
+        # below. So our stack looks something like this:
+        #       outer main event loop
+        #         run()
+        #           <running program with traces>
+        #             callback to debugger's interaction()
+        #               nested event loop
+        #                 run() for second command
+        #
+        # This kind of nesting of event loops causes all kinds of problems
+        # (see e.g. issue #24455) especially when dealing with running as a
+        # subprocess, where there's all kinds of extra stuff happening in
+        # there - insert a traceback.print_stack() to check it out.
+        #
+        # By this point, we've already called restart_subprocess() in
+        # ScriptBinding. However, we also need to unwind the stack back to
+        # that outer event loop.  To accomplish this, we:
+        #   - return immediately from the nested run()
+        #   - abort_loop ensures the nested event loop will terminate
+        #   - the debugger's interaction routine completes normally
+        #   - the restart_subprocess() will have taken care of stopping
+        #     the running program, which will also let the outer run complete
+        #
+        # That leaves us back at the outer main event loop, at which point our
+        # after event can fire, and we'll come back to this routine with a
+        # clean stack.
+        if self.nesting_level > 0:
+            self.abort_loop()
+            self.root.after(100, lambda: self.run(*args))
+            return
+        try:
+            self.interacting = True
+            return self.idb.run(*args)
+        finally:
+            self.interacting = False
+
+    def close(self, event=None):
+        """Close the debugger and window."""
+        try:
+            self.quit()
+        except Exception:
+            pass
+        if self.interacting:
+            self.top.bell()
+            return
+        if self.stackviewer:
+            self.stackviewer.close(); self.stackviewer = None
+        # Clean up pyshell if user clicked debugger control close widget.
+        # (Causes a harmless extra cycle through close_debugger() if user
+        # toggled debugger from pyshell Debug menu)
+        self.pyshell.close_debugger()
+        # Now close the debugger control window....
+        self.top.destroy()
+
+    def make_gui(self):
+        """Draw the debugger gui on the screen."""
+        pyshell = self.pyshell
+        self.flist = pyshell.flist
+        self.root = root = pyshell.root
+        self.top = top = ListedToplevel(root)
+        self.top.wm_title("Debug Control")
+        self.top.wm_iconname("Debug")
+        top.wm_protocol("WM_DELETE_WINDOW", self.close)
+        self.top.bind("<Escape>", self.close)
+
+        self.bframe = bframe = Frame(top)
+        self.bframe.pack(anchor="w")
+        self.buttons = bl = []
+
+        self.bcont = b = Button(bframe, text="Go", command=self.cont)
+        bl.append(b)
+        self.bstep = b = Button(bframe, text="Step", command=self.step)
+        bl.append(b)
+        self.bnext = b = Button(bframe, text="Over", command=self.next)
+        bl.append(b)
+        self.bret = b = Button(bframe, text="Out", command=self.ret)
+        bl.append(b)
+        self.bret = b = Button(bframe, text="Quit", command=self.quit)
+        bl.append(b)
+
+        for b in bl:
+            b.configure(state="disabled")
+            b.pack(side="left")
+
+        self.cframe = cframe = Frame(bframe)
+        self.cframe.pack(side="left")
+
+        if not self.vstack:
+            self.__class__.vstack = BooleanVar(top)
+            self.vstack.set(1)
+        self.bstack = Checkbutton(cframe,
+            text="Stack", command=self.show_stack, variable=self.vstack)
+        self.bstack.grid(row=0, column=0)
+        if not self.vsource:
+            self.__class__.vsource = BooleanVar(top)
+        self.bsource = Checkbutton(cframe,
+            text="Source", command=self.show_source, variable=self.vsource)
+        self.bsource.grid(row=0, column=1)
+        if not self.vlocals:
+            self.__class__.vlocals = BooleanVar(top)
+            self.vlocals.set(1)
+        self.blocals = Checkbutton(cframe,
+            text="Locals", command=self.show_locals, variable=self.vlocals)
+        self.blocals.grid(row=1, column=0)
+        if not self.vglobals:
+            self.__class__.vglobals = BooleanVar(top)
+        self.bglobals = Checkbutton(cframe,
+            text="Globals", command=self.show_globals, variable=self.vglobals)
+        self.bglobals.grid(row=1, column=1)
+
+        self.status = Label(top, anchor="w")
+        self.status.pack(anchor="w")
+        self.error = Label(top, anchor="w")
+        self.error.pack(anchor="w", fill="x")
+        self.errorbg = self.error.cget("background")
+
+        self.fstack = Frame(top, height=1)
+        self.fstack.pack(expand=1, fill="both")
+        self.flocals = Frame(top)
+        self.flocals.pack(expand=1, fill="both")
+        self.fglobals = Frame(top, height=1)
+        self.fglobals.pack(expand=1, fill="both")
+
+        if self.vstack.get():
+            self.show_stack()
+        if self.vlocals.get():
+            self.show_locals()
+        if self.vglobals.get():
+            self.show_globals()
+
+    def interaction(self, message, frame, info=None):
+        self.frame = frame
+        self.status.configure(text=message)
+
+        if info:
+            type, value, tb = info
+            try:
+                m1 = type.__name__
+            except AttributeError:
+                m1 = "%s" % str(type)
+            if value is not None:
+                try:
+                   # TODO redo entire section, tries not needed.
+                    m1 = f"{m1}: {value}"
+                except:
+                    pass
+            bg = "yellow"
+        else:
+            m1 = ""
+            tb = None
+            bg = self.errorbg
+        self.error.configure(text=m1, background=bg)
+
+        sv = self.stackviewer
+        if sv:
+            stack, i = self.idb.get_stack(self.frame, tb)
+            sv.load_stack(stack, i)
+
+        self.show_variables(1)
+
+        if self.vsource.get():
+            self.sync_source_line()
+
+        for b in self.buttons:
+            b.configure(state="normal")
+
+        self.top.wakeup()
+        # Nested main loop: Tkinter's main loop is not reentrant, so use
+        # Tcl's vwait facility, which reenters the event loop until an
+        # event handler sets the variable we're waiting on.
+        self.nesting_level += 1
+        self.root.tk.call('vwait', '::idledebugwait')
+        self.nesting_level -= 1
+
+        for b in self.buttons:
+            b.configure(state="disabled")
+        self.status.configure(text="")
+        self.error.configure(text="", background=self.errorbg)
+        self.frame = None
+
+    def sync_source_line(self):
+        frame = self.frame
+        if not frame:
+            return
+        filename, lineno = self.__frame2fileline(frame)
+        if filename[:1] + filename[-1:] != "<>" and os.path.exists(filename):
+            self.flist.gotofileline(filename, lineno)
+
+    def __frame2fileline(self, frame):
+        code = frame.f_code
+        filename = code.co_filename
+        lineno = frame.f_lineno
+        return filename, lineno
+
+    def cont(self):
+        self.idb.set_continue()
+        self.abort_loop()
+
+    def step(self):
+        self.idb.set_step()
+        self.abort_loop()
+
+    def next(self):
+        self.idb.set_next(self.frame)
+        self.abort_loop()
+
+    def ret(self):
+        self.idb.set_return(self.frame)
+        self.abort_loop()
+
+    def quit(self):
+        self.idb.set_quit()
+        self.abort_loop()
+
+    def abort_loop(self):
+        self.root.tk.call('set', '::idledebugwait', '1')
+
+    def show_stack(self):
+        if not self.stackviewer and self.vstack.get():
+            self.stackviewer = sv = StackViewer(self.fstack, self.flist, self)
+            if self.frame:
+                stack, i = self.idb.get_stack(self.frame, None)
+                sv.load_stack(stack, i)
+        else:
+            sv = self.stackviewer
+            if sv and not self.vstack.get():
+                self.stackviewer = None
+                sv.close()
+            self.fstack['height'] = 1
+
+    def show_source(self):
+        if self.vsource.get():
+            self.sync_source_line()
+
+    def show_frame(self, stackitem):
+        self.frame = stackitem[0]  # lineno is stackitem[1]
+        self.show_variables()
+
+    def show_locals(self):
+        lv = self.localsviewer
+        if self.vlocals.get():
+            if not lv:
+                self.localsviewer = NamespaceViewer(self.flocals, "Locals")
+        else:
+            if lv:
+                self.localsviewer = None
+                lv.close()
+                self.flocals['height'] = 1
+        self.show_variables()
+
+    def show_globals(self):
+        gv = self.globalsviewer
+        if self.vglobals.get():
+            if not gv:
+                self.globalsviewer = NamespaceViewer(self.fglobals, "Globals")
+        else:
+            if gv:
+                self.globalsviewer = None
+                gv.close()
+                self.fglobals['height'] = 1
+        self.show_variables()
+
+    def show_variables(self, force=0):
+        lv = self.localsviewer
+        gv = self.globalsviewer
+        frame = self.frame
+        if not frame:
+            ldict = gdict = None
+        else:
+            ldict = frame.f_locals
+            gdict = frame.f_globals
+            if lv and gv and ldict is gdict:
+                ldict = None
+        if lv:
+            lv.load_dict(ldict, force, self.pyshell.interp.rpcclt)
+        if gv:
+            gv.load_dict(gdict, force, self.pyshell.interp.rpcclt)
+
+    def set_breakpoint(self, filename, lineno):
+        """Set a filename-lineno breakpoint in the debugger.
+
+        Called from self.load_breakpoints and EW.setbreakpoint
+        """
+        self.idb.set_break(filename, lineno)
+
+    def clear_breakpoint(self, filename, lineno):
+        self.idb.clear_break(filename, lineno)
+
+    def clear_file_breaks(self, filename):
+        self.idb.clear_all_file_breaks(filename)
+
+    def load_breakpoints(self):
+        """Load PyShellEditorWindow breakpoints into subprocess debugger."""
+        for editwin in self.pyshell.flist.inversedict:
+            filename = editwin.io.filename
+            try:
+                for lineno in editwin.breakpoints:
+                    self.set_breakpoint(filename, lineno)
+            except AttributeError:
+                continue
+
+
+class StackViewer(ScrolledList):
+    "Code stack viewer for debugger GUI."
+
+    def __init__(self, master, flist, gui):
+        if macosx.isAquaTk():
+            # At least on with the stock AquaTk version on OSX 10.4 you'll
+            # get a shaking GUI that eventually kills IDLE if the width
+            # argument is specified.
+            ScrolledList.__init__(self, master)
+        else:
+            ScrolledList.__init__(self, master, width=80)
+        self.flist = flist
+        self.gui = gui
+        self.stack = []
+
+    def load_stack(self, stack, index=None):
+        self.stack = stack
+        self.clear()
+        for i in range(len(stack)):
+            frame, lineno = stack[i]
+            try:
+                modname = frame.f_globals["__name__"]
+            except:
+                modname = "?"
+            code = frame.f_code
+            filename = code.co_filename
+            funcname = code.co_name
+            import linecache
+            sourceline = linecache.getline(filename, lineno)
+            sourceline = sourceline.strip()
+            if funcname in ("?", "", None):
+                item = "%s, line %d: %s" % (modname, lineno, sourceline)
+            else:
+                item = "%s.%s(), line %d: %s" % (modname, funcname,
+                                                 lineno, sourceline)
+            if i == index:
+                item = "> " + item
+            self.append(item)
+        if index is not None:
+            self.select(index)
+
+    def popup_event(self, event):
+        "Override base method."
+        if self.stack:
+            return ScrolledList.popup_event(self, event)
+
+    def fill_menu(self):
+        "Override base method."
+        menu = self.menu
+        menu.add_command(label="Go to source line",
+                         command=self.goto_source_line)
+        menu.add_command(label="Show stack frame",
+                         command=self.show_stack_frame)
+
+    def on_select(self, index):
+        "Override base method."
+        if 0 <= index < len(self.stack):
+            self.gui.show_frame(self.stack[index])
+
+    def on_double(self, index):
+        "Override base method."
+        self.show_source(index)
+
+    def goto_source_line(self):
+        index = self.listbox.index("active")
+        self.show_source(index)
+
+    def show_stack_frame(self):
+        index = self.listbox.index("active")
+        if 0 <= index < len(self.stack):
+            self.gui.show_frame(self.stack[index])
+
+    def show_source(self, index):
+        if not (0 <= index < len(self.stack)):
+            return
+        frame, lineno = self.stack[index]
+        code = frame.f_code
+        filename = code.co_filename
+        if os.path.isfile(filename):
+            edit = self.flist.open(filename)
+            if edit:
+                edit.gotoline(lineno)
+
+
+class NamespaceViewer:
+    "Global/local namespace viewer for debugger GUI."
+
+    def __init__(self, master, title, odict=None):  # XXX odict never passed.
+        width = 0
+        height = 40
+        if odict:
+            height = 20*len(odict) # XXX 20 == observed height of Entry widget
+        self.master = master
+        self.title = title
+        import reprlib
+        self.repr = reprlib.Repr()
+        self.repr.maxstring = 60
+        self.repr.maxother = 60
+        self.frame = frame = Frame(master)
+        self.frame.pack(expand=1, fill="both")
+        self.label = Label(frame, text=title, borderwidth=2, relief="groove")
+        self.label.pack(fill="x")
+        self.vbar = vbar = Scrollbar(frame, name="vbar")
+        vbar.pack(side="right", fill="y")
+        self.canvas = canvas = Canvas(frame,
+                                      height=min(300, max(40, height)),
+                                      scrollregion=(0, 0, width, height))
+        canvas.pack(side="left", fill="both", expand=1)
+        vbar["command"] = canvas.yview
+        canvas["yscrollcommand"] = vbar.set
+        self.subframe = subframe = Frame(canvas)
+        self.sfid = canvas.create_window(0, 0, window=subframe, anchor="nw")
+        self.load_dict(odict)
+
+    prev_odict = -1  # Needed for initial comparison below.
+
+    def load_dict(self, odict, force=0, rpc_client=None):
+        if odict is self.prev_odict and not force:
+            return
+        subframe = self.subframe
+        frame = self.frame
+        for c in list(subframe.children.values()):
+            c.destroy()
+        self.prev_odict = None
+        if not odict:
+            l = Label(subframe, text="None")
+            l.grid(row=0, column=0)
+        else:
+            #names = sorted(dict)
+            #
+            # Because of (temporary) limitations on the dict_keys type (not yet
+            # public or pickleable), have the subprocess to send a list of
+            # keys, not a dict_keys object.  sorted() will take a dict_keys
+            # (no subprocess) or a list.
+            #
+            # There is also an obscure bug in sorted(dict) where the
+            # interpreter gets into a loop requesting non-existing dict[0],
+            # dict[1], dict[2], etc from the debugger_r.DictProxy.
+            # TODO recheck above; see debugger_r 159ff, debugobj 60.
+            keys_list = odict.keys()
+            names = sorted(keys_list)
+
+            row = 0
+            for name in names:
+                value = odict[name]
+                svalue = self.repr.repr(value) # repr(value)
+                # Strip extra quotes caused by calling repr on the (already)
+                # repr'd value sent across the RPC interface:
+                if rpc_client:
+                    svalue = svalue[1:-1]
+                l = Label(subframe, text=name)
+                l.grid(row=row, column=0, sticky="nw")
+                l = Entry(subframe, width=0, borderwidth=0)
+                l.insert(0, svalue)
+                l.grid(row=row, column=1, sticky="nw")
+                row = row+1
+        self.prev_odict = odict
+        # XXX Could we use a <Configure> callback for the following?
+        subframe.update_idletasks() # Alas!
+        width = subframe.winfo_reqwidth()
+        height = subframe.winfo_reqheight()
+        canvas = self.canvas
+        self.canvas["scrollregion"] = (0, 0, width, height)
+        if height > 300:
+            canvas["height"] = 300
+            frame.pack(expand=1)
+        else:
+            canvas["height"] = height
+            frame.pack(expand=0)
+
+    def close(self):
+        self.frame.destroy()
+
+
+if __name__ == "__main__":
+    from unittest import main
+    main('idlelib.idle_test.test_debugger', verbosity=2, exit=False)
+
+# TODO: htest?

Lib/idlelib/debugger_r.py

@@ -0,0 +1,390 @@
+"""Support for remote Python debugging.
+
+Some ASCII art to describe the structure:
+
+       IN PYTHON SUBPROCESS          #             IN IDLE PROCESS
+                                     #
+                                     #        oid='gui_adapter'
+                 +----------+        #       +------------+          +-----+
+                 | GUIProxy |--remote#call-->| GUIAdapter |--calls-->| GUI |
++-----+--calls-->+----------+        #       +------------+          +-----+
+| Idb |                               #                             /
++-----+<-calls--+------------+         #      +----------+<--calls-/
+                | IdbAdapter |<--remote#call--| IdbProxy |
+                +------------+         #      +----------+
+                oid='idb_adapter'      #
+
+The purpose of the Proxy and Adapter classes is to translate certain
+arguments and return values that cannot be transported through the RPC
+barrier, in particular frame and traceback objects.
+
+"""
+import reprlib
+import types
+from idlelib import debugger
+
+debugging = 0
+
+idb_adap_oid = "idb_adapter"
+gui_adap_oid = "gui_adapter"
+
+#=======================================
+#
+# In the PYTHON subprocess:
+
+frametable = {}
+dicttable = {}
+codetable = {}
+tracebacktable = {}
+
+def wrap_frame(frame):
+    fid = id(frame)
+    frametable[fid] = frame
+    return fid
+
+def wrap_info(info):
+    "replace info[2], a traceback instance, by its ID"
+    if info is None:
+        return None
+    else:
+        traceback = info[2]
+        assert isinstance(traceback, types.TracebackType)
+        traceback_id = id(traceback)
+        tracebacktable[traceback_id] = traceback
+        modified_info = (info[0], info[1], traceback_id)
+        return modified_info
+
+class GUIProxy:
+
+    def __init__(self, conn, gui_adap_oid):
+        self.conn = conn
+        self.oid = gui_adap_oid
+
+    def interaction(self, message, frame, info=None):
+        # calls rpc.SocketIO.remotecall() via run.MyHandler instance
+        # pass frame and traceback object IDs instead of the objects themselves
+        self.conn.remotecall(self.oid, "interaction",
+                             (message, wrap_frame(frame), wrap_info(info)),
+                             {})
+
+class IdbAdapter:
+
+    def __init__(self, idb):
+        self.idb = idb
+
+    #----------called by an IdbProxy----------
+
+    def set_step(self):
+        self.idb.set_step()
+
+    def set_quit(self):
+        self.idb.set_quit()
+
+    def set_continue(self):
+        self.idb.set_continue()
+
+    def set_next(self, fid):
+        frame = frametable[fid]
+        self.idb.set_next(frame)
+
+    def set_return(self, fid):
+        frame = frametable[fid]
+        self.idb.set_return(frame)
+
+    def get_stack(self, fid, tbid):
+        frame = frametable[fid]
+        if tbid is None:
+            tb = None
+        else:
+            tb = tracebacktable[tbid]
+        stack, i = self.idb.get_stack(frame, tb)
+        stack = [(wrap_frame(frame2), k) for frame2, k in stack]
+        return stack, i
+
+    def run(self, cmd):
+        import __main__
+        self.idb.run(cmd, __main__.__dict__)
+
+    def set_break(self, filename, lineno):
+        msg = self.idb.set_break(filename, lineno)
+        return msg
+
+    def clear_break(self, filename, lineno):
+        msg = self.idb.clear_break(filename, lineno)
+        return msg
+
+    def clear_all_file_breaks(self, filename):
+        msg = self.idb.clear_all_file_breaks(filename)
+        return msg
+
+    #----------called by a FrameProxy----------
+
+    def frame_attr(self, fid, name):
+        frame = frametable[fid]
+        return getattr(frame, name)
+
+    def frame_globals(self, fid):
+        frame = frametable[fid]
+        gdict = frame.f_globals
+        did = id(gdict)
+        dicttable[did] = gdict
+        return did
+
+    def frame_locals(self, fid):
+        frame = frametable[fid]
+        ldict = frame.f_locals
+        did = id(ldict)
+        dicttable[did] = ldict
+        return did
+
+    def frame_code(self, fid):
+        frame = frametable[fid]
+        code = frame.f_code
+        cid = id(code)
+        codetable[cid] = code
+        return cid
+
+    #----------called by a CodeProxy----------
+
+    def code_name(self, cid):
+        code = codetable[cid]
+        return code.co_name
+
+    def code_filename(self, cid):
+        code = codetable[cid]
+        return code.co_filename
+
+    #----------called by a DictProxy----------
+
+    def dict_keys(self, did):
+        raise NotImplementedError("dict_keys not public or pickleable")
+##         return dicttable[did].keys()
+
+    ### Needed until dict_keys type is finished and pickleable.
+    # xxx finished. pickleable?
+    ### Will probably need to extend rpc.py:SocketIO._proxify at that time.
+    def dict_keys_list(self, did):
+        return list(dicttable[did].keys())
+
+    def dict_item(self, did, key):
+        value = dicttable[did][key]
+        return reprlib.repr(value) # Can't pickle module 'builtins'.
+
+#----------end class IdbAdapter----------
+
+
+def start_debugger(rpchandler, gui_adap_oid):
+    """Start the debugger and its RPC link in the Python subprocess
+
+    Start the subprocess side of the split debugger and set up that side of the
+    RPC link by instantiating the GUIProxy, Idb debugger, and IdbAdapter
+    objects and linking them together.  Register the IdbAdapter with the
+    RPCServer to handle RPC requests from the split debugger GUI via the
+    IdbProxy.
+
+    """
+    gui_proxy = GUIProxy(rpchandler, gui_adap_oid)
+    idb = debugger.Idb(gui_proxy)
+    idb_adap = IdbAdapter(idb)
+    rpchandler.register(idb_adap_oid, idb_adap)
+    return idb_adap_oid
+
+
+#=======================================
+#
+# In the IDLE process:
+
+
+class FrameProxy:
+
+    def __init__(self, conn, fid):
+        self._conn = conn
+        self._fid = fid
+        self._oid = "idb_adapter"
+        self._dictcache = {}
+
+    def __getattr__(self, name):
+        if name[:1] == "_":
+            raise AttributeError(name)
+        if name == "f_code":
+            return self._get_f_code()
+        if name == "f_globals":
+            return self._get_f_globals()
+        if name == "f_locals":
+            return self._get_f_locals()
+        return self._conn.remotecall(self._oid, "frame_attr",
+                                     (self._fid, name), {})
+
+    def _get_f_code(self):
+        cid = self._conn.remotecall(self._oid, "frame_code", (self._fid,), {})
+        return CodeProxy(self._conn, self._oid, cid)
+
+    def _get_f_globals(self):
+        did = self._conn.remotecall(self._oid, "frame_globals",
+                                    (self._fid,), {})
+        return self._get_dict_proxy(did)
+
+    def _get_f_locals(self):
+        did = self._conn.remotecall(self._oid, "frame_locals",
+                                    (self._fid,), {})
+        return self._get_dict_proxy(did)
+
+    def _get_dict_proxy(self, did):
+        if did in self._dictcache:
+            return self._dictcache[did]
+        dp = DictProxy(self._conn, self._oid, did)
+        self._dictcache[did] = dp
+        return dp
+
+
+class CodeProxy:
+
+    def __init__(self, conn, oid, cid):
+        self._conn = conn
+        self._oid = oid
+        self._cid = cid
+
+    def __getattr__(self, name):
+        if name == "co_name":
+            return self._conn.remotecall(self._oid, "code_name",
+                                         (self._cid,), {})
+        if name == "co_filename":
+            return self._conn.remotecall(self._oid, "code_filename",
+                                         (self._cid,), {})
+
+
+class DictProxy:
+
+    def __init__(self, conn, oid, did):
+        self._conn = conn
+        self._oid = oid
+        self._did = did
+
+##    def keys(self):
+##        return self._conn.remotecall(self._oid, "dict_keys", (self._did,), {})
+
+    # 'temporary' until dict_keys is a pickleable built-in type
+    def keys(self):
+        return self._conn.remotecall(self._oid,
+                                     "dict_keys_list", (self._did,), {})
+
+    def __getitem__(self, key):
+        return self._conn.remotecall(self._oid, "dict_item",
+                                     (self._did, key), {})
+
+    def __getattr__(self, name):
+        ##print("*** Failed DictProxy.__getattr__:", name)
+        raise AttributeError(name)
+
+
+class GUIAdapter:
+
+    def __init__(self, conn, gui):
+        self.conn = conn
+        self.gui = gui
+
+    def interaction(self, message, fid, modified_info):
+        ##print("*** Interaction: (%s, %s, %s)" % (message, fid, modified_info))
+        frame = FrameProxy(self.conn, fid)
+        self.gui.interaction(message, frame, modified_info)
+
+
+class IdbProxy:
+
+    def __init__(self, conn, shell, oid):
+        self.oid = oid
+        self.conn = conn
+        self.shell = shell
+
+    def call(self, methodname, /, *args, **kwargs):
+        ##print("*** IdbProxy.call %s %s %s" % (methodname, args, kwargs))
+        value = self.conn.remotecall(self.oid, methodname, args, kwargs)
+        ##print("*** IdbProxy.call %s returns %r" % (methodname, value))
+        return value
+
+    def run(self, cmd, locals):
+        # Ignores locals on purpose!
+        seq = self.conn.asyncqueue(self.oid, "run", (cmd,), {})
+        self.shell.interp.active_seq = seq
+
+    def get_stack(self, frame, tbid):
+        # passing frame and traceback IDs, not the objects themselves
+        stack, i = self.call("get_stack", frame._fid, tbid)
+        stack = [(FrameProxy(self.conn, fid), k) for fid, k in stack]
+        return stack, i
+
+    def set_continue(self):
+        self.call("set_continue")
+
+    def set_step(self):
+        self.call("set_step")
+
+    def set_next(self, frame):
+        self.call("set_next", frame._fid)
+
+    def set_return(self, frame):
+        self.call("set_return", frame._fid)
+
+    def set_quit(self):
+        self.call("set_quit")
+
+    def set_break(self, filename, lineno):
+        msg = self.call("set_break", filename, lineno)
+        return msg
+
+    def clear_break(self, filename, lineno):
+        msg = self.call("clear_break", filename, lineno)
+        return msg
+
+    def clear_all_file_breaks(self, filename):
+        msg = self.call("clear_all_file_breaks", filename)
+        return msg
+
+def start_remote_debugger(rpcclt, pyshell):
+    """Start the subprocess debugger, initialize the debugger GUI and RPC link
+
+    Request the RPCServer start the Python subprocess debugger and link.  Set
+    up the Idle side of the split debugger by instantiating the IdbProxy,
+    debugger GUI, and debugger GUIAdapter objects and linking them together.
+
+    Register the GUIAdapter with the RPCClient to handle debugger GUI
+    interaction requests coming from the subprocess debugger via the GUIProxy.
+
+    The IdbAdapter will pass execution and environment requests coming from the
+    Idle debugger GUI to the subprocess debugger via the IdbProxy.
+
+    """
+    global idb_adap_oid
+
+    idb_adap_oid = rpcclt.remotecall("exec", "start_the_debugger",\
+                                   (gui_adap_oid,), {})
+    idb_proxy = IdbProxy(rpcclt, pyshell, idb_adap_oid)
+    gui = debugger.Debugger(pyshell, idb_proxy)
+    gui_adap = GUIAdapter(rpcclt, gui)
+    rpcclt.register(gui_adap_oid, gui_adap)
+    return gui
+
+def close_remote_debugger(rpcclt):
+    """Shut down subprocess debugger and Idle side of debugger RPC link
+
+    Request that the RPCServer shut down the subprocess debugger and link.
+    Unregister the GUIAdapter, which will cause a GC on the Idle process
+    debugger and RPC link objects.  (The second reference to the debugger GUI
+    is deleted in pyshell.close_remote_debugger().)
+
+    """
+    close_subprocess_debugger(rpcclt)
+    rpcclt.unregister(gui_adap_oid)
+
+def close_subprocess_debugger(rpcclt):
+    rpcclt.remotecall("exec", "stop_the_debugger", (idb_adap_oid,), {})
+
+def restart_subprocess_debugger(rpcclt):
+    idb_adap_oid_ret = rpcclt.remotecall("exec", "start_the_debugger",\
+                                         (gui_adap_oid,), {})
+    assert idb_adap_oid_ret == idb_adap_oid, 'Idb restarted with different oid'
+
+
+if __name__ == "__main__":
+    from unittest import main
+    main('idlelib.idle_test.test_debugger_r', verbosity=2, exit=False)

Lib/idlelib/debugobj.py

@@ -0,0 +1,146 @@
+"""Define tree items for debug stackviewer, which is only user.
+"""
+# XXX TO DO:
+# - popup menu
+# - support partial or total redisplay
+# - more doc strings
+# - tooltips
+
+# object browser
+
+# XXX TO DO:
+# - for classes/modules, add "open source" to object browser
+from reprlib import Repr
+
+from idlelib.tree import TreeItem, TreeNode, ScrolledCanvas
+
+myrepr = Repr()
+myrepr.maxstring = 100
+myrepr.maxother = 100
+
+class ObjectTreeItem(TreeItem):
+    def __init__(self, labeltext, object_, setfunction=None):
+        self.labeltext = labeltext
+        self.object = object_
+        self.setfunction = setfunction
+    def GetLabelText(self):
+        return self.labeltext
+    def GetText(self):
+        return myrepr.repr(self.object)
+    def GetIconName(self):
+        if not self.IsExpandable():
+            return "python"
+    def IsEditable(self):
+        return self.setfunction is not None
+    def SetText(self, text):
+        try:
+            value = eval(text)
+            self.setfunction(value)
+        except:
+            pass
+        else:
+            self.object = value
+    def IsExpandable(self):
+        return not not dir(self.object)
+    def GetSubList(self):
+        keys = dir(self.object)
+        sublist = []
+        for key in keys:
+            try:
+                value = getattr(self.object, key)
+            except AttributeError:
+                continue
+            item = make_objecttreeitem(
+                str(key) + " =",
+                value,
+                lambda value, key=key, object_=self.object:
+                    setattr(object_, key, value))
+            sublist.append(item)
+        return sublist
+
+class ClassTreeItem(ObjectTreeItem):
+    def IsExpandable(self):
+        return True
+    def GetSubList(self):
+        sublist = ObjectTreeItem.GetSubList(self)
+        if len(self.object.__bases__) == 1:
+            item = make_objecttreeitem("__bases__[0] =",
+                self.object.__bases__[0])
+        else:
+            item = make_objecttreeitem("__bases__ =", self.object.__bases__)
+        sublist.insert(0, item)
+        return sublist
+
+class AtomicObjectTreeItem(ObjectTreeItem):
+    def IsExpandable(self):
+        return False
+
+class SequenceTreeItem(ObjectTreeItem):
+    def IsExpandable(self):
+        return len(self.object) > 0
+    def keys(self):
+        return range(len(self.object))
+    def GetSubList(self):
+        sublist = []
+        for key in self.keys():
+            try:
+                value = self.object[key]
+            except KeyError:
+                continue
+            def setfunction(value, key=key, object_=self.object):
+                object_[key] = value
+            item = make_objecttreeitem(f"{key!r}:", value, setfunction)
+            sublist.append(item)
+        return sublist
+
+class DictTreeItem(SequenceTreeItem):
+    def keys(self):
+        # TODO return sorted(self.object)
+        keys = list(self.object)
+        try:
+            keys.sort()
+        except:
+            pass
+        return keys
+
+dispatch = {
+    int: AtomicObjectTreeItem,
+    float: AtomicObjectTreeItem,
+    str: AtomicObjectTreeItem,
+    tuple: SequenceTreeItem,
+    list: SequenceTreeItem,
+    dict: DictTreeItem,
+    type: ClassTreeItem,
+}
+
+def make_objecttreeitem(labeltext, object_, setfunction=None):
+    t = type(object_)
+    if t in dispatch:
+        c = dispatch[t]
+    else:
+        c = ObjectTreeItem
+    return c(labeltext, object_, setfunction)
+
+
+def _debug_object_browser(parent):  # htest #
+    import sys
+    from tkinter import Toplevel
+    top = Toplevel(parent)
+    top.title("Test debug object browser")
+    x, y = map(int, parent.geometry().split('+')[1:])
+    top.geometry("+%d+%d" % (x + 100, y + 175))
+    top.configure(bd=0, bg="yellow")
+    top.focus_set()
+    sc = ScrolledCanvas(top, bg="white", highlightthickness=0, takefocus=1)
+    sc.frame.pack(expand=1, fill="both")
+    item = make_objecttreeitem("sys", sys)
+    node = TreeNode(sc.canvas, None, item)
+    node.update()
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_debugobj', verbosity=2, exit=False)
+
+    from idlelib.idle_test.htest import run
+    run(_debug_object_browser)

Lib/idlelib/debugobj_r.py

@@ -0,0 +1,41 @@
+from idlelib import rpc
+
+def remote_object_tree_item(item):
+    wrapper = WrappedObjectTreeItem(item)
+    oid = id(wrapper)
+    rpc.objecttable[oid] = wrapper
+    return oid
+
+class WrappedObjectTreeItem:
+    # Lives in PYTHON subprocess
+
+    def __init__(self, item):
+        self.__item = item
+
+    def __getattr__(self, name):
+        value = getattr(self.__item, name)
+        return value
+
+    def _GetSubList(self):
+        sub_list = self.__item._GetSubList()
+        return list(map(remote_object_tree_item, sub_list))
+
+class StubObjectTreeItem:
+    # Lives in IDLE process
+
+    def __init__(self, sockio, oid):
+        self.sockio = sockio
+        self.oid = oid
+
+    def __getattr__(self, name):
+        value = rpc.MethodProxy(self.sockio, self.oid, name)
+        return value
+
+    def _GetSubList(self):
+        sub_list = self.sockio.remotecall(self.oid, "_GetSubList", (), {})
+        return [StubObjectTreeItem(self.sockio, oid) for oid in sub_list]
+
+
+if __name__ == '__main__':
+    from unittest import main
+    main('idlelib.idle_test.test_debugobj_r', verbosity=2)