diff --git a/.github/workflows/pyperformance-comment.yaml b/.github/workflows/pyperformance-comment.yaml new file mode 100644 index 000000000..565d933e9 --- /dev/null +++ b/.github/workflows/pyperformance-comment.yaml @@ -0,0 +1,81 @@ +on: + workflow_run: + workflows: [pyperformance] + types: [completed] + +name: pyperformance comment + +# Same split as `codspeed-comment.yaml`: a `pull_request` run from a fork gets +# a read-only token however it is configured, so the comparison is computed +# there and handed over as an artifact, and this workflow -- which +# `workflow_run` always runs in the base repo's context -- is the one trusted +# with a token that may write the comment. +permissions: + contents: read + actions: read # needed by download-artifact to fetch another run's artifact + pull-requests: write + +jobs: + comment: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Download the comparison produced by the pyperformance workflow + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pyperformance-diff + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + continue-on-error: true # absent when the job was skipped or dispatched + + - name: Post or update the PR comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('fs'); + if (!fs.existsSync('diff.md')) { + core.info('No diff.md in this run; nothing to comment.'); + return; + } + + const marker = ''; + const run = context.payload.workflow_run; + let prNumber = (run.pull_requests || []).map((pr) => pr.number)[0]; + if (!prNumber) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: run.head_sha, + }); + prNumber = data.find((pr) => pr.state === 'open')?.number; + } + if (!prNumber) { + core.info(`No open pull request for ${run.head_sha}; nothing to comment.`); + return; + } + + const raw = fs.readFileSync('diff.md', 'utf8').slice(0, 60000); + const body = `${marker}\n\n${raw.replace(/'; + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + const existing = comments.find((c) => c.body && c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body, + }); + } diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 02ceb805c..43d8f1603 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,6 +4,11 @@ rules: # pull_request_target is needed to label and assign PRs from forks with issues: write. # The workflow does not check out or execute pull request code. - oscca-pr.yml:3 + # workflow_run is needed to post a PR comment from a fork's run, which is + # handed a read-only token however the triggering workflow is configured. + # The workflow does not check out or execute pull request code: it reads + # one artifact and writes a comment. + - pyperformance-comment.yaml:1 excessive-permissions: ignore: # pull_request_target is needed to post PR comments with pull-requests: write. diff --git a/.gitignore b/.gitignore index fdacf4f23..cb22084d3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,8 @@ Lib/test/data/* !Lib/test/data/README cpython/ .claude/ -docs/superpowers/ \ No newline at end of file +docs/superpowers/ + +scripts/pyperformance/results/ +scripts/pyperformance/stub_psutil/build/ +scripts/pyperformance/stub_psutil/src/psutil.egg-info/ \ No newline at end of file diff --git a/scripts/pyperformance/README.md b/scripts/pyperformance/README.md new file mode 100644 index 000000000..5212397bd --- /dev/null +++ b/scripts/pyperformance/README.md @@ -0,0 +1,100 @@ +# Running upstream pyperformance against RustPython + +Two different interpreters are involved whenever +[pyperformance](https://github.com/python/pyperformance) is pointed at +RustPython, and only one of them is RustPython itself: + +- **The `pyperformance` CLI** (`pyperformance run --python `) is the + *host* process -- it parses arguments, decides which benchmarks to run, and + collects results. It always runs under a real CPython, regardless of what + `--python` points at; RustPython never has to run this command. +- **The benchmark being measured** runs as the `--python` target instead. + `pyperf`'s `Runner` re-execs that target as a subprocess to actually run the + timing loop, and the benchmark script that subprocess runs does + `import pyperf` itself to call the measurement API. So it's RustPython, as + that subprocess, that needs to import `pyperf` -- not run `pyperformance`. + +`pyperf` hard-depends on `psutil`, a C-extension package, and RustPython has +no CPython C-API / extension-module loading support (`_imp.create_dynamic` / +`_imp.exec_dynamic` are not implemented, and `_sysconfigdata` doesn't provide +a full compiler config, so even building a C extension from source fails). +That's what breaks: not "installing the pyperformance CLI on RustPython" -- +RustPython importing `pyperf` as a benchmark subprocess. + +Workaround: `pyperf` already disables psutil usage on interpreters that +report `Py_GIL_DISABLED=1` in `sysconfig` (see `pyperf._utils.USE_PSUTIL`) -- +this is exactly what RustPython reports, since it has no GIL. So `pyperf` +never actually calls into psutil at runtime on RustPython; a +functionality-free pure-Python `psutil` stub (see `stub_psutil/`) is enough +to satisfy pip's dependency resolution and let installation proceed. + +This only unblocks *pure-Python* benchmarks. Any benchmark whose own +workload needs a real C extension (numpy, lxml, greenlet, PyYAML's C +accelerator, ...) will still fail to install -- and that failure is a real, +separate finding about RustPython's lack of C-extension support, not an +artifact of this workaround. + +## Usage + +```shell +# Build RustPython first (SSL feature needed for --install-pip, not required here) +cargo build --release + +# Run every benchmark pyperformance knows about against it, and write a catalog +python3 scripts/pyperformance/run_all.py --rustpython target/release/rustpython + +# Or just a subset +python3 scripts/pyperformance/run_all.py --benchmarks nqueens,richards,pyflate +``` + +Output goes to `scripts/pyperformance/results/`: +- `CATALOG.md` -- human-readable pass/fail table with the mean time and a + failure reason for each benchmark. +- `catalog.json` -- the same data, machine-readable. +- `raw/.json` -- pyperf's own result file for benchmarks that + passed. + +The run resumes automatically: re-running the script skips any benchmark +already present in `catalog.json`. Pass `--force` to redo everything. + +A host CPython venv (pyperformance itself only runs under a real CPython; +RustPython is just the `--python` target it benchmarks) and the stub wheel +are cached under `target/pyperformance/` (gitignored) and reused across +runs. + +See `python3 scripts/pyperformance/run_all.py --help` for all options +(`--timeout`, `--rigorous` instead of `--fast`, `--cache-dir`, ...). + +## Comparing against real CPython + +`run_all.py` can target any Python executable, not just RustPython, via +`--python` + `--label`. For a real CPython you don't need (and don't want) +the psutil stub -- pass `--no-psutil-stub` so it installs and uses the +genuine `psutil`: + +```shell +python3 scripts/pyperformance/run_all.py \ + --python "$(command -v python3.13)" --label cpython3.13 --no-psutil-stub + +python3 scripts/pyperformance/run_all.py \ + --python target/release/rustpython --label rustpython +``` + +Each `--label` gets its own subdirectory under `results/` (its own +`catalog.json`, `CATALOG.md`, `raw/`), so multiple targets' catalogs coexist. + +Then compare two catalogs: + +```shell +python3 scripts/pyperformance/compare.py --baseline cpython3.13 --candidate rustpython +``` + +This writes `results/COMPARE-rustpython-vs-cpython3.13.md`: a per-benchmark +table with both targets' status/mean and the candidate/baseline time ratio, +plus the median slowdown across benchmarks both targets passed. + +Note: pass an actual interpreter binary to `--python`, not a version-manager +shim (e.g. an `asdf`/`mise`/`pyenv` shim) -- pyperformance runs a helper +script through it directly, which some shims don't support. If unsure what a +`python3` on your PATH really resolves to, use +`python3 -c "import sys; print(sys.executable)"` and pass that path. diff --git a/scripts/pyperformance/compare.py b/scripts/pyperformance/compare.py new file mode 100644 index 000000000..3fc71a869 --- /dev/null +++ b/scripts/pyperformance/compare.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Compare two `run_all.py` catalogs (e.g. RustPython vs. CPython) side by side. + +Usage +----- + # Catalog a real CPython too (no psutil stub needed -- it has a real one): + python3 scripts/pyperformance/run_all.py --python "$(which python3.13)" \\ + --label cpython3.13 --no-psutil-stub + + # RustPython catalog, if not already generated: + python3 scripts/pyperformance/run_all.py --python target/release/rustpython \\ + --label rustpython + + # Compare them: + python3 scripts/pyperformance/compare.py --baseline cpython3.13 --candidate rustpython + +Writes `scripts/pyperformance/results/COMPARE--vs-.md`. +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_RESULTS_DIR = SCRIPT_DIR / "results" + +TIME_RE = re.compile(r"^\s*([\d.]+)\s*(ns|us|ms|sec)\b") +UNIT_SECONDS = {"ns": 1e-9, "us": 1e-6, "ms": 1e-3, "sec": 1.0} + + +def parse_mean_seconds(mean: str | None) -> float | None: + """Parse a pyperf " +- " string into seconds.""" + if not mean: + return None + m = TIME_RE.match(mean) + if not m: + return None + value, unit = m.groups() + return float(value) * UNIT_SECONDS[unit] + + +def load_catalog(results_dir: Path, label: str) -> dict[str, dict]: + catalog_json = results_dir / label / "catalog.json" + if not catalog_json.exists(): + raise SystemExit( + f"no catalog for label {label!r} at {catalog_json} -- run " + f"`run_all.py --label {label} ...` first" + ) + return {r["benchmark"]: r for r in json.loads(catalog_json.read_text())} + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--baseline", + required=True, + help="label of the baseline catalog, e.g. cpython3.13", + ) + parser.add_argument( + "--candidate", + required=True, + help="label of the catalog to compare against it, e.g. rustpython", + ) + parser.add_argument( + "--results-dir", + type=Path, + default=DEFAULT_RESULTS_DIR, + help="parent directory containing