RustPython/.github/workflows/pyperformance.yaml

on:
  pull_request:
    types: [labeled, opened, synchronize, reopened]
  workflow_dispatch:

name: pyperformance

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_INCREMENTAL: 0
  CARGO_TERM_COLOR: always
  CI: true
  # Pure-Python benchmarks only: anything whose workload needs a real C
  # extension cannot install under RustPython at all, and the async/SSL ones
  # are far too slow here to be worth a runner's time. Keep this list short --
  # every entry is paid for three times (CPython, base, head) in one job.
  BENCHMARKS: >-
    chaos,comprehensions,coroutines,deepcopy,deltablue,fannkuch,float,
    generators,go,hexiom,json_dumps,json_loads,logging,mdp,nbody,nqueens,
    pickle_pure_python,raytrace,regex_compile,regex_v8,richards,scimark,
    spectral_norm,telco,typing_runtime_protocols,unpack_sequence,
    unpickle_pure_python,xml_etree
  # A benchmark that needs longer than this on the base commit is one this
  # comparison cannot afford to wait for; it shows up as a failure in the
  # table, which is itself the interesting signal.
  BENCH_TIMEOUT: 300

jobs:
  compare:
    # `skip:ci` mirrors the other workflows; `run:pyperformance` opts a PR in,
    # because the job costs well over an hour and most PRs do not need it.
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (!contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
       contains(github.event.pull_request.labels.*.name, 'run:pyperformance'))
    name: base vs. head vs. CPython
    runs-on: ubuntu-latest
    timeout-minutes: 300
    permissions:
      contents: read
      # Honoured for a pull request opened from this repo, which is then
      # commented on directly by the last step; a fork's run is handed
      # read-only whatever is asked for here and goes the artifact route.
      pull-requests: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
          persist-credentials: false

      - uses: dtolnay/rust-toolchain@stable

      # Both the third interpreter under test and the host pyperformance
      # itself runs on (it only runs under a real CPython; RustPython is just
      # the `--python` target it drives).
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        id: cpython
        with:
          python-version: "3.15.0"

      - name: Restore cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: pyperformance-
        continue-on-error: true

      # The base commit may predate these scripts, and `git checkout <base>`
      # below would then delete them; run them from a copy outside the tree.
      - name: Copy the benchmark scripts out of the checkout
        run: cp -r scripts/pyperformance "${RUNNER_TEMP}/pyperformance-scripts"

      # Each build is copied aside with the stdlib of its own commit, so the
      # second does not overwrite the first and a PR that touches `Lib/` is
      # still compared against the base commit's `Lib/`. A binary moved away
      # from its checkout finds the stdlib only through RUSTPYTHONPATH, which
      # `run_all.py` forwards through pyperf into the venv it builds.
      - name: Build head
        run: |
          cargo build --release --locked
          mkdir -p "${RUNNER_TEMP}/head"
          cp target/release/rustpython "${RUNNER_TEMP}/head/rustpython"
          cp -r Lib "${RUNNER_TEMP}/head/Lib"

      - name: Build base
        continue-on-error: true # a base that cannot build leaves head vs. CPython
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          git checkout --quiet "${BASE_SHA}" --
          cargo build --release --locked
          mkdir -p "${RUNNER_TEMP}/base"
          cp target/release/rustpython "${RUNNER_TEMP}/base/rustpython"
          cp -r Lib "${RUNNER_TEMP}/base/Lib"

      - name: Restore head checkout
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: git checkout --quiet "${HEAD_SHA}" --

      - name: Save cache
        if: always()
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
        continue-on-error: true

      # The three runs are deliberately adjacent, with no build between them,
      # so nothing but the interpreter differs across the comparison.
      - name: Measure CPython
        env:
          CPYTHON: ${{ steps.cpython.outputs.python-path }}
        run: |
          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
            --python "${CPYTHON}" \
            --label cpython --no-psutil-stub \
            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"

      - name: Measure base
        continue-on-error: true # as above: the table falls back to head alone
        env:
          RUSTPYTHONPATH: ${{ runner.temp }}/base/Lib
        run: |
          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
            --python "${RUNNER_TEMP}/base/rustpython" \
            --label base \
            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"

      - name: Measure head
        env:
          RUSTPYTHONPATH: ${{ runner.temp }}/head/Lib
        run: |
          python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
            --python "${RUNNER_TEMP}/head/rustpython" \
            --label head \
            --benchmarks "${BENCHMARKS//[$'\n' ]/}" \
            --timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"

      - name: Render the comparison
        env:
          CPYTHON_VERSION: ${{ steps.cpython.outputs.python-version }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          python3 "${RUNNER_TEMP}/pyperformance-scripts/pr_diff.py" \
            --results-dir "${RUNNER_TEMP}/pyperf-results" \
            --cpython-version "CPython ${CPYTHON_VERSION}" \
            --md diff.md --json diff.json
          echo "${PR_NUMBER}" > pr_number.txt

      # A `pull_request` run triggered from a fork carries a read-only token no
      # matter what this workflow grants, so the comment is posted from
      # `pyperformance-comment.yaml`, which `workflow_run` runs in the base
      # repo's context. This artifact is the hand-off.
      - name: Upload the comparison
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: pyperformance-diff
          path: |
            diff.md
            diff.json
            pr_number.txt
          retention-days: 14
          if-no-files-found: ignore

      # A pull request opened from this repo can be commented on from here,
      # which puts the table on the PR in the same run instead of waiting for
      # the companion workflow. Forks fall through to that workflow instead.
      - name: Post or update the PR comment
        if: >-
          github.event_name == 'pull_request' &&
          github.event.pull_request.head.repo.full_name == github.repository
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('diff.md', 'utf8');
            const marker = '<!-- pyperformance-local-diff -->';

            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.payload.pull_request.number,
            });
            const existing = comments.find((c) => c.body && c.body.includes(marker));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.payload.pull_request.number,
                body,
              });
            }