From c6b629ead8c75d28c353f4cbfbecf646c512ecfc Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 1 Oct 2026 05:12:19 -0400 Subject: [PATCH] Gate CI on Linux and run macOS only when a change needs it - Run the full deterministic E2E suite on Linux in four shards on every pull request, plus the MCP conformance baseline, all against one shared build of fx. - Run macOS arm64 unit tests and platform E2E files only when a change touches macOS-specific code, build.zig, macOS signing, or the native SDK addon, or carries the ci:macos label. Otherwise the macOS arm64 check passes without a macOS runner. - Remove the four-platform Full CI workflow and the separate cross-target compile jobs. - Smoke-test the Linux arm64, macOS x86_64, and macOS arm64 binaries in Binary Size, and reuse the base build for later pushes to the same pull request. - Build the SDK WASM surfaces and Node-API addons once and share them across the SDK test jobs. - Run PGSO on pull requests only when the PGSO pipeline changes, and keep release PGSO evidence for 90 days. - Run the long-turn memory, libfx runtime, and live ACP jobs only on request. - Limit libfx publishing to an archive check, an install smoke test, and a registry byte comparison. - Add a warning when an E2E file passes only on retry. - Remove the unused Prepare Release workflow and document the new CI rules. --- .github/workflows/bench.yml | 12 +- .github/workflows/binary-size.yml | 35 +- .github/workflows/ci.yml | 488 ++++++++++++++----------- .github/workflows/full-ci.yml | 269 -------------- .github/workflows/macos.yml | 162 ++++++++ .github/workflows/pgso-macos-arm64.yml | 13 +- .github/workflows/prepare-release.yml | 316 ---------------- .github/workflows/publish-libfx.yml | 73 +--- .github/workflows/release.yml | 4 + AGENTS.md | 51 ++- CONTRIBUTING.md | 17 +- scripts/detect-macos-need.sh | 102 ++++++ scripts/pgso/README.md | 3 +- scripts/smoke-binary.sh | 37 ++ sdk/AGENTS.md | 2 +- tests/e2e/ci-run-files.sh | 35 ++ tests/e2e/ci-shards.test.ts | 26 +- tests/e2e/macos-platform-tests.json | 18 + 18 files changed, 761 insertions(+), 902 deletions(-) delete mode 100644 .github/workflows/full-ci.yml create mode 100644 .github/workflows/macos.yml delete mode 100644 .github/workflows/prepare-release.yml create mode 100755 scripts/detect-macos-need.sh create mode 100755 scripts/smoke-binary.sh create mode 100755 tests/e2e/ci-run-files.sh create mode 100644 tests/e2e/macos-platform-tests.json diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 2f141ddf5..cad624278 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -3,6 +3,14 @@ name: Benchmarks on: workflow_dispatch: inputs: + long_turn_memory: + description: Run the long-turn memory benchmark + type: boolean + default: false + libfx_runtime: + description: Run the libfx runtime benchmark + type: boolean + default: false signed_run: description: Successful main Release validation run to compare (no signing credentials used) type: string @@ -78,7 +86,7 @@ jobs: if-no-files-found: warn long-turn-memory: - if: inputs.signed_run == '' && inputs.pgso_control_artifact == '' && inputs.pgso_candidate_artifact == '' + if: github.event_name == 'workflow_dispatch' && inputs.long_turn_memory && inputs.signed_run == '' && inputs.pgso_control_artifact == '' && inputs.pgso_candidate_artifact == '' name: Long-turn memory runs-on: ubuntu-latest timeout-minutes: 15 @@ -111,7 +119,7 @@ jobs: if-no-files-found: warn libfx-ttft: - if: inputs.signed_run == '' && inputs.pgso_control_artifact == '' && inputs.pgso_candidate_artifact == '' + if: github.event_name == 'workflow_dispatch' && inputs.libfx_runtime && inputs.signed_run == '' && inputs.pgso_control_artifact == '' && inputs.pgso_candidate_artifact == '' name: libfx Runtime runs-on: ubuntu-latest steps: diff --git a/.github/workflows/binary-size.yml b/.github/workflows/binary-size.yml index 4b27de5de..59e45aeb1 100644 --- a/.github/workflows/binary-size.yml +++ b/.github/workflows/binary-size.yml @@ -83,11 +83,27 @@ jobs: fi cat "$evidence_dir/head-sections.txt" + # Linux x86_64 gets the same smoke test in CI. The other platforms run + # nowhere else on a pull request, so this is where a broken binary shows up. + - name: Smoke test native binary + if: matrix.name != 'linux-x86_64' + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + run: scripts/smoke-binary.sh "$RUNNER_TEMP/fx-binary-size-evidence/head-fx" + + - name: Restore base binary + id: base-cache + uses: actions/cache/restore@v4 + with: + path: ${{ runner.temp }}/fx-binary-size-base-bin + key: binary-size-base-v1-${{ matrix.target }}-zig-0.16.0-${{ env.BASE_SHA }} + - name: Build base binary + if: steps.base-cache.outputs.cache-hit != 'true' shell: bash run: | set -euo pipefail - evidence_dir="$RUNNER_TEMP/fx-binary-size-evidence" base_worktree="$RUNNER_TEMP/fx-binary-size-base" cleanup() { git worktree remove --force "$base_worktree" >/dev/null 2>&1 || true @@ -103,7 +119,22 @@ jobs: --cache-dir "$RUNNER_TEMP/fx-base-cache" \ --global-cache-dir "$RUNNER_TEMP/fx-global-cache" ) - cp "$base_worktree/zig-out/bin/fx" "$evidence_dir/base-fx" + mkdir -p "$RUNNER_TEMP/fx-binary-size-base-bin" + cp "$base_worktree/zig-out/bin/fx" "$RUNNER_TEMP/fx-binary-size-base-bin/fx" + + - name: Save base binary + if: steps.base-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/fx-binary-size-base-bin + key: binary-size-base-v1-${{ matrix.target }}-zig-0.16.0-${{ env.BASE_SHA }} + + - name: Measure base binary + shell: bash + run: | + set -euo pipefail + evidence_dir="$RUNNER_TEMP/fx-binary-size-evidence" + cp "$RUNNER_TEMP/fx-binary-size-base-bin/fx" "$evidence_dir/base-fx" file "$evidence_dir/base-fx" > "$evidence_dir/base-file.txt" cat "$evidence_dir/base-file.txt" grep -Eq "$EXPECTED_FILE" "$evidence_dir/base-file.txt" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75aff005b..836f7df8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,15 @@ name: CI +# Linux is the gate for every change. Behavior that can differ on macOS runs in +# .github/workflows/macos.yml only when a change needs it. + on: workflow_dispatch: + inputs: + acp_live: + description: Also run the full ACP suite with live model calls + type: boolean + default: false push: branches: [main] pull_request: @@ -38,6 +46,9 @@ jobs: - name: Audit public surface run: ./scripts/check-public-surface.sh + - name: Test signing and release routing + run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos -q + - name: Build run: zig build -Doptimize=${{ matrix.optimize }} @@ -60,9 +71,40 @@ jobs: MATRIX_RESULT: ${{ needs.zig_mode.result }} run: test "$MATRIX_RESULT" = success - sdk-node-wasm: - name: SDK (Node + WASM) + build: + name: Build fx (ReleaseSafe) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + steps: + - uses: actions/checkout@v4 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: "0.16.0" + cache-key: ci-build-ReleaseSafe + + - name: Build + run: zig build -Doptimize=ReleaseSafe + + - name: Smoke test fresh binary + run: scripts/smoke-binary.sh zig-out/bin/fx + + - name: Share the binary with E2E jobs + uses: actions/upload-artifact@v4 + with: + name: fx-linux-x86_64 + path: zig-out/bin/fx + if-no-files-found: error + retention-days: 1 + + sdk-wasm: + name: SDK build (WASM) runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -71,15 +113,77 @@ jobs: with: version: "0.16.0" + - name: Build WASM surfaces (ReleaseSmall) + run: | + zig build fx-core-wasm -Dwasm-surface=core -Doptimize=ReleaseSmall + zig build fx-term-wasm -Dwasm-surface=term -Doptimize=ReleaseSmall + + - name: Share WASM surfaces with SDK jobs + uses: actions/upload-artifact@v4 + with: + name: sdk-wasm + path: | + zig-out/bin/fx-core.wasm + zig-out/bin/fx-term.wasm + if-no-files-found: error + retention-days: 1 + + sdk-napi: + name: SDK build (N-API, ${{ matrix.platform }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + target_flags: -Dtarget=x86_64-linux-gnu.2.34 -Dcpu=baseline + - platform: linux-arm64 + target_flags: -Dtarget=aarch64-linux-gnu.2.34 -Dcpu=baseline + - platform: darwin-x64 + target_flags: -Dtarget=x86_64-macos -Dcpu=baseline + - platform: darwin-arm64 + target_flags: -Dtarget=aarch64-macos -Dcpu=baseline + steps: + - uses: actions/checkout@v4 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: "0.16.0" + cache-key: ci-sdk-napi-${{ matrix.platform }} + + - name: Build N-API addon (ReleaseSafe) + run: | + zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe ${{ matrix.target_flags }} + mkdir -p native-addons + cp zig-out/lib/libfx.node native-addons/libfx.${{ matrix.platform }}.node + + - name: Share the addon with SDK jobs + uses: actions/upload-artifact@v4 + with: + name: sdk-napi-${{ matrix.platform }} + path: native-addons/libfx.${{ matrix.platform }}.node + if-no-files-found: error + retention-days: 1 + + sdk-node-wasm: + name: SDK (Node + WASM) + needs: sdk-wasm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node uses: actions/setup-node@v4 with: node-version: "24" - - name: Build WASM surfaces (ReleaseSmall) - run: | - zig build -Dwasm-surface=core -Doptimize=ReleaseSmall - zig build -Dwasm-surface=term -Doptimize=ReleaseSmall + - name: Download WASM surfaces + uses: actions/download-artifact@v4 + with: + name: sdk-wasm + path: zig-out/bin - name: Install terminal test dependencies run: npm ci --prefix sdk/node @@ -96,22 +200,26 @@ jobs: sdk-node-napi: name: SDK (Node + N-API) + needs: sdk-napi runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - - name: Setup Node uses: actions/setup-node@v4 with: node-version: "24" - - name: Build N-API addon (ReleaseSafe) - run: zig build -Dnapi-surface=core -Doptimize=ReleaseSafe + - name: Download Linux addon + uses: actions/download-artifact@v4 + with: + name: sdk-napi-linux-x64 + path: native-addons + + - name: Install the addon + run: | + mkdir -p zig-out/lib + cp native-addons/libfx.linux-x64.node zig-out/lib/libfx.node - name: Test Node + N-API run: npm run --prefix sdk test:node-napi @@ -126,15 +234,11 @@ jobs: sdk-bun: name: SDK (Bun + N-API/WASM) + needs: [sdk-wasm, sdk-napi] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - - name: Setup Node uses: actions/setup-node@v4 with: @@ -145,11 +249,23 @@ jobs: with: bun-version: "1.4.2" - - name: Build SDK artifacts + - name: Download WASM surfaces + uses: actions/download-artifact@v4 + with: + name: sdk-wasm + path: zig-out/bin + + - name: Download native addons + uses: actions/download-artifact@v4 + with: + pattern: sdk-napi-* + path: native-addons + merge-multiple: true + + - name: Install the Linux addon run: | - zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe -Dtarget=x86_64-linux-gnu.2.34 -Dcpu=baseline - zig build -Dwasm-surface=core -Doptimize=ReleaseSmall - zig build -Dwasm-surface=term -Doptimize=ReleaseSmall + mkdir -p zig-out/lib + cp native-addons/libfx.linux-x64.node zig-out/lib/libfx.node - name: Install terminal test dependencies run: npm ci --prefix sdk/node @@ -189,20 +305,6 @@ jobs: node sdk/tests/test-host-tool-cancel.mjs native ' - - name: Build foreign addons for package qualification - run: | - mkdir -p native-addons - cp zig-out/lib/libfx.node native-addons/libfx.linux-x64.node - prefix="$(mktemp -d "$RUNNER_TEMP/libfx-linux-arm64.XXXXXX")" - zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe -Dtarget=aarch64-linux-gnu.2.34 -Dcpu=baseline --prefix "$prefix" - cp "$prefix/lib/libfx.node" native-addons/libfx.linux-arm64.node - prefix="$(mktemp -d "$RUNNER_TEMP/libfx-darwin-x64.XXXXXX")" - zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe -Dtarget=x86_64-macos -Dcpu=baseline --prefix "$prefix" - cp "$prefix/lib/libfx.node" native-addons/libfx.darwin-x64.node - prefix="$(mktemp -d "$RUNNER_TEMP/libfx-darwin-arm64.XXXXXX")" - zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe -Dtarget=aarch64-macos -Dcpu=baseline --prefix "$prefix" - cp "$prefix/lib/libfx.node" native-addons/libfx.darwin-arm64.node - - name: Test Node package and backend diagnostics run: | node --experimental-wasm-jspi sdk/tests/test-backend-info.mjs @@ -250,15 +352,11 @@ jobs: sdk-browser-wasm: name: SDK (Browser + WASM) + needs: sdk-wasm runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - - name: Setup Node uses: actions/setup-node@v4 with: @@ -268,10 +366,11 @@ jobs: id: setup-chrome uses: browser-actions/setup-chrome@v2 - - name: Build WASM surfaces (ReleaseSmall) - run: | - zig build -Dwasm-surface=core -Doptimize=ReleaseSmall - zig build -Dwasm-surface=term -Doptimize=ReleaseSmall + - name: Download WASM surfaces + uses: actions/download-artifact@v4 + with: + name: sdk-wasm + path: zig-out/bin - name: Test Browser + WASM env: @@ -279,23 +378,136 @@ jobs: run: npm run --prefix sdk test:browser-wasm e2e_mode: - name: E2E (deterministic, ${{ matrix.optimize }}) - runs-on: ubuntu-latest + name: E2E (deterministic, shard ${{ matrix.shard.label }}) + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 60 strategy: fail-fast: false matrix: - optimize: [ReleaseSafe] + shard: + - index: 0 + label: "1/4" + - index: 1 + label: "2/4" + - index: 2 + label: "3/4" + - index: 3 + label: "4/4" + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + FX_REQUIRE_TMUX: "1" + SHARD_INDEX: ${{ matrix.shard.index }} + SHARD_COUNT: 4 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download fx + uses: actions/download-artifact@v4 + with: + name: fx-linux-x86_64 + path: zig-out/bin + + - name: Restore executable mode + run: chmod +x zig-out/bin/fx + + # Some E2E files build their own Zig fixtures. - name: Setup Zig uses: mlugg/setup-zig@v2 with: version: "0.16.0" - cache-key: ci-e2e-${{ matrix.optimize }} + cache-key: ci-e2e-fixtures - - name: Build - run: zig build -Doptimize=${{ matrix.optimize }} + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" + + - name: Install E2E system dependencies + timeout-minutes: 17 + run: | + missing=() + command -v tmux >/dev/null 2>&1 || missing+=(tmux) + command -v zsh >/dev/null 2>&1 || missing+=(zsh) + if [ "${#missing[@]}" -gt 0 ]; then + retry() { + local attempt + for attempt in 1 2; do + if "$@"; then + return 0 + fi + if [ "$attempt" -lt 2 ]; then + printf 'Dependency install attempt %s failed; retrying in 5s\n' "$attempt" >&2 + sleep 5 + fi + done + return 1 + } + # The Azure mirror can stall after APT has already fetched release + # metadata from the direct Ubuntu fallback. Prefer that fallback + # for the whole update when the runner's mirror list provides it. + if [ -f /etc/apt/apt-mirrors.txt ] && + grep -Eq '^https?://(archive|ports)\.ubuntu\.com/' /etc/apt/apt-mirrors.txt + then + sudo sed -i '/:\/\/azure\..*ubuntu\.com\//d' /etc/apt/apt-mirrors.txt + fi + retry sudo timeout --kill-after=15s 4m apt-get \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + update + retry sudo timeout --kill-after=15s 3m apt-get \ + -o Dpkg::Use-Pty=0 \ + install -y "${missing[@]}" + fi + printf 'set-option -g history-limit 100000\n' > "$HOME/.tmux.conf" + tmux -V + zsh --version + + - name: Install E2E dependencies + run: bun install --frozen-lockfile + working-directory: tests/e2e + + - name: Run deterministic E2E shard + working-directory: tests/e2e + run: | + selected=() + selection_file="$RUNNER_TEMP/fx-e2e-shard-${SHARD_INDEX}.txt" + bun ci-shards.ts \ + --shard-count "$SHARD_COUNT" \ + --shard-index "$SHARD_INDEX" > "$selection_file" + while IFS= read -r test_file; do + selected+=("$test_file") + done < "$selection_file" + + test "${#selected[@]}" -gt 0 + printf 'Running E2E shard %s/%s (%s files):\n' "$((SHARD_INDEX + 1))" "$SHARD_COUNT" "${#selected[@]}" + printf ' %s\n' "${selected[@]}" + ./ci-run-files.sh "${selected[@]}" + + e2e_conformance: + name: E2E (MCP conformance) + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + FX_SOUND: "0" + steps: + - uses: actions/checkout@v4 + + - name: Download fx + uses: actions/download-artifact@v4 + with: + name: fx-linux-x86_64 + path: zig-out/bin + + - name: Restore executable mode + run: chmod +x zig-out/bin/fx - name: Setup Bun uses: oven-sh/setup-bun@v2 @@ -307,161 +519,31 @@ jobs: with: node-version: "22" - - name: Install e2e deps - run: bun install + - name: Install E2E dependencies + run: bun install --frozen-lockfile working-directory: tests/e2e - name: Install isolated MCP conformance package run: npm ci working-directory: tests/e2e/conformance - - name: Install tmux - run: | - if ! command -v tmux >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y tmux - fi - - - name: Run CLI tests - run: bun test cli.test.ts - working-directory: tests/e2e - - - name: Run ACP tests (no-key subset) - run: bun test acp.test.ts - working-directory: tests/e2e - - - name: Run legacy stdio MCP tool lifecycle test - run: bun test gateway-stream-lifecycle.test.ts --test-name-pattern "selected dynamic MCP review cautions with zero sends and clears exactly once" - working-directory: tests/e2e - - - name: Run stdio MCP dispatcher lifecycle tests - run: bun test mcp-stdio.test.ts - working-directory: tests/e2e - - - name: Run modern MCP Streamable HTTP tests - run: bun test mcp-http.test.ts - working-directory: tests/e2e - - - name: Run legacy MCP remote compatibility tests - run: bun test mcp-legacy-remote.test.ts - working-directory: tests/e2e - - name: Run isolated MCP conformance baseline run: npm test working-directory: tests/e2e/conformance - env: - FX_SOUND: "0" - - - name: Run persistence tests - run: bun test config-persistence.test.ts session-recovery.test.ts prompt-history.test.ts - working-directory: tests/e2e - - - name: Run gateway lifecycle recovery regressions - run: bun test ./gateway-stream-lifecycle.test.ts --test-name-pattern "saved malformed recovery resumes without re-executing the historical call|OS filesystem access denial reaches the next gateway request" - working-directory: tests/e2e - - - name: Run lean Auto Mode reliability contract - run: bun test auto-mode-reliability.test.ts - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" - - - name: Run Vision route and headless SIGINT regressions - run: | - bun test vision-route-fake-gateway.test.ts - bun test ./tui-command-permissions.test.ts --test-name-pattern "fx ask SIGINT during classifier wait terminates before decision or execution" - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - - - name: Run TUI input line deletion regression - run: | - tmux -V - bun test tui-input-line-delete.test.ts - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" - - - name: Run multiline input navigation TUI tests - run: | - tmux -V - bun test tui-input-navigation.test.ts - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" - - - name: Run valid question recovery TUI regression - run: | - tmux -V - bun test ./tui-decision-prompts.test.ts --test-name-pattern "valid serialized ask control with matching identity opens the question and continues" - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" - - - name: Run streamed lifecycle, direct-write, and command scrollback regressions - run: | - tmux -V - bun test tui-gateway-stream-lifecycle.test.ts tui-direct-write-audit.test.ts - bun test ./tui-command-permissions.test.ts --test-name-pattern "TUI preserves completed transcript when a later auto-approved command starts|TUI isolates approved foreground commands from terminal ownership" - working-directory: tests/e2e - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" e2e: name: E2E (deterministic) if: ${{ always() }} - needs: e2e_mode + needs: [e2e_mode, e2e_conformance] runs-on: ubuntu-latest steps: - - name: Require ReleaseSafe + - name: Require every shard and the conformance baseline env: - MATRIX_RESULT: ${{ needs.e2e_mode.result }} - run: test "$MATRIX_RESULT" = success - - cross-target: - name: Cross-target Compile (${{ matrix.name }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: linux-x86_64 - target: x86_64-linux - - name: linux-aarch64 - target: aarch64-linux - - name: macos-x86_64 - target: x86_64-macos - - name: macos-aarch64 - target: aarch64-macos - steps: - - uses: actions/checkout@v4 - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - cache-key: ci-cross-target-${{ matrix.name }} - - - name: Build target - run: zig build -Doptimize=ReleaseSafe -Dtarget=${{ matrix.target }} - - - name: Report binary size + SHARD_RESULT: ${{ needs.e2e_mode.result }} + CONFORMANCE_RESULT: ${{ needs.e2e_conformance.result }} run: | - bytes=$(stat -c%s zig-out/bin/fx) - mib=$(awk -v b="$bytes" 'BEGIN { printf "%.2f", b / 1048576 }') - echo "${{ matrix.name }}: $bytes bytes ($mib MiB)" - echo "${{ matrix.name }}: $bytes bytes ($mib MiB)" >> "$GITHUB_STEP_SUMMARY" + test "$SHARD_RESULT" = success + test "$CONFORMANCE_RESULT" = success lint-scripts: name: Shellcheck @@ -474,18 +556,17 @@ jobs: scripts=( benchmarks/startup.sh scripts/check-public-surface.sh + scripts/detect-macos-need.sh + scripts/smoke-binary.sh + tests/e2e/ci-run-files.sh ) bash -n "${scripts[@]}" shellcheck "${scripts[@]}" - e2e_acp_full_mode: - name: E2E ACP (full, ${{ matrix.optimize }}) + e2e_acp_live: + name: E2E ACP (live models) + if: github.event_name == 'workflow_dispatch' && inputs.acp_live runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - strategy: - fail-fast: false - matrix: - optimize: [ReleaseSafe] steps: - uses: actions/checkout@v4 @@ -493,10 +574,10 @@ jobs: uses: mlugg/setup-zig@v2 with: version: "0.16.0" - cache-key: ci-acp-full-${{ matrix.optimize }} + cache-key: ci-acp-full-ReleaseSafe - name: Build - run: zig build -Doptimize=${{ matrix.optimize }} + run: zig build -Doptimize=ReleaseSafe - name: Setup Bun uses: oven-sh/setup-bun@v2 @@ -512,14 +593,3 @@ jobs: working-directory: tests/e2e env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} - - e2e-acp-full: - name: E2E ACP (full) - if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - needs: e2e_acp_full_mode - runs-on: ubuntu-latest - steps: - - name: Require ReleaseSafe - env: - MATRIX_RESULT: ${{ needs.e2e_acp_full_mode.result }} - run: test "$MATRIX_RESULT" = success diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml deleted file mode 100644 index d89ba4ed0..000000000 --- a/.github/workflows/full-ci.yml +++ /dev/null @@ -1,269 +0,0 @@ -name: Full CI - -on: - workflow_dispatch: - push: - branches-ignore: - - main - -concurrency: - group: full-ci-${{ github.ref }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - -jobs: - native: - name: Native checks (${{ matrix.optimize }}, ${{ matrix.platform.name }}) - runs-on: ${{ matrix.platform.runner }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - platform: - - name: linux-x86_64 - runner: ubuntu-24.04 - - name: linux-aarch64 - runner: ubuntu-24.04-arm - - name: macos-x86_64 - runner: macos-15-intel - - name: macos-aarch64 - runner: macos-15 - optimize: [ReleaseSafe] - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - steps: - - uses: actions/checkout@v4 - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - cache-key: full-ci-native-${{ matrix.platform.name }}-${{ matrix.optimize }} - - - name: Check formatting - run: zig fmt --check src/ - - - name: Audit public surface - run: ./scripts/check-public-surface.sh - - - name: Test signing and release routing - run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos -q - - - name: Build - run: zig build -Doptimize=${{ matrix.optimize }} - - - name: Run unit tests - run: zig build test -Doptimize=${{ matrix.optimize }} - - - name: Smoke test fresh binary - shell: bash - run: | - help_stdout="$(mktemp)" - help_stderr="$(mktemp)" - status_stdout="$(mktemp)" - status_stderr="$(mktemp)" - trap 'rm -f "$help_stdout" "$help_stderr" "$status_stdout" "$status_stderr"' EXIT - - ./zig-out/bin/fx help >"$help_stdout" 2>"$help_stderr" - test -s "$help_stdout" - test ! -s "$help_stderr" - - ./zig-out/bin/fx status --json >"$status_stdout" 2>"$status_stderr" - test -s "$status_stdout" - test ! -s "$status_stderr" - - e2e: - name: E2E (${{ matrix.optimize }}, ${{ matrix.platform.name }}, shard ${{ matrix.shard.label }}) - runs-on: ${{ matrix.platform.runner }} - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - platform: - - name: linux-x86_64 - runner: ubuntu-24.04 - zig_tarball: zig-x86_64-linux - - name: linux-aarch64 - runner: ubuntu-24.04-arm - zig_tarball: zig-aarch64-linux - - name: macos-x86_64 - runner: macos-15-intel - zig_tarball: zig-x86_64-macos - - name: macos-aarch64 - runner: macos-15 - zig_tarball: zig-aarch64-macos - optimize: [ReleaseSafe] - shard: - - index: 0 - shard_count: 4 - label: "1/4" - - index: 1 - shard_count: 4 - label: "2/4" - - index: 2 - shard_count: 4 - label: "3/4" - - index: 3 - shard_count: 4 - label: "4/4" - env: - AI_GATEWAY_API_KEY: "" - VERCEL_OIDC_TOKEN: "" - FX_REQUIRE_TMUX: "1" - SHARD_INDEX: ${{ matrix.shard.index }} - SHARD_COUNT: ${{ matrix.shard.shard_count }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Zig - uses: mlugg/setup-zig@v2 - with: - version: "0.16.0" - use-cache: false - - - name: Restore native Zig cache - uses: actions/cache/restore@v4 - with: - path: .zig-cache - key: full-ci-e2e-read-only-${{ matrix.platform.name }}-${{ matrix.optimize }}-${{ github.run_id }}-${{ github.run_attempt }} - restore-keys: | - setup-zig-cache-v2-native-${{ matrix.platform.zig_tarball }}-0.16.0-full-ci-native-${{ matrix.platform.name }}-${{ matrix.optimize }}- - - - name: Build - run: zig build -Doptimize=${{ matrix.optimize }} - - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: "1.3.14" - - - name: Install E2E system dependencies - shell: bash - timeout-minutes: 17 - run: | - if [ "$RUNNER_OS" = "Linux" ]; then - missing=() - command -v tmux >/dev/null 2>&1 || missing+=(tmux) - command -v zsh >/dev/null 2>&1 || missing+=(zsh) - if [ "${#missing[@]}" -gt 0 ]; then - retry() { - local attempt - for attempt in 1 2; do - if "$@"; then - return 0 - fi - if [ "$attempt" -lt 2 ]; then - printf 'Dependency install attempt %s failed; retrying in 5s\n' "$attempt" >&2 - sleep 5 - fi - done - return 1 - } - # The Azure mirror can stall after APT has already fetched release - # metadata from the direct Ubuntu fallback. Prefer that fallback - # for the whole update when the runner's mirror list provides it. - if [ -f /etc/apt/apt-mirrors.txt ] && - grep -Eq '^https?://(archive|ports)\.ubuntu\.com/' /etc/apt/apt-mirrors.txt - then - sudo sed -i '/:\/\/azure\..*ubuntu\.com\//d' /etc/apt/apt-mirrors.txt - fi - retry sudo timeout --kill-after=15s 4m apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=30 \ - -o Acquire::https::Timeout=30 \ - update - retry sudo timeout --kill-after=15s 3m apt-get \ - -o Dpkg::Use-Pty=0 \ - install -y "${missing[@]}" - fi - elif ! command -v tmux >/dev/null 2>&1; then - brew install tmux - fi - printf 'set-option -g history-limit 100000\n' > "$HOME/.tmux.conf" - tmux -V - zsh --version - - - name: Install E2E dependencies - run: bun install --frozen-lockfile - working-directory: tests/e2e - - - name: Run deterministic E2E shard - shell: bash - working-directory: tests/e2e - run: | - selected=() - selection_file="$RUNNER_TEMP/fx-e2e-shard-${SHARD_INDEX}.txt" - bun ci-shards.ts \ - --shard-count "$SHARD_COUNT" \ - --shard-index "$SHARD_INDEX" > "$selection_file" - while IFS= read -r test_file; do - selected+=("$test_file") - done < "$selection_file" - - test "${#selected[@]}" -gt 0 - printf 'Running E2E shard %s/%s (%s files):\n' "$((SHARD_INDEX + 1))" "$SHARD_COUNT" "${#selected[@]}" - printf ' %s\n' "${selected[@]}" - - shard_status=0 - test_number=0 - for test_file in "${selected[@]}"; do - tmux_tmp_dir="$RUNNER_TEMP/fx-e2e-tmux-${SHARD_INDEX}-${test_number}" - mkdir -p "$tmux_tmp_dir" - if ! TMUX_TMPDIR="$tmux_tmp_dir" bun test --max-concurrency 1 "./$test_file"; then - printf 'Retrying failed E2E file after resetting tmux: %s\n' "$test_file" - TMUX_TMPDIR="$tmux_tmp_dir" tmux kill-server 2>/dev/null || true - if ! TMUX_TMPDIR="$tmux_tmp_dir" bun test --max-concurrency 1 "./$test_file"; then - shard_status=1 - fi - fi - TMUX_TMPDIR="$tmux_tmp_dir" tmux kill-server 2>/dev/null || true - test_number=$((test_number + 1)) - done - exit "$shard_status" - - full-suite: - name: Full suite (${{ matrix.name }}) - if: ${{ always() }} - needs: - - native - - e2e - runs-on: ubuntu-24.04 - timeout-minutes: 5 - strategy: - fail-fast: false - matrix: - name: - - linux-x86_64 - - linux-aarch64 - - macos-x86_64 - - macos-aarch64 - steps: - - name: Verify native and E2E matrices - env: - GH_TOKEN: ${{ github.token }} - TARGET: ${{ matrix.name }} - run: | - jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100")" - jq -e --arg target "$TARGET" ' - ([ - "Native checks (ReleaseSafe, " + $target + ")", - (range(1; 5) as $shard - | "E2E (ReleaseSafe, " + $target + ", shard " + ($shard | tostring) + "/4)") - ] | sort) as $expected - | - [ - .jobs[] - | select( - (.name == ("Native checks (ReleaseSafe, " + $target + ")")) or - (.name | startswith("E2E (ReleaseSafe, " + $target + ", shard ")) - ) - ] as $selected - | (($selected | map(.name) | sort) == $expected and - all($selected[]; .conclusion == "success")) - ' <<< "$jobs" diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..9603608b6 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,162 @@ +name: macOS + +# Linux CI covers everything that behaves the same on every platform. These +# checks run only for changes that can behave differently on macOS; see +# scripts/detect-macos-need.sh. Add the ci:macos label or dispatch this +# workflow to request them for any other change. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + sdk_native: + description: Also build and load the libfx Node-API addon + type: boolean + default: false + +concurrency: + group: macos-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + detect: + name: Detect macOS need + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + needed: ${{ steps.detect.outputs.needed }} + sdk_native: ${{ steps.detect.outputs.sdk_native }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Decide whether this change needs macOS + id: detect + env: + MACOS_REQUESTED: ${{ github.event_name == 'workflow_dispatch' && 'manual dispatch' || (contains(github.event.pull_request.labels.*.name, 'ci:macos') && 'ci:macos label' || '') }} + SDK_NATIVE_REQUESTED: ${{ github.event_name == 'workflow_dispatch' && inputs.sdk_native || false }} + run: | + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + scripts/detect-macos-need.sh HEAD HEAD + else + # The pull request merge commit's first parent is the base branch. + scripts/detect-macos-need.sh HEAD^1 HEAD + fi + + unit: + name: macOS arm64 unit tests + needs: detect + if: needs.detect.outputs.needed == 'true' + runs-on: macos-15 + timeout-minutes: 60 + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + steps: + - uses: actions/checkout@v4 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: "0.16.0" + cache-key: macos-unit-ReleaseSafe + + - name: Test signing and release routing + run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos -q + + - name: Run unit tests + run: zig build test -Doptimize=ReleaseSafe + + e2e: + name: macOS arm64 platform E2E + needs: detect + if: needs.detect.outputs.needed == 'true' + runs-on: macos-15 + timeout-minutes: 90 + env: + AI_GATEWAY_API_KEY: "" + VERCEL_OIDC_TOKEN: "" + FX_REQUIRE_TMUX: "1" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Zig + uses: mlugg/setup-zig@v2 + with: + version: "0.16.0" + cache-key: macos-e2e-ReleaseSafe + + - name: Build + run: zig build -Doptimize=ReleaseSafe + + - name: Smoke test fresh binary + run: scripts/smoke-binary.sh zig-out/bin/fx + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.14" + + - name: Install E2E system dependencies + run: | + command -v tmux >/dev/null 2>&1 || brew install tmux + printf 'set-option -g history-limit 100000\n' > "$HOME/.tmux.conf" + tmux -V + zsh --version + + - name: Install E2E dependencies + run: bun install --frozen-lockfile + working-directory: tests/e2e + + - name: Run macOS platform E2E files + shell: bash + run: | + files=() + while IFS= read -r test_file; do + files+=("$test_file") + done < <(jq -r '.[]' tests/e2e/macos-platform-tests.json) + test "${#files[@]}" -gt 0 + printf 'Running %s macOS platform E2E files:\n' "${#files[@]}" + printf ' %s\n' "${files[@]}" + tests/e2e/ci-run-files.sh "${files[@]}" + + - name: Setup Node + if: needs.detect.outputs.sdk_native == 'true' + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Build and load the native SDK addon + if: needs.detect.outputs.sdk_native == 'true' + run: | + zig build libfx-napi -Dnapi-surface=core -Doptimize=ReleaseSafe + npm run --prefix sdk test:node-napi + + macos: + name: macOS arm64 + if: ${{ always() }} + needs: [detect, unit, e2e] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require macOS results when the change needs them + env: + DETECT_RESULT: ${{ needs.detect.result }} + NEEDED: ${{ needs.detect.outputs.needed }} + UNIT_RESULT: ${{ needs.unit.result }} + E2E_RESULT: ${{ needs.e2e.result }} + run: | + test "$DETECT_RESULT" = success + if [ "$NEEDED" != true ]; then + echo "No macOS-specific change; nothing to require." + exit 0 + fi + test "$UNIT_RESULT" = success + test "$E2E_RESULT" = success diff --git a/.github/workflows/pgso-macos-arm64.yml b/.github/workflows/pgso-macos-arm64.yml index cac61c800..f151dcbf7 100644 --- a/.github/workflows/pgso-macos-arm64.yml +++ b/.github/workflows/pgso-macos-arm64.yml @@ -1,7 +1,15 @@ name: macOS arm64 PGSO candidate +# Pull requests run PGSO only when the PGSO pipeline itself changes. Dispatch +# this workflow on any branch to qualify other changes; the stable release +# always runs it. on: workflow_call: + inputs: + evidence-retention-days: + description: Days to keep the qualified aggregate evidence + type: number + default: 14 outputs: artifact-id: description: Qualified PGSO evidence artifact ID @@ -9,11 +17,10 @@ on: workflow_dispatch: pull_request: paths: - - build.zig - scripts/pgso/** + - "!scripts/pgso/corpus.json" - .github/actions/setup-pgso/** - .github/workflows/pgso-macos-arm64.yml - - .github/workflows/release.yml concurrency: group: pgso-macos-arm64-${{ github.ref }} @@ -488,4 +495,4 @@ jobs: overwrite: true path: ${{ runner.temp }}/fx-pgso-aggregate if-no-files-found: error - retention-days: 14 + retention-days: ${{ inputs.evidence-retention-days || 14 }} diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml deleted file mode 100644 index 1dda8b145..000000000 --- a/.github/workflows/prepare-release.yml +++ /dev/null @@ -1,316 +0,0 @@ -name: Prepare Release - -on: - workflow_dispatch: - inputs: - bump: - description: "Version bump type" - required: true - type: choice - options: - - patch - - minor - - major - -permissions: - actions: write - contents: write - pull-requests: write - -concurrency: - group: prepare-release - cancel-in-progress: false - -jobs: - prepare: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Compute new version - id: version - run: | - CURRENT=$(grep 'pub const version' src/main.zig | head -1 | sed 's/.*"\(.*\)".*/\1/') - IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT" - case "${{ inputs.bump }}" in - major) MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 ;; - minor) MINOR=$((MINOR + 1)); PATCH=0 ;; - patch) PATCH=$((PATCH + 1)) ;; - esac - NEW="${MAJOR}.${MINOR}.${PATCH}" - echo "current=$CURRENT" >> "$GITHUB_OUTPUT" - echo "new=$NEW" >> "$GITHUB_OUTPUT" - echo "Bumping $CURRENT -> $NEW (${{ inputs.bump }})" - - - name: Collect changes since last tag - id: changes - run: | - PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null || git rev-list --max-parents=0 HEAD) - echo "prev_tag=$PREV_TAG" >> "$GITHUB_OUTPUT" - - git diff --stat "${PREV_TAG}..HEAD" > /tmp/diff-stat.txt - - git diff "${PREV_TAG}..HEAD" -- src/ > /tmp/full-diff.txt - DIFF_SIZE=$(wc -c < /tmp/full-diff.txt | tr -d ' ') - if [ "$DIFF_SIZE" -gt 81920 ]; then - head -c 81920 /tmp/full-diff.txt > /tmp/diff.txt - printf '\n\n[diff truncated at 80KB — %s bytes total]' "$DIFF_SIZE" >> /tmp/diff.txt - else - cp /tmp/full-diff.txt /tmp/diff.txt - fi - - git log "${PREV_TAG}..HEAD" --oneline > /tmp/log.txt - echo "Diff: ${DIFF_SIZE} bytes, $(wc -l < /tmp/diff.txt) lines" - echo "Commits: $(wc -l < /tmp/log.txt)" - - - name: Generate changelog with AI - env: - AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} - run: | - NEW_VERSION="${{ steps.version.outputs.new }}" - PREV_TAG="${{ steps.changes.outputs.prev_tag }}" - - cat > /tmp/system-prompt.txt <<'SYSPROMPT' - You write changelogs for fx, an open-source AI-powered CLI tool written in Zig. - - You will receive the actual code diff since the last release, a diff stat summary, and a commit log. Treat the commit log as private research context. - - Rules: - - Base your changelog ONLY on what the diff actually shows. Do not trust commit messages or PR descriptions as authoritative — they go stale. The diff is the source of truth. - - Write public, user-facing product notes. Describe observable behavior and outcomes, not how the work was implemented or delivered. - - Always spell the product name fx. Preserve different casing only for exact code identifiers such as FX_MODEL. - - Group changes under ### Breaking Changes, ### New Features, ### Improvements, ### Bug Fixes, and ### Security as appropriate. Omit empty sections. - - Bold a short feature or fix name, then describe the user-visible change after a colon. - - Do not include pull request or issue numbers, links to trackers, commit hashes, contributor names, author attribution, or a Contributors section. - - Do not include internal details such as repository moves, website or marketing work, CDN layout, CI workflows, tests or fixtures, branch history, or implementation-only refactors. Translate relevant work into its public user outcome or omit it. - - Do not force every commit into the changelog. Omit changes without a public user outcome. - - Output ONLY the changelog body (the content that goes between the release markers). Do not include the ## version heading, do not include the markers, do not include any preamble or explanation. - - Do not use emojis. - SYSPROMPT - - cat > /tmp/user-header.txt < /tmp/request.json - - if ! jq -e ' - (.prompt | type == "array") and - (.prompt[0].role == "system") and - (.prompt[0].content | type == "string") and - (.prompt[1].role == "user") and - (.prompt[1].content | type == "array") and - (.prompt[1].content | length == 1) and - (.prompt[1].content[0].type == "text") and - (.prompt[1].content[0].text | type == "string") - ' /tmp/request.json >/dev/null; then - echo "::error::Generated AI Gateway payload does not match the expected prompt schema" - exit 1 - fi - - echo "Request payload: $(wc -c < /tmp/request.json) bytes" - - HTTP_STATUS=$(curl -sS -N -X POST "https://ai-gateway.vercel.sh/v4/ai/language-model" \ - -H "Authorization: Bearer ${AI_GATEWAY_API_KEY}" \ - -H "Content-Type: application/json" \ - -H "ai-gateway-protocol-version: 0.0.1" \ - -H "ai-language-model-specification-version: 4" \ - -H "ai-language-model-id: anthropic/claude-opus-4.7" \ - -H "ai-language-model-streaming: true" \ - -H "HTTP-Referer: https://github.com/vercel-labs/fx" \ - -H "X-Title: fx" \ - -d @/tmp/request.json -o /tmp/sse-response.txt -w '%{http_code}') - - if [ "$HTTP_STATUS" -ne 200 ]; then - echo "::error::AI Gateway returned HTTP ${HTTP_STATUS}" - head -c 2000 /tmp/sse-response.txt - exit 1 - fi - - grep '^data: ' /tmp/sse-response.txt \ - | sed 's/^data: //' \ - | grep -v '^\[DONE\]$' \ - | jq -rj 'select(.type == "text-delta") | .delta' \ - > /tmp/changelog-body.md - - LINES=$(wc -l < /tmp/changelog-body.md | tr -d ' ') - echo "Generated changelog: ${LINES} lines" - if [ "$LINES" -lt 2 ]; then - echo "::error::AI generated an empty or too-short changelog" - echo "Raw SSE response (first 2000 chars):" - head -c 2000 /tmp/sse-response.txt - exit 1 - fi - cat /tmp/changelog-body.md - - python3 <<'PYEOF' - import re - from pathlib import Path - - body = Path("/tmp/changelog-body.md").read_text() - forbidden = { - "contributor attribution": r"(?im)^###\s+contributors\b|\bcontributors?\b|\bauthors?\b", - "tracker references": r"(?i)#[0-9]+\b|\bPRs?\s*#?[0-9]+\b|\bpull requests?\b|github\.com/[^\s)]+/(?:pull|issues|commit)/", - "internal web or delivery details": r"(?i)\b(?:fx-web|website|marketing|CDN|deployments?|repository moves?|repository split|repo(?:sitory)? migration)\b|vercel-labs/[A-Za-z0-9_.-]+", - "internal engineering details": r"(?i)\b(?:CI|workflows?|tests?|fixtures?|branch history|Git history|commit hashes?|implementation-only refactors?|refactors?)\b", - } - errors = [label for label, pattern in forbidden.items() if re.search(pattern, body)] - if re.search(r"(?\n", "") - content = content.replace("\n", "") - - entry = f"## {new_version}\n\n\n{body}\n\n" - - marker = "# fx\n\n" - idx = content.find(marker) - if idx >= 0: - pos = idx + len(marker) - content = content[:pos] + entry + "\n" + content[pos:] - else: - content = entry + "\n" + content - - with open("CHANGELOG.md", "w") as f: - f.write(content) - - print(f"Changelog updated with {new_version} entry") - PYEOF - - - name: Create pull request - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - NEW="${{ steps.version.outputs.new }}" - BRANCH="prepare-v${NEW}" - REF="refs/heads/${BRANCH}" - - EXISTING_PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') - if [ -n "$EXISTING_PR" ]; then - echo "Closing existing PR #${EXISTING_PR} for ${BRANCH}" - gh pr close "$EXISTING_PR" --comment "Superseded by a new prepare-release run." - fi - - gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/heads/${BRANCH}" >/dev/null 2>&1 || true - gh api -X POST "repos/${GITHUB_REPOSITORY}/git/refs" \ - -f ref="$REF" \ - -f sha="$GITHUB_SHA" \ - >/dev/null - - jq -n \ - --arg repository "$GITHUB_REPOSITORY" \ - --arg branch "$BRANCH" \ - --arg expected_head "$GITHUB_SHA" \ - --arg headline "Prepare v${NEW} release" \ - --arg main_zig "$(base64 -w 0 src/main.zig)" \ - --arg readme "$(base64 -w 0 README.md)" \ - --arg changelog "$(base64 -w 0 CHANGELOG.md)" \ - '{ - query: "mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }", - variables: { - input: { - branch: { - repositoryNameWithOwner: $repository, - branchName: $branch - }, - expectedHeadOid: $expected_head, - message: {headline: $headline}, - fileChanges: { - additions: [ - {path: "src/main.zig", contents: $main_zig}, - {path: "README.md", contents: $readme}, - {path: "CHANGELOG.md", contents: $changelog} - ] - } - } - } - }' > /tmp/create-release-commit.json - - COMMIT_SHA=$(gh api graphql --input /tmp/create-release-commit.json --jq '.data.createCommitOnBranch.commit.oid') - if [ -z "$COMMIT_SHA" ]; then - echo "::error::GitHub did not return the release commit SHA" - exit 1 - fi - - VERIFIED=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${COMMIT_SHA}" --jq '.commit.verification.verified') - if [ "$VERIFIED" != "true" ]; then - echo "::error::Release commit ${COMMIT_SHA} does not have a verified signature" - exit 1 - fi - - gh pr create \ - --title "Prepare v${NEW} release" \ - --body "$(cat < /tmp/libfx-registry-pack.json mv "$(jq -r '.[0].filename' /tmp/libfx-registry-pack.json)" libfx-registry.tgz cmp libfx-package.tgz libfx-registry.tgz - - name: Test the registry package in Node and Next - env: - LIBFX_TEST_ARTIFACT_ROOT: ${{ runner.temp }}/libfx-registry-evidence - run: | - node sdk/tests/test-packed-example.mjs libfx-registry.tgz native esm - node --no-experimental-require-module sdk/tests/test-packed-example.mjs libfx-registry.tgz native cjs - node --experimental-wasm-jspi sdk/tests/test-packed-example.mjs libfx-registry.tgz wasm esm - node --experimental-wasm-jspi --no-experimental-require-module sdk/tests/test-packed-example.mjs libfx-registry.tgz wasm cjs - node sdk/tests/test-next-package.mjs libfx-registry.tgz - node sdk/tests/test-next-package.mjs libfx-registry.tgz --webpack - node sdk/tests/test-next-package.mjs libfx-registry.tgz --next15 - - name: Retain registry qualification evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: libfx-registry-evidence - path: | - ${{ runner.temp }}/libfx-registry-evidence/libfx-next-*/*.log - ${{ runner.temp }}/libfx-registry-evidence/libfx-next-*/results.json - ${{ runner.temp }}/libfx-registry-evidence/libfx-trace-*/*.log - ${{ runner.temp }}/libfx-registry-evidence/libfx-trace-*/results.json - if-no-files-found: ignore diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0336a807e..561449d0f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -154,6 +154,10 @@ jobs: actions: read contents: read uses: ./.github/workflows/pgso-macos-arm64.yml + with: + # Keep release evidence long enough to serve as the main control for + # manual PGSO comparisons. + evidence-retention-days: 90 sign-macos-arm64: needs: [check-version, build-macos-arm64] diff --git a/AGENTS.md b/AGENTS.md index 1e1972201..5f0c47ae9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Before reporting the work as ready: 1. Build succeeds. 2. Focused tests for the changed path pass locally. -3. The **Full CI** run for the exact current commit passes on every required Linux and macOS runner. +3. **CI** passes for the exact current commit, including the **macOS arm64** check. See **CI on Pull Requests**. 4. Run the built binary locally and drive at least one real interaction that exercises the change end to end. 5. Confirm the process did not abort, stderr is clean, and the behavior matches what you are about to tell the user. @@ -214,7 +214,7 @@ Do not bypass the permission system for new tools. * Zig unit tests go inside the source file they test, using `test "description" { ... }` blocks. -* Run the narrowest relevant tests while developing. The complete `zig build test` suite runs in ReleaseSafe in **Full CI** after the feature branch is pushed, and it must pass before the draft PR is marked ready. +* Run the narrowest relevant tests while developing. The complete `zig build test` suite runs in ReleaseSafe in **CI** after the feature branch is pushed, and it must pass before the draft PR is marked ready. * Use `std.testing.expect`, `std.testing.expectEqual`, `std.testing.expectEqualStrings` for assertions. @@ -270,20 +270,23 @@ Assign the label when the PR is opened and keep it accurate when the PR changes. Keep PR titles as clean imperative sentences, such as `Restore feedback report file clipboard`. Do not add bracketed prefixes such as `[bug]`, `[feature]`, or `[improvement]`. Type belongs in the label, not the title. -## Full CI on Feature Branches +## CI on Pull Requests Do not run the complete deterministic test suite locally as the default development loop. Run the focused test for the changed path, build the binary, and exercise that path with `./zig-out/bin/fx`. -After the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. `.github/workflows/full-ci.yml` runs the following on all four supported native runner architectures: +After the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. Linux is the gate for every change because most fx behavior is identical on every platform: -* `ubuntu-24.04` (x86_64) -* `ubuntu-24.04-arm` (aarch64) -* `macos-15-intel` (x86_64) -* `macos-15` (aarch64) +* `.github/workflows/ci.yml` runs on Linux x86_64. It checks formatting, the PGSO corpus, and the public surface, then runs the ReleaseSafe unit tests. It builds fx once, smoke-tests it, and shares that binary with four duration-balanced E2E shards and the MCP conformance baseline. Checked-in weights assign every E2E file to exactly one shard, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after its tmux server is reset. A file that passes only on retry gets a warning annotation; investigate it as a possible race. The SDK jobs build WASM and the Node-API addons once and run the complete package qualification. +* `.github/workflows/binary-size.yml` builds each release target on its native runner, reports the size change, and smoke-tests the Linux arm64, macOS x86_64, and macOS arm64 binaries. See **Binary Size Observability**. +* `.github/workflows/bench.yml` enforces the startup latency budget. See **Benchmarks**. -The native matrix builds, tests, and smoke-tests ReleaseSafe on every platform; formatting and the public-surface audit run in those ReleaseSafe jobs. The E2E matrix runs four duration-balanced, isolated ReleaseSafe shards per platform with Bun and tmux. Checked-in weights assign every test file to exactly one shard on each platform, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one bounded retry after its tmux server is reset. Live model evals remain separate because they require credentials and are not deterministic. +macOS runs only when a change can behave differently there. `.github/workflows/macos.yml` uses `scripts/detect-macos-need.sh` to check the diff. The macOS arm64 unit tests and the E2E files in `tests/e2e/macos-platform-tests.json` run when the change touches a Zig file with macOS-specific code, `build.zig`, the macOS signing script, the native SDK addon, a listed E2E file, or the macOS checks themselves. Native SDK addon changes also build and load the addon on macOS. To request macOS for any other change, add the `ci:macos` label or run `gh workflow run macos.yml --ref `. When the change does not need macOS, the `macOS arm64` check passes without starting a macOS runner. Every E2E file that branches on `process.platform === "darwin"` must appear in `tests/e2e/macos-platform-tests.json`, and `tests/e2e/ci-shards.test.ts` enforces that. -A Full CI result is valid only when it belongs to the exact current commit and all four `Full suite (...)` jobs succeed. Each platform aggregate requires its ReleaseSafe native check plus all four ReleaseSafe E2E shards. Do not mark the draft PR ready or request review from a stale, partial, queued, cancelled, skipped, or failed run. If Full CI fails, make the smallest repair, rerun the focused local proof, push the new commit to the same draft PR, and wait for Full CI on the new exact commit. After CI passes, run the final ship gate and mark the PR ready only when it reports `SHIP` for that exact commit. +The macOS arm64 PGSO workflow runs on a pull request only when the PGSO pipeline changes: `scripts/pgso/` other than `corpus.json`, the `setup-pgso` action, or `.github/workflows/pgso-macos-arm64.yml`. To qualify any other change, run `gh workflow run pgso-macos-arm64.yml --ref `. The stable release always runs it. + +Live model evals and the live ACP suite need credentials and are not deterministic, so they run only on request. Run the live ACP suite with `gh workflow run ci.yml --ref -f acp_live=true`. + +A CI result is valid only when it belongs to the exact current commit and `Build & Test`, `E2E (deterministic)`, `Shellcheck`, `Startup Latency`, and `macOS arm64` all succeed. Do not mark the draft PR ready or request review from a stale, partial, queued, cancelled, skipped, or failed run. If CI fails, make the smallest repair, rerun the focused local proof, push the new commit to the same draft PR, and wait for CI on the new exact commit. After CI passes, run the final ship gate and mark the PR ready only when it reports `SHIP` for that exact commit. ## Reproducing Render Bugs @@ -339,6 +342,8 @@ Startup latency benchmarks live in `benchmarks/` and run in CI via `.github/work The CI workflow builds a ReleaseSafe binary, measures six CLI paths with hyperfine, and enforces per-command latency budgets. PRs that exceed a budget fail the check. On `main`, results are uploaded to Vercel Blob for historical tracking. +The long-turn memory and libfx runtime benchmarks run only on request. Start them with `gh workflow run bench.yml --ref -f long_turn_memory=true` or `-f libfx_runtime=true`. + The startup benchmark uses `FX_BENCH=1`, an environment variable that runs through arg parsing and CLI dispatch, then exits before TTY initialization. This lives in `src/core/app/app_entry_runtime.zig`. Current raw wall-clock contract: @@ -359,7 +364,11 @@ Every pull request runs `.github/workflows/binary-size.yml` across Linux x86_64, Linux arm64, macOS x86_64, and macOS arm64. Each matrix job builds the pull request merge commit and its base commit as stripped ReleaseSafe binaries on the same native runner, then reports the exact byte and MiB delta plus ELF or -Mach-O section changes. +Mach-O section changes. The base binary is cached by base commit, so later +pushes to the same pull request reuse it until the base branch moves. The Linux +arm64, macOS x86_64, and macOS arm64 jobs also smoke-test the pull request +binary with `scripts/smoke-binary.sh`, because those binaries run nowhere else +on a pull request. Each platform check is informational. An increase of at least 52,429 bytes (0.050000 MiB) emits a warning and retains that platform's binaries for @@ -380,21 +389,11 @@ Do not document intended behavior as if it already exists. ## Releasing -Releases use a two-workflow pipeline. The maintainer controls the changelog voice and format. +Releases are prepared by hand in a release PR. The maintainer controls the changelog voice and format. -### Automated flow (preferred) +### Preparing a release -1. Go to **Actions > Prepare Release** on GitHub -2. Select the bump type (`patch`, `minor`, or `major`) and run the workflow -3. The workflow bumps the version, feeds the actual `git diff` to an LLM to draft the changelog, and opens a PR -4. Review the PR — edit the AI-drafted changelog if needed — then merge -5. The existing `release.yml` detects the version change and handles build, publish, tagging, and the GitHub Release - -The `prepare-release.yml` workflow uses the Vercel AI Gateway (`AI_GATEWAY_API_KEY` secret) to generate the changelog from the real code diff, not from commit messages or PR descriptions. - -### Manual flow - -To prepare a release by hand: +To prepare a release: 1. Create a branch (e.g. `prepare-v0.3.0`) 2. Bump `pub const version` in `src/main.zig` @@ -406,7 +405,7 @@ When the PR merges, CI compares the version tag to what exists in git. If the ta ### Writing the changelog -Whether automated or manual, the changelog is public product copy. Describe observable user behavior, not the engineering process behind it. Use the diff, commits, and merged pull requests as research evidence only. +The changelog is public product copy. Describe observable user behavior, not the engineering process behind it. Use the diff, commits, and merged pull requests as research evidence only. Public changelog entries must: @@ -468,5 +467,5 @@ The canonical repository is `vercel-labs/fx` on GitHub. All URLs, links, and ref 1. Run `zig fmt --check src/` and the focused tests for the changed path. 2. Build and exercise the change locally with `./zig-out/bin/fx`. 3. Push a clean checkpoint commit and open a draft PR immediately. -4. Require **Full CI** and the final ship gate to pass on the exact current commit across all four native runners. +4. Require **CI**, including the **macOS arm64** check, and the final ship gate to pass on the exact current commit. 5. Update docs if behavior changed. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb17da436..c2591b765 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,11 +39,13 @@ zig build run Keep the local development loop focused: run the narrowest test that covers the changed path, build fx, and exercise the change using `./zig-out/bin/fx`. The installed `fx` on `PATH` is not valid development evidence. -Once the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. The **Full CI** workflow runs the complete deterministic suite on native Linux x86_64, Linux aarch64, macOS x86_64, and macOS aarch64 runners. The native matrix builds, tests, and smoke-tests ReleaseSafe on every platform; formatting and the public-surface audit run in those ReleaseSafe jobs. Four duration-balanced, isolated ReleaseSafe E2E shards per platform use checked-in weights to assign every Bun test file once; files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one bounded retry after tmux is reset. +Once the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. **CI** runs the complete deterministic suite on Linux x86_64, which is the gate for every change. It checks formatting, the PGSO corpus, and the public surface, runs the ReleaseSafe unit tests, and builds fx once for the E2E jobs. Four duration-balanced E2E shards use checked-in weights to assign every Bun test file once, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after tmux is reset, and a file that passes only on retry gets a warning annotation. -Standard PR CI reports ReleaseSafe Build & Test and deterministic E2E results. Do not mark the draft PR ready until all four Full CI jobs and the final ship gate have succeeded for the exact current commit. Each platform aggregate requires its ReleaseSafe native check and all four ReleaseSafe E2E shards. A result from an older commit does not count. Live model evals are separate from this gate because they require credentials and are not deterministic. +macOS checks run only when a change can behave differently on macOS: a Zig file with macOS-specific code, `build.zig`, the macOS signing script, the native SDK addon, or an E2E file listed in `tests/e2e/macos-platform-tests.json`. Add the `ci:macos` label or run `gh workflow run macos.yml --ref ` to request them for any other change. The `macOS arm64` check passes without a macOS runner when the change does not need one. -Changes to `build.zig` or `scripts/pgso/` also run the native macOS arm64 PGSO candidate workflow. That lane produces retained size, behavior, and performance evidence but does not alter any release artifact or update channel. Its pinned toolchain, local reproduction command, corpus exclusions, and failure rules are documented in [`scripts/pgso/README.md`](scripts/pgso/README.md). +Do not mark the draft PR ready until `Build & Test`, `E2E (deterministic)`, `Shellcheck`, `Startup Latency`, `macOS arm64`, and the final ship gate have succeeded for the exact current commit. A result from an older commit does not count. Live model evals are separate from this gate because they require credentials and are not deterministic. + +Changes to the PGSO pipeline also run the native macOS arm64 PGSO candidate workflow: `scripts/pgso/` other than `corpus.json`, the `setup-pgso` action, or the PGSO workflow. Run it on any other branch with `gh workflow run pgso-macos-arm64.yml --ref `. That lane produces retained size, behavior, and performance evidence but does not alter any release artifact or update channel. Its pinned toolchain, local reproduction command, corpus exclusions, and failure rules are documented in [`scripts/pgso/README.md`](scripts/pgso/README.md). Every pull request also receives informational ReleaseSafe binary-size comparisons for Linux x86_64, Linux arm64, macOS x86_64, and macOS arm64. Each @@ -51,7 +53,8 @@ comparison builds the pull request merge commit and base commit on the same native runner, reports exact file and ELF or Mach-O section deltas, and emits a warning at increases of 52,429 bytes (0.050000 MiB) or more. The warning requests investigation but does not replace the full PGSO release gate or reject a valid -feature solely for adding code. +feature solely for adding code. The same jobs smoke-test the Linux arm64, macOS +x86_64, and macOS arm64 binaries, which run nowhere else on a pull request. ## Pull Requests @@ -601,6 +604,10 @@ CI uses `--runs 100` with a reduced warmup and skips the build step because the workflow builds ReleaseSafe first. Results are written to `benchmarks/results/` (gitignored). +The libfx runtime and long-turn memory jobs run only on request. Start them by +dispatching **Benchmarks** with `libfx_runtime` or `long_turn_memory` set, for +example `gh workflow run bench.yml --ref -f libfx_runtime=true`. + The libfx runtime job measures cold startup, warm prompts, host-tool calls, stream throughput, and Agent cleanup. Its direct Pi comparison uses an external Zig HTTP server, Pi 0.84.4, and three alternating 100-sample rounds. On Bun, @@ -626,5 +633,5 @@ Minimum checklist: 1. Run `zig fmt --check src/` and the focused tests for the changed path. 2. Run `zig build`, then exercise the change with `./zig-out/bin/fx`. 3. Push the feature branch and open a draft PR immediately. -4. Require all four **Full CI** jobs and the final ship gate to pass for the exact current commit before marking the PR ready. +4. Require **CI**, including the **macOS arm64** check, and the final ship gate to pass for the exact current commit before marking the PR ready. 5. Update `README.md` if user-facing behavior changed. diff --git a/scripts/detect-macos-need.sh b/scripts/detect-macos-need.sh new file mode 100755 index 000000000..101400acb --- /dev/null +++ b/scripts/detect-macos-need.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Decide whether a change needs the macOS arm64 checks. +# +# Usage: detect-macos-need.sh +# +# macOS runs only when a change touches behavior that can differ on macOS: a +# Zig file with macOS-specific code before or after the change, build.zig, the +# macOS signing script, the native SDK addon, an E2E file listed in +# tests/e2e/macos-platform-tests.json, or the macOS checks themselves. Set +# MACOS_REQUESTED to a reason to force the checks, and SDK_NATIVE_REQUESTED=true +# to force the native SDK addon check. +# +# Writes needed=true|false and sdk_native=true|false to $GITHUB_OUTPUT and the +# reasons to $GITHUB_STEP_SUMMARY when those files are set. +set -euo pipefail + +if [ "$#" -ne 2 ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +base="$1" +head="$2" +platform_list=tests/e2e/macos-platform-tests.json + +needed=false +sdk_native=false +reasons=() + +require() { + needed=true + reasons+=("$1") +} + +require_sdk_native() { + sdk_native=true + require "$1" +} + +has_macos_code() { + local content + content="$({ git show "$head:$1" 2>/dev/null; git show "$base:$1" 2>/dev/null; } || true)" + grep -Eq '\.macos|isDarwin|[Dd]arwin' <<<"$content" +} + +is_platform_test() { + jq -e --arg file "$1" 'index($file) != null' "$platform_list" >/dev/null +} + +if [ -n "${MACOS_REQUESTED:-}" ]; then + require "$MACOS_REQUESTED" +fi +if [ "${SDK_NATIVE_REQUESTED:-false}" = true ]; then + require_sdk_native "native SDK addon check requested" +fi + +while IFS= read -r path; do + case "$path" in + .github/workflows/macos.yml) + require_sdk_native "$path: the macOS checks changed" ;; + scripts/detect-macos-need.sh | scripts/smoke-binary.sh | tests/e2e/ci-run-files.sh | "$platform_list") + require "$path: the macOS checks changed" ;; + build.zig | build.zig.zon) + require "$path: build configuration changed" ;; + scripts/sign-and-notarize-macos.sh) + require "$path: macOS signing changed" ;; + src/napi_*.zig | sdk/node.js | sdk/tests/test-node-napi.mjs) + require_sdk_native "$path: native SDK addon changed" ;; + *.zig) + if has_macos_code "$path"; then + require "$path: contains macOS-specific code" + fi ;; + tests/e2e/*.test.ts) + name="${path#tests/e2e/}" + if [[ "$name" != */* ]] && is_platform_test "$name"; then + require "$path: macOS platform E2E file changed" + fi ;; + esac +done < <(git diff --name-only --no-renames "$base" "$head") + +if [ "$needed" = true ]; then + printf 'macOS arm64 checks needed:\n' + printf ' %s\n' "${reasons[@]}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + printf '### macOS arm64 checks run\n\n' + printf -- '- %s\n' "${reasons[@]}" + } >>"$GITHUB_STEP_SUMMARY" + fi +else + printf 'No macOS-specific change; macOS arm64 checks skipped.\n' + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf 'No macOS-specific change; macOS arm64 checks skipped.\n' >>"$GITHUB_STEP_SUMMARY" + fi +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + { + printf 'needed=%s\n' "$needed" + printf 'sdk_native=%s\n' "$sdk_native" + } >>"$GITHUB_OUTPUT" +fi diff --git a/scripts/pgso/README.md b/scripts/pgso/README.md index 202dc81b4..d074b6df2 100644 --- a/scripts/pgso/README.md +++ b/scripts/pgso/README.md @@ -161,7 +161,8 @@ manual branch head with its exact main ancestor, dispatch the **Benchmarks** workflow on the comparison branch with `pgso_control_artifact` and `pgso_candidate_artifact` set to the two qualified aggregate artifact IDs. The main control may come from the dedicated PGSO workflow or the Release -workflow that invokes it. Both artifacts must have completed qualification. +workflow that invokes it. Release runs keep their aggregate evidence for 90 +days, so the last release can serve as the control. Both artifacts must have completed qualification. The candidate must come from the dedicated PGSO workflow. For source-only PRs, run that workflow manually on the PR branch. Its manifest source must match the run head, and GitHub's immutable SHA comparison must prove that exact control diff --git a/scripts/smoke-binary.sh b/scripts/smoke-binary.sh new file mode 100755 index 000000000..5b90ae5a5 --- /dev/null +++ b/scripts/smoke-binary.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Run a freshly built fx binary through quick commands. Each command must exit +# zero, print to stdout, and leave stderr empty. +set -euo pipefail + +if [ "$#" -ne 1 ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +binary="$1" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +check() { + local label="$1" + shift + if ! "$binary" "$@" >"$work_dir/stdout" 2>"$work_dir/stderr"; then + printf '::error title=Binary smoke test::fx %s exited with an error\n' "$label" + cat "$work_dir/stderr" >&2 + return 1 + fi + if [ ! -s "$work_dir/stdout" ]; then + printf '::error title=Binary smoke test::fx %s printed nothing\n' "$label" + return 1 + fi + if [ -s "$work_dir/stderr" ]; then + printf '::error title=Binary smoke test::fx %s wrote to stderr\n' "$label" + cat "$work_dir/stderr" >&2 + return 1 + fi + printf 'fx %s: ok\n' "$label" +} + +check "--version" --version +check "help" help +check "status --json" status --json diff --git a/sdk/AGENTS.md b/sdk/AGENTS.md index 62f60a4f2..20eb6b04e 100644 --- a/sdk/AGENTS.md +++ b/sdk/AGENTS.md @@ -90,7 +90,7 @@ sdk_package_dir="$(mktemp -d)" node sdk/scripts/package-term-demo.mjs "$sdk_package_dir" ``` -Do not commit `zig-out/`, `sdk/dist/`, or other generated artifacts. Run `git diff --check` before creating a checkpoint. The repository-level build, binary exercise, and Full CI ship gates still apply. +Do not commit `zig-out/`, `sdk/dist/`, or other generated artifacts. Run `git diff --check` before creating a checkpoint. The repository-level build, binary exercise, and CI ship gates still apply. ## Code review rules diff --git a/tests/e2e/ci-run-files.sh b/tests/e2e/ci-run-files.sh new file mode 100755 index 000000000..70f337c21 --- /dev/null +++ b/tests/e2e/ci-run-files.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Run E2E files one at a time, each in its own Bun process with its own tmux +# server, so terminal fixtures and process state cannot leak between files. +# A failing file gets one retry after its tmux server is reset. A file that +# passes only on retry is reported as a warning so flaky files stay visible. +set -uo pipefail + +if [ "$#" -eq 0 ]; then + printf 'usage: %s ...\n' "$0" >&2 + exit 2 +fi + +cd "$(dirname "$0")" || exit 1 + +tmux_root="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/fx-e2e-tmux" +status=0 +index=0 + +for test_file in "$@"; do + tmux_dir="$tmux_root-$index" + mkdir -p "$tmux_dir" + if ! TMUX_TMPDIR="$tmux_dir" bun test --max-concurrency 1 "./$test_file"; then + printf 'Retrying failed E2E file after resetting tmux: %s\n' "$test_file" + TMUX_TMPDIR="$tmux_dir" tmux kill-server 2>/dev/null || true + if TMUX_TMPDIR="$tmux_dir" bun test --max-concurrency 1 "./$test_file"; then + printf '::warning title=E2E passed only on retry::%s failed and then passed on retry. Investigate it as a possible race.\n' "$test_file" + else + status=1 + fi + fi + TMUX_TMPDIR="$tmux_dir" tmux kill-server 2>/dev/null || true + index=$((index + 1)) +done + +exit "$status" diff --git a/tests/e2e/ci-shards.test.ts b/tests/e2e/ci-shards.test.ts index df1d4940b..16eed216c 100644 --- a/tests/e2e/ci-shards.test.ts +++ b/tests/e2e/ci-shards.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildShardPlan, selectShard } from "./ci-shards"; @@ -131,3 +131,27 @@ describe("CI shard planner", () => { } }); }); + +describe("macOS platform E2E list", () => { + const listed: unknown = JSON.parse( + readFileSync(join(import.meta.dir, "macos-platform-tests.json"), "utf8"), + ); + const rootTests = readdirSync(import.meta.dir).filter((name) => name.endsWith(".test.ts")); + + test("names existing root E2E files once, in sorted order", () => { + expect(Array.isArray(listed)).toBe(true); + const names = listed as string[]; + expect(names.length).toBeGreaterThan(0); + for (const name of names) expect(rootTests).toContain(name); + expect(names).toEqual([...new Set(names)].sort()); + }); + + test("includes every E2E file that branches on macOS", () => { + const macosBranch = /(process\.platform|platform\(\))\s*[!=]==\s*["']darwin["']/; + const missing = rootTests.filter((name) => + macosBranch.test(readFileSync(join(import.meta.dir, name), "utf8")) && + !(listed as string[]).includes(name) + ); + expect(missing).toEqual([]); + }); +}); diff --git a/tests/e2e/macos-platform-tests.json b/tests/e2e/macos-platform-tests.json new file mode 100644 index 000000000..1e67005ef --- /dev/null +++ b/tests/e2e/macos-platform-tests.json @@ -0,0 +1,18 @@ +[ + "cli.test.ts", + "file-tool-permissions.test.ts", + "gateway-stream-lifecycle.test.ts", + "mcp-auth.test.ts", + "oauth-keychain-migration.test.ts", + "sessions-v2.test.ts", + "terminal-host.test.ts", + "tui-auth-source-selection.test.ts", + "tui-command-permissions.test.ts", + "tui-full-transcript-brutal.test.ts", + "tui-interrupt-recovery.test.ts", + "tui-resume-brutal.test.ts", + "tui-slash-extra.test.ts", + "tui-terminal-tool.test.ts", + "vision-route-fake-gateway.test.ts", + "yolo-permission-mode.test.ts" +]