From f851e5e95b04134d6559c83c69a8daf06ae6e931 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 02:02:27 +0200 Subject: [PATCH] ci: run reduced test subsets on macOS x64 and mas (#54013) Pull requests now run the macOS x64 test legs and the mas test legs as a single shard each over a curated allowlist of spec files (script/macos-x64-spec-allowlist.txt, script/macos-mas-spec-allowlist.txt), while the darwin arm64 legs keep running the whole suite. The whole suite still runs on every macOS leg for pushes to main and release branches, for roller/ pull requests and for pull requests carrying the run-full-macos label. This takes a PR from 13 macOS runners to 8. The gate is a new full-macos-tests boolean input on the build-and-test pipelines and the test segment, computed in build.yml from the event name, head ref and PR labels, and defaulting to true for every other caller. Claude-Session: https://claude.ai/code/session_015NrhHJaEg6V3V9G6yF9SHA Co-authored-by: Claude --- .github/workflows/build.yml | 9 ++++ .../pipeline-electron-build-and-test.yml | 6 +++ ...eline-electron-build-and-tidy-and-test.yml | 6 +++ .../pipeline-segment-electron-test.yml | 36 ++++++++++++++-- script/gen-spec-weights.js | 4 +- script/macos-mas-spec-allowlist.txt | 40 ++++++++++++++++++ script/macos-x64-spec-allowlist.txt | 41 +++++++++++++++++++ 7 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 script/macos-mas-spec-allowlist.txt create mode 100644 script/macos-x64-spec-allowlist.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb6964bd53..2afbaab0ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -327,6 +327,13 @@ jobs: ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }} # Build Jobs - These cascade into testing jobs + # + # Pull requests run a reduced macOS test matrix: the x64 legs and the mas + # legs each run one shard over an allowlist (script/macos-*-spec-allowlist.txt) + # while darwin arm64 runs the whole suite. The whole suite runs on every leg + # for pushes to main and release branches, for roller/ PRs and for PRs + # carrying the run-full-macos label (labels are read from the event that + # started the run, so add the label before the next push). macos-x64: permissions: contents: read @@ -344,6 +351,7 @@ jobs: generate-symbols: false upload-to-storage: '0' enable-ssh: ${{ inputs.enable-ssh || false }} + full-macos-tests: ${{ github.event_name != 'pull_request' || startsWith(github.head_ref, 'roller/') || contains(github.event.pull_request.labels.*.name, 'run-full-macos') }} secrets: CI_STEP_DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }} CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }} @@ -380,6 +388,7 @@ jobs: generate-symbols: false upload-to-storage: '0' enable-ssh: ${{ inputs.enable-ssh || false }} + full-macos-tests: ${{ github.event_name != 'pull_request' || startsWith(github.head_ref, 'roller/') || contains(github.event.pull_request.labels.*.name, 'run-full-macos') }} secrets: CI_STEP_DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }} CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }} diff --git a/.github/workflows/pipeline-electron-build-and-test.yml b/.github/workflows/pipeline-electron-build-and-test.yml index cc032bf9a0..1eff5592a5 100644 --- a/.github/workflows/pipeline-electron-build-and-test.yml +++ b/.github/workflows/pipeline-electron-build-and-test.yml @@ -69,6 +69,11 @@ on: required: false type: boolean default: false + full-macos-tests: + description: 'Run the whole spec suite on every macOS test leg; see pipeline-segment-electron-test.yml' + required: false + type: boolean + default: true secrets: # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the # workflows that read it map this back to the ACTIONS_STEP_DEBUG env. @@ -140,6 +145,7 @@ jobs: target-platform: ${{ inputs.target-platform }} test-runs-on: ${{ inputs.test-runs-on }} test-container: ${{ inputs.test-container }} + full-macos-tests: ${{ inputs.full-macos-tests }} is-asan: ${{ inputs.is-asan }} is-ubsan: ${{ inputs.is-ubsan }} enable-ssh: ${{ inputs.enable-ssh }} diff --git a/.github/workflows/pipeline-electron-build-and-tidy-and-test.yml b/.github/workflows/pipeline-electron-build-and-tidy-and-test.yml index 8e92aaf503..bca6dd9585 100644 --- a/.github/workflows/pipeline-electron-build-and-tidy-and-test.yml +++ b/.github/workflows/pipeline-electron-build-and-tidy-and-test.yml @@ -73,6 +73,11 @@ on: required: false type: boolean default: false + full-macos-tests: + description: 'Run the whole spec suite on every macOS test leg; see pipeline-segment-electron-test.yml' + required: false + type: boolean + default: true secrets: # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the # workflows that read it map this back to the ACTIONS_STEP_DEBUG env. @@ -156,6 +161,7 @@ jobs: target-platform: ${{ inputs.target-platform }} test-runs-on: ${{ inputs.test-runs-on }} test-container: ${{ inputs.test-container }} + full-macos-tests: ${{ inputs.full-macos-tests }} is-asan: ${{ inputs.is-asan }} enable-ssh: ${{ inputs.enable-ssh }} secrets: diff --git a/.github/workflows/pipeline-segment-electron-test.yml b/.github/workflows/pipeline-segment-electron-test.yml index ae60fcba4c..27244d1027 100644 --- a/.github/workflows/pipeline-segment-electron-test.yml +++ b/.github/workflows/pipeline-segment-electron-test.yml @@ -40,6 +40,11 @@ on: required: false type: string default: x11 + full-macos-tests: + description: 'Run the whole spec suite on every macOS leg. When false the x64 legs and the mas legs each run one shard over the allowlist in script/macos-{x64,mas}-spec-allowlist.txt; darwin arm64 always runs the whole suite' + required: false + type: boolean + default: true secrets: # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the # workflows that read it map this back to the ACTIONS_STEP_DEBUG env. @@ -90,11 +95,19 @@ jobs: fail-fast: false matrix: build-type: ${{ inputs.target-platform == 'macos' && (inputs.is-ubsan && fromJSON('["darwin"]') || fromJSON('["darwin","mas"]')) || (inputs.target-platform == 'win' && fromJSON('["win"]') || fromJSON('["linux"]')) }} - shard: ${{ case(inputs.display-server == 'wayland', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.is-ubsan, fromJSON('[1, 2, 3]'), inputs.target-platform == 'linux', fromJSON('[1, 2, 3]'), inputs.target-platform == 'macos' && inputs.target-arch == 'x64', fromJSON('[1, 2, 3]'), fromJSON('[1, 2]')) }} + shard: ${{ case(inputs.display-server == 'wayland', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.is-ubsan, fromJSON('[1, 2, 3]'), inputs.target-platform == 'linux', fromJSON('[1, 2, 3]'), inputs.target-platform == 'macos' && !inputs.full-macos-tests && inputs.target-arch == 'x64', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.target-arch == 'x64', fromJSON('[1, 2, 3]'), fromJSON('[1, 2]')) }} + # The reduced macOS arm64 leg keeps both darwin shards and runs mas on + # a single one; the reduced x64 leg is already one shard per variant. + exclude: ${{ inputs.target-platform == 'macos' && !inputs.full-macos-tests && inputs.target-arch == 'arm64' && fromJSON('[{"build-type":"mas","shard":2}]') || fromJSON('[]') }} env: BUILD_TYPE: ${{ matrix.build-type }} TARGET_ARCH: ${{ inputs.target-arch }} ARTIFACT_KEY: ${{ matrix.build-type }}_${{ inputs.target-arch }} + # Set on the reduced macOS legs (x64, and mas on any arch): the file + # listing the spec files to run instead of the whole suite. Empty + # elsewhere. The mas list wins on mas_x64: its arch coverage comes from + # the darwin_x64 leg of the same run. + MACOS_SPEC_ALLOWLIST: ${{ inputs.target-platform == 'macos' && !inputs.full-macos-tests && (matrix.build-type == 'mas' && 'script/macos-mas-spec-allowlist.txt' || inputs.target-arch == 'x64' && 'script/macos-x64-spec-allowlist.txt') || '' }} # depot_tools' unbootstrapped python3 wrapper (crrev.com/c/8162440) breaks system python3 resolution; bypass it DEPOT_TOOLS_PYTHON_BYPASS: 1 steps: @@ -341,14 +354,31 @@ jobs: TARGET_PLATFORM: ${{ inputs.target-platform }} DISPLAY_SERVER: ${{ inputs.display-server }} SHARD: ${{ matrix.shard }} - SHARD_COUNT: ${{ case(inputs.display-server == 'wayland', 1, inputs.target-platform == 'macos' && inputs.is-ubsan, 3, inputs.target-platform == 'linux', 3, inputs.target-platform == 'macos' && inputs.target-arch == 'x64', 3, 2) }} + SHARD_COUNT: ${{ case(inputs.display-server == 'wayland', 1, env.MACOS_SPEC_ALLOWLIST != '', 1, inputs.target-platform == 'macos' && inputs.is-ubsan, 3, inputs.target-platform == 'linux', 3, inputs.target-platform == 'macos' && inputs.target-arch == 'x64', 3, 2) }} run: | cd src/electron export ELECTRON_TEST_RESULTS_DIR=`pwd`/junit # Get which tests are on this shard tests_files=$(node script/split-tests "$SHARD" "$SHARD_COUNT") + # The Wayland leg and the reduced macOS legs run only the spec files + # named in an allowlist (one path per line, # comments ignored). + allowlist_file="" if [ "$DISPLAY_SERVER" = "wayland" ]; then allowlist_file=script/wayland-test-allowlist.txt + elif [ -n "$MACOS_SPEC_ALLOWLIST" ]; then + allowlist_file="$MACOS_SPEC_ALLOWLIST" + # A renamed or deleted spec would otherwise silently drop out of the + # reduced run. + while IFS= read -r allowed; do + case "$allowed" in ''|'#'*) continue ;; esac + if [ ! -f "$allowed" ]; then + echo "::error file=$allowlist_file::lists $allowed, which does not exist" + exit 1 + fi + done < "$allowlist_file" + fi + if [ -n "$allowlist_file" ]; then + echo "Running only the spec files listed in $allowlist_file" filtered_tests="" for test_file in $tests_files; do if grep -Fxq "$test_file" "$allowlist_file"; then @@ -358,7 +388,7 @@ jobs: tests_files="${filtered_tests# }" if [ -z "$tests_files" ]; then - echo "No tests matched Wayland filter, skipping." + echo "No tests matched $allowlist_file, skipping." exit 0 fi fi diff --git a/script/gen-spec-weights.js b/script/gen-spec-weights.js index 497e7a929b..d8fc0b5ae0 100644 --- a/script/gen-spec-weights.js +++ b/script/gen-spec-weights.js @@ -2,7 +2,9 @@ // test job, used by script/split-tests.js) from the spec-timings.json files // the test jobs upload in their test_artifacts_* bundles. // -// Usage, from a recent green build.yml run on the branch: +// Usage, from a recent green build.yml run on the branch (one with full macOS +// legs: a push to main or a PR carrying the run-full-macos label, since the +// reduced PR legs only time the spec files in script/macos-*-spec-allowlist.txt): // gh run download --repo electron/electron -D /tmp/spec-timings -p 'test_artifacts_*' // node script/gen-spec-weights.js /tmp/spec-timings // diff --git a/script/macos-mas-spec-allowlist.txt b/script/macos-mas-spec-allowlist.txt new file mode 100644 index 0000000000..afe6ff9d2e --- /dev/null +++ b/script/macos-mas-spec-allowlist.txt @@ -0,0 +1,40 @@ +# Spec files the mas (Mac App Store) test legs run on pull requests, one per +# line, matched exactly against script/split-tests output. Everything else in +# spec/ is covered by the darwin legs of the same run; this list is the areas +# where the mas build differs from darwin (IS_MAS_BUILD() in shell/, +# is_mas_build in BUILD.gn and the patches/chromium/mas_* and +# *_mas_* patches). The full suite still runs on pushes to main and release +# branches, on roller/ PRs and on PRs carrying the run-full-macos label. +# Used by .github/workflows/pipeline-segment-electron-test.yml. +# +# private API symbol scan of every MAS binary (mas_avoid_private_macos_api_usage) +spec/mas-spec.ts +# app sandbox: getAppMetrics().sandboxed, startAccessingSecurityScopedResource, +# login items (electron_login_helper), process.mas gating +spec/api-app-spec.ts +# security-scoped bookmarks returned by the open/save dialogs +spec/api-dialog-spec.ts +# keychain account name differs in MAS builds (feat_ensure_mas_builds_..._safestorage) +spec/api-safe-storage-spec.ts +# StoreKit, only meaningful for the App Store build +spec/api-in-app-purchase-spec.ts +# frameless window mouseDown/swipe swizzling, no remote CoreAnimation, +# capturePage and hasCapturableScreen branches on process.mas +spec/api-browser-window-spec.ts +# screen capture without CGSSetWindowCaptureExcludeShape / remote CA +spec/api-desktop-capturer-spec.ts +# ui/display/mac/screen_mac.mm is patched for MAS +spec/api-screen-spec.ts +# menu_controller_cocoa_delegate_impl.mm / NSToolbar private API removal +spec/api-menu-spec.ts +# forcefullyCrashRenderer without CrashDumpHungChildProcess, patched +# render_widget_host_view_mac paths +spec/api-web-contents-spec.ts +# sandboxMain branches on process.mas +spec/api-subframe-spec.ts +# renderer/main crash paths without crashpad +spec/crash-spec.ts +# node_service.cc drops crash keys in MAS builds +spec/api-utility-process-spec.ts +# kill ring, font matching, remote cocoa and sandbox paths patched for MAS +spec/chromium-spec.ts diff --git a/script/macos-x64-spec-allowlist.txt b/script/macos-x64-spec-allowlist.txt new file mode 100644 index 0000000000..90e561580f --- /dev/null +++ b/script/macos-x64-spec-allowlist.txt @@ -0,0 +1,41 @@ +# Spec files the macOS x64 (darwin) test leg runs on pull requests, one per +# line, matched exactly against script/split-tests output. The arm64 legs run +# the whole suite; this list is the code whose behavior can differ by CPU +# architecture (V8 code cache, JIT and WebAssembly, the Node.js startup +# snapshot, native addons, crash reporting) plus Squirrel.Mac, whose specs +# have x64-only cases. The full suite still runs on pushes to main and +# release branches, on roller/ PRs and on PRs carrying the run-full-macos +# label. Used by .github/workflows/pipeline-segment-electron-test.yml. +# +# Squirrel.Mac update lifecycle; unsigned-app cases only run on x64 +spec/api-autoupdater-darwin-spec.ts +# Squirrel.Mac update policy (version rules, tampered payloads, JSON mode) +spec/api-autoupdater-darwin-policy-spec.ts +# autoUpdater module API surface +spec/api-auto-updater-spec.ts +# Node.js startup snapshot captures process.arch and friends; native addons +spec/node-spec.ts +# electron_xcache: the Node startup snapshot / code cache blob for this arch +spec/xcache-spec.ts +# js2c build-time V8 code cache and LoadBrowserProcessSpecificV8Snapshot +spec/api-js2c-code-cache-spec.ts +# sandboxed preload V8 code cache; the largest window-level surface +spec/api-browser-window-spec.ts +# codeCache for privileged custom protocols +spec/api-protocol-spec.ts +# wasm-eval CSP, WebAssembly streaming compilation, workers, heap snapshots +spec/chromium-spec.ts +# utility process Node isolate, ESM loading, heap snapshots near heap limit +spec/api-utility-process-spec.ts +# heap statistics, memory info and takeHeapSnapshot +spec/api-process-spec.ts +# native module loading (compiled per arch) and uv_dlopen +spec/modules-spec.ts +# child_process.execFile out of asar has a darwin x64-only case +spec/asar-spec.ts +# runningUnderARM64Translation, x64-only recent-documents skip, sandbox options +spec/api-app-spec.ts +# crashpad minidumps and stack walking are architecture specific +spec/api-crash-reporter-spec.ts +# renderer, main and node crash paths +spec/crash-spec.ts