ci: run reduced test subsets on macOS x64 and mas (#54013)

Pull requests now run the macOS x64 test legs and the mas test legs as a
single shard each over a curated allowlist of spec files
(script/macos-x64-spec-allowlist.txt, script/macos-mas-spec-allowlist.txt),
while the darwin arm64 legs keep running the whole suite. The whole suite
still runs on every macOS leg for pushes to main and release branches, for
roller/ pull requests and for pull requests carrying the run-full-macos
label. This takes a PR from 13 macOS runners to 8.

The gate is a new full-macos-tests boolean input on the build-and-test
pipelines and the test segment, computed in build.yml from the event name,
head ref and PR labels, and defaulting to true for every other caller.


Claude-Session: https://claude.ai/code/session_015NrhHJaEg6V3V9G6yF9SHA

Co-authored-by: Claude <noreply@anthropic.com>

.github/workflows/build.yml

@@ -327,6 +327,13 @@ jobs:
       ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
 
   # Build Jobs - These cascade into testing jobs
+  #
+  # Pull requests run a reduced macOS test matrix: the x64 legs and the mas
+  # legs each run one shard over an allowlist (script/macos-*-spec-allowlist.txt)
+  # while darwin arm64 runs the whole suite. The whole suite runs on every leg
+  # for pushes to main and release branches, for roller/ PRs and for PRs
+  # carrying the run-full-macos label (labels are read from the event that
+  # started the run, so add the label before the next push).
   macos-x64:
     permissions:
       contents: read
@@ -344,6 +351,7 @@ jobs:
       generate-symbols: false
       upload-to-storage: '0'
       enable-ssh: ${{ inputs.enable-ssh || false }}
+      full-macos-tests: ${{ github.event_name != 'pull_request' || startsWith(github.head_ref, 'roller/') || contains(github.event.pull_request.labels.*.name, 'run-full-macos') }}
     secrets:
       CI_STEP_DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }}
       CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
@@ -380,6 +388,7 @@ jobs:
       generate-symbols: false
       upload-to-storage: '0'
       enable-ssh: ${{ inputs.enable-ssh || false }}
+      full-macos-tests: ${{ github.event_name != 'pull_request' || startsWith(github.head_ref, 'roller/') || contains(github.event.pull_request.labels.*.name, 'run-full-macos') }}
     secrets:
       CI_STEP_DEBUG: ${{ secrets.ACTIONS_STEP_DEBUG }}
       CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}

.github/workflows/pipeline-electron-build-and-test.yml

@@ -69,6 +69,11 @@ on:
         required: false
         type: boolean
         default: false
+      full-macos-tests:
+        description: 'Run the whole spec suite on every macOS test leg; see pipeline-segment-electron-test.yml'
+        required: false
+        type: boolean
+        default: true
     secrets:
       # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the
       # workflows that read it map this back to the ACTIONS_STEP_DEBUG env.
@@ -140,6 +145,7 @@ jobs:
       target-platform: ${{ inputs.target-platform }}
       test-runs-on: ${{ inputs.test-runs-on }}
       test-container: ${{ inputs.test-container }}
+      full-macos-tests: ${{ inputs.full-macos-tests }}
       is-asan: ${{ inputs.is-asan }}
       is-ubsan: ${{ inputs.is-ubsan }}
       enable-ssh: ${{ inputs.enable-ssh }}

.github/workflows/pipeline-electron-build-and-tidy-and-test.yml

@@ -73,6 +73,11 @@ on:
         required: false
         type: boolean
         default: false
+      full-macos-tests:
+        description: 'Run the whole spec suite on every macOS test leg; see pipeline-segment-electron-test.yml'
+        required: false
+        type: boolean
+        default: true
     secrets:
       # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the
       # workflows that read it map this back to the ACTIONS_STEP_DEBUG env.
@@ -156,6 +161,7 @@ jobs:
       target-platform: ${{ inputs.target-platform }}
       test-runs-on: ${{ inputs.test-runs-on }}
       test-container: ${{ inputs.test-container }}
+      full-macos-tests: ${{ inputs.full-macos-tests }}
       is-asan: ${{ inputs.is-asan }}
       enable-ssh: ${{ inputs.enable-ssh }}
     secrets:

.github/workflows/pipeline-segment-electron-test.yml

@@ -40,6 +40,11 @@ on:
         required: false
         type: string
         default: x11
+      full-macos-tests:
+        description: 'Run the whole spec suite on every macOS leg. When false the x64 legs and the mas legs each run one shard over the allowlist in script/macos-{x64,mas}-spec-allowlist.txt; darwin arm64 always runs the whole suite'
+        required: false
+        type: boolean
+        default: true
     secrets:
       # ACTIONS_STEP_DEBUG is a reserved name for workflow_call secrets; the
       # workflows that read it map this back to the ACTIONS_STEP_DEBUG env.
@@ -90,11 +95,19 @@ jobs:
       fail-fast: false
       matrix:
         build-type: ${{ inputs.target-platform == 'macos' && (inputs.is-ubsan && fromJSON('["darwin"]') || fromJSON('["darwin","mas"]')) || (inputs.target-platform == 'win' && fromJSON('["win"]') || fromJSON('["linux"]')) }}
-        shard: ${{ case(inputs.display-server == 'wayland', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.is-ubsan, fromJSON('[1, 2, 3]'), inputs.target-platform == 'linux', fromJSON('[1, 2, 3]'), inputs.target-platform == 'macos' && inputs.target-arch == 'x64', fromJSON('[1, 2, 3]'), fromJSON('[1, 2]')) }}
+        shard: ${{ case(inputs.display-server == 'wayland', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.is-ubsan, fromJSON('[1, 2, 3]'), inputs.target-platform == 'linux', fromJSON('[1, 2, 3]'), inputs.target-platform == 'macos' && !inputs.full-macos-tests && inputs.target-arch == 'x64', fromJSON('[1]'), inputs.target-platform == 'macos' && inputs.target-arch == 'x64', fromJSON('[1, 2, 3]'), fromJSON('[1, 2]')) }}
+        # The reduced macOS arm64 leg keeps both darwin shards and runs mas on
+        # a single one; the reduced x64 leg is already one shard per variant.
+        exclude: ${{ inputs.target-platform == 'macos' && !inputs.full-macos-tests && inputs.target-arch == 'arm64' && fromJSON('[{"build-type":"mas","shard":2}]') || fromJSON('[]') }}
     env:
       BUILD_TYPE: ${{ matrix.build-type }}
       TARGET_ARCH: ${{ inputs.target-arch }}
       ARTIFACT_KEY: ${{ matrix.build-type }}_${{ inputs.target-arch }}
+      # Set on the reduced macOS legs (x64, and mas on any arch): the file
+      # listing the spec files to run instead of the whole suite. Empty
+      # elsewhere. The mas list wins on mas_x64: its arch coverage comes from
+      # the darwin_x64 leg of the same run.
+      MACOS_SPEC_ALLOWLIST: ${{ inputs.target-platform == 'macos' && !inputs.full-macos-tests && (matrix.build-type == 'mas' && 'script/macos-mas-spec-allowlist.txt' || inputs.target-arch == 'x64' && 'script/macos-x64-spec-allowlist.txt') || '' }}
       # depot_tools' unbootstrapped python3 wrapper (crrev.com/c/8162440) breaks system python3 resolution; bypass it
       DEPOT_TOOLS_PYTHON_BYPASS: 1
     steps:
@@ -341,14 +354,31 @@ jobs:
         TARGET_PLATFORM: ${{ inputs.target-platform }}
         DISPLAY_SERVER: ${{ inputs.display-server }}
         SHARD: ${{ matrix.shard }}
-        SHARD_COUNT: ${{ case(inputs.display-server == 'wayland', 1, inputs.target-platform == 'macos' && inputs.is-ubsan, 3, inputs.target-platform == 'linux', 3, inputs.target-platform == 'macos' && inputs.target-arch == 'x64', 3, 2) }}
+        SHARD_COUNT: ${{ case(inputs.display-server == 'wayland', 1, env.MACOS_SPEC_ALLOWLIST != '', 1, inputs.target-platform == 'macos' && inputs.is-ubsan, 3, inputs.target-platform == 'linux', 3, inputs.target-platform == 'macos' && inputs.target-arch == 'x64', 3, 2) }}
       run: |
         cd src/electron
         export ELECTRON_TEST_RESULTS_DIR=`pwd`/junit
         # Get which tests are on this shard
         tests_files=$(node script/split-tests "$SHARD" "$SHARD_COUNT")
+        # The Wayland leg and the reduced macOS legs run only the spec files
+        # named in an allowlist (one path per line, # comments ignored).
+        allowlist_file=""
         if [ "$DISPLAY_SERVER" = "wayland" ]; then
           allowlist_file=script/wayland-test-allowlist.txt
+        elif [ -n "$MACOS_SPEC_ALLOWLIST" ]; then
+          allowlist_file="$MACOS_SPEC_ALLOWLIST"
+          # A renamed or deleted spec would otherwise silently drop out of the
+          # reduced run.
+          while IFS= read -r allowed; do
+            case "$allowed" in ''|'#'*) continue ;; esac
+            if [ ! -f "$allowed" ]; then
+              echo "::error file=$allowlist_file::lists $allowed, which does not exist"
+              exit 1
+            fi
+          done < "$allowlist_file"
+        fi
+        if [ -n "$allowlist_file" ]; then
+          echo "Running only the spec files listed in $allowlist_file"
           filtered_tests=""
           for test_file in $tests_files; do
             if grep -Fxq "$test_file" "$allowlist_file"; then
@@ -358,7 +388,7 @@ jobs:
           tests_files="${filtered_tests# }"
 
           if [ -z "$tests_files" ]; then
-            echo "No tests matched Wayland filter, skipping."
+            echo "No tests matched $allowlist_file, skipping."
             exit 0
           fi
         fi

script/gen-spec-weights.js

@@ -2,7 +2,9 @@
 // test job, used by script/split-tests.js) from the spec-timings.json files
 // the test jobs upload in their test_artifacts_* bundles.
 //
-// Usage, from a recent green build.yml run on the branch:
+// Usage, from a recent green build.yml run on the branch (one with full macOS
+// legs: a push to main or a PR carrying the run-full-macos label, since the
+// reduced PR legs only time the spec files in script/macos-*-spec-allowlist.txt):
 //   gh run download <run-id> --repo electron/electron -D /tmp/spec-timings -p 'test_artifacts_*'
 //   node script/gen-spec-weights.js /tmp/spec-timings
 //

script/macos-mas-spec-allowlist.txt

@@ -0,0 +1,40 @@
+# Spec files the mas (Mac App Store) test legs run on pull requests, one per
+# line, matched exactly against script/split-tests output. Everything else in
+# spec/ is covered by the darwin legs of the same run; this list is the areas
+# where the mas build differs from darwin (IS_MAS_BUILD() in shell/,
+# is_mas_build in BUILD.gn and the patches/chromium/mas_* and
+# *_mas_* patches). The full suite still runs on pushes to main and release
+# branches, on roller/ PRs and on PRs carrying the run-full-macos label.
+# Used by .github/workflows/pipeline-segment-electron-test.yml.
+#
+# private API symbol scan of every MAS binary (mas_avoid_private_macos_api_usage)
+spec/mas-spec.ts
+# app sandbox: getAppMetrics().sandboxed, startAccessingSecurityScopedResource,
+# login items (electron_login_helper), process.mas gating
+spec/api-app-spec.ts
+# security-scoped bookmarks returned by the open/save dialogs
+spec/api-dialog-spec.ts
+# keychain account name differs in MAS builds (feat_ensure_mas_builds_..._safestorage)
+spec/api-safe-storage-spec.ts
+# StoreKit, only meaningful for the App Store build
+spec/api-in-app-purchase-spec.ts
+# frameless window mouseDown/swipe swizzling, no remote CoreAnimation,
+# capturePage and hasCapturableScreen branches on process.mas
+spec/api-browser-window-spec.ts
+# screen capture without CGSSetWindowCaptureExcludeShape / remote CA
+spec/api-desktop-capturer-spec.ts
+# ui/display/mac/screen_mac.mm is patched for MAS
+spec/api-screen-spec.ts
+# menu_controller_cocoa_delegate_impl.mm / NSToolbar private API removal
+spec/api-menu-spec.ts
+# forcefullyCrashRenderer without CrashDumpHungChildProcess, patched
+# render_widget_host_view_mac paths
+spec/api-web-contents-spec.ts
+# sandboxMain branches on process.mas
+spec/api-subframe-spec.ts
+# renderer/main crash paths without crashpad
+spec/crash-spec.ts
+# node_service.cc drops crash keys in MAS builds
+spec/api-utility-process-spec.ts
+# kill ring, font matching, remote cocoa and sandbox paths patched for MAS
+spec/chromium-spec.ts

script/macos-x64-spec-allowlist.txt

@@ -0,0 +1,41 @@
+# Spec files the macOS x64 (darwin) test leg runs on pull requests, one per
+# line, matched exactly against script/split-tests output. The arm64 legs run
+# the whole suite; this list is the code whose behavior can differ by CPU
+# architecture (V8 code cache, JIT and WebAssembly, the Node.js startup
+# snapshot, native addons, crash reporting) plus Squirrel.Mac, whose specs
+# have x64-only cases. The full suite still runs on pushes to main and
+# release branches, on roller/ PRs and on PRs carrying the run-full-macos
+# label. Used by .github/workflows/pipeline-segment-electron-test.yml.
+#
+# Squirrel.Mac update lifecycle; unsigned-app cases only run on x64
+spec/api-autoupdater-darwin-spec.ts
+# Squirrel.Mac update policy (version rules, tampered payloads, JSON mode)
+spec/api-autoupdater-darwin-policy-spec.ts
+# autoUpdater module API surface
+spec/api-auto-updater-spec.ts
+# Node.js startup snapshot captures process.arch and friends; native addons
+spec/node-spec.ts
+# electron_xcache: the Node startup snapshot / code cache blob for this arch
+spec/xcache-spec.ts
+# js2c build-time V8 code cache and LoadBrowserProcessSpecificV8Snapshot
+spec/api-js2c-code-cache-spec.ts
+# sandboxed preload V8 code cache; the largest window-level surface
+spec/api-browser-window-spec.ts
+# codeCache for privileged custom protocols
+spec/api-protocol-spec.ts
+# wasm-eval CSP, WebAssembly streaming compilation, workers, heap snapshots
+spec/chromium-spec.ts
+# utility process Node isolate, ESM loading, heap snapshots near heap limit
+spec/api-utility-process-spec.ts
+# heap statistics, memory info and takeHeapSnapshot
+spec/api-process-spec.ts
+# native module loading (compiled per arch) and uv_dlopen
+spec/modules-spec.ts
+# child_process.execFile out of asar has a darwin x64-only case
+spec/asar-spec.ts
+# runningUnderARM64Translation, x64-only recent-documents skip, sandbox options
+spec/api-app-spec.ts
+# crashpad minidumps and stack walking are architecture specific
+spec/api-crash-reporter-spec.ts
+# renderer, main and node crash paths
+spec/crash-spec.ts