build: fail the PGO collect attempt on renderer death and relaunch (#54177)

* build: fail PGO workload when the renderer dies

A renderer OOM 13 seconds into the network workload left the benchmark app
awaiting webContents.executeJavaScript forever, so the collect job hung
until the 330-minute step timeout and collect-profile.js's retry loop,
which only runs once the app exits, never ran. Every page evaluation now
races against a deadline and a renderer-death signal, so a dead renderer
fails the running workload and the app quits into the retry path.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CMnZca8hbAx3iAsLvGkMzk

* build: recover PGO workload when the renderer dies

A renderer dying mid-workload no longer costs the whole collection
attempt. The benchmark app destroys the dead BrowserWindow, creates a
fresh one and retries that workload (PGO_RENDERER_RETRIES, default 2);
if the renderer keeps dying the workload is recorded with
`recovered: false` and the run continues so every other process still
shuts down cleanly and writes its counters. Only the dead renderer's own
counters are lost (%m exit-time writing on Linux/Windows; under Darwin's
%c continuous mode even those survive).

collect-profile.js accepts a run that gave up on a workload after in-app
renderer recovery instead of wiping the profraw dir and relaunching -
the loss is bounded and a relaunch would most likely die the same way.
The whole-attempt relaunch is kept for a run that was actually lost (the
app exited non-zero, or hung until killed, or left no results file) or
that failed for a reason the app does not recover in place;
PGO_ABORT_ON_FAILURE only cuts a run short for those. A clean run that
wrote no profraw at all (binary not instrumented) still fails fast.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CMnZca8hbAx3iAsLvGkMzk

* build: fail the PGO collect attempt on renderer death and relaunch

Drop the in-place renderer recovery: recreating the window and rerunning
the workload double-counts the aborted workload's counters in every
other process (and, under continuous mode, in the renderer's own file),
and giving up on a workload skipped part of the run while still passing
the attempt. A renderer death now records the workload with
rendererDeath: true, exits the app non-zero at once, and collect-profile
treats that as a lost attempt: the profraw dir is wiped and the whole
run relaunched, and when no retry is left the job fails without merging
or copying anything partial.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CMnZca8hbAx3iAsLvGkMzk

---------

Co-authored-by: Claude <noreply@anthropic.com>

script/pgo/benchmark-app/main.js

@@ -155,20 +155,16 @@ async function sleep(ms) {
 // of failing one workload and moving on.
 const NAVIGATION_TIMEOUT_MS = 2 * 60 * 1000;
 
-async function loadURLWithTimeout(win, url) {
-  const load = win.loadURL(url);
-  // If the timeout wins the race the eventual loadURL rejection has no
+async function withTimeout(promise, timeoutMs, label) {
+  // If the timeout wins the race the eventual rejection of `promise` has no
   // listener; swallow it so it does not surface as an unhandled rejection.
-  load.catch(() => {});
+  promise.catch(() => {});
   let timer;
   try {
-    await Promise.race([
-      load,
+    return await Promise.race([
+      promise,
       new Promise((_resolve, reject) => {
-        timer = setTimeout(
-          () => reject(new Error(`navigation to ${url.slice(0, 80)} timed out after ${NAVIGATION_TIMEOUT_MS / 1000}s`)),
-          NAVIGATION_TIMEOUT_MS
-        );
+        timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs / 1000}s`)), timeoutMs);
       })
     ]);
   } finally {
@@ -176,6 +172,53 @@ async function loadURLWithTimeout(win, url) {
   }
 }
 
+async function loadURLWithTimeout(win, url) {
+  await withTimeout(win.loadURL(url), NAVIGATION_TIMEOUT_MS, `navigation to ${url.slice(0, 80)}`);
+}
+
+// executeJavaScript never settles if the renderer dies mid-evaluation: the
+// reply simply never arrives. Observed on the windows-x86 collect job, where a
+// renderer OOM 13 seconds into the network workload left the app awaiting its
+// driver script for 5+ hours until the CI step timeout - collect-profile.js
+// only retries once the app exits, so the retry loop never ran. Every page
+// evaluation therefore races against both a deadline and the diagnostics'
+// renderer-death signal (attachDiagnostics), so a dead renderer fails the
+// running workload instead of hanging collection.
+//
+// POLL_TIMEOUT_MS bounds the short state queries runWorkload polls with;
+// EVALUATE_GRACE_MS is how long past their own deadline the renderer-side
+// drivers (IPC, network) may run before they are presumed hung - their loops
+// await in-flight requests that have no timeout of their own.
+const POLL_TIMEOUT_MS = 30 * 1000;
+const EVALUATE_GRACE_MS = 60 * 1000;
+
+async function evaluateInPage(win, diag, script, timeoutMs, label) {
+  const racers = [win.webContents.executeJavaScript(script, true)];
+  if (diag) racers.push(diag.rendererDeath);
+  return withTimeout(Promise.race(racers), timeoutMs, label);
+}
+
+function describeRendererGone(details) {
+  return `renderer process gone (reason: ${details.reason}, exit code: ${details.exitCode})`;
+}
+
+// Failures caused by the renderer dying carry the render-process-gone details
+// so the run loop can tell a renderer death (which aborts the whole attempt -
+// see below) from a genuine workload failure. Every failure path in the
+// window-driven workloads goes through tagRendererDeath so a death that
+// surfaced indirectly (a failed navigation, a driver that never became
+// startable) is classified the same way as one caught directly.
+function rendererGoneError(details) {
+  const err = new Error(describeRendererGone(details));
+  err.rendererGone = details;
+  return err;
+}
+
+function tagRendererDeath(err, diag) {
+  if (diag && diag.rendererGone && !err.rendererGone) err.rendererGone = diag.rendererGone;
+  return err;
+}
+
 // ---------------------------------------------------------------------------
 // Failure diagnostics
 //
@@ -187,20 +230,31 @@ async function loadURLWithTimeout(win, url) {
 // ---------------------------------------------------------------------------
 
 function attachDiagnostics(win) {
+  let failRenderer;
   const diag = {
     consoleErrors: [],
     rendererGone: null,
     unresponsive: false,
+    // Rejects once the renderer dies; page evaluations race against it (see
+    // evaluateInPage). Re-armed by reset() because each workload's loadURL
+    // spawns a fresh renderer.
+    rendererDeath: null,
     reset() {
       this.consoleErrors = [];
       this.rendererGone = null;
       this.unresponsive = false;
+      this.rendererDeath = new Promise((_resolve, reject) => {
+        failRenderer = reject;
+      });
+      this.rendererDeath.catch(() => {});
     }
   };
+  diag.reset();
 
   win.webContents.on('render-process-gone', (_event, details) => {
     diag.rendererGone = details;
     log(`DIAGNOSTIC: renderer process gone: ${JSON.stringify(details)}`);
+    failRenderer(rendererGoneError(details));
   });
   win.webContents.on('unresponsive', () => {
     diag.unresponsive = true;
@@ -240,6 +294,9 @@ async function reportWorkloadFailure(win, workloadName, diag) {
 
   // Memory + DOM snapshot, best effort: a hung renderer may never respond,
   // so give it a strict deadline rather than hanging the collection further.
+  // A dead renderer cannot answer at all; skip the query so the abort is not
+  // delayed by the deadline.
+  if (diag.rendererGone) return;
   try {
     const state = await Promise.race([
       win.webContents.executeJavaScript(
@@ -280,11 +337,23 @@ async function runWorkload(win, workload, diag) {
   log(`starting workload: ${workload.name}`);
   if (diag) diag.reset();
   const startTime = Date.now();
+
+  // A dead renderer never answers a poll (the evaluation fails instantly via
+  // diag.rendererDeath and is swallowed as "page busy"), so without this the
+  // loops below would spin until the workload deadline - up to 45 minutes.
+  const throwIfRendererGone = async () => {
+    if (!diag || !diag.rendererGone) return;
+    await reportWorkloadFailure(win, workload.name, diag);
+    throw tagRendererDeath(
+      new Error(`workload ${workload.name} aborted: ${describeRendererGone(diag.rendererGone)}`),
+      diag
+    );
+  };
   try {
     await loadURLWithTimeout(win, workload.url);
   } catch (err) {
     if (diag) await reportWorkloadFailure(win, workload.name, diag);
-    throw err;
+    throw tagRendererDeath(err, diag);
   }
 
   // Install page-side error capture for failure diagnostics. Benchmark
@@ -293,7 +362,9 @@ async function runWorkload(win, workload, diag) {
   // happened during static page load (before this injection) are detected by
   // checking driver state flags where they exist.
   try {
-    await win.webContents.executeJavaScript(
+    await evaluateInPage(
+      win,
+      diag,
       `(() => {
         window.__pgoErrors = window.__pgoErrors || [];
         if (typeof allIsGood !== 'undefined' && !allIsGood) {
@@ -311,7 +382,8 @@ async function runWorkload(win, workload, diag) {
         });
         return true;
       })()`,
-      true
+      POLL_TIMEOUT_MS,
+      'error capture injection'
     );
   } catch {
     /* page busy - diagnostics only */
@@ -328,15 +400,16 @@ async function runWorkload(win, workload, diag) {
     let started = false;
     for (let i = 0; i < 300 && !started; i++) {
       await sleep(1000);
+      await throwIfRendererGone();
       try {
-        started = await win.webContents.executeJavaScript(workload.startExpr, true);
+        started = await evaluateInPage(win, diag, workload.startExpr, POLL_TIMEOUT_MS, 'start poll');
       } catch {
         /* page busy - retry */
       }
       if (!started && workload.startFailExpr) {
         let unstartable = false;
         try {
-          unstartable = await win.webContents.executeJavaScript(workload.startFailExpr, true);
+          unstartable = await evaluateInPage(win, diag, workload.startFailExpr, POLL_TIMEOUT_MS, 'start-fail poll');
         } catch {
           /* page busy - keep polling */
         }
@@ -345,7 +418,7 @@ async function runWorkload(win, workload, diag) {
     }
     if (!started) {
       if (diag) await reportWorkloadFailure(win, workload.name, diag);
-      throw new Error(`workload ${workload.name} never became startable`);
+      throw tagRendererDeath(new Error(`workload ${workload.name} never became startable`), diag);
     }
   }
 
@@ -353,6 +426,7 @@ async function runWorkload(win, workload, diag) {
   const deadline = Date.now() + workload.timeoutMin * 60 * 1000;
   while (Date.now() < deadline) {
     await sleep(5000);
+    await throwIfRendererGone();
     // Soft cap: treat a long-running benchmark as complete. Profile counters
     // accumulate continuously, so everything up to this point is kept.
     if (workload.softCapMs && Date.now() - startTime > workload.softCapMs) {
@@ -363,7 +437,7 @@ async function runWorkload(win, workload, diag) {
     }
     let done = false;
     try {
-      done = await win.webContents.executeJavaScript(workload.doneExpr, true);
+      done = await evaluateInPage(win, diag, workload.doneExpr, POLL_TIMEOUT_MS, 'completion poll');
     } catch {
       /* page busy running benchmark - retry */
     }
@@ -374,21 +448,21 @@ async function runWorkload(win, workload, diag) {
       let succeeded = true;
       if (workload.successExpr) {
         try {
-          succeeded = await win.webContents.executeJavaScript(workload.successExpr, true);
+          succeeded = await evaluateInPage(win, diag, workload.successExpr, POLL_TIMEOUT_MS, 'success poll');
         } catch {
           succeeded = false;
         }
       }
       if (!succeeded) {
         if (diag) await reportWorkloadFailure(win, workload.name, diag);
-        throw new Error(`workload ${workload.name} ended in an error state after ${elapsed}s`);
+        throw tagRendererDeath(new Error(`workload ${workload.name} ended in an error state after ${elapsed}s`), diag);
       }
       log(`finished workload: ${workload.name} in ${elapsed}s`);
       return { name: workload.name, ok: true, seconds: Number(elapsed) };
     }
   }
   if (diag) await reportWorkloadFailure(win, workload.name, diag);
-  throw new Error(`workload ${workload.name} timed out after ${workload.timeoutMin} minutes`);
+  throw tagRendererDeath(new Error(`workload ${workload.name} timed out after ${workload.timeoutMin} minutes`), diag);
 }
 
 // ---------------------------------------------------------------------------
@@ -643,7 +717,9 @@ async function runIpcBridgeWorkload(durationMs, maxCalls) {
     // The driver runs in the main world and calls across the bridge. Payload
     // sizes cover the spectrum apps use: small control messages, medium JSON
     // payloads, and large binary transfers.
-    result = await win.webContents.executeJavaScript(
+    result = await evaluateInPage(
+      win,
+      diag,
       `(async () => {
     const small = { id: 1, type: 'msg', body: 'hello world' };
     const medium = { rows: Array.from({ length: 200 }, (_, i) => ({ i, name: 'row-' + i, values: [i, i * 2, i * 3] })) };
@@ -676,11 +752,12 @@ async function runIpcBridgeWorkload(durationMs, maxCalls) {
     }
     return { bridgeCalls, ipcCalls };
   })()`,
-      true
+      durationMs + EVALUATE_GRACE_MS,
+      'ipc-contextbridge renderer driver'
     );
   } catch (err) {
     await reportWorkloadFailure(win, 'ipc-contextbridge', diag);
-    throw err;
+    throw tagRendererDeath(err, diag);
   } finally {
     win.destroy();
     ipcMain.removeHandler('pgo-ping');
@@ -708,8 +785,9 @@ async function runIpcBridgeWorkload(durationMs, maxCalls) {
 // to point at the collection CA (set by collect-profile.js).
 // ---------------------------------------------------------------------------
 
-async function runNetworkWorkload(win, durationMs, maxRequests) {
+async function runNetworkWorkload(win, diag, durationMs, maxRequests) {
   log('starting workload: network');
+  diag.reset();
   const startTime = Date.now();
   const isTls = BASE_URL.startsWith('https:');
   const wsUrl = BASE_URL.replace(/^http/, 'ws') + '/__pgo/ws';
@@ -717,9 +795,13 @@ async function runNetworkWorkload(win, durationMs, maxRequests) {
   // Renderer: parallel fetches + WebSocket echo. Runs for the first half of
   // the budget; the Node-side loop runs for the second half.
   const rendererBudget = Math.floor(durationMs / 2);
-  await loadURLWithTimeout(win, `${BASE_URL}/speedometer/`);
-  const rendererResult = await win.webContents.executeJavaScript(
-    `(async () => {
+  let rendererResult;
+  try {
+    await loadURLWithTimeout(win, `${BASE_URL}/speedometer/`);
+    rendererResult = await evaluateInPage(
+      win,
+      diag,
+      `(async () => {
     const deadline = Date.now() + ${rendererBudget};
     const maxRequests = ${maxRequests};
     let requests = 0;
@@ -767,8 +849,13 @@ async function runNetworkWorkload(win, durationMs, maxRequests) {
 
     return { requests, wsMessages };
   })()`,
-    true
-  );
+      rendererBudget + EVALUATE_GRACE_MS,
+      'network renderer driver'
+    );
+  } catch (err) {
+    await reportWorkloadFailure(win, 'network', diag);
+    throw tagRendererDeath(err, diag);
+  }
 
   // Main process: Node-side HTTPS/HTTP requests.
   let nodeRequests = 0;
@@ -846,7 +933,7 @@ app.whenReady().then(async () => {
     () => runAsyncChurnWorkload(ASYNC_CHURN_MAX_OPS),
     () => runPackagedAppWorkload(),
     () => runIpcBridgeWorkload(IPC_BRIDGE_TIMEOUT_MS, IPC_BRIDGE_MAX_CALLS),
-    () => runNetworkWorkload(win, NETWORK_TIMEOUT_MS, NETWORK_MAX_REQUESTS)
+    () => runNetworkWorkload(win, diag, NETWORK_TIMEOUT_MS, NETWORK_MAX_REQUESTS)
   ];
   let phaseNames = [
     ...WORKLOADS.map((w) => w.name),
@@ -874,13 +961,27 @@ app.whenReady().then(async () => {
   // remaining workloads is wasted work. On the final attempt the variable is
   // unset and the run continues past failures so the partial profile keeps
   // as much coverage as possible.
+  //
+  // A renderer death is the exception: it always aborts the attempt. The
+  // aborted workload already ran partway in every other process (browser,
+  // GPU, network service), so neither carrying on nor retrying it in place
+  // can yield a profile weighted like one complete run - the orchestrator
+  // discards this attempt's counters and relaunches from scratch.
+  let rendererDied = false;
   for (let i = 0; i < phases.length; i++) {
     try {
       results.push(await phases[i]());
     } catch (err) {
       log(`ERROR: ${err.message}`);
-      results.push({ name: phaseNames[i], ok: false, error: err.message });
+      const result = { name: phaseNames[i], ok: false, error: err.message };
+      if (err.rendererGone) result.rendererDeath = true;
+      results.push(result);
       exitCode = 1;
+      if (err.rendererGone) {
+        rendererDied = true;
+        log('aborting the attempt: a renderer died, so its profile is unusable; the orchestrator will discard it');
+        break;
+      }
       if (process.env.PGO_ABORT_ON_FAILURE) {
         log('aborting remaining workloads; the orchestrator will wipe profiles and relaunch');
         break;
@@ -891,6 +992,14 @@ app.whenReady().then(async () => {
   fs.writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
   log(`results written to ${RESULTS_FILE}`);
 
+  // A renderer death leaves nothing worth flushing, so skip the clean
+  // shutdown and exit non-zero: collect-profile.js treats a non-zero exit as
+  // a lost attempt regardless of the results file.
+  if (rendererDied) {
+    app.exit(1);
+    return;
+  }
+
   // Clean shutdown is what flushes the PGO counters from every process. Note
   // that app.quit() always exits 0 regardless of process.exitCode (app.exit()
   // would honor a code but skips the clean shutdown we need), so failures are

script/pgo/collect-profile.js

@@ -375,6 +375,13 @@ async function main() {
   // abort on the first workload failure (PGO_ABORT_ON_FAILURE) since their
   // output is discarded anyway; the final attempt runs to completion so a
   // persistent failure still yields a maximal partial profile.
+  //
+  // The app exits non-zero when it aborts an attempt itself: a renderer death
+  // mid-workload leaves that workload half-run in every other process, so
+  // the app records it (rendererDeath: true) and bails immediately instead of
+  // finishing the run. Such an attempt is lost - its counters are wiped and
+  // the app relaunched - and if it is the last one, nothing is merged: a
+  // profile weighted by a partial workload must never be published.
   const maxAttempts = parseInt(args.attempts || '5', 10);
   // Attempts can be slow when a workload burns its own timeout before
   // failing (jetstream2 alone allows 45 minutes), so the attempt count alone
@@ -386,7 +393,9 @@ async function main() {
   const collectionStart = Date.now();
   let exitCode = 1;
   let failedWorkloads = [];
-  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+  let attemptLost = false;
+  let attempt = 0;
+  while (attempt++ < maxAttempts) {
     const finalAttempt = attempt === maxAttempts;
     // Always start clean: counters from a failed attempt (or stale %m pool
     // files from a previous run, which append) must not merge into this one.
@@ -425,21 +434,39 @@ async function main() {
 
     // Electron's clean-shutdown path (app.quit()) always exits 0, so workload
     // failures are reported through the results file rather than the exit code.
+    // A non-zero exit (the app aborted on a renderer death, crashed, or was
+    // killed) or a missing results file means the attempt did not run to a
+    // clean shutdown: its counters are not those of one complete run.
+    attemptLost = exitCode !== 0;
     failedWorkloads = [];
     if (fs.existsSync(resultsFile)) {
       const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
       log(`workload results:\n${JSON.stringify(results, null, 2)}`);
       failedWorkloads = results.filter((r) => !r.ok);
+      const rendererDeaths = failedWorkloads.filter((r) => r.rendererDeath);
+      if (rendererDeaths.length > 0) {
+        log(`renderer died during: ${rendererDeaths.map((w) => w.name).join(', ')} - attempt aborted`);
+        attemptLost = true;
+      }
     } else {
       log('WARNING: no results file was written - the app may have crashed');
       failedWorkloads = [{ name: 'all', error: 'results file missing' }];
+      attemptLost = true;
     }
     if (exitCode === 0 && failedWorkloads.length > 0) {
       exitCode = 1;
     }
     if (exitCode === 0 || finalAttempt) break;
+    // A clean run that wrote no profraw at all means the binary is not
+    // instrumented (or LLVM_PROFILE_FILE is not honoured); relaunching cannot
+    // fix that, so fail fast - the merge step below reports it.
+    const profrawCount = fs.readdirSync(profrawDir).filter((f) => f.endsWith('.profraw')).length;
+    if (!attemptLost && profrawCount === 0) {
+      log('no .profraw files written by a run that exited cleanly - not retrying');
+      break;
+    }
     if (Date.now() - collectionStart > retryDeadlineMs) {
-      log(`retry deadline reached after attempt ${attempt} - keeping this attempt's partial output`);
+      log(`retry deadline reached after attempt ${attempt} - not launching another`);
       break;
     }
     const reason = failedWorkloads.map((w) => w.name).join(', ') || `exit code ${exitCode}`;
@@ -447,6 +474,17 @@ async function main() {
   }
   server.close();
 
+  // The last attempt was lost and no retry is left, so there is no complete
+  // run to merge. Wipe the leftovers so nothing partial reaches --output
+  // (which CI uploads even when this step fails) and fail the job.
+  if (attemptLost) {
+    fs.rmSync(profrawDir, { recursive: true, force: true });
+    throw new Error(
+      `collection attempt ${attempt} was lost (${failedWorkloads.map((w) => w.name).join(', ')}) ` +
+        'and no retries remain - no profile to merge'
+    );
+  }
+
   // 3. Merge (or hand off the raw files for later merging).
   const profrawFiles = fs
     .readdirSync(profrawDir)