From b6ade12c7d5498fe2eaa0c55bc5cae81bedca0dd Mon Sep 17 00:00:00 2001 From: James M Snell Date: Sat, 29 Aug 2026 01:38:29 +0000 Subject: [PATCH] lib: add runFile api to node:bench Signed-off-by: James M Snell Assisted-by: Opencode PR-URL: https://github.com/nodejs/node/pull/65631 Reviewed-By: Filip Skokan --- doc/api/bench.md | 48 +- lib/bench.js | 2 + lib/internal/bench_runner/cli.js | 584 +++++++++++++++--- lib/internal/bench_runner/runner.js | 5 + .../bench-runner/run-file-blocked.cjs | 7 + .../bench-runner/run-file-lingering.cjs | 11 + .../bench-runner/run-file-partial.cjs | 11 + test/fixtures/bench-runner/run-file.cjs | 21 + test/parallel/test-bench-cli.js | 2 + test/parallel/test-bench-run-file.js | 318 ++++++++++ 10 files changed, 920 insertions(+), 89 deletions(-) create mode 100644 test/fixtures/bench-runner/run-file-blocked.cjs create mode 100644 test/fixtures/bench-runner/run-file-lingering.cjs create mode 100644 test/fixtures/bench-runner/run-file-partial.cjs create mode 100644 test/fixtures/bench-runner/run-file.cjs create mode 100644 test/parallel/test-bench-run-file.js diff --git a/doc/api/bench.md b/doc/api/bench.md index c36ce3eed29..2eb6eb6a382 100644 --- a/doc/api/bench.md +++ b/doc/api/bench.md @@ -11,7 +11,8 @@ added: REPLACEME The `node:bench` module supports defining and running JavaScript benchmarks in -the current process. To access it: +the current process, and running one benchmark file in a fresh child process. +To access it: ```mjs import { bench, suite } from 'node:bench'; @@ -497,6 +498,51 @@ for await (const { type, data } of run()) { } ``` +## `runFile(path[, options])` + + + +* `path` {string} The absolute path of one benchmark module. +* `options` {Object} + * `env` {Object} The child process environment. Property values must be + strings or `undefined`. This replaces, rather than extends, the parent + environment. **Default:** A snapshot of `process.env`. + * `execArgv` {string\[]} Node.js command-line options for the child process. + This replaces, rather than extends, inherited options. Benchmark runner + options, positional arguments, and options that select another execution + mode are not allowed. **Default:** Compatible options inherited from the + current process. + * `signal` {AbortSignal} Terminates the child process when aborted. +* Returns: {BenchmarksStream} + +Runs exactly one benchmark module in a fresh child process and returns its +object-mode event stream. `path` is not interpreted as a glob. Unless the signal +is aborted or the stream is destroyed before startup, every call uses a new +child. Input discovery, ordering, concurrency, retries, and multi-file +scheduling remain the caller's responsibility. + +Records use advanced child process serialization, preserving supported +structured values such as `bigint` and errors. Child writes to stdout and stderr +become `'bench:diagnostic'` records. A module loading error, abnormal child exit, +or cancellation also emits an error diagnostic and produces a terminal +`'bench:summary'` whose `success` property is `false`; these execution failures +do not error the stream. If module evaluation fails after declaring benchmarks, +those declarations still run before the unsuccessful summary. + +`env`, effective inherited options, and an explicitly provided `execArgv` are +copied when `runFile()` is called. The runner removes `NODE_OPTIONS`, replaces +IPC-related environment variables, and sets its private child-context, run +identity, and file identity variables, overriding properties with those names +in `env`. Pass child Node.js options through `execArgv`, not `NODE_OPTIONS`. +Standard `child_process` environment propagation still applies, including +`NODE_V8_COVERAGE`, permission-model options, and required z/OS variables. +Aborting `signal` before the child starts produces an `AbortError` diagnostic +without spawning it. Aborting during execution sends `SIGTERM` to the child and +escalates to `SIGKILL` if it does not exit. Destroying the returned stream +follows the same termination procedure. + ## Class: `BenchContext` An instance of `BenchContext` is passed to every benchmark invocation. A new diff --git a/lib/bench.js b/lib/bench.js index 9d1bd24fceb..96e199773d9 100644 --- a/lib/bench.js +++ b/lib/bench.js @@ -16,6 +16,7 @@ const { const { createRunner, run, + runFile, } = require('internal/bench_runner/runner'); if (process.env.NODE_BENCH_CONTEXT !== 'child' || @@ -33,5 +34,6 @@ ObjectAssign(module.exports, { createRunner, describe: suite, run, + runFile, suite, }); diff --git a/lib/internal/bench_runner/cli.js b/lib/internal/bench_runner/cli.js index 38611e49a99..4b14493e391 100644 --- a/lib/internal/bench_runner/cli.js +++ b/lib/internal/bench_runner/cli.js @@ -8,13 +8,17 @@ const { ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypePushApply, + ArrayPrototypeSlice, ArrayPrototypeSome, ArrayPrototypeSort, + MathMax, NumberIsFinite, NumberIsSafeInteger, ObjectGetOwnPropertyDescriptor, ObjectGetPrototypeOf, + ObjectKeys, ObjectPrototype, + ObjectPrototypeHasOwnProperty, ObjectValues, Promise, PromisePrototypeThen, @@ -27,8 +31,11 @@ const { SafeSet, String, StringPrototypeIndexOf, + StringPrototypeReplaceAll, StringPrototypeSlice, StringPrototypeStartsWith, + StringPrototypeToUpperCase, + SymbolDispose, } = primordials; const { spawn } = require('child_process'); const { createWriteStream, statSync } = require('fs'); @@ -44,19 +51,32 @@ const { } = require('internal/bench_runner/harness'); const { deserializeError, serializeError } = require('internal/error_serdes'); const { + AbortError, codes: { + ERR_INVALID_ARG_TYPE, ERR_INVALID_ARG_VALUE, ERR_INVALID_STATE, }, } = require('internal/errors'); -const { getOptionValue, getOptionsAsFlagsFromBinding } = require('internal/options'); +const { addAbortListener } = require('internal/events/abort_listener'); +const { + getCLIOptionsInfo, + getOptionValue, + getOptionsAsFlagsFromBinding, +} = require('internal/options'); const { TIMEOUT_MAX } = require('internal/timers'); const { kEmptyObject } = require('internal/util'); -const { validateUint32 } = require('internal/validators'); +const { + validateAbortSignal, + validateArray, + validateObject, + validateStringWithoutNullBytes, + validateUint32, +} = require('internal/validators'); const { pathToFileURL } = require('internal/url'); const { pipeline } = require('stream/promises'); -const { once } = require('events'); -const { resolve, sep } = require('path'); +const { isAbsolute, resolve, sep } = require('path'); +const { clearTimeout, setTimeout } = require('timers'); const console = require('internal/console/global'); const esmLoader = require('internal/modules/esm/loader'); @@ -90,6 +110,51 @@ const kFilterArgValues = [ '--bench-warmup', '--experimental-config-file', ]; +const kIncompatibleExecArgv = new SafeSet([ + '--build-sea', + '--build-snapshot', + '--build-snapshot-config', + '--check', + '--completion-bash', + '--eval', + '--experimental-sea-config', + '--help', + '--help-all', + '--input-type', + '--interactive', + '--print', + '--prof-process', + '--run', + '--test', + '--version', + '--v8-options', + '--watch', + '--watch-path', + '-c', + '-e', + '-h', + '-i', + '-p', + '-v', +]); +const kExecArgvWithValue = new SafeSet([ + '--eval', + '--input-type', + '--print', + '--run', + '--watch-path', + '-e', + '-p', +]); +const kIPCEnvironmentVariables = new SafeSet([ + 'NODE_CHANNEL_FD', + 'NODE_CHANNEL_SERIALIZATION_MODE', + 'NODE_BENCH_CONTEXT', + 'NODE_BENCH_FILE_RUN_ID', + 'NODE_BENCH_RUN_ID', + 'NODE_OPTIONS', +]); +const kForceKillDelay = 1_000; function createBenchmarkFileList(patterns, cwd) { if (patterns.length === 0) { @@ -533,6 +598,26 @@ async function loadBenchmarkFiles(files, options, modules, onRecord) { }, true); let summary; for await (const record of stream) { + if (record.type === 'bench:summary' && (process.exitCode ?? 0) !== 0) { + const scope = files.length === 1 ? options.fileScopes[0] : { + __proto__: null, + entryFile: null, + fileRunId: null, + }; + await onRecord({ + __proto__: null, + type: 'bench:diagnostic', + data: { + __proto__: null, + runId: options.runId, + ...scope, + message: `Benchmark process set exit code ${process.exitCode}`, + level: 'error', + file: files.length === 1 ? + resolve(options.cwd, files[0]) : null, + }, + }); + } if (record.type === 'bench:summary' && (loadFailed || (process.exitCode ?? 0) !== 0)) { record.data.success = false; @@ -552,18 +637,62 @@ async function loadBenchmarkFiles(files, options, modules, onRecord) { } function filterExecArgv(arg, index, args) { - return !ArrayPrototypeIncludes(kFilterArgs, arg) && + const name = getOptionName(arg); + return !ArrayPrototypeIncludes(kFilterArgs, name) && !ArrayPrototypeSome(kFilterArgValues, (option) => { - return arg === option || - StringPrototypeStartsWith(arg, `${option}=`) || + return name === option || (option !== '--experimental-config-file' && - index > 0 && args[index - 1] === option); + index > 0 && getOptionName(args[index - 1]) === option); }); } function getOptionName(arg) { const equals = StringPrototypeIndexOf(arg, '='); - return equals === -1 ? arg : StringPrototypeSlice(arg, 0, equals); + const name = equals === -1 ? arg : StringPrototypeSlice(arg, 0, equals); + return StringPrototypeReplaceAll(name, '_', '-'); +} + +function getRunFileOptionName(arg) { + if (StringPrototypeStartsWith(arg, '-') && + !StringPrototypeStartsWith(arg, '--') && arg.length > 2) { + const shortName = StringPrototypeSlice(arg, 0, 2); + if (kIncompatibleExecArgv.has(shortName)) return shortName; + } + return getOptionName(arg); +} + +function filterRunFileExecArgv(arg, index, args) { + if (!filterExecArgv(arg, index, args)) return false; + if (!StringPrototypeStartsWith(arg, '-') || arg === '-' || arg === '--') { + return false; + } + const name = getRunFileOptionName(arg); + if (kIncompatibleExecArgv.has(name)) return false; + if (index > 0) { + const previous = getRunFileOptionName(args[index - 1]); + if (kExecArgvWithValue.has(previous) && + StringPrototypeIndexOf(args[index - 1], '=') === -1) { + return false; + } + } + return true; +} + +function runFileOptionRequiresValue(arg) { + const equals = StringPrototypeIndexOf(arg, '='); + let name = getRunFileOptionName(arg); + const { aliases, options } = getCLIOptionsInfo(); + let info = options.get(name); + if (info === undefined) { + const alias = aliases.get(name); + if (alias !== undefined) info = options.get(alias[0]); + } + if (info === undefined && StringPrototypeStartsWith(name, '--no-')) { + name = `--${StringPrototypeSlice(name, 5)}`; + info = options.get(name); + } + return info !== undefined && info.type >= 3 && + (equals === -1 || equals === arg.length - 1); } const kOptionAliases = new SafeMap([ @@ -572,7 +701,7 @@ const kOptionAliases = new SafeMap([ ['-r', '--require'], ]); -function getChildArgs(path, options) { +function getInheritedChildArgs() { const nodeOptions = getOptionsAsFlagsFromBinding(); const args = ArrayPrototypeFilter(nodeOptions, filterExecArgv); const nodeOptionNames = new SafeSet(); @@ -597,6 +726,12 @@ function getChildArgs(path, options) { ArrayPrototypePushApply(args, unknownExecArgv); // Option serialization omits port 0, which would otherwise become 9229. if (process.debugPort === 0) ArrayPrototypePush(args, '--inspect-port=0'); + return args; +} + +function getChildArgs(path, options) { + const args = options.execArgv === undefined ? + getInheritedChildArgs() : ArrayPrototypeSlice(options.execArgv); ArrayPrototypePush(args, '--bench', '--bench-isolation=none'); if (options.namePatternSource.length > 0) { ArrayPrototypePush( @@ -613,24 +748,100 @@ function getChildArgs(path, options) { } async function runChild(path, options, scope, onRecord) { - const child = spawn(process.execPath, getChildArgs(path, options), { - __proto__: null, - cwd: options.cwd, - env: { + if (options.signal?.aborted) { + return { __proto__: null, - ...process.env, - NODE_BENCH_CONTEXT: 'child', - NODE_BENCH_FILE_RUN_ID: scope.fileRunId, - NODE_BENCH_RUN_ID: options.runId, + aborted: true, + error: new AbortError(undefined, { + __proto__: null, + cause: options.signal.reason, + }), + }; + } + const child = spawn( + options.execPath ?? process.execPath, + getChildArgs(path, options), + { + __proto__: null, + cwd: options.cwd, + env: { + __proto__: null, + ...(options.env ?? process.env), + NODE_BENCH_CONTEXT: 'child', + NODE_BENCH_FILE_RUN_ID: scope.fileRunId, + NODE_BENCH_RUN_ID: options.runId, + }, + serialization: 'advanced', + stdio: ['inherit', 'pipe', 'pipe', 'ipc'], }, - serialization: 'advanced', - stdio: ['inherit', 'pipe', 'pipe', 'ipc'], - }); + ); + let childClosed = false; + let forceKillTimer; + const terminateChild = () => { + if (childClosed) return; + child.kill(); + forceKillTimer ??= setTimeout(() => { + child.kill('SIGKILL'); + }, kForceKillDelay); + }; + const closed = PromiseWithResolvers(); + let closeTracked = false; + let spawnError; + try { + child.once('close', (...status) => closed.resolve(status)); + closeTracked = true; + child.once('error', (error) => { + spawnError ??= error; + terminateChild(); + }); + } catch (error) { + terminateChild(); + if (closeTracked) { + try { + await closed.promise; + } catch { + // Preserve the setup error. + } + } + throw error; + } let protocolError; + let abortError; + let aborted = false; let activeBenchId; let plansComplete = false; let recordPending = false; let summaryReceived = false; + let abortListener; + const closeListener = () => { + terminateChild(); + }; + try { + if (options.signal !== undefined) { + abortListener = addAbortListener(options.signal, () => { + aborted = true; + abortError = new AbortError(undefined, { + __proto__: null, + cause: options.signal.reason, + }); + terminateChild(); + }); + } + options.output?.once('close', closeListener); + if (options.output?.destroyed) closeListener(); + } catch (error) { + terminateChild(); + try { + await closed.promise; + } catch { + // Preserve the setup error. + } + childClosed = true; + if (forceKillTimer !== undefined) clearTimeout(forceKillTimer); + abortListener?.[SymbolDispose](); + options.output?.removeListener('close', closeListener); + throw error; + } const pendingRecords = new SafeSet(); const handleRecord = (record) => { if (protocolError !== undefined) return; @@ -638,7 +849,7 @@ async function runChild(path, options, scope, onRecord) { return onRecord(record); } catch (error) { protocolError = error; - child.kill(); + terminateChild(); } }; const trackPending = (pending, source) => { @@ -649,8 +860,8 @@ async function runChild(path, options, scope, onRecord) { source?.resume(); }, (error) => { pendingRecords.delete(tracked); - protocolError ??= error; - child.kill(); + if (!aborted) protocolError ??= error; + terminateChild(); }); pendingRecords.add(tracked); }; @@ -670,69 +881,99 @@ async function runChild(path, options, scope, onRecord) { }); trackPending(pending, source); }; - child.stdout.setEncoding('utf8'); - child.stderr.setEncoding('utf8'); - child.stdout.on('data', (data) => { - reportOutput(child.stdout, 'stdout', data); - }); - child.stderr.on('data', (data) => { - reportOutput(child.stderr, 'stderr', data); - }); - child.on('message', (message) => { - if (message?.type !== kChildMessageType) return; + try { + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (data) => { + reportOutput(child.stdout, 'stdout', data); + }); + child.stderr.on('data', (data) => { + reportOutput(child.stderr, 'stderr', data); + }); + child.on('message', (message) => { + if (message?.type !== kChildMessageType) return; + try { + if (!NumberIsSafeInteger(message.id) || message.id < 0 || recordPending) { + throw new ERR_INVALID_ARG_VALUE( + 'benchmark child message', message, + 'does not have a valid record sequence'); + } + recordPending = true; + const record = deserializeRecord(validateRecord(message.record)); + const contextDiagnostic = isContextDiagnostic(record); + if (summaryReceived || (record.type === 'bench:plan' && plansComplete) || + (record.type === 'bench:start' && activeBenchId !== undefined) || + ((record.type === 'bench:sample' || contextDiagnostic) && + activeBenchId !== record.data.benchId) || + (record.type === 'bench:complete' && activeBenchId !== undefined && + activeBenchId !== record.data.benchId) || + (record.type === 'bench:summary' && activeBenchId !== undefined)) { + throw new ERR_INVALID_ARG_VALUE( + 'benchmark child message', message, + 'does not have a valid lifecycle sequence'); + } + if (record.type !== 'bench:plan' && + record.type !== 'bench:diagnostic') { + plansComplete = true; + } + if (record.type === 'bench:start') { + activeBenchId = record.data.benchId; + } else if (record.type === 'bench:complete' && + activeBenchId === record.data.benchId) { + activeBenchId = undefined; + } + if (record.type === 'bench:summary') summaryReceived = true; + record.data.runId = options.runId; + record.data.fileRunId = scope.fileRunId; + if (record.data.entryFile !== null) { + record.data.entryFile = scope.entryFile; + } + const pending = handleRecord(record); + if (protocolError === undefined) { + const acknowledged = PromisePrototypeThen( + PromiseResolve(pending), () => sendAck(child, message.id)); + trackPending(PromisePrototypeThen(acknowledged, () => { + recordPending = false; + })); + } + } catch (error) { + if (!aborted) protocolError ??= error; + terminateChild(); + } + }); + } catch (error) { + terminateChild(); try { - if (!NumberIsSafeInteger(message.id) || message.id < 0 || recordPending) { - throw new ERR_INVALID_ARG_VALUE( - 'benchmark child message', message, - 'does not have a valid record sequence'); - } - recordPending = true; - const record = deserializeRecord(validateRecord(message.record)); - const contextDiagnostic = isContextDiagnostic(record); - if (summaryReceived || (record.type === 'bench:plan' && plansComplete) || - (record.type === 'bench:start' && activeBenchId !== undefined) || - ((record.type === 'bench:sample' || contextDiagnostic) && - activeBenchId !== record.data.benchId) || - (record.type === 'bench:complete' && activeBenchId !== undefined && - activeBenchId !== record.data.benchId) || - (record.type === 'bench:summary' && activeBenchId !== undefined)) { - throw new ERR_INVALID_ARG_VALUE( - 'benchmark child message', message, - 'does not have a valid lifecycle sequence'); - } - if (record.type !== 'bench:plan' && - record.type !== 'bench:diagnostic') { - plansComplete = true; - } - if (record.type === 'bench:start') { - activeBenchId = record.data.benchId; - } else if (record.type === 'bench:complete' && - activeBenchId === record.data.benchId) { - activeBenchId = undefined; - } - if (record.type === 'bench:summary') summaryReceived = true; - record.data.runId = options.runId; - record.data.fileRunId = scope.fileRunId; - if (record.data.entryFile !== null) { - record.data.entryFile = scope.entryFile; - } - const pending = handleRecord(record); - if (protocolError === undefined) { - const acknowledged = PromisePrototypeThen( - PromiseResolve(pending), () => sendAck(child, message.id)); - trackPending(PromisePrototypeThen(acknowledged, () => { - recordPending = false; - })); - } - } catch (error) { - protocolError ??= error; - child.kill(); + await closed.promise; + } catch { + // Preserve the setup error. } - }); - const { 0: code, 1: signal } = await once(child, 'close'); + childClosed = true; + if (forceKillTimer !== undefined) clearTimeout(forceKillTimer); + abortListener?.[SymbolDispose](); + options.output?.removeListener('close', closeListener); + throw error; + } + let status; + try { + status = await closed.promise; + } finally { + childClosed = true; + if (forceKillTimer !== undefined) clearTimeout(forceKillTimer); + abortListener?.[SymbolDispose](); + options.output?.removeListener('close', closeListener); + } + const { 0: code, 1: signal } = status; await SafePromiseAllReturnVoid(ArrayFrom(pendingRecords)); - if (protocolError !== undefined) throw protocolError; - return { __proto__: null, code, signal }; + if (!aborted && spawnError !== undefined) throw spawnError; + if (!aborted && protocolError !== undefined) throw protocolError; + return { + __proto__: null, + aborted, + code, + error: abortError, + signal, + }; } async function runIsolated(files, options, output) { @@ -749,6 +990,13 @@ async function runIsolated(files, options, output) { for (let i = 0; i < files.length; i++) { const path = files[i]; const scope = options.fileScopes[i]; + const observedCounts = { + __proto__: null, + completed: 0, + failed: 0, + skipped: 0, + total: 0, + }; let childSummary; let result; try { @@ -757,6 +1005,16 @@ async function runIsolated(files, options, output) { childSummary = record.data; return; } + if (record.type === 'bench:plan') observedCounts.total++; + if (record.type === 'bench:complete') { + if (ObjectPrototypeHasOwnProperty(record.data, 'error')) { + observedCounts.failed++; + } else if (ObjectPrototypeHasOwnProperty(record.data, 'skip')) { + observedCounts.skipped++; + } else { + observedCounts.completed++; + } + } return emitRecordAndWait(output, record); }); } catch (error) { @@ -774,18 +1032,49 @@ async function runIsolated(files, options, output) { file: path, }, }); + counts.completed += observedCounts.completed; + counts.failed += observedCounts.failed; + counts.skipped += observedCounts.skipped; + counts.total += MathMax( + observedCounts.total, + observedCounts.completed + observedCounts.failed + + observedCounts.skipped); continue; } + if (result.aborted) { + success = false; + await emitRecordAndWait(output, { + __proto__: null, + type: 'bench:diagnostic', + data: { + __proto__: null, + runId: options.runId, + ...scope, + message: result.error.message, + error: result.error, + level: 'error', + file: path, + }, + }); + } if (childSummary !== undefined) { success &&= childSummary.success; counts.completed += childSummary.counts.completed; counts.failed += childSummary.counts.failed; counts.skipped += childSummary.counts.skipped; counts.total += childSummary.counts.total; + } else { + counts.completed += observedCounts.completed; + counts.failed += observedCounts.failed; + counts.skipped += observedCounts.skipped; + counts.total += MathMax( + observedCounts.total, + observedCounts.completed + observedCounts.failed + + observedCounts.skipped); } - if (childSummary === undefined || result.code !== 0 || - result.signal !== null) { + if (!result.aborted && (childSummary === undefined || result.code !== 0 || + result.signal !== null)) { success = false; if (childSummary === undefined || childSummary.success) { const status = result.signal === null ? @@ -812,7 +1101,8 @@ async function runIsolated(files, options, output) { runId: options.runId, fileRunId: scope?.fileRunId ?? null, entryFile: scope?.entryFile ?? null, - success: success && (process.exitCode ?? 0) === 0, + success: success && (options.useProcessExitCode === false || + (process.exitCode ?? 0) === 0), counts, duration_ns: hrtime() - start, file: files.length === 1 ? resolve(options.cwd, files[0]) : null, @@ -825,6 +1115,124 @@ async function runIsolated(files, options, output) { return summary; } +function runFile(path, options = kEmptyObject) { + validateStringWithoutNullBytes(path, 'path'); + if (!isAbsolute(path)) { + throw new ERR_INVALID_ARG_VALUE('path', path, 'must be an absolute path'); + } + const file = resolve(path); + validateObject(options, 'options'); + const { + env = process.env, + execArgv, + signal, + } = options; + let childExecArgv; + if (execArgv !== undefined) { + validateArray(execArgv, 'options.execArgv'); + childExecArgv = []; + const length = execArgv.length; + for (let i = 0; i < length; i++) { + const arg = execArgv[i]; + validateStringWithoutNullBytes(arg, `options.execArgv[${i}]`); + if (!StringPrototypeStartsWith(arg, '-') || arg === '-' || arg === '--') { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, arg, + 'must be a Node.js command-line option'); + } + if (kIncompatibleExecArgv.has(getRunFileOptionName(arg))) { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, arg, + 'is not compatible with benchmark execution'); + } + if (runFileOptionRequiresValue(arg)) { + const equals = StringPrototypeIndexOf(arg, '='); + if (equals !== -1) { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, arg, + 'must include a non-empty value'); + } + if (i + 1 >= length) { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, arg, + 'must be followed by a value'); + } + ArrayPrototypePush(childExecArgv, arg); + i++; + const value = execArgv[i]; + validateStringWithoutNullBytes(value, `options.execArgv[${i}]`); + if (value.length === 0) { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, value, + 'must not be empty'); + } + ArrayPrototypePush(childExecArgv, value); + continue; + } + if (!StringPrototypeStartsWith(arg, '--') && + StringPrototypeIndexOf(arg, '=') !== -1) { + throw new ERR_INVALID_ARG_VALUE( + `options.execArgv[${i}]`, arg, + 'must not use = with a short option'); + } + ArrayPrototypePush(childExecArgv, arg); + } + if (ArrayPrototypeFilter(childExecArgv, filterExecArgv).length !== + childExecArgv.length) { + throw new ERR_INVALID_ARG_VALUE( + 'options.execArgv', childExecArgv, + 'must not contain benchmark runner options'); + } + } else { + childExecArgv = ArrayPrototypeFilter( + getInheritedChildArgs(), filterRunFileExecArgv); + } + validateObject(env, 'options.env'); + const childEnv = { __proto__: null }; + const envKeys = ObjectKeys(env); + for (let i = 0; i < envKeys.length; i++) { + const key = envKeys[i]; + validateStringWithoutNullBytes(key, 'options.env key'); + const value = env[key]; + if (value === undefined) continue; + validateStringWithoutNullBytes(value, `options.env.${key}`); + if (kIPCEnvironmentVariables.has(StringPrototypeToUpperCase(key))) continue; + childEnv[key] = value; + } + validateAbortSignal(signal, 'options.signal'); + if (signal !== undefined && + (typeof signal.addEventListener !== 'function' || + typeof signal.removeEventListener !== 'function')) { + throw new ERR_INVALID_ARG_TYPE('options.signal', 'AbortSignal', signal); + } + + const runId = createRunId(); + const scope = { + __proto__: null, + entryFile: file, + fileRunId: createRunId(), + }; + const output = new BenchmarksStream(); + const runOptions = { + __proto__: null, + cwd: process.cwd(), + env: childEnv, + execPath: process.execPath, + execArgv: childExecArgv, + fileScopes: [scope], + namePatternSource: '', + output, + runId, + signal, + useProcessExitCode: false, + }; + const execution = PromisePrototypeThen(PromiseResolve(), () => + (output.destroyed ? undefined : runIsolated([file], runOptions, output))); + PromisePrototypeThen(execution, () => output.end(), + (error) => output.destroy(error)); + return output; +} + async function run(patterns) { const options = parseCommandLine(); const files = options.isChild ? @@ -883,4 +1291,4 @@ async function run(patterns) { return summary; } -module.exports = { run }; +module.exports = { run, runFile }; diff --git a/lib/internal/bench_runner/runner.js b/lib/internal/bench_runner/runner.js index fc8595d2a35..c48a0a5339f 100644 --- a/lib/internal/bench_runner/runner.js +++ b/lib/internal/bench_runner/runner.js @@ -10,7 +10,12 @@ function run(options = kEmptyObject) { return runBenchmarks(options); } +function runFile(path, options = kEmptyObject) { + return require('internal/bench_runner/cli').runFile(path, options); +} + module.exports = { createRunner, run, + runFile, }; diff --git a/test/fixtures/bench-runner/run-file-blocked.cjs b/test/fixtures/bench-runner/run-file-blocked.cjs new file mode 100644 index 00000000000..9298d0dccc4 --- /dev/null +++ b/test/fixtures/bench-runner/run-file-blocked.cjs @@ -0,0 +1,7 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('blocked run file', { samples: 1 }, async () => { + await new Promise(() => {}); +}); diff --git a/test/fixtures/bench-runner/run-file-lingering.cjs b/test/fixtures/bench-runner/run-file-lingering.cjs new file mode 100644 index 00000000000..954bc108d01 --- /dev/null +++ b/test/fixtures/bench-runner/run-file-lingering.cjs @@ -0,0 +1,11 @@ +'use strict'; + +const { bench } = require('node:bench'); + +setInterval(() => {}, 1_000); +if (process.platform !== 'win32') process.on('SIGTERM', () => {}); +process.on('disconnect', () => process.stdout.write('child disconnected\n')); + +bench('lingering run file', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1 }); +}); diff --git a/test/fixtures/bench-runner/run-file-partial.cjs b/test/fixtures/bench-runner/run-file-partial.cjs new file mode 100644 index 00000000000..9e3019e94e6 --- /dev/null +++ b/test/fixtures/bench-runner/run-file-partial.cjs @@ -0,0 +1,11 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('completed before abort', { samples: 1 }, (b) => { + b.record({ duration_ns: 1n, operations: 1 }); +}); + +bench('aborted run file', { samples: 1 }, async () => { + await new Promise(() => {}); +}); diff --git a/test/fixtures/bench-runner/run-file.cjs b/test/fixtures/bench-runner/run-file.cjs new file mode 100644 index 00000000000..382af4cc42f --- /dev/null +++ b/test/fixtures/bench-runner/run-file.cjs @@ -0,0 +1,21 @@ +'use strict'; + +const { bench } = require('node:bench'); + +bench('run file', { + samples: 1, + params: { + context: process.env.NODE_BENCH_CONTEXT, + exposed: typeof globalThis.gc === 'function', + value: process.env.NODE_BENCH_RUN_FILE ?? 'unset', + }, +}, (b) => { + b.record({ + duration_ns: 1n, + operations: 1, + detail: { + execArgv: process.execArgv, + pid: process.pid, + }, + }); +}); diff --git a/test/parallel/test-bench-cli.js b/test/parallel/test-bench-cli.js index bd67b4b45bd..d0f9629ad9a 100644 --- a/test/parallel/test-bench-cli.js +++ b/test/parallel/test-bench-cli.js @@ -709,6 +709,8 @@ if (common.hasInspector) { ]); assert.strictEqual(result.status, 1); const records = parseRecords(result); + assert(records.some(({ type, data }) => + type === 'bench:diagnostic' && /set exit code/.test(data.message))); assert.strictEqual(records.at(-1).data.success, false); } diff --git a/test/parallel/test-bench-run-file.js b/test/parallel/test-bench-run-file.js new file mode 100644 index 00000000000..eccbf2e7296 --- /dev/null +++ b/test/parallel/test-bench-run-file.js @@ -0,0 +1,318 @@ +// Flags: --no-warnings +'use strict'; + +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const { once } = require('events'); +const path = require('path'); +const { runFile } = require('node:bench'); + +const fixture = fixtures.path('bench-runner/run-file.cjs'); + +assert.throws(() => runFile(null), { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => runFile('relative.cjs'), { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => runFile(fixture, null), { code: 'ERR_INVALID_ARG_TYPE' }); +assert.throws(() => runFile(fixture, { execArgv: null }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { execArgv: [1] }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['--bench'] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: [fixture] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['-e', '0'] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['--require'] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['--require='] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: [`-r=${fixture}`] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['--prof-process'] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { execArgv: ['--prof_process'] }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { + execArgv: ['--bench_name_pattern=run'], +}), { code: 'ERR_INVALID_ARG_VALUE' }); +assert.throws(() => runFile(`${fixture}\0`), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { env: null }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { env: { INVALID: 1 } }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { env: { INVALID: 'x\0' } }), { + code: 'ERR_INVALID_ARG_VALUE', +}); +assert.throws(() => runFile(fixture, { env: { NODE_CHANNEL_FD: 1 } }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { signal: {} }), { + code: 'ERR_INVALID_ARG_TYPE', +}); +assert.throws(() => runFile(fixture, { signal: { aborted: false } }), { + code: 'ERR_INVALID_ARG_TYPE', +}); + +async function testRunFile() { + const execArgv = ['--expose-gc', '-r', 'fs']; + const env = { + __proto__: null, + ...process.env, + NODE_BENCH_CONTEXT: 'not-a-child', + NODE_BENCH_RUN_FILE: 'original', + NODE_CHANNEL_FD: '999', + NODE_CHANNEL_SERIALIZATION_MODE: 'json', + }; + const stream = runFile(fixture, { env, execArgv }); + execArgv.length = 0; + env.NODE_BENCH_RUN_FILE = 'mutated'; + const records = await stream.toArray(); + const plan = records.find(({ type }) => type === 'bench:plan').data; + const result = records.find(({ type }) => type === 'bench:complete').data; + const summary = records.at(-1).data; + + assert.strictEqual(records.at(-1).type, 'bench:summary'); + assert.strictEqual(plan.params.context, 'child'); + assert.strictEqual(plan.params.exposed, true); + assert.strictEqual(plan.params.value, 'original'); + assert.strictEqual(result.error, undefined); + assert.strictEqual(result.samples.length, 1); + assert.strictEqual(typeof result.samples[0].duration_ns, 'bigint'); + assert.notStrictEqual(result.samples[0].detail.pid, process.pid); + assert(result.samples[0].detail.execArgv.includes('--expose-gc')); + assert(result.samples[0].detail.execArgv.includes('-r')); + assert.strictEqual(summary.success, true); + assert.strictEqual(summary.file, fixture); + assert.strictEqual(summary.entryFile, fixture); + assert.strictEqual(summary.fileRunId, result.fileRunId); + assert.strictEqual(summary.runId, result.runId); + assert.strictEqual(records.filter( + ({ type }) => type === 'bench:summary').length, 1); +} + +async function testConcurrentCalls() { + const [cjsRecords, esmRecords] = await Promise.all([ + runFile(fixtures.path('bench-runner/a.cjs')).toArray(), + runFile(fixtures.path('bench-runner/b.mjs')).toArray(), + ]); + const cjsResult = cjsRecords.find( + ({ type }) => type === 'bench:complete').data; + const esmResult = esmRecords.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(cjsResult.name, 'alpha'); + assert.strictEqual(esmResult.name, 'beta'); + assert.notStrictEqual(cjsResult.params.pid, esmResult.params.pid); + assert.notStrictEqual(cjsResult.runId, esmResult.runId); +} + +async function testLoadFailure() { + const missing = path.resolve(fixtures.fixturesDir, 'does-not-exist.cjs'); + const records = await runFile(missing).toArray(); + const diagnostics = records.filter( + ({ type }) => type === 'bench:diagnostic'); + assert(diagnostics.some( + ({ data }) => data.level === 'error' && /failed with exit code/.test( + data.message))); + assert.strictEqual(records.some( + ({ type }) => type === 'bench:complete'), false); + assert.strictEqual(records.at(-1).type, 'bench:summary'); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testEvaluationFailure() { + const records = await runFile(fixtures.path( + 'bench-runner/load-error-after-declaration.cjs')).toArray(); + assert(records.some(({ type, data }) => + type === 'bench:diagnostic' && + data.message === 'load failed after declaration')); + const result = records.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(result.name, 'declared before load error'); + assert.strictEqual(result.error, undefined); + assert.strictEqual(records.at(-1).data.counts.completed, 1); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testStructuredError() { + const records = await runFile( + fixtures.path('bench-runner/error.cjs')).toArray(); + const result = records.find( + ({ type }) => type === 'bench:complete').data; + assert.strictEqual(result.error.code, 'ERR_BENCHMARK_FIXTURE'); + assert.deepStrictEqual(result.error.cause, { value: 42n }); + assert.match(result.error.stack, /error\.cjs/); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testAbort() { + const controller = new AbortController(); + const reason = new Error('cancel run file'); + const stream = runFile( + fixtures.path('bench-runner/run-file-blocked.cjs'), + { signal: controller.signal }); + stream.once('bench:start', common.mustCall(() => controller.abort(reason))); + const records = await stream.toArray(); + const diagnostic = records.find(({ type, data }) => + type === 'bench:diagnostic' && data.error?.code === 'ABORT_ERR').data; + assert.strictEqual(diagnostic.error.cause.message, reason.message); + assert.strictEqual(records.some( + ({ type }) => type === 'bench:complete'), false); + assert.strictEqual(records.at(-1).type, 'bench:summary'); + assert.strictEqual(records.at(-1).data.success, false); + assert.deepStrictEqual(records.at(-1).data.counts, { + __proto__: null, + completed: 0, + failed: 0, + skipped: 0, + total: 1, + }); +} + +async function testPartialAbort() { + const controller = new AbortController(); + const stream = runFile( + fixtures.path('bench-runner/run-file-partial.cjs'), + { signal: controller.signal }); + stream.on('bench:start', ({ name }) => { + if (name === 'aborted run file') controller.abort(); + }); + const records = await stream.toArray(); + const summary = records.at(-1).data; + assert.strictEqual(records.filter( + ({ type }) => type === 'bench:complete').length, 1); + assert.deepStrictEqual(summary.counts, { + __proto__: null, + completed: 1, + failed: 0, + skipped: 0, + total: 2, + }); + assert.strictEqual(summary.success, false); +} + +async function testPostSummaryAbort() { + const controller = new AbortController(); + const stream = runFile( + fixtures.path('bench-runner/run-file-lingering.cjs'), + { signal: controller.signal }); + stream.on('bench:diagnostic', common.mustCall(({ message, stream }) => { + if (stream === 'stdout' && /child disconnected/.test(message)) { + controller.abort(new Error('stop lingering child')); + } + }, 2)); + const records = await stream.toArray(); + assert(records.some(({ type, data }) => + type === 'bench:diagnostic' && data.error?.code === 'ABORT_ERR')); + assert.strictEqual(records.at(-1).type, 'bench:summary'); + assert.strictEqual(records.at(-1).data.counts.completed, 1); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testExitCode() { + const records = await runFile( + fixtures.path('bench-runner/exit-code.cjs')).toArray(); + assert(records.some(({ type, data }) => + type === 'bench:diagnostic' && + /set exit code/.test(data.message))); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testParentExitCode() { + process.exitCode = 42; + try { + const records = await runFile(fixture).toArray(); + assert.strictEqual(records.at(-1).data.success, true); + } finally { + process.exitCode = undefined; + } +} + +async function testDefaultExecArgvSnapshot() { + const stream = runFile(fixture); + process.execArgv.push('--require=/does/not/exist.cjs'); + try { + const records = await stream.toArray(); + assert.strictEqual(records.at(-1).data.success, true); + } finally { + process.execArgv.pop(); + } +} + +async function testExecPathSnapshot() { + const stream = runFile(fixture); + const execPath = process.execPath; + process.execPath = '/does/not/exist'; + try { + const records = await stream.toArray(); + assert.strictEqual(records.at(-1).data.success, true); + } finally { + process.execPath = execPath; + } +} + +function testEvalParent() { + const script = ` + require('node:bench').runFile(${JSON.stringify(fixture)}) + .on('bench:summary', (summary) => console.log(summary.success)); + `; + const result = spawnSync(process.execPath, [ + '--no-warnings', + '-e', + script, + ], { encoding: 'utf8' }); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(result.stdout, 'true\n'); +} + +async function testPreAborted() { + const records = await runFile(fixture, { + signal: AbortSignal.abort(new Error('already cancelled')), + }).toArray(); + assert.strictEqual(records.some(({ type }) => type === 'bench:start'), false); + assert.strictEqual(records.find(({ type }) => + type === 'bench:diagnostic').data.error.code, 'ABORT_ERR'); + assert.strictEqual(records.at(-1).type, 'bench:summary'); + assert.strictEqual(records.at(-1).data.success, false); +} + +async function testDestroy() { + const stream = runFile( + fixtures.path('bench-runner/run-file-blocked.cjs')); + stream.once('bench:start', common.mustCall(() => stream.destroy())); + await once(stream, 'close'); + assert.strictEqual(stream.destroyed, true); +} + +(async () => { + await testRunFile(); + await testConcurrentCalls(); + await testLoadFailure(); + await testEvaluationFailure(); + await testStructuredError(); + await testAbort(); + await testPartialAbort(); + await testPostSummaryAbort(); + await testExitCode(); + await testParentExitCode(); + await testDefaultExecArgvSnapshot(); + await testExecPathSnapshot(); + await testPreAborted(); + await testDestroy(); + testEvalParent(); +})().then(common.mustCall());