test: schedule WPT variants individually

Discover WPT tasks through their existing JavaScript drivers so the
Python runner can schedule, report, and rerun generated paths.

Assisted-by: Codex
Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: https://github.com/nodejs/node/pull/65984
Fixes: https://github.com/nodejs/node/issues/51854
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

test/common/wpt.js

@@ -96,8 +96,8 @@ class ReportResult {
 // Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
 // for more details.
 class WPTReport {
-  constructor(testPath) {
-    this.filename = `report-${testPath.replaceAll('/', '-')}.json`;
+  constructor(testPath, suffix = '') {
+    this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`;
     this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`);
     /** @type {Map<string, ReportResult>} */
     this.results = new Map();
@@ -461,12 +461,14 @@ class WPTTestSpec {
   /**
    * Whether a command line argument selects this spec. Accepts the source file
    * name, which selects every global and variant generated from it, or a test
-   * path as printed alongside the results, which selects only this one.
+   * path as printed alongside the results. Omitting its query selects all
+   * variants of that global.
    * @param {string} arg
    * @returns {boolean}
    */
   isSelectedBy(arg) {
-    if (arg === this.getTestPath()) {
+    const testPath = this.getTestPath();
+    if (arg === testPath || arg === testPath.split('?')[0]) {
       return true;
     }
     const [filename, variant = ''] = arg.split('?');
@@ -608,7 +610,7 @@ class StatusLoader {
     return result;
   }
 
-  load() {
+  load(source) {
     const dir = path.join(__dirname, '..', 'wpt');
     let result;
 
@@ -625,7 +627,7 @@ class StatusLoader {
     this.rules.addRules(result);
 
     const subDir = fixtures.path('wpt', this.path);
-    const list = this.grep(subDir);
+    const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)];
     for (const file of list) {
       const relativePath = path.relative(subDir, file);
       const match = this.rules.match(relativePath);
@@ -813,14 +815,29 @@ const backends = {
 };
 
 class WPTRunner {
-  constructor(path, {
-    concurrency = os.availableParallelism() - 1 || 1,
-    backend = 'thread',
-  } = {}) {
+  constructor(path, options = {}) {
+    let {
+      concurrency = os.availableParallelism() - 1 || 1,
+      backend = 'thread',
+    } = options;
     if (!Number.isInteger(concurrency) || concurrency < 1) {
       throw new TypeError('WPT concurrency must be a positive integer');
     }
 
+    if (process.env.NODE_TEST_WPT !== undefined) {
+      this.managed = JSON.parse(process.env.NODE_TEST_WPT);
+      if (!this.managed || !['list', 'run'].includes(this.managed.mode) ||
+          (this.managed.mode === 'run' &&
+           (['source', 'key'].some((key) =>
+             typeof this.managed[key] !== 'string' || !this.managed[key]) ||
+            (this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) {
+        throw new Error('Invalid WPT runner configuration');
+      }
+    }
+    this.isListing = this.managed?.mode === 'list';
+    this.serial = options.concurrency === 1;
+    if (this.managed?.mode === 'run') concurrency = 1;
+
     // RISC-V has very limited virtual address space in the currently common
     // sv39 mode, in which we can only create a very limited number of wasm
     // memories(27 from a fresh node repl). Limit the concurrency to avoid
@@ -860,7 +877,7 @@ class WPTRunner {
     this.initScript = null;
 
     this.status = new StatusLoader(path);
-    this.status.load();
+    this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined);
     this.statusFile = this.status.statusFile;
     this.specs = new Set(this.status.specs);
 
@@ -872,8 +889,9 @@ class WPTRunner {
 
     this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
 
-    if (process.env.WPT_REPORT != null) {
-      this.report = new WPTReport(path);
+    if (process.env.WPT_REPORT != null && !this.isListing) {
+      const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : '';
+      this.report = new WPTReport(path, suffix);
     }
   }
 
@@ -958,7 +976,40 @@ class WPTRunner {
   // TODO(joyeecheung): work with the upstream to port more tests in .html
   // to .js.
   async runJsTests() {
+    if (this.isListing) {
+      const groups = new Map();
+      for (const spec of this.specs) {
+        const key = spec.getStatusKey();
+        if (!groups.has(key)) groups.set(key, []);
+        groups.get(key).push(spec);
+      }
+      const tests = [...groups.values()].flatMap((specs) => {
+        // Strict expected failures are checked across all query variants.
+        // Keep that group together; all other variants are independent tasks.
+        const grouped = specs.some((spec) =>
+          spec.failedTests.some((name) => isUnexpectedPass(spec, name)));
+        return (grouped ? [specs[0]] : specs).map((spec) => {
+          const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath();
+          return {
+            source: spec.filename.split(path.sep).join('/'),
+            key: spec.getStatusKey(),
+            ...(grouped ? {} : { variant: spec.variant }),
+            id: selector.slice(this.path.length + 1),
+            selector,
+          };
+        });
+      });
+      console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({
+        version: 1,
+        serial: this.serial,
+        tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)),
+      })}`);
+      return;
+    }
     const queue = this.buildQueue();
+    if (this.managed && queue.length === 0) {
+      console.log('1..0 # SKIP No runnable WPT variants');
+    }
 
     const run = limit(this.concurrency);
     const jobs = [];
@@ -1313,11 +1364,19 @@ class WPTRunner {
   buildQueue() {
     const queue = [];
     this.skippedSpecCount = 0;
-    const arg = process.argv[2];
+    const key = this.managed?.mode === 'run' ? this.managed.key : undefined;
+    const variant = this.managed?.variant;
+    const matches = (spec) => spec.getStatusKey() === key &&
+      (variant === undefined || spec.variant === variant);
+    const arg = key === undefined ? process.argv[2] : undefined;
+    if (key !== undefined && ![...this.specs].some(matches)) {
+      throw new Error(`${key}${variant ?? ''} not found!`);
+    }
     if (this.inspectBrk && !arg) {
       throw new Error('WPT_INSPECT requires a WPT test path');
     }
     for (const spec of this.specs) {
+      if (key !== undefined && !matches(spec)) continue;
       if (arg) {
         if (spec.isSelectedBy(arg)) {
           queue.push(spec);

test/parallel/test-common-wpt-runner.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const tmpdir = require('../common/tmpdir');
+
+if (process.env.NODE_TEST_WPT_REPORT_DIR) {
+  const { WPTRunner } = require('../common/wpt');
+  const runner = new WPTRunner('html/webappapis/atob');
+  runner.report.filepath = path.join(process.env.NODE_TEST_WPT_REPORT_DIR, runner.report.filename);
+  runner.runJsTests();
+} else if (process.env.NODE_TEST_WPT_QUERY_PROBE) {
+  const { WPTRunner, WPTTestSpec } = require('../common/wpt');
+  const runner = new WPTRunner('compression');
+  if (runner.managed?.mode === 'run') assert.strictEqual(runner.concurrency, 1);
+  runner.specs = new Set(['?pass', '?fail'].map((query) => {
+    const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window');
+    spec.failedTests = ['expected across queries'];
+    if (process.env.NODE_TEST_WPT_QUERY_PROBE === 'flaky' ||
+        (process.env.NODE_TEST_WPT_QUERY_PROBE === 'mixed' && query === '?pass')) {
+      spec.flakyTests = [...spec.failedTests];
+    }
+    return spec;
+  }));
+  runner.setScriptModifier((script) => {
+    if (!script.filename.endsWith('compression-bad-chunks.any.js')) return;
+    script.code = `test(() => assert_true(${process.env.NODE_TEST_WPT_QUERY_PROBE === 'missing'} ||
+      location.search === '?pass'), 'expected across queries');`;
+  });
+  runner.runJsTests();
+} else {
+  main();
+}
+
+function main() {
+  tmpdir.refresh();
+  const env = { ...process.env };
+  for (const key of ['NODE_TEST_WPT', 'WPT_REPORT', 'WPT_INSPECT']) delete env[key];
+  const driver = (name) => path.join(__dirname, '../wpt', `test-${name}.js`);
+  function invoke(file, config, overrides = {}, status = 0) {
+    const result = spawnSync(process.execPath, [file], {
+      env: { ...env, ...overrides, NODE_TEST_WPT: JSON.stringify(config) },
+      encoding: 'utf8', timeout: common.platformTimeout(10_000),
+      maxBuffer: 10 * 1024 * 1024,
+    });
+    assert.ifError(result.error);
+    assert.strictEqual(result.status, status, result.stdout + result.stderr);
+    return result.stdout + result.stderr;
+  }
+
+  function discover(name, overrides, file = driver(name)) {
+    const stdout = invoke(file, { mode: 'list' }, overrides);
+    const lines = stdout.split('\n').filter((line) => line.startsWith('NODE_TEST_WPT_MANIFEST:'));
+    assert.strictEqual(lines.length, 1);
+    assert.doesNotMatch(stdout, /\[PASS\]/);
+    const manifest = JSON.parse(lines[0].slice('NODE_TEST_WPT_MANIFEST:'.length));
+    assert.strictEqual(manifest.version, 1);
+    assert.strictEqual(new Set(manifest.tests.map((test) => test.id)).size, manifest.tests.length);
+    return manifest;
+  }
+
+  const atob = discover('atob');
+  assert.strictEqual(atob.serial, false);
+  assert.deepStrictEqual(atob.tests.map((test) => test.id), ['base64.any.html', 'base64.any.worker.html']);
+  const reportRoot = path.join(tmpdir.path, 'reports');
+  const canReport = ['darwin', 'linux', 'win32'].includes(process.platform);
+  if (canReport) fs.mkdirSync(reportRoot, { recursive: true });
+  for (const [index, group] of atob.tests.entries()) {
+    const serial = `group-probe-${process.pid}-${index}`;
+    const reportPath = path.join(reportRoot, `report-html-webappapis-atob-${serial}.json`);
+    try {
+      const stdout = invoke(canReport ? __filename : driver('atob'),
+                            { mode: 'run', source: group.source, key: group.key, variant: group.variant }, canReport ? {
+                              WPT_REPORT: '1', TEST_SERIAL_ID: serial, NODE_TEST_WPT_REPORT_DIR: reportRoot,
+                            } : {});
+      const results = stdout.split('\n').filter((line) => line.startsWith('[PASS]'));
+      assert.ok(results.length > 0);
+      assert.ok(results.every((line) => line.includes(`${group.id}:`)));
+      if (canReport) {
+        const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+        assert.deepStrictEqual(report.results.map((result) => result.test),
+                               [`/html/webappapis/atob/${group.id}`]);
+      }
+    } finally {
+      fs.rmSync(reportPath, { force: true });
+    }
+  }
+
+  const encoding = discover('encoding');
+  const queryGroups = encoding.tests.filter((test) => test.source === 'api-invalid-label.any.js');
+  assert.strictEqual(queryGroups.length, 8);
+  assert.deepStrictEqual([...new Set(queryGroups.map((test) => test.key))],
+                         ['api-invalid-label.any.html', 'api-invalid-label.any.worker.html']);
+  const timers = discover('timers');
+  assert.strictEqual(timers.serial, true);
+  const skipped = timers.tests.find((test) => test.source === 'negative-settimeout.any.js');
+  assert.ok(skipped);
+  const skippedOutput = invoke(driver('timers'), { mode: 'run', source: skipped.source, key: skipped.key });
+  assert.match(skippedOutput, /\[SKIPPED\].*unreliable in Node\.js/);
+  assert.match(skippedOutput, /1\.\.0 # SKIP/);
+  assert.doesNotMatch(skippedOutput, /\[PASS\]/);
+
+  if (common.hasSQLite) {
+    const root = path.join(tmpdir.path, 'discovery');
+    const directory = path.join(root, '.tmp.probe');
+    fs.mkdirSync(directory, { recursive: true });
+    const sentinel = path.join(directory, 'sentinel');
+    fs.writeFileSync(sentinel, 'preserved');
+    assert.strictEqual(discover('webstorage', { NODE_TEST_DIR: root, TEST_SERIAL_ID: 'probe' }).serial, true);
+    assert.strictEqual(fs.readFileSync(sentinel, 'utf8'), 'preserved');
+  }
+
+  const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' };
+  for (const probe of ['combined', 'mixed']) {
+    const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename);
+    assert.deepStrictEqual(strict.tests, [{
+      source: config.source, key: config.key, id: config.key, selector: `compression/${config.key}`,
+    }]);
+  }
+  const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename);
+  assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']);
+  assert.deepStrictEqual(flaky.tests.map((test) => test.id).sort(),
+                         [`${config.key}?fail`, `${config.key}?pass`]);
+  const single = invoke(__filename, { ...config, variant: '?pass' }, { NODE_TEST_WPT_QUERY_PROBE: 'flaky' });
+  assert.match(single, /\.any\.html\?pass:/);
+  assert.doesNotMatch(single, /\.any\.html\?fail:/);
+  const combined = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'combined' });
+  assert.match(combined, /\.any\.html\?pass:/);
+  assert.match(combined, /\.any\.html\?fail:/);
+  const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1);
+  assert.match(missing, /Found 2 unexpected passes/);
+
+  for (const query of ['', '?fail']) {
+    const direct = spawnSync(process.execPath, [__filename, `compression/${config.key}${query}`], {
+      env: { ...env, NODE_TEST_WPT_QUERY_PROBE: 'combined' },
+      encoding: 'utf8', timeout: common.platformTimeout(10_000),
+    });
+    assert.ifError(direct.error);
+    assert.strictEqual(direct.status, 0, direct.stdout + direct.stderr);
+    assert.match(direct.stdout, /\.any\.html\?fail:/);
+    if (query) assert.doesNotMatch(direct.stdout, /\.any\.html\?pass:/);
+    else assert.match(direct.stdout, /\.any\.html\?pass:/);
+  }
+}

test/tools/test_wpt_runner.py

@@ -0,0 +1,170 @@
+import contextlib
+import json
+import os
+import shlex
+import sys
+import tempfile
+import unittest
+import warnings
+from unittest import mock
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+sys.path.insert(0, os.path.join(ROOT, 'tools'))
+import test as runner
+
+wpt = runner.get_module('testcfg', os.path.join(ROOT, 'test', 'wpt'))
+
+
+class WPTConfigurationTest(unittest.TestCase):
+  def setUp(self):
+    self.stack = contextlib.ExitStack()
+    self.addCleanup(self.stack.close)
+    self.stack.enter_context(warnings.catch_warnings())
+    # SimpleTestCase's existing source reader does not explicitly close files.
+    warnings.filterwarnings('ignore', category=ResourceWarning, module='testpy',
+                            message=r'unclosed file .*test-example\.js')
+    self.root = self.stack.enter_context(tempfile.TemporaryDirectory())
+    self.wrapper = os.path.join(self.root, 'test-example.js')
+    with open(self.wrapper, 'w', encoding='utf8') as source:
+      source.write('// Flags: --expose-gc\n// Env: GROUP_TEST=kept\n')
+    self.context = runner.Context(
+      ROOT, False, sys.executable, [], False, 5, lambda args: args,
+      False, False, 1, False)
+    self.config = wpt.GetConfiguration(self.context, self.root)
+    self.manifest = {'version': 1, 'serial': False, 'tests': [
+      {'source': 'nested/a.any.js', 'key': 'nested/a.any.html', 'id': 'nested/a.any.html',
+       'selector': 'example/nested/a.any.html'},
+      {'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html',
+       'id': 'nested/a.any.worker.html', 'selector': 'example/nested/a.any.worker.html'},
+      {'source': 'z.any.js', 'key': 'z.any.html', 'id': 'z.any.html', 'selector': 'example/z.any.html'},
+    ]}
+    self.discovery = self.stack.enter_context(mock.patch.object(
+      runner, 'Execute', side_effect=self.discover))
+
+  def discover(self, command, context, timeout=None, env=None, **kwargs):
+    self.assertEqual(json.loads(env['NODE_TEST_WPT']), {'mode': 'list'})
+    self.assertEqual(command[-1], self.wrapper)
+    return runner.CommandOutput(
+      0, False, wpt.MANIFEST_PREFIX + json.dumps(self.manifest) + '\n', '')
+
+  def cases(self, selector='wpt/test-example'):
+    return self.config.ListTests(['wpt'], runner.SplitPath(selector), 'none', 'release')
+
+  def test_discovery_creates_one_case_per_status_group(self):
+    cases = self.cases()
+    self.assertEqual([case.group for case in cases], self.manifest['tests'])
+    self.assertTrue(all(case.parallel for case in cases))
+    self.assertEqual([case.GetName() for case in cases],
+                     ['wpt/test-example/' + group['id'] for group in self.manifest['tests']])
+    self.assertEqual(len({tuple(case.path) for case in cases}), 3)
+    self.assertEqual(self.discovery.call_count, 1)
+
+  def test_group_request_preserves_wrapper_startup_flags(self):
+    self.config.additional_flags = ['--trace-warnings']
+    case = self.cases('wpt/test-example/nested/a.any.worker.html')[0]
+    configuration = case.GetRunConfiguration()
+    self.assertEqual(configuration['command'],
+                     [sys.executable, '--expose-gc', '--trace-warnings', self.wrapper])
+    self.assertEqual(configuration['envs']['GROUP_TEST'], 'kept')
+    self.assertEqual(json.loads(configuration['envs']['NODE_TEST_WPT']), {
+      'mode': 'run', 'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html',
+    })
+    self.assertEqual(case.GetReportingName(configuration['command']),
+                     'wpt/test-example/nested/a.any.worker.html')
+
+  def test_group_selection_and_serial_suites(self):
+    self.manifest['serial'] = True
+    cases = self.cases('wpt/test-example/nested/*')
+    self.assertEqual(len(cases), 2)
+    self.assertTrue(all(not case.parallel for case in cases))
+    selected = self.cases('wpt/test-example/nested/a.any.worker.html')
+    self.assertEqual([case.group['id'] for case in selected], ['nested/a.any.worker.html'])
+
+  def test_variant_queries_are_literal_and_base_selects_all_queries(self):
+    group = self.manifest['tests'][0]
+    variants = ['', '?q=[a]+(b)|c', '?q=aaab', '?q=*', '?q=source.js']
+    self.manifest['tests'] = [
+      {**group, 'id': group['id'] + variant, 'selector': group['selector'] + variant,
+       'variant': variant} for variant in variants]
+    base = 'wpt/test-example/' + group['id']
+    self.assertEqual([case.group['variant'] for case in self.cases(base)], variants)
+    for variant in variants[1:]:
+      with self.subTest(variant=variant):
+        selector = base + variant
+        self.assertEqual(runner.NormalizePath(selector), selector)
+        selected = self.cases(selector)
+        self.assertEqual(len(selected), 1)
+        case = selected[0]
+        self.assertEqual(case.GetName(), selector)
+        request = json.loads(case.GetRunConfiguration()['envs']['NODE_TEST_WPT'])
+        self.assertEqual(request, {'mode': 'run', 'source': group['source'],
+                                  'key': group['key'], 'variant': variant})
+    empty = self.cases(base)[0]
+    self.assertEqual(json.loads(empty.GetRunConfiguration()['envs']['NODE_TEST_WPT'])['variant'], '')
+
+  def test_discovery_is_cached_and_uses_existing_directory(self):
+    absent = os.path.join(self.root, 'not-created')
+    with mock.patch.dict(os.environ, {'NODE_TEST_DIR': absent}):
+      self.cases()
+      self.cases('wpt/test-example/nested/*')
+    self.assertEqual(self.discovery.call_count, 1)
+    self.assertFalse(os.path.exists(absent))
+    command, _, _, env = self.discovery.call_args.args
+    self.assertIn('--expose-gc', command)
+    self.assertEqual(env['NODE_TEST_DIR'], self.root)
+
+  def test_whole_wrapper_feature_skip_keeps_original_test(self):
+    self.discovery.side_effect = None
+    self.discovery.return_value = runner.CommandOutput(0, False, '1..0 # Skipped: no feature\n', '')
+    cases = self.cases()
+    self.assertEqual(len(cases), 1)
+    self.assertEqual(cases[0].path, ['wpt', 'test-example'])
+    self.assertEqual(cases[0].GetRunConfiguration()['command'][-1], self.wrapper)
+    self.assertNotIn('NODE_TEST_WPT', cases[0].GetRunConfiguration()['envs'])
+
+  def test_malformed_discovery_is_an_error(self):
+    self.manifest['tests'] = 'not a list'
+    with self.assertRaisesRegex(RuntimeError, 'WPT discovery failed'):
+      self.cases()
+
+  def test_failure_commands_preserve_actual_command_and_selection(self):
+    self.config.additional_flags = ["--title=space ' $|?", '--trace-warnings']
+    query = "?q=space ' | $(echo)"
+    for group in self.manifest['tests'][:2]:
+      group.update(id=group['id'] + query, selector=group['selector'] + query, variant=query)
+    self.manifest['tests'].append({'source': 'empty.any.js', 'key': 'empty.any.html',
+                                   'id': 'empty.any.html', 'selector': 'example/empty.any.html', 'variant': ''})
+    cases = self.cases()
+    self.assertEqual(len(cases), 4)
+    self.context.processor = lambda args: ['valgrind', '--tool=memcheck', *args, 'suffix with |']
+    with mock.patch.object(sys, 'platform', 'linux'):
+      for case, group in zip(cases, self.manifest['tests']):
+        command = case.GetRunConfiguration()['command']
+        expected = [sys.executable, '--expose-gc', "--title=space ' $|?", '--trace-warnings',
+                    self.wrapper]
+        self.assertEqual(command, expected)
+        command = self.context.processor(command)
+        rerun = ['valgrind', '--tool=memcheck', *expected, group['selector'], 'suffix with |']
+        self.assertEqual(shlex.split(case.GetFailureCommand(command)), rerun)
+        failure = runner.TestOutput(case, command,
+                                    runner.CommandOutput(1, False, '', 'probe failure'), False)
+        printer = runner.ProgressIndicator([], runner.RUN, 0)
+        self.assertIn('Command: ' + shlex.join(rerun), printer.GetFailureOutput(failure))
+
+  def test_failure_command_quotes_powershell_metacharacters(self):
+    case = self.cases()[0]
+    case.file = 'test driver.js'
+    case.group['selector'] = "example/a.any.html?q=O'Brien|x"
+    command = ['node', case.file]
+    with mock.patch.object(sys, 'platform', 'win32'):
+      self.assertEqual(case.GetFailureCommand(command),
+                       "& 'node' 'test driver.js' 'example/a.any.html?q=O''Brien|x'")
+
+  def test_ordinary_failure_command_is_unchanged(self):
+    case = runner.TestCase(self.context, ['parallel', 'test-example'], 'none', 'release')
+    command = [sys.executable, '--expose-gc', 'test/parallel/path with spaces.js']
+    self.assertEqual(case.GetFailureCommand(command), runner.EscapeCommand(command))
+
+
+if __name__ == '__main__':
+  unittest.main()

test/wpt/README.md

@@ -20,6 +20,15 @@ Run a WPT module through the Python test runner:
 tools/test.py wpt/test-url
 ```
 
+Select a generated path from the Python runner's output:
+
+```bash
+tools/test.py 'wpt/test-webcrypto/derive_bits_keys/hkdf.https.any.worker.html?1-1000'
+```
+
+Omit the query string to select all its variants. Variants sharing strict
+expected-failure rules run together.
+
 Pass a source file to its module runner to run all globals and variants
 generated from that file:
 

test/wpt/test-compression.js

@@ -2,8 +2,7 @@
 
 const { WPTRunner } = require('../common/wpt');
 
-// Runs each spec in its own process; this suite has crashed the runner in CI.
-const runner = new WPTRunner('compression', { backend: 'process' });
+const runner = new WPTRunner('compression');
 
 runner.pretendGlobalThisAs('Window');
 

test/wpt/test-streams.js

@@ -2,8 +2,7 @@
 
 const { WPTRunner } = require('../common/wpt');
 
-// Runs each spec in its own process; this suite has crashed the runner in CI.
-const runner = new WPTRunner('streams', { backend: 'process' });
+const runner = new WPTRunner('streams');
 
 // Set a script that will be executed in the worker before running the tests.
 runner.pretendGlobalThisAs('Window');

test/wpt/test-wasm-jsapi.mjs

@@ -15,8 +15,7 @@ try {
 }
 
 if (supportsSimd) {
-  // Runs each spec in its own process; this suite has crashed the runner in CI.
-  const runner = new WPTRunner('wasm/jsapi', { backend: 'process' });
+  const runner = new WPTRunner('wasm/jsapi');
   runner.setFlags(['--experimental-wasm-modules']);
 
   runner.runJsTests();

test/wpt/test-webcrypto.js

@@ -6,8 +6,7 @@ if (!common.hasCrypto)
 
 const { WPTRunner } = require('../common/wpt');
 
-// Runs each spec in its own process; this suite has crashed the runner in CI.
-const runner = new WPTRunner('WebCryptoAPI', { backend: 'process' });
+const runner = new WPTRunner('WebCryptoAPI');
 
 // Experimental warnings drown out the actual test output.
 runner.setFlags(['--disable-warning=ExperimentalWarning']);

test/wpt/test-webstorage.js

@@ -6,7 +6,7 @@ const { WPTRunner } = require('../common/wpt');
 const { join } = require('node:path');
 const runner = new WPTRunner('webstorage', { concurrency: 1 });
 
-tmpdir.refresh();
+if (!runner.isListing) tmpdir.refresh();
 
 runner.setFlags([
   '--localstorage-file', join(tmpdir.path, 'wpt-tests.localstorage'),

test/wpt/testcfg.py

@@ -1,6 +1,114 @@
-import sys, os
+import json
+import os
+import re
+import shlex
+import sys
+
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import testpy
+import test
+
+
+MANIFEST_PREFIX = 'NODE_TEST_WPT_MANIFEST:'
+
+
+class WPTTestCase(testpy.SimpleTestCase):
+  def __init__(self, path, file, arch, mode, context, config, group, serial):
+    super(WPTTestCase, self).__init__(
+        path, file, arch, mode, context, config, config.additional_flags)
+    self.group = group
+    self.parallel = not serial
+
+  def GetName(self):
+    return '/'.join(self.path)
+
+  def GetReportingName(self, command):
+    return self.GetName()
+
+  def GetFailureCommand(self, command):
+    command = list(command)
+    command.insert(command.index(self.file) + 1, self.group['selector'])
+    if sys.platform == 'win32':
+      # PowerShell quoting also protects query metacharacters such as | and &.
+      return '& ' + ' '.join("'" + arg.replace("'", "''") + "'" for arg in command)
+    return shlex.join(command)
+
+  def GetRunConfiguration(self):
+    configuration = super(WPTTestCase, self).GetRunConfiguration()
+    request = {
+        'mode': 'run', 'source': self.group['source'], 'key': self.group['key'],
+    }
+    if 'variant' in self.group:
+      request['variant'] = self.group['variant']
+    configuration['envs']['NODE_TEST_WPT'] = json.dumps(request)
+    return configuration
+
+
+class WPTTestConfiguration(testpy.SimpleTestConfiguration):
+  def __init__(self, context, root):
+    super(WPTTestConfiguration, self).__init__(context, root, 'wpt')
+    self.manifests = {}
+
+  def _Discover(self, wrapper):
+    key = (wrapper.file, wrapper.arch, wrapper.mode)
+    if key in self.manifests:
+      return self.manifests[key]
+    configuration = wrapper.GetRunConfiguration()
+    configuration['envs']['NODE_TEST_WPT'] = json.dumps({'mode': 'list'})
+    # common/tmpdir resolves this path before Main creates the execution dir.
+    configuration['envs']['NODE_TEST_DIR'] = os.path.abspath(self.root)
+    output = test.Execute(
+        self.context.processor(configuration['command']), self.context,
+        self.context.GetTimeout(wrapper.mode), configuration['envs'])
+    if output.exit_code != 0 or output.timed_out:
+      raise RuntimeError('WPT discovery failed for %s:\n%s%s' % (
+          wrapper.file, output.stdout, output.stderr))
+    manifests = [line[len(MANIFEST_PREFIX):] for line in output.stdout.splitlines()
+                 if line.startswith(MANIFEST_PREFIX)]
+    if not manifests and test.skip_regex.search(output.stdout):
+      self.manifests[key] = None
+      return None
+    try:
+      if len(manifests) != 1:
+        raise ValueError('expected exactly one WPT manifest')
+      manifest = json.loads(manifests[0])
+      if (manifest.get('version') != 1 or
+          not isinstance(manifest.get('serial'), bool) or
+          not isinstance(manifest.get('tests'), list) or
+          any(not isinstance(group.get(field), str)
+              for group in manifest['tests'] for field in ['source', 'key', 'id', 'selector']) or
+          any('variant' in group and not isinstance(group['variant'], str)
+              for group in manifest['tests'])):
+        raise ValueError('invalid WPT manifest')
+    except (AttributeError, TypeError, ValueError) as error:
+      raise RuntimeError('WPT discovery failed for %s: %s' % (
+          wrapper.file, error)) from error
+    self.manifests[key] = manifest
+    return manifest
+
+  def ListTests(self, current_path, path, arch, mode):
+    wrappers = super(WPTTestConfiguration, self).ListTests(
+        current_path, path[:2], arch, mode)
+    selector = '/'.join(part.pattern for part in path[2:])
+    pattern = re.escape(selector).replace(r'\*', '.*') + r'(?:/.*)?'
+    result = []
+    for wrapper in wrappers:
+      manifest = self._Discover(wrapper)
+      if manifest is None:
+        result.append(wrapper)
+        continue
+      for group in manifest['tests']:
+        case_path = wrapper.path + group['id'].split('/')
+        # Query selectors are literal; a query-free path selects all its variants.
+        if '?' in selector:
+          selected = group['id'] == selector
+        else:
+          selected = not selector or re.fullmatch(pattern, group['id'].split('?')[0])
+        if selected:
+          result.append(WPTTestCase(case_path, wrapper.file, arch, mode,
+                                    self.context, self, group, manifest['serial']))
+    return result
+
 
 def GetConfiguration(context, root):
-  return testpy.SimpleTestConfiguration(context, root, 'wpt')
+  return WPTTestConfiguration(context, root)

tools/test.py

@@ -126,7 +126,7 @@ class ProgressIndicator(object):
     if failure.output.stdout:
       output += ["--- stdout ---"]
       output += [failure.output.stdout.strip()]
-    output += ["Command: %s" % EscapeCommand(failure.command)]
+    output += ["Command: %s" % failure.test.GetFailureCommand(failure.command)]
     if failure.HasCrashed():
       output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)]
     if failure.HasTimedOut():
@@ -360,9 +360,7 @@ class TapProgressIndicator(SimpleProgressIndicator):
     # Print test name as (for example) "parallel/test-assert".  Tests that are
     # scraped from the addons documentation are all named test.js, making it
     # hard to decipher what test is running when only the filename is printed.
-    prefix = abspath(join(dirname(__file__), '../test')) + os.sep
-    command = output.command[-1]
-    command = NormalizePath(command, prefix)
+    command = output.test.GetReportingName(output.command)
 
     if output.UnexpectedOutput():
       status_line = 'not ok %i %s' % (self._done, command)
@@ -425,9 +423,7 @@ class DeoptsCheckProgressIndicator(SimpleProgressIndicator):
     # Print test name as (for example) "parallel/test-assert".  Tests that are
     # scraped from the addons documentation are all named test.js, making it
     # hard to decipher what test is running when only the filename is printed.
-    prefix = abspath(join(dirname(__file__), '../test')) + os.sep
-    command = output.command[-1]
-    command = NormalizePath(command, prefix)
+    command = output.test.GetReportingName(output.command)
 
     stdout = output.output.stdout.strip()
     printed_file = False
@@ -473,7 +469,7 @@ class CompactProgressIndicator(ProgressIndicator):
       stderr = output.output.stderr.strip()
       if len(stderr):
         print(self.templates['stderr'] % stderr)
-      print("Command: %s" % EscapeCommand(output.command))
+      print("Command: %s" % output.test.GetFailureCommand(output.command))
       if output.HasCrashed():
         print("--- %s ---" % PrintCrashed(output.output.exit_code))
       if output.HasTimedOut():
@@ -573,6 +569,13 @@ class TestCase(object):
     self.serial_id = 0
     self.thread_id = 0
 
+  def GetReportingName(self, command):
+    prefix = abspath(join(dirname(__file__), '../test')) + os.sep
+    return NormalizePath(command[-1], prefix)
+
+  def GetFailureCommand(self, command):
+    return EscapeCommand(command)
+
   def IsNegative(self):
     return self.context.expect_fail
 
@@ -1545,6 +1548,8 @@ def NormalizePath(path, prefix='test/'):
   path = path.replace('\\', '/')
   if path.startswith(prefix):
     path = path[len(prefix):]
+  if '?' in path or '#' in path:
+    return path
   if path.endswith('.js'):
     path = path[:-3]
   elif path.endswith('.mjs'):
@@ -1800,7 +1805,8 @@ def Main():
         sys.exit(1)
 
   def should_keep(case):
-    if any((s in case.file) for s in options.skip_tests):
+    if any(s in case.file or s in '/'.join(case.path)
+           for s in options.skip_tests):
       return False
     elif SKIP in case.outcomes:
       return False
@@ -1826,7 +1832,7 @@ def Main():
     # Must ensure the list of tests is sorted before selecting, to avoid
     # silent errors if this file is changed to list the tests in a way that
     # can be different in different machines
-    cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file))
+    cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file, c.path))
     cases_to_run = [ cases_to_run[i] for i
                      in range(options.run[0],
                                len(cases_to_run),
@@ -1860,7 +1866,7 @@ def Main():
   elif result['failed']:
     print("\nFailed tests:")
     for failure in result['failed']:
-      print(EscapeCommand(failure.command))
+      print(failure.test.GetFailureCommand(failure.command))
   else:
     print("\nTest aborted.")
   return exitcode