node/benchmark/compare.js

'use strict';

const { spawn, fork } = require('node:child_process');
const { closeSync, openSync, writeSync } = require('node:fs');
const { inspect } = require('util');
const path = require('path');
const CLI = require('./_cli.js');
const BenchmarkProgress = require('./_benchmark_progress.js');

//
// Parse arguments
//
const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
  Run each benchmark in the <category> directory many times using two different
  node versions. More than one <category> directory can be specified.
  The output is formatted as csv, which can be processed using for
  example 'compare.R'. Use --analyze to perform statistical analysis
  directly without R.

  --new      ./new-node-binary  new node binary (required)
  --old      ./old-node-binary  old node binary (required)
  --runs     30                 number of samples
  --filter   pattern            includes only benchmark scripts matching
                                <pattern> (can be repeated)
  --exclude  pattern            excludes scripts matching <pattern> (can be
                                repeated)
  --set      variable=value     set benchmark variable (can be repeated)
  --no-progress                 don't show benchmark progress indicator
  --analyze                     perform statistical analysis after benchmarks
                                complete (Welch's t-test, effect size) instead
                                of printing csv output to stdout
  --csv      filename           write csv output to filename (can be combined
                                with --analyze). Use - to write to stdout.
  --scale    1000               rate-to-integer multiplier for histogram
                                 precision when using --analyze (default: 1000)
  --max-regression  N           exit with code 1 if any statistically
                                 significant regression exceeds N% (implies
                                 --analyze)

  Examples:
    --set CPUSET=0            Runs benchmarks on CPU core 0.
    --set CPUSET=0-2          Specifies that benchmarks should run on CPU cores 0 to 2.

  Note: The CPUSET format should match the specifications of the 'taskset' command
`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress', 'analyze'] });

if (!cli.optional.new || !cli.optional.old) {
  cli.abort(cli.usage);
}

const binaries = ['old', 'new'];
const runs = cli.optional.runs ? parseInt(cli.optional.runs, 10) : 30;
const maxRegression = cli.optional['max-regression'] ?
  parseFloat(cli.optional['max-regression']) :
  0;
const analyze = !!cli.optional.analyze || maxRegression > 0;
const scale = cli.optional.scale ? parseInt(cli.optional.scale, 10) : 1000;
const benchmarks = cli.benchmarks();

if (benchmarks.length === 0) {
  console.error('No benchmarks found');
  process.exitCode = 1;
  return;
}

const cvsToStdout = cli.optional.csv === '-';
const csvFd = cli.optional.csv === undefined || cvsToStdout ?
  null :
  openSync(cli.optional.csv, 'w');
const outputCsv = !analyze || csvFd !== null || cvsToStdout;

function writeCsv(line) {
  writeSync(csvFd || process.stdout.fd, `${line}\n`);
}

// When --analyze is set, collect results for statistical analysis.
const results = analyze ? new Map() : null;

// Create queue from the benchmarks list such both node versions are tested
// `runs` amount of times each.
// Note: BenchmarkProgress relies on this order to estimate
// how much runs remaining for a file. All benchmarks generated from
// the same file must be run consecutively.
const queue = [];
for (const filename of benchmarks) {
  for (let iter = 0; iter < runs; iter++) {
    for (const binary of binaries) {
      queue.push({ binary, filename, iter });
    }
  }
}
// queue.length = binary.length * runs * benchmarks.length

// Print csv header unless only analyzing inline.
if (outputCsv) {
  writeCsv('"binary","filename","configuration","rate","time"');
}

const kStartOfQueue = 0;

const showProgress = !cli.optional['no-progress'] && !cvsToStdout;
let progress;
if (showProgress) {
  progress = new BenchmarkProgress(queue, benchmarks, {
    analyze: analyze || csvFd !== null,
  });
  progress.startQueue(kStartOfQueue);
}

(function recursive(i) {
  const job = queue[i];
  const resolvedPath = path.resolve(__dirname, job.filename);

  const cpuCore = cli.getCpuCoreSetting();
  let child;
  if (cpuCore !== null) {
    const spawnArgs = ['-c', cpuCore, cli.optional[job.binary], resolvedPath, ...cli.optional.set];
    child = spawn('taskset', spawnArgs, {
      env: process.env,
      stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
    });
  } else {
    child = fork(resolvedPath, cli.optional.set, {
      execPath: cli.optional[job.binary],
    });
  }

  child.on('message', (data) => {
    if (data.type === 'report') {
      // Construct configuration string, " A=a, B=b, ..."
      let conf = '';
      for (const key of Object.keys(data.conf)) {
        conf += ` ${key}=${inspect(data.conf[key])}`;
      }
      conf = conf.slice(1);

      if (analyze) {
        // Collect results for post-run analysis.
        const name = `${job.filename} ${conf}`;
        if (!results.has(name)) {
          results.set(name, { old: [], new: [] });
        }
        results.get(name)[job.binary].push(data.rate);
      }

      if (outputCsv) {
        // Escape quotes (") for correct csv formatting
        const csvConf = conf.replace(/"/g, '""');
        writeCsv(`"${job.binary}","${job.filename}","${csvConf}",` +
                 `${data.rate},${data.time}`);
      }
      if (showProgress) {
        // One item in the subqueue has been completed.
        progress.completeConfig(data);
      }
    } else if (showProgress && data.type === 'config') {
      // The child has computed the configurations, ready to run subqueue.
      progress.startSubqueue(data, i);
    }
  });

  child.once('close', (code) => {
    if (code) {
      process.exit(code);
    }
    if (showProgress) {
      progress.completeRun(job);
    }

    // If there are more benchmarks execute the next
    if (i + 1 < queue.length) {
      recursive(i + 1);
    } else {
      if (csvFd !== null) closeSync(csvFd);
      if (analyze) printAnalysis(results, scale, maxRegression);
    }
  });
})(kStartOfQueue);

// Holm-Bonferroni step-down adjustment. Controls the probability of *any*
// false positive across the whole comparison set, which is what a pass/fail
// gate needs: an uncorrected suite of 169 comparisons at 5% has a 99.98%
// chance of flagging something that is not there. Uniformly more powerful
// than plain Bonferroni, and makes no assumption about independence.
function holmAdjust(pValues) {
  const order = pValues
    .map((p, i) => ({ p, i }))
    .sort((a, b) => a.p - b.p);
  const m = order.length;
  const adjusted = new Array(m);
  let running = 0;
  for (let k = 0; k < m; k++) {
    // Step down, enforcing monotonicity so an adjusted value can never be
    // smaller than one belonging to a more significant raw p-value.
    running = Math.max(running, Math.min(1, (m - k) * order[k].p));
    adjusted[order[k].i] = running;
  }
  return adjusted;
}

function printAnalysis(results, scale, maxRegression) {
  const { createHistogram } = require('node:perf_hooks');

  // Build per-benchmark histograms and run statistical tests.
  const rows = [];
  let maxNameLen = 0;

  let skipped = 0;

  for (const [name, { old: oldRates, new: newRates }] of results) {
    if (oldRates.length < 2 || newRates.length < 2) {
      skipped++;
      continue;
    }

    const hOld = createHistogram({ figures: 3 });
    const hNew = createHistogram({ figures: 3 });

    for (const r of oldRates) hOld.record(Math.max(1, Math.round(r * scale)));
    for (const r of newRates) hNew.record(Math.max(1, Math.round(r * scale)));

    const oldMean = oldRates.reduce((a, b) => a + b, 0) / oldRates.length;
    const newMean = newRates.reduce((a, b) => a + b, 0) / newRates.length;
    const improvement = ((newMean - oldMean) / oldMean) * 100;

    // Query the three confidence levels. The p-value and t-statistic
    // are the same regardless of the confidence level, so we extract
    // them from the first result.
    const w95 = hOld.welchTest(hNew, { confidence: 0.95 });
    const w99 = hOld.welchTest(hNew, { confidence: 0.99 });
    const w999 = hOld.welchTest(hNew, { confidence: 0.999 });

    // Significance stars matching compare.R convention.
    let stars = '';
    if (w95.pValue < 0.001) stars = '***';
    else if (w95.pValue < 0.01) stars = ' **';
    else if (w95.pValue < 0.05) stars = '  *';

    // Confidence intervals expressed as percentage of the old mean.
    const ciPct = (w) => {
      const half =
        (w.confidenceInterval.upper - w.confidenceInterval.lower) / 2;
      return (half / (oldMean * scale)) * 100;
    };

    rows.push({
      name,
      stars,
      improvement,
      ci95: ciPct(w95),
      ci99: ciPct(w99),
      ci999: ciPct(w999),
      pValue: w95.pValue,
    });

    if (name.length > maxNameLen) maxNameLen = name.length;
  }

  // Adjust for the size of the comparison set. The raw p-value answers "is
  // this one benchmark different", but a suite is read as a whole, so the
  // relevant question is "is anything here different".
  const adjusted = holmAdjust(rows.map((r) => r.pValue));
  for (let i = 0; i < rows.length; i++) rows[i].pAdjusted = adjusted[i];

  // A comparison can only rule out an effect it was able to resolve. Where no
  // threshold has been given there is no definition of "worth detecting", so
  // nothing is claimed. `maxRegression` is exactly such a declaration, so it
  // is reused rather than inventing a second constant.
  const resolution = maxRegression > 0 ? maxRegression : null;
  let underpowered = 0;
  for (const row of rows) {
    row.inconclusive = resolution !== null &&
                       row.stars.trim() === '' &&
                       row.ci95 > resolution;
    if (row.inconclusive) underpowered++;
  }

  // Print header.
  const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));
  const rpad = (s, n) => ' '.repeat(Math.max(0, n - s.length)) + s;

  writeSync(process.stdout.fd, `${pad('', maxNameLen)}  confidence` +
              `   improvement   accuracy (*)    (**)   (***)\n`);

  for (const row of rows) {
    const imp = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
    writeSync(process.stdout.fd,
              `${pad(row.name, maxNameLen)}  ${pad(row.stars, 10)}` +
      `  ${rpad(imp, 11)}` +
      `   ±${row.ci95.toFixed(2)}%` +
      `  ±${row.ci99.toFixed(2)}%` +
      `  ±${row.ci999.toFixed(2)}%` +
      `${row.inconclusive ? '  (inconclusive)' : ''}\n`,
    );
  }

  if (skipped > 0) {
    writeSync(process.stdout.fd, '\n');
    writeSync(process.stdout.fd,
              `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` +
      ` skipped because Welch's t-test requires at least 2 samples per` +
      ` binary. Use --runs 2 or higher.\n`,
    );
  }

  // --- Bar chart visualization ---
  printChart(rows, maxNameLen);

  writeSync(process.stdout.fd, '\n');
  writeSync(process.stdout.fd,
            `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` +
    `Use --scale to adjust precision if needed.\n\n`,
  );
  const anyFamilyWise = rows.filter((r) => r.pAdjusted < 0.05).length;
  writeSync(process.stdout.fd,
            `Be aware that when doing many comparisons the risk of a false-positive\n` +
    `result increases. In this case, there are ${rows.length} comparisons, ` +
    `you can thus\nexpect the following amount of false-positive results:\n` +
    `  ${(rows.length * 0.05).toFixed(2)} false positives, when considering ` +
    `a   5% risk acceptance (*, **, ***),\n` +
    `  ${(rows.length * 0.01).toFixed(2)} false positives, when considering ` +
    `a   1% risk acceptance (**, ***),\n` +
    `  ${(rows.length * 0.001).toFixed(2)} false positives, when considering ` +
    `a 0.1% risk acceptance (***)\n` +
    `\nThe stars above are per-benchmark and uncorrected. Adjusting for the ` +
    `size of\nthis comparison set (Holm-Bonferroni), ${anyFamilyWise} ` +
    `comparison${anyFamilyWise === 1 ? '' : 's'} remain${anyFamilyWise === 1 ? 's' : ''} ` +
    `significant at 5%.\n--max-regression uses the corrected values.\n`,
  );

  // Gate: exit with error if any regression is shown to exceed the limit.
  if (maxRegression > 0) {
    if (underpowered > 0) {
      writeSync(process.stdout.fd, '\n');
      writeSync(process.stdout.fd,
                `Note: ${underpowered} of ${rows.length} comparison` +
        `${rows.length === 1 ? '' : 's'} could not resolve an effect as ` +
        `small as ${maxRegression}%, and are marked (inconclusive). They are ` +
        `not\nevidence of no regression -- the samples are too noisy to tell. ` +
        `Raise --runs,\nor pin cores with --set CPUSET, to narrow them.\n`,
      );
    }

    // Two conditions, both required.
    //
    // The confidence interval must lie entirely beyond the threshold. A small
    // p-value only says the effect is not exactly zero; claiming it exceeds
    // `maxRegression` is a statement about magnitude, so the interval has to
    // exclude that magnitude. Testing the point estimate instead systematically
    // fires on the noisiest benchmarks, because a large point estimate is
    // easiest to obtain when the interval is wide.
    //
    // The p-value must also survive adjustment for the size of the comparison
    // set, so that a suite of hundreds of benchmarks does not fail purely
    // because one of them drifted.
    const failures = rows.filter(
      (r) => r.pAdjusted < 0.05 && r.improvement + r.ci95 < -maxRegression,
    );

    if (failures.length > 0) {
      writeSync(process.stdout.fd, '\n');
      writeSync(process.stdout.fd,
                `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` +
        ` regressed by more than ${maxRegression}%` +
        ` (interval excludes the threshold,\n` +
        `family-wise corrected across ${rows.length} comparisons):\n`,
      );
      for (const f of failures) {
        writeSync(process.stdout.fd,
                  `  ${f.name}  ${f.improvement.toFixed(2)}% ` +
          `(95% CI up to ${(f.improvement + f.ci95).toFixed(2)}%, ` +
          `adjusted p=${f.pAdjusted.toExponential(2)})\n`,
        );
      }
      process.exitCode = 1;
    }
  }
}

function printChart(rows, maxNameLen) {
  if (rows.length === 0) return;

  // Determine the chart scale from the data. The bar region covers
  // the range [-maxAbs, +maxAbs] so the zero line sits in the center.
  const barWidth = 40;
  const halfWidth = barWidth / 2;
  let maxAbs = 0;
  for (const row of rows) {
    const extent = Math.abs(row.improvement) + row.ci95;
    if (extent > maxAbs) maxAbs = extent;
  }
  if (maxAbs === 0) maxAbs = 1;

  const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));

  // Scale axis labels.
  const axisLeft = `-${maxAbs.toFixed(1)}%`;
  const axisRight = `+${maxAbs.toFixed(1)}%`;
  const axisCenter = '0%';

  // Print axis header.
  const labelPad = maxNameLen + 5;
  const leftLabel = ' '.repeat(labelPad) +
    axisLeft +
    ' '.repeat(Math.max(0, halfWidth - axisLeft.length - Math.floor(axisCenter.length / 2))) +
    axisCenter +
    ' '.repeat(Math.max(0, halfWidth - Math.ceil(axisCenter.length / 2) - axisRight.length)) +
    axisRight;
  writeSync(process.stdout.fd, '\n');
  writeSync(process.stdout.fd, `${leftLabel}\n`);

  for (const row of rows) {
    const imp = row.improvement;
    const ci = row.ci95;

    // Position of the improvement value in the bar region [0, barWidth].
    const center = halfWidth;
    const impPos = center + (imp / maxAbs) * halfWidth;

    // CI extent in bar positions.
    const ciLeft = center + ((imp - ci) / maxAbs) * halfWidth;
    const ciRight = center + ((imp + ci) / maxAbs) * halfWidth;

    // Build the bar character by character.
    const chars = [];
    for (let x = 0; x < barWidth; x++) {
      const pos = x + 0.5; // Center of this character cell.
      if (x === Math.floor(center)) {
        chars.push('|');
      } else if ((imp >= 0 && pos > center && pos <= impPos) ||
                 (imp < 0 && pos < center && pos >= impPos)) {
        chars.push(row.stars ? '\u2588' : '\u2593'); // solid or dark shade
      } else if (pos >= ciLeft && pos <= ciRight) {
        chars.push('\u2591'); // Light shade for CI region
      } else {
        chars.push(' ');
      }
    }

    const label = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
    const sig = row.stars.trim();
    writeSync(process.stdout.fd, `${pad(row.name, maxNameLen)}  ${chars.join('')}  ${label} ${sig}\n`);
  }
}