'use strict'; const { createHistogram } = require('node:perf_hooks'); const { inspect } = require('node:util'); function createRateHistogram(rates, scale, figures) { const histogram = createHistogram({ figures }); for (const rate of rates) { const value = Math.max(1, Math.round(rate * scale)); if (!Number.isSafeInteger(value)) { throw new RangeError('Benchmark rate is too large for the histogram scale'); } histogram.record(value); } return histogram; } function holmAdjust(pValues) { const order = pValues .map((p, index) => ({ index, p })) .sort((a, b) => a.p - b.p); const adjusted = new Array(order.length); let running = 0; for (let index = 0; index < order.length; index++) { running = Math.max( running, Math.min(1, (order.length - index) * order[index].p), ); adjusted[order[index].index] = running; } return adjusted; } function thresholdPValue(oldRates, newHistogram, scale, maxRegression) { const factor = 1 - maxRegression / 100; if (factor <= 0) return 1; const thresholdHistogram = createRateHistogram( oldRates.map((rate) => rate * factor), scale, 3); const result = thresholdHistogram.welchTest(newHistogram); if (Number.isNaN(result.pValue)) return 1; return result.tStatistic > 0 ? result.pValue / 2 : 1 - result.pValue / 2; } function isRegressionFailure(row, maxRegression) { return row.pThresholdAdjusted < 0.05 && row.improvement + row.ci95 < -maxRegression; } function analyzeCompare(samples, scale, maxRegression) { const groups = new Map(); for (const sample of samples) { let group = groups.get(sample.identity); if (group === undefined) { const suffix = sample.configuration === '' ? '' : ` ${sample.configuration}`; group = { name: `${sample.name}${suffix}`, new: [], old: [], }; groups.set(sample.identity, group); } group[sample.binary].push(sample.rate); } const rows = []; let skipped = 0; for (const { name, old: oldRates, new: newRates } of groups.values()) { if (oldRates.length < 2 || newRates.length < 2) { skipped++; continue; } const oldHistogram = createRateHistogram(oldRates, scale, 3); const newHistogram = createRateHistogram(newRates, scale, 3); const oldMean = oldRates.reduce((sum, rate) => sum + rate, 0) / oldRates.length; const newMean = newRates.reduce((sum, rate) => sum + rate, 0) / newRates.length; const improvement = ((newMean - oldMean) / oldMean) * 100; const w95 = oldHistogram.welchTest(newHistogram, { confidence: 0.95 }); const w99 = oldHistogram.welchTest(newHistogram, { confidence: 0.99 }); const w999 = oldHistogram.welchTest(newHistogram, { confidence: 0.999 }); let stars = ''; if (w95.pValue < 0.001) stars = '***'; else if (w95.pValue < 0.01) stars = ' **'; else if (w95.pValue < 0.05) stars = ' *'; const ciPercent = (result) => { const half = (result.confidenceInterval.upper - result.confidenceInterval.lower) / 2; return (half / (oldMean * scale)) * 100; }; const row = { ci95: ciPercent(w95), ci99: ciPercent(w99), ci999: ciPercent(w999), improvement, name, pValue: Number.isNaN(w95.pValue) ? 1 : w95.pValue, stars, }; if (maxRegression !== undefined) { row.pThreshold = thresholdPValue( oldRates, newHistogram, scale, maxRegression); } rows.push(row); } const adjusted = holmAdjust(rows.map(({ pValue }) => pValue)); const thresholdAdjusted = maxRegression === undefined ? null : holmAdjust(rows.map(({ pThreshold }) => pThreshold)); let underpowered = 0; for (let index = 0; index < rows.length; index++) { const row = rows[index]; row.pAdjusted = adjusted[index]; if (thresholdAdjusted !== null) { row.pThresholdAdjusted = thresholdAdjusted[index]; } row.inconclusive = maxRegression > 0 && row.stars.trim() === '' && row.ci95 > maxRegression; if (row.inconclusive) underpowered++; } const output = []; const maxNameLength = rows.reduce( (maximum, { name }) => Math.max(maximum, name.length), 0); const pad = (value, length) => value + ' '.repeat(Math.max(0, length - value.length)); const padStart = (value, length) => ' '.repeat(Math.max(0, length - value.length)) + value; output.push(`${pad('', maxNameLength)} confidence` + ' improvement accuracy (*) (**) (***)'); for (const row of rows) { const improvement = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`; output.push( `${pad(row.name, maxNameLength)} ${pad(row.stars, 10)}` + ` ${padStart(improvement, 11)}` + ` ±${row.ci95.toFixed(2)}%` + ` ±${row.ci99.toFixed(2)}%` + ` ±${row.ci999.toFixed(2)}%` + `${row.inconclusive ? ' (inconclusive)' : ''}`, ); } if (skipped > 0) { output.push(''); output.push( `Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` + ' skipped because Welch\'s t-test requires at least 2 samples per' + ' binary. Use --runs 2 or higher.', ); } printCompareChart(output, rows, maxNameLength); output.push(''); output.push( `Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).`, 'Use --scale to adjust precision if needed.', '', ); const significant = rows.filter(({ pAdjusted }) => pAdjusted < 0.05).length; output.push( 'The confidence markers above are per-benchmark and uncorrected. ' + `After Holm-Bonferroni correction across ${rows.length} comparison` + `${rows.length === 1 ? '' : 's'}, ${significant} remain` + `${significant === 1 ? 's' : ''} significant at 5%.`, ); if (maxRegression !== undefined) { output.push( `For --max-regression, one-sided p-values against the ` + `${maxRegression}% threshold were corrected separately.`, ); } if (maxRegression > 0 && underpowered > 0) { output.push(''); output.push( `Note: ${underpowered} of ${rows.length} comparison` + `${rows.length === 1 ? '' : 's'} could not resolve an effect as small ` + `as ${maxRegression}% and are marked (inconclusive). Raise --runs to ` + 'narrow their confidence intervals.', ); } const failures = maxRegression !== undefined ? rows.filter((row) => isRegressionFailure(row, maxRegression)) : []; if (failures.length > 0) { output.push(''); output.push( `FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` + ` regressed by more than ${maxRegression}% (the 95% interval excludes ` + `the threshold and its one-sided test is family-wise corrected across ` + `${rows.length} comparisons):`, ); for (const failure of failures) { output.push( ` ${failure.name} ${failure.improvement.toFixed(2)}% ` + `(95% CI up to ${(failure.improvement + failure.ci95).toFixed(2)}%, ` + `adjusted threshold p=` + `${failure.pThresholdAdjusted.toExponential(2)})`, ); } } return { failed: failures.length > 0, output: `${output.join('\n')}\n`, rows, }; } function printCompareChart(output, rows, maxNameLength) { if (rows.length === 0) return; const width = 40; const halfWidth = width / 2; let maximum = 0; for (const row of rows) { maximum = Math.max(maximum, Math.abs(row.improvement) + row.ci95); } if (maximum === 0) maximum = 1; const left = `-${maximum.toFixed(1)}%`; const right = `+${maximum.toFixed(1)}%`; const centerLabel = '0%'; const labelPadding = maxNameLength + 5; output.push(''); output.push( ' '.repeat(labelPadding) + left + ' '.repeat(Math.max( 0, halfWidth - left.length - Math.floor(centerLabel.length / 2))) + centerLabel + ' '.repeat(Math.max( 0, halfWidth - Math.ceil(centerLabel.length / 2) - right.length)) + right, ); for (const row of rows) { const center = halfWidth; const result = center + (row.improvement / maximum) * halfWidth; const lower = center + ((row.improvement - row.ci95) / maximum) * halfWidth; const upper = center + ((row.improvement + row.ci95) / maximum) * halfWidth; let bar = ''; for (let index = 0; index < width; index++) { const position = index + 0.5; if (index === Math.floor(center)) { bar += '|'; } else if ((row.improvement >= 0 && position > center && position <= result) || (row.improvement < 0 && position < center && position >= result)) { bar += row.stars === '' ? '▓' : '█'; } else if (position >= lower && position <= upper) { bar += '░'; } else { bar += ' '; } } const label = `${row.improvement >= 0 ? '+' : ''}` + `${row.improvement.toFixed(2)}%`; output.push( `${row.name.padEnd(maxNameLength)} ${bar} ${label} ${row.stars.trim()}`, ); } } function histogramScale(rates) { let minimum = Infinity; let maximum = 0; for (const rate of rates) { if (rate > 0 && rate < minimum) minimum = rate; if (rate > maximum) maximum = rate; } if (!Number.isFinite(minimum) || maximum === 0) return 1; let scale = 1; while (minimum * scale < 1e6 && maximum * scale < 1e15) scale *= 10; return scale; } function validateScatterParameters(samples, xAxis, category) { if (category !== undefined && category === xAxis) { throw new Error('--xaxis and --category must name different parameters'); } for (const key of [xAxis, category]) { if (key === undefined) continue; if (samples.some(({ params }) => !Object.hasOwn(params, key))) { const available = [...new Set(samples.flatMap( ({ params }) => Object.keys(params)))].sort(); throw new Error( `The variable '${key}' is not present in every configuration. ` + `Available variables: ${available.join(', ')}`, ); } } } function analyzeScatter(samples, xAxis, category, showChart) { validateScatterParameters(samples, xAxis, category); const parameterNames = [...new Set(samples.flatMap( ({ params }) => Object.keys(params)))]; const aggregated = parameterNames.filter((name) => { if (name === xAxis || name === category) return false; const first = samples[0].params[name]; return samples.some(({ params }) => params[name] !== first); }); const groups = new Map(); for (const sample of samples) { const xValue = sample.params[xAxis]; const categoryValue = category === undefined ? undefined : sample.params[category]; const key = valueKey([xValue, categoryValue]); let group = groups.get(key); if (group === undefined) { group = { categoryValue, members: [], observations: new Map(), xValue, }; groups.set(key, group); } group.members.push(sample); let rates = group.observations.get(sample.observation); if (rates === undefined) { rates = []; group.observations.set(sample.observation, rates); } rates.push(sample.rate); } for (const group of groups.values()) { group.processRates = [...group.observations].map(([observation, rates]) => ({ observation, rate: rates.reduce((sum, rate) => sum + rate, 0) / rates.length, })); group.rates = group.processRates.map(({ rate }) => rate); } const scale = histogramScale(samples.map(({ rate }) => rate)); const compareValues = (a, b) => { if (typeof a === 'number' && typeof b === 'number') return a - b; return String(a).localeCompare(String(b)); }; const rows = [...groups.values()] .sort((a, b) => compareValues(a.xValue, b.xValue) || compareValues(a.categoryValue, b.categoryValue)) .map((group) => { const histogram = createRateHistogram(group.rates, scale, 5); const count = group.rates.length; const mean = group.rates.reduce((sum, rate) => sum + rate, 0) / count; const meanInterval = histogram.meanCI(); const confidenceInterval = count > 1 ? (meanInterval.upper - meanInterval.lower) / (2 * scale) : NaN; const medianInterval = histogram.percentileCI(50); const median = rawMedian(group.rates); const skewed = count > 1 && (Math.abs(histogram.skewness) > 1 || Math.abs(median - mean) > confidenceInterval); return { ...group, confidenceInterval, count, histogram, mean, median, medianLower: medianInterval.lower / scale, medianUpper: medianInterval.upper / scale, skewed, }; }); const legend = assignLabels(rows, 'xValue', 'xLabel'); if (category !== undefined) { legend.push(...assignLabels(rows, 'categoryValue', 'categoryLabel')); } const output = []; const contamination = new Map(); for (const variable of aggregated) { const share = varianceShare([...groups.values()], variable); contamination.set(variable, share); const percent = share === 1 ? '100' : (share >= 0.995 ? '>99' : (share * 100).toFixed(0)); const suffix = Number.isNaN(share) ? '' : ` (explains ${percent}% of within-group variance)`; output.push(`aggregating variable: ${variable}${suffix}`); } const dominant = aggregated.filter( (variable) => contamination.get(variable) > 0.5); if (dominant.length > 0) { output.push(''); wrapOutput( output, `${dominant.join(', ')} ${dominant.length === 1 ? 'explains' : 'explain'} ` + 'most of the spread within each group. Pin the parameter or use it as ' + '--category; increasing --runs will not remove this source of variance.', ); } if (aggregated.length > 0) output.push(''); printScatterTable(output, rows, xAxis, category); if (showChart) printScatterChart(output, rows, xAxis, category); printScatterComparisons(output, rows, xAxis, category); if (legend.length > 0) { output.push('', 'Abbreviated values:'); for (const { full, label } of legend) { output.push(` ${label}`, ` = ${full}`); } } const singleSample = rows.filter(({ count }) => count < 2).length; if (singleSample > 0) { output.push(''); wrapOutput( output, `Note: ${singleSample} group${singleSample === 1 ? ' has' : 's have'} ` + 'only one sample, so no confidence interval could be estimated. Use ' + '--runs 2 or higher.', ); } if (rows.some(({ skewed }) => skewed)) { output.push(''); wrapOutput( output, '(!) marks groups where the median falls outside the mean confidence ' + 'interval or the sample is strongly skewed. The median and its interval ' + 'describe the typical run better for those groups.', ); } return `${output.join('\n')}\n`; } function rawMedian(rates) { const sorted = [...rates].sort((a, b) => a - b); const middle = Math.floor(sorted.length / 2); return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; } function valueKey(value) { return JSON.stringify(value, (_, item) => { if (typeof item === 'bigint') return { bigint: String(item) }; return item; }); } function varianceShare(groups, variable) { let between = 0; let total = 0; for (const group of groups) { if (group.members.length < 2) continue; const mean = group.members.reduce( (sum, sample) => sum + sample.rate, 0) / group.members.length; const levels = new Map(); for (const sample of group.members) { const key = valueKey(sample.params[variable]); let level = levels.get(key); if (level === undefined) { level = { count: 0, sum: 0 }; levels.set(key, level); } level.count++; level.sum += sample.rate; } for (const level of levels.values()) { between += level.count * ((level.sum / level.count) - mean) ** 2; } for (const sample of group.members) total += (sample.rate - mean) ** 2; } return total === 0 ? NaN : Math.min(1, between / total); } function effectSizeLabel(delta) { const absolute = Math.abs(delta); if (absolute < 0.147) return 'negligible'; if (absolute < 0.33) return 'small'; if (absolute < 0.474) return 'medium'; return 'large'; } const mannWhitneyFloors = new Map(); function mannWhitneyFloor(firstCount, secondCount) { const key = `${firstCount},${secondCount}`; let floor = mannWhitneyFloors.get(key); if (floor !== undefined) return floor; const low = createHistogram({ figures: 5 }); const high = createHistogram({ figures: 5 }); for (let index = 0; index < firstCount; index++) low.record(10000 + index); for (let index = 0; index < secondCount; index++) { high.record(10000 + firstCount + index); } floor = high.mannWhitneyTest(low).pValue; mannWhitneyFloors.set(key, floor); return floor; } function printScatterComparisons(output, rows, xAxis, category) { const usable = rows.filter(({ count }) => count > 1); if (usable.length < 2) return; const series = new Map(); for (const row of usable) { const key = valueKey(row.categoryValue); if (!series.has(key)) series.set(key, []); series.get(key).push(row); } const sections = []; let floor = 0; for (const group of series.values()) { if (group.length < 2) continue; const entries = []; for (let index = 1; index < group.length; index++) { const previous = group[index - 1]; const current = group[index]; // Configurations in one file share a process. Split consecutive groups // across disjoint outer-process sets so the unpaired test does not treat // correlated observations as independent. const parity = index % 2; const previousRates = previous.processRates .filter(({ observation }) => observation % 2 === parity) .map(({ rate }) => rate); const currentRates = current.processRates .filter(({ observation }) => observation % 2 !== parity) .map(({ rate }) => rate); if (previousRates.length === 0 || currentRates.length === 0) continue; const comparisonScale = histogramScale([ ...previousRates, ...currentRates, ]); const previousHistogram = createRateHistogram(previousRates, comparisonScale, 5); const currentHistogram = createRateHistogram(currentRates, comparisonScale, 5); const { pValue } = currentHistogram.mannWhitneyTest(previousHistogram); const delta = currentHistogram.cliffsD(previousHistogram); const previousMean = previousRates.reduce( (sum, rate) => sum + rate, 0) / previousRates.length; const currentMean = currentRates.reduce( (sum, rate) => sum + rate, 0) / currentRates.length; const change = ((currentMean - previousMean) / previousMean) * 100; floor = Math.max( floor, mannWhitneyFloor(previousRates.length, currentRates.length)); let ratio = ''; let exponent = ''; if (typeof previous.xValue === 'number' && typeof current.xValue === 'number' && previous.xValue > 0 && current.xValue > 0 && previous.xValue !== current.xValue && previousMean > 0 && currentMean > 0) { const xRatio = current.xValue / previous.xValue; const value = Math.log(currentMean / previousMean) / Math.log(xRatio); ratio = `${xRatio.toFixed(1)}x`; exponent = `${value >= 0 ? '+' : ''}${value.toFixed(2)}`; } entries.push( ` ${previous.xLabel} -> ${current.xLabel}` + ` ${change >= 0 ? '+' : ''}${change.toFixed(2)}%` + (exponent === '' ? '' : ` ${ratio} exponent=${exponent}`) + ` p=${pValue < 1e-4 ? pValue.toExponential(1) : pValue.toFixed(4)}` + ` delta=${delta >= 0 ? '+' : ''}${delta.toFixed(3)}` + ` (${effectSizeLabel(delta)})`, ); } if (entries.length > 0) { sections.push({ entries, heading: category === undefined ? undefined : `${category}=${group[0].categoryLabel}`, }); } } if (sections.length === 0) return; output.push('', `Change between consecutive ${xAxis} values ` + `(Mann-Whitney U on disjoint process sets, Cliff's delta):`); for (const section of sections) { output.push(''); if (section.heading !== undefined) output.push(` ${section.heading}`); output.push(...section.entries); } if (floor >= 0.05) { output.push(''); wrapOutput( output, `Warning: at this sample size the smallest p-value this test can ` + `produce is ${floor.toFixed(4)}, so no comparison above can reach ` + 'significance. Raise --runs.', ); } else if (floor >= 0.005) { output.push(''); wrapOutput( output, `Note: at this sample size the smallest p-value this test can produce ` + `is ${floor.toFixed(4)}. Raise --runs to strengthen non-significant ` + 'results.', ); } } function formatRate(rate) { return rate.toLocaleString('en-US', { maximumFractionDigits: 1, minimumFractionDigits: 1, }); } function displayWidth(value) { return [...value].length; } function pad(value, width, right) { const padding = ' '.repeat(Math.max(0, width - displayWidth(value))); return right ? padding + value : value + padding; } function truncateMiddle(value, maximum = 24) { const characters = [...value]; if (characters.length <= maximum) return value; const retained = maximum - 3; const head = Math.ceil(retained / 2); const tail = Math.floor(retained / 2); return `${characters.slice(0, head).join('')}...` + characters.slice(-tail).join(''); } function assignLabels(rows, valueName, labelName) { const assigned = new Map(); const used = new Map(); const legend = []; for (const row of rows) { const key = valueKey(row[valueName]); let label = assigned.get(key); if (label === undefined) { const full = typeof row[valueName] === 'string' ? inspect(row[valueName]) : String(row[valueName]); const abbreviated = truncateMiddle(full); const collisions = used.get(abbreviated) ?? 0; used.set(abbreviated, collisions + 1); label = collisions === 0 ? abbreviated : `${abbreviated}~${collisions + 1}`; assigned.set(key, label); if (label !== full) legend.push({ full, label }); } row[labelName] = label; } return legend; } function printScatterTable(output, rows, xAxis, category) { const header = [xAxis]; if (category !== undefined) header.push(category); header.push( 'samples', 'rate', 'confidence.interval', 'median', 'median.interval', ''); const body = rows.map((row) => { const values = [row.xLabel]; if (category !== undefined) values.push(row.categoryLabel); const medianInterval = row.count > 1 ? `[${(((row.medianLower - row.median) / row.median) * 100).toFixed(2)}%, ` + `+${(((row.medianUpper - row.median) / row.median) * 100).toFixed(2)}%]` : 'NA'; values.push( String(row.count), formatRate(row.mean), Number.isNaN(row.confidenceInterval) ? 'NA' : `${formatRate(row.confidenceInterval)} ` + `(±${((row.confidenceInterval / row.mean) * 100).toFixed(2)}%)`, formatRate(row.median), medianInterval, row.skewed ? '(!)' : '', ); return values; }); const widths = header.map((value, index) => Math.max( displayWidth(value), ...body.map((values) => displayWidth(values[index])), )); const right = [typeof rows[0].xValue === 'number']; if (category !== undefined) { right.push(typeof rows[0].categoryValue === 'number'); } right.push(true, true, true, true, true, false); const format = (values) => values.map( (value, index) => pad(value, widths[index], right[index])).join(' ').trimEnd(); output.push(format(header)); for (const values of body) output.push(format(values)); } function printScatterChart(output, rows, xAxis, category) { if (rows.length === 0) return; const width = 40; let maximum = 0; for (const row of rows) { maximum = Math.max( maximum, row.mean + (Number.isNaN(row.confidenceInterval) ? 0 : row.confidenceInterval), ); } if (maximum === 0) return; const labels = rows.map((row) => { let label = `${xAxis}=${row.xLabel}`; if (category !== undefined) label += ` ${category}=${row.categoryLabel}`; return truncateMiddle(label, 44); }); const labelWidth = Math.max(...labels.map(displayWidth)); const rateWidth = Math.max(...rows.map(({ mean }) => displayWidth(formatRate(mean)))); const axis = formatRate(maximum); const indent = ' '.repeat(labelWidth + 2); output.push( '', 'Rate in operations/second; longer is faster. │ marks the mean and the', 'shaded band (░) is its 95% confidence interval.', '', `${indent}0${' '.repeat(Math.max(1, width - 1 - displayWidth(axis)))}${axis}`, `${indent}+${'-'.repeat(width - 2)}+`, ); let previous; for (let index = 0; index < rows.length; index++) { const row = rows[index]; if (previous !== undefined && previous !== row.xValue) output.push(''); previous = row.xValue; const interval = Number.isNaN(row.confidenceInterval) ? 0 : row.confidenceInterval; const end = (row.mean / maximum) * width; const lower = ((row.mean - interval) / maximum) * width; const upper = ((row.mean + interval) / maximum) * width; const meanCell = Math.min(width - 1, Math.floor(end)); let bar = ''; for (let cell = 0; cell < width; cell++) { const position = cell + 0.5; if (cell === meanCell) bar += '│'; else if (position >= lower && position <= upper) bar += '░'; else if (position <= end) bar += '█'; else bar += ' '; } output.push( `${pad(labels[index], labelWidth, false)} ${bar} ` + pad(formatRate(row.mean), rateWidth, true), ); } } function wrapOutput(output, text, width = 76) { let line = ''; for (const word of text.split(/\s+/)) { if (line === '') line = word; else if (displayWidth(line) + displayWidth(word) + 1 <= width) { line += ` ${word}`; } else { output.push(line); line = word; } } if (line !== '') output.push(line); } module.exports = { analyzeCompare, analyzeScatter, holmAdjust, isRegressionFailure, validateScatterParameters, };