#!/usr/bin/env node import { spawnSync } from 'node:child_process' const args = process.argv.slice(2) const runsArgIndex = args.findIndex(arg => arg === '--runs' || arg === '-n') const warmupArgIndex = args.findIndex(arg => arg === '--warmup' || arg === '-w') const runs = runsArgIndex >= 0 ? Number(args[runsArgIndex + 1]) : Number(process.env.RUNS || 10) const warmup = warmupArgIndex >= 0 ? Number(args[warmupArgIndex + 1]) : Number(process.env.WARMUP || 5) const quiet = args.includes('--quiet') if (!Number.isFinite(runs) || runs <= 0) { console.error('Invalid runs value. Use --runs or set RUNS.') process.exit(1) } if (!Number.isFinite(warmup) || warmup < 0) { console.error('Invalid warmup value. Use --warmup or set WARMUP.') process.exit(1) } const cases = [ { title: 'build vs build-rollup', a: { label: 'build', cmd: 'node', args: ['scripts/build.js'], }, b: { label: 'build-rollup', cmd: 'node', args: ['scripts/build-with-rollup.js'], }, }, { title: 'build-dts vs build-dts-tsc', a: { label: 'build-dts', cmd: 'node', args: ['scripts/build-types.js'] }, b: { label: 'build-dts-tsc', cmd: 'node', args: ['scripts/build-dts-tsc.js'], }, }, ] function extractBuiltInMs(output) { const cleaned = output.replace(/\u001b\[[0-9;]*m/g, '').replace(/\s+/g, ' ') const matches = [...cleaned.matchAll(/built in\s+([\d.]+)ms/gi)] if (!matches.length) return null return matches.reduce((sum, match) => sum + Number(match[1]), 0) } function runOnce(command, commandArgs, label, index) { const result = spawnSync(command, commandArgs, { encoding: 'utf8', stdio: 'pipe', shell: false, }) if (!quiet) { if (result.stdout) process.stdout.write(result.stdout) if (result.stderr) process.stderr.write(result.stderr) } if (result.error) throw result.error if (result.status !== 0) { throw new Error( `${label} failed with exit code ${result.status} on run ${index + 1}`, ) } const output = `${result.stdout ?? ''}${result.stderr ?? ''}` const builtInMs = extractBuiltInMs(output) if (!Number.isFinite(builtInMs)) { const lines = output.trim().split(/\r?\n/) const tail = lines.slice(-20).join('\n') throw new Error( `${label} did not emit a "built in " timing line on run ${ index + 1 }.\nLast output:\n${tail}`, ) } return builtInMs } function summarize(times) { const sorted = [...times].sort((a, b) => a - b) const sum = times.reduce((acc, t) => acc + t, 0) const avg = sum / times.length const min = sorted[0] const max = sorted[sorted.length - 1] const mid = Math.floor(sorted.length / 2) const median = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid] const variance = times.reduce((acc, t) => acc + (t - avg) ** 2, 0) / times.length const stddev = Math.sqrt(variance) return { avg, min, max, median, stddev } } function formatMs(ms) { return `${(ms / 1000).toFixed(2)}s` } function buildTable(rows) { const widths = rows[0].map((_, col) => Math.max(...rows.map(row => String(row[col]).length)), ) const formatRow = row => `| ${row.map((cell, i) => String(cell).padEnd(widths[i], ' ')).join(' | ')} |` const separator = `| ${widths.map(w => '-'.repeat(w)).join(' | ')} |` return [formatRow(rows[0]), separator, ...rows.slice(1).map(formatRow)].join( '\n', ) } function renderStatsTable(labelA, labelB, statsA, statsB) { const diff = statsB.avg - statsA.avg const ratio = statsB.avg / statsA.avg const sign = diff >= 0 ? '+' : '-' const table = buildTable([ ['metric', labelA, labelB], ['avg', formatMs(statsA.avg), formatMs(statsB.avg)], ['median', formatMs(statsA.median), formatMs(statsB.median)], ['min', formatMs(statsA.min), formatMs(statsB.min)], ['max', formatMs(statsA.max), formatMs(statsB.max)], ['stddev', formatMs(statsA.stddev), formatMs(statsB.stddev)], ['avg diff', '-', `${sign}${formatMs(Math.abs(diff))}`], ['avg ratio', '-', `${ratio.toFixed(2)}x`], ]) console.log(table) } for (const testCase of cases) { console.log(`\n== ${testCase.title} ==`) const timesA = [] const timesB = [] if (warmup > 0) { console.log(`\nWarmup x${warmup} - ${testCase.a.label}`) for (let i = 0; i < warmup; i += 1) { runOnce(testCase.a.cmd, testCase.a.args, testCase.a.label, i) } console.log(`\nWarmup x${warmup} - ${testCase.b.label}`) for (let i = 0; i < warmup; i += 1) { runOnce(testCase.b.cmd, testCase.b.args, testCase.b.label, i) } } for (let i = 0; i < runs; i += 1) { console.log(`\nRun ${i + 1}/${runs} - ${testCase.a.label}`) const timeA = runOnce(testCase.a.cmd, testCase.a.args, testCase.a.label, i) timesA.push(timeA) console.log(`time: ${formatMs(timeA)}`) } for (let i = 0; i < runs; i += 1) { console.log(`\nRun ${i + 1}/${runs} - ${testCase.b.label}`) const timeB = runOnce(testCase.b.cmd, testCase.b.args, testCase.b.label, i) timesB.push(timeB) console.log(`time: ${formatMs(timeB)}`) } const statsA = summarize(timesA) const statsB = summarize(timesB) console.log('') renderStatsTable(testCase.a.label, testCase.b.label, statsA, statsB) } console.log('\nDone.')