Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Repair six CI gates that cannot fail on their charter defect (#4174)

Nick Gomez91b4b419bc5a
diff --git a/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsx b/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsx
new file mode 100644
index 000000000..dbb5b216c
--- /dev/null
+++ b/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsx
@@ -0,0 +1,180 @@
+import { act, cleanup, render } from '@testing-library/react';
+import type { Editor } from '@tiptap/core';
+import Collaboration from '@tiptap/extension-collaboration';
+import { EditorContent, useEditor } from '@tiptap/react';
+import StarterKit from '@tiptap/starter-kit';
+import { useEffect, useState } from 'react';
+import { createPortal } from 'react-dom';
+import { afterEach, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { JsxInline } from '../../src/editor/extensions/jsx-inline';
+
+const KEYSTROKE_COUNT = 100;
+const OBSERVED_HEALTHY_DELTA = 1;
+const MAX_ITEM_DELTA = 20;
+
+function totalStructs(doc: Y.Doc): number {
+ let total = 0;
+ for (const [, structs] of doc.store.clients) total += structs.length;
+ return total;
+}
+
+function jsxInlineElement(doc: Y.Doc): Y.XmlElement | null {
+ const paragraph = doc.getXmlFragment('default').get(0);
+ if (!(paragraph instanceof Y.XmlElement)) return null;
+ for (let i = 0; i < paragraph.length; i += 1) {
+ const child = paragraph.get(i);
+ if (child instanceof Y.XmlElement && child.nodeName === 'jsxInline') return child;
+ }
+ return null;
+}
+
+function jsxInlineRange(editor: Editor): { pos: number; contentSize: number } {
+ let pos = -1;
+ let contentSize = 0;
+ editor.state.doc.descendants((node, at) => {
+ if (node.type.name !== 'jsxInline') return undefined;
+ pos = at;
+ contentSize = node.content.size;
+ return false;
+ });
+ return { pos, contentSize };
+}
+
+function Host({ ydoc, onEditor }: { ydoc: Y.Doc; onEditor: (editor: Editor) => void }) {
+ const editor = useEditor({
+ extensions: [
+ StarterKit.configure({ link: false, undoRedo: false }),
+ JsxInline.configure({ docName: 'jsx-inline-item-growth' }),
+ Collaboration.configure({ document: ydoc }),
+ ],
+ editable: true,
+ immediatelyRender: true,
+ });
+ const [portalTarget] = useState(() => document.createElement('div'));
+ useEffect(() => {
+ document.body.appendChild(portalTarget);
+ return () => portalTarget.remove();
+ }, [portalTarget]);
+ if (editor) onEditor(editor);
+ return createPortal(
+ // biome-ignore lint/plugin/no-unportaled-editor-content: portalled per the H6 contract, with a per-render exclusive target owned by this harness
+ <EditorContent editor={editor} />,
+ portalTarget,
+ );
+}
+
+function mountJsxInlineEditor(ydoc: Y.Doc): Editor {
+ let captured: Editor | null = null;
+ render(
+ <Host
+ ydoc={ydoc}
+ onEditor={(editor) => {
+ captured = editor;
+ }}
+ />,
+ );
+ if (!captured) throw new Error('editor did not mount');
+ const editor = captured as Editor;
+ editor.commands.setContent({
+ type: 'doc',
+ content: [
+ {
+ type: 'paragraph',
+ content: [
+ { type: 'text', text: 'before ' },
+ {
+ type: 'jsxInline',
+ attrs: { componentName: 'Callout' },
+ content: [{ type: 'text', text: 'x' }],
+ },
+ { type: 'text', text: ' after' },
+ ],
+ },
+ ],
+ });
+ return editor;
+}
+
+function recreateJsxInlineElement(ydoc: Y.Doc): void {
+ const paragraph = ydoc.getXmlFragment('default').get(0);
+ if (!(paragraph instanceof Y.XmlElement)) return;
+ for (let i = 0; i < paragraph.length; i += 1) {
+ const child = paragraph.get(i);
+ if (!(child instanceof Y.XmlElement) || child.nodeName !== 'jsxInline') continue;
+ const clone = child.clone();
+ ydoc.transact(() => {
+ paragraph.delete(i, 1);
+ paragraph.insert(i, [clone]);
+ });
+ return;
+ }
+}
+
+function typeIntoJsxInline(
+ editor: Editor,
+ ydoc: Y.Doc,
+ options: { mode?: 'append' | 'alternate'; afterKeystroke?: (ydoc: Y.Doc) => void } = {},
+): { delta: number; elementPreserved: boolean } {
+ const mode = options.mode ?? 'append';
+ const elementBefore = jsxInlineElement(ydoc);
+ const before = totalStructs(ydoc);
+ act(() => {
+ for (let i = 0; i < KEYSTROKE_COUNT; i += 1) {
+ const { pos, contentSize } = jsxInlineRange(editor);
+ if (pos < 0) throw new Error('jsxInline node vanished mid-typing');
+ const at = mode === 'alternate' && i % 2 === 0 ? pos + 1 : pos + 1 + contentSize;
+ editor.view.dispatch(editor.state.tr.insertText(String.fromCharCode(97 + (i % 26)), at, at));
+ options.afterKeystroke?.(ydoc);
+ }
+ });
+ const elementAfter = jsxInlineElement(ydoc);
+ return {
+ delta: totalStructs(ydoc) - before,
+ elementPreserved: elementBefore !== null && elementBefore === elementAfter,
+ };
+}
+
+afterEach(cleanup);
+
+describe('Y.Item growth under jsxInline typing in a mounted collaborative editor', () => {
+ test('typing 100 keystrokes inside a jsxInline node keeps struct growth sublinear', () => {
+ const ydoc = new Y.Doc();
+ const editor = mountJsxInlineEditor(ydoc);
+
+ expect(jsxInlineElement(ydoc)).toBeInstanceOf(Y.XmlElement);
+
+ const { delta, elementPreserved } = typeIntoJsxInline(editor, ydoc);
+
+ expect(
+ delta,
+ `Y struct delta ${delta} over ${KEYSTROKE_COUNT} keystrokes exceeds the bound ${MAX_ITEM_DELTA} (observed healthy value: ${OBSERVED_HEALTHY_DELTA}). ` +
+ `The bound sits 5x and 10x below the defect arms, which land near 100 and ${KEYSTROKE_COUNT * 2 + 1}, so a delta just above ${MAX_ITEM_DELTA} and still far below ${KEYSTROKE_COUNT} after a routine ` +
+ 'Yjs / ProseMirror / TipTap bump is a change in their struct-merging internals, not content loss: re-measure and widen this constant. Hunt for a content-loss bug only when the delta approaches per-keystroke growth.',
+ ).toBeLessThanOrEqual(MAX_ITEM_DELTA);
+ expect(elementPreserved).toBe(true);
+ expect(editor.state.doc.textContent).toContain('xabcdefghij');
+ });
+
+ test('recreating the jsxInline Y.XmlElement on every keystroke breaks the bound', () => {
+ const ydoc = new Y.Doc();
+ const editor = mountJsxInlineEditor(ydoc);
+
+ const { delta, elementPreserved } = typeIntoJsxInline(editor, ydoc, {
+ afterKeystroke: recreateJsxInlineElement,
+ });
+
+ expect(delta).toBeGreaterThan(MAX_ITEM_DELTA);
+ expect(elementPreserved).toBe(false);
+ });
+
+ test('one unmergeable struct per keystroke breaks the bound', () => {
+ const ydoc = new Y.Doc();
+ const editor = mountJsxInlineEditor(ydoc);
+
+ const { delta } = typeIntoJsxInline(editor, ydoc, { mode: 'alternate' });
+
+ expect(delta).toBe(KEYSTROKE_COUNT);
+ expect(delta).toBeGreaterThan(MAX_ITEM_DELTA);
+ });
+});
diff --git a/packages/app/tests/stress/component-blocks.perf.test.ts b/packages/app/tests/stress/component-blocks.perf.test.ts
index 518c97603..47cab915c 100644
--- a/packages/app/tests/stress/component-blocks.perf.test.ts
+++ b/packages/app/tests/stress/component-blocks.perf.test.ts
@@ -1,5 +1,4 @@
import { describe, expect, test } from 'vitest';
-import * as Y from 'yjs';
import { sharedExtensions } from '../../../../packages/core/src/extensions/shared.ts';
import { MarkdownManager } from '../../../../packages/core/src/markdown/index.ts';
@@ -66,45 +65,3 @@ describe('PF03: parseWithFallback cycle time under load', () => {
expect(maxVal).toBeLessThan(MAX_CATASTROPHIC_MS);
});
});
-
-describe('PF05: Y.Item growth under jsxInline typing', () => {
- test('100-keystroke typing in jsxInline content — Y.Item delta ≤ keystroke_count + constant', () => {
- const ydoc = new Y.Doc();
- const ytext = ydoc.getText('source');
-
- const initialContent = 'Hello <Icon name="check" /> world';
- ytext.insert(0, initialContent);
-
- const countItems = (yt: Y.Text): number => {
- let count = 0;
- let item = yt._start;
- while (item !== null) {
- if (!item.deleted) count++;
- item = item.right;
- }
- return count;
- };
-
- const initialItems = countItems(ytext);
- const KEYSTROKE_COUNT = 100;
-
- for (let i = 0; i < KEYSTROKE_COUNT; i++) {
- const insertPos = ytext.toString().length;
- ydoc.transact(() => {
- ytext.insert(insertPos, String.fromCharCode(97 + (i % 26)));
- });
- }
-
- const finalItems = countItems(ytext);
- const itemDelta = finalItems - initialItems;
-
- console.log(
- `PF05 results: initialItems=${initialItems}, finalItems=${finalItems}, delta=${itemDelta}, keystrokes=${KEYSTROKE_COUNT}`,
- );
-
- const CONSTANT_OVERHEAD = 10;
- expect(itemDelta).toBeLessThanOrEqual(KEYSTROKE_COUNT + CONSTANT_OVERHEAD);
-
- expect(itemDelta).toBeLessThan(KEYSTROKE_COUNT * 2);
- });
-});
diff --git a/packages/core/package.json b/packages/core/package.json
index cc56fdfda..e993652e7 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -145,9 +145,9 @@
"scripts": {
"build": "tsdown && pnpm run verify:exports",
"verify:exports": "node scripts/verify-exports-targets.mjs",
- "typecheck": "tsc --noEmit",
+ "typecheck": "tsc --noEmit && tsc -p tsconfig.perf.json --noEmit",
"test": "vitest run src/ tests/contract/ scripts/",
- "test:perf:bench": "RUN_BENCH=1 vitest run tests/perf/markdown-bench.test.ts",
+ "test:perf:bench": "NODE_OPTIONS=\"${NODE_OPTIONS:-} --expose-gc\" RUN_BENCH=1 vitest run tests/perf/markdown-bench.test.ts",
"test:perf:regression:unit": "vitest run tests/perf/regression-gate.test.ts",
"test:perf:regression": "tsx tests/perf/run-regression-gate.ts",
"test:health:unit": "vitest run tests/health/parse-health-gate.test.ts",
diff --git a/packages/core/tests/perf/baseline.json b/packages/core/tests/perf/baseline.json
index 24d2a9ebf..a6f54aed4 100644
--- a/packages/core/tests/perf/baseline.json
+++ b/packages/core/tests/perf/baseline.json
@@ -1,5 +1,5 @@
{
- "schemaVersion": 1,
+ "schemaVersion": 2,
"capturedAt": "2026-04-16T08:28:01.616Z",
"runnerClass": "local-m-series-calibration",
"methodology": {
@@ -7,10 +7,14 @@
"measuredIters": 10,
"gcBetweenRuns": true
},
- "toolchain": {
+ "capturedUnder": {
"runtime": "[email protected]",
"testRunner": "bun test"
},
+ "targetToolchain": {
+ "runtime": "[email protected]",
+ "testRunner": "[email protected]"
+ },
"calibrationRuns": 4,
"threshold": {
"floorPct": 0.1,
diff --git a/packages/core/tests/perf/gc.ts b/packages/core/tests/perf/gc.ts
new file mode 100644
index 000000000..4529c5ca2
--- /dev/null
+++ b/packages/core/tests/perf/gc.ts
@@ -0,0 +1,19 @@
+type GcHost = { gc?: () => void };
+
+export const EXPOSE_GC_FLAG = '--expose-gc';
+
+export function gcAvailable(): boolean {
+ return typeof (globalThis as GcHost).gc === 'function';
+}
+
+export function collectGarbage(): void {
+ (globalThis as GcHost).gc?.();
+}
+
+export function withForcedGc<T extends NodeJS.ProcessEnv>(
+ env: T,
+): Omit<T, 'NODE_OPTIONS'> & { NODE_OPTIONS: string } {
+ const current = env.NODE_OPTIONS ?? '';
+ if (current.split(/\s+/).includes(EXPOSE_GC_FLAG)) return { ...env, NODE_OPTIONS: current };
+ return { ...env, NODE_OPTIONS: current ? `${current} ${EXPOSE_GC_FLAG}` : EXPOSE_GC_FLAG };
+}
diff --git a/packages/core/tests/perf/markdown-bench.test.ts b/packages/core/tests/perf/markdown-bench.test.ts
index fc2af2971..dd5d106cd 100644
--- a/packages/core/tests/perf/markdown-bench.test.ts
+++ b/packages/core/tests/perf/markdown-bench.test.ts
@@ -12,6 +12,8 @@ import {
type PerfBlockCount,
} from '../../src/markdown/fixtures/index.ts';
import { MarkdownManager } from '../../src/markdown/index.ts';
+import { collectGarbage, gcAvailable } from './gc.ts';
+import type { FreshResults } from './regression-gate.ts';
const BENCH_ENABLED = process.env.RUN_BENCH === '1' || process.env.RUN_BENCH === 'true';
@@ -69,7 +71,7 @@ function readVitestVersion(): string {
}
}
-interface RunnerInfo {
+type RunnerInfo = {
nodeVersion: string;
vitestVersion: string;
gitSha: string;
@@ -79,7 +81,7 @@ interface RunnerInfo {
ramGB: number;
platform: string;
runnerClass: string;
-}
+};
function runnerInfo(): RunnerInfo {
const cpuList = cpus();
@@ -96,12 +98,12 @@ function runnerInfo(): RunnerInfo {
};
}
-const GC_FORCED = typeof (globalThis as { gc?: () => void }).gc === 'function';
+const GC_FORCED = gcAvailable();
function measure(op: () => void, n: number): number[] {
const samples: number[] = [];
for (let i = 0; i < n; i++) {
- (globalThis as { gc?: () => void }).gc?.();
+ collectGarbage();
const t0 = performance.now();
op();
samples.push(performance.now() - t0);
@@ -164,7 +166,7 @@ describeBench('markdown pipeline benchmark harness (R1)', () => {
}
const output = {
- schemaVersion: 1,
+ schemaVersion: 2,
startedAt,
finishedAt: new Date().toISOString(),
methodology: {
@@ -172,9 +174,13 @@ describeBench('markdown pipeline benchmark harness (R1)', () => {
measuredIters: MEASURED_ITERS,
gcBetweenRuns: GC_FORCED,
},
+ toolchain: {
+ runtime: `node@${process.versions.node}`,
+ testRunner: `vitest@${readVitestVersion()}`,
+ },
runner: runnerInfo(),
results,
- };
+ } satisfies FreshResults;
const stamp = startedAt.replace(/[:.]/g, '-');
const target = resolve(HARNESS_DIR, `results.${stamp}.json`);
diff --git a/packages/core/tests/perf/regression-gate.test.ts b/packages/core/tests/perf/regression-gate.test.ts
index 113f8d06d..9acc94fd6 100644
--- a/packages/core/tests/perf/regression-gate.test.ts
+++ b/packages/core/tests/perf/regression-gate.test.ts
@@ -1,22 +1,38 @@
-import { mkdtempSync, writeFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, test } from 'vitest';
+import { withForcedGc } from './gc.ts';
import {
+ ACCEPT_MISMATCH_FLAG,
type Baseline,
+ baselineResultsToolchain,
checkMethodologyMismatches,
+ comparabilityBlockers,
+ DataError,
+ EXIT_DATA,
+ EXIT_INCONCLUSIVE,
+ EXIT_USAGE,
evaluateRegression,
+ exitCodeForFailure,
type FreshResults,
formatReport,
loadBaseline,
loadFreshResults,
} from './regression-gate.ts';
+import {
+ forcedGcPlumbingFailure,
+ InconclusiveError,
+ resultsStalenessFailure,
+} from './run-regression-gate.ts';
function makeBaseline(overrides: Partial<Baseline> = {}): Baseline {
return {
- schemaVersion: 1,
+ schemaVersion: 2,
capturedAt: '2026-04-16T00:00:00.000Z',
runnerClass: 'test-fixture',
+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },
calibrationRuns: 5,
threshold: { floorPct: 0.1, varianceMultiplier: 2 },
results: [
@@ -41,10 +57,11 @@ function makeBaseline(overrides: Partial<Baseline> = {}): Baseline {
function makeFresh(overrides: Partial<FreshResults> = {}): FreshResults {
return {
- schemaVersion: 1,
+ schemaVersion: 2,
startedAt: '2026-04-16T01:00:00.000Z',
finishedAt: '2026-04-16T01:05:00.000Z',
methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
runner: {},
results: [
{
@@ -83,8 +100,8 @@ describe('checkMethodologyMismatches', () => {
makeFresh({ methodology: { ...METHODOLOGY, gcBetweenRuns: false } }),
);
expect(warnings).toHaveLength(1);
- expect(warnings[0]).toContain('gcBetweenRuns');
- expect(warnings[0]).toContain('--expose-gc');
+ expect(warnings[0].message).toContain('gcBetweenRuns');
+ expect(warnings[0].message).toContain('--expose-gc');
});
test('a different measured-iteration count warns', () => {
@@ -93,7 +110,7 @@ describe('checkMethodologyMismatches', () => {
makeFresh({ methodology: { ...METHODOLOGY, measuredIters: 100 } }),
);
expect(warnings).toHaveLength(1);
- expect(warnings[0]).toContain('measuredIters');
+ expect(warnings[0].message).toContain('measuredIters');
});
test('both differing produces both warnings', () => {
@@ -112,10 +129,16 @@ describe('checkMethodologyMismatches', () => {
expect(warnings).toEqual([]);
});
- test('a fresh run without recorded methodology produces no warnings', () => {
+ test('FreshResults.methodology is required at the type level', () => {
+ // @ts-expect-error methodology is required on FreshResults; the silent no-warning path is unreachable
+ const invalid: FreshResults = { ...makeFresh(), methodology: undefined };
+ expect(invalid.methodology).toBeUndefined();
+ });
+
+ test('a baseline without a methodology block still compares, which is the surviving optional side', () => {
const warnings = checkMethodologyMismatches(
- makeBaseline({ methodology: METHODOLOGY }),
- makeFresh({ methodology: undefined }),
+ makeBaseline({ methodology: undefined }),
+ makeFresh({ methodology: METHODOLOGY }),
);
expect(warnings).toEqual([]);
});
@@ -272,6 +295,62 @@ describe('loadBaseline / loadFreshResults finite-value validation', () => {
expect(() => loadFreshResults(path)).toThrow(/parseMs\.p95 is not finite/);
});
+ test('RED: a string blockCount in the baseline is rejected rather than joining to nothing', () => {
+ const baseline = makeBaseline();
+ const path = writeTmp('baseline.json', {
+ ...baseline,
+ results: [{ ...baseline.results[0], blockCount: '100' }, baseline.results[1]],
+ });
+ expect(() => loadBaseline(path)).toThrow(DataError);
+ expect(() => loadBaseline(path)).toThrow(
+ /results\[0\]\.blockCount must be a finite number \(got "100"\)\. blockCount is the key baseline rows are joined to fresh rows on/,
+ );
+ });
+
+ test('a string blockCount would otherwise have been reported as a regression, not as bad data', () => {
+ const baseline = makeBaseline();
+ const corrupt: Baseline = {
+ ...baseline,
+ results: [
+ // @ts-expect-error blockCount is a number in the type; a hand-edited baseline.json can carry a string
+ { ...baseline.results[0], blockCount: '100' },
+ baseline.results[1],
+ ],
+ };
+ const report = evaluateRegression(corrupt, makeFresh());
+ expect(report.missingFresh).toEqual(['100']);
+ expect(report.pass).toBe(false);
+ });
+
+ test('RED: a non-object results entry is rejected before anything dereferences it', () => {
+ const baseline = makeBaseline();
+ const bare = writeTmp('baseline.json', { ...baseline, results: [42, baseline.results[1]] });
+ expect(() => loadBaseline(bare)).toThrow(DataError);
+ expect(() => loadBaseline(bare)).toThrow(/results\[0\] must be an object \(got 42\)/);
+ const nulled = writeTmp('baseline.json', { ...baseline, results: [null, baseline.results[1]] });
+ expect(() => loadBaseline(nulled)).toThrow(DataError);
+ expect(() => loadBaseline(nulled)).toThrow(/results\[0\] must be an object \(got null\)/);
+ const arrayed = writeTmp('baseline.json', { ...baseline, results: [[], baseline.results[1]] });
+ expect(() => loadBaseline(arrayed)).toThrow(DataError);
+ expect(() => loadBaseline(arrayed)).toThrow(/results\[0\] must be an object \(got \[\]\)/);
+ });
+
+ test('RED: a docSizeChars that is not a number is rejected in both loaders', () => {
+ const baseline = makeBaseline();
+ const withStringSize = {
+ ...baseline,
+ results: [baseline.results[0], { ...baseline.results[1], docSizeChars: '200000' }],
+ };
+ expect(() => loadBaseline(writeTmp('baseline.json', withStringSize))).toThrow(
+ /results\[1\]\.docSizeChars must be a finite number \(got "200000"\)\. docSizeChars is provenance the comparator never reads/,
+ );
+ const fresh = makeFresh();
+ fresh.results[0].docSizeChars = Number.NaN;
+ expect(() => loadFreshResults(writeTmp('results.json', fresh))).toThrow(
+ /results\[0\]\.docSizeChars must be a finite number \(got null\)\. docSizeChars is provenance the comparator never reads/,
+ );
+ });
+
test('loadBaseline accepts a valid baseline', () => {
const path = writeTmp('baseline.json', makeBaseline());
expect(() => loadBaseline(path)).not.toThrow();
@@ -282,3 +361,979 @@ describe('loadBaseline / loadFreshResults finite-value validation', () => {
expect(() => loadFreshResults(path)).not.toThrow();
});
});
+
+const PERF_DIR = import.meta.dirname;
+const COMMITTED_BASELINE_PATH = join(PERF_DIR, 'baseline.json');
+
+type OpKey = 'parseMs' | 'serializeMs' | 'roundTripMs';
+
+function freshFromBaseline(
+ baseline: Baseline,
+ scale: Partial<Record<OpKey, number>> = {},
+ overrides: Partial<FreshResults> = {},
+): FreshResults {
+ return {
+ schemaVersion: 2,
+ startedAt: '2026-09-04T00:00:00.000Z',
+ finishedAt: '2026-09-04T00:10:00.000Z',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ runner: {},
+ results: baseline.results.map((entry) => {
+ const op = (name: OpKey) => {
+ const p99 = entry[name].p99 * (scale[name] ?? 1);
+ return { mean: p99 * 0.8, min: p99 * 0.7, max: p99, p50: p99 * 0.8, p95: p99 * 0.95, p99 };
+ };
+ return {
+ blockCount: entry.blockCount,
+ docSizeChars: entry.docSizeChars,
+ parseMs: op('parseMs'),
+ serializeMs: op('serializeMs'),
+ roundTripMs: op('roundTripMs'),
+ };
+ }),
+ ...overrides,
+ };
+}
+
+function regressedParseBlockCounts(baseline: Baseline, scale: number): number[] {
+ return evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: scale }))
+ .rows.filter((r) => r.op === 'parseMs' && r.regression)
+ .map((r) => r.blockCount)
+ .sort((a, b) => a - b);
+}
+
+describe('committed baseline.json as the gate substrate', () => {
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+
+ test('the committed baseline loads and records the provenance of its results', () => {
+ expect(baseline.results).toHaveLength(5);
+ expect(baseline.capturedUnder).toEqual({ runtime: '[email protected]', testRunner: 'bun test' });
+ expect(baseline.targetToolchain).toEqual({
+ runtime: '[email protected]',
+ testRunner: '[email protected]',
+ });
+ expect(baselineResultsToolchain(baseline)?.runtime).toBe('[email protected]');
+ });
+
+ test('a fresh run identical to the committed baseline passes', () => {
+ const report = evaluateRegression(baseline, freshFromBaseline(baseline));
+ expect(report.pass).toBe(true);
+ expect(report.missingFresh).toEqual([]);
+ expect(report.rows.filter((r) => r.regression)).toEqual([]);
+ });
+
+ test('parse p99 raised 9% passes on 0 of the 5 parse rows', () => {
+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 1.09 }));
+ expect(report.pass).toBe(true);
+ expect(regressedParseBlockCounts(baseline, 1.09)).toEqual([]);
+ });
+
+ test('parse p99 raised 11% fails on the 4 of 5 parse rows the 10% floor governs', () => {
+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 1.11 }));
+ expect(report.pass).toBe(false);
+ expect(regressedParseBlockCounts(baseline, 1.11)).toEqual([1000, 5000, 10000, 20000]);
+ const smallest = report.rows.find((r) => r.blockCount === 100 && r.op === 'parseMs');
+ expect(smallest?.regression).toBe(false);
+ expect(smallest?.allowedDeltaMs).toBeCloseTo(1.78, 6);
+ expect(report.rows.filter((r) => r.op !== 'parseMs' && r.regression)).toEqual([]);
+ });
+
+ test('parse p99 raised 20% fails on all 5 parse rows, including the variance-governed one', () => {
+ expect(regressedParseBlockCounts(baseline, 1.2)).toEqual([100, 1000, 5000, 10000, 20000]);
+ });
+});
+
+describe('toolchain provenance warning', () => {
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+
+ test('a node/vitest run against the bun-era committed results warns and names both toolchains', () => {
+ const warnings = checkMethodologyMismatches(baseline, freshFromBaseline(baseline));
+ const toolchainWarnings = warnings.filter((w) => w.kind === 'toolchain');
+ expect(toolchainWarnings).toHaveLength(1);
+ expect(toolchainWarnings[0].message).toContain('[email protected]');
+ expect(toolchainWarnings[0].message).toContain('bun test');
+ expect(toolchainWarnings[0].message).toContain('[email protected]');
+ expect(toolchainWarnings[0].message).toContain('[email protected]');
+ });
+
+ test('a fresh run under the same toolchain as the results produces no toolchain warning', () => {
+ const fresh = freshFromBaseline(
+ baseline,
+ {},
+ {
+ toolchain: { runtime: '[email protected]', testRunner: 'bun test' },
+ },
+ );
+ expect(checkMethodologyMismatches(baseline, fresh)).toEqual([]);
+ });
+
+ test('RED: a schemaVersion 1 baseline is rejected, because `toolchain` meant something else there', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ const legacy = { ...makeBaseline(), schemaVersion: 1 };
+ writeFileSync(path, JSON.stringify(legacy));
+ expect(() => loadBaseline(path)).toThrow(/schemaVersion must be 2/);
+ });
+
+ test('RED: targetToolchain without capturedUnder is rejected as unrecorded provenance', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ const { capturedUnder: _dropped, ...noProvenance } = makeBaseline();
+ writeFileSync(
+ path,
+ JSON.stringify({
+ ...noProvenance,
+ targetToolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ }),
+ );
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);
+ });
+
+ test('RED: the version-1 `toolchain` spelling under schemaVersion 2 is rejected, not silently accepted', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ const { capturedUnder, ...rest } = makeBaseline();
+ writeFileSync(path, JSON.stringify({ ...rest, toolchain: capturedUnder }));
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);
+ expect(() => loadBaseline(path)).toThrow(/version-1 `toolchain` key/);
+ });
+
+ test('RED: capturedUnder null is rejected the same way absence is', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), capturedUnder: null }));
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);
+ });
+
+ test('RED: a baseline with no provenance at all is rejected, so no run can be silently comparable', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ const { capturedUnder: _c, targetToolchain: _t, ...stripped } = makeBaseline();
+ writeFileSync(path, JSON.stringify(stripped));
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);
+ });
+
+ for (const [label, value] of [
+ ['an empty string', ''],
+ ['false', false],
+ ['zero', 0],
+ ['an array', []],
+ ['an empty object', {}],
+ ['an object whose fields are empty strings', { runtime: '', testRunner: '' }],
+ ] as const) {
+ test(`RED: capturedUnder as ${label} is rejected, not carried through as silent provenance`, () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), capturedUnder: value }));
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);
+ });
+ }
+
+ test('RED: a results file with no toolchain is rejected, so a foreign capture cannot compare silently', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ const { toolchain: _dropped, ...bare } = makeFresh();
+ writeFileSync(path, JSON.stringify(bare));
+ expect(() => loadFreshResults(path)).toThrow(/toolchain must be an object declaring runtime/);
+ });
+
+ test('RED: a version-1 results file is rejected on the VERSION, not on a field version 1 never had', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ const { toolchain: _dropped, ...preToolchain } = makeFresh();
+ writeFileSync(path, JSON.stringify({ ...preToolchain, schemaVersion: 1 }));
+ expect(() => loadFreshResults(path)).toThrow(/results\.json schemaVersion must be 2/);
+ expect(() => loadFreshResults(path)).toThrow(/declares no `toolchain`/);
+ expect(() => loadFreshResults(path)).not.toThrow(/renumbering it to 2/);
+ });
+
+ test('a version-2 results file carrying toolchain loads, so the bump is not a blanket rejection', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ writeFileSync(path, JSON.stringify(makeFresh()));
+ const loaded = loadFreshResults(path);
+ expect(loaded.schemaVersion).toBe(2);
+ expect(loaded.toolchain).toEqual({ runtime: '[email protected]', testRunner: '[email protected]' });
+ });
+
+ test('RED: a version-1 results file that already declares a toolchain is told to renumber, not to re-capture', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ writeFileSync(path, JSON.stringify({ ...makeFresh(), schemaVersion: 1 }));
+ expect(() => loadFreshResults(path)).toThrow(/results\.json schemaVersion must be 2/);
+ expect(() => loadFreshResults(path)).toThrow(/renumbering it to 2 preserves/);
+ expect(() => loadFreshResults(path)).not.toThrow(/declares no `toolchain`/);
+ });
+
+ for (const [label, value] of [
+ ['an empty object', {}],
+ ['a bare string', '[email protected]'],
+ ['an array', []],
+ ['null', null],
+ ['an object whose fields are empty strings', { runtime: '', testRunner: '' }],
+ ] as const) {
+ test(`RED: a version-1 results file whose toolchain is ${label} is told to re-capture, not to renumber`, () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ writeFileSync(path, JSON.stringify({ ...makeFresh(), schemaVersion: 1, toolchain: value }));
+ expect(() => loadFreshResults(path)).toThrow(/declares no `toolchain`/);
+ expect(() => loadFreshResults(path)).not.toThrow(/renumbering it to 2/);
+ });
+ }
+
+ test('FreshResults.schemaVersion is the literal 2 at the type level', () => {
+ // @ts-expect-error schemaVersion is pinned to 2; a version-1 capture is not a FreshResults
+ const invalid: FreshResults = { ...makeFresh(), schemaVersion: 1 };
+ expect(invalid.schemaVersion).toBe(1);
+ });
+
+ test('RED: a results file with no methodology block is rejected', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));
+ const path = join(dir, 'results.json');
+ const { methodology: _dropped, ...bare } = makeFresh();
+ writeFileSync(path, JSON.stringify(bare));
+ expect(() => loadFreshResults(path)).toThrow(/methodology must be an object recording/);
+ });
+
+ for (const [label, threshold] of [
+ ['an empty object', {}],
+ ['a missing floorPct', { varianceMultiplier: 2 }],
+ ['a non-finite varianceMultiplier', { floorPct: 0.1, varianceMultiplier: Number.NaN }],
+ ] as const) {
+ test(`RED: threshold with ${label} is rejected rather than computing an NaN allowance`, () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-threshold-'));
+ const path = join(dir, 'baseline.json');
+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), threshold }));
+ expect(() => loadBaseline(path)).toThrow(
+ /must be a finite number|threshold must be an object/,
+ );
+ });
+ }
+
+ test('NEGATIVE: a 50x slower run against a well-formed threshold still FAILS', () => {
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 50 }));
+ expect(report.pass).toBe(false);
+ });
+
+ test('RED: capturedUnder equal to targetToolchain is rejected as a finished re-baseline', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const path = join(dir, 'baseline.json');
+ const same = { runtime: '[email protected]', testRunner: '[email protected]' };
+ writeFileSync(
+ path,
+ JSON.stringify(makeBaseline({ capturedUnder: same, targetToolchain: { ...same } })),
+ );
+ expect(() => loadBaseline(path)).toThrow(/capturedUnder equals targetToolchain/);
+ });
+
+ test('NEGATIVE: capturedUnder alone, and capturedUnder differing from targetToolchain, both load', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));
+ const soloPath = join(dir, 'solo.json');
+ writeFileSync(
+ soloPath,
+ JSON.stringify(makeBaseline({ capturedUnder: { runtime: '[email protected]' } })),
+ );
+ expect(loadBaseline(soloPath).capturedUnder?.runtime).toBe('[email protected]');
+ expect(loadBaseline(COMMITTED_BASELINE_PATH).capturedUnder?.runtime).toBe('[email protected]');
+ });
+
+ test('a re-baselined file carries capturedUnder alone and warns only on a real change', () => {
+ const rebaselined = makeBaseline({
+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },
+ });
+ expect(baselineResultsToolchain(rebaselined)?.runtime).toBe('[email protected]');
+ expect(checkMethodologyMismatches(rebaselined, freshFromBaseline(rebaselined))).toEqual([]);
+ const stale = freshFromBaseline(
+ rebaselined,
+ {},
+ {
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ },
+ );
+ expect(checkMethodologyMismatches(rebaselined, stale)).toHaveLength(1);
+ });
+
+ test('FreshResults.toolchain is required at the type level', () => {
+ // @ts-expect-error toolchain is required on FreshResults; the silent no-warning path is unreachable
+ const invalid: FreshResults = { ...freshFromBaseline(baseline), toolchain: undefined };
+ expect(invalid.toolchain).toBeUndefined();
+ });
+});
+
+describe('resultsStalenessFailure', () => {
+ const SPAWNED_AT = 1_000_000;
+
+ test('NEGATIVE: a capture written after the spawn is not stale', () => {
+ expect(
+ resultsStalenessFailure([{ f: 'results.1.json', mtimeMs: SPAWNED_AT + 1 }], SPAWNED_AT),
+ ).toBeNull();
+ });
+
+ test('BOUNDARY: a capture written exactly at the spawn instant counts as this run', () => {
+ expect(
+ resultsStalenessFailure([{ f: 'results.1.json', mtimeMs: SPAWNED_AT }], SPAWNED_AT),
+ ).toBeNull();
+ });
+
+ test('RED: a capture older than the spawn names RUN_BENCH as the propagation suspect', () => {
+ const failure = resultsStalenessFailure(
+ [{ f: 'results.old.json', mtimeMs: SPAWNED_AT - 1 }],
+ SPAWNED_AT,
+ );
+ expect(failure).toMatch(/predates the bench spawn/);
+ expect(failure).toMatch(/RUN_BENCH/);
+ });
+
+ test('RED: an empty directory names the missing capture rather than grading nothing', () => {
+ expect(resultsStalenessFailure([], SPAWNED_AT)).toMatch(/no results\.\*\.json found/);
+ });
+
+ test('the freshest entry wins regardless of directory order, so a stale sibling cannot mask it', () => {
+ const entries = [
+ { f: 'results.old.json', mtimeMs: SPAWNED_AT - 5000 },
+ { f: 'results.new.json', mtimeMs: SPAWNED_AT + 10 },
+ ];
+ expect(resultsStalenessFailure(entries, SPAWNED_AT)).toBeNull();
+ expect(resultsStalenessFailure([...entries].reverse(), SPAWNED_AT)).toBeNull();
+ });
+
+ test('an InconclusiveError is what the orchestrator maps onto EXIT_INCONCLUSIVE', () => {
+ expect(new InconclusiveError('x')).toBeInstanceOf(Error);
+ expect(EXIT_INCONCLUSIVE).not.toBe(EXIT_DATA);
+ expect(EXIT_INCONCLUSIVE).not.toBe(EXIT_USAGE);
+ expect([EXIT_INCONCLUSIVE, EXIT_DATA, EXIT_USAGE]).not.toContain(1);
+ });
+});
+
+describe('forcedGcPlumbingFailure', () => {
+ const captured = (gcBetweenRuns: boolean | undefined) =>
+ ({
+ ...makeFresh(),
+ methodology:
+ gcBetweenRuns === undefined
+ ? undefined
+ : { warmupIters: 10, measuredIters: 10, gcBetweenRuns },
+ }) as FreshResults;
+
+ test('NEGATIVE: a capture that recorded forced GC is not a plumbing failure', () => {
+ expect(forcedGcPlumbingFailure(captured(true))).toBeNull();
+ });
+
+ test('RED: a capture that recorded gcBetweenRuns=false names the plumbing', () => {
+ expect(forcedGcPlumbingFailure(captured(false))).toMatch(
+ /--expose-gc did not reach the worker/,
+ );
+ });
+
+ test('RED: an ABSENT methodology block is a failure, not a silent pass', () => {
+ expect(forcedGcPlumbingFailure(captured(undefined))).toMatch(/no methodology block/);
+ });
+});
+
+describe('the standalone comparator CLI surfaces methodology warnings and refuses a verdict', () => {
+ function runEntryPoint(
+ script: string,
+ extraArgs: string[] = [],
+ ): { status: number | null; stdout: string; stderr: string } {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-entrypoint-'));
+ const freshPath = join(dir, 'fresh.json');
+ writeFileSync(
+ freshPath,
+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),
+ );
+ const result = spawnSync(
+ 'pnpm',
+ ['exec', 'tsx', join(PERF_DIR, script), COMMITTED_BASELINE_PATH, freshPath, ...extraArgs],
+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },
+ );
+ if (result.error) {
+ throw new Error(
+ `spawn of ${script} failed (${result.signal ?? 'no signal'}): ${result.error.message}`,
+ );
+ }
+ return { status: result.status, stdout: result.stdout, stderr: result.stderr };
+ }
+
+ test('the standalone comparator CLI warns that the committed results are bun-era', () => {
+ const { stderr } = runEntryPoint('regression-gate.ts');
+ expect(stderr).toContain('toolchain mismatch');
+ expect(stderr).toContain('[email protected]');
+ }, 60_000);
+
+ test('RED: an unreadable baseline exits EXIT_DATA with the validator message, not a stack on 1', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-baddata-'));
+ const badPath = join(dir, 'baseline.json');
+ writeFileSync(badPath, JSON.stringify({ ...makeBaseline(), capturedUnder: {} }));
+ const freshPath = join(dir, 'fresh.json');
+ writeFileSync(
+ freshPath,
+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),
+ );
+ const result = spawnSync(
+ 'pnpm',
+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), badPath, freshPath],
+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },
+ );
+ expect(result.status).toBe(EXIT_DATA);
+ expect(result.stderr).toContain('capturedUnder must be an object declaring runtime');
+ expect(result.stderr).not.toMatch(/^\s+at /m);
+ }, 60_000);
+
+ function runComparator(baselinePath: string, freshPath: string) {
+ const result = spawnSync(
+ 'pnpm',
+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), baselinePath, freshPath],
+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },
+ );
+ if (result.error) throw result.error;
+ return result;
+ }
+
+ function badInputFixtures() {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-badinput-'));
+ const freshPath = join(dir, 'fresh.json');
+ writeFileSync(
+ freshPath,
+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),
+ );
+ const write = (name: string, body: string) => {
+ const p = join(dir, name);
+ writeFileSync(p, body);
+ return p;
+ };
+ const committed = JSON.parse(readFileSync(COMMITTED_BASELINE_PATH, 'utf8'));
+ const missingStats = structuredClone(committed);
+ delete missingStats.results[0].parseMs;
+ return {
+ freshPath,
+ nonexistent: join(dir, 'never-written.json'),
+ truncated: write('truncated.json', '{ "schemaVersion": 2, '),
+ notAnObject: write('array.json', '[1, 2, 3]'),
+ missingStats: write('nostats.json', JSON.stringify(missingStats)),
+ };
+ }
+
+ test('RED: every unreadable-input shape exits EXIT_DATA with a named message and no stack', () => {
+ const f = badInputFixtures();
+ for (const [label, path, expected] of [
+ ['a nonexistent path', f.nonexistent, /cannot read/],
+ ['truncated JSON', f.truncated, /is not valid JSON/],
+ ['a JSON array instead of an object', f.notAnObject, /must contain a JSON object/],
+ ['a results row missing an op block', f.missingStats, /is missing its parseMs stats block/],
+ ] as const) {
+ const result = runComparator(path, f.freshPath);
+ expect(result.status, `${label} should exit EXIT_DATA\n${result.stderr}`).toBe(EXIT_DATA);
+ expect(result.stderr, `${label} should name the problem`).toMatch(expected);
+ expect(result.stderr, `${label} should not dump a stack`).not.toMatch(/^\s+at /m);
+ }
+ }, 120_000);
+
+ test('the failure classifier routes unreadable input to EXIT_DATA and rethrows everything else', () => {
+ expect(exitCodeForFailure(new DataError('baseline.json: cannot read'))).toBe(EXIT_DATA);
+ for (const internal of [
+ new TypeError('an invariant inside the gate broke'),
+ new RangeError('index out of bounds'),
+ new Error('plain failure'),
+ ]) {
+ expect(
+ exitCodeForFailure(internal),
+ `${internal.constructor.name} is an internal fault: it must rethrow so its stack survives, ` +
+ 'not be reported to the operator as unreadable input',
+ ).toBeNull();
+ }
+ });
+
+ test('RED: an incomparable run exits INCONCLUSIVE rather than PASS/0', () => {
+ const { status, stdout, stderr } = runEntryPoint('regression-gate.ts');
+ expect(status).toBe(EXIT_INCONCLUSIVE);
+ expect(stderr).toContain('INCONCLUSIVE');
+ expect(stdout).toContain('perf regression gate');
+ }, 60_000);
+
+ test('NEGATIVE: the escape hatch restores the PASS verdict on the same inputs', () => {
+ const { status, stderr } = runEntryPoint('regression-gate.ts', [ACCEPT_MISMATCH_FLAG]);
+ expect(status).toBe(0);
+ expect(stderr).toContain('toolchain mismatch');
+ expect(stderr).not.toContain('INCONCLUSIVE');
+ }, 60_000);
+
+ test('NEGATIVE: a comparable run needs no escape hatch', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'perf-comparable-'));
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+ const baselinePath = join(dir, 'baseline.json');
+ const freshPath = join(dir, 'fresh.json');
+ const { targetToolchain: _dropped, ...rest } = baseline;
+ writeFileSync(
+ baselinePath,
+ JSON.stringify({
+ ...rest,
+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },
+ }),
+ );
+ writeFileSync(freshPath, JSON.stringify(freshFromBaseline(baseline)));
+ const result = spawnSync(
+ 'pnpm',
+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), baselinePath, freshPath],
+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },
+ );
+ expect(result.stderr).not.toContain('INCONCLUSIVE');
+ expect(result.status).toBe(0);
+ }, 60_000);
+});
+
+describe('p50 and p95 are recorded alongside p99', () => {
+ test('every row carries the fresh p50/p95 while the verdict stays on p99', () => {
+ const baseline = makeBaseline();
+ const fresh = makeFresh();
+ const report = evaluateRegression(baseline, fresh);
+ const row = report.rows.find((r) => r.blockCount === 1000 && r.op === 'parseMs');
+ expect(row?.freshP50).toBe(90);
+ expect(row?.freshP95).toBe(100);
+ expect(row?.regression).toBe(false);
+ });
+
+ test('baseline p50/p95 are carried when the baseline records them and omitted when it does not', () => {
+ const withPercentiles = makeBaseline({
+ results: [
+ {
+ blockCount: 100,
+ docSizeChars: 20_000,
+ parseMs: { p99: 10, p99StdevMs: 0.25, p50: 8, p95: 9.5 },
+ serializeMs: { p99: 2, p99StdevMs: 0.1 },
+ roundTripMs: { p99: 12, p99StdevMs: 0.3 },
+ },
+ ],
+ });
+ const rows = evaluateRegression(withPercentiles, makeFresh()).rows;
+ const parse = rows.find((r) => r.op === 'parseMs');
+ expect(parse?.baselineP50).toBe(8);
+ expect(parse?.baselineP95).toBe(9.5);
+ const serialize = rows.find((r) => r.op === 'serializeMs');
+ expect(serialize?.baselineP50).toBeUndefined();
+ expect(serialize?.baselineP95).toBeUndefined();
+ });
+
+ test('formatReport prints the fresh percentiles and the baseline pair when recorded', () => {
+ const withPercentiles = makeBaseline({
+ results: [
+ {
+ blockCount: 100,
+ docSizeChars: 20_000,
+ parseMs: { p99: 10, p99StdevMs: 0.25, p50: 8, p95: 9.5 },
+ serializeMs: { p99: 2, p99StdevMs: 0.1 },
+ roundTripMs: { p99: 12, p99StdevMs: 0.3 },
+ },
+ ],
+ });
+ const text = formatReport(evaluateRegression(withPercentiles, makeFresh()));
+ expect(text).toContain('p50=9.00ms p95=10.00ms');
+ expect(text).toContain('(baseline p50=8.00ms p95=9.50ms)');
+ const serializeLine = text.split('\n').find((line) => line.includes('serializeMs'));
+ expect(serializeLine).toContain('p50=1.80ms p95=2.00ms');
+ expect(serializeLine).not.toContain('baseline p50=');
+ });
+
+ test('a baseline recording a non-finite p50 is rejected at load', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'regression-gate-percentiles-'));
+ const path = join(dir, 'baseline.json');
+ const baseline = makeBaseline();
+ baseline.results[0].parseMs.p50 = Number.NaN;
+ writeFileSync(path, JSON.stringify(baseline));
+ expect(() => loadBaseline(path)).toThrow(/parseMs\.p50 is not finite/);
+ });
+});
+
+describe('forced GC is the default bench invocation', () => {
+ function probeGcInSpawnedNode(nodeArgs: string[]): string {
+ const result = spawnSync(
+ process.execPath,
+ [
+ ...nodeArgs,
+ '--input-type=module',
+ '-e',
+ `import { gcAvailable } from ${JSON.stringify(join(PERF_DIR, 'gc.ts'))}; process.stdout.write(String(gcAvailable()));`,
+ ],
+ { encoding: 'utf8', env: { ...process.env, NODE_OPTIONS: '' }, timeout: 15_000 },
+ );
+ if (result.error) {
+ throw new Error(
+ `gc probe spawn failed (${result.signal ?? 'no signal'}): ${result.error.message}`,
+ );
+ }
+ expect(result.status, result.stderr).toBe(0);
+ return result.stdout;
+ }
+
+ test('a node process without --expose-gc reports no forced GC', () => {
+ expect(probeGcInSpawnedNode([])).toBe('false');
+ });
+
+ test('a node process with --expose-gc reports forced GC', () => {
+ expect(probeGcInSpawnedNode(['--expose-gc'])).toBe('true');
+ });
+
+ test('withForcedGc adds the flag, preserves existing NODE_OPTIONS, and is idempotent', () => {
+ expect(withForcedGc({}).NODE_OPTIONS).toBe('--expose-gc');
+ expect(withForcedGc({ NODE_OPTIONS: '--max-old-space-size=4096' }).NODE_OPTIONS).toBe(
+ '--max-old-space-size=4096 --expose-gc',
+ );
+ expect(withForcedGc({ NODE_OPTIONS: '--expose-gc' }).NODE_OPTIONS).toBe('--expose-gc');
+ expect(withForcedGc({ RUN_BENCH: '1' }).RUN_BENCH).toBe('1');
+ });
+
+ test('the core test:perf:bench script forces GC', () => {
+ const pkg = JSON.parse(readFileSync(join(PERF_DIR, '..', '..', 'package.json'), 'utf8'));
+ expect(pkg.scripts['test:perf:bench']).toContain('--expose-gc');
+ expect(pkg.scripts['test:perf:bench']).toContain('RUN_BENCH=1');
+ });
+
+ test('the bench records the GC state it actually ran under', () => {
+ const bench = readFileSync(join(PERF_DIR, 'markdown-bench.test.ts'), 'utf8');
+ expect(bench).toContain('const GC_FORCED = gcAvailable();');
+ expect(bench).toContain('gcBetweenRuns: GC_FORCED,');
+ });
+
+ test('the regression orchestrator spawns the bench with forced GC', () => {
+ const orchestrator = readFileSync(join(PERF_DIR, 'run-regression-gate.ts'), 'utf8');
+ expect(orchestrator).toContain("env: withForcedGc({ ...process.env, RUN_BENCH: '1' })");
+ });
+
+ test('a fresh run without forced GC is flagged against the committed baseline', () => {
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+ const fresh = freshFromBaseline(
+ baseline,
+ {},
+ {
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: false },
+ },
+ );
+ const warnings = checkMethodologyMismatches(baseline, fresh);
+ const gcWarnings = warnings.filter((w) => w.kind === 'gc-regime');
+ expect(gcWarnings).toHaveLength(1);
+ expect(gcWarnings[0].message).toContain('fresh gcBetweenRuns=false');
+ expect(gcWarnings[0].message).toContain('--expose-gc');
+ expect(comparabilityBlockers(warnings).length).toBeGreaterThan(0);
+ expect(comparabilityBlockers(warnings)).toContain(gcWarnings[0]);
+ });
+
+ test('an iteration-count mismatch warns but is deliberately NOT a comparability blocker', () => {
+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);
+ const fresh = freshFromBaseline(
+ baseline,
+ {},
+ {
+ methodology: { warmupIters: 10, measuredIters: 40, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: 'bun test' },
+ },
+ );
+ const warnings = checkMethodologyMismatches(baseline, fresh);
+ expect(warnings.filter((w) => w.kind === 'iteration-count')).toHaveLength(1);
+ expect(comparabilityBlockers(warnings)).toEqual([]);
+ });
+});
+
+function opStats(
+ mean: number,
+ min: number,
+ max: number,
+ p50: number,
+ p95: number,
+ p99: number,
+): { mean: number; min: number; max: number; p50: number; p95: number; p99: number } {
+ return { mean, min, max, p50, p95, p99 };
+}
+
+const CALIBRATION_CAPTURE_1: FreshResults = {
+ schemaVersion: 2,
+ startedAt: '2026-09-04T12:20:15.610Z',
+ finishedAt: '2026-09-04T12:24:39.000Z',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ runner: { runnerClass: 'local-m-series-calibration' },
+ results: [
+ {
+ blockCount: 100,
+ docSizeChars: 20293,
+ parseMs: opStats(22.78, 21.76, 23.34, 23.01, 23.34, 23.34),
+ serializeMs: opStats(3.82, 3.24, 4.86, 3.81, 4.86, 4.86),
+ roundTripMs: opStats(26.0, 25.03, 27.08, 26.02, 27.08, 27.08),
+ },
+ {
+ blockCount: 1000,
+ docSizeChars: 194243,
+ parseMs: opStats(173.46, 168.62, 179.25, 174.62, 179.25, 179.25),
+ serializeMs: opStats(28.55, 26.38, 30.84, 28.59, 30.84, 30.84),
+ roundTripMs: opStats(198.17, 190.58, 206.49, 198.6, 206.49, 206.49),
+ },
+ {
+ blockCount: 5000,
+ docSizeChars: 952068,
+ parseMs: opStats(858.23, 838.26, 882.04, 860.69, 882.04, 882.04),
+ serializeMs: opStats(116.83, 109.4, 125.21, 115.74, 125.21, 125.21),
+ roundTripMs: opStats(940.96, 909.94, 968.7, 938.71, 968.7, 968.7),
+ },
+ {
+ blockCount: 10000,
+ docSizeChars: 1902300,
+ parseMs: opStats(1770.17, 1752.23, 1817.97, 1765.8, 1817.97, 1817.97),
+ serializeMs: opStats(212.92, 206.64, 223.86, 210.3, 223.86, 223.86),
+ roundTripMs: opStats(2000.19, 1973.64, 2023.4, 2003.95, 2023.4, 2023.4),
+ },
+ {
+ blockCount: 20000,
+ docSizeChars: 3837177,
+ parseMs: opStats(4625.82, 4565.56, 4694.37, 4620.68, 4694.37, 4694.37),
+ serializeMs: opStats(424.5, 411.92, 437.56, 420.32, 437.56, 437.56),
+ roundTripMs: opStats(5223.79, 5095.89, 5302.72, 5240.68, 5302.72, 5302.72),
+ },
+ ],
+};
+
+const CALIBRATION_CAPTURE_2: FreshResults = {
+ schemaVersion: 2,
+ startedAt: '2026-09-04T12:24:49.393Z',
+ finishedAt: '2026-09-04T12:29:25.000Z',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ runner: { runnerClass: 'local-m-series-calibration' },
+ results: [
+ {
+ blockCount: 100,
+ docSizeChars: 20293,
+ parseMs: opStats(22.68, 20.75, 24.44, 22.69, 24.44, 24.44),
+ serializeMs: opStats(3.58, 3.0, 4.46, 3.55, 4.46, 4.46),
+ roundTripMs: opStats(25.3, 23.82, 26.2, 25.83, 26.2, 26.2),
+ },
+ {
+ blockCount: 1000,
+ docSizeChars: 194243,
+ parseMs: opStats(164.29, 157.63, 171.97, 164.2, 171.97, 171.97),
+ serializeMs: opStats(26.52, 24.16, 28.38, 26.99, 28.38, 28.38),
+ roundTripMs: opStats(191.85, 184.05, 199.62, 191.75, 199.62, 199.62),
+ },
+ {
+ blockCount: 5000,
+ docSizeChars: 952068,
+ parseMs: opStats(838.32, 817.42, 899.96, 835.3, 899.96, 899.96),
+ serializeMs: opStats(115.95, 110.76, 122.85, 114.27, 122.85, 122.85),
+ roundTripMs: opStats(922.31, 884.99, 974.84, 927.15, 974.84, 974.84),
+ },
+ {
+ blockCount: 10000,
+ docSizeChars: 1902300,
+ parseMs: opStats(1739.65, 1715.93, 1771.42, 1749.17, 1771.42, 1771.42),
+ serializeMs: opStats(207.65, 200.59, 216.24, 206.62, 216.24, 216.24),
+ roundTripMs: opStats(1975.12, 1937.2, 2152.98, 1960.42, 2152.98, 2152.98),
+ },
+ {
+ blockCount: 20000,
+ docSizeChars: 3837177,
+ parseMs: opStats(5321.17, 5116.1, 5470.61, 5342.77, 5470.61, 5470.61),
+ serializeMs: opStats(467.29, 452.19, 505.48, 462.36, 505.48, 505.48),
+ roundTripMs: opStats(5589.69, 5458.21, 5788.72, 5585.02, 5788.72, 5788.72),
+ },
+ ],
+};
+
+const CONTENDED_FOUR_RUN_AGGREGATE: Baseline = {
+ schemaVersion: 2,
+ capturedAt: '2026-09-04T12:20:15.610Z',
+ runnerClass: 'local-m-series-calibration',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },
+ calibrationRuns: 4,
+ threshold: { floorPct: 0.1, varianceMultiplier: 2 },
+ results: [
+ {
+ blockCount: 100,
+ docSizeChars: 20293,
+ parseMs: { p99: 37.45, p99StdevMs: 21.3 },
+ serializeMs: { p99: 7.07, p99StdevMs: 3.99 },
+ roundTripMs: { p99: 48.36, p99StdevMs: 36.61 },
+ },
+ {
+ blockCount: 1000,
+ docSizeChars: 194243,
+ parseMs: { p99: 398.44, p99StdevMs: 280.31 },
+ serializeMs: { p99: 47.92, p99StdevMs: 23.11 },
+ roundTripMs: { p99: 365.25, p99StdevMs: 187.32 },
+ },
+ {
+ blockCount: 5000,
+ docSizeChars: 952068,
+ parseMs: { p99: 1338.45, p99StdevMs: 581.95 },
+ serializeMs: { p99: 217.89, p99StdevMs: 127.8 },
+ roundTripMs: { p99: 1271.04, p99StdevMs: 356.18 },
+ },
+ {
+ blockCount: 10000,
+ docSizeChars: 1902300,
+ parseMs: { p99: 3572.92, p99StdevMs: 2466.94 },
+ serializeMs: { p99: 258.39, p99StdevMs: 71.11 },
+ roundTripMs: { p99: 2220.6, p99StdevMs: 207.23 },
+ },
+ {
+ blockCount: 20000,
+ docSizeChars: 3837177,
+ parseMs: { p99: 5167.96, p99StdevMs: 463.46 },
+ serializeMs: { p99: 468.86, p99StdevMs: 33.64 },
+ roundTripMs: { p99: 10325.58, p99StdevMs: 6599.79 },
+ },
+ ],
+};
+
+function sampleStdev(a: number, b: number): number {
+ const mean = (a + b) / 2;
+ return Math.sqrt((a - mean) ** 2 + (b - mean) ** 2);
+}
+
+function baselineFromCapturePair(first: FreshResults, second: FreshResults): Baseline {
+ return {
+ schemaVersion: 2,
+ capturedAt: first.startedAt,
+ runnerClass: 'local-m-series-calibration',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },
+ calibrationRuns: 2,
+ threshold: { floorPct: 0.1, varianceMultiplier: 2 },
+ results: first.results.map((row) => {
+ const other = second.results.find((r) => r.blockCount === row.blockCount);
+ if (!other) throw new Error(`capture pair disagrees on block count ${row.blockCount}`);
+ const op = (name: 'parseMs' | 'serializeMs' | 'roundTripMs') => ({
+ p99: row[name].p99,
+ p99StdevMs: sampleStdev(row[name].p99, other[name].p99),
+ p50: row[name].p50,
+ p95: row[name].p95,
+ });
+ return {
+ blockCount: row.blockCount,
+ docSizeChars: row.docSizeChars,
+ parseMs: op('parseMs'),
+ serializeMs: op('serializeMs'),
+ roundTripMs: op('roundTripMs'),
+ };
+ }),
+ };
+}
+
+function scaleCapture(capture: FreshResults, scale: number): FreshResults {
+ return {
+ ...capture,
+ results: capture.results.map((row) => ({
+ ...row,
+ parseMs: { ...row.parseMs, p99: row.parseMs.p99 * scale },
+ serializeMs: { ...row.serializeMs, p99: row.serializeMs.p99 * scale },
+ roundTripMs: { ...row.roundTripMs, p99: row.roundTripMs.p99 * scale },
+ })),
+ };
+}
+
+function uniformlyScaledFresh(baseline: Baseline, scale: number): FreshResults {
+ return {
+ schemaVersion: 2,
+ startedAt: '2026-09-04T13:00:00.000Z',
+ finishedAt: '2026-09-04T13:05:00.000Z',
+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },
+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },
+ runner: {},
+ results: baseline.results.map((row) => {
+ const op = (name: 'parseMs' | 'serializeMs' | 'roundTripMs') => {
+ const p99 = row[name].p99 * scale;
+ return { mean: p99, min: p99, max: p99, p50: p99, p95: p99, p99 };
+ };
+ return {
+ blockCount: row.blockCount,
+ docSizeChars: row.docSizeChars,
+ parseMs: op('parseMs'),
+ serializeMs: op('serializeMs'),
+ roundTripMs: op('roundTripMs'),
+ };
+ }),
+ };
+}
+
+function regressedRowCount(baseline: Baseline, fresh: FreshResults): number {
+ return evaluateRegression(baseline, fresh).rows.filter((r) => r.regression).length;
+}
+
+describe('node-era calibration captures (R7 re-capture)', () => {
+ const quietPairGate = baselineFromCapturePair(CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2);
+
+ test('a gate built from calibration capture 1 does not fail on capture 2', () => {
+ const report = evaluateRegression(quietPairGate, CALIBRATION_CAPTURE_2);
+ expect(report.pass).toBe(true);
+ expect(report.missingFresh).toEqual([]);
+ expect(report.rows).toHaveLength(15);
+ expect(report.rows.filter((r) => r.regression)).toEqual([]);
+ });
+
+ test('that same gate reds 7 of its 15 rows when capture 2 is raised 11 percent', () => {
+ const report = evaluateRegression(quietPairGate, scaleCapture(CALIBRATION_CAPTURE_2, 1.11));
+ expect(report.pass).toBe(false);
+ expect(report.rows.filter((r) => r.regression)).toHaveLength(7);
+ });
+
+ test('that same gate reds all 15 rows when capture 2 is raised 30 percent', () => {
+ expect(regressedRowCount(quietPairGate, scaleCapture(CALIBRATION_CAPTURE_2, 1.3))).toBe(15);
+ });
+
+ test('the captures record forced GC and the node toolchain', () => {
+ for (const capture of [CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2]) {
+ expect(capture.methodology?.gcBetweenRuns).toBe(true);
+ expect(capture.toolchain).toEqual({ runtime: '[email protected]', testRunner: '[email protected]' });
+ }
+ });
+
+ test('every capture row reports p95 equal to p99, so p99 at 10 iterations is the maximum', () => {
+ for (const capture of [CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2]) {
+ for (const row of capture.results) {
+ for (const op of ['parseMs', 'serializeMs', 'roundTripMs'] as const) {
+ expect(row[op].p95).toBe(row[op].p99);
+ expect(row[op].p99).toBe(row[op].max);
+ }
+ }
+ }
+ });
+
+ test('aggregating the contended four-run calibration produces a gate that cannot fire', () => {
+ expect(
+ regressedRowCount(
+ CONTENDED_FOUR_RUN_AGGREGATE,
+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 1.11),
+ ),
+ ).toBe(0);
+ expect(
+ regressedRowCount(
+ CONTENDED_FOUR_RUN_AGGREGATE,
+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 2),
+ ),
+ ).toBe(7);
+ expect(
+ regressedRowCount(
+ CONTENDED_FOUR_RUN_AGGREGATE,
+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 3),
+ ),
+ ).toBe(15);
+ });
+
+ test('the contended aggregate carries a variance band wider than the ten percent floor everywhere', () => {
+ const bands = CONTENDED_FOUR_RUN_AGGREGATE.results.flatMap((row) =>
+ (['parseMs', 'serializeMs', 'roundTripMs'] as const).map(
+ (op) => (2 * row[op].p99StdevMs) / row[op].p99,
+ ),
+ );
+ expect(bands).toHaveLength(15);
+ expect(Math.min(...bands)).toBeGreaterThan(0.1);
+ expect(Math.max(...bands)).toBeGreaterThan(1.5);
+ });
+
+ test('the committed baseline results are not replaced by this calibration', () => {
+ const committed = loadBaseline(COMMITTED_BASELINE_PATH);
+ const smallestParse = committed.results.find((r) => r.blockCount === 100)?.parseMs;
+ expect(smallestParse).toEqual({ p99: 9.69, p99StdevMs: 0.89 });
+ expect(committed.calibrationRuns).toBe(4);
+ expect(committed.capturedAt).toBe('2026-04-16T08:28:01.616Z');
+ });
+});
diff --git a/packages/core/tests/perf/regression-gate.ts b/packages/core/tests/perf/regression-gate.ts
index 3d103eee0..8d2dc5e44 100644
--- a/packages/core/tests/perf/regression-gate.ts
+++ b/packages/core/tests/perf/regression-gate.ts
@@ -16,6 +16,11 @@ interface OpStats {
p99: number;
}
+export interface Toolchain {
+ runtime?: string;
+ testRunner?: string;
+}
+
interface FreshBlockResult {
blockCount: number;
docSizeChars: number;
@@ -25,31 +30,37 @@ interface FreshBlockResult {
}
export interface FreshResults {
- schemaVersion: number;
+ schemaVersion: 2;
startedAt: string;
finishedAt: string;
- methodology?: Methodology;
+ methodology: Methodology;
+ toolchain: Toolchain;
runner: Record<string, unknown>;
results: FreshBlockResult[];
}
+interface BaselineOpStats {
+ p99: number;
+ p99StdevMs: number;
+ p50?: number;
+ p95?: number;
+}
+
interface BaselineBlockEntry {
blockCount: number;
docSizeChars: number;
- parseMs: { p99: number; p99StdevMs: number };
- serializeMs: { p99: number; p99StdevMs: number };
- roundTripMs: { p99: number; p99StdevMs: number };
+ parseMs: BaselineOpStats;
+ serializeMs: BaselineOpStats;
+ roundTripMs: BaselineOpStats;
}
export interface Baseline {
- schemaVersion: 1;
+ schemaVersion: 2;
capturedAt: string;
runnerClass: string;
methodology?: Methodology;
- toolchain?: {
- runtime?: string;
- testRunner?: string;
- };
+ capturedUnder: Toolchain;
+ targetToolchain?: Toolchain;
calibrationRuns: number;
threshold: {
floorPct: number;
@@ -68,6 +79,10 @@ interface OpRegressionRow {
deltaMs: number;
allowedDeltaMs: number;
regression: boolean;
+ baselineP50?: number;
+ baselineP95?: number;
+ freshP50: number;
+ freshP95: number;
}
interface RegressionReport {
@@ -108,6 +123,10 @@ export function evaluateRegression(baseline: Baseline, fresh: FreshResults): Reg
deltaMs,
allowedDeltaMs,
regression: deltaMs > allowedDeltaMs,
+ baselineP50: b[op].p50,
+ baselineP95: b[op].p95,
+ freshP50: f[op].p50,
+ freshP95: f[op].p95,
});
}
}
@@ -121,6 +140,10 @@ export function evaluateRegression(baseline: Baseline, fresh: FreshResults): Reg
return { pass, rows, missingFresh, extraFresh };
}
+function fmtOptionalMs(value: number | undefined): string {
+ return value === undefined ? 'n/a' : `${value.toFixed(2)}ms`;
+}
+
export function formatReport(report: RegressionReport): string {
const lines: string[] = [];
lines.push(`perf regression gate: ${report.pass ? 'PASS' : 'FAIL'}`);
@@ -133,54 +156,156 @@ export function formatReport(report: RegressionReport): string {
for (const row of report.rows) {
const marker = row.regression ? '✗' : '✓';
const deltaSign = row.deltaMs >= 0 ? '+' : '';
+ const baselineTail =
+ row.baselineP50 === undefined && row.baselineP95 === undefined
+ ? ''
+ : ` (baseline p50=${fmtOptionalMs(row.baselineP50)} p95=${fmtOptionalMs(row.baselineP95)})`;
lines.push(
` ${marker} ${String(row.blockCount).padStart(5)} ${row.op.padEnd(12)}` +
` baseline=${row.baselineP99.toFixed(2)}ms` +
` fresh=${row.freshP99.toFixed(2)}ms` +
` Δ=${deltaSign}${row.deltaMs.toFixed(2)}ms` +
- ` allowed=${row.allowedDeltaMs.toFixed(2)}ms`,
+ ` allowed=${row.allowedDeltaMs.toFixed(2)}ms` +
+ ` p50=${row.freshP50.toFixed(2)}ms p95=${row.freshP95.toFixed(2)}ms` +
+ baselineTail,
);
}
return lines.join('\n');
}
-export function checkMethodologyMismatches(baseline: Baseline, fresh: FreshResults): string[] {
- const warnings: string[] = [];
+export function baselineResultsToolchain(baseline: Baseline): Toolchain {
+ return baseline.capturedUnder;
+}
+
+function describeToolchain(toolchain: Toolchain): string {
+ return `runtime=${toolchain.runtime ?? 'unknown'} testRunner=${toolchain.testRunner ?? 'unknown'}`;
+}
+
+export const EXIT_INCONCLUSIVE = 3;
+
+export const EXIT_USAGE = 64;
+
+export const EXIT_DATA = 65;
+
+export const EXIT_SOFTWARE = 70;
+
+export class DataError extends Error {}
+
+export const ACCEPT_MISMATCH_FLAG = '--accept-methodology-mismatch';
+
+export interface MethodologyWarning {
+ kind: 'toolchain' | 'gc-regime' | 'iteration-count';
+ blocking: boolean;
+ message: string;
+}
+
+export function comparabilityBlockers(warnings: MethodologyWarning[]): MethodologyWarning[] {
+ return warnings.filter((w) => w.blocking);
+}
+
+export function checkMethodologyMismatches(
+ baseline: Baseline,
+ fresh: FreshResults,
+): MethodologyWarning[] {
+ const warnings: MethodologyWarning[] = [];
+ const baselineToolchain = baselineResultsToolchain(baseline);
+ const freshToolchain = fresh.toolchain;
+ if (
+ baselineToolchain.runtime !== freshToolchain.runtime ||
+ baselineToolchain.testRunner !== freshToolchain.testRunner
+ ) {
+ warnings.push({
+ kind: 'toolchain',
+ blocking: true,
+ message:
+ `toolchain mismatch: the committed baseline results were captured under ` +
+ `${describeToolchain(baselineToolchain)}, this run used ` +
+ `${describeToolchain(freshToolchain)}. A different engine or test runner ` +
+ `changes the allocation and JIT regime, so the p99 deltas below are not ` +
+ `attributable to code alone; re-baseline before trusting them.`,
+ });
+ }
const b = baseline.methodology;
const f = fresh.methodology;
- if (!b || !f) return warnings;
+ if (!b) return warnings;
if (b.gcBetweenRuns !== f.gcBetweenRuns) {
- warnings.push(
- `methodology mismatch: baseline gcBetweenRuns=${b.gcBetweenRuns} ` +
+ warnings.push({
+ kind: 'gc-regime',
+ blocking: true,
+ message:
+ `methodology mismatch: baseline gcBetweenRuns=${b.gcBetweenRuns} ` +
`fresh gcBetweenRuns=${f.gcBetweenRuns}. Re-run with ` +
`NODE_OPTIONS=--expose-gc to force GC, or re-baseline; the numbers ` +
`below compare different allocation regimes.`,
- );
+ });
}
if (b.measuredIters !== f.measuredIters) {
- warnings.push(
- `iteration-count mismatch: baseline measuredIters=${b.measuredIters} ` +
+ warnings.push({
+ kind: 'iteration-count',
+ blocking: false,
+ message:
+ `iteration-count mismatch: baseline measuredIters=${b.measuredIters} ` +
`fresh measuredIters=${f.measuredIters}; p99 is a worst-of-N observation, ` +
`so changing N changes what the number means.`,
- );
+ });
}
return warnings;
}
+function readJson(file: string, path: string): Record<string, unknown> {
+ let text: string;
+ try {
+ text = readFileSync(path, 'utf8');
+ } catch (error) {
+ throw new DataError(`${file}: cannot read ${path} (${(error as Error).message})`);
+ }
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(text);
+ } catch (error) {
+ throw new DataError(`${file}: ${path} is not valid JSON (${(error as Error).message})`);
+ }
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+ throw new DataError(
+ `${file}: ${path} must contain a JSON object (got ${JSON.stringify(parsed)})`,
+ );
+ }
+ return parsed as Record<string, unknown>;
+}
+
+function assertStatsPresent(ctx: string, blockCount: number, opName: OpName, stats: unknown): void {
+ if (typeof stats !== 'object' || stats === null || Array.isArray(stats)) {
+ throw new DataError(
+ `${ctx}: blockCount=${blockCount} is missing its ${opName} stats block (got ${JSON.stringify(stats)})`,
+ );
+ }
+}
+
function assertFiniteStats(
ctx: string,
blockCount: number,
opName: OpName,
- stats: { p99: number; p99StdevMs: number },
+ stats: BaselineOpStats,
): void {
+ assertStatsPresent(ctx, blockCount, opName, stats);
if (!Number.isFinite(stats.p99)) {
- throw new Error(`${ctx}: blockCount=${blockCount} ${opName}.p99 is not finite (${stats.p99})`);
+ throw new DataError(
+ `${ctx}: blockCount=${blockCount} ${opName}.p99 is not finite (${stats.p99})`,
+ );
}
if (!Number.isFinite(stats.p99StdevMs)) {
- throw new Error(
+ throw new DataError(
`${ctx}: blockCount=${blockCount} ${opName}.p99StdevMs is not finite (${stats.p99StdevMs})`,
);
}
+ for (const key of ['p50', 'p95'] as const) {
+ const value = stats[key];
+ if (value !== undefined && !Number.isFinite(value)) {
+ throw new DataError(
+ `${ctx}: blockCount=${blockCount} ${opName}.${key} is not finite (${value})`,
+ );
+ }
+ }
}
function assertFiniteOpStats(
@@ -189,60 +314,213 @@ function assertFiniteOpStats(
opName: OpName,
stats: OpStats,
): void {
+ assertStatsPresent(ctx, blockCount, opName, stats);
for (const key of ['mean', 'min', 'max', 'p50', 'p95', 'p99'] as const) {
if (!Number.isFinite(stats[key])) {
- throw new Error(
+ throw new DataError(
`${ctx}: blockCount=${blockCount} ${opName}.${key} is not finite (${stats[key]})`,
);
}
}
}
+function assertFiniteEntryKeys(ctx: string, index: number, entry: unknown): void {
+ if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
+ throw new DataError(
+ `${ctx}: results[${index}] must be an object (got ${JSON.stringify(entry)})`,
+ );
+ }
+ for (const key of ['blockCount', 'docSizeChars'] as const) {
+ const value = (entry as Record<string, unknown>)[key];
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
+ throw new DataError(
+ `${ctx}: results[${index}].${key} must be a finite number (got ${JSON.stringify(value)}). ` +
+ (key === 'blockCount'
+ ? 'blockCount is the key baseline rows are joined to fresh rows on, so a non-number ' +
+ 'matches nothing, renders every baseline row as missingFresh, and prints FAIL as ' +
+ 'though the run regressed.'
+ : 'docSizeChars is provenance the comparator never reads, so a non-number changes no ' +
+ 'verdict; it means the capture that wrote this row is corrupt.'),
+ );
+ }
+ }
+}
+
+function declaresToolchain(value: unknown): boolean {
+ return (
+ typeof value === 'object' &&
+ value !== null &&
+ !Array.isArray(value) &&
+ (['runtime', 'testRunner'] as const).some((key) => {
+ const declared = (value as Record<string, unknown>)[key];
+ return typeof declared === 'string' && declared.length > 0;
+ })
+ );
+}
+
+function assertToolchain(file: string, field: string, value: unknown, why: string): void {
+ if (!declaresToolchain(value)) {
+ throw new DataError(
+ `${file}: ${field} must be an object declaring runtime and/or testRunner (got ${JSON.stringify(value)}). ${why}`,
+ );
+ }
+}
+
+function assertThreshold(value: unknown): void {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new DataError(
+ `baseline.json: threshold must be an object (got ${JSON.stringify(value)})`,
+ );
+ }
+ for (const field of ['floorPct', 'varianceMultiplier'] as const) {
+ const n = (value as Record<string, unknown>)[field];
+ if (typeof n !== 'number' || !Number.isFinite(n)) {
+ throw new DataError(
+ `baseline.json: threshold.${field} must be a finite number (got ${JSON.stringify(n)}). ` +
+ 'A missing term makes allowedDeltaMs NaN, and every `delta > NaN` comparison is false, so ' +
+ 'the gate prints PASS against an arbitrarily slower run.',
+ );
+ }
+ }
+}
+
export function loadBaseline(path: string): Baseline {
- const raw = JSON.parse(readFileSync(path, 'utf8'));
- if (raw.schemaVersion !== 1) {
- throw new Error(`baseline.json schemaVersion must be 1 (got ${raw.schemaVersion})`);
+ const raw = readJson('baseline.json', path);
+ if (raw.schemaVersion !== 2) {
+ throw new DataError(
+ `baseline.json schemaVersion must be 2 (got ${raw.schemaVersion}). Version 1 spelled the ` +
+ 'provenance of results[] as `toolchain`; version 2 splits it into `capturedUnder` (what ' +
+ 'the numbers were measured under, the only field the comparator reads) and ' +
+ '`targetToolchain` (what a re-capture should target).',
+ );
}
if (!Array.isArray(raw.results)) {
- throw new Error('baseline.json: results must be an array');
+ throw new DataError('baseline.json: results must be an array');
}
- for (const entry of raw.results as BaselineBlockEntry[]) {
+ assertToolchain(
+ 'baseline.json',
+ 'capturedUnder',
+ raw.capturedUnder,
+ 'It is the only provenance field the comparator reads, and a missing or falsy value ' +
+ 'short-circuits the toolchain-mismatch arm rather than failing loudly, which returns the ' +
+ 'gate to rendering PASS on an incomparable pair.' +
+ (raw.toolchain !== undefined
+ ? ' This file still carries the version-1 `toolchain` key; rename it to capturedUnder rather than renumbering.'
+ : ''),
+ );
+ if (raw.targetToolchain !== undefined) {
+ assertToolchain(
+ 'baseline.json',
+ 'targetToolchain',
+ raw.targetToolchain,
+ 'Omit the key entirely rather than declaring an empty one.',
+ );
+ }
+ assertThreshold(raw.threshold);
+ if (
+ raw.targetToolchain !== undefined &&
+ JSON.stringify(raw.capturedUnder) === JSON.stringify(raw.targetToolchain)
+ ) {
+ throw new DataError(
+ 'baseline.json: capturedUnder equals targetToolchain, which is the state a completed ' +
+ 're-baseline removes. Drop targetToolchain once the numbers are captured under it.',
+ );
+ }
+ for (const [index, entry] of (raw.results as BaselineBlockEntry[]).entries()) {
+ assertFiniteEntryKeys('baseline', index, entry);
for (const op of OP_NAMES) {
assertFiniteStats('baseline', entry.blockCount, op, entry[op]);
}
}
- return raw as Baseline;
+ return raw as unknown as Baseline;
}
export function loadFreshResults(path: string): FreshResults {
- const raw = JSON.parse(readFileSync(path, 'utf8'));
- if (raw.schemaVersion !== 1) {
- throw new Error(`results.json schemaVersion must be 1 (got ${raw.schemaVersion})`);
+ const raw = readJson('results.json', path);
+ if (raw.schemaVersion !== 2) {
+ throw new DataError(
+ `results.json schemaVersion must be 2 (got ${raw.schemaVersion}). ` +
+ (declaresToolchain(raw.toolchain)
+ ? 'This file already carries a version-2-shaped `toolchain`, so renumbering it to 2 ' +
+ 'preserves that provenance rather than discarding it. Re-run the bench only if you ' +
+ 'would rather discard the capture.'
+ : 'It declares no `toolchain`, so it carries no provenance for the toolchain-mismatch ' +
+ 'arm to read. Re-run the bench to produce a version-2 capture rather than renumbering ' +
+ 'the file.'),
+ );
}
if (!Array.isArray(raw.results)) {
- throw new Error('results.json: results must be an array');
+ throw new DataError('results.json: results must be an array');
}
- for (const entry of raw.results as FreshBlockResult[]) {
+ assertToolchain(
+ 'results.json',
+ 'toolchain',
+ raw.toolchain,
+ 'Without it the toolchain-mismatch arm cannot fire, so a capture from a different runtime ' +
+ 'compares silently against the baseline.',
+ );
+ if (
+ typeof raw.methodology !== 'object' ||
+ raw.methodology === null ||
+ Array.isArray(raw.methodology)
+ ) {
+ throw new DataError(
+ `results.json: methodology must be an object recording what this run measured under (got ${JSON.stringify(raw.methodology)}). ` +
+ 'Without it the gcBetweenRuns and measuredIters arms cannot fire.',
+ );
+ }
+ for (const [index, entry] of (raw.results as FreshBlockResult[]).entries()) {
+ assertFiniteEntryKeys('results', index, entry);
for (const op of OP_NAMES) {
assertFiniteOpStats('results', entry.blockCount, op, entry[op]);
}
}
- return raw as FreshResults;
+ return raw as unknown as FreshResults;
}
async function main(): Promise<void> {
- const [, , baselineArg, freshArg] = process.argv;
+ const argv = process.argv.slice(2);
+ const accepted = argv.includes(ACCEPT_MISMATCH_FLAG);
+ const [baselineArg, freshArg] = argv.filter((arg) => !arg.startsWith('--'));
if (!baselineArg || !freshArg) {
- console.error('usage: regression-gate.ts <baseline.json> <fresh-results.json>');
- process.exit(2);
+ console.error(
+ `usage: regression-gate.ts <baseline.json> <fresh-results.json> [${ACCEPT_MISMATCH_FLAG}]\n` +
+ ` exit 0 no regression, 1 regression, ${EXIT_INCONCLUSIVE} the comparison is not valid, ` +
+ `${EXIT_USAGE} bad invocation, ${EXIT_DATA} unreadable input`,
+ );
+ process.exit(EXIT_USAGE);
}
const baseline = loadBaseline(resolve(baselineArg));
const fresh = loadFreshResults(resolve(freshArg));
+ const warnings = checkMethodologyMismatches(baseline, fresh);
+ for (const warning of warnings) {
+ console.warn(warning.message);
+ }
const report = evaluateRegression(baseline, fresh);
console.log(formatReport(report));
+ const blockers = comparabilityBlockers(warnings);
+ if (blockers.length > 0 && !accepted) {
+ console.error(
+ `perf regression gate: INCONCLUSIVE - ${blockers.length} warning(s) above say this run and ` +
+ 'the baseline are not comparable, so neither PASS nor FAIL is a verdict this data ' +
+ `supports. Re-baseline, or pass ${ACCEPT_MISMATCH_FLAG} to read the deltas anyway.`,
+ );
+ process.exit(EXIT_INCONCLUSIVE);
+ }
process.exit(report.pass ? 0 : 1);
}
+export function exitCodeForFailure(error: unknown): number | null {
+ return error instanceof DataError ? EXIT_DATA : null;
+}
+
if (import.meta.main) {
- await main();
+ try {
+ await main();
+ } catch (error) {
+ const code = exitCodeForFailure(error);
+ if (code === null) throw error;
+ console.error((error as Error).message);
+ process.exit(code);
+ }
}
diff --git a/packages/core/tests/perf/run-regression-gate.ts b/packages/core/tests/perf/run-regression-gate.ts
index 17f0ec76a..6bf5e7806 100644
--- a/packages/core/tests/perf/run-regression-gate.ts
+++ b/packages/core/tests/perf/run-regression-gate.ts
@@ -2,8 +2,16 @@ import { spawnSync } from 'node:child_process';
import { readdirSync, statSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
+import { withForcedGc } from './gc.ts';
+import type { FreshResults } from './regression-gate.ts';
import {
+ ACCEPT_MISMATCH_FLAG,
checkMethodologyMismatches,
+ comparabilityBlockers,
+ DataError,
+ EXIT_DATA,
+ EXIT_INCONCLUSIVE,
+ EXIT_SOFTWARE,
evaluateRegression,
formatReport,
loadBaseline,
@@ -13,32 +21,67 @@ import {
const HERE = dirname(fileURLToPath(import.meta.url));
const BASELINE_PATH = resolve(HERE, 'baseline.json');
-function findFreshestResults(): string {
- const entries = readdirSync(HERE)
- .filter((f) => f.startsWith('results.') && f.endsWith('.json'))
- .map((f) => ({ f, mtime: statSync(resolve(HERE, f)).mtimeMs }))
- .sort((a, b) => b.mtime - a.mtime);
+export function forcedGcPlumbingFailure(fresh: FreshResults): string | null {
+ if (fresh.methodology?.gcBetweenRuns === true) return null;
+ const reported =
+ fresh.methodology === undefined
+ ? 'no methodology block'
+ : `gcBetweenRuns=${fresh.methodology.gcBetweenRuns}`;
+ return (
+ `the bench was spawned through withForcedGc but the capture reports ${reported}: either ` +
+ '--expose-gc did not reach the worker or the run did not record what it ran under, so this ' +
+ 'capture was taken under a different allocation regime than the one it claims. Fix the ' +
+ 'plumbing before reading the deltas.'
+ );
+}
+
+export class InconclusiveError extends Error {}
+
+export interface ResultsEntry {
+ f: string;
+ mtimeMs: number;
+}
+
+export function resultsStalenessFailure(entries: ResultsEntry[], notBefore: number): string | null {
if (entries.length === 0) {
- throw new Error(`no results.*.json found in ${HERE}; did the bench run fail?`);
+ return `no results.*.json found in ${HERE}; the bench produced no capture, so there is nothing to grade.`;
}
- return resolve(HERE, entries[0].f);
+ const freshest = [...entries].sort((a, b) => b.mtimeMs - a.mtimeMs)[0];
+ if (freshest.mtimeMs >= notBefore) return null;
+ return (
+ `the freshest results file (${freshest.f}) predates the bench spawn, so the bench wrote ` +
+ 'nothing and this run would have graded a leftover capture. markdown-bench.test.ts gates its ' +
+ 'whole describe on RUN_BENCH, so a propagation failure skips every test and still exits 0 - ' +
+ 'that is what this check catches.'
+ );
+}
+
+function findFreshestResults(notBefore: number): string {
+ const entries = readdirSync(HERE)
+ .filter((f) => f.startsWith('results.') && f.endsWith('.json'))
+ .map((f) => ({ f, mtimeMs: statSync(resolve(HERE, f)).mtimeMs }));
+ const failure = resultsStalenessFailure(entries, notBefore);
+ if (failure !== null) throw new InconclusiveError(failure);
+ return resolve(HERE, [...entries].sort((a, b) => b.mtimeMs - a.mtimeMs)[0].f);
}
async function main(): Promise<void> {
+ const accepted = process.argv.includes(ACCEPT_MISMATCH_FLAG);
+ const spawnedAt = Date.now();
const bench = spawnSync(
'pnpm',
['exec', 'vitest', 'run', resolve(HERE, 'markdown-bench.test.ts')],
{
- env: { ...process.env, RUN_BENCH: '1' },
+ env: withForcedGc({ ...process.env, RUN_BENCH: '1' }),
stdio: 'inherit',
},
);
if (bench.status !== 0) {
console.error(`bench run failed with exit code ${bench.status ?? 'null'}`);
- process.exit(bench.status ?? 1);
+ process.exit(EXIT_SOFTWARE);
}
- const freshPath = findFreshestResults();
+ const freshPath = findFreshestResults(spawnedAt);
const baseline = loadBaseline(BASELINE_PATH);
const fresh = loadFreshResults(freshPath);
@@ -53,17 +96,45 @@ async function main(): Promise<void> {
);
}
- for (const warning of checkMethodologyMismatches(baseline, fresh)) {
- console.warn(`[r4-gate] ${warning}`);
+ const plumbingFailure = forcedGcPlumbingFailure(fresh);
+ if (plumbingFailure !== null) {
+ console.error(`[r4-gate] ${plumbingFailure}`);
+ process.exit(EXIT_INCONCLUSIVE);
+ }
+
+ const warnings = checkMethodologyMismatches(baseline, fresh);
+ for (const warning of warnings) {
+ console.warn(`[r4-gate] ${warning.message}`);
}
const report = evaluateRegression(baseline, fresh);
console.log(formatReport(report));
console.log(` baseline=${BASELINE_PATH}`);
console.log(` fresh =${freshPath}`);
+ const blockers = comparabilityBlockers(warnings);
+ if (blockers.length > 0 && !accepted) {
+ console.error(
+ `[r4-gate] INCONCLUSIVE - ${blockers.length} warning(s) above say this run and the baseline ` +
+ 'are not comparable, so neither PASS nor FAIL is a verdict this data supports. ' +
+ `Re-baseline, or pass ${ACCEPT_MISMATCH_FLAG} to read the deltas anyway.`,
+ );
+ process.exit(EXIT_INCONCLUSIVE);
+ }
process.exit(report.pass ? 0 : 1);
}
if (import.meta.main) {
- await main();
+ try {
+ await main();
+ } catch (error) {
+ if (error instanceof InconclusiveError) {
+ console.error(`[r4-gate] INCONCLUSIVE - ${error.message}`);
+ process.exit(EXIT_INCONCLUSIVE);
+ }
+ if (error instanceof DataError) {
+ console.error(`[r4-gate] ${error.message}`);
+ process.exit(EXIT_DATA);
+ }
+ throw error;
+ }
}
diff --git a/packages/core/tsconfig.perf.json b/packages/core/tsconfig.perf.json
new file mode 100644
index 000000000..d146ec4aa
--- /dev/null
+++ b/packages/core/tsconfig.perf.json
@@ -0,0 +1,8 @@
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "types": ["node"]
+ },
+ "include": ["tests/perf"],
+ "exclude": ["node_modules", "dist"]
+}