Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Repair six CI gates that cannot fail on their charter defect (#4174)
Nick Gomez91b4b419bc5a
diff --git a/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsx b/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsxnew file mode 100644index 000000000..dbb5b216c--- /dev/null+++ b/packages/app/tests/dom/jsx-inline-item-growth.dom.test.tsx@@ -0,0 +1,180 @@+import { act, cleanup, render } from '@testing-library/react';+import type { Editor } from '@tiptap/core';+import Collaboration from '@tiptap/extension-collaboration';+import { EditorContent, useEditor } from '@tiptap/react';+import StarterKit from '@tiptap/starter-kit';+import { useEffect, useState } from 'react';+import { createPortal } from 'react-dom';+import { afterEach, describe, expect, test } from 'vitest';+import * as Y from 'yjs';+import { JsxInline } from '../../src/editor/extensions/jsx-inline';++const KEYSTROKE_COUNT = 100;+const OBSERVED_HEALTHY_DELTA = 1;+const MAX_ITEM_DELTA = 20;++function totalStructs(doc: Y.Doc): number {+ let total = 0;+ for (const [, structs] of doc.store.clients) total += structs.length;+ return total;+}++function jsxInlineElement(doc: Y.Doc): Y.XmlElement | null {+ const paragraph = doc.getXmlFragment('default').get(0);+ if (!(paragraph instanceof Y.XmlElement)) return null;+ for (let i = 0; i < paragraph.length; i += 1) {+ const child = paragraph.get(i);+ if (child instanceof Y.XmlElement && child.nodeName === 'jsxInline') return child;+ }+ return null;+}++function jsxInlineRange(editor: Editor): { pos: number; contentSize: number } {+ let pos = -1;+ let contentSize = 0;+ editor.state.doc.descendants((node, at) => {+ if (node.type.name !== 'jsxInline') return undefined;+ pos = at;+ contentSize = node.content.size;+ return false;+ });+ return { pos, contentSize };+}++function Host({ ydoc, onEditor }: { ydoc: Y.Doc; onEditor: (editor: Editor) => void }) {+ const editor = useEditor({+ extensions: [+ StarterKit.configure({ link: false, undoRedo: false }),+ JsxInline.configure({ docName: 'jsx-inline-item-growth' }),+ Collaboration.configure({ document: ydoc }),+ ],+ editable: true,+ immediatelyRender: true,+ });+ const [portalTarget] = useState(() => document.createElement('div'));+ useEffect(() => {+ document.body.appendChild(portalTarget);+ return () => portalTarget.remove();+ }, [portalTarget]);+ if (editor) onEditor(editor);+ return createPortal(+ // biome-ignore lint/plugin/no-unportaled-editor-content: portalled per the H6 contract, with a per-render exclusive target owned by this harness+ <EditorContent editor={editor} />,+ portalTarget,+ );+}++function mountJsxInlineEditor(ydoc: Y.Doc): Editor {+ let captured: Editor | null = null;+ render(+ <Host+ ydoc={ydoc}+ onEditor={(editor) => {+ captured = editor;+ }}+ />,+ );+ if (!captured) throw new Error('editor did not mount');+ const editor = captured as Editor;+ editor.commands.setContent({+ type: 'doc',+ content: [+ {+ type: 'paragraph',+ content: [+ { type: 'text', text: 'before ' },+ {+ type: 'jsxInline',+ attrs: { componentName: 'Callout' },+ content: [{ type: 'text', text: 'x' }],+ },+ { type: 'text', text: ' after' },+ ],+ },+ ],+ });+ return editor;+}++function recreateJsxInlineElement(ydoc: Y.Doc): void {+ const paragraph = ydoc.getXmlFragment('default').get(0);+ if (!(paragraph instanceof Y.XmlElement)) return;+ for (let i = 0; i < paragraph.length; i += 1) {+ const child = paragraph.get(i);+ if (!(child instanceof Y.XmlElement) || child.nodeName !== 'jsxInline') continue;+ const clone = child.clone();+ ydoc.transact(() => {+ paragraph.delete(i, 1);+ paragraph.insert(i, [clone]);+ });+ return;+ }+}++function typeIntoJsxInline(+ editor: Editor,+ ydoc: Y.Doc,+ options: { mode?: 'append' | 'alternate'; afterKeystroke?: (ydoc: Y.Doc) => void } = {},+): { delta: number; elementPreserved: boolean } {+ const mode = options.mode ?? 'append';+ const elementBefore = jsxInlineElement(ydoc);+ const before = totalStructs(ydoc);+ act(() => {+ for (let i = 0; i < KEYSTROKE_COUNT; i += 1) {+ const { pos, contentSize } = jsxInlineRange(editor);+ if (pos < 0) throw new Error('jsxInline node vanished mid-typing');+ const at = mode === 'alternate' && i % 2 === 0 ? pos + 1 : pos + 1 + contentSize;+ editor.view.dispatch(editor.state.tr.insertText(String.fromCharCode(97 + (i % 26)), at, at));+ options.afterKeystroke?.(ydoc);+ }+ });+ const elementAfter = jsxInlineElement(ydoc);+ return {+ delta: totalStructs(ydoc) - before,+ elementPreserved: elementBefore !== null && elementBefore === elementAfter,+ };+}++afterEach(cleanup);++describe('Y.Item growth under jsxInline typing in a mounted collaborative editor', () => {+ test('typing 100 keystrokes inside a jsxInline node keeps struct growth sublinear', () => {+ const ydoc = new Y.Doc();+ const editor = mountJsxInlineEditor(ydoc);++ expect(jsxInlineElement(ydoc)).toBeInstanceOf(Y.XmlElement);++ const { delta, elementPreserved } = typeIntoJsxInline(editor, ydoc);++ expect(+ delta,+ `Y struct delta ${delta} over ${KEYSTROKE_COUNT} keystrokes exceeds the bound ${MAX_ITEM_DELTA} (observed healthy value: ${OBSERVED_HEALTHY_DELTA}). ` ++ `The bound sits 5x and 10x below the defect arms, which land near 100 and ${KEYSTROKE_COUNT * 2 + 1}, so a delta just above ${MAX_ITEM_DELTA} and still far below ${KEYSTROKE_COUNT} after a routine ` ++ 'Yjs / ProseMirror / TipTap bump is a change in their struct-merging internals, not content loss: re-measure and widen this constant. Hunt for a content-loss bug only when the delta approaches per-keystroke growth.',+ ).toBeLessThanOrEqual(MAX_ITEM_DELTA);+ expect(elementPreserved).toBe(true);+ expect(editor.state.doc.textContent).toContain('xabcdefghij');+ });++ test('recreating the jsxInline Y.XmlElement on every keystroke breaks the bound', () => {+ const ydoc = new Y.Doc();+ const editor = mountJsxInlineEditor(ydoc);++ const { delta, elementPreserved } = typeIntoJsxInline(editor, ydoc, {+ afterKeystroke: recreateJsxInlineElement,+ });++ expect(delta).toBeGreaterThan(MAX_ITEM_DELTA);+ expect(elementPreserved).toBe(false);+ });++ test('one unmergeable struct per keystroke breaks the bound', () => {+ const ydoc = new Y.Doc();+ const editor = mountJsxInlineEditor(ydoc);++ const { delta } = typeIntoJsxInline(editor, ydoc, { mode: 'alternate' });++ expect(delta).toBe(KEYSTROKE_COUNT);+ expect(delta).toBeGreaterThan(MAX_ITEM_DELTA);+ });+});diff --git a/packages/app/tests/stress/component-blocks.perf.test.ts b/packages/app/tests/stress/component-blocks.perf.test.tsindex 518c97603..47cab915c 100644--- a/packages/app/tests/stress/component-blocks.perf.test.ts+++ b/packages/app/tests/stress/component-blocks.perf.test.ts@@ -1,5 +1,4 @@import { describe, expect, test } from 'vitest';-import * as Y from 'yjs';import { sharedExtensions } from '../../../../packages/core/src/extensions/shared.ts';import { MarkdownManager } from '../../../../packages/core/src/markdown/index.ts';@@ -66,45 +65,3 @@ describe('PF03: parseWithFallback cycle time under load', () => {expect(maxVal).toBeLessThan(MAX_CATASTROPHIC_MS);});});--describe('PF05: Y.Item growth under jsxInline typing', () => {- test('100-keystroke typing in jsxInline content — Y.Item delta ≤ keystroke_count + constant', () => {- const ydoc = new Y.Doc();- const ytext = ydoc.getText('source');-- const initialContent = 'Hello <Icon name="check" /> world';- ytext.insert(0, initialContent);-- const countItems = (yt: Y.Text): number => {- let count = 0;- let item = yt._start;- while (item !== null) {- if (!item.deleted) count++;- item = item.right;- }- return count;- };-- const initialItems = countItems(ytext);- const KEYSTROKE_COUNT = 100;-- for (let i = 0; i < KEYSTROKE_COUNT; i++) {- const insertPos = ytext.toString().length;- ydoc.transact(() => {- ytext.insert(insertPos, String.fromCharCode(97 + (i % 26)));- });- }-- const finalItems = countItems(ytext);- const itemDelta = finalItems - initialItems;-- console.log(- `PF05 results: initialItems=${initialItems}, finalItems=${finalItems}, delta=${itemDelta}, keystrokes=${KEYSTROKE_COUNT}`,- );-- const CONSTANT_OVERHEAD = 10;- expect(itemDelta).toBeLessThanOrEqual(KEYSTROKE_COUNT + CONSTANT_OVERHEAD);-- expect(itemDelta).toBeLessThan(KEYSTROKE_COUNT * 2);- });-});diff --git a/packages/core/package.json b/packages/core/package.jsonindex cc56fdfda..e993652e7 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -145,9 +145,9 @@"scripts": {"build": "tsdown && pnpm run verify:exports","verify:exports": "node scripts/verify-exports-targets.mjs",- "typecheck": "tsc --noEmit",+ "typecheck": "tsc --noEmit && tsc -p tsconfig.perf.json --noEmit","test": "vitest run src/ tests/contract/ scripts/",- "test:perf:bench": "RUN_BENCH=1 vitest run tests/perf/markdown-bench.test.ts",+ "test:perf:bench": "NODE_OPTIONS=\"${NODE_OPTIONS:-} --expose-gc\" RUN_BENCH=1 vitest run tests/perf/markdown-bench.test.ts","test:perf:regression:unit": "vitest run tests/perf/regression-gate.test.ts","test:perf:regression": "tsx tests/perf/run-regression-gate.ts","test:health:unit": "vitest run tests/health/parse-health-gate.test.ts",diff --git a/packages/core/tests/perf/baseline.json b/packages/core/tests/perf/baseline.jsonindex 24d2a9ebf..a6f54aed4 100644--- a/packages/core/tests/perf/baseline.json+++ b/packages/core/tests/perf/baseline.json@@ -1,5 +1,5 @@{- "schemaVersion": 1,+ "schemaVersion": 2,"capturedAt": "2026-04-16T08:28:01.616Z","runnerClass": "local-m-series-calibration","methodology": {@@ -7,10 +7,14 @@"measuredIters": 10,"gcBetweenRuns": true},- "toolchain": {+ "capturedUnder": {"runtime": "[email protected]","testRunner": "bun test"},+ "targetToolchain": {+ "runtime": "[email protected]",+ "testRunner": "[email protected]"+ },"calibrationRuns": 4,"threshold": {"floorPct": 0.1,diff --git a/packages/core/tests/perf/gc.ts b/packages/core/tests/perf/gc.tsnew file mode 100644index 000000000..4529c5ca2--- /dev/null+++ b/packages/core/tests/perf/gc.ts@@ -0,0 +1,19 @@+type GcHost = { gc?: () => void };++export const EXPOSE_GC_FLAG = '--expose-gc';++export function gcAvailable(): boolean {+ return typeof (globalThis as GcHost).gc === 'function';+}++export function collectGarbage(): void {+ (globalThis as GcHost).gc?.();+}++export function withForcedGc<T extends NodeJS.ProcessEnv>(+ env: T,+): Omit<T, 'NODE_OPTIONS'> & { NODE_OPTIONS: string } {+ const current = env.NODE_OPTIONS ?? '';+ if (current.split(/\s+/).includes(EXPOSE_GC_FLAG)) return { ...env, NODE_OPTIONS: current };+ return { ...env, NODE_OPTIONS: current ? `${current} ${EXPOSE_GC_FLAG}` : EXPOSE_GC_FLAG };+}diff --git a/packages/core/tests/perf/markdown-bench.test.ts b/packages/core/tests/perf/markdown-bench.test.tsindex fc2af2971..dd5d106cd 100644--- a/packages/core/tests/perf/markdown-bench.test.ts+++ b/packages/core/tests/perf/markdown-bench.test.ts@@ -12,6 +12,8 @@ import {type PerfBlockCount,} from '../../src/markdown/fixtures/index.ts';import { MarkdownManager } from '../../src/markdown/index.ts';+import { collectGarbage, gcAvailable } from './gc.ts';+import type { FreshResults } from './regression-gate.ts';const BENCH_ENABLED = process.env.RUN_BENCH === '1' || process.env.RUN_BENCH === 'true';@@ -69,7 +71,7 @@ function readVitestVersion(): string {}}-interface RunnerInfo {+type RunnerInfo = {nodeVersion: string;vitestVersion: string;gitSha: string;@@ -79,7 +81,7 @@ interface RunnerInfo {ramGB: number;platform: string;runnerClass: string;-}+};function runnerInfo(): RunnerInfo {const cpuList = cpus();@@ -96,12 +98,12 @@ function runnerInfo(): RunnerInfo {};}-const GC_FORCED = typeof (globalThis as { gc?: () => void }).gc === 'function';+const GC_FORCED = gcAvailable();function measure(op: () => void, n: number): number[] {const samples: number[] = [];for (let i = 0; i < n; i++) {- (globalThis as { gc?: () => void }).gc?.();+ collectGarbage();const t0 = performance.now();op();samples.push(performance.now() - t0);@@ -164,7 +166,7 @@ describeBench('markdown pipeline benchmark harness (R1)', () => {}const output = {- schemaVersion: 1,+ schemaVersion: 2,startedAt,finishedAt: new Date().toISOString(),methodology: {@@ -172,9 +174,13 @@ describeBench('markdown pipeline benchmark harness (R1)', () => {measuredIters: MEASURED_ITERS,gcBetweenRuns: GC_FORCED,},+ toolchain: {+ runtime: `node@${process.versions.node}`,+ testRunner: `vitest@${readVitestVersion()}`,+ },runner: runnerInfo(),results,- };+ } satisfies FreshResults;const stamp = startedAt.replace(/[:.]/g, '-');const target = resolve(HARNESS_DIR, `results.${stamp}.json`);diff --git a/packages/core/tests/perf/regression-gate.test.ts b/packages/core/tests/perf/regression-gate.test.tsindex 113f8d06d..9acc94fd6 100644--- a/packages/core/tests/perf/regression-gate.test.ts+++ b/packages/core/tests/perf/regression-gate.test.ts@@ -1,22 +1,38 @@-import { mkdtempSync, writeFileSync } from 'node:fs';+import { spawnSync } from 'node:child_process';+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { describe, expect, test } from 'vitest';+import { withForcedGc } from './gc.ts';import {+ ACCEPT_MISMATCH_FLAG,type Baseline,+ baselineResultsToolchain,checkMethodologyMismatches,+ comparabilityBlockers,+ DataError,+ EXIT_DATA,+ EXIT_INCONCLUSIVE,+ EXIT_USAGE,evaluateRegression,+ exitCodeForFailure,type FreshResults,formatReport,loadBaseline,loadFreshResults,} from './regression-gate.ts';+import {+ forcedGcPlumbingFailure,+ InconclusiveError,+ resultsStalenessFailure,+} from './run-regression-gate.ts';function makeBaseline(overrides: Partial<Baseline> = {}): Baseline {return {- schemaVersion: 1,+ schemaVersion: 2,capturedAt: '2026-04-16T00:00:00.000Z',runnerClass: 'test-fixture',+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },calibrationRuns: 5,threshold: { floorPct: 0.1, varianceMultiplier: 2 },results: [@@ -41,10 +57,11 @@ function makeBaseline(overrides: Partial<Baseline> = {}): Baseline {function makeFresh(overrides: Partial<FreshResults> = {}): FreshResults {return {- schemaVersion: 1,+ schemaVersion: 2,startedAt: '2026-04-16T01:00:00.000Z',finishedAt: '2026-04-16T01:05:00.000Z',methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },runner: {},results: [{@@ -83,8 +100,8 @@ describe('checkMethodologyMismatches', () => {makeFresh({ methodology: { ...METHODOLOGY, gcBetweenRuns: false } }),);expect(warnings).toHaveLength(1);- expect(warnings[0]).toContain('gcBetweenRuns');- expect(warnings[0]).toContain('--expose-gc');+ expect(warnings[0].message).toContain('gcBetweenRuns');+ expect(warnings[0].message).toContain('--expose-gc');});test('a different measured-iteration count warns', () => {@@ -93,7 +110,7 @@ describe('checkMethodologyMismatches', () => {makeFresh({ methodology: { ...METHODOLOGY, measuredIters: 100 } }),);expect(warnings).toHaveLength(1);- expect(warnings[0]).toContain('measuredIters');+ expect(warnings[0].message).toContain('measuredIters');});test('both differing produces both warnings', () => {@@ -112,10 +129,16 @@ describe('checkMethodologyMismatches', () => {expect(warnings).toEqual([]);});- test('a fresh run without recorded methodology produces no warnings', () => {+ test('FreshResults.methodology is required at the type level', () => {+ // @ts-expect-error methodology is required on FreshResults; the silent no-warning path is unreachable+ const invalid: FreshResults = { ...makeFresh(), methodology: undefined };+ expect(invalid.methodology).toBeUndefined();+ });++ test('a baseline without a methodology block still compares, which is the surviving optional side', () => {const warnings = checkMethodologyMismatches(- makeBaseline({ methodology: METHODOLOGY }),- makeFresh({ methodology: undefined }),+ makeBaseline({ methodology: undefined }),+ makeFresh({ methodology: METHODOLOGY }),);expect(warnings).toEqual([]);});@@ -272,6 +295,62 @@ describe('loadBaseline / loadFreshResults finite-value validation', () => {expect(() => loadFreshResults(path)).toThrow(/parseMs\.p95 is not finite/);});+ test('RED: a string blockCount in the baseline is rejected rather than joining to nothing', () => {+ const baseline = makeBaseline();+ const path = writeTmp('baseline.json', {+ ...baseline,+ results: [{ ...baseline.results[0], blockCount: '100' }, baseline.results[1]],+ });+ expect(() => loadBaseline(path)).toThrow(DataError);+ expect(() => loadBaseline(path)).toThrow(+ /results\[0\]\.blockCount must be a finite number \(got "100"\)\. blockCount is the key baseline rows are joined to fresh rows on/,+ );+ });++ test('a string blockCount would otherwise have been reported as a regression, not as bad data', () => {+ const baseline = makeBaseline();+ const corrupt: Baseline = {+ ...baseline,+ results: [+ // @ts-expect-error blockCount is a number in the type; a hand-edited baseline.json can carry a string+ { ...baseline.results[0], blockCount: '100' },+ baseline.results[1],+ ],+ };+ const report = evaluateRegression(corrupt, makeFresh());+ expect(report.missingFresh).toEqual(['100']);+ expect(report.pass).toBe(false);+ });++ test('RED: a non-object results entry is rejected before anything dereferences it', () => {+ const baseline = makeBaseline();+ const bare = writeTmp('baseline.json', { ...baseline, results: [42, baseline.results[1]] });+ expect(() => loadBaseline(bare)).toThrow(DataError);+ expect(() => loadBaseline(bare)).toThrow(/results\[0\] must be an object \(got 42\)/);+ const nulled = writeTmp('baseline.json', { ...baseline, results: [null, baseline.results[1]] });+ expect(() => loadBaseline(nulled)).toThrow(DataError);+ expect(() => loadBaseline(nulled)).toThrow(/results\[0\] must be an object \(got null\)/);+ const arrayed = writeTmp('baseline.json', { ...baseline, results: [[], baseline.results[1]] });+ expect(() => loadBaseline(arrayed)).toThrow(DataError);+ expect(() => loadBaseline(arrayed)).toThrow(/results\[0\] must be an object \(got \[\]\)/);+ });++ test('RED: a docSizeChars that is not a number is rejected in both loaders', () => {+ const baseline = makeBaseline();+ const withStringSize = {+ ...baseline,+ results: [baseline.results[0], { ...baseline.results[1], docSizeChars: '200000' }],+ };+ expect(() => loadBaseline(writeTmp('baseline.json', withStringSize))).toThrow(+ /results\[1\]\.docSizeChars must be a finite number \(got "200000"\)\. docSizeChars is provenance the comparator never reads/,+ );+ const fresh = makeFresh();+ fresh.results[0].docSizeChars = Number.NaN;+ expect(() => loadFreshResults(writeTmp('results.json', fresh))).toThrow(+ /results\[0\]\.docSizeChars must be a finite number \(got null\)\. docSizeChars is provenance the comparator never reads/,+ );+ });+test('loadBaseline accepts a valid baseline', () => {const path = writeTmp('baseline.json', makeBaseline());expect(() => loadBaseline(path)).not.toThrow();@@ -282,3 +361,979 @@ describe('loadBaseline / loadFreshResults finite-value validation', () => {expect(() => loadFreshResults(path)).not.toThrow();});});++const PERF_DIR = import.meta.dirname;+const COMMITTED_BASELINE_PATH = join(PERF_DIR, 'baseline.json');++type OpKey = 'parseMs' | 'serializeMs' | 'roundTripMs';++function freshFromBaseline(+ baseline: Baseline,+ scale: Partial<Record<OpKey, number>> = {},+ overrides: Partial<FreshResults> = {},+): FreshResults {+ return {+ schemaVersion: 2,+ startedAt: '2026-09-04T00:00:00.000Z',+ finishedAt: '2026-09-04T00:10:00.000Z',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ runner: {},+ results: baseline.results.map((entry) => {+ const op = (name: OpKey) => {+ const p99 = entry[name].p99 * (scale[name] ?? 1);+ return { mean: p99 * 0.8, min: p99 * 0.7, max: p99, p50: p99 * 0.8, p95: p99 * 0.95, p99 };+ };+ return {+ blockCount: entry.blockCount,+ docSizeChars: entry.docSizeChars,+ parseMs: op('parseMs'),+ serializeMs: op('serializeMs'),+ roundTripMs: op('roundTripMs'),+ };+ }),+ ...overrides,+ };+}++function regressedParseBlockCounts(baseline: Baseline, scale: number): number[] {+ return evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: scale }))+ .rows.filter((r) => r.op === 'parseMs' && r.regression)+ .map((r) => r.blockCount)+ .sort((a, b) => a - b);+}++describe('committed baseline.json as the gate substrate', () => {+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);++ test('the committed baseline loads and records the provenance of its results', () => {+ expect(baseline.results).toHaveLength(5);+ expect(baseline.capturedUnder).toEqual({ runtime: '[email protected]', testRunner: 'bun test' });+ expect(baseline.targetToolchain).toEqual({+ runtime: '[email protected]',+ testRunner: '[email protected]',+ });+ expect(baselineResultsToolchain(baseline)?.runtime).toBe('[email protected]');+ });++ test('a fresh run identical to the committed baseline passes', () => {+ const report = evaluateRegression(baseline, freshFromBaseline(baseline));+ expect(report.pass).toBe(true);+ expect(report.missingFresh).toEqual([]);+ expect(report.rows.filter((r) => r.regression)).toEqual([]);+ });++ test('parse p99 raised 9% passes on 0 of the 5 parse rows', () => {+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 1.09 }));+ expect(report.pass).toBe(true);+ expect(regressedParseBlockCounts(baseline, 1.09)).toEqual([]);+ });++ test('parse p99 raised 11% fails on the 4 of 5 parse rows the 10% floor governs', () => {+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 1.11 }));+ expect(report.pass).toBe(false);+ expect(regressedParseBlockCounts(baseline, 1.11)).toEqual([1000, 5000, 10000, 20000]);+ const smallest = report.rows.find((r) => r.blockCount === 100 && r.op === 'parseMs');+ expect(smallest?.regression).toBe(false);+ expect(smallest?.allowedDeltaMs).toBeCloseTo(1.78, 6);+ expect(report.rows.filter((r) => r.op !== 'parseMs' && r.regression)).toEqual([]);+ });++ test('parse p99 raised 20% fails on all 5 parse rows, including the variance-governed one', () => {+ expect(regressedParseBlockCounts(baseline, 1.2)).toEqual([100, 1000, 5000, 10000, 20000]);+ });+});++describe('toolchain provenance warning', () => {+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);++ test('a node/vitest run against the bun-era committed results warns and names both toolchains', () => {+ const warnings = checkMethodologyMismatches(baseline, freshFromBaseline(baseline));+ const toolchainWarnings = warnings.filter((w) => w.kind === 'toolchain');+ expect(toolchainWarnings).toHaveLength(1);+ expect(toolchainWarnings[0].message).toContain('[email protected]');+ expect(toolchainWarnings[0].message).toContain('bun test');+ expect(toolchainWarnings[0].message).toContain('[email protected]');+ expect(toolchainWarnings[0].message).toContain('[email protected]');+ });++ test('a fresh run under the same toolchain as the results produces no toolchain warning', () => {+ const fresh = freshFromBaseline(+ baseline,+ {},+ {+ toolchain: { runtime: '[email protected]', testRunner: 'bun test' },+ },+ );+ expect(checkMethodologyMismatches(baseline, fresh)).toEqual([]);+ });++ test('RED: a schemaVersion 1 baseline is rejected, because `toolchain` meant something else there', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ const legacy = { ...makeBaseline(), schemaVersion: 1 };+ writeFileSync(path, JSON.stringify(legacy));+ expect(() => loadBaseline(path)).toThrow(/schemaVersion must be 2/);+ });++ test('RED: targetToolchain without capturedUnder is rejected as unrecorded provenance', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ const { capturedUnder: _dropped, ...noProvenance } = makeBaseline();+ writeFileSync(+ path,+ JSON.stringify({+ ...noProvenance,+ targetToolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ }),+ );+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);+ });++ test('RED: the version-1 `toolchain` spelling under schemaVersion 2 is rejected, not silently accepted', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ const { capturedUnder, ...rest } = makeBaseline();+ writeFileSync(path, JSON.stringify({ ...rest, toolchain: capturedUnder }));+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);+ expect(() => loadBaseline(path)).toThrow(/version-1 `toolchain` key/);+ });++ test('RED: capturedUnder null is rejected the same way absence is', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), capturedUnder: null }));+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);+ });++ test('RED: a baseline with no provenance at all is rejected, so no run can be silently comparable', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ const { capturedUnder: _c, targetToolchain: _t, ...stripped } = makeBaseline();+ writeFileSync(path, JSON.stringify(stripped));+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);+ });++ for (const [label, value] of [+ ['an empty string', ''],+ ['false', false],+ ['zero', 0],+ ['an array', []],+ ['an empty object', {}],+ ['an object whose fields are empty strings', { runtime: '', testRunner: '' }],+ ] as const) {+ test(`RED: capturedUnder as ${label} is rejected, not carried through as silent provenance`, () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), capturedUnder: value }));+ expect(() => loadBaseline(path)).toThrow(/capturedUnder must be an object declaring runtime/);+ });+ }++ test('RED: a results file with no toolchain is rejected, so a foreign capture cannot compare silently', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ const { toolchain: _dropped, ...bare } = makeFresh();+ writeFileSync(path, JSON.stringify(bare));+ expect(() => loadFreshResults(path)).toThrow(/toolchain must be an object declaring runtime/);+ });++ test('RED: a version-1 results file is rejected on the VERSION, not on a field version 1 never had', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ const { toolchain: _dropped, ...preToolchain } = makeFresh();+ writeFileSync(path, JSON.stringify({ ...preToolchain, schemaVersion: 1 }));+ expect(() => loadFreshResults(path)).toThrow(/results\.json schemaVersion must be 2/);+ expect(() => loadFreshResults(path)).toThrow(/declares no `toolchain`/);+ expect(() => loadFreshResults(path)).not.toThrow(/renumbering it to 2/);+ });++ test('a version-2 results file carrying toolchain loads, so the bump is not a blanket rejection', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ writeFileSync(path, JSON.stringify(makeFresh()));+ const loaded = loadFreshResults(path);+ expect(loaded.schemaVersion).toBe(2);+ expect(loaded.toolchain).toEqual({ runtime: '[email protected]', testRunner: '[email protected]' });+ });++ test('RED: a version-1 results file that already declares a toolchain is told to renumber, not to re-capture', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ writeFileSync(path, JSON.stringify({ ...makeFresh(), schemaVersion: 1 }));+ expect(() => loadFreshResults(path)).toThrow(/results\.json schemaVersion must be 2/);+ expect(() => loadFreshResults(path)).toThrow(/renumbering it to 2 preserves/);+ expect(() => loadFreshResults(path)).not.toThrow(/declares no `toolchain`/);+ });++ for (const [label, value] of [+ ['an empty object', {}],+ ['a bare string', '[email protected]'],+ ['an array', []],+ ['null', null],+ ['an object whose fields are empty strings', { runtime: '', testRunner: '' }],+ ] as const) {+ test(`RED: a version-1 results file whose toolchain is ${label} is told to re-capture, not to renumber`, () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ writeFileSync(path, JSON.stringify({ ...makeFresh(), schemaVersion: 1, toolchain: value }));+ expect(() => loadFreshResults(path)).toThrow(/declares no `toolchain`/);+ expect(() => loadFreshResults(path)).not.toThrow(/renumbering it to 2/);+ });+ }++ test('FreshResults.schemaVersion is the literal 2 at the type level', () => {+ // @ts-expect-error schemaVersion is pinned to 2; a version-1 capture is not a FreshResults+ const invalid: FreshResults = { ...makeFresh(), schemaVersion: 1 };+ expect(invalid.schemaVersion).toBe(1);+ });++ test('RED: a results file with no methodology block is rejected', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-fresh-schema-'));+ const path = join(dir, 'results.json');+ const { methodology: _dropped, ...bare } = makeFresh();+ writeFileSync(path, JSON.stringify(bare));+ expect(() => loadFreshResults(path)).toThrow(/methodology must be an object recording/);+ });++ for (const [label, threshold] of [+ ['an empty object', {}],+ ['a missing floorPct', { varianceMultiplier: 2 }],+ ['a non-finite varianceMultiplier', { floorPct: 0.1, varianceMultiplier: Number.NaN }],+ ] as const) {+ test(`RED: threshold with ${label} is rejected rather than computing an NaN allowance`, () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-threshold-'));+ const path = join(dir, 'baseline.json');+ writeFileSync(path, JSON.stringify({ ...makeBaseline(), threshold }));+ expect(() => loadBaseline(path)).toThrow(+ /must be a finite number|threshold must be an object/,+ );+ });+ }++ test('NEGATIVE: a 50x slower run against a well-formed threshold still FAILS', () => {+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);+ const report = evaluateRegression(baseline, freshFromBaseline(baseline, { parseMs: 50 }));+ expect(report.pass).toBe(false);+ });++ test('RED: capturedUnder equal to targetToolchain is rejected as a finished re-baseline', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const path = join(dir, 'baseline.json');+ const same = { runtime: '[email protected]', testRunner: '[email protected]' };+ writeFileSync(+ path,+ JSON.stringify(makeBaseline({ capturedUnder: same, targetToolchain: { ...same } })),+ );+ expect(() => loadBaseline(path)).toThrow(/capturedUnder equals targetToolchain/);+ });++ test('NEGATIVE: capturedUnder alone, and capturedUnder differing from targetToolchain, both load', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baseline-schema-'));+ const soloPath = join(dir, 'solo.json');+ writeFileSync(+ soloPath,+ JSON.stringify(makeBaseline({ capturedUnder: { runtime: '[email protected]' } })),+ );+ expect(loadBaseline(soloPath).capturedUnder?.runtime).toBe('[email protected]');+ expect(loadBaseline(COMMITTED_BASELINE_PATH).capturedUnder?.runtime).toBe('[email protected]');+ });++ test('a re-baselined file carries capturedUnder alone and warns only on a real change', () => {+ const rebaselined = makeBaseline({+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },+ });+ expect(baselineResultsToolchain(rebaselined)?.runtime).toBe('[email protected]');+ expect(checkMethodologyMismatches(rebaselined, freshFromBaseline(rebaselined))).toEqual([]);+ const stale = freshFromBaseline(+ rebaselined,+ {},+ {+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ },+ );+ expect(checkMethodologyMismatches(rebaselined, stale)).toHaveLength(1);+ });++ test('FreshResults.toolchain is required at the type level', () => {+ // @ts-expect-error toolchain is required on FreshResults; the silent no-warning path is unreachable+ const invalid: FreshResults = { ...freshFromBaseline(baseline), toolchain: undefined };+ expect(invalid.toolchain).toBeUndefined();+ });+});++describe('resultsStalenessFailure', () => {+ const SPAWNED_AT = 1_000_000;++ test('NEGATIVE: a capture written after the spawn is not stale', () => {+ expect(+ resultsStalenessFailure([{ f: 'results.1.json', mtimeMs: SPAWNED_AT + 1 }], SPAWNED_AT),+ ).toBeNull();+ });++ test('BOUNDARY: a capture written exactly at the spawn instant counts as this run', () => {+ expect(+ resultsStalenessFailure([{ f: 'results.1.json', mtimeMs: SPAWNED_AT }], SPAWNED_AT),+ ).toBeNull();+ });++ test('RED: a capture older than the spawn names RUN_BENCH as the propagation suspect', () => {+ const failure = resultsStalenessFailure(+ [{ f: 'results.old.json', mtimeMs: SPAWNED_AT - 1 }],+ SPAWNED_AT,+ );+ expect(failure).toMatch(/predates the bench spawn/);+ expect(failure).toMatch(/RUN_BENCH/);+ });++ test('RED: an empty directory names the missing capture rather than grading nothing', () => {+ expect(resultsStalenessFailure([], SPAWNED_AT)).toMatch(/no results\.\*\.json found/);+ });++ test('the freshest entry wins regardless of directory order, so a stale sibling cannot mask it', () => {+ const entries = [+ { f: 'results.old.json', mtimeMs: SPAWNED_AT - 5000 },+ { f: 'results.new.json', mtimeMs: SPAWNED_AT + 10 },+ ];+ expect(resultsStalenessFailure(entries, SPAWNED_AT)).toBeNull();+ expect(resultsStalenessFailure([...entries].reverse(), SPAWNED_AT)).toBeNull();+ });++ test('an InconclusiveError is what the orchestrator maps onto EXIT_INCONCLUSIVE', () => {+ expect(new InconclusiveError('x')).toBeInstanceOf(Error);+ expect(EXIT_INCONCLUSIVE).not.toBe(EXIT_DATA);+ expect(EXIT_INCONCLUSIVE).not.toBe(EXIT_USAGE);+ expect([EXIT_INCONCLUSIVE, EXIT_DATA, EXIT_USAGE]).not.toContain(1);+ });+});++describe('forcedGcPlumbingFailure', () => {+ const captured = (gcBetweenRuns: boolean | undefined) =>+ ({+ ...makeFresh(),+ methodology:+ gcBetweenRuns === undefined+ ? undefined+ : { warmupIters: 10, measuredIters: 10, gcBetweenRuns },+ }) as FreshResults;++ test('NEGATIVE: a capture that recorded forced GC is not a plumbing failure', () => {+ expect(forcedGcPlumbingFailure(captured(true))).toBeNull();+ });++ test('RED: a capture that recorded gcBetweenRuns=false names the plumbing', () => {+ expect(forcedGcPlumbingFailure(captured(false))).toMatch(+ /--expose-gc did not reach the worker/,+ );+ });++ test('RED: an ABSENT methodology block is a failure, not a silent pass', () => {+ expect(forcedGcPlumbingFailure(captured(undefined))).toMatch(/no methodology block/);+ });+});++describe('the standalone comparator CLI surfaces methodology warnings and refuses a verdict', () => {+ function runEntryPoint(+ script: string,+ extraArgs: string[] = [],+ ): { status: number | null; stdout: string; stderr: string } {+ const dir = mkdtempSync(join(tmpdir(), 'perf-entrypoint-'));+ const freshPath = join(dir, 'fresh.json');+ writeFileSync(+ freshPath,+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),+ );+ const result = spawnSync(+ 'pnpm',+ ['exec', 'tsx', join(PERF_DIR, script), COMMITTED_BASELINE_PATH, freshPath, ...extraArgs],+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },+ );+ if (result.error) {+ throw new Error(+ `spawn of ${script} failed (${result.signal ?? 'no signal'}): ${result.error.message}`,+ );+ }+ return { status: result.status, stdout: result.stdout, stderr: result.stderr };+ }++ test('the standalone comparator CLI warns that the committed results are bun-era', () => {+ const { stderr } = runEntryPoint('regression-gate.ts');+ expect(stderr).toContain('toolchain mismatch');+ expect(stderr).toContain('[email protected]');+ }, 60_000);++ test('RED: an unreadable baseline exits EXIT_DATA with the validator message, not a stack on 1', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-baddata-'));+ const badPath = join(dir, 'baseline.json');+ writeFileSync(badPath, JSON.stringify({ ...makeBaseline(), capturedUnder: {} }));+ const freshPath = join(dir, 'fresh.json');+ writeFileSync(+ freshPath,+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),+ );+ const result = spawnSync(+ 'pnpm',+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), badPath, freshPath],+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },+ );+ expect(result.status).toBe(EXIT_DATA);+ expect(result.stderr).toContain('capturedUnder must be an object declaring runtime');+ expect(result.stderr).not.toMatch(/^\s+at /m);+ }, 60_000);++ function runComparator(baselinePath: string, freshPath: string) {+ const result = spawnSync(+ 'pnpm',+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), baselinePath, freshPath],+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },+ );+ if (result.error) throw result.error;+ return result;+ }++ function badInputFixtures() {+ const dir = mkdtempSync(join(tmpdir(), 'perf-badinput-'));+ const freshPath = join(dir, 'fresh.json');+ writeFileSync(+ freshPath,+ JSON.stringify(freshFromBaseline(loadBaseline(COMMITTED_BASELINE_PATH))),+ );+ const write = (name: string, body: string) => {+ const p = join(dir, name);+ writeFileSync(p, body);+ return p;+ };+ const committed = JSON.parse(readFileSync(COMMITTED_BASELINE_PATH, 'utf8'));+ const missingStats = structuredClone(committed);+ delete missingStats.results[0].parseMs;+ return {+ freshPath,+ nonexistent: join(dir, 'never-written.json'),+ truncated: write('truncated.json', '{ "schemaVersion": 2, '),+ notAnObject: write('array.json', '[1, 2, 3]'),+ missingStats: write('nostats.json', JSON.stringify(missingStats)),+ };+ }++ test('RED: every unreadable-input shape exits EXIT_DATA with a named message and no stack', () => {+ const f = badInputFixtures();+ for (const [label, path, expected] of [+ ['a nonexistent path', f.nonexistent, /cannot read/],+ ['truncated JSON', f.truncated, /is not valid JSON/],+ ['a JSON array instead of an object', f.notAnObject, /must contain a JSON object/],+ ['a results row missing an op block', f.missingStats, /is missing its parseMs stats block/],+ ] as const) {+ const result = runComparator(path, f.freshPath);+ expect(result.status, `${label} should exit EXIT_DATA\n${result.stderr}`).toBe(EXIT_DATA);+ expect(result.stderr, `${label} should name the problem`).toMatch(expected);+ expect(result.stderr, `${label} should not dump a stack`).not.toMatch(/^\s+at /m);+ }+ }, 120_000);++ test('the failure classifier routes unreadable input to EXIT_DATA and rethrows everything else', () => {+ expect(exitCodeForFailure(new DataError('baseline.json: cannot read'))).toBe(EXIT_DATA);+ for (const internal of [+ new TypeError('an invariant inside the gate broke'),+ new RangeError('index out of bounds'),+ new Error('plain failure'),+ ]) {+ expect(+ exitCodeForFailure(internal),+ `${internal.constructor.name} is an internal fault: it must rethrow so its stack survives, ` ++ 'not be reported to the operator as unreadable input',+ ).toBeNull();+ }+ });++ test('RED: an incomparable run exits INCONCLUSIVE rather than PASS/0', () => {+ const { status, stdout, stderr } = runEntryPoint('regression-gate.ts');+ expect(status).toBe(EXIT_INCONCLUSIVE);+ expect(stderr).toContain('INCONCLUSIVE');+ expect(stdout).toContain('perf regression gate');+ }, 60_000);++ test('NEGATIVE: the escape hatch restores the PASS verdict on the same inputs', () => {+ const { status, stderr } = runEntryPoint('regression-gate.ts', [ACCEPT_MISMATCH_FLAG]);+ expect(status).toBe(0);+ expect(stderr).toContain('toolchain mismatch');+ expect(stderr).not.toContain('INCONCLUSIVE');+ }, 60_000);++ test('NEGATIVE: a comparable run needs no escape hatch', () => {+ const dir = mkdtempSync(join(tmpdir(), 'perf-comparable-'));+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);+ const baselinePath = join(dir, 'baseline.json');+ const freshPath = join(dir, 'fresh.json');+ const { targetToolchain: _dropped, ...rest } = baseline;+ writeFileSync(+ baselinePath,+ JSON.stringify({+ ...rest,+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },+ }),+ );+ writeFileSync(freshPath, JSON.stringify(freshFromBaseline(baseline)));+ const result = spawnSync(+ 'pnpm',+ ['exec', 'tsx', join(PERF_DIR, 'regression-gate.ts'), baselinePath, freshPath],+ { cwd: join(PERF_DIR, '..', '..'), encoding: 'utf8', timeout: 30_000 },+ );+ expect(result.stderr).not.toContain('INCONCLUSIVE');+ expect(result.status).toBe(0);+ }, 60_000);+});++describe('p50 and p95 are recorded alongside p99', () => {+ test('every row carries the fresh p50/p95 while the verdict stays on p99', () => {+ const baseline = makeBaseline();+ const fresh = makeFresh();+ const report = evaluateRegression(baseline, fresh);+ const row = report.rows.find((r) => r.blockCount === 1000 && r.op === 'parseMs');+ expect(row?.freshP50).toBe(90);+ expect(row?.freshP95).toBe(100);+ expect(row?.regression).toBe(false);+ });++ test('baseline p50/p95 are carried when the baseline records them and omitted when it does not', () => {+ const withPercentiles = makeBaseline({+ results: [+ {+ blockCount: 100,+ docSizeChars: 20_000,+ parseMs: { p99: 10, p99StdevMs: 0.25, p50: 8, p95: 9.5 },+ serializeMs: { p99: 2, p99StdevMs: 0.1 },+ roundTripMs: { p99: 12, p99StdevMs: 0.3 },+ },+ ],+ });+ const rows = evaluateRegression(withPercentiles, makeFresh()).rows;+ const parse = rows.find((r) => r.op === 'parseMs');+ expect(parse?.baselineP50).toBe(8);+ expect(parse?.baselineP95).toBe(9.5);+ const serialize = rows.find((r) => r.op === 'serializeMs');+ expect(serialize?.baselineP50).toBeUndefined();+ expect(serialize?.baselineP95).toBeUndefined();+ });++ test('formatReport prints the fresh percentiles and the baseline pair when recorded', () => {+ const withPercentiles = makeBaseline({+ results: [+ {+ blockCount: 100,+ docSizeChars: 20_000,+ parseMs: { p99: 10, p99StdevMs: 0.25, p50: 8, p95: 9.5 },+ serializeMs: { p99: 2, p99StdevMs: 0.1 },+ roundTripMs: { p99: 12, p99StdevMs: 0.3 },+ },+ ],+ });+ const text = formatReport(evaluateRegression(withPercentiles, makeFresh()));+ expect(text).toContain('p50=9.00ms p95=10.00ms');+ expect(text).toContain('(baseline p50=8.00ms p95=9.50ms)');+ const serializeLine = text.split('\n').find((line) => line.includes('serializeMs'));+ expect(serializeLine).toContain('p50=1.80ms p95=2.00ms');+ expect(serializeLine).not.toContain('baseline p50=');+ });++ test('a baseline recording a non-finite p50 is rejected at load', () => {+ const dir = mkdtempSync(join(tmpdir(), 'regression-gate-percentiles-'));+ const path = join(dir, 'baseline.json');+ const baseline = makeBaseline();+ baseline.results[0].parseMs.p50 = Number.NaN;+ writeFileSync(path, JSON.stringify(baseline));+ expect(() => loadBaseline(path)).toThrow(/parseMs\.p50 is not finite/);+ });+});++describe('forced GC is the default bench invocation', () => {+ function probeGcInSpawnedNode(nodeArgs: string[]): string {+ const result = spawnSync(+ process.execPath,+ [+ ...nodeArgs,+ '--input-type=module',+ '-e',+ `import { gcAvailable } from ${JSON.stringify(join(PERF_DIR, 'gc.ts'))}; process.stdout.write(String(gcAvailable()));`,+ ],+ { encoding: 'utf8', env: { ...process.env, NODE_OPTIONS: '' }, timeout: 15_000 },+ );+ if (result.error) {+ throw new Error(+ `gc probe spawn failed (${result.signal ?? 'no signal'}): ${result.error.message}`,+ );+ }+ expect(result.status, result.stderr).toBe(0);+ return result.stdout;+ }++ test('a node process without --expose-gc reports no forced GC', () => {+ expect(probeGcInSpawnedNode([])).toBe('false');+ });++ test('a node process with --expose-gc reports forced GC', () => {+ expect(probeGcInSpawnedNode(['--expose-gc'])).toBe('true');+ });++ test('withForcedGc adds the flag, preserves existing NODE_OPTIONS, and is idempotent', () => {+ expect(withForcedGc({}).NODE_OPTIONS).toBe('--expose-gc');+ expect(withForcedGc({ NODE_OPTIONS: '--max-old-space-size=4096' }).NODE_OPTIONS).toBe(+ '--max-old-space-size=4096 --expose-gc',+ );+ expect(withForcedGc({ NODE_OPTIONS: '--expose-gc' }).NODE_OPTIONS).toBe('--expose-gc');+ expect(withForcedGc({ RUN_BENCH: '1' }).RUN_BENCH).toBe('1');+ });++ test('the core test:perf:bench script forces GC', () => {+ const pkg = JSON.parse(readFileSync(join(PERF_DIR, '..', '..', 'package.json'), 'utf8'));+ expect(pkg.scripts['test:perf:bench']).toContain('--expose-gc');+ expect(pkg.scripts['test:perf:bench']).toContain('RUN_BENCH=1');+ });++ test('the bench records the GC state it actually ran under', () => {+ const bench = readFileSync(join(PERF_DIR, 'markdown-bench.test.ts'), 'utf8');+ expect(bench).toContain('const GC_FORCED = gcAvailable();');+ expect(bench).toContain('gcBetweenRuns: GC_FORCED,');+ });++ test('the regression orchestrator spawns the bench with forced GC', () => {+ const orchestrator = readFileSync(join(PERF_DIR, 'run-regression-gate.ts'), 'utf8');+ expect(orchestrator).toContain("env: withForcedGc({ ...process.env, RUN_BENCH: '1' })");+ });++ test('a fresh run without forced GC is flagged against the committed baseline', () => {+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);+ const fresh = freshFromBaseline(+ baseline,+ {},+ {+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: false },+ },+ );+ const warnings = checkMethodologyMismatches(baseline, fresh);+ const gcWarnings = warnings.filter((w) => w.kind === 'gc-regime');+ expect(gcWarnings).toHaveLength(1);+ expect(gcWarnings[0].message).toContain('fresh gcBetweenRuns=false');+ expect(gcWarnings[0].message).toContain('--expose-gc');+ expect(comparabilityBlockers(warnings).length).toBeGreaterThan(0);+ expect(comparabilityBlockers(warnings)).toContain(gcWarnings[0]);+ });++ test('an iteration-count mismatch warns but is deliberately NOT a comparability blocker', () => {+ const baseline = loadBaseline(COMMITTED_BASELINE_PATH);+ const fresh = freshFromBaseline(+ baseline,+ {},+ {+ methodology: { warmupIters: 10, measuredIters: 40, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: 'bun test' },+ },+ );+ const warnings = checkMethodologyMismatches(baseline, fresh);+ expect(warnings.filter((w) => w.kind === 'iteration-count')).toHaveLength(1);+ expect(comparabilityBlockers(warnings)).toEqual([]);+ });+});++function opStats(+ mean: number,+ min: number,+ max: number,+ p50: number,+ p95: number,+ p99: number,+): { mean: number; min: number; max: number; p50: number; p95: number; p99: number } {+ return { mean, min, max, p50, p95, p99 };+}++const CALIBRATION_CAPTURE_1: FreshResults = {+ schemaVersion: 2,+ startedAt: '2026-09-04T12:20:15.610Z',+ finishedAt: '2026-09-04T12:24:39.000Z',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ runner: { runnerClass: 'local-m-series-calibration' },+ results: [+ {+ blockCount: 100,+ docSizeChars: 20293,+ parseMs: opStats(22.78, 21.76, 23.34, 23.01, 23.34, 23.34),+ serializeMs: opStats(3.82, 3.24, 4.86, 3.81, 4.86, 4.86),+ roundTripMs: opStats(26.0, 25.03, 27.08, 26.02, 27.08, 27.08),+ },+ {+ blockCount: 1000,+ docSizeChars: 194243,+ parseMs: opStats(173.46, 168.62, 179.25, 174.62, 179.25, 179.25),+ serializeMs: opStats(28.55, 26.38, 30.84, 28.59, 30.84, 30.84),+ roundTripMs: opStats(198.17, 190.58, 206.49, 198.6, 206.49, 206.49),+ },+ {+ blockCount: 5000,+ docSizeChars: 952068,+ parseMs: opStats(858.23, 838.26, 882.04, 860.69, 882.04, 882.04),+ serializeMs: opStats(116.83, 109.4, 125.21, 115.74, 125.21, 125.21),+ roundTripMs: opStats(940.96, 909.94, 968.7, 938.71, 968.7, 968.7),+ },+ {+ blockCount: 10000,+ docSizeChars: 1902300,+ parseMs: opStats(1770.17, 1752.23, 1817.97, 1765.8, 1817.97, 1817.97),+ serializeMs: opStats(212.92, 206.64, 223.86, 210.3, 223.86, 223.86),+ roundTripMs: opStats(2000.19, 1973.64, 2023.4, 2003.95, 2023.4, 2023.4),+ },+ {+ blockCount: 20000,+ docSizeChars: 3837177,+ parseMs: opStats(4625.82, 4565.56, 4694.37, 4620.68, 4694.37, 4694.37),+ serializeMs: opStats(424.5, 411.92, 437.56, 420.32, 437.56, 437.56),+ roundTripMs: opStats(5223.79, 5095.89, 5302.72, 5240.68, 5302.72, 5302.72),+ },+ ],+};++const CALIBRATION_CAPTURE_2: FreshResults = {+ schemaVersion: 2,+ startedAt: '2026-09-04T12:24:49.393Z',+ finishedAt: '2026-09-04T12:29:25.000Z',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ runner: { runnerClass: 'local-m-series-calibration' },+ results: [+ {+ blockCount: 100,+ docSizeChars: 20293,+ parseMs: opStats(22.68, 20.75, 24.44, 22.69, 24.44, 24.44),+ serializeMs: opStats(3.58, 3.0, 4.46, 3.55, 4.46, 4.46),+ roundTripMs: opStats(25.3, 23.82, 26.2, 25.83, 26.2, 26.2),+ },+ {+ blockCount: 1000,+ docSizeChars: 194243,+ parseMs: opStats(164.29, 157.63, 171.97, 164.2, 171.97, 171.97),+ serializeMs: opStats(26.52, 24.16, 28.38, 26.99, 28.38, 28.38),+ roundTripMs: opStats(191.85, 184.05, 199.62, 191.75, 199.62, 199.62),+ },+ {+ blockCount: 5000,+ docSizeChars: 952068,+ parseMs: opStats(838.32, 817.42, 899.96, 835.3, 899.96, 899.96),+ serializeMs: opStats(115.95, 110.76, 122.85, 114.27, 122.85, 122.85),+ roundTripMs: opStats(922.31, 884.99, 974.84, 927.15, 974.84, 974.84),+ },+ {+ blockCount: 10000,+ docSizeChars: 1902300,+ parseMs: opStats(1739.65, 1715.93, 1771.42, 1749.17, 1771.42, 1771.42),+ serializeMs: opStats(207.65, 200.59, 216.24, 206.62, 216.24, 216.24),+ roundTripMs: opStats(1975.12, 1937.2, 2152.98, 1960.42, 2152.98, 2152.98),+ },+ {+ blockCount: 20000,+ docSizeChars: 3837177,+ parseMs: opStats(5321.17, 5116.1, 5470.61, 5342.77, 5470.61, 5470.61),+ serializeMs: opStats(467.29, 452.19, 505.48, 462.36, 505.48, 505.48),+ roundTripMs: opStats(5589.69, 5458.21, 5788.72, 5585.02, 5788.72, 5788.72),+ },+ ],+};++const CONTENDED_FOUR_RUN_AGGREGATE: Baseline = {+ schemaVersion: 2,+ capturedAt: '2026-09-04T12:20:15.610Z',+ runnerClass: 'local-m-series-calibration',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },+ calibrationRuns: 4,+ threshold: { floorPct: 0.1, varianceMultiplier: 2 },+ results: [+ {+ blockCount: 100,+ docSizeChars: 20293,+ parseMs: { p99: 37.45, p99StdevMs: 21.3 },+ serializeMs: { p99: 7.07, p99StdevMs: 3.99 },+ roundTripMs: { p99: 48.36, p99StdevMs: 36.61 },+ },+ {+ blockCount: 1000,+ docSizeChars: 194243,+ parseMs: { p99: 398.44, p99StdevMs: 280.31 },+ serializeMs: { p99: 47.92, p99StdevMs: 23.11 },+ roundTripMs: { p99: 365.25, p99StdevMs: 187.32 },+ },+ {+ blockCount: 5000,+ docSizeChars: 952068,+ parseMs: { p99: 1338.45, p99StdevMs: 581.95 },+ serializeMs: { p99: 217.89, p99StdevMs: 127.8 },+ roundTripMs: { p99: 1271.04, p99StdevMs: 356.18 },+ },+ {+ blockCount: 10000,+ docSizeChars: 1902300,+ parseMs: { p99: 3572.92, p99StdevMs: 2466.94 },+ serializeMs: { p99: 258.39, p99StdevMs: 71.11 },+ roundTripMs: { p99: 2220.6, p99StdevMs: 207.23 },+ },+ {+ blockCount: 20000,+ docSizeChars: 3837177,+ parseMs: { p99: 5167.96, p99StdevMs: 463.46 },+ serializeMs: { p99: 468.86, p99StdevMs: 33.64 },+ roundTripMs: { p99: 10325.58, p99StdevMs: 6599.79 },+ },+ ],+};++function sampleStdev(a: number, b: number): number {+ const mean = (a + b) / 2;+ return Math.sqrt((a - mean) ** 2 + (b - mean) ** 2);+}++function baselineFromCapturePair(first: FreshResults, second: FreshResults): Baseline {+ return {+ schemaVersion: 2,+ capturedAt: first.startedAt,+ runnerClass: 'local-m-series-calibration',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ capturedUnder: { runtime: '[email protected]', testRunner: '[email protected]' },+ calibrationRuns: 2,+ threshold: { floorPct: 0.1, varianceMultiplier: 2 },+ results: first.results.map((row) => {+ const other = second.results.find((r) => r.blockCount === row.blockCount);+ if (!other) throw new Error(`capture pair disagrees on block count ${row.blockCount}`);+ const op = (name: 'parseMs' | 'serializeMs' | 'roundTripMs') => ({+ p99: row[name].p99,+ p99StdevMs: sampleStdev(row[name].p99, other[name].p99),+ p50: row[name].p50,+ p95: row[name].p95,+ });+ return {+ blockCount: row.blockCount,+ docSizeChars: row.docSizeChars,+ parseMs: op('parseMs'),+ serializeMs: op('serializeMs'),+ roundTripMs: op('roundTripMs'),+ };+ }),+ };+}++function scaleCapture(capture: FreshResults, scale: number): FreshResults {+ return {+ ...capture,+ results: capture.results.map((row) => ({+ ...row,+ parseMs: { ...row.parseMs, p99: row.parseMs.p99 * scale },+ serializeMs: { ...row.serializeMs, p99: row.serializeMs.p99 * scale },+ roundTripMs: { ...row.roundTripMs, p99: row.roundTripMs.p99 * scale },+ })),+ };+}++function uniformlyScaledFresh(baseline: Baseline, scale: number): FreshResults {+ return {+ schemaVersion: 2,+ startedAt: '2026-09-04T13:00:00.000Z',+ finishedAt: '2026-09-04T13:05:00.000Z',+ methodology: { warmupIters: 10, measuredIters: 10, gcBetweenRuns: true },+ toolchain: { runtime: '[email protected]', testRunner: '[email protected]' },+ runner: {},+ results: baseline.results.map((row) => {+ const op = (name: 'parseMs' | 'serializeMs' | 'roundTripMs') => {+ const p99 = row[name].p99 * scale;+ return { mean: p99, min: p99, max: p99, p50: p99, p95: p99, p99 };+ };+ return {+ blockCount: row.blockCount,+ docSizeChars: row.docSizeChars,+ parseMs: op('parseMs'),+ serializeMs: op('serializeMs'),+ roundTripMs: op('roundTripMs'),+ };+ }),+ };+}++function regressedRowCount(baseline: Baseline, fresh: FreshResults): number {+ return evaluateRegression(baseline, fresh).rows.filter((r) => r.regression).length;+}++describe('node-era calibration captures (R7 re-capture)', () => {+ const quietPairGate = baselineFromCapturePair(CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2);++ test('a gate built from calibration capture 1 does not fail on capture 2', () => {+ const report = evaluateRegression(quietPairGate, CALIBRATION_CAPTURE_2);+ expect(report.pass).toBe(true);+ expect(report.missingFresh).toEqual([]);+ expect(report.rows).toHaveLength(15);+ expect(report.rows.filter((r) => r.regression)).toEqual([]);+ });++ test('that same gate reds 7 of its 15 rows when capture 2 is raised 11 percent', () => {+ const report = evaluateRegression(quietPairGate, scaleCapture(CALIBRATION_CAPTURE_2, 1.11));+ expect(report.pass).toBe(false);+ expect(report.rows.filter((r) => r.regression)).toHaveLength(7);+ });++ test('that same gate reds all 15 rows when capture 2 is raised 30 percent', () => {+ expect(regressedRowCount(quietPairGate, scaleCapture(CALIBRATION_CAPTURE_2, 1.3))).toBe(15);+ });++ test('the captures record forced GC and the node toolchain', () => {+ for (const capture of [CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2]) {+ expect(capture.methodology?.gcBetweenRuns).toBe(true);+ expect(capture.toolchain).toEqual({ runtime: '[email protected]', testRunner: '[email protected]' });+ }+ });++ test('every capture row reports p95 equal to p99, so p99 at 10 iterations is the maximum', () => {+ for (const capture of [CALIBRATION_CAPTURE_1, CALIBRATION_CAPTURE_2]) {+ for (const row of capture.results) {+ for (const op of ['parseMs', 'serializeMs', 'roundTripMs'] as const) {+ expect(row[op].p95).toBe(row[op].p99);+ expect(row[op].p99).toBe(row[op].max);+ }+ }+ }+ });++ test('aggregating the contended four-run calibration produces a gate that cannot fire', () => {+ expect(+ regressedRowCount(+ CONTENDED_FOUR_RUN_AGGREGATE,+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 1.11),+ ),+ ).toBe(0);+ expect(+ regressedRowCount(+ CONTENDED_FOUR_RUN_AGGREGATE,+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 2),+ ),+ ).toBe(7);+ expect(+ regressedRowCount(+ CONTENDED_FOUR_RUN_AGGREGATE,+ uniformlyScaledFresh(CONTENDED_FOUR_RUN_AGGREGATE, 3),+ ),+ ).toBe(15);+ });++ test('the contended aggregate carries a variance band wider than the ten percent floor everywhere', () => {+ const bands = CONTENDED_FOUR_RUN_AGGREGATE.results.flatMap((row) =>+ (['parseMs', 'serializeMs', 'roundTripMs'] as const).map(+ (op) => (2 * row[op].p99StdevMs) / row[op].p99,+ ),+ );+ expect(bands).toHaveLength(15);+ expect(Math.min(...bands)).toBeGreaterThan(0.1);+ expect(Math.max(...bands)).toBeGreaterThan(1.5);+ });++ test('the committed baseline results are not replaced by this calibration', () => {+ const committed = loadBaseline(COMMITTED_BASELINE_PATH);+ const smallestParse = committed.results.find((r) => r.blockCount === 100)?.parseMs;+ expect(smallestParse).toEqual({ p99: 9.69, p99StdevMs: 0.89 });+ expect(committed.calibrationRuns).toBe(4);+ expect(committed.capturedAt).toBe('2026-04-16T08:28:01.616Z');+ });+});diff --git a/packages/core/tests/perf/regression-gate.ts b/packages/core/tests/perf/regression-gate.tsindex 3d103eee0..8d2dc5e44 100644--- a/packages/core/tests/perf/regression-gate.ts+++ b/packages/core/tests/perf/regression-gate.ts@@ -16,6 +16,11 @@ interface OpStats {p99: number;}+export interface Toolchain {+ runtime?: string;+ testRunner?: string;+}+interface FreshBlockResult {blockCount: number;docSizeChars: number;@@ -25,31 +30,37 @@ interface FreshBlockResult {}export interface FreshResults {- schemaVersion: number;+ schemaVersion: 2;startedAt: string;finishedAt: string;- methodology?: Methodology;+ methodology: Methodology;+ toolchain: Toolchain;runner: Record<string, unknown>;results: FreshBlockResult[];}+interface BaselineOpStats {+ p99: number;+ p99StdevMs: number;+ p50?: number;+ p95?: number;+}+interface BaselineBlockEntry {blockCount: number;docSizeChars: number;- parseMs: { p99: number; p99StdevMs: number };- serializeMs: { p99: number; p99StdevMs: number };- roundTripMs: { p99: number; p99StdevMs: number };+ parseMs: BaselineOpStats;+ serializeMs: BaselineOpStats;+ roundTripMs: BaselineOpStats;}export interface Baseline {- schemaVersion: 1;+ schemaVersion: 2;capturedAt: string;runnerClass: string;methodology?: Methodology;- toolchain?: {- runtime?: string;- testRunner?: string;- };+ capturedUnder: Toolchain;+ targetToolchain?: Toolchain;calibrationRuns: number;threshold: {floorPct: number;@@ -68,6 +79,10 @@ interface OpRegressionRow {deltaMs: number;allowedDeltaMs: number;regression: boolean;+ baselineP50?: number;+ baselineP95?: number;+ freshP50: number;+ freshP95: number;}interface RegressionReport {@@ -108,6 +123,10 @@ export function evaluateRegression(baseline: Baseline, fresh: FreshResults): RegdeltaMs,allowedDeltaMs,regression: deltaMs > allowedDeltaMs,+ baselineP50: b[op].p50,+ baselineP95: b[op].p95,+ freshP50: f[op].p50,+ freshP95: f[op].p95,});}}@@ -121,6 +140,10 @@ export function evaluateRegression(baseline: Baseline, fresh: FreshResults): Regreturn { pass, rows, missingFresh, extraFresh };}+function fmtOptionalMs(value: number | undefined): string {+ return value === undefined ? 'n/a' : `${value.toFixed(2)}ms`;+}+export function formatReport(report: RegressionReport): string {const lines: string[] = [];lines.push(`perf regression gate: ${report.pass ? 'PASS' : 'FAIL'}`);@@ -133,54 +156,156 @@ export function formatReport(report: RegressionReport): string {for (const row of report.rows) {const marker = row.regression ? '✗' : '✓';const deltaSign = row.deltaMs >= 0 ? '+' : '';+ const baselineTail =+ row.baselineP50 === undefined && row.baselineP95 === undefined+ ? ''+ : ` (baseline p50=${fmtOptionalMs(row.baselineP50)} p95=${fmtOptionalMs(row.baselineP95)})`;lines.push(` ${marker} ${String(row.blockCount).padStart(5)} ${row.op.padEnd(12)}` +` baseline=${row.baselineP99.toFixed(2)}ms` +` fresh=${row.freshP99.toFixed(2)}ms` +` Δ=${deltaSign}${row.deltaMs.toFixed(2)}ms` +- ` allowed=${row.allowedDeltaMs.toFixed(2)}ms`,+ ` allowed=${row.allowedDeltaMs.toFixed(2)}ms` ++ ` p50=${row.freshP50.toFixed(2)}ms p95=${row.freshP95.toFixed(2)}ms` ++ baselineTail,);}return lines.join('\n');}-export function checkMethodologyMismatches(baseline: Baseline, fresh: FreshResults): string[] {- const warnings: string[] = [];+export function baselineResultsToolchain(baseline: Baseline): Toolchain {+ return baseline.capturedUnder;+}++function describeToolchain(toolchain: Toolchain): string {+ return `runtime=${toolchain.runtime ?? 'unknown'} testRunner=${toolchain.testRunner ?? 'unknown'}`;+}++export const EXIT_INCONCLUSIVE = 3;++export const EXIT_USAGE = 64;++export const EXIT_DATA = 65;++export const EXIT_SOFTWARE = 70;++export class DataError extends Error {}++export const ACCEPT_MISMATCH_FLAG = '--accept-methodology-mismatch';++export interface MethodologyWarning {+ kind: 'toolchain' | 'gc-regime' | 'iteration-count';+ blocking: boolean;+ message: string;+}++export function comparabilityBlockers(warnings: MethodologyWarning[]): MethodologyWarning[] {+ return warnings.filter((w) => w.blocking);+}++export function checkMethodologyMismatches(+ baseline: Baseline,+ fresh: FreshResults,+): MethodologyWarning[] {+ const warnings: MethodologyWarning[] = [];+ const baselineToolchain = baselineResultsToolchain(baseline);+ const freshToolchain = fresh.toolchain;+ if (+ baselineToolchain.runtime !== freshToolchain.runtime ||+ baselineToolchain.testRunner !== freshToolchain.testRunner+ ) {+ warnings.push({+ kind: 'toolchain',+ blocking: true,+ message:+ `toolchain mismatch: the committed baseline results were captured under ` ++ `${describeToolchain(baselineToolchain)}, this run used ` ++ `${describeToolchain(freshToolchain)}. A different engine or test runner ` ++ `changes the allocation and JIT regime, so the p99 deltas below are not ` ++ `attributable to code alone; re-baseline before trusting them.`,+ });+ }const b = baseline.methodology;const f = fresh.methodology;- if (!b || !f) return warnings;+ if (!b) return warnings;if (b.gcBetweenRuns !== f.gcBetweenRuns) {- warnings.push(- `methodology mismatch: baseline gcBetweenRuns=${b.gcBetweenRuns} ` ++ warnings.push({+ kind: 'gc-regime',+ blocking: true,+ message:+ `methodology mismatch: baseline gcBetweenRuns=${b.gcBetweenRuns} ` +`fresh gcBetweenRuns=${f.gcBetweenRuns}. Re-run with ` +`NODE_OPTIONS=--expose-gc to force GC, or re-baseline; the numbers ` +`below compare different allocation regimes.`,- );+ });}if (b.measuredIters !== f.measuredIters) {- warnings.push(- `iteration-count mismatch: baseline measuredIters=${b.measuredIters} ` ++ warnings.push({+ kind: 'iteration-count',+ blocking: false,+ message:+ `iteration-count mismatch: baseline measuredIters=${b.measuredIters} ` +`fresh measuredIters=${f.measuredIters}; p99 is a worst-of-N observation, ` +`so changing N changes what the number means.`,- );+ });}return warnings;}+function readJson(file: string, path: string): Record<string, unknown> {+ let text: string;+ try {+ text = readFileSync(path, 'utf8');+ } catch (error) {+ throw new DataError(`${file}: cannot read ${path} (${(error as Error).message})`);+ }+ let parsed: unknown;+ try {+ parsed = JSON.parse(text);+ } catch (error) {+ throw new DataError(`${file}: ${path} is not valid JSON (${(error as Error).message})`);+ }+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {+ throw new DataError(+ `${file}: ${path} must contain a JSON object (got ${JSON.stringify(parsed)})`,+ );+ }+ return parsed as Record<string, unknown>;+}++function assertStatsPresent(ctx: string, blockCount: number, opName: OpName, stats: unknown): void {+ if (typeof stats !== 'object' || stats === null || Array.isArray(stats)) {+ throw new DataError(+ `${ctx}: blockCount=${blockCount} is missing its ${opName} stats block (got ${JSON.stringify(stats)})`,+ );+ }+}+function assertFiniteStats(ctx: string,blockCount: number,opName: OpName,- stats: { p99: number; p99StdevMs: number },+ stats: BaselineOpStats,): void {+ assertStatsPresent(ctx, blockCount, opName, stats);if (!Number.isFinite(stats.p99)) {- throw new Error(`${ctx}: blockCount=${blockCount} ${opName}.p99 is not finite (${stats.p99})`);+ throw new DataError(+ `${ctx}: blockCount=${blockCount} ${opName}.p99 is not finite (${stats.p99})`,+ );}if (!Number.isFinite(stats.p99StdevMs)) {- throw new Error(+ throw new DataError(`${ctx}: blockCount=${blockCount} ${opName}.p99StdevMs is not finite (${stats.p99StdevMs})`,);}+ for (const key of ['p50', 'p95'] as const) {+ const value = stats[key];+ if (value !== undefined && !Number.isFinite(value)) {+ throw new DataError(+ `${ctx}: blockCount=${blockCount} ${opName}.${key} is not finite (${value})`,+ );+ }+ }}function assertFiniteOpStats(@@ -189,60 +314,213 @@ function assertFiniteOpStats(opName: OpName,stats: OpStats,): void {+ assertStatsPresent(ctx, blockCount, opName, stats);for (const key of ['mean', 'min', 'max', 'p50', 'p95', 'p99'] as const) {if (!Number.isFinite(stats[key])) {- throw new Error(+ throw new DataError(`${ctx}: blockCount=${blockCount} ${opName}.${key} is not finite (${stats[key]})`,);}}}+function assertFiniteEntryKeys(ctx: string, index: number, entry: unknown): void {+ if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {+ throw new DataError(+ `${ctx}: results[${index}] must be an object (got ${JSON.stringify(entry)})`,+ );+ }+ for (const key of ['blockCount', 'docSizeChars'] as const) {+ const value = (entry as Record<string, unknown>)[key];+ if (typeof value !== 'number' || !Number.isFinite(value)) {+ throw new DataError(+ `${ctx}: results[${index}].${key} must be a finite number (got ${JSON.stringify(value)}). ` ++ (key === 'blockCount'+ ? 'blockCount is the key baseline rows are joined to fresh rows on, so a non-number ' ++ 'matches nothing, renders every baseline row as missingFresh, and prints FAIL as ' ++ 'though the run regressed.'+ : 'docSizeChars is provenance the comparator never reads, so a non-number changes no ' ++ 'verdict; it means the capture that wrote this row is corrupt.'),+ );+ }+ }+}++function declaresToolchain(value: unknown): boolean {+ return (+ typeof value === 'object' &&+ value !== null &&+ !Array.isArray(value) &&+ (['runtime', 'testRunner'] as const).some((key) => {+ const declared = (value as Record<string, unknown>)[key];+ return typeof declared === 'string' && declared.length > 0;+ })+ );+}++function assertToolchain(file: string, field: string, value: unknown, why: string): void {+ if (!declaresToolchain(value)) {+ throw new DataError(+ `${file}: ${field} must be an object declaring runtime and/or testRunner (got ${JSON.stringify(value)}). ${why}`,+ );+ }+}++function assertThreshold(value: unknown): void {+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ throw new DataError(+ `baseline.json: threshold must be an object (got ${JSON.stringify(value)})`,+ );+ }+ for (const field of ['floorPct', 'varianceMultiplier'] as const) {+ const n = (value as Record<string, unknown>)[field];+ if (typeof n !== 'number' || !Number.isFinite(n)) {+ throw new DataError(+ `baseline.json: threshold.${field} must be a finite number (got ${JSON.stringify(n)}). ` ++ 'A missing term makes allowedDeltaMs NaN, and every `delta > NaN` comparison is false, so ' ++ 'the gate prints PASS against an arbitrarily slower run.',+ );+ }+ }+}+export function loadBaseline(path: string): Baseline {- const raw = JSON.parse(readFileSync(path, 'utf8'));- if (raw.schemaVersion !== 1) {- throw new Error(`baseline.json schemaVersion must be 1 (got ${raw.schemaVersion})`);+ const raw = readJson('baseline.json', path);+ if (raw.schemaVersion !== 2) {+ throw new DataError(+ `baseline.json schemaVersion must be 2 (got ${raw.schemaVersion}). Version 1 spelled the ` ++ 'provenance of results[] as `toolchain`; version 2 splits it into `capturedUnder` (what ' ++ 'the numbers were measured under, the only field the comparator reads) and ' ++ '`targetToolchain` (what a re-capture should target).',+ );}if (!Array.isArray(raw.results)) {- throw new Error('baseline.json: results must be an array');+ throw new DataError('baseline.json: results must be an array');}- for (const entry of raw.results as BaselineBlockEntry[]) {+ assertToolchain(+ 'baseline.json',+ 'capturedUnder',+ raw.capturedUnder,+ 'It is the only provenance field the comparator reads, and a missing or falsy value ' ++ 'short-circuits the toolchain-mismatch arm rather than failing loudly, which returns the ' ++ 'gate to rendering PASS on an incomparable pair.' ++ (raw.toolchain !== undefined+ ? ' This file still carries the version-1 `toolchain` key; rename it to capturedUnder rather than renumbering.'+ : ''),+ );+ if (raw.targetToolchain !== undefined) {+ assertToolchain(+ 'baseline.json',+ 'targetToolchain',+ raw.targetToolchain,+ 'Omit the key entirely rather than declaring an empty one.',+ );+ }+ assertThreshold(raw.threshold);+ if (+ raw.targetToolchain !== undefined &&+ JSON.stringify(raw.capturedUnder) === JSON.stringify(raw.targetToolchain)+ ) {+ throw new DataError(+ 'baseline.json: capturedUnder equals targetToolchain, which is the state a completed ' ++ 're-baseline removes. Drop targetToolchain once the numbers are captured under it.',+ );+ }+ for (const [index, entry] of (raw.results as BaselineBlockEntry[]).entries()) {+ assertFiniteEntryKeys('baseline', index, entry);for (const op of OP_NAMES) {assertFiniteStats('baseline', entry.blockCount, op, entry[op]);}}- return raw as Baseline;+ return raw as unknown as Baseline;}export function loadFreshResults(path: string): FreshResults {- const raw = JSON.parse(readFileSync(path, 'utf8'));- if (raw.schemaVersion !== 1) {- throw new Error(`results.json schemaVersion must be 1 (got ${raw.schemaVersion})`);+ const raw = readJson('results.json', path);+ if (raw.schemaVersion !== 2) {+ throw new DataError(+ `results.json schemaVersion must be 2 (got ${raw.schemaVersion}). ` ++ (declaresToolchain(raw.toolchain)+ ? 'This file already carries a version-2-shaped `toolchain`, so renumbering it to 2 ' ++ 'preserves that provenance rather than discarding it. Re-run the bench only if you ' ++ 'would rather discard the capture.'+ : 'It declares no `toolchain`, so it carries no provenance for the toolchain-mismatch ' ++ 'arm to read. Re-run the bench to produce a version-2 capture rather than renumbering ' ++ 'the file.'),+ );}if (!Array.isArray(raw.results)) {- throw new Error('results.json: results must be an array');+ throw new DataError('results.json: results must be an array');}- for (const entry of raw.results as FreshBlockResult[]) {+ assertToolchain(+ 'results.json',+ 'toolchain',+ raw.toolchain,+ 'Without it the toolchain-mismatch arm cannot fire, so a capture from a different runtime ' ++ 'compares silently against the baseline.',+ );+ if (+ typeof raw.methodology !== 'object' ||+ raw.methodology === null ||+ Array.isArray(raw.methodology)+ ) {+ throw new DataError(+ `results.json: methodology must be an object recording what this run measured under (got ${JSON.stringify(raw.methodology)}). ` ++ 'Without it the gcBetweenRuns and measuredIters arms cannot fire.',+ );+ }+ for (const [index, entry] of (raw.results as FreshBlockResult[]).entries()) {+ assertFiniteEntryKeys('results', index, entry);for (const op of OP_NAMES) {assertFiniteOpStats('results', entry.blockCount, op, entry[op]);}}- return raw as FreshResults;+ return raw as unknown as FreshResults;}async function main(): Promise<void> {- const [, , baselineArg, freshArg] = process.argv;+ const argv = process.argv.slice(2);+ const accepted = argv.includes(ACCEPT_MISMATCH_FLAG);+ const [baselineArg, freshArg] = argv.filter((arg) => !arg.startsWith('--'));if (!baselineArg || !freshArg) {- console.error('usage: regression-gate.ts <baseline.json> <fresh-results.json>');- process.exit(2);+ console.error(+ `usage: regression-gate.ts <baseline.json> <fresh-results.json> [${ACCEPT_MISMATCH_FLAG}]\n` ++ ` exit 0 no regression, 1 regression, ${EXIT_INCONCLUSIVE} the comparison is not valid, ` ++ `${EXIT_USAGE} bad invocation, ${EXIT_DATA} unreadable input`,+ );+ process.exit(EXIT_USAGE);}const baseline = loadBaseline(resolve(baselineArg));const fresh = loadFreshResults(resolve(freshArg));+ const warnings = checkMethodologyMismatches(baseline, fresh);+ for (const warning of warnings) {+ console.warn(warning.message);+ }const report = evaluateRegression(baseline, fresh);console.log(formatReport(report));+ const blockers = comparabilityBlockers(warnings);+ if (blockers.length > 0 && !accepted) {+ console.error(+ `perf regression gate: INCONCLUSIVE - ${blockers.length} warning(s) above say this run and ` ++ 'the baseline are not comparable, so neither PASS nor FAIL is a verdict this data ' ++ `supports. Re-baseline, or pass ${ACCEPT_MISMATCH_FLAG} to read the deltas anyway.`,+ );+ process.exit(EXIT_INCONCLUSIVE);+ }process.exit(report.pass ? 0 : 1);}+export function exitCodeForFailure(error: unknown): number | null {+ return error instanceof DataError ? EXIT_DATA : null;+}+if (import.meta.main) {- await main();+ try {+ await main();+ } catch (error) {+ const code = exitCodeForFailure(error);+ if (code === null) throw error;+ console.error((error as Error).message);+ process.exit(code);+ }}diff --git a/packages/core/tests/perf/run-regression-gate.ts b/packages/core/tests/perf/run-regression-gate.tsindex 17f0ec76a..6bf5e7806 100644--- a/packages/core/tests/perf/run-regression-gate.ts+++ b/packages/core/tests/perf/run-regression-gate.ts@@ -2,8 +2,16 @@ import { spawnSync } from 'node:child_process';import { readdirSync, statSync } from 'node:fs';import { dirname, resolve } from 'node:path';import { fileURLToPath } from 'node:url';+import { withForcedGc } from './gc.ts';+import type { FreshResults } from './regression-gate.ts';import {+ ACCEPT_MISMATCH_FLAG,checkMethodologyMismatches,+ comparabilityBlockers,+ DataError,+ EXIT_DATA,+ EXIT_INCONCLUSIVE,+ EXIT_SOFTWARE,evaluateRegression,formatReport,loadBaseline,@@ -13,32 +21,67 @@ import {const HERE = dirname(fileURLToPath(import.meta.url));const BASELINE_PATH = resolve(HERE, 'baseline.json');-function findFreshestResults(): string {- const entries = readdirSync(HERE)- .filter((f) => f.startsWith('results.') && f.endsWith('.json'))- .map((f) => ({ f, mtime: statSync(resolve(HERE, f)).mtimeMs }))- .sort((a, b) => b.mtime - a.mtime);+export function forcedGcPlumbingFailure(fresh: FreshResults): string | null {+ if (fresh.methodology?.gcBetweenRuns === true) return null;+ const reported =+ fresh.methodology === undefined+ ? 'no methodology block'+ : `gcBetweenRuns=${fresh.methodology.gcBetweenRuns}`;+ return (+ `the bench was spawned through withForcedGc but the capture reports ${reported}: either ` ++ '--expose-gc did not reach the worker or the run did not record what it ran under, so this ' ++ 'capture was taken under a different allocation regime than the one it claims. Fix the ' ++ 'plumbing before reading the deltas.'+ );+}++export class InconclusiveError extends Error {}++export interface ResultsEntry {+ f: string;+ mtimeMs: number;+}++export function resultsStalenessFailure(entries: ResultsEntry[], notBefore: number): string | null {if (entries.length === 0) {- throw new Error(`no results.*.json found in ${HERE}; did the bench run fail?`);+ return `no results.*.json found in ${HERE}; the bench produced no capture, so there is nothing to grade.`;}- return resolve(HERE, entries[0].f);+ const freshest = [...entries].sort((a, b) => b.mtimeMs - a.mtimeMs)[0];+ if (freshest.mtimeMs >= notBefore) return null;+ return (+ `the freshest results file (${freshest.f}) predates the bench spawn, so the bench wrote ` ++ 'nothing and this run would have graded a leftover capture. markdown-bench.test.ts gates its ' ++ 'whole describe on RUN_BENCH, so a propagation failure skips every test and still exits 0 - ' ++ 'that is what this check catches.'+ );+}++function findFreshestResults(notBefore: number): string {+ const entries = readdirSync(HERE)+ .filter((f) => f.startsWith('results.') && f.endsWith('.json'))+ .map((f) => ({ f, mtimeMs: statSync(resolve(HERE, f)).mtimeMs }));+ const failure = resultsStalenessFailure(entries, notBefore);+ if (failure !== null) throw new InconclusiveError(failure);+ return resolve(HERE, [...entries].sort((a, b) => b.mtimeMs - a.mtimeMs)[0].f);}async function main(): Promise<void> {+ const accepted = process.argv.includes(ACCEPT_MISMATCH_FLAG);+ const spawnedAt = Date.now();const bench = spawnSync('pnpm',['exec', 'vitest', 'run', resolve(HERE, 'markdown-bench.test.ts')],{- env: { ...process.env, RUN_BENCH: '1' },+ env: withForcedGc({ ...process.env, RUN_BENCH: '1' }),stdio: 'inherit',},);if (bench.status !== 0) {console.error(`bench run failed with exit code ${bench.status ?? 'null'}`);- process.exit(bench.status ?? 1);+ process.exit(EXIT_SOFTWARE);}- const freshPath = findFreshestResults();+ const freshPath = findFreshestResults(spawnedAt);const baseline = loadBaseline(BASELINE_PATH);const fresh = loadFreshResults(freshPath);@@ -53,17 +96,45 @@ async function main(): Promise<void> {);}- for (const warning of checkMethodologyMismatches(baseline, fresh)) {- console.warn(`[r4-gate] ${warning}`);+ const plumbingFailure = forcedGcPlumbingFailure(fresh);+ if (plumbingFailure !== null) {+ console.error(`[r4-gate] ${plumbingFailure}`);+ process.exit(EXIT_INCONCLUSIVE);+ }++ const warnings = checkMethodologyMismatches(baseline, fresh);+ for (const warning of warnings) {+ console.warn(`[r4-gate] ${warning.message}`);}const report = evaluateRegression(baseline, fresh);console.log(formatReport(report));console.log(` baseline=${BASELINE_PATH}`);console.log(` fresh =${freshPath}`);+ const blockers = comparabilityBlockers(warnings);+ if (blockers.length > 0 && !accepted) {+ console.error(+ `[r4-gate] INCONCLUSIVE - ${blockers.length} warning(s) above say this run and the baseline ` ++ 'are not comparable, so neither PASS nor FAIL is a verdict this data supports. ' ++ `Re-baseline, or pass ${ACCEPT_MISMATCH_FLAG} to read the deltas anyway.`,+ );+ process.exit(EXIT_INCONCLUSIVE);+ }process.exit(report.pass ? 0 : 1);}if (import.meta.main) {- await main();+ try {+ await main();+ } catch (error) {+ if (error instanceof InconclusiveError) {+ console.error(`[r4-gate] INCONCLUSIVE - ${error.message}`);+ process.exit(EXIT_INCONCLUSIVE);+ }+ if (error instanceof DataError) {+ console.error(`[r4-gate] ${error.message}`);+ process.exit(EXIT_DATA);+ }+ throw error;+ }}diff --git a/packages/core/tsconfig.perf.json b/packages/core/tsconfig.perf.jsonnew file mode 100644index 000000000..d146ec4aa--- /dev/null+++ b/packages/core/tsconfig.perf.json@@ -0,0 +1,8 @@+{+ "extends": "./tsconfig.json",+ "compilerOptions": {+ "types": ["node"]+ },+ "include": ["tests/perf"],+ "exclude": ["node_modules", "dist"]+}