An open source design system that's fully customizable and agent ready
TypeScript
test(markdown): isolate FR23 and remove biased sample selection (#6402)
Cindy Zhang400998fdbe5f
diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjsindex 85883ae00a..9c32a9d9a0 100644--- a/.github/scripts/ci-test-routing.test.mjs+++ b/.github/scripts/ci-test-routing.test.mjs@@ -4,7 +4,8 @@* @file ci-test-routing.test.mjs** The contract between `vitest.config.ts` and the CI jobs that run it, in both- * workflows that run the suite.+ * workflows that run the suite. FR23 must run after the other UI workers exit+ * so its helper-overhead measurement does not compete with unrelated tests.** This repo has now lost test coverage to routing twice. First a suite* belonged to no Vitest project, so `pnpm test` never collected it. Then the@@ -110,6 +111,24 @@ describe.each(Object.entries(WORKFLOWS))(}});+ it('runs FR23 only after the parallel UI suite has exited', () => {+ const benchmark = 'packages/core/src/Markdown/Markdown.fr23.perf.test.ts';+ const job = workflow.jobs['test-ui'];+ const suites = job.steps.filter(step =>+ step.run?.includes('--project ui'),+ );++ // One conditional step preserves the existing scope gate for both runs.+ // Exact foreground commands also reject backgrounding or ignored exits.+ expect(suites).toHaveLength(1);+ expect(suites[0].run.trim().split('\n')).toEqual([+ `pnpm vitest run --project ui --exclude ${benchmark}`,+ `pnpm vitest run --project ui ${benchmark} --no-file-parallelism`,+ ]);+ expect(job['continue-on-error']).not.toBe(true);+ expect(suites[0]['continue-on-error']).not.toBe(true);+ });+it('gives no lane the whole suite', () => {// A bare `pnpm test` runs every project in one job — the shape that hit// the wall.diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 0414028d15..22c8e12b0e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -266,7 +266,12 @@ jobs:if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'run: pnpm lab:readiness:check- - run: pnpm vitest run --project ui+ # A child process isolates engine state, not CPU or memory contention.+ # FR23 must wait until the other UI workers have exited (AST-036).+ - name: Run UI tests and isolated Markdown benchmark+ run: |+ pnpm vitest run --project ui --exclude packages/core/src/Markdown/Markdown.fr23.perf.test.ts+ pnpm vitest run --project ui packages/core/src/Markdown/Markdown.fr23.perf.test.ts --no-file-parallelismif: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'test-node:diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.ymlindex 32797bd17a..cf3fa3ec74 100644--- a/.github/workflows/deploy.yml+++ b/.github/workflows/deploy.yml@@ -93,7 +93,11 @@ jobs:- uses: ./.github/actions/setup- - run: pnpm vitest run --project ui+ # Match the PR gate: FR23 needs an idle test runner, not just a new process.+ - name: Run UI tests and isolated Markdown benchmark+ run: |+ pnpm vitest run --project ui --exclude packages/core/src/Markdown/Markdown.fr23.perf.test.ts+ pnpm vitest run --project ui packages/core/src/Markdown/Markdown.fr23.perf.test.ts --no-file-parallelismtest-node:runs-on: 2-core-ubuntu-armdiff --git a/packages/core/src/Markdown/Markdown.fr23.bench.ts b/packages/core/src/Markdown/Markdown.fr23.bench.tsindex cd72986b26..819265672c 100644--- a/packages/core/src/Markdown/Markdown.fr23.bench.ts+++ b/packages/core/src/Markdown/Markdown.fr23.bench.ts@@ -7,27 +7,22 @@* @position Benchmark body for Markdown.fr23.perf.test.ts; not a test itself** Run as its own process, one measurement at a time, by- * `Markdown.fr23.perf.test.ts`. Nothing else may execute here: the ratio is- * sensitive both to other work competing for the machine and to engine state- * left behind by other plugin configurations in the same process, and this- * file exists so neither can reach it.+ * `Markdown.fr23.perf.test.ts`. The fresh process isolates engine state left+ * behind by other plugin configurations. The harness must also run separately+ * from other tests: a child process still shares machine resources, and CPU+ * time does not remove cache or memory-bandwidth contention.** The timed region is exactly the spec's protocol — ten untimed warmups,* then nine alternating paired rounds averaging 20 iterations a side,* reported as the median of the nine paired ratios — against the omitted- * baseline, for a stable list and a recreated equivalent one.+ * baseline, for a stable list and a recreated equivalent one. Each ratio+ * comes from one sample, never a selection by the fastest baseline.*- * That protocol is run under a validity gate. A paired design cancels a- * machine that is uniformly slow, but not one that deschedules the two- * sides unequally, and this process can still be sharing cores with the- * test runner that spawned it. So each attempt also records how much the- * BASELINE's own per-round time moved: when the same work varies by more- * than a few percent across the attempt, the machine was not holding still- * and the attempt is discarded and retried, not averaged in. The accepted- * figure is one untouched run of the spec's statistic; only invalid- * attempts are dropped, and exhausting them fails loudly.+ * The harness retries a fresh process only when the A/A control is invalid.+ * A helper-overhead failure is never retried to obtain a passing sample.*/+import {measureMarkdownPairedRatio} from './Markdown.fr23.sampling';import {parseMarkdown} from './parser';import {createMarkdownPlugin} from './plugins';import type {MarkdownExtensionNode} from './plugins';@@ -61,114 +56,6 @@ function benchmarkDocument(sections: number): string {).join('\n\n');}-function median(values: ReadonlyArray<number>): number {- const sorted = [...values].sort((left, right) => left - right);- return sorted[Math.floor(sorted.length / 2)];-}--/**- * The CPU time this process spent on 20 iterations, in microseconds.- *- * CPU time, not wall clock: this process can be sharing cores with the test- * runner that spawned it, and a wall clock counts the milliseconds it spent- * descheduled as if the work had taken them. It does not distribute that- * penalty evenly between the two sides of a paired round either, which is- * exactly the error the pairing is supposed to remove. CPU time counts only- * cycles actually spent on this work, so the ratio reports the helpers- * whether or not the machine is busy.- */-function measureAverage(callback: () => number): number {- const start = process.cpuUsage();- for (let iteration = 0; iteration < 20; iteration++) {- benchmarkSink ^= callback();- }- const used = process.cpuUsage(start);- return (used.user + used.system) / 20;-}--interface Attempt {- /** The spec's statistic: the median of the nine paired ratios. */- readonly ratio: number;- /**- * The baseline's own median round time. Identical work every attempt, so- * it says nothing about the helpers and everything about how much of the- * machine this attempt actually had.- */- readonly baselineCost: number;-}--/** Ten untimed warmups, then the median of nine alternating paired rounds. */-function pairedMedianRatio(- baseline: () => number,- candidate: () => number,-): Attempt {- for (let warmup = 0; warmup < 10; warmup++) {- benchmarkSink ^= baseline();- benchmarkSink ^= candidate();- }- const ratios: number[] = [];- const baselineTimes: number[] = [];- for (let round = 0; round < 9; round++) {- const baselineFirst = round % 2 === 0;- const first = measureAverage(baselineFirst ? baseline : candidate);- const second = measureAverage(baselineFirst ? candidate : baseline);- const baselineTime = baselineFirst ? first : second;- const candidateTime = baselineFirst ? second : first;- baselineTimes.push(baselineTime);- ratios.push(candidateTime / baselineTime);- }- void benchmarkSink;- return {ratio: median(ratios), baselineCost: median(baselineTimes)};-}--/** Sleeps without spinning, so a busy machine can finish what it is doing. */-function pause(milliseconds: number): void {- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);-}--/**- * The spec's statistic, taken from the least-contended attempt.- *- * Each attempt is one untouched run of the protocol — ten warmups, one- * nine-pair median — and nothing is averaged across them. What varies- * between attempts is only how much of the machine this process had, and- * `baselineCost` measures exactly that: the same baseline work, timed the- * same way, independent of anything the helpers do. Taking the attempt with- * the cheapest baseline therefore picks the run least disturbed by whatever- * else the box was doing, without touching the statistic itself.- *- * CPU time already removes descheduling, but not cache and memory-bandwidth- * pressure from a neighbour, which lands hardest on whichever side of the- * pair touches more memory. That is the residue this selection removes.- */-function leastContendedRatio(- baseline: () => number,- candidate: () => number,-): number {- let best: Attempt | undefined;- for (let attempt = 0; attempt < 5; attempt++) {- const measured = pairedMedianRatio(baseline, candidate);- if (best === undefined || measured.baselineCost < best.baselineCost) {- best = measured;- }- pause(250);- }- return (best as Attempt).ratio;-}--/**- * The same protocol with the SAME work on both sides.- *- * An A/A run has no overhead to find, so whatever it reports above 1.0 is- * the measurement's own error on this machine right now. It is the honest- * way to tell a helper regression from a busy box: without it, a contended- * run blames the helpers for time they never spent. The result bounds what- * any A/B number here can be trusted to resolve.- */-function controlRatio(baseline: () => number): number {- return leastContendedRatio(baseline, baseline);-}-/** Matches a prose identifier in every fixture section. */const identifierPlugin = createMarkdownPlugin({name: 'representative-identifiers',@@ -266,13 +153,12 @@ function measure(sections: number): {const withRecreatedList = () =>parseMarkdown(source, {plugins: [...representative]}).length;primePipeline([empty, withStableList, withRecreatedList]);- // The control is taken first: it is the cheapest way to learn the machine- // cannot measure, and taking it before the pair keeps it from being the- // one attempt that happened to land in a quiet moment.- const control = controlRatio(empty);+ // The A/A control measures identical work. Only a failed control can+ // invalidate this sample; neither helper ratio participates in selection.+ const control = measureMarkdownPairedRatio(empty, empty);return {- stable: leastContendedRatio(empty, withStableList),- recreated: leastContendedRatio(empty, withRecreatedList),+ stable: measureMarkdownPairedRatio(empty, withStableList),+ recreated: measureMarkdownPairedRatio(empty, withRecreatedList),control,};}diff --git a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts b/packages/core/src/Markdown/Markdown.fr23.perf.test.tsindex 04edd689d0..06f988cb1b 100644--- a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts+++ b/packages/core/src/Markdown/Markdown.fr23.perf.test.ts@@ -10,13 +10,11 @@* `Markdown.fr23.bench.ts` as a dedicated Node process, one section size at* a time, and only asserts on what that process reports.*- * It has to work that way. The ratio is sensitive to two things a test- * runner cannot hold still: other Vitest files competing for the machine,- * and engine state — polymorphic call sites through the parser — left by- * every other plugin configuration the worker already exercised. The same- * code measured ~1.17 alone and up to ~1.54 beside the rest of the suite.- * A fresh process with nothing else in it removes both, so the number the- * budget is applied to reflects the helpers rather than the scheduler.+ * A fresh process isolates engine state left by other plugin configurations,+ * but it still shares CPU and memory with other Vitest workers. CI and Deploy+ * exclude this file from the parallel UI suite, then run it alone after those+ * workers exit, as the spec's isolated-process-and-runner protocol requires.+ * Local measurements need the same isolation from other test and build work.** The timed region inside that process is the spec's protocol, unsmoothed:* ten untimed warmups, then one median of nine alternating paired rounds,@@ -29,6 +27,7 @@ import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';import path from 'node:path';import {fileURLToPath} from 'node:url';import {afterAll, describe, expect, it} from 'vitest';+import {measureMarkdownPairedRatio} from './Markdown.fr23.sampling';interface BenchmarkResult {readonly sections: number;@@ -135,6 +134,45 @@ function benchmarkArgv(sections: number): ReadonlyArray<string> {}describe('Markdown FR23 helper overhead', () => {+ it('reports one nine-pair median without selecting the cheapest baseline', () => {+ // Paired ratios: four 1.1s, one 1.2, four 10s. The median is 1.2,+ // whereas dividing the separate medians would give 10. Later attempts+ // have a cheaper baseline but a ratio of 2: selecting them biases the+ // result even though the selector never inspects candidate times.+ const baselineCosts = [1000, 1000, 1000, 1000, 10, 10, 10, 10, 10];+ const candidateCosts = [1100, 1100, 1100, 1100, 12, 100, 100, 100, 100];+ const calls: string[] = [];+ let elapsed = 0;+ const callback = (name: string, costs: ReadonlyArray<number>) => {+ let count = 0;+ return () => {+ count++;+ calls.push(name);+ elapsed +=+ count <= 10+ ? 100+ : (costs[Math.floor((count - 11) / 20)] ?? (name === 'B' ? 5 : 10));+ return count;+ };+ };++ expect(+ measureMarkdownPairedRatio(+ callback('B', baselineCosts),+ callback('C', candidateCosts),+ () => elapsed,+ ),+ ).toBeCloseTo(1.2, 10);+ expect(calls).toHaveLength(2 * (10 + 9 * 20));+ expect(calls.slice(0, 20).join('')).toBe('BC'.repeat(10));+ for (let round = 0; round < 9; round++) {+ const pair = ['B'.repeat(20), 'C'.repeat(20)];+ expect(calls.slice(20 + round * 40, 60 + round * 40).join('')).toBe(+ (round % 2 === 0 ? pair : pair.reverse()).join(''),+ );+ }+ });+it.each([200, 500])('keeps the representative three-helper set within 25 percent of the empty pipeline at %i sections',sections => {diff --git a/packages/core/src/Markdown/Markdown.fr23.sampling.ts b/packages/core/src/Markdown/Markdown.fr23.sampling.tsnew file mode 100644index 0000000000..462a10c903--- /dev/null+++ b/packages/core/src/Markdown/Markdown.fr23.sampling.ts@@ -0,0 +1,49 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Markdown.fr23.sampling.ts+ * @input Baseline/candidate callbacks and a microsecond CPU clock+ * @output The FR23 paired-median ratio+ * @position Test-only sampling for Markdown.fr23.bench.ts+ */++function cpuTime(): number {+ const used = process.cpuUsage();+ return used.user + used.system;+}++function median(values: ReadonlyArray<number>): number {+ const sorted = [...values].sort((left, right) => left - right);+ return sorted[Math.floor(sorted.length / 2)];+}++export function measureMarkdownPairedRatio(+ baseline: () => number,+ candidate: () => number,+ now: () => number = cpuTime,+): number {+ let sink = 0;+ const average = (callback: () => number): number => {+ const start = now();+ for (let iteration = 0; iteration < 20; iteration++) {+ sink ^= callback();+ }+ return (now() - start) / 20;+ };+ for (let warmup = 0; warmup < 10; warmup++) {+ sink ^= baseline();+ sink ^= candidate();+ }+ const ratios: number[] = [];+ for (let round = 0; round < 9; round++) {+ const baselineFirst = round % 2 === 0;+ const first = average(baselineFirst ? baseline : candidate);+ const second = average(baselineFirst ? candidate : baseline);+ const baselineTime = baselineFirst ? first : second;+ const candidateTime = baselineFirst ? second : first;+ ratios.push(candidateTime / baselineTime);+ }+ void sink;+ // Selecting a sample by its cheapest baseline also selects denominator noise.+ return median(ratios);+}