Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

test(markdown): isolate FR23 and remove biased sample selection (#6402)

Cindy Zhang400998fdbe5f
diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs
index 85883ae00a..9c32a9d9a0 100644
--- a/.github/scripts/ci-test-routing.test.mjs
+++ b/.github/scripts/ci-test-routing.test.mjs
@@ -4,7 +4,8 @@
* @file ci-test-routing.test.mjs
*
* The contract between `vitest.config.ts` and the CI jobs that run it, in both
- * workflows that run the suite.
+ * workflows that run the suite. FR23 must run after the other UI workers exit
+ * so its helper-overhead measurement does not compete with unrelated tests.
*
* This repo has now lost test coverage to routing twice. First a suite
* belonged to no Vitest project, so `pnpm test` never collected it. Then the
@@ -110,6 +111,24 @@ describe.each(Object.entries(WORKFLOWS))(
}
});
+ it('runs FR23 only after the parallel UI suite has exited', () => {
+ const benchmark = 'packages/core/src/Markdown/Markdown.fr23.perf.test.ts';
+ const job = workflow.jobs['test-ui'];
+ const suites = job.steps.filter(step =>
+ step.run?.includes('--project ui'),
+ );
+
+ // One conditional step preserves the existing scope gate for both runs.
+ // Exact foreground commands also reject backgrounding or ignored exits.
+ expect(suites).toHaveLength(1);
+ expect(suites[0].run.trim().split('\n')).toEqual([
+ `pnpm vitest run --project ui --exclude ${benchmark}`,
+ `pnpm vitest run --project ui ${benchmark} --no-file-parallelism`,
+ ]);
+ expect(job['continue-on-error']).not.toBe(true);
+ expect(suites[0]['continue-on-error']).not.toBe(true);
+ });
+
it('gives no lane the whole suite', () => {
// A bare `pnpm test` runs every project in one job — the shape that hit
// the wall.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0414028d15..22c8e12b0e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -266,7 +266,12 @@ jobs:
if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
run: pnpm lab:readiness:check
- - run: pnpm vitest run --project ui
+ # A child process isolates engine state, not CPU or memory contention.
+ # FR23 must wait until the other UI workers have exited (AST-036).
+ - name: Run UI tests and isolated Markdown benchmark
+ run: |
+ pnpm vitest run --project ui --exclude packages/core/src/Markdown/Markdown.fr23.perf.test.ts
+ pnpm vitest run --project ui packages/core/src/Markdown/Markdown.fr23.perf.test.ts --no-file-parallelism
if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
test-node:
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 32797bd17a..cf3fa3ec74 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -93,7 +93,11 @@ jobs:
- uses: ./.github/actions/setup
- - run: pnpm vitest run --project ui
+ # Match the PR gate: FR23 needs an idle test runner, not just a new process.
+ - name: Run UI tests and isolated Markdown benchmark
+ run: |
+ pnpm vitest run --project ui --exclude packages/core/src/Markdown/Markdown.fr23.perf.test.ts
+ pnpm vitest run --project ui packages/core/src/Markdown/Markdown.fr23.perf.test.ts --no-file-parallelism
test-node:
runs-on: 2-core-ubuntu-arm
diff --git a/packages/core/src/Markdown/Markdown.fr23.bench.ts b/packages/core/src/Markdown/Markdown.fr23.bench.ts
index cd72986b26..819265672c 100644
--- a/packages/core/src/Markdown/Markdown.fr23.bench.ts
+++ b/packages/core/src/Markdown/Markdown.fr23.bench.ts
@@ -7,27 +7,22 @@
* @position Benchmark body for Markdown.fr23.perf.test.ts; not a test itself
*
* Run as its own process, one measurement at a time, by
- * `Markdown.fr23.perf.test.ts`. Nothing else may execute here: the ratio is
- * sensitive both to other work competing for the machine and to engine state
- * left behind by other plugin configurations in the same process, and this
- * file exists so neither can reach it.
+ * `Markdown.fr23.perf.test.ts`. The fresh process isolates engine state left
+ * behind by other plugin configurations. The harness must also run separately
+ * from other tests: a child process still shares machine resources, and CPU
+ * time does not remove cache or memory-bandwidth contention.
*
* The timed region is exactly the spec's protocol — ten untimed warmups,
* then nine alternating paired rounds averaging 20 iterations a side,
* reported as the median of the nine paired ratios — against the omitted
- * baseline, for a stable list and a recreated equivalent one.
+ * baseline, for a stable list and a recreated equivalent one. Each ratio
+ * comes from one sample, never a selection by the fastest baseline.
*
- * That protocol is run under a validity gate. A paired design cancels a
- * machine that is uniformly slow, but not one that deschedules the two
- * sides unequally, and this process can still be sharing cores with the
- * test runner that spawned it. So each attempt also records how much the
- * BASELINE's own per-round time moved: when the same work varies by more
- * than a few percent across the attempt, the machine was not holding still
- * and the attempt is discarded and retried, not averaged in. The accepted
- * figure is one untouched run of the spec's statistic; only invalid
- * attempts are dropped, and exhausting them fails loudly.
+ * The harness retries a fresh process only when the A/A control is invalid.
+ * A helper-overhead failure is never retried to obtain a passing sample.
*/
+import {measureMarkdownPairedRatio} from './Markdown.fr23.sampling';
import {parseMarkdown} from './parser';
import {createMarkdownPlugin} from './plugins';
import type {MarkdownExtensionNode} from './plugins';
@@ -61,114 +56,6 @@ function benchmarkDocument(sections: number): string {
).join('\n\n');
}
-function median(values: ReadonlyArray<number>): number {
- const sorted = [...values].sort((left, right) => left - right);
- return sorted[Math.floor(sorted.length / 2)];
-}
-
-/**
- * The CPU time this process spent on 20 iterations, in microseconds.
- *
- * CPU time, not wall clock: this process can be sharing cores with the test
- * runner that spawned it, and a wall clock counts the milliseconds it spent
- * descheduled as if the work had taken them. It does not distribute that
- * penalty evenly between the two sides of a paired round either, which is
- * exactly the error the pairing is supposed to remove. CPU time counts only
- * cycles actually spent on this work, so the ratio reports the helpers
- * whether or not the machine is busy.
- */
-function measureAverage(callback: () => number): number {
- const start = process.cpuUsage();
- for (let iteration = 0; iteration < 20; iteration++) {
- benchmarkSink ^= callback();
- }
- const used = process.cpuUsage(start);
- return (used.user + used.system) / 20;
-}
-
-interface Attempt {
- /** The spec's statistic: the median of the nine paired ratios. */
- readonly ratio: number;
- /**
- * The baseline's own median round time. Identical work every attempt, so
- * it says nothing about the helpers and everything about how much of the
- * machine this attempt actually had.
- */
- readonly baselineCost: number;
-}
-
-/** Ten untimed warmups, then the median of nine alternating paired rounds. */
-function pairedMedianRatio(
- baseline: () => number,
- candidate: () => number,
-): Attempt {
- for (let warmup = 0; warmup < 10; warmup++) {
- benchmarkSink ^= baseline();
- benchmarkSink ^= candidate();
- }
- const ratios: number[] = [];
- const baselineTimes: number[] = [];
- for (let round = 0; round < 9; round++) {
- const baselineFirst = round % 2 === 0;
- const first = measureAverage(baselineFirst ? baseline : candidate);
- const second = measureAverage(baselineFirst ? candidate : baseline);
- const baselineTime = baselineFirst ? first : second;
- const candidateTime = baselineFirst ? second : first;
- baselineTimes.push(baselineTime);
- ratios.push(candidateTime / baselineTime);
- }
- void benchmarkSink;
- return {ratio: median(ratios), baselineCost: median(baselineTimes)};
-}
-
-/** Sleeps without spinning, so a busy machine can finish what it is doing. */
-function pause(milliseconds: number): void {
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
-}
-
-/**
- * The spec's statistic, taken from the least-contended attempt.
- *
- * Each attempt is one untouched run of the protocol — ten warmups, one
- * nine-pair median — and nothing is averaged across them. What varies
- * between attempts is only how much of the machine this process had, and
- * `baselineCost` measures exactly that: the same baseline work, timed the
- * same way, independent of anything the helpers do. Taking the attempt with
- * the cheapest baseline therefore picks the run least disturbed by whatever
- * else the box was doing, without touching the statistic itself.
- *
- * CPU time already removes descheduling, but not cache and memory-bandwidth
- * pressure from a neighbour, which lands hardest on whichever side of the
- * pair touches more memory. That is the residue this selection removes.
- */
-function leastContendedRatio(
- baseline: () => number,
- candidate: () => number,
-): number {
- let best: Attempt | undefined;
- for (let attempt = 0; attempt < 5; attempt++) {
- const measured = pairedMedianRatio(baseline, candidate);
- if (best === undefined || measured.baselineCost < best.baselineCost) {
- best = measured;
- }
- pause(250);
- }
- return (best as Attempt).ratio;
-}
-
-/**
- * The same protocol with the SAME work on both sides.
- *
- * An A/A run has no overhead to find, so whatever it reports above 1.0 is
- * the measurement's own error on this machine right now. It is the honest
- * way to tell a helper regression from a busy box: without it, a contended
- * run blames the helpers for time they never spent. The result bounds what
- * any A/B number here can be trusted to resolve.
- */
-function controlRatio(baseline: () => number): number {
- return leastContendedRatio(baseline, baseline);
-}
-
/** Matches a prose identifier in every fixture section. */
const identifierPlugin = createMarkdownPlugin({
name: 'representative-identifiers',
@@ -266,13 +153,12 @@ function measure(sections: number): {
const withRecreatedList = () =>
parseMarkdown(source, {plugins: [...representative]}).length;
primePipeline([empty, withStableList, withRecreatedList]);
- // The control is taken first: it is the cheapest way to learn the machine
- // cannot measure, and taking it before the pair keeps it from being the
- // one attempt that happened to land in a quiet moment.
- const control = controlRatio(empty);
+ // The A/A control measures identical work. Only a failed control can
+ // invalidate this sample; neither helper ratio participates in selection.
+ const control = measureMarkdownPairedRatio(empty, empty);
return {
- stable: leastContendedRatio(empty, withStableList),
- recreated: leastContendedRatio(empty, withRecreatedList),
+ stable: measureMarkdownPairedRatio(empty, withStableList),
+ recreated: measureMarkdownPairedRatio(empty, withRecreatedList),
control,
};
}
diff --git a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts b/packages/core/src/Markdown/Markdown.fr23.perf.test.ts
index 04edd689d0..06f988cb1b 100644
--- a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts
+++ b/packages/core/src/Markdown/Markdown.fr23.perf.test.ts
@@ -10,13 +10,11 @@
* `Markdown.fr23.bench.ts` as a dedicated Node process, one section size at
* a time, and only asserts on what that process reports.
*
- * It has to work that way. The ratio is sensitive to two things a test
- * runner cannot hold still: other Vitest files competing for the machine,
- * and engine state — polymorphic call sites through the parser — left by
- * every other plugin configuration the worker already exercised. The same
- * code measured ~1.17 alone and up to ~1.54 beside the rest of the suite.
- * A fresh process with nothing else in it removes both, so the number the
- * budget is applied to reflects the helpers rather than the scheduler.
+ * A fresh process isolates engine state left by other plugin configurations,
+ * but it still shares CPU and memory with other Vitest workers. CI and Deploy
+ * exclude this file from the parallel UI suite, then run it alone after those
+ * workers exit, as the spec's isolated-process-and-runner protocol requires.
+ * Local measurements need the same isolation from other test and build work.
*
* The timed region inside that process is the spec's protocol, unsmoothed:
* ten untimed warmups, then one median of nine alternating paired rounds,
@@ -29,6 +27,7 @@ import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {afterAll, describe, expect, it} from 'vitest';
+import {measureMarkdownPairedRatio} from './Markdown.fr23.sampling';
interface BenchmarkResult {
readonly sections: number;
@@ -135,6 +134,45 @@ function benchmarkArgv(sections: number): ReadonlyArray<string> {
}
describe('Markdown FR23 helper overhead', () => {
+ it('reports one nine-pair median without selecting the cheapest baseline', () => {
+ // Paired ratios: four 1.1s, one 1.2, four 10s. The median is 1.2,
+ // whereas dividing the separate medians would give 10. Later attempts
+ // have a cheaper baseline but a ratio of 2: selecting them biases the
+ // result even though the selector never inspects candidate times.
+ const baselineCosts = [1000, 1000, 1000, 1000, 10, 10, 10, 10, 10];
+ const candidateCosts = [1100, 1100, 1100, 1100, 12, 100, 100, 100, 100];
+ const calls: string[] = [];
+ let elapsed = 0;
+ const callback = (name: string, costs: ReadonlyArray<number>) => {
+ let count = 0;
+ return () => {
+ count++;
+ calls.push(name);
+ elapsed +=
+ count <= 10
+ ? 100
+ : (costs[Math.floor((count - 11) / 20)] ?? (name === 'B' ? 5 : 10));
+ return count;
+ };
+ };
+
+ expect(
+ measureMarkdownPairedRatio(
+ callback('B', baselineCosts),
+ callback('C', candidateCosts),
+ () => elapsed,
+ ),
+ ).toBeCloseTo(1.2, 10);
+ expect(calls).toHaveLength(2 * (10 + 9 * 20));
+ expect(calls.slice(0, 20).join('')).toBe('BC'.repeat(10));
+ for (let round = 0; round < 9; round++) {
+ const pair = ['B'.repeat(20), 'C'.repeat(20)];
+ expect(calls.slice(20 + round * 40, 60 + round * 40).join('')).toBe(
+ (round % 2 === 0 ? pair : pair.reverse()).join(''),
+ );
+ }
+ });
+
it.each([200, 500])(
'keeps the representative three-helper set within 25 percent of the empty pipeline at %i sections',
sections => {
diff --git a/packages/core/src/Markdown/Markdown.fr23.sampling.ts b/packages/core/src/Markdown/Markdown.fr23.sampling.ts
new file mode 100644
index 0000000000..462a10c903
--- /dev/null
+++ b/packages/core/src/Markdown/Markdown.fr23.sampling.ts
@@ -0,0 +1,49 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Markdown.fr23.sampling.ts
+ * @input Baseline/candidate callbacks and a microsecond CPU clock
+ * @output The FR23 paired-median ratio
+ * @position Test-only sampling for Markdown.fr23.bench.ts
+ */
+
+function cpuTime(): number {
+ const used = process.cpuUsage();
+ return used.user + used.system;
+}
+
+function median(values: ReadonlyArray<number>): number {
+ const sorted = [...values].sort((left, right) => left - right);
+ return sorted[Math.floor(sorted.length / 2)];
+}
+
+export function measureMarkdownPairedRatio(
+ baseline: () => number,
+ candidate: () => number,
+ now: () => number = cpuTime,
+): number {
+ let sink = 0;
+ const average = (callback: () => number): number => {
+ const start = now();
+ for (let iteration = 0; iteration < 20; iteration++) {
+ sink ^= callback();
+ }
+ return (now() - start) / 20;
+ };
+ for (let warmup = 0; warmup < 10; warmup++) {
+ sink ^= baseline();
+ sink ^= candidate();
+ }
+ const ratios: number[] = [];
+ for (let round = 0; round < 9; round++) {
+ const baselineFirst = round % 2 === 0;
+ const first = average(baselineFirst ? baseline : candidate);
+ const second = average(baselineFirst ? candidate : baseline);
+ const baselineTime = baselineFirst ? first : second;
+ const candidateTime = baselineFirst ? second : first;
+ ratios.push(candidateTime / baselineTime);
+ }
+ void sink;
+ // Selecting a sample by its cheapest baseline also selects denominator noise.
+ return median(ratios);
+}