An open source design system that's fully customizable and agent ready
TypeScript
test(Markdown): decide each FR23 path by its own majority of nine attempts (#7462)
Cindy Zhangfb02ee9a5885
diff --git a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts b/packages/core/src/Markdown/Markdown.fr23.perf.test.tsindex 646267182f..66ad28d7a1 100644--- a/packages/core/src/Markdown/Markdown.fr23.perf.test.ts+++ b/packages/core/src/Markdown/Markdown.fr23.perf.test.ts@@ -19,8 +19,10 @@* The timed region inside each process is the spec's protocol, unsmoothed:* ten untimed warmups, then one median of nine alternating paired rounds,* for a stable plugin list and a recreated equivalent one, against the- * omitted baseline. One over-budget A/B triggers two fresh-process attempts;- * the unchanged budget fails only when at least two of the three exceed it.+ * omitted baseline. A first attempt within budget on both paths certifies+ * them. Otherwise up to nine fresh-process attempts decide each path on its+ * own: a path fails the unchanged budget when a majority of the nine (five)+ * exceed it, and collection stops as soon as both paths are decided.*/import {execFileSync} from 'node:child_process';@@ -166,18 +168,64 @@ function logBenchmarkResult(result: BenchmarkResult, attempt: number): void {);}+/** The two paths FR23 measures; each is held to the budget on its own. */+const FR23_PATHS = ['stable', 'recreated'] as const;+type FR23Path = (typeof FR23_PATHS)[number];++/**+ * Fresh-process attempts that decide a path after a noisy first one. On CI a+ * clean representative set measures about 1.19 at 500 sections with about+ * ±0.04 of sample noise, so a single attempt lands over the budget roughly+ * one time in ten; a majority of nine does so well under one time in a+ * thousand, while a slowdown of 0.1 or more still exceeds it nearly always.+ */+const FR23_ATTEMPTS = 9;+const FR23_MAJORITY = Math.floor(FR23_ATTEMPTS / 2) + 1;++function attemptsOverBudget(+ attempts: ReadonlyArray<BenchmarkResult>,+ fr23Path: FR23Path,+): number {+ return attempts.filter(result => result[fr23Path] > FR23_BUDGET).length;+}++/** Whether the remaining attempts can no longer change this path's verdict. */+function pathDecided(+ attempts: ReadonlyArray<BenchmarkResult>,+ fr23Path: FR23Path,+): boolean {+ const over = attemptsOverBudget(attempts, fr23Path);+ return over >= FR23_MAJORITY || attempts.length - over >= FR23_MAJORITY;+}+function collectFR23Attempts(sections: number,runAttempt: (sections: number) => BenchmarkResult = runSettledBenchmark,): ReadonlyArray<BenchmarkResult> {const attempts = [runAttempt(sections)];- if (exceedsFR23Budget(attempts[0])) {- attempts.push(runAttempt(sections));+ if (!exceedsFR23Budget(attempts[0])) {+ return attempts;+ }+ while (+ attempts.length < FR23_ATTEMPTS &&+ !FR23_PATHS.every(fr23Path => pathDecided(attempts, fr23Path))+ ) {attempts.push(runAttempt(sections));}return attempts;}+/** The paths whose attempts exceed the budget in a majority of nine. */+function pathsOverBudget(+ attempts: ReadonlyArray<BenchmarkResult>,+): ReadonlyArray<FR23Path> {+ return FR23_PATHS.filter(fr23Path =>+ attempts.length > 1+ ? attemptsOverBudget(attempts, fr23Path) >= FR23_MAJORITY+ : attemptsOverBudget(attempts, fr23Path) > 0,+ );+}+describe('Markdown FR23 helper overhead', () => {it('reports one nine-pair median without selecting the cheapest baseline', () => {// Paired ratios: four 1.1s, one 1.2, four 10s. The median is 1.2,@@ -218,35 +266,104 @@ describe('Markdown FR23 helper overhead', () => {}});- it('retries one over-budget A/B twice and uses the three-attempt majority', () => {+ describe('deciding each path from fresh-process attempts', () => {const result = (stable: number, recreated: number): BenchmarkResult => ({- sections: 200,+ sections: 500,stable,recreated,control: 1,});- const passFirst = [result(1.2, 1.22)];- expect(collectFR23Attempts(200, () => passFirst.shift()!)).toHaveLength(1);-- const retryResults = [- result(1.27, 1.1),- result(1.2, 1.22),- result(1.18, 1.19),- ];- const attempts = collectFR23Attempts(200, () => retryResults.shift()!);- expect(attempts).toHaveLength(3);- expect(attempts.filter(exceedsFR23Budget)).toHaveLength(1);-- const majorityResults = [- result(1.27, 1.1),- result(1.2, 1.26),- result(1.18, 1.19),- ];- expect(- collectFR23Attempts(200, () => majorityResults.shift()!).filter(- exceedsFR23Budget,- ),- ).toHaveLength(2);+ const run = (results: BenchmarkResult[]) => {+ const queue = [...results];+ let calls = 0;+ const attempts = collectFR23Attempts(500, () => {+ calls++;+ const next = queue.shift();+ if (next == null) {+ throw new Error('asked for an unplanned attempt');+ }+ return next;+ });+ return {attempts, calls, failing: pathsOverBudget(attempts)};+ };++ it('certifies both paths from one attempt within budget', () => {+ const {calls, failing} = run([result(1.2, 1.22)]);+ expect(calls).toBe(1);+ expect(failing).toEqual([]);+ });++ it('fails a single attempt over budget only if no retry ran', () => {+ expect(pathsOverBudget([result(1.26, 1.2)])).toEqual(['stable']);+ });++ it('judges each path by its own majority, not either path per attempt', () => {+ // Main run 38393555595: stable was over in attempt 1 and recreated in+ // attempt 2, which the old either-path-per-attempt count read as a+ // two-of-three failure although neither path was over in a majority.+ const {attempts, failing} = run([+ result(1.256, 1.201),+ result(1.145, 1.275),+ result(1.183, 1.187),+ result(1.18, 1.2),+ result(1.19, 1.21),+ result(1.2, 1.22),+ result(1.17, 1.19),+ ]);+ // Each path needs five attempts within budget; each had one over.+ expect(attempts).toHaveLength(6);+ expect(failing).toEqual([]);+ });++ it('stops as soon as both paths are decided', () => {+ // Recreated is decided after five passes, stable after one more.+ const {calls} = run([+ result(1.3, 1.2),+ result(1.2, 1.2),+ result(1.2, 1.2),+ result(1.2, 1.2),+ result(1.2, 1.2),+ result(1.2, 1.2),+ ]);+ expect(calls).toBe(6);+ });++ it('fails a path that is over budget in a majority of nine', () => {+ const {attempts, failing} = run([+ result(1.3, 1.2),+ result(1.27, 1.2),+ result(1.2, 1.2),+ result(1.26, 1.21),+ result(1.22, 1.2),+ result(1.28, 1.19),+ result(1.26, 1.2),+ ]);+ expect(attempts).toHaveLength(7);+ expect(failing).toEqual(['stable']);+ });++ it('fails a consistent slowdown on both paths after five attempts', () => {+ const {calls, failing} = run(+ Array.from({length: 9}, () => result(1.4, 1.42)),+ );+ expect(calls).toBe(5);+ expect(failing).toEqual(['stable', 'recreated']);+ });++ it('never runs more than nine attempts', () => {+ const {calls} = run([+ result(1.3, 1.3),+ result(1.2, 1.2),+ result(1.3, 1.3),+ result(1.2, 1.2),+ result(1.3, 1.3),+ result(1.2, 1.2),+ result(1.3, 1.3),+ result(1.2, 1.2),+ result(1.3, 1.3),+ ]);+ expect(calls).toBe(9);+ });});it.each([200, 500])(@@ -257,28 +374,32 @@ describe('Markdown FR23 helper overhead', () => {// attempt counts: a value over budget is machine noise, so that process// is replaced before its A/B result can certify or fail the helpers.//- // A single over-budget A/B is also noisy at this threshold. Retry that- // result twice in fresh processes and fail only when a majority (at- // least two of the three independent attempts) exceeds the unchanged- // 1.25 budget.+ // A single over-budget A/B is also noisy at this threshold. Each path+ // is then decided by a majority of up to nine fresh-process attempts+ // against the unchanged 1.25 budget.const attempts = collectFR23Attempts(sections);- if (attempts.length === 3) {+ if (attempts.length > 1) {console.log(` ${sections} sections: first A/B exceeded ${FR23_BUDGET};` +- ' retried twice in fresh processes',+ ` decided each path from ${attempts.length} fresh processes`,);}attempts.forEach((result, index) =>logBenchmarkResult(result, index + 1),);- const overBudget = attempts.filter(exceedsFR23Budget).length;+ const failing = pathsOverBudget(attempts);expect(- overBudget,- `${overBudget} of ${attempts.length} fresh FR23 attempts exceeded ` +- `the ${FR23_BUDGET} budget`,- ).toBeLessThan(2);+ failing,+ failing+ .map(+ fr23Path =>+ `${fr23Path}: ${attemptsOverBudget(attempts, fr23Path)} of ` ++ `${attempts.length} fresh FR23 attempts exceeded the ${FR23_BUDGET} budget`,+ )+ .join('; '),+ ).toEqual([]);},- 300_000,+ 600_000,);it('fails loudly when the benchmark process cannot produce a result', () => {