An open source design system that's fully customizable and agent ready
TypeScript
ci: the Chromium interaction proofs run in as few legs as fit ten minutes each (#10258)
Cindy Zhang010db3f355d9
diff --git a/.github/playwright-leg-durations.json b/.github/playwright-leg-durations.jsonindex 629aadd1ee..38639d6e30 100644--- a/.github/playwright-leg-durations.json+++ b/.github/playwright-leg-durations.json@@ -1,32 +1,74 @@{"interaction-chromium": {- "internal/interaction-harness/src/harness.interaction.browser.spec.ts": 118.3,- "internal/interaction-harness/src/storybook.interaction.browser.spec.ts": 1.6,- "packages/core/src/AlertDialog/__tests__/AlertDialog.interaction.browser.spec.ts": 20.3,- "packages/core/src/AppShell/__tests__/AppShellLayoutRoot.interaction.browser.spec.ts": 4.6,- "packages/core/src/ButtonGroup/__tests__/ButtonGroupCorners.interaction.browser.spec.ts": 0.9,- "packages/core/src/Chat/__tests__/ChatMessageAvatar.interaction.browser.spec.ts": 1.2,- "packages/core/src/DateInput/__tests__/DateInputFocusLeave.interaction.browser.spec.ts": 2.6,- "packages/core/src/DateInput/__tests__/MonthScroller.interaction.browser.spec.ts": 5.1,- "packages/core/src/DateInput/__tests__/MonthScrollerSteps.interaction.browser.spec.ts": 7.8,- "packages/core/src/DateRangeInput/__tests__/DateRangeInputReflow.interaction.browser.spec.ts": 3.5,- "packages/core/src/Field/__tests__/SearchKeyboard.interaction.browser.spec.ts": 5.6,- "packages/core/src/SegmentedControl/__tests__/SegmentedControl.interaction.browser.spec.ts": 85.9,- "packages/core/src/SideNav/__tests__/SideNavRailMotion.interaction.browser.spec.ts": 16.1,- "packages/core/src/Slider/__tests__/SliderFinePointer.interaction.browser.spec.ts": 5.6,- "packages/core/src/Slider/__tests__/SliderTouch.interaction.browser.spec.ts": 50.4,- "packages/core/src/Switch/__tests__/Switch.interaction.browser.spec.ts": 98.1,- "packages/core/src/Switch/__tests__/SwitchPointer.interaction.browser.spec.ts": 18.3,- "packages/core/src/TabList/__tests__/TabListOverflow.interaction.browser.spec.ts": 1.6,- "packages/core/src/TabList/__tests__/TabPressPointer.interaction.browser.spec.ts": 2.7,- "packages/core/src/TreeList/__tests__/TreeListGuideAlignment.interaction.browser.spec.ts": 2.9,- "packages/core/src/animator/__tests__/createAnimator.interaction.browser.spec.ts": 6.8,- "packages/core/src/gestures/__tests__/browser/gestures.interaction.browser.spec.ts": 41.7,- "packages/core/src/gestures/__tests__/browser/presentedHitTesting.interaction.browser.spec.ts": 4.5,- "packages/core/src/hooks/__tests__/usePager.interaction.browser.spec.ts": 38.5,- "packages/core/src/motion/__tests__/animatedLayout.interaction.browser.spec.ts": 24.4,- "packages/core/src/motion/__tests__/animatedLayoutAdopters.interaction.browser.spec.ts": 24.5,- "packages/core/src/motion/__tests__/animator.interaction.browser.spec.ts": 16.6+ "internal/interaction-harness/src/harness.interaction.browser.spec.ts": 135.6,+ "internal/interaction-harness/src/pointer.interaction.browser.spec.ts": 0.2,+ "internal/interaction-harness/src/storybook.interaction.browser.spec.ts": 4.2,+ "packages/core/src/AlertDialog/__tests__/AlertDialog.interaction.browser.spec.ts": 21.1,+ "packages/core/src/AppShell/__tests__/AppShellLayoutRoot.interaction.browser.spec.ts": 6.4,+ "packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts": 16.5,+ "packages/core/src/BottomSheet/__tests__/BottomSheetKeyboardInset.interaction.browser.spec.ts": 21.9,+ "packages/core/src/Button/__tests__/ButtonPress.interaction.browser.spec.ts": 10.7,+ "packages/core/src/Button/__tests__/ButtonPressRelease.interaction.browser.spec.ts": 17.9,+ "packages/core/src/Button/__tests__/ButtonPressTiming.interaction.browser.spec.ts": 8.0,+ "packages/core/src/ButtonGroup/__tests__/ButtonGroupCorners.interaction.browser.spec.ts": 1.1,+ "packages/core/src/ButtonGroup/__tests__/ButtonGroupTracking.interaction.browser.spec.ts": 23.9,+ "packages/core/src/Chat/__tests__/ChatMessageAvatar.interaction.browser.spec.ts": 1.6,+ "packages/core/src/DateInput/__tests__/DateInputFocusLeave.interaction.browser.spec.ts": 3.5,+ "packages/core/src/DateInput/__tests__/MonthScroller.interaction.browser.spec.ts": 5.8,+ "packages/core/src/DateInput/__tests__/MonthScrollerSteps.interaction.browser.spec.ts": 8.5,+ "packages/core/src/DateInput/__tests__/Wheel.interaction.browser.spec.ts": 47.9,+ "packages/core/src/DateInput/__tests__/WheelActivation.interaction.browser.spec.ts": 4.6,+ "packages/core/src/DateRangeInput/__tests__/DateRangeInputReflow.interaction.browser.spec.ts": 4.4,+ "packages/core/src/Dialog/__tests__/DialogTouchEntrance.interaction.browser.spec.ts": 11.2,+ "packages/core/src/Drawer/__tests__/DrawerTouchEntrance.interaction.browser.spec.ts": 9.7,+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuDragHighlight.interaction.browser.spec.ts": 19.2,+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuEntranceHold.interaction.browser.spec.ts": 5.0,+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuTouchHandoff.interaction.browser.spec.ts": 5.6,+ "packages/core/src/Field/__tests__/SearchKeyboard.interaction.browser.spec.ts": 8.6,+ "packages/core/src/Indicator/__tests__/BoxHold.interaction.browser.spec.ts": 73.2,+ "packages/core/src/Indicator/__tests__/TouchTargets.interaction.browser.spec.ts": 11.2,+ "packages/core/src/Layer/__tests__/inputHold.interaction.browser.spec.ts": 11.0,+ "packages/core/src/Layer/__tests__/keyboardCappedLists.interaction.browser.spec.ts": 12.2,+ "packages/core/src/Layer/__tests__/preserveAnimations.interaction.browser.spec.ts": 0.1,+ "packages/core/src/Layout/__tests__/collapsedPanel.interaction.browser.spec.ts": 2.1,+ "packages/core/src/List/__tests__/ListRowPress.interaction.browser.spec.ts": 16.3,+ "packages/core/src/List/__tests__/ListRowPressPaths.interaction.browser.spec.ts": 9.7,+ "packages/core/src/List/__tests__/ListTouchHitArea.interaction.browser.spec.ts": 26.0,+ "packages/core/src/Popover/__tests__/PopoverKeyboard.interaction.browser.spec.ts": 12.3,+ "packages/core/src/Resizable/__tests__/grabZone.interaction.browser.spec.ts": 24.6,+ "packages/core/src/SegmentedControl/__tests__/SegmentedControl.interaction.browser.spec.ts": 95.1,+ "packages/core/src/SegmentedControl/__tests__/SegmentedControlHitArea.interaction.browser.spec.ts": 5.4,+ "packages/core/src/SideNav/__tests__/SideNavRailMotion.interaction.browser.spec.ts": 23.9,+ "packages/core/src/Slider/__tests__/SliderFinePointer.interaction.browser.spec.ts": 6.6,+ "packages/core/src/Slider/__tests__/SliderTouch.interaction.browser.spec.ts": 54.0,+ "packages/core/src/SnapPaging/__tests__/SnapPaging.interaction.browser.spec.ts": 53.1,+ "packages/core/src/Switch/__tests__/Switch.interaction.browser.spec.ts": 107.8,+ "packages/core/src/Switch/__tests__/SwitchPointer.interaction.browser.spec.ts": 21.6,+ "packages/core/src/TabList/__tests__/TabListOverflow.interaction.browser.spec.ts": 1.8,+ "packages/core/src/TabList/__tests__/TabPressPointer.interaction.browser.spec.ts": 2.9,+ "packages/core/src/TimeInput/__tests__/TimeWheels.interaction.browser.spec.ts": 15.1,+ "packages/core/src/Toast/__tests__/ToastEmptyViewport.interaction.browser.spec.ts": 3.3,+ "packages/core/src/Toast/__tests__/ToastSwipeScroll.interaction.browser.spec.ts": 52.9,+ "packages/core/src/Toast/__tests__/ToastViewport.hostMove.interaction.browser.spec.ts": 7.7,+ "packages/core/src/Tooltip/__tests__/TooltipClosedTriggerAnchor.interaction.browser.spec.ts": 6.4,+ "packages/core/src/TreeList/__tests__/TreeListGuideAlignment.interaction.browser.spec.ts": 3.4,+ "packages/core/src/animator/__tests__/createAnimator.interaction.browser.spec.ts": 6.9,+ "packages/core/src/dragToSnapPoints/__tests__/createDragToSnapPoints.interaction.browser.spec.ts": 6.1,+ "packages/core/src/gestures/__tests__/browser/gestures.interaction.browser.spec.ts": 44.8,+ "packages/core/src/gestures/__tests__/browser/presentedHitTesting.interaction.browser.spec.ts": 4.6,+ "packages/core/src/hooks/__tests__/usePager.interaction.browser.spec.ts": 38.6,+ "packages/core/src/motion/__tests__/animatedLayout.interaction.browser.spec.ts": 30.6,+ "packages/core/src/motion/__tests__/animatedLayoutAdopters.interaction.browser.spec.ts": 24.8,+ "packages/core/src/motion/__tests__/animator.interaction.browser.spec.ts": 16.8,+ "packages/core/src/motion/__tests__/themeTempo.interaction.browser.spec.ts": 3.3,+ "packages/core/src/motion/__tests__/themeTiming.interaction.browser.spec.ts": 0.4,+ "packages/core/src/utils/__tests__/hitArea.interaction.browser.spec.ts": 2.0,+ "packages/lab/src/Chart/__tests__/ChartZoomWheel.interaction.browser.spec.ts": 2.8,+ "packages/lab/src/CircleCheckIndicator/__tests__/CircleCheckIndicator.interaction.browser.spec.ts": 12.6,+ "packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.ts": 111.7,+ "packages/lab/src/NavigationStack/__tests__/NavigationStackScrollEdge.interaction.browser.spec.ts": 14.7,+ "packages/lab/src/RefreshControl/__tests__/RefreshControl.interaction.browser.spec.ts": 79.2,+ "packages/lab/src/TabBar/__tests__/TabBar.interaction.browser.spec.ts": 85.5},"interaction-webkit": {"internal/interaction-harness/src/harness.interaction.browser.spec.ts": 168.8,@@ -44,5 +86,9 @@"internal-docs-prompt.spec.ts": 98,"mobile-navigation.spec.ts": 8,"playground-preview-isolation.spec.ts": 113.8+ },+ "seconds-per-test": {+ "interaction-chromium": 1.56,+ "interaction-webkit": 1.28}}diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjsindex e8219ee5d8..08557da740 100644--- a/.github/scripts/ci-interaction.test.mjs+++ b/.github/scripts/ci-interaction.test.mjs@@ -68,16 +68,26 @@ describe('interaction proof routing', () => {const guards = new Set(["Require this pull request's Storybook build",'Skip the proofs outside their scope',+ 'Leave the proofs to the other legs',]);- for (const candidate of proofs.steps.slice(2)) {- if (guards.has(candidate.name)) continue;+ // Every step after the leg count also waits for this leg to have a share.+ const counted = proofs.steps.indexOf(step(proofs, "Count this run's legs"));+ for (const [index, candidate] of proofs.steps.entries()) {+ if (index < 2 || guards.has(candidate.name)) continue;+ const gate =+ index <= counted+ ? applies+ : `${applies} && steps.legs.outputs.runs == 'true'`;// A leg whose plan holds no spec file runs no proofs.expect(candidate.if, candidate.name ?? candidate.uses).toMatch(new RegExp(- `^(always\\(\\) && )?${applies.replace(/[.()|]/g, '\\$&')}( && steps\\.plan\\.outputs\\.files != '0')?$`,+ `^((always|failure)\\(\\) && )?${gate.replace(/[.()|]/g, '\\$&')}( && steps\\.plan\\.outputs\\.files != '0')?$`,),);}+ expect(step(proofs, 'Leave the proofs to the other legs').if).toBe(+ `${applies} && steps.legs.outputs.runs != 'true'`,+ );});it('fails closed when Storybook did not build for proofs that apply', () => {diff --git a/.github/scripts/playwright-legs.mjs b/.github/scripts/playwright-legs.mjsindex a6ecc40785..6ce37c0dd1 100644--- a/.github/scripts/playwright-legs.mjs+++ b/.github/scripts/playwright-legs.mjs@@ -58,25 +58,121 @@ export function planLegs(files, durations, legs) {}/**- * The seconds a suite's proofs take: each listed spec file a filter selects,- * and each changed spec, at its recorded time. A spec with no time of its own- * counts its time in `fallback` (another browser's), else the median.+ * The seconds a suite's proofs take: each listed spec file a filter selects+ * (every listed spec with no filter), and each changed spec, at its recorded+ * time. A spec with no time of its own counts its time in `fallback`+ * (another browser's); else its test count (`testsOf`) times the suite's mean+ * time a test (`perTest`); else the median time a file.*/export function estimateSeconds(times,filters,- {listed = [], changed = [], fallback = {}} = {},+ {listed = [], changed = [], fallback = {}, testsOf = () => 0, perTest} = {},) {+ // No filter: the suite plans every listed spec.const files = new Set(- listed.filter(file => filters.some(filter => file.includes(filter))),+ filters.length === 0+ ? listed+ : listed.filter(file => filters.some(filter => file.includes(filter))),);- for (const file of changed) files.add(file);+ // A changed spec the merge commit no longer holds (deleted, or renamed+ // from) runs nowhere and costs nothing.+ const present = new Set(listed);+ for (const file of changed) if (present.has(file)) files.add(file);const median = medianOf(Object.values(times));let seconds = 0;- for (const file of files) seconds += times[file] ?? fallback[file] ?? median;+ for (const file of files) {+ const tests =+ times[file] == null && fallback[file] == null && perTest > 0+ ? testsOf(file)+ : 0;+ seconds +=+ times[file] ?? fallback[file] ?? (tests > 0 ? tests * perTest : median);+ }return seconds;}+/**+ * Reads a spec's test count from the repository at `source`'s HEAD. A spec+ * that cannot be read stops the leg: one leg guessing while another reads+ * would count different legs and leave a share of the proofs to no leg.+ */+export function testsReader(source) {+ return file => {+ if (!source) return 0;+ let text;+ try {+ text = execFileSync('git', ['-C', source, 'show', `HEAD:${file}`], {+ encoding: 'utf8',+ stdio: ['ignore', 'pipe', 'pipe'],+ });+ } catch (error) {+ throw new Error(+ `cannot read ${file} from ${source} to count its tests: ${String(error.stderr ?? error.message).trim()}`,+ );+ }+ return countTests(text);+ };+}++/**+ * The test counts the leg count read, from the file it wrote. A spec it did+ * not count has none.+ */+function testsFrom(file) {+ const counts =+ file && fs.existsSync(file)+ ? JSON.parse(fs.readFileSync(file, 'utf8'))+ : {};+ return spec => counts[spec] ?? 0;+}++/**+ * Each file's seconds for the split: its recorded time, else its time in+ * `fallback`, else its test count times `perTest`. A file with none of them+ * is left out, so the splitter counts it at the median.+ */+export function costsFor(+ files,+ times,+ {fallback = {}, testsOf = () => 0, perTest} = {},+) {+ const costs = {};+ for (const file of files) {+ const tests =+ times[file] == null && fallback[file] == null && perTest > 0+ ? testsOf(file)+ : 0;+ const cost =+ times[file] ??+ fallback[file] ??+ (tests > 0 ? tests * perTest : undefined);+ if (cost != null) costs[file] = cost;+ }+ return costs;+}++// A test declaration at the start of a line: test(), test.only() and the+// like, not test.describe() or test.use().+const TEST_CALL = /^\s*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow))?\s*\(/gm;++/** How many tests a spec file declares; a test inside a loop counts once. */+export function countTests(source) {+ return (source.match(TEST_CALL) ?? []).length;+}++/** The mean seconds a test, over every test of every log. */+function meanTestSeconds(texts, project, strip) {+ let seconds = 0;+ let tests = 0;+ for (const text of texts)+ for (const each of Object.values(recordLog(text, project, strip, true))) {+ seconds += each.seconds;+ tests += each.tests;+ }+ return tests > 0 ? Math.round((seconds / tests) * 100) / 100 : undefined;+}+/*** The fewest legs, at most `maxLegs`, in which each leg's setup and share of* the proofs fits `maxSeconds`. One leg while the whole estimate fits.@@ -123,8 +219,9 @@ function stamp(line) {* `strip` turns the printed path into the one Playwright lists (relative to* the config's test directory).*/-export function recordLog(text, project, strip = '') {+export function recordLog(text, project, strip = '', counts = false) {const seconds = {};+ const tests = {};let previous = null;for (const line of text.split('\n')) {const at = stamp(line);@@ -140,9 +237,17 @@ export function recordLog(text, project, strip = '') {? match[3].slice(strip.length): match[3];seconds[file] = (seconds[file] ?? 0) + (time - previous) / 1000;+ tests[file] = (tests[file] ?? 0) + 1;previous = time;}- return seconds;+ return counts+ ? Object.fromEntries(+ Object.entries(seconds).map(([file, value]) => [+ file,+ {seconds: value, tests: tests[file]},+ ]),+ )+ : seconds;}function readDurations(file = DURATIONS) {@@ -207,13 +312,23 @@ function plan(args) {leg > legs) {throw new Error(- 'usage: plan --key <times key> --config <file> [--project <name>] --leg <i> --of <n> --out <file> [--durations <file>] [-- <filters...>]',+ 'usage: plan --key <times key> [--fallback-key <times key>] [--tests <file>] --config <file> [--project <name>] --leg <i> --of <n> --out <file> [--durations <file>] [-- <filters...>]',);}const files = list(suite, out, filters);if (files.length === 0)throw new Error(`Playwright listed no ${key} spec files`);- const legsPlan = planLegs(files, readDurations(durations)[key] ?? {}, legs);+ const recorded = readDurations(durations);+ // The same costs the leg count used, so the split matches its estimate.+ const legsPlan = planLegs(+ files,+ costsFor(files, recorded[key] ?? {}, {+ fallback: recorded[option(own, '--fallback-key')] ?? {},+ testsOf: testsFrom(option(own, '--tests')),+ perTest: recorded['seconds-per-test']?.[key],+ }),+ legs,+ );const mine = legsPlan[leg - 1];fs.writeFileSync(out, mine.files.map(file => `${file}\n`).join(''));// Playwright runs nothing, and passes, for a test list that matches no@@ -251,9 +366,9 @@ function legs(args) {maxSeconds: Number(option(own, '--max-seconds')),maxLegs: Number(option(own, '--max-legs')),};- if (!key || !Number.isInteger(leg) || leg < 1 || filters.length === 0)+ if (!key || !Number.isInteger(leg) || leg < 1)throw new Error(- 'usage: legs --key <times key> --leg <i> --listed <file> --setup-seconds <s> --max-seconds <s> --max-legs <n> [--fallback-key <times key>] [--durations <file>] [--changed <file>] -- <filters...>',+ 'usage: legs --key <times key> --leg <i> --listed <file> [--source <git dir>] [--tests-out <file>] --setup-seconds <s> --max-seconds <s> --max-legs <n> [--fallback-key <times key>] [--durations <file>] [--changed <file>] [-- <filters...>]',);const specs = file =>fs.readFileSync(file, 'utf8').split('\n').filter(isInteractionSpec);@@ -281,11 +396,23 @@ function legs(args) {return;}const durations = readDurations(option(own, '--durations') ?? DURATIONS);+ // A spec with no recorded time counts its tests, read from the merge+ // commit (the scope checkout) without a checkout of its own.+ const source = option(own, '--source');+ const read = testsReader(source);+ const counted = {};+ const testsOf = file => (counted[file] = read(file));const seconds = estimateSeconds(durations[key] ?? {}, filters, {listed,changed: changedFile ? specs(changedFile) : [],fallback: durations[option(own, '--fallback-key')] ?? {},+ testsOf,+ perTest: durations['seconds-per-test']?.[key],});+ // The plan step runs after the head checkout has replaced the scope+ // checkout, so it reads the counts from here.+ const testsOut = option(own, '--tests-out');+ if (testsOut) fs.writeFileSync(testsOut, `${JSON.stringify(counted)}\n`);const count = legCount({seconds, ...limits});const runs = leg <= count;console.log(@@ -311,9 +438,10 @@ function record(args) {'usage: record --key <times key> [--project <name>] [--strip <path prefix>] <job log...>',);const seconds = {};- for (const log of logs) {+ const texts = logs.map(log => fs.readFileSync(log, 'utf8'));+ for (const text of texts) {for (const [file, value] of Object.entries(- recordLog(fs.readFileSync(log, 'utf8'), project, strip),+ recordLog(text, project, strip),)) {seconds[file] = (seconds[file] ?? 0) + value;}@@ -321,6 +449,11 @@ function record(args) {if (Object.keys(seconds).length === 0)throw new Error(`no ${key} test lines in the logs`);const durations = readDurations();+ // The mean time a test, for a spec that has no recorded time yet.+ durations['seconds-per-test'] = {+ ...durations['seconds-per-test'],+ [key]: meanTestSeconds(texts, project, strip),+ };durations[key] = Object.fromEntries(Object.entries(seconds).sort(([a], [b]) => (a < b ? -1 : 1))diff --git a/.github/scripts/playwright-legs.test.mjs b/.github/scripts/playwright-legs.test.mjsindex 15bc4b8f5c..7951ba0516 100644--- a/.github/scripts/playwright-legs.test.mjs+++ b/.github/scripts/playwright-legs.test.mjs@@ -10,11 +10,14 @@ import {describe, expect, it} from 'vitest';import yaml from 'yaml';import {+ costsFor,+ countTests,estimateSeconds,isInteractionSpec,legCount,planLegs,recordLog,+ testsReader,} from './playwright-legs.mjs';const root = path.resolve(@@ -106,17 +109,57 @@ describe('estimateSeconds and legCount', () => {// then the median of this suite's times.expect(estimateSeconds(times, ['internal/'], {- listed,+ listed: [...listed, 'new.spec.ts'],changed: ['new.spec.ts'],fallback: {'new.spec.ts': 12},}),).toBe(182);expect(estimateSeconds(times, ['internal/'], {- listed,+ listed: [...listed, 'other.spec.ts'],changed: ['other.spec.ts'],}),).toBe(170 + 50);+ // A changed spec the merge commit no longer holds (deleted, or renamed+ // from) costs nothing, and its tests are never read.+ expect(+ estimateSeconds(times, ['internal/'], {+ listed,+ changed: ['gone.spec.ts'],+ testsOf: () => {+ throw new Error('read a deleted spec');+ },+ perTest: 1.5,+ }),+ ).toBe(170);+ });++ it('costs a spec with no recorded time by its tests at the mean time a test', () => {+ // A spec with 50 tests at 1.56 s a test, not the 10.7 s median file.+ const source = Array.from(+ {length: 50},+ (_, i) => ` test('case ${i}', async () => {});`,+ ).join('\n');+ expect(countTests(source)).toBe(50);+ expect(+ countTests(+ "test.describe('group', () => {\n test.skip('a', () => {});\n it('b', () => {});\n test.use({});\n});",+ ),+ ).toBe(2);+ const estimate = estimateSeconds(times, ['packages/lab/'], {+ listed: [...listed, 'packages/lab/TabBar.spec.ts'],+ testsOf: file => (file === 'packages/lab/TabBar.spec.ts' ? 50 : 0),+ perTest: 1.56,+ });+ expect(estimate).toBeCloseTo(78, 5);+ // With no tests to count, the file median stands.+ expect(+ estimateSeconds(times, ['packages/lab/'], {+ listed: ['packages/lab/Empty.spec.ts'],+ testsOf: () => 0,+ perTest: 1.56,+ }),+ ).toBe(50);});it('counts a listed spec in the lane that has no recorded time yet', () => {@@ -229,13 +272,16 @@ describe('the interaction proof legs in ci.yml', () => {{id: 'pr-interaction',project: 'chromium',- legs: [1, 2],+ legs: [1, 2, 3, 4, 5, 6],run: 'Run interaction proofs',plan: "Plan this leg's interaction proofs",evidence: 'interaction-evidence-${{ matrix.leg }}',- gate: "steps.scope.outputs.applies == 'true'",- leg: '${{ matrix.leg }}',- of: '${{ strategy.job-total }}',+ // As few legs as fit; see the leg counts below.+ gate: "steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'",+ leg: '${{ steps.legs.outputs.leg }}',+ of: '${{ steps.legs.outputs.count }}',+ count: "Count this run's legs",+ fallback: 'webkit',},{id: 'pr-interaction-webkit',@@ -248,6 +294,8 @@ describe('the interaction proof legs in ci.yml', () => {gate: "steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'",leg: '${{ steps.legs.outputs.leg }}',of: '${{ steps.legs.outputs.count }}',+ count: "Count this run's WebKit legs",+ fallback: 'chromium',},];@@ -296,7 +344,10 @@ describe('the interaction proof legs in ci.yml', () => {`cp .test-scope/${file} "$RUNNER_TEMP/interaction-legs/${dir}"`,);expect(plan.run).toContain(- `--key interaction-${lane.project} --config playwright.interaction.config.ts --project ${lane.project}`,+ `--key interaction-${lane.project} --fallback-key interaction-${lane.fallback} --tests "$RUNNER_TEMP/interaction-legs/tests.json"`,+ );+ expect(plan.run).toContain(+ `--config playwright.interaction.config.ts --project ${lane.project}`,);expect(plan.run).toContain('--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',@@ -347,13 +398,19 @@ describe('the interaction proof legs in ci.yml', () => {'--leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}',// Every spec file in the merge commit, recorded or not.'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"',- '--listed "$RUNNER_TEMP/interaction-legs/listed.txt"',+ '--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',])expect(count.run).toContain(part);});// Runs the count step with the real planner on these recorded times.- function countFor(leg, webkitTimes, changed = [], unrecorded = []) {+ function countFor(+ leg,+ webkitTimes,+ changed = [],+ unrecorded = [],+ {deleted = [], sources = {}, perTest} = {},+ ) {// The real path: the planner runs only when invoked as its own file.const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'webkit-legs-')),@@ -373,13 +430,24 @@ describe('the interaction proof legs in ci.yml', () => {JSON.stringify({'interaction-webkit': webkitTimes,'interaction-chromium': {},+ ...(perTest+ ? {'seconds-per-test': {'interaction-webkit': perTest}}+ : {}),}),);// The scope checkout: a repository whose tree holds the spec files.const scope = path.join(dir, '.test-scope');- for (const file of [...Object.keys(webkitTimes), ...unrecorded]) {+ // A changed spec is in the merge commit too, unless it was deleted.+ const present = (changed ?? []).filter(+ file => isInteractionSpec(file) && !deleted.includes(file),+ );+ for (const file of [+ ...Object.keys(webkitTimes),+ ...unrecorded,+ ...present,+ ]) {fs.mkdirSync(path.join(scope, path.dirname(file)), {recursive: true});- fs.writeFileSync(path.join(scope, file), '');+ fs.writeFileSync(path.join(scope, file), sources[file] ?? '');}const git = args =>execFileSync('git', ['-C', scope, ...args], {stdio: 'ignore'});@@ -421,18 +489,42 @@ describe('the interaction proof legs in ci.yml', () => {},},);- return Object.fromEntries(+ const result = Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split('\n').map(line => line.split('=')),);+ // The counts the plan step reads after the head checkout.+ const testsFile = path.join(dir, 'interaction-legs/tests.json');+ if (fs.existsSync(testsFile))+ Object.defineProperty(result, 'tests', {+ value: JSON.parse(fs.readFileSync(testsFile, 'utf8')),+ });+ return result;} finally {fs.rmSync(dir, {recursive: true, force: true});}}+ it('writes the test counts it read for the plan, and skips a deleted spec', () => {+ const times = {+ 'internal/interaction-harness/src/harness.interaction.browser.spec.ts': 100,+ };+ const fresh =+ 'internal/interaction-harness/src/fresh.interaction.browser.spec.ts';+ const gone =+ 'packages/core/src/List/__tests__/Gone.interaction.browser.spec.ts';+ const result = countFor(2, times, [fresh, gone], [], {+ deleted: [gone],+ sources: {[fresh]: "test('a', () => {});\ntest('b', () => {});\n"},+ perTest: 2,+ });+ expect(result).toEqual({count: '1', runs: 'false', leg: '2'});+ expect(result.tests).toEqual({[fresh]: 2});+ });+it('gives the whole suite to the first leg at today’s recorded times', () => {const times = durations['interaction-webkit'];expect(countFor(1, times)).toEqual({count: '1', runs: 'true', leg: '1'});@@ -594,14 +686,132 @@ describe('the interaction proof legs in ci.yml', () => {expect(leg.seconds).toBeLessThanOrEqual(6 * 60);});- it('records a time for each project the legs plan, each under the legs\u2019 per-runner budget', () => {+ it("pins the Chromium count's arguments: every spec, the matrix's legs, the shared costs", () => {+ const job = ci.jobs['pr-interaction'];+ const count = step(job, "Count this run's legs");+ expect(count.id).toBe('legs');+ expect(count.if).toBe("steps.scope.outputs.applies == 'true'");+ for (const part of [+ 'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"',+ '--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',+ '--tests-out "$RUNNER_TEMP/interaction-legs/tests.json"',+ '--key interaction-chromium --fallback-key interaction-webkit',+ '--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',+ '--setup-seconds 90 --max-seconds 600',+ '--leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}',+ ])+ expect(count.run).toContain(part);+ // No path filter: the Chromium lane counts every spec.+ expect(count.run).not.toContain(' -- ');+ const at = job.steps.indexOf(count);+ expect(at).toBeGreaterThan(+ job.steps.indexOf(+ step(job, "Keep the leg planner from the workflow's commit"),+ ),+ );+ expect(at).toBeLessThan(+ job.steps.indexOf(step(job, 'Check out the heavy-suite caps')),+ );+ // Both plans cost an unrecorded spec as their counts do, from the+ // counts the count step wrote: the head checkout has replaced the scope+ // checkout by then.+ expect(step(job, "Plan this leg's interaction proofs").run).toContain(+ '--key interaction-chromium --fallback-key interaction-webkit --tests "$RUNNER_TEMP/interaction-legs/tests.json"',+ );+ const webkit = ci.jobs['pr-interaction-webkit'];+ expect(step(webkit, "Count this run's WebKit legs").run).toContain(+ '--tests-out "$RUNNER_TEMP/interaction-legs/tests.json"',+ );+ expect(+ step(webkit, "Plan this leg's WebKit interaction proofs").run,+ ).toContain(+ '--key interaction-webkit --fallback-key interaction-chromium --tests "$RUNNER_TEMP/interaction-legs/tests.json"',+ );+ // The scope checkout is gone by the plan step.+ for (const [id, name] of [+ ['pr-interaction', "Plan this leg's interaction proofs"],+ ['pr-interaction-webkit', "Plan this leg's WebKit interaction proofs"],+ ])+ expect(step(ci.jobs[id], name).run).not.toContain('.test-scope');+ });++ it('splits by the same costs it counts, and stops on a spec it cannot read', () => {+ const times = {'a.spec.ts': 100};+ expect(+ costsFor(['a.spec.ts', 'b.spec.ts', 'c.spec.ts', 'd.spec.ts'], times, {+ fallback: {'b.spec.ts': 40},+ testsOf: file => (file === 'c.spec.ts' ? 10 : 0),+ perTest: 1.5,+ }),+ ).toEqual({'a.spec.ts': 100, 'b.spec.ts': 40, 'c.spec.ts': 15});+ const read = testsReader(root);+ expect(read('.github/scripts/playwright-legs.test.mjs')).toBeGreaterThan(5);+ expect(() => read('no/such.spec.ts')).toThrow(+ /cannot read no\/such\.spec\.ts/,+ );+ expect(testsReader(undefined)('anything.spec.ts')).toBe(0);+ });++ it('fits every leg in ten minutes at today’s recorded times, within the matrix', () => {+ // Every interaction spec in the tree, as the count steps list them.+ const specs = execFileSync('git', ['ls-files'], {+ cwd: root,+ encoding: 'utf8',+ })+ .split('\n')+ .filter(isInteractionSpec);for (const lane of lanes) {const times = durations[`interaction-${lane.project}`];- expect(Object.keys(times).length, lane.project).toBeGreaterThan(0);- const plan = planLegs(Object.keys(times), times, lane.legs.length);- // About six minutes a leg under load; the recorded run was a loaded one.- for (const leg of plan)- expect(leg.seconds, lane.project).toBeLessThanOrEqual(6 * 60);+ const fallback = durations[`interaction-${lane.fallback}`];+ const count = step(ci.jobs[lane.id], lane.count);+ const filters =+ lane.project === 'webkit'+ ? ci.jobs[lane.id].env.WEBKIT_SPEC_PATHS.split(' ')+ : [];+ const perTest =+ durations['seconds-per-test'][`interaction-${lane.project}`];+ const testsOf = file =>+ countTests(fs.readFileSync(path.join(root, file), 'utf8'));+ const seconds = estimateSeconds(times, filters, {+ listed: specs,+ fallback,+ testsOf,+ perTest,+ });+ const limits = {+ setupSeconds: 90,+ maxSeconds: 600,+ maxLegs: lane.legs.length,+ };+ expect(count.run, lane.id).toContain(+ '--setup-seconds 90 --max-seconds 600',+ );+ // The matrix still has room: the count stays under its legs.+ const needed = Math.ceil(+ seconds / (limits.maxSeconds - limits.setupSeconds),+ );+ expect(+ needed,+ `${lane.id}: ${Math.round(seconds)} s of proofs`,+ ).toBeLessThan(lane.legs.length);+ const legsNow = legCount({seconds, ...limits});+ const planned = planLegs(+ specs.filter(+ file =>+ filters.length === 0 || filters.some(each => file.includes(each)),+ ),+ Object.fromEntries(+ specs.map(file => [+ file,+ times[file] ?? fallback[file] ?? testsOf(file) * perTest,+ ]),+ ),+ legsNow,+ );+ for (const leg of planned)+ expect(leg.seconds + limits.setupSeconds, lane.id).toBeLessThanOrEqual(+ limits.maxSeconds + 60,+ );}});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a12e059107..62b8a03a0c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -2471,16 +2471,15 @@ jobs:needs: [room, admission, heavy-ci, build-storybook]if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}# Frame evidence measures timing: four cores, one Playwright worker.- # One runner ran every proof in about ten minutes and passed the job's- # fifteen once its checkout was slow. Two legs share the spec files by- # their recorded times (playwright-legs.mjs), so each runs in about five- # minutes, and a leg with the slowest checkout seen still ends with about- # four minutes to spare. `test` reads the job's result, which is success- # only when every leg passed.+ # The legs share the spec files by their recorded times+ # (playwright-legs.mjs), as few legs as keep each leg's setup and proofs+ # within ten minutes, which leaves a third of the job's fifteen for a slow+ # runner or checkout. A leg with no share passes. `test` reads the job's+ # result, which is success only when every leg passed.strategy:fail-fast: falsematrix:- leg: [1, 2]+ leg: [1, 2, 3, 4, 5, 6]runs-on: ubuntu-latesttimeout-minutes: 15permissions:@@ -2538,11 +2537,33 @@ jobs:cp .test-scope/.github/scripts/lane-environment.mjs "$RUNNER_TEMP/interaction-legs/.github/scripts/"cp .test-scope/.github/lane-environments.json "$RUNNER_TEMP/interaction-legs/.github/"+ # The legs this run needs: every spec file in the merge commit, which the+ # scope checkout fetched whole, at its recorded time (a WebKit time, or+ # its test count times the mean time a test, for one with none), plus+ # 90 s of setup a leg, within ten minutes a leg. Each leg decides the+ # same way before its own setup.+ - name: Count this run's legs+ id: legs+ if: steps.scope.outputs.applies == 'true'+ run: |+ git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"+ node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" legs \+ --listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope \+ --tests-out "$RUNNER_TEMP/interaction-legs/tests.json" \+ --key interaction-chromium --fallback-key interaction-webkit \+ --durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \+ --setup-seconds 90 --max-seconds 600 \+ --leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}++ - name: Leave the proofs to the other legs+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs != 'true'+ run: echo "This run's interaction proofs fit ${{ steps.legs.outputs.count }} legs, so this leg runs none."+# The caps step comes from the merge commit, not the pull request's# head: a head that predates the step does not have it. The head# checkout below replaces this directory.- name: Check out the heavy-suite caps- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: actions/checkout@v7with:path: .heavy-caps@@ -2550,22 +2571,22 @@ jobs:persist-credentials: false- name: Per-author heavy-suite caps- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: ./.heavy-caps/.github/actions/heavy-caps# The daily heavy-run cap counts this step (spec:AST-030 FR16).- name: Heavy suite runs- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'run: 'true'- name: Checkout- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: actions/checkout@v7with:ref: ${{ github.event.pull_request.head.sha }}- name: Verify evidence checkout- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'env:EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA"@@ -2573,46 +2594,47 @@ jobs:# The head's specs run in this lane's environment, which comes from the# merge commit. A head older than that environment stops here.- name: Require this lane's environment in the branch- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'run: >-node "$RUNNER_TEMP/interaction-legs/.github/scripts/lane-environment.mjs" check --lane interaction-chromium--workflow "$RUNNER_TEMP/interaction-legs/.github/lane-environments.json"--head .github/lane-environments.json- name: Setup Node and pnpm- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: ./.github/actions/setupwith:install: 'false'- name: Download Storybook artifact- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: actions/download-artifact@v8with:name: storybook-${{ needs.build-storybook.outputs.short_hash }}path: apps/storybook/dist- name: Install Playwright- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'run: pnpm install --frozen-lockfile && npx playwright install chromium# This leg's share of the spec files, by their recorded times. Every# spec file is in exactly one leg; one with no recorded time counts the- # median. The planner, the shared splitter it imports and the times come+ # test count the leg count read, times the mean time a test. The planner, the shared splitter it imports and the times come# from the scope checkout, the workflow's own commit (kept above): a# head cut before them has none.- name: Plan this leg's interaction proofsid: plan- if: steps.scope.outputs.applies == 'true'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'run: |node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" plan \- --key interaction-chromium --config playwright.interaction.config.ts --project chromium \- --leg ${{ matrix.leg }} --of ${{ strategy.job-total }} \+ --key interaction-chromium --fallback-key interaction-webkit --tests "$RUNNER_TEMP/interaction-legs/tests.json" \+ --config playwright.interaction.config.ts --project chromium \+ --leg ${{ steps.legs.outputs.leg }} --of ${{ steps.legs.outputs.count }} \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \--out "$RUNNER_TEMP/interaction-leg.txt"- name: Run interaction proofs- if: steps.scope.outputs.applies == 'true' && steps.plan.outputs.files != '0'+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true' && steps.plan.outputs.files != '0'run: |pnpm exec playwright test \--test-list "$RUNNER_TEMP/interaction-leg.txt" \@@ -2622,7 +2644,7 @@ jobs:# Storyboards (PNG contact sheets) and curve traces, for review.- name: Upload interaction evidence- if: always() && steps.scope.outputs.applies == 'true'+ if: always() && steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'uses: actions/upload-artifact@v7with:name: interaction-evidence-${{ matrix.leg }}@@ -2648,7 +2670,7 @@ jobs:# Frame evidence measures timing: four cores, one Playwright worker.# One leg runs every spec while the planner's estimate fits; past that two# legs share the spec files by their recorded times, as the Chromium- # proofs' two do.+ # proofs' legs do.strategy:fail-fast: falsematrix:@@ -2749,7 +2771,8 @@ jobs:read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" legs \- --listed "$RUNNER_TEMP/interaction-legs/listed.txt" \+ --listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope \+ --tests-out "$RUNNER_TEMP/interaction-legs/tests.json" \--key interaction-webkit --fallback-key interaction-chromium \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \--changed "${CHANGED_FILES:-}" \@@ -2850,7 +2873,8 @@ jobs:echo "::warning title=Changed specs not planned::The scope step could not list the changed files, so a changed interaction spec outside this lane's paths does not run here."finode "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" plan \- --key interaction-webkit --config playwright.interaction.config.ts --project webkit \+ --key interaction-webkit --fallback-key interaction-chromium --tests "$RUNNER_TEMP/interaction-legs/tests.json" \+ --config playwright.interaction.config.ts --project webkit \--leg ${{ steps.legs.outputs.leg }} --of ${{ steps.legs.outputs.count }} \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \--out "$RUNNER_TEMP/interaction-leg.txt" -- \