Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: the Chromium interaction proofs run in as few legs as fit ten minutes each (#10258)

Cindy Zhang010db3f355d9
diff --git a/.github/playwright-leg-durations.json b/.github/playwright-leg-durations.json
index 629aadd1ee..38639d6e30 100644
--- a/.github/playwright-leg-durations.json
+++ b/.github/playwright-leg-durations.json
@@ -1,32 +1,74 @@
{
"interaction-chromium": {
- "internal/interaction-harness/src/harness.interaction.browser.spec.ts": 118.3,
- "internal/interaction-harness/src/storybook.interaction.browser.spec.ts": 1.6,
- "packages/core/src/AlertDialog/__tests__/AlertDialog.interaction.browser.spec.ts": 20.3,
- "packages/core/src/AppShell/__tests__/AppShellLayoutRoot.interaction.browser.spec.ts": 4.6,
- "packages/core/src/ButtonGroup/__tests__/ButtonGroupCorners.interaction.browser.spec.ts": 0.9,
- "packages/core/src/Chat/__tests__/ChatMessageAvatar.interaction.browser.spec.ts": 1.2,
- "packages/core/src/DateInput/__tests__/DateInputFocusLeave.interaction.browser.spec.ts": 2.6,
- "packages/core/src/DateInput/__tests__/MonthScroller.interaction.browser.spec.ts": 5.1,
- "packages/core/src/DateInput/__tests__/MonthScrollerSteps.interaction.browser.spec.ts": 7.8,
- "packages/core/src/DateRangeInput/__tests__/DateRangeInputReflow.interaction.browser.spec.ts": 3.5,
- "packages/core/src/Field/__tests__/SearchKeyboard.interaction.browser.spec.ts": 5.6,
- "packages/core/src/SegmentedControl/__tests__/SegmentedControl.interaction.browser.spec.ts": 85.9,
- "packages/core/src/SideNav/__tests__/SideNavRailMotion.interaction.browser.spec.ts": 16.1,
- "packages/core/src/Slider/__tests__/SliderFinePointer.interaction.browser.spec.ts": 5.6,
- "packages/core/src/Slider/__tests__/SliderTouch.interaction.browser.spec.ts": 50.4,
- "packages/core/src/Switch/__tests__/Switch.interaction.browser.spec.ts": 98.1,
- "packages/core/src/Switch/__tests__/SwitchPointer.interaction.browser.spec.ts": 18.3,
- "packages/core/src/TabList/__tests__/TabListOverflow.interaction.browser.spec.ts": 1.6,
- "packages/core/src/TabList/__tests__/TabPressPointer.interaction.browser.spec.ts": 2.7,
- "packages/core/src/TreeList/__tests__/TreeListGuideAlignment.interaction.browser.spec.ts": 2.9,
- "packages/core/src/animator/__tests__/createAnimator.interaction.browser.spec.ts": 6.8,
- "packages/core/src/gestures/__tests__/browser/gestures.interaction.browser.spec.ts": 41.7,
- "packages/core/src/gestures/__tests__/browser/presentedHitTesting.interaction.browser.spec.ts": 4.5,
- "packages/core/src/hooks/__tests__/usePager.interaction.browser.spec.ts": 38.5,
- "packages/core/src/motion/__tests__/animatedLayout.interaction.browser.spec.ts": 24.4,
- "packages/core/src/motion/__tests__/animatedLayoutAdopters.interaction.browser.spec.ts": 24.5,
- "packages/core/src/motion/__tests__/animator.interaction.browser.spec.ts": 16.6
+ "internal/interaction-harness/src/harness.interaction.browser.spec.ts": 135.6,
+ "internal/interaction-harness/src/pointer.interaction.browser.spec.ts": 0.2,
+ "internal/interaction-harness/src/storybook.interaction.browser.spec.ts": 4.2,
+ "packages/core/src/AlertDialog/__tests__/AlertDialog.interaction.browser.spec.ts": 21.1,
+ "packages/core/src/AppShell/__tests__/AppShellLayoutRoot.interaction.browser.spec.ts": 6.4,
+ "packages/core/src/BottomSheet/__tests__/BottomSheetDetent.interaction.browser.spec.ts": 16.5,
+ "packages/core/src/BottomSheet/__tests__/BottomSheetKeyboardInset.interaction.browser.spec.ts": 21.9,
+ "packages/core/src/Button/__tests__/ButtonPress.interaction.browser.spec.ts": 10.7,
+ "packages/core/src/Button/__tests__/ButtonPressRelease.interaction.browser.spec.ts": 17.9,
+ "packages/core/src/Button/__tests__/ButtonPressTiming.interaction.browser.spec.ts": 8.0,
+ "packages/core/src/ButtonGroup/__tests__/ButtonGroupCorners.interaction.browser.spec.ts": 1.1,
+ "packages/core/src/ButtonGroup/__tests__/ButtonGroupTracking.interaction.browser.spec.ts": 23.9,
+ "packages/core/src/Chat/__tests__/ChatMessageAvatar.interaction.browser.spec.ts": 1.6,
+ "packages/core/src/DateInput/__tests__/DateInputFocusLeave.interaction.browser.spec.ts": 3.5,
+ "packages/core/src/DateInput/__tests__/MonthScroller.interaction.browser.spec.ts": 5.8,
+ "packages/core/src/DateInput/__tests__/MonthScrollerSteps.interaction.browser.spec.ts": 8.5,
+ "packages/core/src/DateInput/__tests__/Wheel.interaction.browser.spec.ts": 47.9,
+ "packages/core/src/DateInput/__tests__/WheelActivation.interaction.browser.spec.ts": 4.6,
+ "packages/core/src/DateRangeInput/__tests__/DateRangeInputReflow.interaction.browser.spec.ts": 4.4,
+ "packages/core/src/Dialog/__tests__/DialogTouchEntrance.interaction.browser.spec.ts": 11.2,
+ "packages/core/src/Drawer/__tests__/DrawerTouchEntrance.interaction.browser.spec.ts": 9.7,
+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuDragHighlight.interaction.browser.spec.ts": 19.2,
+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuEntranceHold.interaction.browser.spec.ts": 5.0,
+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuTouchHandoff.interaction.browser.spec.ts": 5.6,
+ "packages/core/src/Field/__tests__/SearchKeyboard.interaction.browser.spec.ts": 8.6,
+ "packages/core/src/Indicator/__tests__/BoxHold.interaction.browser.spec.ts": 73.2,
+ "packages/core/src/Indicator/__tests__/TouchTargets.interaction.browser.spec.ts": 11.2,
+ "packages/core/src/Layer/__tests__/inputHold.interaction.browser.spec.ts": 11.0,
+ "packages/core/src/Layer/__tests__/keyboardCappedLists.interaction.browser.spec.ts": 12.2,
+ "packages/core/src/Layer/__tests__/preserveAnimations.interaction.browser.spec.ts": 0.1,
+ "packages/core/src/Layout/__tests__/collapsedPanel.interaction.browser.spec.ts": 2.1,
+ "packages/core/src/List/__tests__/ListRowPress.interaction.browser.spec.ts": 16.3,
+ "packages/core/src/List/__tests__/ListRowPressPaths.interaction.browser.spec.ts": 9.7,
+ "packages/core/src/List/__tests__/ListTouchHitArea.interaction.browser.spec.ts": 26.0,
+ "packages/core/src/Popover/__tests__/PopoverKeyboard.interaction.browser.spec.ts": 12.3,
+ "packages/core/src/Resizable/__tests__/grabZone.interaction.browser.spec.ts": 24.6,
+ "packages/core/src/SegmentedControl/__tests__/SegmentedControl.interaction.browser.spec.ts": 95.1,
+ "packages/core/src/SegmentedControl/__tests__/SegmentedControlHitArea.interaction.browser.spec.ts": 5.4,
+ "packages/core/src/SideNav/__tests__/SideNavRailMotion.interaction.browser.spec.ts": 23.9,
+ "packages/core/src/Slider/__tests__/SliderFinePointer.interaction.browser.spec.ts": 6.6,
+ "packages/core/src/Slider/__tests__/SliderTouch.interaction.browser.spec.ts": 54.0,
+ "packages/core/src/SnapPaging/__tests__/SnapPaging.interaction.browser.spec.ts": 53.1,
+ "packages/core/src/Switch/__tests__/Switch.interaction.browser.spec.ts": 107.8,
+ "packages/core/src/Switch/__tests__/SwitchPointer.interaction.browser.spec.ts": 21.6,
+ "packages/core/src/TabList/__tests__/TabListOverflow.interaction.browser.spec.ts": 1.8,
+ "packages/core/src/TabList/__tests__/TabPressPointer.interaction.browser.spec.ts": 2.9,
+ "packages/core/src/TimeInput/__tests__/TimeWheels.interaction.browser.spec.ts": 15.1,
+ "packages/core/src/Toast/__tests__/ToastEmptyViewport.interaction.browser.spec.ts": 3.3,
+ "packages/core/src/Toast/__tests__/ToastSwipeScroll.interaction.browser.spec.ts": 52.9,
+ "packages/core/src/Toast/__tests__/ToastViewport.hostMove.interaction.browser.spec.ts": 7.7,
+ "packages/core/src/Tooltip/__tests__/TooltipClosedTriggerAnchor.interaction.browser.spec.ts": 6.4,
+ "packages/core/src/TreeList/__tests__/TreeListGuideAlignment.interaction.browser.spec.ts": 3.4,
+ "packages/core/src/animator/__tests__/createAnimator.interaction.browser.spec.ts": 6.9,
+ "packages/core/src/dragToSnapPoints/__tests__/createDragToSnapPoints.interaction.browser.spec.ts": 6.1,
+ "packages/core/src/gestures/__tests__/browser/gestures.interaction.browser.spec.ts": 44.8,
+ "packages/core/src/gestures/__tests__/browser/presentedHitTesting.interaction.browser.spec.ts": 4.6,
+ "packages/core/src/hooks/__tests__/usePager.interaction.browser.spec.ts": 38.6,
+ "packages/core/src/motion/__tests__/animatedLayout.interaction.browser.spec.ts": 30.6,
+ "packages/core/src/motion/__tests__/animatedLayoutAdopters.interaction.browser.spec.ts": 24.8,
+ "packages/core/src/motion/__tests__/animator.interaction.browser.spec.ts": 16.8,
+ "packages/core/src/motion/__tests__/themeTempo.interaction.browser.spec.ts": 3.3,
+ "packages/core/src/motion/__tests__/themeTiming.interaction.browser.spec.ts": 0.4,
+ "packages/core/src/utils/__tests__/hitArea.interaction.browser.spec.ts": 2.0,
+ "packages/lab/src/Chart/__tests__/ChartZoomWheel.interaction.browser.spec.ts": 2.8,
+ "packages/lab/src/CircleCheckIndicator/__tests__/CircleCheckIndicator.interaction.browser.spec.ts": 12.6,
+ "packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.ts": 111.7,
+ "packages/lab/src/NavigationStack/__tests__/NavigationStackScrollEdge.interaction.browser.spec.ts": 14.7,
+ "packages/lab/src/RefreshControl/__tests__/RefreshControl.interaction.browser.spec.ts": 79.2,
+ "packages/lab/src/TabBar/__tests__/TabBar.interaction.browser.spec.ts": 85.5
},
"interaction-webkit": {
"internal/interaction-harness/src/harness.interaction.browser.spec.ts": 168.8,
@@ -44,5 +86,9 @@
"internal-docs-prompt.spec.ts": 98,
"mobile-navigation.spec.ts": 8,
"playground-preview-isolation.spec.ts": 113.8
+ },
+ "seconds-per-test": {
+ "interaction-chromium": 1.56,
+ "interaction-webkit": 1.28
}
}
diff --git a/.github/scripts/ci-interaction.test.mjs b/.github/scripts/ci-interaction.test.mjs
index e8219ee5d8..08557da740 100644
--- a/.github/scripts/ci-interaction.test.mjs
+++ b/.github/scripts/ci-interaction.test.mjs
@@ -68,16 +68,26 @@ describe('interaction proof routing', () => {
const guards = new Set([
"Require this pull request's Storybook build",
'Skip the proofs outside their scope',
+ 'Leave the proofs to the other legs',
]);
- for (const candidate of proofs.steps.slice(2)) {
- if (guards.has(candidate.name)) continue;
+ // Every step after the leg count also waits for this leg to have a share.
+ const counted = proofs.steps.indexOf(step(proofs, "Count this run's legs"));
+ for (const [index, candidate] of proofs.steps.entries()) {
+ if (index < 2 || guards.has(candidate.name)) continue;
+ const gate =
+ index <= counted
+ ? applies
+ : `${applies} && steps.legs.outputs.runs == 'true'`;
// A leg whose plan holds no spec file runs no proofs.
expect(candidate.if, candidate.name ?? candidate.uses).toMatch(
new RegExp(
- `^(always\\(\\) && )?${applies.replace(/[.()|]/g, '\\$&')}( && steps\\.plan\\.outputs\\.files != '0')?$`,
+ `^((always|failure)\\(\\) && )?${gate.replace(/[.()|]/g, '\\$&')}( && steps\\.plan\\.outputs\\.files != '0')?$`,
),
);
}
+ expect(step(proofs, 'Leave the proofs to the other legs').if).toBe(
+ `${applies} && steps.legs.outputs.runs != 'true'`,
+ );
});
it('fails closed when Storybook did not build for proofs that apply', () => {
diff --git a/.github/scripts/playwright-legs.mjs b/.github/scripts/playwright-legs.mjs
index a6ecc40785..6ce37c0dd1 100644
--- a/.github/scripts/playwright-legs.mjs
+++ b/.github/scripts/playwright-legs.mjs
@@ -58,25 +58,121 @@ export function planLegs(files, durations, legs) {
}
/**
- * The seconds a suite's proofs take: each listed spec file a filter selects,
- * and each changed spec, at its recorded time. A spec with no time of its own
- * counts its time in `fallback` (another browser's), else the median.
+ * The seconds a suite's proofs take: each listed spec file a filter selects
+ * (every listed spec with no filter), and each changed spec, at its recorded
+ * time. A spec with no time of its own counts its time in `fallback`
+ * (another browser's); else its test count (`testsOf`) times the suite's mean
+ * time a test (`perTest`); else the median time a file.
*/
export function estimateSeconds(
times,
filters,
- {listed = [], changed = [], fallback = {}} = {},
+ {listed = [], changed = [], fallback = {}, testsOf = () => 0, perTest} = {},
) {
+ // No filter: the suite plans every listed spec.
const files = new Set(
- listed.filter(file => filters.some(filter => file.includes(filter))),
+ filters.length === 0
+ ? listed
+ : listed.filter(file => filters.some(filter => file.includes(filter))),
);
- for (const file of changed) files.add(file);
+ // A changed spec the merge commit no longer holds (deleted, or renamed
+ // from) runs nowhere and costs nothing.
+ const present = new Set(listed);
+ for (const file of changed) if (present.has(file)) files.add(file);
const median = medianOf(Object.values(times));
let seconds = 0;
- for (const file of files) seconds += times[file] ?? fallback[file] ?? median;
+ for (const file of files) {
+ const tests =
+ times[file] == null && fallback[file] == null && perTest > 0
+ ? testsOf(file)
+ : 0;
+ seconds +=
+ times[file] ?? fallback[file] ?? (tests > 0 ? tests * perTest : median);
+ }
return seconds;
}
+/**
+ * Reads a spec's test count from the repository at `source`'s HEAD. A spec
+ * that cannot be read stops the leg: one leg guessing while another reads
+ * would count different legs and leave a share of the proofs to no leg.
+ */
+export function testsReader(source) {
+ return file => {
+ if (!source) return 0;
+ let text;
+ try {
+ text = execFileSync('git', ['-C', source, 'show', `HEAD:${file}`], {
+ encoding: 'utf8',
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ } catch (error) {
+ throw new Error(
+ `cannot read ${file} from ${source} to count its tests: ${String(error.stderr ?? error.message).trim()}`,
+ );
+ }
+ return countTests(text);
+ };
+}
+
+/**
+ * The test counts the leg count read, from the file it wrote. A spec it did
+ * not count has none.
+ */
+function testsFrom(file) {
+ const counts =
+ file && fs.existsSync(file)
+ ? JSON.parse(fs.readFileSync(file, 'utf8'))
+ : {};
+ return spec => counts[spec] ?? 0;
+}
+
+/**
+ * Each file's seconds for the split: its recorded time, else its time in
+ * `fallback`, else its test count times `perTest`. A file with none of them
+ * is left out, so the splitter counts it at the median.
+ */
+export function costsFor(
+ files,
+ times,
+ {fallback = {}, testsOf = () => 0, perTest} = {},
+) {
+ const costs = {};
+ for (const file of files) {
+ const tests =
+ times[file] == null && fallback[file] == null && perTest > 0
+ ? testsOf(file)
+ : 0;
+ const cost =
+ times[file] ??
+ fallback[file] ??
+ (tests > 0 ? tests * perTest : undefined);
+ if (cost != null) costs[file] = cost;
+ }
+ return costs;
+}
+
+// A test declaration at the start of a line: test(), test.only() and the
+// like, not test.describe() or test.use().
+const TEST_CALL = /^\s*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow))?\s*\(/gm;
+
+/** How many tests a spec file declares; a test inside a loop counts once. */
+export function countTests(source) {
+ return (source.match(TEST_CALL) ?? []).length;
+}
+
+/** The mean seconds a test, over every test of every log. */
+function meanTestSeconds(texts, project, strip) {
+ let seconds = 0;
+ let tests = 0;
+ for (const text of texts)
+ for (const each of Object.values(recordLog(text, project, strip, true))) {
+ seconds += each.seconds;
+ tests += each.tests;
+ }
+ return tests > 0 ? Math.round((seconds / tests) * 100) / 100 : undefined;
+}
+
/**
* The fewest legs, at most `maxLegs`, in which each leg's setup and share of
* the proofs fits `maxSeconds`. One leg while the whole estimate fits.
@@ -123,8 +219,9 @@ function stamp(line) {
* `strip` turns the printed path into the one Playwright lists (relative to
* the config's test directory).
*/
-export function recordLog(text, project, strip = '') {
+export function recordLog(text, project, strip = '', counts = false) {
const seconds = {};
+ const tests = {};
let previous = null;
for (const line of text.split('\n')) {
const at = stamp(line);
@@ -140,9 +237,17 @@ export function recordLog(text, project, strip = '') {
? match[3].slice(strip.length)
: match[3];
seconds[file] = (seconds[file] ?? 0) + (time - previous) / 1000;
+ tests[file] = (tests[file] ?? 0) + 1;
previous = time;
}
- return seconds;
+ return counts
+ ? Object.fromEntries(
+ Object.entries(seconds).map(([file, value]) => [
+ file,
+ {seconds: value, tests: tests[file]},
+ ]),
+ )
+ : seconds;
}
function readDurations(file = DURATIONS) {
@@ -207,13 +312,23 @@ function plan(args) {
leg > legs
) {
throw new Error(
- 'usage: plan --key <times key> --config <file> [--project <name>] --leg <i> --of <n> --out <file> [--durations <file>] [-- <filters...>]',
+ 'usage: plan --key <times key> [--fallback-key <times key>] [--tests <file>] --config <file> [--project <name>] --leg <i> --of <n> --out <file> [--durations <file>] [-- <filters...>]',
);
}
const files = list(suite, out, filters);
if (files.length === 0)
throw new Error(`Playwright listed no ${key} spec files`);
- const legsPlan = planLegs(files, readDurations(durations)[key] ?? {}, legs);
+ const recorded = readDurations(durations);
+ // The same costs the leg count used, so the split matches its estimate.
+ const legsPlan = planLegs(
+ files,
+ costsFor(files, recorded[key] ?? {}, {
+ fallback: recorded[option(own, '--fallback-key')] ?? {},
+ testsOf: testsFrom(option(own, '--tests')),
+ perTest: recorded['seconds-per-test']?.[key],
+ }),
+ legs,
+ );
const mine = legsPlan[leg - 1];
fs.writeFileSync(out, mine.files.map(file => `${file}\n`).join(''));
// Playwright runs nothing, and passes, for a test list that matches no
@@ -251,9 +366,9 @@ function legs(args) {
maxSeconds: Number(option(own, '--max-seconds')),
maxLegs: Number(option(own, '--max-legs')),
};
- if (!key || !Number.isInteger(leg) || leg < 1 || filters.length === 0)
+ if (!key || !Number.isInteger(leg) || leg < 1)
throw new Error(
- 'usage: legs --key <times key> --leg <i> --listed <file> --setup-seconds <s> --max-seconds <s> --max-legs <n> [--fallback-key <times key>] [--durations <file>] [--changed <file>] -- <filters...>',
+ 'usage: legs --key <times key> --leg <i> --listed <file> [--source <git dir>] [--tests-out <file>] --setup-seconds <s> --max-seconds <s> --max-legs <n> [--fallback-key <times key>] [--durations <file>] [--changed <file>] [-- <filters...>]',
);
const specs = file =>
fs.readFileSync(file, 'utf8').split('\n').filter(isInteractionSpec);
@@ -281,11 +396,23 @@ function legs(args) {
return;
}
const durations = readDurations(option(own, '--durations') ?? DURATIONS);
+ // A spec with no recorded time counts its tests, read from the merge
+ // commit (the scope checkout) without a checkout of its own.
+ const source = option(own, '--source');
+ const read = testsReader(source);
+ const counted = {};
+ const testsOf = file => (counted[file] = read(file));
const seconds = estimateSeconds(durations[key] ?? {}, filters, {
listed,
changed: changedFile ? specs(changedFile) : [],
fallback: durations[option(own, '--fallback-key')] ?? {},
+ testsOf,
+ perTest: durations['seconds-per-test']?.[key],
});
+ // The plan step runs after the head checkout has replaced the scope
+ // checkout, so it reads the counts from here.
+ const testsOut = option(own, '--tests-out');
+ if (testsOut) fs.writeFileSync(testsOut, `${JSON.stringify(counted)}\n`);
const count = legCount({seconds, ...limits});
const runs = leg <= count;
console.log(
@@ -311,9 +438,10 @@ function record(args) {
'usage: record --key <times key> [--project <name>] [--strip <path prefix>] <job log...>',
);
const seconds = {};
- for (const log of logs) {
+ const texts = logs.map(log => fs.readFileSync(log, 'utf8'));
+ for (const text of texts) {
for (const [file, value] of Object.entries(
- recordLog(fs.readFileSync(log, 'utf8'), project, strip),
+ recordLog(text, project, strip),
)) {
seconds[file] = (seconds[file] ?? 0) + value;
}
@@ -321,6 +449,11 @@ function record(args) {
if (Object.keys(seconds).length === 0)
throw new Error(`no ${key} test lines in the logs`);
const durations = readDurations();
+ // The mean time a test, for a spec that has no recorded time yet.
+ durations['seconds-per-test'] = {
+ ...durations['seconds-per-test'],
+ [key]: meanTestSeconds(texts, project, strip),
+ };
durations[key] = Object.fromEntries(
Object.entries(seconds)
.sort(([a], [b]) => (a < b ? -1 : 1))
diff --git a/.github/scripts/playwright-legs.test.mjs b/.github/scripts/playwright-legs.test.mjs
index 15bc4b8f5c..7951ba0516 100644
--- a/.github/scripts/playwright-legs.test.mjs
+++ b/.github/scripts/playwright-legs.test.mjs
@@ -10,11 +10,14 @@ import {describe, expect, it} from 'vitest';
import yaml from 'yaml';
import {
+ costsFor,
+ countTests,
estimateSeconds,
isInteractionSpec,
legCount,
planLegs,
recordLog,
+ testsReader,
} from './playwright-legs.mjs';
const root = path.resolve(
@@ -106,17 +109,57 @@ describe('estimateSeconds and legCount', () => {
// then the median of this suite's times.
expect(
estimateSeconds(times, ['internal/'], {
- listed,
+ listed: [...listed, 'new.spec.ts'],
changed: ['new.spec.ts'],
fallback: {'new.spec.ts': 12},
}),
).toBe(182);
expect(
estimateSeconds(times, ['internal/'], {
- listed,
+ listed: [...listed, 'other.spec.ts'],
changed: ['other.spec.ts'],
}),
).toBe(170 + 50);
+ // A changed spec the merge commit no longer holds (deleted, or renamed
+ // from) costs nothing, and its tests are never read.
+ expect(
+ estimateSeconds(times, ['internal/'], {
+ listed,
+ changed: ['gone.spec.ts'],
+ testsOf: () => {
+ throw new Error('read a deleted spec');
+ },
+ perTest: 1.5,
+ }),
+ ).toBe(170);
+ });
+
+ it('costs a spec with no recorded time by its tests at the mean time a test', () => {
+ // A spec with 50 tests at 1.56 s a test, not the 10.7 s median file.
+ const source = Array.from(
+ {length: 50},
+ (_, i) => ` test('case ${i}', async () => {});`,
+ ).join('\n');
+ expect(countTests(source)).toBe(50);
+ expect(
+ countTests(
+ "test.describe('group', () => {\n test.skip('a', () => {});\n it('b', () => {});\n test.use({});\n});",
+ ),
+ ).toBe(2);
+ const estimate = estimateSeconds(times, ['packages/lab/'], {
+ listed: [...listed, 'packages/lab/TabBar.spec.ts'],
+ testsOf: file => (file === 'packages/lab/TabBar.spec.ts' ? 50 : 0),
+ perTest: 1.56,
+ });
+ expect(estimate).toBeCloseTo(78, 5);
+ // With no tests to count, the file median stands.
+ expect(
+ estimateSeconds(times, ['packages/lab/'], {
+ listed: ['packages/lab/Empty.spec.ts'],
+ testsOf: () => 0,
+ perTest: 1.56,
+ }),
+ ).toBe(50);
});
it('counts a listed spec in the lane that has no recorded time yet', () => {
@@ -229,13 +272,16 @@ describe('the interaction proof legs in ci.yml', () => {
{
id: 'pr-interaction',
project: 'chromium',
- legs: [1, 2],
+ legs: [1, 2, 3, 4, 5, 6],
run: 'Run interaction proofs',
plan: "Plan this leg's interaction proofs",
evidence: 'interaction-evidence-${{ matrix.leg }}',
- gate: "steps.scope.outputs.applies == 'true'",
- leg: '${{ matrix.leg }}',
- of: '${{ strategy.job-total }}',
+ // As few legs as fit; see the leg counts below.
+ gate: "steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'",
+ leg: '${{ steps.legs.outputs.leg }}',
+ of: '${{ steps.legs.outputs.count }}',
+ count: "Count this run's legs",
+ fallback: 'webkit',
},
{
id: 'pr-interaction-webkit',
@@ -248,6 +294,8 @@ describe('the interaction proof legs in ci.yml', () => {
gate: "steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'",
leg: '${{ steps.legs.outputs.leg }}',
of: '${{ steps.legs.outputs.count }}',
+ count: "Count this run's WebKit legs",
+ fallback: 'chromium',
},
];
@@ -296,7 +344,10 @@ describe('the interaction proof legs in ci.yml', () => {
`cp .test-scope/${file} "$RUNNER_TEMP/interaction-legs/${dir}"`,
);
expect(plan.run).toContain(
- `--key interaction-${lane.project} --config playwright.interaction.config.ts --project ${lane.project}`,
+ `--key interaction-${lane.project} --fallback-key interaction-${lane.fallback} --tests "$RUNNER_TEMP/interaction-legs/tests.json"`,
+ );
+ expect(plan.run).toContain(
+ `--config playwright.interaction.config.ts --project ${lane.project}`,
);
expect(plan.run).toContain(
'--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',
@@ -347,13 +398,19 @@ describe('the interaction proof legs in ci.yml', () => {
'--leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}',
// Every spec file in the merge commit, recorded or not.
'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"',
- '--listed "$RUNNER_TEMP/interaction-legs/listed.txt"',
+ '--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',
])
expect(count.run).toContain(part);
});
// Runs the count step with the real planner on these recorded times.
- function countFor(leg, webkitTimes, changed = [], unrecorded = []) {
+ function countFor(
+ leg,
+ webkitTimes,
+ changed = [],
+ unrecorded = [],
+ {deleted = [], sources = {}, perTest} = {},
+ ) {
// The real path: the planner runs only when invoked as its own file.
const dir = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'webkit-legs-')),
@@ -373,13 +430,24 @@ describe('the interaction proof legs in ci.yml', () => {
JSON.stringify({
'interaction-webkit': webkitTimes,
'interaction-chromium': {},
+ ...(perTest
+ ? {'seconds-per-test': {'interaction-webkit': perTest}}
+ : {}),
}),
);
// The scope checkout: a repository whose tree holds the spec files.
const scope = path.join(dir, '.test-scope');
- for (const file of [...Object.keys(webkitTimes), ...unrecorded]) {
+ // A changed spec is in the merge commit too, unless it was deleted.
+ const present = (changed ?? []).filter(
+ file => isInteractionSpec(file) && !deleted.includes(file),
+ );
+ for (const file of [
+ ...Object.keys(webkitTimes),
+ ...unrecorded,
+ ...present,
+ ]) {
fs.mkdirSync(path.join(scope, path.dirname(file)), {recursive: true});
- fs.writeFileSync(path.join(scope, file), '');
+ fs.writeFileSync(path.join(scope, file), sources[file] ?? '');
}
const git = args =>
execFileSync('git', ['-C', scope, ...args], {stdio: 'ignore'});
@@ -421,18 +489,42 @@ describe('the interaction proof legs in ci.yml', () => {
},
},
);
- return Object.fromEntries(
+ const result = Object.fromEntries(
fs
.readFileSync(output, 'utf8')
.trim()
.split('\n')
.map(line => line.split('=')),
);
+ // The counts the plan step reads after the head checkout.
+ const testsFile = path.join(dir, 'interaction-legs/tests.json');
+ if (fs.existsSync(testsFile))
+ Object.defineProperty(result, 'tests', {
+ value: JSON.parse(fs.readFileSync(testsFile, 'utf8')),
+ });
+ return result;
} finally {
fs.rmSync(dir, {recursive: true, force: true});
}
}
+ it('writes the test counts it read for the plan, and skips a deleted spec', () => {
+ const times = {
+ 'internal/interaction-harness/src/harness.interaction.browser.spec.ts': 100,
+ };
+ const fresh =
+ 'internal/interaction-harness/src/fresh.interaction.browser.spec.ts';
+ const gone =
+ 'packages/core/src/List/__tests__/Gone.interaction.browser.spec.ts';
+ const result = countFor(2, times, [fresh, gone], [], {
+ deleted: [gone],
+ sources: {[fresh]: "test('a', () => {});\ntest('b', () => {});\n"},
+ perTest: 2,
+ });
+ expect(result).toEqual({count: '1', runs: 'false', leg: '2'});
+ expect(result.tests).toEqual({[fresh]: 2});
+ });
+
it('gives the whole suite to the first leg at today’s recorded times', () => {
const times = durations['interaction-webkit'];
expect(countFor(1, times)).toEqual({count: '1', runs: 'true', leg: '1'});
@@ -594,14 +686,132 @@ describe('the interaction proof legs in ci.yml', () => {
expect(leg.seconds).toBeLessThanOrEqual(6 * 60);
});
- it('records a time for each project the legs plan, each under the legs\u2019 per-runner budget', () => {
+ it("pins the Chromium count's arguments: every spec, the matrix's legs, the shared costs", () => {
+ const job = ci.jobs['pr-interaction'];
+ const count = step(job, "Count this run's legs");
+ expect(count.id).toBe('legs');
+ expect(count.if).toBe("steps.scope.outputs.applies == 'true'");
+ for (const part of [
+ 'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"',
+ '--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',
+ '--tests-out "$RUNNER_TEMP/interaction-legs/tests.json"',
+ '--key interaction-chromium --fallback-key interaction-webkit',
+ '--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',
+ '--setup-seconds 90 --max-seconds 600',
+ '--leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}',
+ ])
+ expect(count.run).toContain(part);
+ // No path filter: the Chromium lane counts every spec.
+ expect(count.run).not.toContain(' -- ');
+ const at = job.steps.indexOf(count);
+ expect(at).toBeGreaterThan(
+ job.steps.indexOf(
+ step(job, "Keep the leg planner from the workflow's commit"),
+ ),
+ );
+ expect(at).toBeLessThan(
+ job.steps.indexOf(step(job, 'Check out the heavy-suite caps')),
+ );
+ // Both plans cost an unrecorded spec as their counts do, from the
+ // counts the count step wrote: the head checkout has replaced the scope
+ // checkout by then.
+ expect(step(job, "Plan this leg's interaction proofs").run).toContain(
+ '--key interaction-chromium --fallback-key interaction-webkit --tests "$RUNNER_TEMP/interaction-legs/tests.json"',
+ );
+ const webkit = ci.jobs['pr-interaction-webkit'];
+ expect(step(webkit, "Count this run's WebKit legs").run).toContain(
+ '--tests-out "$RUNNER_TEMP/interaction-legs/tests.json"',
+ );
+ expect(
+ step(webkit, "Plan this leg's WebKit interaction proofs").run,
+ ).toContain(
+ '--key interaction-webkit --fallback-key interaction-chromium --tests "$RUNNER_TEMP/interaction-legs/tests.json"',
+ );
+ // The scope checkout is gone by the plan step.
+ for (const [id, name] of [
+ ['pr-interaction', "Plan this leg's interaction proofs"],
+ ['pr-interaction-webkit', "Plan this leg's WebKit interaction proofs"],
+ ])
+ expect(step(ci.jobs[id], name).run).not.toContain('.test-scope');
+ });
+
+ it('splits by the same costs it counts, and stops on a spec it cannot read', () => {
+ const times = {'a.spec.ts': 100};
+ expect(
+ costsFor(['a.spec.ts', 'b.spec.ts', 'c.spec.ts', 'd.spec.ts'], times, {
+ fallback: {'b.spec.ts': 40},
+ testsOf: file => (file === 'c.spec.ts' ? 10 : 0),
+ perTest: 1.5,
+ }),
+ ).toEqual({'a.spec.ts': 100, 'b.spec.ts': 40, 'c.spec.ts': 15});
+ const read = testsReader(root);
+ expect(read('.github/scripts/playwright-legs.test.mjs')).toBeGreaterThan(5);
+ expect(() => read('no/such.spec.ts')).toThrow(
+ /cannot read no\/such\.spec\.ts/,
+ );
+ expect(testsReader(undefined)('anything.spec.ts')).toBe(0);
+ });
+
+ it('fits every leg in ten minutes at today’s recorded times, within the matrix', () => {
+ // Every interaction spec in the tree, as the count steps list them.
+ const specs = execFileSync('git', ['ls-files'], {
+ cwd: root,
+ encoding: 'utf8',
+ })
+ .split('\n')
+ .filter(isInteractionSpec);
for (const lane of lanes) {
const times = durations[`interaction-${lane.project}`];
- expect(Object.keys(times).length, lane.project).toBeGreaterThan(0);
- const plan = planLegs(Object.keys(times), times, lane.legs.length);
- // About six minutes a leg under load; the recorded run was a loaded one.
- for (const leg of plan)
- expect(leg.seconds, lane.project).toBeLessThanOrEqual(6 * 60);
+ const fallback = durations[`interaction-${lane.fallback}`];
+ const count = step(ci.jobs[lane.id], lane.count);
+ const filters =
+ lane.project === 'webkit'
+ ? ci.jobs[lane.id].env.WEBKIT_SPEC_PATHS.split(' ')
+ : [];
+ const perTest =
+ durations['seconds-per-test'][`interaction-${lane.project}`];
+ const testsOf = file =>
+ countTests(fs.readFileSync(path.join(root, file), 'utf8'));
+ const seconds = estimateSeconds(times, filters, {
+ listed: specs,
+ fallback,
+ testsOf,
+ perTest,
+ });
+ const limits = {
+ setupSeconds: 90,
+ maxSeconds: 600,
+ maxLegs: lane.legs.length,
+ };
+ expect(count.run, lane.id).toContain(
+ '--setup-seconds 90 --max-seconds 600',
+ );
+ // The matrix still has room: the count stays under its legs.
+ const needed = Math.ceil(
+ seconds / (limits.maxSeconds - limits.setupSeconds),
+ );
+ expect(
+ needed,
+ `${lane.id}: ${Math.round(seconds)} s of proofs`,
+ ).toBeLessThan(lane.legs.length);
+ const legsNow = legCount({seconds, ...limits});
+ const planned = planLegs(
+ specs.filter(
+ file =>
+ filters.length === 0 || filters.some(each => file.includes(each)),
+ ),
+ Object.fromEntries(
+ specs.map(file => [
+ file,
+ times[file] ?? fallback[file] ?? testsOf(file) * perTest,
+ ]),
+ ),
+ legsNow,
+ );
+ for (const leg of planned)
+ expect(leg.seconds + limits.setupSeconds, lane.id).toBeLessThanOrEqual(
+ limits.maxSeconds + 60,
+ );
}
});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a12e059107..62b8a03a0c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2471,16 +2471,15 @@ jobs:
needs: [room, admission, heavy-ci, build-storybook]
if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}
# Frame evidence measures timing: four cores, one Playwright worker.
- # One runner ran every proof in about ten minutes and passed the job's
- # fifteen once its checkout was slow. Two legs share the spec files by
- # their recorded times (playwright-legs.mjs), so each runs in about five
- # minutes, and a leg with the slowest checkout seen still ends with about
- # four minutes to spare. `test` reads the job's result, which is success
- # only when every leg passed.
+ # The legs share the spec files by their recorded times
+ # (playwright-legs.mjs), as few legs as keep each leg's setup and proofs
+ # within ten minutes, which leaves a third of the job's fifteen for a slow
+ # runner or checkout. A leg with no share passes. `test` reads the job's
+ # result, which is success only when every leg passed.
strategy:
fail-fast: false
matrix:
- leg: [1, 2]
+ leg: [1, 2, 3, 4, 5, 6]
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -2538,11 +2537,33 @@ jobs:
cp .test-scope/.github/scripts/lane-environment.mjs "$RUNNER_TEMP/interaction-legs/.github/scripts/"
cp .test-scope/.github/lane-environments.json "$RUNNER_TEMP/interaction-legs/.github/"
+ # The legs this run needs: every spec file in the merge commit, which the
+ # scope checkout fetched whole, at its recorded time (a WebKit time, or
+ # its test count times the mean time a test, for one with none), plus
+ # 90 s of setup a leg, within ten minutes a leg. Each leg decides the
+ # same way before its own setup.
+ - name: Count this run's legs
+ id: legs
+ if: steps.scope.outputs.applies == 'true'
+ run: |
+ git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"
+ node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" legs \
+ --listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope \
+ --tests-out "$RUNNER_TEMP/interaction-legs/tests.json" \
+ --key interaction-chromium --fallback-key interaction-webkit \
+ --durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
+ --setup-seconds 90 --max-seconds 600 \
+ --leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }}
+
+ - name: Leave the proofs to the other legs
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs != 'true'
+ run: echo "This run's interaction proofs fit ${{ steps.legs.outputs.count }} legs, so this leg runs none."
+
# The caps step comes from the merge commit, not the pull request's
# head: a head that predates the step does not have it. The head
# checkout below replaces this directory.
- name: Check out the heavy-suite caps
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: actions/checkout@v7
with:
path: .heavy-caps
@@ -2550,22 +2571,22 @@ jobs:
persist-credentials: false
- name: Per-author heavy-suite caps
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: ./.heavy-caps/.github/actions/heavy-caps
# The daily heavy-run cap counts this step (spec:AST-030 FR16).
- name: Heavy suite runs
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
run: 'true'
- name: Checkout
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Verify evidence checkout
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
env:
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA"
@@ -2573,46 +2594,47 @@ jobs:
# The head's specs run in this lane's environment, which comes from the
# merge commit. A head older than that environment stops here.
- name: Require this lane's environment in the branch
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
run: >-
node "$RUNNER_TEMP/interaction-legs/.github/scripts/lane-environment.mjs" check --lane interaction-chromium
--workflow "$RUNNER_TEMP/interaction-legs/.github/lane-environments.json"
--head .github/lane-environments.json
- name: Setup Node and pnpm
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: ./.github/actions/setup
with:
install: 'false'
- name: Download Storybook artifact
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: actions/download-artifact@v8
with:
name: storybook-${{ needs.build-storybook.outputs.short_hash }}
path: apps/storybook/dist
- name: Install Playwright
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
run: pnpm install --frozen-lockfile && npx playwright install chromium
# This leg's share of the spec files, by their recorded times. Every
# spec file is in exactly one leg; one with no recorded time counts the
- # median. The planner, the shared splitter it imports and the times come
+ # test count the leg count read, times the mean time a test. The planner, the shared splitter it imports and the times come
# from the scope checkout, the workflow's own commit (kept above): a
# head cut before them has none.
- name: Plan this leg's interaction proofs
id: plan
- if: steps.scope.outputs.applies == 'true'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
run: |
node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" plan \
- --key interaction-chromium --config playwright.interaction.config.ts --project chromium \
- --leg ${{ matrix.leg }} --of ${{ strategy.job-total }} \
+ --key interaction-chromium --fallback-key interaction-webkit --tests "$RUNNER_TEMP/interaction-legs/tests.json" \
+ --config playwright.interaction.config.ts --project chromium \
+ --leg ${{ steps.legs.outputs.leg }} --of ${{ steps.legs.outputs.count }} \
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
--out "$RUNNER_TEMP/interaction-leg.txt"
- name: Run interaction proofs
- if: steps.scope.outputs.applies == 'true' && steps.plan.outputs.files != '0'
+ if: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true' && steps.plan.outputs.files != '0'
run: |
pnpm exec playwright test \
--test-list "$RUNNER_TEMP/interaction-leg.txt" \
@@ -2622,7 +2644,7 @@ jobs:
# Storyboards (PNG contact sheets) and curve traces, for review.
- name: Upload interaction evidence
- if: always() && steps.scope.outputs.applies == 'true'
+ if: always() && steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs == 'true'
uses: actions/upload-artifact@v7
with:
name: interaction-evidence-${{ matrix.leg }}
@@ -2648,7 +2670,7 @@ jobs:
# Frame evidence measures timing: four cores, one Playwright worker.
# One leg runs every spec while the planner's estimate fits; past that two
# legs share the spec files by their recorded times, as the Chromium
- # proofs' two do.
+ # proofs' legs do.
strategy:
fail-fast: false
matrix:
@@ -2749,7 +2771,8 @@ jobs:
read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"
git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"
node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" legs \
- --listed "$RUNNER_TEMP/interaction-legs/listed.txt" \
+ --listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope \
+ --tests-out "$RUNNER_TEMP/interaction-legs/tests.json" \
--key interaction-webkit --fallback-key interaction-chromium \
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
--changed "${CHANGED_FILES:-}" \
@@ -2850,7 +2873,8 @@ jobs:
echo "::warning title=Changed specs not planned::The scope step could not list the changed files, so a changed interaction spec outside this lane's paths does not run here."
fi
node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" plan \
- --key interaction-webkit --config playwright.interaction.config.ts --project webkit \
+ --key interaction-webkit --fallback-key interaction-chromium --tests "$RUNNER_TEMP/interaction-legs/tests.json" \
+ --config playwright.interaction.config.ts --project webkit \
--leg ${{ steps.legs.outputs.leg }} --of ${{ steps.legs.outputs.count }} \
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
--out "$RUNNER_TEMP/interaction-leg.txt" -- \