Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: interaction lanes cap their legs at ten, and the leg test warns when a plan needs the last one (#10677)

Cindy Zhang7299aabd7f44
diff --git a/.github/scripts/playwright-legs.mjs b/.github/scripts/playwright-legs.mjs
index 59325ae181..2aa127ef75 100644
--- a/.github/scripts/playwright-legs.mjs
+++ b/.github/scripts/playwright-legs.mjs
@@ -63,6 +63,18 @@ export function planLegs(files, durations, legs) {
}));
}
+/**
+ * A note that the proofs need the last leg the matrix allows, with the room
+ * left before it is full; null while a whole leg of room remains, and null
+ * past the ceiling, which `legsFitMessage` reports.
+ */
+export function nearCeilingMessage({seconds, setupSeconds, maxSeconds, maxLegs}) {
+ const needed = legsNeeded({seconds, setupSeconds, maxSeconds});
+ if (needed !== maxLegs) return null;
+ const room = Math.round(maxLegs * (maxSeconds - setupSeconds) - seconds);
+ return `${Math.round(seconds)} s of proofs need ${needed} of the ${maxLegs} legs; ${room} s of room is left before the ceiling`;
+}
+
/**
* Why a suite's proofs do not fit its matrix, or null when they do: the leg
* count the planner takes for `seconds` must be no more than `maxLegs`, and
diff --git a/.github/scripts/playwright-legs.test.mjs b/.github/scripts/playwright-legs.test.mjs
index 8ba466ccfb..5a512a4dc7 100644
--- a/.github/scripts/playwright-legs.test.mjs
+++ b/.github/scripts/playwright-legs.test.mjs
@@ -17,6 +17,7 @@ import {
isInteractionSpec,
legCount,
legsFitMessage,
+ nearCeilingMessage,
planLegs,
recordLog,
selectProofs,
@@ -39,6 +40,20 @@ const durations = JSON.parse(
);
const step = (job, name) => job.steps.find(each => each.name === name);
+describe('nearCeilingMessage', () => {
+ const limits = {setupSeconds: 90, maxSeconds: 600, maxLegs: 6};
+
+ it('stays quiet while a whole leg of room is left', () => {
+ expect(nearCeilingMessage({seconds: 2549, ...limits})).toBeNull();
+ });
+
+ it('names the room left once the proofs need the last leg', () => {
+ expect(nearCeilingMessage({seconds: 2600, ...limits})).toBe(
+ '2600 s of proofs need 6 of the 6 legs; 460 s of room is left before the ceiling',
+ );
+ });
+});
+
describe('legsFitMessage', () => {
const limits = {setupSeconds: 90, maxSeconds: 600, maxLegs: 3};
const costs = {'a.spec.ts': 500, 'b.spec.ts': 500, 'c.spec.ts': 480};
@@ -324,7 +339,7 @@ describe('the interaction leg count job in ci.yml', () => {
it('checks out and keeps the planner exactly as each lane does', () => {
// A file the lanes' planner needs but this job did not copy would fail
- // its count and send every run back to six legs.
+ // its count and send every run back to the ten-leg fallback.
for (const lane of lanes) {
const job = ci.jobs[lane.id];
expect(counter.steps[0]).toEqual(job.steps[0]);
@@ -344,7 +359,7 @@ describe('the interaction leg count job in ci.yml', () => {
const mine = step(counter, lane.mine);
expect(mine.id).toBe(`count-${lane.key}`);
expect(mine.if).toBe(`steps.scope-${lane.key}.outputs.applies == 'true'`);
- // A failed count falls back to six legs that each decide for themselves.
+ // A failed count falls back to ten legs that each decide for themselves.
expect(mine['continue-on-error']).toBe(true);
expect(mine.env ?? {}).toEqual(
JSON.parse(
@@ -352,11 +367,11 @@ describe('the interaction leg count job in ci.yml', () => {
),
);
expect(mine.run).toBe(theirs.run.replaceAll('${{ matrix.leg }}', '1'));
- expect(theirs.run).toContain('--max-legs 6');
+ expect(theirs.run).toContain('--max-legs 10');
});
}
- it('names each lane one leg out of scope, its count in scope, and six when the count failed', () => {
+ it('names each lane one leg out of scope, its count in scope, and ten when the count failed', () => {
const name = step(counter, "Name each lane's legs");
expect(name.id).toBe('matrix');
const outputs = (chromium, webkit) => {
@@ -381,16 +396,34 @@ describe('the interaction leg count job in ci.yml', () => {
}
};
expect(outputs(['success', '3'], ['success', '0'])).toBe('chromium=[1, 2, 3]\nwebkit=[1]\n');
- expect(outputs(['skipped', ''], ['failure', ''])).toBe('chromium=[1]\nwebkit=[1, 2, 3, 4, 5, 6]\n');
- expect(outputs(['success', '9'], ['success', '6'])).toBe(
- 'chromium=[1, 2, 3, 4, 5, 6]\nwebkit=[1, 2, 3, 4, 5, 6]\n',
+ expect(outputs(['skipped', ''], ['failure', ''])).toBe(
+ 'chromium=[1]\nwebkit=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n',
);
+ expect(outputs(['success', '12'], ['success', '6'])).toBe(
+ 'chromium=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\nwebkit=[1, 2, 3, 4, 5, 6]\n',
+ );
+ });
+
+ it('caps every count at the ceiling the lanes\' fallback matrix lists, ten legs', () => {
+ const ceiling = matrixFor(ci.jobs['pr-interaction'].strategy.matrix.leg, 'pull_request').length;
+ expect(ceiling).toBe(10);
+ expect(matrixFor(ci.jobs['pr-interaction-webkit'].strategy.matrix.leg, 'pull_request')).toHaveLength(ceiling);
+ for (const lane of lanes) {
+ expect(step(ci.jobs[lane.id], lane.count).run).toContain(`--max-legs ${ceiling}`);
+ expect(step(counter, lane.mine).run).toContain(`--max-legs ${ceiling}`);
+ }
+ const name = step(counter, "Name each lane's legs").run;
+ expect(name).toContain(`*) count=${ceiling} ;;`);
+ expect(name).toContain(`[ "$count" -le ${ceiling} ] || count=${ceiling}`);
});
it('reads a counted matrix as its fallback list, the most legs it starts', () => {
expect(
- matrixFor("${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6]') }}", 'pull_request'),
- ).toEqual([1, 2, 3, 4, 5, 6]);
+ matrixFor(
+ "${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}",
+ 'pull_request',
+ ),
+ ).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
});
});
@@ -399,7 +432,7 @@ describe('the interaction proof legs in ci.yml', () => {
{
id: 'pr-interaction',
project: 'chromium',
- legs: [1, 2, 3, 4, 5, 6],
+ legs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
matrix: 'chromium',
run: 'Run interaction proofs',
plan: "Plan this leg's interaction proofs",
@@ -414,7 +447,7 @@ describe('the interaction proof legs in ci.yml', () => {
{
id: 'pr-interaction-webkit',
project: 'webkit',
- legs: [1, 2, 3, 4, 5, 6],
+ legs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
matrix: 'webkit',
run: 'Run WebKit interaction proofs',
plan: "Plan this leg's WebKit interaction proofs",
@@ -526,7 +559,7 @@ describe('the interaction proof legs in ci.yml', () => {
'--key interaction-webkit --fallback-key interaction-chromium',
'--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',
'--setup-seconds 90 --max-seconds 600',
- '--leg ${{ matrix.leg }} --max-legs 6',
+ '--leg ${{ matrix.leg }} --max-legs 10',
// Every spec file in the merge commit, recorded or not.
'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"',
'--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',
@@ -905,7 +938,7 @@ describe('the interaction proof legs in ci.yml', () => {
'--key interaction-chromium --fallback-key interaction-webkit',
'--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"',
'--setup-seconds 90 --max-seconds 600',
- '--leg ${{ matrix.leg }} --max-legs 6',
+ '--leg ${{ matrix.leg }} --max-legs 10',
])
expect(count.run).toContain(part);
// No path filter: the Chromium lane counts every spec.
@@ -1002,6 +1035,11 @@ describe('the interaction proof legs in ci.yml', () => {
// The planner fits the matrix: no more legs than it has, and no leg
// over ten minutes (plus one spec's slack). It fails naming the
// heaviest specs, not by a margin under the matrix.
+ // Within one leg of the ceiling: a warning in the log, not a failure,
+ // so the next step is proposed before the guard trips.
+ const near = nearCeilingMessage({seconds, ...limits});
+ if (near)
+ console.log(`::warning title=Interaction legs near the ceiling::${label}: ${near}`);
expect(
legsFitMessage({
seconds,
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8a3db19173..479f0f1913 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2536,7 +2536,7 @@ jobs:
delete-merged: true
# How many legs each interaction lane starts. It counts each lane the way
- # its legs do (same scope, planner and recorded times, at most six legs),
+ # its legs do (same scope, planner and recorded times, at most ten legs),
# so a lane starts only the legs that have a share. A leg fails when its
# own count needs more legs than started.
interaction-legs:
@@ -2625,7 +2625,7 @@ jobs:
--key interaction-chromium --fallback-key interaction-webkit \
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
--setup-seconds 90 --max-seconds 600 \
- --leg 1 --max-legs 6
+ --leg 1 --max-legs 10
- name: Count the WebKit legs
id: count-webkit
@@ -2660,11 +2660,11 @@ jobs:
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
"${CHANGED[@]}" \
--setup-seconds 90 --max-seconds 600 \
- --leg 1 --max-legs 6 \
+ --leg 1 --max-legs 10 \
-- "${SELECTED[@]}"
# The matrix each lane starts: its counted legs, one leg (which skips)
- # outside its scope or with nothing to run, and all six when its count
+ # outside its scope or with nothing to run, and all ten when its count
# failed, so each leg decides for itself as before.
- name: Name each lane's legs
id: matrix
@@ -2679,9 +2679,9 @@ jobs:
case "$outcome" in
success) [ "${count:-0}" -ge 1 ] 2>/dev/null || count=1 ;;
skipped) count=1 ;;
- *) count=6 ;;
+ *) count=10 ;;
esac
- [ "$count" -le 6 ] || count=6
+ [ "$count" -le 10 ] || count=10
local list=1 leg
for ((leg = 2; leg <= count; leg++)); do list="$list, $leg"; done
printf '[%s]' "$list"
@@ -2691,7 +2691,7 @@ jobs:
for lane in "Chromium:$CHROMIUM_OUTCOME" "WebKit:$WEBKIT_OUTCOME"; do
case "${lane#*:}" in
success | skipped) ;;
- *) echo "::warning title=Leg count fell back to six::The ${lane%%:*} leg count failed, so its lane starts six legs that each decide for themselves." ;;
+ *) echo "::warning title=Leg count fell back to ten::The ${lane%%:*} leg count failed, so its lane starts ten legs that each decide for themselves." ;;
esac
done
cat "$GITHUB_OUTPUT"
@@ -2705,12 +2705,12 @@ jobs:
# (playwright-legs.mjs), as few legs as keep each leg's setup and proofs
# within ten minutes, which leaves a third of the job's fifteen for a slow
# runner or checkout; interaction-legs starts only those legs, at most
- # six. `test` reads the job's result, which is success only when every
+ # ten. `test` reads the job's result, which is success only when every
# leg passed.
strategy:
fail-fast: false
matrix:
- leg: ${{ fromJSON(needs.interaction-legs.outputs.chromium || '[1, 2, 3, 4, 5, 6]') }}
+ leg: ${{ fromJSON(needs.interaction-legs.outputs.chromium || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -2786,7 +2786,7 @@ jobs:
--key interaction-chromium --fallback-key interaction-webkit \
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
--setup-seconds 90 --max-seconds 600 \
- --leg ${{ matrix.leg }} --max-legs 6
+ --leg ${{ matrix.leg }} --max-legs 10
- name: Require a leg for every share
if: steps.scope.outputs.applies == 'true' && fromJSON(steps.legs.outputs.count || '0') > strategy.job-total
@@ -2923,12 +2923,12 @@ jobs:
if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}
# Frame evidence measures timing: four cores, one Playwright worker.
# The leg count plans as many legs as keep each one's setup and proofs
- # within ten minutes, as the Chromium proofs' count does, at most six;
+ # within ten minutes, as the Chromium proofs' count does, at most ten;
# interaction-legs starts only those legs.
strategy:
fail-fast: false
matrix:
- leg: ${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6]') }}
+ leg: ${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}
runs-on: ubuntu-latest
timeout-minutes: 15
# Playwright's own image for the version the lockfile resolves, with
@@ -3024,7 +3024,7 @@ jobs:
# One leg runs every proof while its setup and the recorded times fit
# ten minutes, which leaves a third of the job's limit for a slow
- # runner. Past that the planner adds legs, up to the matrix, so a
+ # runner. Past that the planner adds legs, up to the ten-leg ceiling, so a
# growing suite or a wide change does not reach the limit. Each leg decides the same way, before its
# own setup, from the workflow's times and the changed files. The spec
# files come from the merge commit's tree, which the scope checkout
@@ -3061,7 +3061,7 @@ jobs:
--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \
"${CHANGED[@]}" \
--setup-seconds 90 --max-seconds 600 \
- --leg ${{ matrix.leg }} --max-legs 6 \
+ --leg ${{ matrix.leg }} --max-legs 10 \
-- "${SELECTED[@]}"
- name: Require a leg for every share