An open source design system that's fully customizable and agent ready
TypeScript
ci: interaction lanes cap their legs at ten, and the leg test warns when a plan needs the last one (#10677)
Cindy Zhang7299aabd7f44
diff --git a/.github/scripts/playwright-legs.mjs b/.github/scripts/playwright-legs.mjsindex 59325ae181..2aa127ef75 100644--- a/.github/scripts/playwright-legs.mjs+++ b/.github/scripts/playwright-legs.mjs@@ -63,6 +63,18 @@ export function planLegs(files, durations, legs) {}));}+/**+ * A note that the proofs need the last leg the matrix allows, with the room+ * left before it is full; null while a whole leg of room remains, and null+ * past the ceiling, which `legsFitMessage` reports.+ */+export function nearCeilingMessage({seconds, setupSeconds, maxSeconds, maxLegs}) {+ const needed = legsNeeded({seconds, setupSeconds, maxSeconds});+ if (needed !== maxLegs) return null;+ const room = Math.round(maxLegs * (maxSeconds - setupSeconds) - seconds);+ return `${Math.round(seconds)} s of proofs need ${needed} of the ${maxLegs} legs; ${room} s of room is left before the ceiling`;+}+/*** Why a suite's proofs do not fit its matrix, or null when they do: the leg* count the planner takes for `seconds` must be no more than `maxLegs`, anddiff --git a/.github/scripts/playwright-legs.test.mjs b/.github/scripts/playwright-legs.test.mjsindex 8ba466ccfb..5a512a4dc7 100644--- a/.github/scripts/playwright-legs.test.mjs+++ b/.github/scripts/playwright-legs.test.mjs@@ -17,6 +17,7 @@ import {isInteractionSpec,legCount,legsFitMessage,+ nearCeilingMessage,planLegs,recordLog,selectProofs,@@ -39,6 +40,20 @@ const durations = JSON.parse();const step = (job, name) => job.steps.find(each => each.name === name);+describe('nearCeilingMessage', () => {+ const limits = {setupSeconds: 90, maxSeconds: 600, maxLegs: 6};++ it('stays quiet while a whole leg of room is left', () => {+ expect(nearCeilingMessage({seconds: 2549, ...limits})).toBeNull();+ });++ it('names the room left once the proofs need the last leg', () => {+ expect(nearCeilingMessage({seconds: 2600, ...limits})).toBe(+ '2600 s of proofs need 6 of the 6 legs; 460 s of room is left before the ceiling',+ );+ });+});+describe('legsFitMessage', () => {const limits = {setupSeconds: 90, maxSeconds: 600, maxLegs: 3};const costs = {'a.spec.ts': 500, 'b.spec.ts': 500, 'c.spec.ts': 480};@@ -324,7 +339,7 @@ describe('the interaction leg count job in ci.yml', () => {it('checks out and keeps the planner exactly as each lane does', () => {// A file the lanes' planner needs but this job did not copy would fail- // its count and send every run back to six legs.+ // its count and send every run back to the ten-leg fallback.for (const lane of lanes) {const job = ci.jobs[lane.id];expect(counter.steps[0]).toEqual(job.steps[0]);@@ -344,7 +359,7 @@ describe('the interaction leg count job in ci.yml', () => {const mine = step(counter, lane.mine);expect(mine.id).toBe(`count-${lane.key}`);expect(mine.if).toBe(`steps.scope-${lane.key}.outputs.applies == 'true'`);- // A failed count falls back to six legs that each decide for themselves.+ // A failed count falls back to ten legs that each decide for themselves.expect(mine['continue-on-error']).toBe(true);expect(mine.env ?? {}).toEqual(JSON.parse(@@ -352,11 +367,11 @@ describe('the interaction leg count job in ci.yml', () => {),);expect(mine.run).toBe(theirs.run.replaceAll('${{ matrix.leg }}', '1'));- expect(theirs.run).toContain('--max-legs 6');+ expect(theirs.run).toContain('--max-legs 10');});}- it('names each lane one leg out of scope, its count in scope, and six when the count failed', () => {+ it('names each lane one leg out of scope, its count in scope, and ten when the count failed', () => {const name = step(counter, "Name each lane's legs");expect(name.id).toBe('matrix');const outputs = (chromium, webkit) => {@@ -381,16 +396,34 @@ describe('the interaction leg count job in ci.yml', () => {}};expect(outputs(['success', '3'], ['success', '0'])).toBe('chromium=[1, 2, 3]\nwebkit=[1]\n');- expect(outputs(['skipped', ''], ['failure', ''])).toBe('chromium=[1]\nwebkit=[1, 2, 3, 4, 5, 6]\n');- expect(outputs(['success', '9'], ['success', '6'])).toBe(- 'chromium=[1, 2, 3, 4, 5, 6]\nwebkit=[1, 2, 3, 4, 5, 6]\n',+ expect(outputs(['skipped', ''], ['failure', ''])).toBe(+ 'chromium=[1]\nwebkit=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n',);+ expect(outputs(['success', '12'], ['success', '6'])).toBe(+ 'chromium=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\nwebkit=[1, 2, 3, 4, 5, 6]\n',+ );+ });++ it('caps every count at the ceiling the lanes\' fallback matrix lists, ten legs', () => {+ const ceiling = matrixFor(ci.jobs['pr-interaction'].strategy.matrix.leg, 'pull_request').length;+ expect(ceiling).toBe(10);+ expect(matrixFor(ci.jobs['pr-interaction-webkit'].strategy.matrix.leg, 'pull_request')).toHaveLength(ceiling);+ for (const lane of lanes) {+ expect(step(ci.jobs[lane.id], lane.count).run).toContain(`--max-legs ${ceiling}`);+ expect(step(counter, lane.mine).run).toContain(`--max-legs ${ceiling}`);+ }+ const name = step(counter, "Name each lane's legs").run;+ expect(name).toContain(`*) count=${ceiling} ;;`);+ expect(name).toContain(`[ "$count" -le ${ceiling} ] || count=${ceiling}`);});it('reads a counted matrix as its fallback list, the most legs it starts', () => {expect(- matrixFor("${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6]') }}", 'pull_request'),- ).toEqual([1, 2, 3, 4, 5, 6]);+ matrixFor(+ "${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}",+ 'pull_request',+ ),+ ).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);});});@@ -399,7 +432,7 @@ describe('the interaction proof legs in ci.yml', () => {{id: 'pr-interaction',project: 'chromium',- legs: [1, 2, 3, 4, 5, 6],+ legs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],matrix: 'chromium',run: 'Run interaction proofs',plan: "Plan this leg's interaction proofs",@@ -414,7 +447,7 @@ describe('the interaction proof legs in ci.yml', () => {{id: 'pr-interaction-webkit',project: 'webkit',- legs: [1, 2, 3, 4, 5, 6],+ legs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],matrix: 'webkit',run: 'Run WebKit interaction proofs',plan: "Plan this leg's WebKit interaction proofs",@@ -526,7 +559,7 @@ describe('the interaction proof legs in ci.yml', () => {'--key interaction-webkit --fallback-key interaction-chromium','--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"','--setup-seconds 90 --max-seconds 600',- '--leg ${{ matrix.leg }} --max-legs 6',+ '--leg ${{ matrix.leg }} --max-legs 10',// Every spec file in the merge commit, recorded or not.'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"','--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',@@ -905,7 +938,7 @@ describe('the interaction proof legs in ci.yml', () => {'--key interaction-chromium --fallback-key interaction-webkit','--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json"','--setup-seconds 90 --max-seconds 600',- '--leg ${{ matrix.leg }} --max-legs 6',+ '--leg ${{ matrix.leg }} --max-legs 10',])expect(count.run).toContain(part);// No path filter: the Chromium lane counts every spec.@@ -1002,6 +1035,11 @@ describe('the interaction proof legs in ci.yml', () => {// The planner fits the matrix: no more legs than it has, and no leg// over ten minutes (plus one spec's slack). It fails naming the// heaviest specs, not by a margin under the matrix.+ // Within one leg of the ceiling: a warning in the log, not a failure,+ // so the next step is proposed before the guard trips.+ const near = nearCeilingMessage({seconds, ...limits});+ if (near)+ console.log(`::warning title=Interaction legs near the ceiling::${label}: ${near}`);expect(legsFitMessage({seconds,diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 8a3db19173..479f0f1913 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -2536,7 +2536,7 @@ jobs:delete-merged: true# How many legs each interaction lane starts. It counts each lane the way- # its legs do (same scope, planner and recorded times, at most six legs),+ # its legs do (same scope, planner and recorded times, at most ten legs),# so a lane starts only the legs that have a share. A leg fails when its# own count needs more legs than started.interaction-legs:@@ -2625,7 +2625,7 @@ jobs:--key interaction-chromium --fallback-key interaction-webkit \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \--setup-seconds 90 --max-seconds 600 \- --leg 1 --max-legs 6+ --leg 1 --max-legs 10- name: Count the WebKit legsid: count-webkit@@ -2660,11 +2660,11 @@ jobs:--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \"${CHANGED[@]}" \--setup-seconds 90 --max-seconds 600 \- --leg 1 --max-legs 6 \+ --leg 1 --max-legs 10 \-- "${SELECTED[@]}"# The matrix each lane starts: its counted legs, one leg (which skips)- # outside its scope or with nothing to run, and all six when its count+ # outside its scope or with nothing to run, and all ten when its count# failed, so each leg decides for itself as before.- name: Name each lane's legsid: matrix@@ -2679,9 +2679,9 @@ jobs:case "$outcome" insuccess) [ "${count:-0}" -ge 1 ] 2>/dev/null || count=1 ;;skipped) count=1 ;;- *) count=6 ;;+ *) count=10 ;;esac- [ "$count" -le 6 ] || count=6+ [ "$count" -le 10 ] || count=10local list=1 legfor ((leg = 2; leg <= count; leg++)); do list="$list, $leg"; doneprintf '[%s]' "$list"@@ -2691,7 +2691,7 @@ jobs:for lane in "Chromium:$CHROMIUM_OUTCOME" "WebKit:$WEBKIT_OUTCOME"; docase "${lane#*:}" insuccess | skipped) ;;- *) echo "::warning title=Leg count fell back to six::The ${lane%%:*} leg count failed, so its lane starts six legs that each decide for themselves." ;;+ *) echo "::warning title=Leg count fell back to ten::The ${lane%%:*} leg count failed, so its lane starts ten legs that each decide for themselves." ;;esacdonecat "$GITHUB_OUTPUT"@@ -2705,12 +2705,12 @@ jobs:# (playwright-legs.mjs), as few legs as keep each leg's setup and proofs# within ten minutes, which leaves a third of the job's fifteen for a slow# runner or checkout; interaction-legs starts only those legs, at most- # six. `test` reads the job's result, which is success only when every+ # ten. `test` reads the job's result, which is success only when every# leg passed.strategy:fail-fast: falsematrix:- leg: ${{ fromJSON(needs.interaction-legs.outputs.chromium || '[1, 2, 3, 4, 5, 6]') }}+ leg: ${{ fromJSON(needs.interaction-legs.outputs.chromium || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}runs-on: ubuntu-latesttimeout-minutes: 15permissions:@@ -2786,7 +2786,7 @@ jobs:--key interaction-chromium --fallback-key interaction-webkit \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \--setup-seconds 90 --max-seconds 600 \- --leg ${{ matrix.leg }} --max-legs 6+ --leg ${{ matrix.leg }} --max-legs 10- name: Require a leg for every shareif: steps.scope.outputs.applies == 'true' && fromJSON(steps.legs.outputs.count || '0') > strategy.job-total@@ -2923,12 +2923,12 @@ jobs:if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}# Frame evidence measures timing: four cores, one Playwright worker.# The leg count plans as many legs as keep each one's setup and proofs- # within ten minutes, as the Chromium proofs' count does, at most six;+ # within ten minutes, as the Chromium proofs' count does, at most ten;# interaction-legs starts only those legs.strategy:fail-fast: falsematrix:- leg: ${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6]') }}+ leg: ${{ fromJSON(needs.interaction-legs.outputs.webkit || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }}runs-on: ubuntu-latesttimeout-minutes: 15# Playwright's own image for the version the lockfile resolves, with@@ -3024,7 +3024,7 @@ jobs:# One leg runs every proof while its setup and the recorded times fit# ten minutes, which leaves a third of the job's limit for a slow- # runner. Past that the planner adds legs, up to the matrix, so a+ # runner. Past that the planner adds legs, up to the ten-leg ceiling, so a# growing suite or a wide change does not reach the limit. Each leg decides the same way, before its# own setup, from the workflow's times and the changed files. The spec# files come from the merge commit's tree, which the scope checkout@@ -3061,7 +3061,7 @@ jobs:--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \"${CHANGED[@]}" \--setup-seconds 90 --max-seconds 600 \- --leg ${{ matrix.leg }} --max-legs 6 \+ --leg ${{ matrix.leg }} --max-legs 10 \-- "${SELECTED[@]}"- name: Require a leg for every share