An open source design system that's fully customizable and agent ready
TypeScript
ci: the WebKit interaction proofs plan as many legs as fit ten minutes each, up to five (#10319)
Cindy Zhang39b5e0e457e4
diff --git a/.github/scripts/ci-interaction-webkit.test.mjs b/.github/scripts/ci-interaction-webkit.test.mjsindex 0a7edc9afb..87dc7af02d 100644--- a/.github/scripts/ci-interaction-webkit.test.mjs+++ b/.github/scripts/ci-interaction-webkit.test.mjs@@ -96,7 +96,7 @@ describe('WebKit interaction proof routing', () => {const guards = new Set(["Require this pull request's Storybook build",'Skip the WebKit proofs outside their scope',- 'Leave the proofs to the first leg',+ 'Leave the proofs to the other legs',]);// Every step after the leg count also waits for this leg to have a share.const counted = webkit.steps.indexOf(@@ -113,7 +113,7 @@ describe('WebKit interaction proof routing', () => {),);}- expect(step(webkit, 'Leave the proofs to the first leg').if).toBe(+ expect(step(webkit, 'Leave the proofs to the other legs').if).toBe(`${applies} && steps.legs.outputs.runs != 'true'`,);});diff --git a/.github/scripts/playwright-legs.test.mjs b/.github/scripts/playwright-legs.test.mjsindex 7951ba0516..2f7ffd07f0 100644--- a/.github/scripts/playwright-legs.test.mjs+++ b/.github/scripts/playwright-legs.test.mjs@@ -286,7 +286,7 @@ describe('the interaction proof legs in ci.yml', () => {{id: 'pr-interaction-webkit',project: 'webkit',- legs: [1, 2],+ legs: [1, 2, 3, 4, 5],run: 'Run WebKit interaction proofs',plan: "Plan this leg's WebKit interaction proofs",evidence: 'interaction-evidence-webkit-${{ matrix.leg }}',@@ -399,6 +399,11 @@ describe('the interaction proof legs in ci.yml', () => {// Every spec file in the merge commit, recorded or not.'git -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"','--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope',+ // The changed files, except for the release check, which plans every+ // spec already.+ 'CHANGED=(--changed "${CHANGED_FILES:-}")',+ 'if [ "${WEBKIT_ALL_SPECS:-}" = true ]; then CHANGED=(); fi',+ '"${CHANGED[@]}"',])expect(count.run).toContain(part);});@@ -409,7 +414,7 @@ describe('the interaction proof legs in ci.yml', () => {webkitTimes,changed = [],unrecorded = [],- {deleted = [], sources = {}, perTest} = {},+ {deleted = [], sources = {}, perTest, env = {}} = {},) {// The real path: the planner runs only when invoked as its own file.const dir = fs.realpathSync(@@ -475,7 +480,10 @@ describe('the interaction proof legs in ci.yml', () => {'-c',count.run.replaceAll('${{ matrix.leg }}', String(leg))- .replaceAll('${{ strategy.job-total }}', '2'),+ .replaceAll(+ '${{ strategy.job-total }}',+ String(job.strategy.matrix.leg.length),+ ),],{cwd: dir,@@ -486,6 +494,7 @@ describe('the interaction proof legs in ci.yml', () => {CHANGED_FILES: list,GITHUB_OUTPUT: output,WEBKIT_SPEC_PATHS: job.env.WEBKIT_SPEC_PATHS,+ ...env,},},);@@ -508,6 +517,62 @@ describe('the interaction proof legs in ci.yml', () => {}}+ it("plans every WebKit spec only for the release tooling's interaction-check/ pull request", () => {+ expect(job.env.WEBKIT_ALL_SPECS).toBe(+ "${{ startsWith(github.head_ref, 'interaction-check/') && github.event.pull_request.user.login == 'cixzhang' && github.event.pull_request.head.repo.full_name == github.repository }}",+ );+ // The expression's three conditions, read against a pull request.+ const fullScope = pr =>+ job.env.WEBKIT_ALL_SPECS.slice(3, -2)+ .trim()+ .split(' && ')+ .every(condition => {+ const value = name =>+ name.startsWith("'") ? name.slice(1, -1) : pr[name];+ const prefix = /^startsWith\(([\w.]+), '([^']+)'\)$/.exec(+ condition,+ );+ if (prefix) return String(value(prefix[1])).startsWith(prefix[2]);+ const [left, right] = condition.split(' == ');+ return value(left) === value(right);+ });+ const tooling = {+ 'github.head_ref': 'interaction-check/3bd5a4c1b4',+ 'github.event.pull_request.user.login': 'cixzhang',+ 'github.event.pull_request.head.repo.full_name': 'meta/astryx',+ 'github.repository': 'meta/astryx',+ };+ expect(fullScope(tooling)).toBe(true);+ // The same branch prefix from another author, or from a fork, keeps+ // the normal scope; so does the tooling account on another branch.+ expect(+ fullScope({+ ...tooling,+ 'github.event.pull_request.user.login': 'someone',+ }),+ ).toBe(false);+ expect(+ fullScope({+ ...tooling,+ 'github.event.pull_request.head.repo.full_name': 'someone/astryx',+ }),+ ).toBe(false);+ expect(+ fullScope({...tooling, 'github.head_ref': 'fix/interaction-check'}),+ ).toBe(false);+ // A spec outside the WebKit paths, long enough for two legs.+ const times = {+ 'packages/core/src/Button/__tests__/Button.interaction.browser.spec.ts': 700,+ };+ expect(countFor(1, times)).toEqual({count: '1', runs: 'true', leg: '1'});+ expect(+ countFor(1, times, [], [], {env: {WEBKIT_ALL_SPECS: 'true'}}),+ ).toEqual({count: '2', runs: 'true', leg: '1'});+ expect(+ countFor(1, times, [], [], {env: {WEBKIT_ALL_SPECS: 'false'}}),+ ).toEqual({count: '1', runs: 'true', leg: '1'});+ });+it('writes the test counts it read for the plan, and skips a deleted spec', () => {const times = {'internal/interaction-harness/src/harness.interaction.browser.spec.ts': 100,@@ -561,6 +626,18 @@ describe('the interaction proof legs in ci.yml', () => {).toEqual({count: '2', runs: 'true', leg: '2'});});+ it('splits the whole suite for the release check even without a changed-file list', () => {+ // With every spec planned, a missing list must not make each leg run+ // the whole suite on its own.+ const times = {+ 'packages/core/src/Button/__tests__/Button.interaction.browser.spec.ts': 700,+ };+ for (const leg of [1, 2])+ expect(+ countFor(leg, times, null, [], {env: {WEBKIT_ALL_SPECS: 'true'}}),+ ).toEqual({count: '2', runs: 'true', leg: String(leg)});+ });+it('runs every proof in a leg that has no changed-file list, as a leg of its own', () => {// Whatever the other leg counts, every proof then runs in some leg.for (const leg of [1, 2])@@ -603,7 +680,7 @@ describe('the interaction proof legs in ci.yml', () => {]) {const {run} = step(job, name);expect(run, name).toContain(- 'read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"',+ 'if [ "${WEBKIT_ALL_SPECS:-}" = true ]; then SPEC_PATHS=(packages/ internal/); else read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"; fi',);expect(run, name).toContain('"${SPEC_PATHS[@]}"');}@@ -772,12 +849,6 @@ describe('the interaction proof legs in ci.yml', () => {durations['seconds-per-test'][`interaction-${lane.project}`];const testsOf = file =>countTests(fs.readFileSync(path.join(root, file), 'utf8'));- const seconds = estimateSeconds(times, filters, {- listed: specs,- fallback,- testsOf,- perTest,- });const limits = {setupSeconds: 90,maxSeconds: 600,@@ -786,32 +857,47 @@ describe('the interaction proof legs in ci.yml', () => {expect(count.run, lane.id).toContain('--setup-seconds 90 --max-seconds 600',);- // The matrix still has room: the count stays under its legs.- const needed = Math.ceil(- seconds / (limits.maxSeconds - limits.setupSeconds),- );- expect(- needed,- `${lane.id}: ${Math.round(seconds)} s of proofs`,- ).toBeLessThan(lane.legs.length);- const legsNow = legCount({seconds, ...limits});- const planned = planLegs(- specs.filter(- file =>- filters.length === 0 || filters.some(each => file.includes(each)),- ),- Object.fromEntries(- specs.map(file => [- file,- times[file] ?? fallback[file] ?? testsOf(file) * perTest,- ]),- ),- legsNow,- );- for (const leg of planned)- expect(leg.seconds + limits.setupSeconds, lane.id).toBeLessThanOrEqual(- limits.maxSeconds + 60,+ // WebKit also runs every changed spec: a release-wide change, which+ // touches them all, must fit too.+ const changes = lane.project === 'webkit' ? [[], specs] : [[]];+ for (const changed of changes) {+ const label = `${lane.id}, ${changed.length} changed specs`;+ const seconds = estimateSeconds(times, filters, {+ listed: specs,+ changed,+ fallback,+ testsOf,+ perTest,+ });+ // The matrix still has room: the count stays under its legs.+ const needed = Math.ceil(+ seconds / (limits.maxSeconds - limits.setupSeconds),);+ expect(+ needed,+ `${label}: ${Math.round(seconds)} s of proofs`,+ ).toBeLessThan(lane.legs.length);+ const legsNow = legCount({seconds, ...limits});+ const planned = planLegs(+ specs.filter(+ file =>+ filters.length === 0 ||+ filters.some(each => file.includes(each)) ||+ changed.includes(file),+ ),+ Object.fromEntries(+ specs.map(file => [+ file,+ times[file] ?? fallback[file] ?? testsOf(file) * perTest,+ ]),+ ),+ legsNow,+ );+ for (const leg of planned)+ expect(leg.seconds + limits.setupSeconds, label).toBeLessThanOrEqual(+ limits.maxSeconds + 60,+ );+ }}});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex d771ab55ce..a748a835a9 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -2681,13 +2681,13 @@ jobs:needs: [room, admission, heavy-ci, build-storybook]if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}# Frame evidence measures timing: four cores, one Playwright worker.- # One leg runs every spec while the planner's estimate fits; past that two- # legs share the spec files by their recorded times, as the Chromium- # proofs' legs do.+ # The leg count plans as many legs as keep each one's setup and proofs+ # within ten minutes, as the Chromium proofs' count does. The matrix is+ # the ceiling; a leg with no share passes at once.strategy:fail-fast: falsematrix:- leg: [1, 2]+ leg: [1, 2, 3, 4, 5]runs-on: ubuntu-latesttimeout-minutes: 15# Playwright's own image for the version the lockfile resolves, with@@ -2703,8 +2703,15 @@ jobs:run:shell: bashenv:- # The specs the WebKit legs run, besides each interaction spec the pull- # request changes.+ # The release interaction check's throwaway pull request runs every+ # interaction spec in WebKit: the release receipt needs the whole suite,+ # which the leg count spreads over the matrix. Only the release tooling+ # opens it: an interaction-check/ branch in this repository, opened by+ # the account the tooling uses. Any other pull request keeps the paths+ # below.+ WEBKIT_ALL_SPECS: ${{ startsWith(github.head_ref, 'interaction-check/') && github.event.pull_request.user.login == 'cixzhang' && github.event.pull_request.head.repo.full_name == github.repository }}+ # The specs the WebKit legs run otherwise, besides each interaction spec+ # the pull request changes.WEBKIT_SPEC_PATHS: >-packages/core/src/gestures/packages/core/src/motion/__tests__/animatedLayout.interaction@@ -2772,8 +2779,8 @@ jobs:# One leg runs every proof while its setup and the recorded times fit# ten minutes, which leaves a third of the job's limit for a slow- # runner. Past that the planner adds the second leg, so a growing suite- # does not reach the limit. Each leg decides the same way, before its+ # runner. Past that the planner adds legs, up to the matrix, so a+ # growing suite or a wide change does not reach the limit. Each leg decides the same way, before its# own setup, from the workflow's times and the changed files. The spec# files come from the merge commit's tree, which the scope checkout# fetched whole, so a spec with no recorded time still counts.@@ -2783,21 +2790,25 @@ jobs:env:CHANGED_FILES: ${{ steps.scope.outputs.changed_files_path }}run: |- read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"+ if [ "${WEBKIT_ALL_SPECS:-}" = true ]; then SPEC_PATHS=(packages/ internal/); else read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"; fi+ # With every spec planned, the changed files add nothing. Leave them+ # out, so a missing list cannot make each leg run the whole suite.+ CHANGED=(--changed "${CHANGED_FILES:-}")+ if [ "${WEBKIT_ALL_SPECS:-}" = true ]; then CHANGED=(); figit -C .test-scope ls-tree -r --name-only HEAD > "$RUNNER_TEMP/interaction-legs/listed.txt"node "$RUNNER_TEMP/interaction-legs/.github/scripts/playwright-legs.mjs" legs \--listed "$RUNNER_TEMP/interaction-legs/listed.txt" --source .test-scope \--tests-out "$RUNNER_TEMP/interaction-legs/tests.json" \--key interaction-webkit --fallback-key interaction-chromium \--durations "$RUNNER_TEMP/interaction-legs/.github/playwright-leg-durations.json" \- --changed "${CHANGED_FILES:-}" \+ "${CHANGED[@]}" \--setup-seconds 90 --max-seconds 600 \--leg ${{ matrix.leg }} --max-legs ${{ strategy.job-total }} \-- "${SPEC_PATHS[@]}"- - name: Leave the proofs to the first leg+ - name: Leave the proofs to the other legsif: steps.scope.outputs.applies == 'true' && steps.legs.outputs.runs != 'true'- run: echo "One leg holds this run's WebKit proofs (${{ steps.legs.outputs.count }} leg planned), so this leg runs none."+ run: echo "This run's WebKit proofs fit ${{ steps.legs.outputs.count }} legs, so this leg runs none."# The caps step comes from the merge commit, not the pull request's# head: a head that predates the step does not have it. The head@@ -2876,7 +2887,7 @@ jobs:env:CHANGED_FILES: ${{ steps.scope.outputs.changed_files_path }}run: |- read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"+ if [ "${WEBKIT_ALL_SPECS:-}" = true ]; then SPEC_PATHS=(packages/ internal/); else read -ra SPEC_PATHS <<< "$WEBKIT_SPEC_PATHS"; fiCHANGED_SPECS=()if [ -n "${CHANGED_FILES:-}" ] && [ -s "$CHANGED_FILES" ]; thenwhile IFS= read -r file; dodiff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.mdindex e784680ff1..ad3487ea1f 100644--- a/docs/specs/AST-030/spec.md+++ b/docs/specs/AST-030/spec.md@@ -581,6 +581,17 @@ they are separately named.help when the reference is unclear. A range it cannot read fails the check. Amaintainer handles a false positive through the normal fix-forward route,with no bypass.+- **FR28 — A release interaction check's throwaway pull request runs the full+ WebKit scope.** A release interaction check's throwaway pull request, on an+ `interaction-check/` branch that the release tooling creates for a named+ release attempt, runs the full WebKit browser spec scope. The workflow+ verifies the branch prefix, the creator identity and that the head repository+ is this one, so a pull request that only picks the prefix, or comes from a+ fork, does not qualify. The creator check matches the shared automation+ account, so another automated pull request from that account on an+ `interaction-check/` branch also qualifies; the prefix is reserved for the+ release tooling. Such pull requests stay non-mergeable, and this widens the+ scope of no other pull request (DEC-24).### Platform support@@ -645,6 +656,7 @@ The nightly perf lane is the one scheduled measurement outside them (FR18).| FR24 | `ci-heavy-lanes.test.mjs` cadence cases | release-check 170 and 176 minutes old; the unit lane hourly beside it; weekday 06:37 Pacific inside the window in standard and daylight time; the pre-cut check already started; the tip already checked; a weekend morning; before 06:30 and after 07:30 | a release-check starts inside the 175-minute window, the pre-cut check is skipped or starts twice, or it runs on a weekend || FR26 | `ci-lab-only-routing.test.mjs`, `change-scope.test.mjs` and `test-scope-action.test.mjs` Lab-only cases | a Lab-only change, with and without a Changeset; a new Lab component with its routes, discovery entry and story; Lab beside core, charts, a core Storybook story, a route or discovery entry for another component, an entry-file diff GitHub leaves out, the docsite, the Sandbox, a workflow, the lockfile, root configuration, internal tooling or a spec record; a rename into or out of Lab; an incomplete list; a base classifier without the lane; each required context, its owners, the UI legs' Lab tests, the UI and Node guard tests, the repository guards, the Storybook build and Lab typecheck, the heavy suites with and without a label, lint and an agent push | a mixed change takes the Lab-only lane; a new Lab component's own entry files make it mixed, or another component's do not; a changed existing story, a story for a component the change does not carry, or a renamed entry file keeps the lane; the CLI official-packages check, the route-lock check when the lock changed, or the Storybook build does not run for one; a required context skips or fails, or passes without its notice; a Lab test, a guard test, a repository guard, the Lab typecheck or the Storybook build does not run; an unlabelled heavy suite, the docsite or the full typecheck runs, or a label's suite cannot audit Lab; lint is skipped || FR27 | `check-public-text.test.mjs` over the check and the Lint workflow step | each identifier shape; a hashed name and host made up in the test; ordinary text; a finding in a commit message, an added line and the description; a match before the merge base; an empty range, a bad sha, a missing commit and a missing digest file | a reference passes; the output holds the matched text; a match on the base branch fails a later pull request; an unreadable range passes; the step runs the pull request's own copy |+| FR28 | `playwright-legs.test.mjs` creator, prefix and head-repository cases ("plans every WebKit spec only for the release tooling's interaction-check/ pull request") | release interaction-check PRs: the release tooling's pull request; the same prefix from another author or from a fork; the tooling account on another branch | A prefix-only, other-author or fork pull request gets the full WebKit scope, or the tooling's interaction-check pull request does not |## Decision log@@ -1147,6 +1159,19 @@ Rejected: dropping the five-component limit for every audit (the visual,accessibility and RTL audits of a wide change do not fit the default check'stime).+### DEC-24 — Release interaction-check pull requests run the full WebKit scope++**Reference:** `spec:AST-030/DEC-24`+**Decider:** the Chief with the release owner, `2026-11-06`++A release interaction check's throwaway pull request runs the full WebKit+browser spec scope (FR28). It qualifies only when its head branch starts with+`interaction-check/`, the release tooling's account opened it, and its head+repository is this repository. Such pull requests stay non-mergeable, and the+rule widens the scope of no other pull request.++Rejected: any pull request on the prefix (a prefix alone is spoofable).+## Open questionsNone.