An open source design system that's fully customizable and agent ready
TypeScript
ci: run the accessibility contracts at the head that built their Storybook (#9971)
Cindy Zhang54166ffe49c1
diff --git a/.github/scripts/a11y-pr-contracts.mjs b/.github/scripts/a11y-pr-contracts.mjsindex 019bcba623..c5441f6731 100644--- a/.github/scripts/a11y-pr-contracts.mjs+++ b/.github/scripts/a11y-pr-contracts.mjs@@ -610,9 +610,9 @@ export function changedSinceMergeBase(base) {}/**- * A spec's routing lines (and the audit config) come from the commit the- * workflow steps come from when it has the file (an older PR head predates- * current routing); otherwise from the checkout.+ * A spec's routing lines, the audit config and the recorded durations come+ * from the commit the workflow steps come from when it has the file (an older+ * PR head predates current routing); otherwise from the checkout.*/function readPolicySource(root, file, policySha) {if (policySha) {@@ -639,10 +639,18 @@ function readAuditConfig(root, file, policySha) {return JSON.parse(source);}-function readDurations(root) {- const file = path.join(root, '.github/a11y-contract-durations.json');- if (!fs.existsSync(file)) return {};- return fileSeconds(JSON.parse(fs.readFileSync(file, 'utf8')));+function readDurations(root, policySha) {+ let source;+ try {+ source = readPolicySource(+ root,+ '.github/a11y-contract-durations.json',+ policySha,+ );+ } catch {+ return {};+ }+ return fileSeconds(JSON.parse(source));}/** The Playwright worker count, as playwright.config.ts reads it. */@@ -765,7 +773,7 @@ function main() {components: ownedBy,changedPaths: readLines(changedFilesFile) ?? changedPaths,}),- seconds: readDurations(root),+ seconds: readDurations(root, policySha),workers: contractWorkers(),}): {run: routed, notRun: []};diff --git a/.github/scripts/a11y-pr-contracts.test.mjs b/.github/scripts/a11y-pr-contracts.test.mjsindex d77a0aa108..f4bf2c9dd7 100644--- a/.github/scripts/a11y-pr-contracts.test.mjs+++ b/.github/scripts/a11y-pr-contracts.test.mjs@@ -475,7 +475,7 @@ describe('the default check of the touched components', () => {expect(runner).toContain('changedPaths: readLines(changedFilesFile) ?? changedPaths',);- expect(runner).toContain('seconds: readDurations(root)');+ expect(runner).toContain('seconds: readDurations(root, policySha)');expect(runner).toContain('workers: contractWorkers()');});});diff --git a/.github/scripts/ci-head-checkout-scripts.test.mjs b/.github/scripts/ci-head-checkout-scripts.test.mjsindex 84b1af5775..04082b3dfc 100644--- a/.github/scripts/ci-head-checkout-scripts.test.mjs+++ b/.github/scripts/ci-head-checkout-scripts.test.mjs@@ -18,8 +18,9 @@* `head_file_present`, the step loads it from the workflow's own commit* (`git show "$WORKFLOW_SHA:<script>"`, or from the job's `.test-scope`* checkout, whose sparse directories hold it, directly or through a copy an- * earlier step made in `$RUNNER_TEMP`), or the step checks for it itself- * (`[ ! -f ... ]`). A job that does not measure the head takes the default+ * earlier step made in `$RUNNER_TEMP`), the step runs in the job's merge+ * checkout before its head checkout replaces the workspace, or the step checks+ * for it itself (`[ ! -f ... ]`). A job that does not measure the head takes the default* merge checkout instead, and this test does not apply to it.** Scope: `.js`, `.mjs` and `.cjs` scripts named in a step's `run:`.@@ -158,6 +159,24 @@ export function beforeHeadCheckout(job, step) {return head === -1 || steps.indexOf(step) < head;}+/**+ * Whether the step runs in the job's merge checkout (the workflow's own+ * commit): a workspace checkout with no `ref` comes before the step, and the+ * head checkout comes after it.+ */+export function inMergeCheckout(job, step) {+ const steps = job.steps ?? [];+ const merge = steps.findIndex(+ each =>+ String(each.uses ?? '').startsWith('actions/checkout@') &&+ each.with?.ref == null &&+ !each.with?.path,+ );+ return (+ merge !== -1 && merge < steps.indexOf(step) && beforeHeadCheckout(job, step)+ );+}+export function unguardedScripts(job) {const scope = (job.steps ?? []).find(step =>String(step.uses ?? '').includes('test-scope'),@@ -180,6 +199,7 @@ export function unguardedScripts(job) {beforeHeadCheckout(job, step))continue;+ if (inMergeCheckout(job, step)) continue;// Run from a copy an earlier step took from that scope checkout.if (copiedFromScope(job, step, script)) continue;// The step skips a head without this script on its own.@@ -246,6 +266,24 @@ describe('pull-request jobs that check out the head', () => {}});+ it('exempts a step in the merge checkout only before the head checkout', () => {+ const merge = {name: 'Merge', uses: 'actions/checkout@v7'};+ const head = {+ name: 'Head',+ uses: 'actions/checkout@v7',+ with: {ref: '${{ github.event.pull_request.head.sha }}'},+ };+ const before = {name: 'Before', run: 'node .github/scripts/new.mjs'};+ const after = {name: 'After', run: 'node .github/scripts/new.mjs'};+ expect(unguardedScripts({steps: [merge, before, head, after]})).toEqual([+ 'After: .github/scripts/new.mjs',+ ]);+ // With no merge checkout, the step reads no tree of the workflow.+ expect(unguardedScripts({steps: [before, head]})).toEqual([+ 'Before: .github/scripts/new.mjs',+ ]);+ });+it("exempts only the script a step's own missing-file check names", () => {const run = ['D7_TEST=.github/scripts/d7.test.mjs',diff --git a/.github/scripts/touched-components.test.mjs b/.github/scripts/touched-components.test.mjsindex e77ac06eb8..b5de2fe10b 100644--- a/.github/scripts/touched-components.test.mjs+++ b/.github/scripts/touched-components.test.mjs@@ -749,14 +749,63 @@ describe('pr-component-a11y-contracts routing', () => {expect(find.run).toContain('import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";',);- const checkout = contractStep('Checkout');- expect(checkout.with?.ref).toBeUndefined();+ // The merge commit, as the touched check: an older head lacks these scripts.+ expect(+ contractStep('Check out the selection scripts').with?.ref,+ ).toBeUndefined();+ });++ it("runs the head's own specs against the Storybook that head built", () => {+ const storybookCheckout = ci.jobs['build-storybook'].steps.find(+ candidate =>+ candidate.uses?.startsWith('actions/checkout@') && candidate.with?.ref,+ );+ const headCheckout = contractStep(+ 'Check out the head that built the Storybook',+ );+ expect(storybookCheckout.with.ref).toBe(+ '${{ github.event.pull_request.head.sha || github.sha }}',+ );+ expect(headCheckout.with.ref).toBe(storybookCheckout.with.ref);+ expect(headCheckout.if).toBe(ran);+ const names = contracts.steps.map(({name}) => name);+ const at = name => names.indexOf(name);+ expect(at('Check out the head that built the Storybook')).toBeGreaterThan(+ Math.max(+ at('Decide the touched components'),+ at('Find the changed browser accessibility specs'),+ ),+ );+ for (const name of [+ 'Setup Node and pnpm',+ 'Download Storybook artifact',+ 'Install Playwright',+ "Run the touched components' accessibility contracts",+ ]) {+ expect(at(name), name).toBeGreaterThan(+ at('Check out the head that built the Storybook'),+ );+ }+ // Only the selection and head checkouts: no later checkout moves the tree.+ expect(+ contracts.steps+ .filter(candidate => candidate.uses?.startsWith('actions/checkout@'))+ .map(({name}) => name),+ ).toEqual([+ 'Check out the scope action',+ 'Check out the selection scripts',+ 'Check out the head that built the Storybook',+ ]);});it('runs the changed specs and the touched owners, on four workers, inside the job budget', () => {expect(runStep.if).toBe(ran);- expect(runStep.run).toBe(- 'node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"',+ // The runner comes from the merge commit; the head may predate it.+ expect(runStep.run).toContain(+ 'git show "$WORKFLOW_SHA:.github/scripts/a11y-pr-contracts.mjs" > "$runner"',+ );+ expect(runStep.run).toContain(+ 'node "$runner" --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"',);// Every changed path, not only the changed specs: a change in a// component folder owns that folder's specs.diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 21870e0b0c..79396151c2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -3851,8 +3851,10 @@ jobs:if: steps.scope.outputs.applies != 'true'run: echo "No component or browser accessibility spec changed; no contract to run."- # The tested merge commit, as the touched-components job checks out.- - name: Checkout+ # The tested merge commit, as the touched-components job checks out: a+ # head that predates these scripts does not have them. The specs run at+ # the head (see "Check out the head that built the Storybook").+ - name: Check out the selection scriptsif: steps.scope.outputs.applies == 'true'uses: actions/checkout@v7with:@@ -3923,6 +3925,17 @@ jobs:echo "No touched component and no changed spec; no contract to run."fi+ # The specs, their helpers and the lockfile come from the same commit as+ # the Storybook build. A merge commit has specs that the head Storybook+ # does not have, and those specs fail. Merge-level interactions are the+ # job of the post-merge and release checks.+ - name: Check out the head that built the Storybook+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ uses: actions/checkout@v7+ with:+ ref: ${{ github.event.pull_request.head.sha || github.sha }}+ persist-credentials: false+- name: Setup Node and pnpmif: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'uses: ./.github/actions/setup@@ -3940,9 +3953,11 @@ jobs:if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'run: pnpm install --frozen-lockfile && npx playwright install chromium- # Routing lines come from this merge commit. A failing or unfinished- # spec fails the job, and so the test join. A file that does not fit the- # budget is not run and does not fail the job.+ # The runner and the routing lines come from the merge commit, so an+ # older head is routed by current policy. The specs run at the head; a+ # spec that the head does not have is not listed and does not run. A+ # failing or unfinished spec fails the job, and so the test join. A file+ # that does not fit the budget is not run and does not fail the job.- name: Run the touched components' accessibility contractsid: contractsif: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'@@ -3956,7 +3971,11 @@ jobs:FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}OWNERS: ${{ steps.touched.outputs.has_components == 'true' && steps.touched.outputs.a11y_owners || '' }}A11Y_CONTRACT_WORKERS: 4- run: node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"+ run: |+ runner="$RUNNER_TEMP/a11y-pr-contracts.mjs"+ git cat-file -e "$WORKFLOW_SHA^{commit}" 2>/dev/null || git fetch --quiet --no-tags --depth=1 origin "$WORKFLOW_SHA"+ git show "$WORKFLOW_SHA:.github/scripts/a11y-pr-contracts.mjs" > "$runner"+ node "$runner" --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"- name: Upload contract evidenceif: always() && (steps.contracts.outcome == 'success' || steps.contracts.outcome == 'failure')