Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: run the accessibility contracts at the head that built their Storybook (#9971)

Cindy Zhang54166ffe49c1
diff --git a/.github/scripts/a11y-pr-contracts.mjs b/.github/scripts/a11y-pr-contracts.mjs
index 019bcba623..c5441f6731 100644
--- a/.github/scripts/a11y-pr-contracts.mjs
+++ b/.github/scripts/a11y-pr-contracts.mjs
@@ -610,9 +610,9 @@ export function changedSinceMergeBase(base) {
}
/**
- * A spec's routing lines (and the audit config) come from the commit the
- * workflow steps come from when it has the file (an older PR head predates
- * current routing); otherwise from the checkout.
+ * A spec's routing lines, the audit config and the recorded durations come
+ * from the commit the workflow steps come from when it has the file (an older
+ * PR head predates current routing); otherwise from the checkout.
*/
function readPolicySource(root, file, policySha) {
if (policySha) {
@@ -639,10 +639,18 @@ function readAuditConfig(root, file, policySha) {
return JSON.parse(source);
}
-function readDurations(root) {
- const file = path.join(root, '.github/a11y-contract-durations.json');
- if (!fs.existsSync(file)) return {};
- return fileSeconds(JSON.parse(fs.readFileSync(file, 'utf8')));
+function readDurations(root, policySha) {
+ let source;
+ try {
+ source = readPolicySource(
+ root,
+ '.github/a11y-contract-durations.json',
+ policySha,
+ );
+ } catch {
+ return {};
+ }
+ return fileSeconds(JSON.parse(source));
}
/** The Playwright worker count, as playwright.config.ts reads it. */
@@ -765,7 +773,7 @@ function main() {
components: ownedBy,
changedPaths: readLines(changedFilesFile) ?? changedPaths,
}),
- seconds: readDurations(root),
+ seconds: readDurations(root, policySha),
workers: contractWorkers(),
})
: {run: routed, notRun: []};
diff --git a/.github/scripts/a11y-pr-contracts.test.mjs b/.github/scripts/a11y-pr-contracts.test.mjs
index d77a0aa108..f4bf2c9dd7 100644
--- a/.github/scripts/a11y-pr-contracts.test.mjs
+++ b/.github/scripts/a11y-pr-contracts.test.mjs
@@ -475,7 +475,7 @@ describe('the default check of the touched components', () => {
expect(runner).toContain(
'changedPaths: readLines(changedFilesFile) ?? changedPaths',
);
- expect(runner).toContain('seconds: readDurations(root)');
+ expect(runner).toContain('seconds: readDurations(root, policySha)');
expect(runner).toContain('workers: contractWorkers()');
});
});
diff --git a/.github/scripts/ci-head-checkout-scripts.test.mjs b/.github/scripts/ci-head-checkout-scripts.test.mjs
index 84b1af5775..04082b3dfc 100644
--- a/.github/scripts/ci-head-checkout-scripts.test.mjs
+++ b/.github/scripts/ci-head-checkout-scripts.test.mjs
@@ -18,8 +18,9 @@
* `head_file_present`, the step loads it from the workflow's own commit
* (`git show "$WORKFLOW_SHA:<script>"`, or from the job's `.test-scope`
* checkout, whose sparse directories hold it, directly or through a copy an
- * earlier step made in `$RUNNER_TEMP`), or the step checks for it itself
- * (`[ ! -f ... ]`). A job that does not measure the head takes the default
+ * earlier step made in `$RUNNER_TEMP`), the step runs in the job's merge
+ * checkout before its head checkout replaces the workspace, or the step checks
+ * for it itself (`[ ! -f ... ]`). A job that does not measure the head takes the default
* merge checkout instead, and this test does not apply to it.
*
* Scope: `.js`, `.mjs` and `.cjs` scripts named in a step's `run:`.
@@ -158,6 +159,24 @@ export function beforeHeadCheckout(job, step) {
return head === -1 || steps.indexOf(step) < head;
}
+/**
+ * Whether the step runs in the job's merge checkout (the workflow's own
+ * commit): a workspace checkout with no `ref` comes before the step, and the
+ * head checkout comes after it.
+ */
+export function inMergeCheckout(job, step) {
+ const steps = job.steps ?? [];
+ const merge = steps.findIndex(
+ each =>
+ String(each.uses ?? '').startsWith('actions/checkout@') &&
+ each.with?.ref == null &&
+ !each.with?.path,
+ );
+ return (
+ merge !== -1 && merge < steps.indexOf(step) && beforeHeadCheckout(job, step)
+ );
+}
+
export function unguardedScripts(job) {
const scope = (job.steps ?? []).find(step =>
String(step.uses ?? '').includes('test-scope'),
@@ -180,6 +199,7 @@ export function unguardedScripts(job) {
beforeHeadCheckout(job, step)
)
continue;
+ if (inMergeCheckout(job, step)) continue;
// Run from a copy an earlier step took from that scope checkout.
if (copiedFromScope(job, step, script)) continue;
// The step skips a head without this script on its own.
@@ -246,6 +266,24 @@ describe('pull-request jobs that check out the head', () => {
}
});
+ it('exempts a step in the merge checkout only before the head checkout', () => {
+ const merge = {name: 'Merge', uses: 'actions/checkout@v7'};
+ const head = {
+ name: 'Head',
+ uses: 'actions/checkout@v7',
+ with: {ref: '${{ github.event.pull_request.head.sha }}'},
+ };
+ const before = {name: 'Before', run: 'node .github/scripts/new.mjs'};
+ const after = {name: 'After', run: 'node .github/scripts/new.mjs'};
+ expect(unguardedScripts({steps: [merge, before, head, after]})).toEqual([
+ 'After: .github/scripts/new.mjs',
+ ]);
+ // With no merge checkout, the step reads no tree of the workflow.
+ expect(unguardedScripts({steps: [before, head]})).toEqual([
+ 'Before: .github/scripts/new.mjs',
+ ]);
+ });
+
it("exempts only the script a step's own missing-file check names", () => {
const run = [
'D7_TEST=.github/scripts/d7.test.mjs',
diff --git a/.github/scripts/touched-components.test.mjs b/.github/scripts/touched-components.test.mjs
index e77ac06eb8..b5de2fe10b 100644
--- a/.github/scripts/touched-components.test.mjs
+++ b/.github/scripts/touched-components.test.mjs
@@ -749,14 +749,63 @@ describe('pr-component-a11y-contracts routing', () => {
expect(find.run).toContain(
'import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";',
);
- const checkout = contractStep('Checkout');
- expect(checkout.with?.ref).toBeUndefined();
+ // The merge commit, as the touched check: an older head lacks these scripts.
+ expect(
+ contractStep('Check out the selection scripts').with?.ref,
+ ).toBeUndefined();
+ });
+
+ it("runs the head's own specs against the Storybook that head built", () => {
+ const storybookCheckout = ci.jobs['build-storybook'].steps.find(
+ candidate =>
+ candidate.uses?.startsWith('actions/checkout@') && candidate.with?.ref,
+ );
+ const headCheckout = contractStep(
+ 'Check out the head that built the Storybook',
+ );
+ expect(storybookCheckout.with.ref).toBe(
+ '${{ github.event.pull_request.head.sha || github.sha }}',
+ );
+ expect(headCheckout.with.ref).toBe(storybookCheckout.with.ref);
+ expect(headCheckout.if).toBe(ran);
+ const names = contracts.steps.map(({name}) => name);
+ const at = name => names.indexOf(name);
+ expect(at('Check out the head that built the Storybook')).toBeGreaterThan(
+ Math.max(
+ at('Decide the touched components'),
+ at('Find the changed browser accessibility specs'),
+ ),
+ );
+ for (const name of [
+ 'Setup Node and pnpm',
+ 'Download Storybook artifact',
+ 'Install Playwright',
+ "Run the touched components' accessibility contracts",
+ ]) {
+ expect(at(name), name).toBeGreaterThan(
+ at('Check out the head that built the Storybook'),
+ );
+ }
+ // Only the selection and head checkouts: no later checkout moves the tree.
+ expect(
+ contracts.steps
+ .filter(candidate => candidate.uses?.startsWith('actions/checkout@'))
+ .map(({name}) => name),
+ ).toEqual([
+ 'Check out the scope action',
+ 'Check out the selection scripts',
+ 'Check out the head that built the Storybook',
+ ]);
});
it('runs the changed specs and the touched owners, on four workers, inside the job budget', () => {
expect(runStep.if).toBe(ran);
- expect(runStep.run).toBe(
- 'node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"',
+ // The runner comes from the merge commit; the head may predate it.
+ expect(runStep.run).toContain(
+ 'git show "$WORKFLOW_SHA:.github/scripts/a11y-pr-contracts.mjs" > "$runner"',
+ );
+ expect(runStep.run).toContain(
+ 'node "$runner" --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"',
);
// Every changed path, not only the changed specs: a change in a
// component folder owns that folder's specs.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 21870e0b0c..79396151c2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3851,8 +3851,10 @@ jobs:
if: steps.scope.outputs.applies != 'true'
run: echo "No component or browser accessibility spec changed; no contract to run."
- # The tested merge commit, as the touched-components job checks out.
- - name: Checkout
+ # The tested merge commit, as the touched-components job checks out: a
+ # head that predates these scripts does not have them. The specs run at
+ # the head (see "Check out the head that built the Storybook").
+ - name: Check out the selection scripts
if: steps.scope.outputs.applies == 'true'
uses: actions/checkout@v7
with:
@@ -3923,6 +3925,17 @@ jobs:
echo "No touched component and no changed spec; no contract to run."
fi
+ # The specs, their helpers and the lockfile come from the same commit as
+ # the Storybook build. A merge commit has specs that the head Storybook
+ # does not have, and those specs fail. Merge-level interactions are the
+ # job of the post-merge and release checks.
+ - name: Check out the head that built the Storybook
+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ persist-credentials: false
+
- name: Setup Node and pnpm
if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'
uses: ./.github/actions/setup
@@ -3940,9 +3953,11 @@ jobs:
if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'
run: pnpm install --frozen-lockfile && npx playwright install chromium
- # Routing lines come from this merge commit. A failing or unfinished
- # spec fails the job, and so the test join. A file that does not fit the
- # budget is not run and does not fail the job.
+ # The runner and the routing lines come from the merge commit, so an
+ # older head is routed by current policy. The specs run at the head; a
+ # spec that the head does not have is not listed and does not run. A
+ # failing or unfinished spec fails the job, and so the test join. A file
+ # that does not fit the budget is not run and does not fail the job.
- name: Run the touched components' accessibility contracts
id: contracts
if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'
@@ -3956,7 +3971,11 @@ jobs:
FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}
OWNERS: ${{ steps.touched.outputs.has_components == 'true' && steps.touched.outputs.a11y_owners || '' }}
A11Y_CONTRACT_WORKERS: 4
- run: node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"
+ run: |
+ runner="$RUNNER_TEMP/a11y-pr-contracts.mjs"
+ git cat-file -e "$WORKFLOW_SHA^{commit}" 2>/dev/null || git fetch --quiet --no-tags --depth=1 origin "$WORKFLOW_SHA"
+ git show "$WORKFLOW_SHA:.github/scripts/a11y-pr-contracts.mjs" > "$runner"
+ node "$runner" --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"
- name: Upload contract evidence
if: always() && (steps.contracts.outcome == 'success' || steps.contracts.outcome == 'failure')