An open source design system that's fully customizable and agent ready
TypeScript
ci(a11y): scan the release check's full axe roster in three shards (#7416)
Cindy Zhang5c2d7f75c9ad
diff --git a/.github/scripts/accessibility-audit.js b/.github/scripts/accessibility-audit.jsindex add366e7c5..8f1ebb181b 100644--- a/.github/scripts/accessibility-audit.js+++ b/.github/scripts/accessibility-audit.js@@ -7,6 +7,8 @@* @input --storybook-dir <path> --output <file> [--components <comma-separated>]* [--stories <comma-separated-story-ids>] [--port <number>]* [--ready-selector <css-selector>] [--concurrency <pages, default 1>]+ * [--shard <i>/<n> --shard-output <file>] (scan every n-th story, write it)+ * [--merge-shards <dir>] (read a complete shard set instead of scanning)* --baseline <path> (compare violations against a checked-in baseline)* --fail-on-new (exit 1 when violations not present in the baseline exist)* --update-baseline (rewrite the baseline file from this run's report)@@ -36,6 +38,14 @@ const {storyKey,} = require('./lib/a11y-baseline');const {waitForStoryReadiness} = require('./lib/a11y-story-readiness');+const {+ mergeA11yShards,+ parseShard,+ readShardFiles,+ readStorybookSha,+ shardFile,+ shardItems,+} = require('./lib/a11y-shards');const {legacyBaselineStoryKeys,ownerForA11yStory,@@ -74,6 +84,19 @@ const concurrency = Number(concurrencyArg ?? 1);if (!Number.isInteger(concurrency) || concurrency < 1) {throw new Error(`Invalid --concurrency value: ${concurrencyArg}`);}+// A sharded audit: each `--shard i/n` job scans every n-th story and writes+// only its per-story results; `--merge-shards` then builds the report (and+// applies the baseline gate) from the complete set, exactly as one run would.+const shardArg = getArg('shard');+const shardOutput = getArg('shard-output');+const mergeShardsDir = getArg('merge-shards');+const shard = shardArg === null ? null : parseShard(shardArg);+if ((shard === null) !== (shardOutput === null)) {+ throw new Error('--shard and --shard-output must be given together');+}+if (shard !== null && mergeShardsDir !== null) {+ throw new Error('--shard and --merge-shards cannot be combined');+}const baselineFile = getArg('baseline');const readySelector = getArg('ready-selector');const failOnNew = hasFlag('fail-on-new');@@ -343,17 +366,21 @@ async function runAccessibilityAudit() {console.log(`Auditing ${Object.keys(storyGroups).length} components`);- const server = await createServer(storybookPath, port);- const browser = await chromium.launch();+ // A merge reads results; it starts no server and no browser.+ const scanning = mergeShardsDir === null;+ const server = scanning ? await createServer(storybookPath, port) : null;+ const browser = scanning ? await chromium.launch() : null;const componentResults = {};const auditedStoryIds = new Set();let totalViolations = 0;try {- const context = await browser.newContext({- viewport: { width: 1280, height: 720 },- reducedMotion: 'reduce',- });+ const context = scanning+ ? await browser.newContext({+ viewport: { width: 1280, height: 720 },+ reducedMotion: 'reduce',+ })+ : null;// Scan one story on a fresh page; returns its axe violations.const scanStory = async (component, story) => {@@ -424,22 +451,55 @@ async function runAccessibilityAudit() {([component, componentStories]) =>componentStories.map(story => ({component, story})),);- const violationsByStory = new Map();- let next = 0;- let failed = false;- await Promise.all(- Array.from({length: Math.min(concurrency, queue.length)}, async () => {- while (!failed && next < queue.length) {- const {component, story} = queue[next++];- try {- violationsByStory.set(story.id, await scanStory(component, story));- } catch (e) {- failed = true;- throw e;+ const queueStoryIds = queue.map(({story}) => story.id);+ const storybookSha = readStorybookSha(storybookPath);+ let violationsByStory = new Map();+ if (!scanning) {+ violationsByStory = mergeA11yShards(readShardFiles(mergeShardsDir), {+ storyIds: queueStoryIds,+ storybookSha,+ });+ console.log(+ `Merged ${violationsByStory.size} audited stories from ${mergeShardsDir}`,+ );+ } else {+ const items = shard === null ? queue : shardItems(queue, shard);+ if (shard !== null) {+ console.log(+ `Shard ${shard.index}/${shard.count}: ${items.length} of ${queue.length} stories`,+ );+ }+ let next = 0;+ let failed = false;+ await Promise.all(+ Array.from({length: Math.min(concurrency, items.length)}, async () => {+ while (!failed && next < items.length) {+ const {component, story} = items[next++];+ try {+ violationsByStory.set(story.id, await scanStory(component, story));+ } catch (e) {+ failed = true;+ throw e;+ }}- }- }),- );+ }),+ );+ if (shard !== null) {+ fs.writeFileSync(+ shardOutput,+ JSON.stringify(+ shardFile({+ storyIds: queueStoryIds,+ shard,+ storybookSha,+ violationsByStory,+ }),+ ),+ );+ console.log(`Shard results written to ${shardOutput}`);+ return null;+ }+ }for (const [component, componentStories] of Object.entries(storyGroups)) {const componentViolations = [];@@ -499,8 +559,8 @@ async function runAccessibilityAudit() {};}} finally {- await browser.close();- server.close();+ await browser?.close();+ server?.close();}const auditedStories = routed.routes@@ -613,6 +673,8 @@ function applyBaselineGate(report) {runAccessibilityAudit().then((report) => {+ // A shard writes per-story results only; the merge run gates.+ if (report === null) return;const code = applyBaselineGate(report);if (code !== 0) process.exitCode = code;})diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjsindex 750642e3d8..5be0e595c7 100644--- a/.github/scripts/ci-test-routing.test.mjs+++ b/.github/scripts/ci-test-routing.test.mjs@@ -233,6 +233,7 @@ describe('ci.yml RTL package sharding', () => {const exactRef = '${{ github.event.pull_request.head.sha || github.sha }}';for (const name of ['build-storybook',+ 'pr-a11y-shard','pr-a11y','a11y-contracts','pr-rtl-shard',diff --git a/.github/scripts/lib/a11y-shards.js b/.github/scripts/lib/a11y-shards.jsnew file mode 100644index 0000000000..46ee0c65ab--- /dev/null+++ b/.github/scripts/lib/a11y-shards.js@@ -0,0 +1,155 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++'use strict';+/* global module */++/**+ * @file Split one accessibility audit across jobs and join the parts.+ * @input The audit's ordered story list, one shard's per-story axe violations.+ * @output A shard file per job, and — only when every shard of the same story+ * list and the same Storybook build is present — the per-story violations a+ * single run would have collected.+ * @position Used by accessibility-audit.js: `--shard`/`--shard-output` write,+ * `--merge-shards` reads. A story with no entry would read as "not audited",+ * so a partial or mismatched set is refused rather than merged.+ */++const crypto = require('node:crypto');+const fs = require('node:fs');+const path = require('node:path');++const SHARD_FILE_VERSION = 1;++function parseShard(value) {+ const match = /^(\d+)\/(\d+)$/.exec(value ?? '');+ const index = match ? Number(match[1]) : NaN;+ const count = match ? Number(match[2]) : NaN;+ if (+ !Number.isInteger(index) ||+ !Number.isInteger(count) ||+ count < 1 ||+ index < 1 ||+ index > count+ ) {+ throw new Error(+ `--shard must be <index>/<count> with 1 <= index <= count, got ${value ?? 'nothing'}`,+ );+ }+ return {index, count};+}++/** Every count-th item starting at the shard's index, so shards stay balanced. */+function shardItems(items, {index, count}) {+ return items.filter((_, position) => position % count === index - 1);+}++function storyListDigest(storyIds) {+ return crypto+ .createHash('sha256')+ .update(JSON.stringify(storyIds))+ .digest('hex');+}++function readStorybookSha(storybookPath) {+ const stamp = path.join(storybookPath, 'astryx-build-sha.txt');+ return fs.existsSync(stamp)+ ? fs.readFileSync(stamp, 'utf8').trim() || null+ : null;+}++function shardFile({storyIds, shard, storybookSha, violationsByStory}) {+ return {+ version: SHARD_FILE_VERSION,+ shard,+ storybookSha,+ storyListDigest: storyListDigest(storyIds),+ storyCount: storyIds.length,+ violationsByStory: Object.fromEntries(violationsByStory),+ };+}++function readShardFiles(dir) {+ if (!dir || !fs.existsSync(dir)) return [];+ return fs+ .readdirSync(dir)+ .filter(name => name.endsWith('.json'))+ .sort()+ .map(name => {+ const file = path.join(dir, name);+ return {file, content: JSON.parse(fs.readFileSync(file, 'utf8'))};+ });+}++/**+ * @returns {Map<string, object[]>} story id -> axe violations, for every story+ */+function mergeA11yShards(files, {storyIds, storybookSha}) {+ if (files.length === 0)+ throw new Error('No accessibility shard files to merge');+ const digest = storyListDigest(storyIds);+ const count = files[0].content?.shard?.count;+ const seen = new Set();+ const merged = new Map();+ for (const {file, content} of files) {+ const where = path.basename(file);+ if (content?.version !== SHARD_FILE_VERSION) {+ throw new Error(+ `${where}: unsupported shard version ${content?.version}`,+ );+ }+ if (content.shard?.count !== count) {+ throw new Error(+ `${where}: shard count ${content.shard?.count} does not match ${count}`,+ );+ }+ if (seen.has(content.shard.index)) {+ throw new Error(+ `${where}: shard ${content.shard.index}/${count} appears twice`,+ );+ }+ seen.add(content.shard.index);+ if (+ content.storyListDigest !== digest ||+ content.storyCount !== storyIds.length+ ) {+ throw new Error(`${where}: audited a different story list than this run`);+ }+ if ((content.storybookSha ?? null) !== (storybookSha ?? null)) {+ throw new Error(+ `${where}: audited Storybook ${content.storybookSha ?? 'unstamped'}, this run reads ${storybookSha ?? 'unstamped'}`,+ );+ }+ const expected = shardItems(storyIds, content.shard);+ const ids = Object.keys(content.violationsByStory ?? {});+ const expectedSet = new Set(expected);+ if (+ ids.length !== expected.length ||+ ids.some(id => !expectedSet.has(id))+ ) {+ throw new Error(+ `${where}: does not hold exactly the stories of shard ${content.shard.index}/${count}`,+ );+ }+ for (const id of ids) {+ if (!Array.isArray(content.violationsByStory[id])) {+ throw new Error(`${where}: ${id} has no violations list`);+ }+ merged.set(id, content.violationsByStory[id]);+ }+ }+ if (seen.size !== count) {+ throw new Error(+ `Accessibility shards ${[...seen].sort((a, b) => a - b).join(', ')} of ${count}: the set is incomplete`,+ );+ }+ return merged;+}++module.exports = {+ mergeA11yShards,+ parseShard,+ readShardFiles,+ readStorybookSha,+ shardFile,+ shardItems,+};diff --git a/.github/scripts/lib/a11y-shards.test.mjs b/.github/scripts/lib/a11y-shards.test.mjsnew file mode 100644index 0000000000..cf4e7f1cb1--- /dev/null+++ b/.github/scripts/lib/a11y-shards.test.mjs@@ -0,0 +1,109 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {createRequire} from 'node:module';+import {describe, expect, it} from 'vitest';++const require = createRequire(import.meta.url);+const {+ mergeA11yShards,+ parseShard,+ shardFile,+ shardItems,+} = require('./a11y-shards.js');++const storyIds = [+ 'button--a',+ 'button--b',+ 'card--a',+ 'chat--a',+ 'chat--b',+ 'dialog--a',+ 'list--a',+];+const violations = id =>+ id.startsWith('chat') ? [{id: 'color-contrast', nodes: [{html: id}]}] : [];++function shards(count, {storybookSha = 'abc', ids = storyIds} = {}) {+ return Array.from({length: count}, (_, offset) => {+ const shard = {index: offset + 1, count};+ return {+ file: `a11y-${shard.index}.json`,+ content: shardFile({+ storyIds: ids,+ shard,+ storybookSha,+ violationsByStory: new Map(+ shardItems(ids, shard).map(id => [id, violations(id)]),+ ),+ }),+ };+ });+}++describe('accessibility audit shards', () => {+ it('parses index/count and refuses anything else', () => {+ expect(parseShard('1/3')).toEqual({index: 1, count: 3});+ for (const bad of [null, '', '0/2', '3/2', '2/0', 'a/b']) {+ expect(() => parseShard(bad), String(bad)).toThrow(/--shard/);+ }+ });++ it('gives every story to exactly one shard', () => {+ const parts = [1, 2, 3].map(index =>+ shardItems(storyIds, {index, count: 3}),+ );+ expect(parts.flat().sort()).toEqual([...storyIds].sort());+ expect(new Set(parts.flat()).size).toBe(storyIds.length);+ });++ it('joins a complete set into every story’s violations', () => {+ const merged = mergeA11yShards(shards(3).reverse(), {+ storyIds,+ storybookSha: 'abc',+ });+ expect([...merged.keys()].sort()).toEqual([...storyIds].sort());+ for (const id of storyIds) expect(merged.get(id)).toEqual(violations(id));+ });++ it('refuses a missing or duplicated shard', () => {+ const set = shards(3);+ expect(() =>+ mergeA11yShards(set.slice(0, 2), {storyIds, storybookSha: 'abc'}),+ ).toThrow(/incomplete/);+ expect(() =>+ mergeA11yShards([set[0], set[0], set[2]], {+ storyIds,+ storybookSha: 'abc',+ }),+ ).toThrow(/twice/);+ });++ it('refuses shards of another story list or another build', () => {+ expect(() =>+ mergeA11yShards(shards(2, {ids: storyIds.slice(1)}), {+ storyIds,+ storybookSha: 'abc',+ }),+ ).toThrow(/different story list/);+ expect(() =>+ mergeA11yShards(shards(2, {storybookSha: 'zzz'}), {+ storyIds,+ storybookSha: 'abc',+ }),+ ).toThrow(/Storybook/);+ });++ it('refuses a shard missing one of its stories', () => {+ const set = shards(2);+ delete set[0].content.violationsByStory[storyIds[0]];+ expect(() => mergeA11yShards(set, {storyIds, storybookSha: 'abc'})).toThrow(+ /exactly the stories of shard 1\/2/,+ );+ });++ it('refuses an empty set', () => {+ expect(() => mergeA11yShards([], {storyIds, storybookSha: 'abc'})).toThrow(+ /No accessibility shard/,+ );+ });+});diff --git a/.github/scripts/release-check-workflow.test.mjs b/.github/scripts/release-check-workflow.test.mjsindex 5ba3470878..8e60df1f17 100644--- a/.github/scripts/release-check-workflow.test.mjs+++ b/.github/scripts/release-check-workflow.test.mjs@@ -25,6 +25,8 @@ const releaseJobs = ['build-storybook','typecheck','pr-visual',+ 'pr-a11y-shard',+ 'pr-a11y-audit','pr-a11y','a11y-contracts','pr-rtl-shard',@@ -220,14 +222,17 @@ describe('explicit release routing', () => {},);- it.each(['pr-a11y', 'a11y-contracts', 'pr-visual', 'pr-rtl-shard'])(- 'blocks %s when its build fails',- name => {- const needs = dependencies(jobs[name]);- needs['build-storybook'].result = 'failure';- expect(runs(jobs[name], {needs})).toBe(false);- },- );+ it.each([+ 'pr-a11y-shard',+ 'pr-a11y',+ 'a11y-contracts',+ 'pr-visual',+ 'pr-rtl-shard',+ ])('blocks %s when its build fails', name => {+ const needs = dependencies(jobs[name]);+ needs['build-storybook'].result = 'failure';+ expect(runs(jobs[name], {needs})).toBe(false);+ });it('keeps dispatch concurrency separate and non-cancelling', () => {expect(workflow.concurrency.group).toContain('|| github.run_id');@@ -368,6 +373,7 @@ describe('exact main and release branch completion', () => {'pr-visual','pr-a11y','a11y-contracts',+ 'pr-a11y-audit','pr-rtl',]);});@@ -568,7 +574,11 @@ describe('full release scope without PR metadata or maintenance writes', () => {});it('runs the canonical accessibility command without a component filter for full scope', () => {- const audit = step('pr-a11y', 'Run accessibility audit');+ const audit = step('pr-a11y-audit', 'Run accessibility audit');+ expect(runs(step('pr-a11y', 'Run accessibility audit'))).toBe(false);+ expect(step('pr-a11y-audit', 'Require every audit shard').run).toContain(+ '"$SHARD_RESULT" != success',+ );const result = shell(audit.run.replaceAll('${{ github.event_name }}', 'workflow_dispatch'),{},@@ -581,6 +591,7 @@ describe('full release scope without PR metadata or maintenance writes', () => {);expect(result.status, result.stderr).toBe(0);expect(result.stdout).toContain('--fail-on-new');+ expect(result.stdout).toContain('--merge-shards');expect(result.stdout).not.toContain('--components');expect(audit['continue-on-error']).toBeUndefined();expect(diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7fb0887c1b..cefdf3aa9e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1110,6 +1110,114 @@ jobs:# Skipped when no components changed. Depends only on build-storybook, so it# starts as soon as the storybook artifact is up (~2min) instead of waiting# for the slower sandbox build.+ # The release check's full axe roster is scanned by three shard jobs and+ # joined, with the baseline gate, in pr-a11y-audit, while pr-a11y runs the+ # browser guards beside them. Pull requests audit only their scoped owners in+ # pr-a11y itself, so neither runs there.+ pr-a11y-shard:+ name: pr-a11y (${{ matrix.shard }}/${{ strategy.job-total }})+ needs: [build-storybook, check-components]+ if: github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && needs.check-components.outputs.has_components == 'true'+ strategy:+ fail-fast: false+ matrix:+ shard: [1, 2, 3]+ runs-on: 2-core-ubuntu-arm+ permissions:+ contents: read+ steps:+ - name: Checkout+ uses: actions/checkout@v7+ with:+ ref: ${{ github.event.pull_request.head.sha || github.sha }}++ - name: Setup Node and pnpm+ uses: ./.github/actions/setup+ with:+ install: 'false'++ - name: Download Storybook artifact+ uses: actions/download-artifact@v8+ with:+ name: storybook-${{ needs.build-storybook.outputs.short_hash }}+ path: apps/storybook/dist++ - name: Install Playwright+ run: pnpm install --frozen-lockfile && npx playwright install chromium++ - name: Run accessibility audit shard+ run: |+ node .github/scripts/accessibility-audit.js \+ --storybook-dir apps/storybook/dist \+ --concurrency 4 \+ --shard "${{ matrix.shard }}/${{ strategy.job-total }}" \+ --shard-output "a11y-shard-${{ matrix.shard }}.json"++ - name: Upload accessibility audit shard+ uses: actions/upload-artifact@v7+ with:+ name: a11y-shard-${{ needs.build-storybook.outputs.short_hash }}-${{ matrix.shard }}+ path: a11y-shard-${{ matrix.shard }}.json+ retention-days: 1+ if-no-files-found: error++ pr-a11y-audit:+ name: pr-a11y (full audit)+ needs: [build-storybook, check-components, pr-a11y-shard]+ if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && needs.check-components.outputs.has_components == 'true' }}+ runs-on: 2-core-ubuntu-arm+ permissions:+ contents: read+ steps:+ - name: Require every audit shard+ env:+ SHARD_RESULT: ${{ needs.pr-a11y-shard.result }}+ run: |+ if [ "$SHARD_RESULT" != success ]; then+ echo "Accessibility audit shards did not all succeed: $SHARD_RESULT"+ exit 1+ fi++ - name: Checkout+ uses: actions/checkout@v7+ with:+ ref: ${{ github.event.pull_request.head.sha || github.sha }}++ - name: Setup Node and pnpm+ uses: ./.github/actions/setup++ - name: Download Storybook artifact+ uses: actions/download-artifact@v8+ with:+ name: storybook-${{ needs.build-storybook.outputs.short_hash }}+ path: apps/storybook/dist++ - name: Download accessibility audit shards+ uses: actions/download-artifact@v8+ with:+ pattern: a11y-shard-${{ needs.build-storybook.outputs.short_hash }}-*+ path: a11y-shards+ merge-multiple: true++ # Builds the report a single full run would write from the complete+ # shard set (or fails), then applies the checked-in baseline.+ - name: Run accessibility audit+ run: |+ node .github/scripts/accessibility-audit.js \+ --storybook-dir apps/storybook/dist \+ --output a11y-report.json \+ --merge-shards a11y-shards \+ --baseline .github/a11y-baseline.json \+ --fail-on-new++ - name: Upload a11y artifact+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: a11y-report+ path: a11y-report.json+ retention-days: 1+pr-a11y:needs: [build-storybook, check-components]if: (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) && needs.check-components.outputs.has_components == 'true'@@ -1152,7 +1260,10 @@ jobs:# Fails on axe violations not present in the checked-in baseline# (.github/a11y-baseline.json). See CONTRIBUTING.md "Accessibility# audits" for the baseline workflow.+ # Release checks scan the full roster in pr-a11y-shard and gate it in+ # pr-a11y-audit; this job keeps the scoped PR audit and the guards.- name: Run accessibility audit+ if: github.event_name == 'pull_request'run: |SCOPE_ARGS=()if [ "${{ github.event_name }}" = 'pull_request' ]; then@@ -1173,7 +1284,7 @@ jobs:# if: always() so the report artifact (and the PR comment built from it)# still exists when the gate above fails.- name: Upload a11y artifact- if: always()+ if: always() && github.event_name == 'pull_request'uses: actions/upload-artifact@v7with:name: a11y-report@@ -2450,6 +2561,7 @@ jobs:pr-visual,pr-a11y,a11y-contracts,+ pr-a11y-audit,pr-rtl,]if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' }}@@ -2468,7 +2580,7 @@ jobs:with:script: |const needs = JSON.parse(process.env.RELEASE_NEEDS);- for (const name of ['check-scope', 'check-components', 'build-storybook', 'typecheck', 'pr-visual', 'pr-a11y', 'a11y-contracts', 'pr-rtl']) {+ for (const name of ['check-scope', 'check-components', 'build-storybook', 'typecheck', 'pr-visual', 'pr-a11y', 'a11y-contracts', 'pr-a11y-audit', 'pr-rtl']) {if (needs[name]?.result !== 'success') {throw new Error(`Required release job ${name} did not succeed: ${needs[name]?.result}`);}