Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: fail pr-a11y on new axe violations against a checked-in baseline (#4386)

Baraa Hamodi4b4399baa5f4
diff --git a/.github/a11y-baseline.json b/.github/a11y-baseline.json
new file mode 100644
index 0000000000..46ae48d210
--- /dev/null
+++ b/.github/a11y-baseline.json
@@ -0,0 +1,6 @@
+{
+ "$comment": "Known axe violations tolerated by the pr-a11y CI gate. Entries are keyed Component::Story::rule-id (see .github/scripts/lib/a11y-baseline.js). Regenerate with `pnpm a11y:baseline` after `pnpm storybook:build` and `npx playwright install chromium`. Remove entries as violations are fixed; anything not listed here fails pr-a11y.",
+ "version": 1,
+ "generatedAt": null,
+ "entries": []
+}
diff --git a/.github/scripts/accessibility-audit.js b/.github/scripts/accessibility-audit.js
index 2687d0359d..2ab65324be 100644
--- a/.github/scripts/accessibility-audit.js
+++ b/.github/scripts/accessibility-audit.js
@@ -5,7 +5,13 @@
/**
* @description Runs accessibility audits on component stories using axe-core
* @input --storybook-dir <path> --output <file> --components <comma-separated>
- * @output JSON report with accessibility violations
+ * --baseline <path> (compare violations against a checked-in baseline)
+ * --fail-on-new (exit 1 when violations not present in the baseline exist)
+ * --update-baseline (rewrite the baseline file from this run's report)
+ * @output JSON report with accessibility violations; with --baseline, a gate
+ * summary (new / baselined / resolved) on stdout and a non-zero exit code
+ * when --fail-on-new finds regressions. Diff logic lives in
+ * lib/a11y-baseline.js.
*/
const { chromium } = require('playwright');
@@ -13,17 +19,26 @@ const { AxeBuilder } = require('@axe-core/playwright');
const fs = require('node:fs');
const path = require('node:path');
const http = require('node:http');
+const {
+ buildBaseline,
+ diffAgainstBaseline,
+ formatDiffSummary,
+} = require('./lib/a11y-baseline');
const args = process.argv.slice(2);
const getArg = (name) => {
const idx = args.indexOf(`--${name}`);
return idx !== -1 ? args[idx + 1] : null;
};
+const hasFlag = (name) => args.includes(`--${name}`);
const storybookDir = getArg('storybook-dir') || 'apps/storybook/dist';
const outputFile = getArg('output') || 'a11y-report.json';
const componentsArg = getArg('components') || '';
const components = componentsArg.split(',').filter(Boolean);
+const baselineFile = getArg('baseline');
+const failOnNew = hasFlag('fail-on-new');
+const updateBaseline = hasFlag('update-baseline');
// Rules to disable — these are Storybook-context false positives, not component issues
const DISABLED_RULES = [
@@ -105,7 +120,7 @@ async function runAccessibilityAudit() {
summary: { total: 0, violations: 0 },
};
fs.writeFileSync(outputFile, JSON.stringify(report, null, 2));
- return;
+ return report;
}
// Start server
@@ -259,9 +274,58 @@ async function runAccessibilityAudit() {
fs.writeFileSync(outputFile, JSON.stringify(report, null, 2));
console.log(`\nAudit complete: ${totalViolations} total violations found`);
console.log(`Report written to ${outputFile}`);
+
+ return report;
+}
+
+// Read the baseline file; a missing file behaves as an empty baseline so
+// every current violation counts as new.
+function loadBaselineFile(filePath) {
+ if (!fs.existsSync(filePath)) {
+ console.warn(`Baseline file not found at ${filePath} — treating as empty`);
+ return { version: 1, entries: [] };
+ }
+ return JSON.parse(fs.readFileSync(filePath, 'utf8'));
}
-runAccessibilityAudit().catch((e) => {
- console.error('Accessibility audit failed:', e);
- process.exit(1);
-});
+// Compare the report against the checked-in baseline (or rewrite it with
+// --update-baseline). Returns the exit code for the process.
+function applyBaselineGate(report) {
+ if (!baselineFile) return 0;
+
+ if (updateBaseline) {
+ // Merge-aware: entries for components outside this (possibly
+ // --components-scoped) run are preserved.
+ const existing = fs.existsSync(baselineFile)
+ ? JSON.parse(fs.readFileSync(baselineFile, 'utf8'))
+ : null;
+ fs.writeFileSync(
+ baselineFile,
+ JSON.stringify(buildBaseline(report, { existing }), null, 2) + '\n'
+ );
+ console.log(`Baseline written to ${baselineFile}`);
+ return 0;
+ }
+
+ const baseline = loadBaselineFile(baselineFile);
+ const diff = diffAgainstBaseline(report, baseline);
+ console.log(formatDiffSummary(diff, { baselinePath: baselineFile }));
+
+ if (diff.newViolations.length > 0 && failOnNew) {
+ console.error(
+ `\nFailing: ${diff.newViolations.length} accessibility violation(s) not in baseline.`
+ );
+ return 1;
+ }
+ return 0;
+}
+
+runAccessibilityAudit()
+ .then((report) => {
+ const code = applyBaselineGate(report);
+ if (code !== 0) process.exitCode = code;
+ })
+ .catch((e) => {
+ console.error('Accessibility audit failed:', e);
+ process.exit(1);
+ });
diff --git a/.github/scripts/lib/a11y-baseline.js b/.github/scripts/lib/a11y-baseline.js
new file mode 100644
index 0000000000..beb3635a8a
--- /dev/null
+++ b/.github/scripts/lib/a11y-baseline.js
@@ -0,0 +1,278 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file a11y-baseline.js
+ * @input An a11y report (the JSON written by accessibility-audit.js) and a
+ * checked-in baseline file (.github/a11y-baseline.json)
+ * @output Pure diff logic for the accessibility CI gate: which violations are
+ * NEW (not in the baseline, fail the build), which baseline entries are
+ * RESOLVED (can be removed), plus baseline (re)generation and a readable
+ * failure summary.
+ * @position Shared by accessibility-audit.js (--baseline / --fail-on-new /
+ * --update-baseline). Kept free of Playwright/axe so it can be unit-tested
+ * without a browser (see a11y-baseline.test.mjs).
+ *
+ * Key design: `Component::Story::rule-id`.
+ * - Component + story + axe rule id is stable across unrelated DOM churn:
+ * axe rule ids are versioned and stable, and story names only change when
+ * someone renames a story (an intentional act).
+ * - Node target selectors / HTML snippets are deliberately EXCLUDED from the
+ * key — they change whenever markup shifts for unrelated reasons, which
+ * would make baseline entries silently stop matching and re-fail the
+ * build on cosmetic refactors.
+ * - Tradeoff: if a story already violates a rule (baselined) and a change
+ * adds MORE nodes violating the same rule in the same story, the gate
+ * will not catch it. That is acceptable — the goal is to stop new rule
+ * regressions while keeping the gate churn-proof.
+ */
+
+const BASELINE_VERSION = 1;
+
+/** Build the stable baseline key for one violation occurrence. */
+function violationKey(component, story, ruleId) {
+ return `${component}::${story}::${ruleId}`;
+}
+
+/**
+ * Flatten an a11y report into one entry per (component, story, rule)
+ * occurrence, deduped by key.
+ *
+ * Prefers the per-story `storyDetails` shape (raw axe violations per story);
+ * falls back to the aggregated `violations` shape (which carries a `stories`
+ * list per rule) for older reports.
+ *
+ * @param {object} report - report object with `.components`
+ * @returns {Array<{key: string, component: string, story: string,
+ * ruleId: string, impact: string, help: string, helpUrl: string,
+ * nodes: number}>}
+ */
+function collectViolations(report) {
+ const occurrences = [];
+ const components = (report && report.components) || {};
+
+ for (const [component, result] of Object.entries(components)) {
+ const storyDetails = Array.isArray(result.storyDetails)
+ ? result.storyDetails
+ : [];
+
+ if (storyDetails.length > 0) {
+ for (const storyResult of storyDetails) {
+ for (const violation of storyResult.violations || []) {
+ occurrences.push({
+ key: violationKey(component, storyResult.story, violation.id),
+ component,
+ story: storyResult.story,
+ ruleId: violation.id,
+ impact: violation.impact || 'unknown',
+ help: violation.help || violation.description || '',
+ helpUrl: violation.helpUrl || '',
+ nodes: (violation.nodes || []).length,
+ });
+ }
+ }
+ } else {
+ // Aggregated-only shape: one occurrence per (rule, story) pair.
+ for (const violation of result.violations || []) {
+ const stories =
+ Array.isArray(violation.stories) && violation.stories.length > 0
+ ? violation.stories
+ : ['*'];
+ for (const story of stories) {
+ occurrences.push({
+ key: violationKey(component, story, violation.id),
+ component,
+ story,
+ ruleId: violation.id,
+ impact: violation.impact || 'unknown',
+ help: violation.help || violation.description || '',
+ helpUrl: violation.helpUrl || '',
+ nodes: violation.totalNodes || 0,
+ });
+ }
+ }
+ }
+ }
+
+ // Dedupe by key (a story name should be unique within a component, but be
+ // defensive), merging node counts.
+ const byKey = new Map();
+ for (const occurrence of occurrences) {
+ const existing = byKey.get(occurrence.key);
+ if (existing) {
+ existing.nodes += occurrence.nodes;
+ } else {
+ byKey.set(occurrence.key, {...occurrence});
+ }
+ }
+ return Array.from(byKey.values());
+}
+
+/** Normalize baseline entries (objects or bare key strings) to a key Set. */
+function baselineKeySet(baseline) {
+ const entries = (baseline && baseline.entries) || [];
+ return new Set(
+ entries
+ .map(entry => (typeof entry === 'string' ? entry : entry && entry.key))
+ .filter(Boolean),
+ );
+}
+
+/**
+ * Build a baseline object from a report (for --update-baseline).
+ *
+ * The audit is often scoped with --components, so the report only covers a
+ * subset of the library. Entries in `existing` that belong to components NOT
+ * audited in this report are preserved; entries for audited components are
+ * replaced wholesale by the report's current violations.
+ *
+ * @param {object} report
+ * @param {{existing?: object|null, now?: Date}} [options]
+ */
+function buildBaseline(report, {existing = null, now = new Date()} = {}) {
+ const audited = new Set(Object.keys((report && report.components) || {}));
+ const preserved = ((existing && existing.entries) || [])
+ .map(entry => (typeof entry === 'string' ? {key: entry} : entry))
+ .filter(
+ entry => entry && entry.key && !audited.has(entry.key.split('::')[0]),
+ );
+ const fresh = collectViolations(report).map(v => ({
+ key: v.key,
+ impact: v.impact,
+ helpUrl: v.helpUrl,
+ }));
+
+ return {
+ $comment:
+ 'Known axe violations tolerated by the pr-a11y CI gate. Entries are ' +
+ 'keyed Component::Story::rule-id. Regenerate with `pnpm a11y:baseline` ' +
+ '(requires a built Storybook + Playwright chromium). Remove entries ' +
+ 'as violations are fixed.',
+ version: BASELINE_VERSION,
+ generatedAt: now.toISOString(),
+ entries: [...preserved, ...fresh].sort((a, b) =>
+ a.key < b.key ? -1 : a.key > b.key ? 1 : 0,
+ ),
+ };
+}
+
+/**
+ * Diff a report against a baseline.
+ *
+ * Anything in the report but missing from the baseline is NEW (a missing or
+ * empty baseline means every violation is new). Baseline entries with no
+ * matching violation are RESOLVED and can be deleted from the baseline —
+ * but only for components that were actually audited in this run. The CI
+ * audit is scoped to changed components, so baseline entries for components
+ * outside this run are reported as `unchecked`, not resolved.
+ *
+ * @param {object} report
+ * @param {object|null|undefined} baseline
+ * @returns {{newViolations: Array<object>, resolved: string[],
+ * unchecked: string[], matched: number}}
+ */
+function diffAgainstBaseline(report, baseline) {
+ const current = collectViolations(report);
+ const known = baselineKeySet(baseline);
+ const currentKeys = new Set(current.map(v => v.key));
+ // Every audited component gets a report entry, even with zero violations.
+ const auditedComponents = new Set(
+ Object.keys((report && report.components) || {}),
+ );
+
+ const newViolations = current.filter(v => !known.has(v.key));
+ const resolved = [];
+ const unchecked = [];
+ for (const key of Array.from(known).sort()) {
+ if (currentKeys.has(key)) continue;
+ const component = key.split('::')[0];
+ if (auditedComponents.has(component)) {
+ resolved.push(key);
+ } else {
+ unchecked.push(key);
+ }
+ }
+
+ return {
+ newViolations,
+ resolved,
+ unchecked,
+ matched: current.length - newViolations.length,
+ };
+}
+
+/**
+ * Render a human-readable gate summary for CI logs.
+ *
+ * @param {{newViolations: Array<object>, resolved: string[],
+ * unchecked?: string[], matched: number}} diff
+ * @param {{baselinePath?: string}} [options]
+ * @returns {string}
+ */
+function formatDiffSummary(
+ diff,
+ {baselinePath = '.github/a11y-baseline.json'} = {},
+) {
+ const lines = [];
+ lines.push('');
+ lines.push('=== Accessibility baseline gate ===');
+ const unchecked = diff.unchecked || [];
+ lines.push(
+ `${diff.newViolations.length} new, ${diff.matched} baselined, ` +
+ `${diff.resolved.length} resolved` +
+ (unchecked.length > 0
+ ? `, ${unchecked.length} baselined for components outside this run`
+ : ''),
+ );
+
+ if (diff.newViolations.length > 0) {
+ lines.push('');
+ lines.push('NEW violations (not in baseline — these fail the build):');
+ for (const v of diff.newViolations) {
+ lines.push(
+ ` ✗ [${v.impact}] ${v.ruleId} — ${v.component} / ${v.story}` +
+ (v.nodes ? ` (${v.nodes} node${v.nodes === 1 ? '' : 's'})` : ''),
+ );
+ if (v.help) lines.push(` ${v.help}`);
+ if (v.helpUrl) lines.push(` ${v.helpUrl}`);
+ }
+ lines.push('');
+ lines.push('To reproduce locally:');
+ lines.push(' pnpm storybook:build && npx playwright install chromium');
+ lines.push(' pnpm a11y:audit -- --components <Component>');
+ lines.push('');
+ lines.push('Fix the violation if at all possible. If it is a known,');
+ lines.push('intentional exception, add it to the baseline:');
+ lines.push(' pnpm a11y:baseline -- --components <Component>');
+ lines.push(
+ ` (or hand-add the key to ${baselinePath} with a reviewer's blessing)`,
+ );
+ }
+
+ if (diff.resolved.length > 0) {
+ lines.push('');
+ lines.push(
+ 'Resolved — these baseline entries no longer occur and can be removed ' +
+ `from ${baselinePath}:`,
+ );
+ for (const key of diff.resolved) {
+ lines.push(` ✓ ${key}`);
+ }
+ }
+
+ if (diff.newViolations.length === 0) {
+ lines.push('');
+ lines.push('No new accessibility violations. Gate passed.');
+ }
+
+ return lines.join('\n');
+}
+
+module.exports = {
+ BASELINE_VERSION,
+ violationKey,
+ collectViolations,
+ baselineKeySet,
+ buildBaseline,
+ diffAgainstBaseline,
+ formatDiffSummary,
+};
diff --git a/.github/scripts/lib/a11y-baseline.test.mjs b/.github/scripts/lib/a11y-baseline.test.mjs
new file mode 100644
index 0000000000..162cf89e73
--- /dev/null
+++ b/.github/scripts/lib/a11y-baseline.test.mjs
@@ -0,0 +1,255 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file a11y-baseline.test.mjs
+ * Tests for the a11y baseline gate: key stability across DOM churn, diff
+ * classification (new / baselined / resolved / unchecked), baseline
+ * generation, and the failure summary format.
+ */
+
+import {describe, expect, it} from 'vitest';
+import {
+ buildBaseline,
+ collectViolations,
+ diffAgainstBaseline,
+ formatDiffSummary,
+ violationKey,
+} from './a11y-baseline.js';
+
+// Build a report in the shape accessibility-audit.js writes: per-component
+// results with per-story raw axe violations under `storyDetails`.
+function makeReport(componentStories) {
+ const components = {};
+ for (const [component, stories] of Object.entries(componentStories)) {
+ const storyDetails = Object.entries(stories).map(([story, violations]) => ({
+ story,
+ violations,
+ }));
+ components[component] = {
+ storiesAudited: storyDetails.length,
+ violations: [],
+ storyDetails,
+ };
+ }
+ return {components, summary: {}};
+}
+
+function axeViolation(id, overrides = {}) {
+ return {
+ id,
+ impact: 'serious',
+ description: `${id} description`,
+ help: `${id} help`,
+ helpUrl: `https://dequeuniversity.com/rules/axe/4.10/${id}`,
+ tags: ['wcag2a'],
+ nodes: [{html: '<button></button>', target: ['#root > button']}],
+ ...overrides,
+ };
+}
+
+describe('violationKey', () => {
+ it('is component + story + rule id, independent of DOM specifics', () => {
+ expect(violationKey('Button', 'Primary', 'button-name')).toBe(
+ 'Button::Primary::button-name',
+ );
+ });
+});
+
+describe('collectViolations', () => {
+ it('keys violations stably across unrelated DOM churn', () => {
+ const before = makeReport({
+ Button: {
+ Primary: [
+ axeViolation('button-name', {
+ nodes: [{html: '<button class="a"></button>', target: ['.a']}],
+ }),
+ ],
+ },
+ });
+ const after = makeReport({
+ Button: {
+ Primary: [
+ axeViolation('button-name', {
+ // Same violation, different selector/markup after a refactor.
+ nodes: [
+ {html: '<button class="b x"></button>', target: ['div > .b']},
+ ],
+ }),
+ ],
+ },
+ });
+ expect(collectViolations(before).map(v => v.key)).toEqual(
+ collectViolations(after).map(v => v.key),
+ );
+ });
+
+ it('falls back to the aggregated shape when storyDetails is absent', () => {
+ const report = {
+ components: {
+ Card: {
+ storiesAudited: 2,
+ violations: [
+ {
+ id: 'color-contrast',
+ impact: 'serious',
+ help: 'contrast',
+ helpUrl: 'https://example.com',
+ stories: ['Default', 'Compact'],
+ totalNodes: 3,
+ },
+ ],
+ },
+ },
+ };
+ expect(
+ collectViolations(report)
+ .map(v => v.key)
+ .sort(),
+ ).toEqual([
+ 'Card::Compact::color-contrast',
+ 'Card::Default::color-contrast',
+ ]);
+ });
+});
+
+describe('diffAgainstBaseline', () => {
+ const report = makeReport({
+ Button: {Primary: [axeViolation('button-name')]},
+ Card: {Default: []},
+ });
+
+ it('flags violations missing from the baseline as new', () => {
+ const diff = diffAgainstBaseline(report, {version: 1, entries: []});
+ expect(diff.newViolations).toHaveLength(1);
+ expect(diff.newViolations[0].key).toBe('Button::Primary::button-name');
+ expect(diff.matched).toBe(0);
+ });
+
+ it('treats a missing baseline as empty (everything is new)', () => {
+ expect(diffAgainstBaseline(report, null).newViolations).toHaveLength(1);
+ expect(diffAgainstBaseline(report, undefined).newViolations).toHaveLength(
+ 1,
+ );
+ });
+
+ it('passes when every violation is baselined', () => {
+ const diff = diffAgainstBaseline(report, {
+ version: 1,
+ entries: [{key: 'Button::Primary::button-name', impact: 'serious'}],
+ });
+ expect(diff.newViolations).toEqual([]);
+ expect(diff.matched).toBe(1);
+ expect(diff.resolved).toEqual([]);
+ });
+
+ it('accepts bare string entries', () => {
+ const diff = diffAgainstBaseline(report, {
+ version: 1,
+ entries: ['Button::Primary::button-name'],
+ });
+ expect(diff.newViolations).toEqual([]);
+ });
+
+ it('reports baseline entries for audited components as resolved', () => {
+ const diff = diffAgainstBaseline(report, {
+ version: 1,
+ entries: [
+ {key: 'Button::Primary::button-name'},
+ {key: 'Card::Default::color-contrast'},
+ ],
+ });
+ expect(diff.newViolations).toEqual([]);
+ expect(diff.resolved).toEqual(['Card::Default::color-contrast']);
+ });
+
+ it('does not mark entries for unaudited components as resolved', () => {
+ // CI only audits changed components; a baseline entry for a component
+ // outside this run is unchecked, not resolved.
+ const diff = diffAgainstBaseline(report, {
+ version: 1,
+ entries: [
+ {key: 'Button::Primary::button-name'},
+ {key: 'Dialog::Basic::aria-dialog-name'},
+ ],
+ });
+ expect(diff.resolved).toEqual([]);
+ expect(diff.unchecked).toEqual(['Dialog::Basic::aria-dialog-name']);
+ });
+});
+
+describe('buildBaseline', () => {
+ it('round-trips: a baseline built from a report gates that report clean', () => {
+ const report = makeReport({
+ Button: {Primary: [axeViolation('button-name')]},
+ Dialog: {Basic: [axeViolation('aria-dialog-name', {impact: 'critical'})]},
+ });
+ const baseline = buildBaseline(report, {
+ now: new Date('2026-07-25T00:00:00Z'),
+ });
+ expect(baseline.generatedAt).toBe('2026-07-25T00:00:00.000Z');
+ expect(baseline.entries.map(e => e.key)).toEqual([
+ 'Button::Primary::button-name',
+ 'Dialog::Basic::aria-dialog-name',
+ ]);
+
+ const diff = diffAgainstBaseline(report, baseline);
+ expect(diff.newViolations).toEqual([]);
+ expect(diff.resolved).toEqual([]);
+ expect(diff.matched).toBe(2);
+ });
+
+ it('preserves entries for components outside a scoped regeneration', () => {
+ // `pnpm a11y:baseline -- --components Button` must not drop baseline
+ // entries belonging to components that were not audited in this run.
+ const existing = {
+ version: 1,
+ entries: [
+ {key: 'Button::Primary::color-contrast', impact: 'serious'},
+ {key: 'Dialog::Basic::aria-dialog-name', impact: 'critical'},
+ 'Toast::Stacked::aria-live-region',
+ ],
+ };
+ const scopedReport = makeReport({
+ Button: {Primary: [axeViolation('button-name')]},
+ });
+ const baseline = buildBaseline(scopedReport, {existing});
+ expect(baseline.entries.map(e => e.key)).toEqual([
+ // Button entries replaced by the fresh audit (color-contrast dropped,
+ // button-name added); Dialog/Toast entries preserved untouched.
+ 'Button::Primary::button-name',
+ 'Dialog::Basic::aria-dialog-name',
+ 'Toast::Stacked::aria-live-region',
+ ]);
+ });
+});
+
+describe('formatDiffSummary', () => {
+ it('describes new violations with rule, impact, location, and remediation', () => {
+ const report = makeReport({
+ Button: {Primary: [axeViolation('button-name')]},
+ });
+ const summary = formatDiffSummary(
+ diffAgainstBaseline(report, {version: 1, entries: []}),
+ {baselinePath: '.github/a11y-baseline.json'},
+ );
+ expect(summary).toContain('button-name');
+ expect(summary).toContain('[serious]');
+ expect(summary).toContain('Button / Primary');
+ expect(summary).toContain('pnpm a11y:audit');
+ expect(summary).toContain('pnpm a11y:baseline');
+ expect(summary).toContain('.github/a11y-baseline.json');
+ });
+
+ it('notes resolved entries as removable without failing language', () => {
+ const report = makeReport({Button: {Primary: []}});
+ const summary = formatDiffSummary(
+ diffAgainstBaseline(report, {
+ version: 1,
+ entries: ['Button::Primary::button-name'],
+ }),
+ );
+ expect(summary).toContain('can be removed');
+ expect(summary).toContain('Button::Primary::button-name');
+ expect(summary).toContain('Gate passed');
+ });
+});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 93a5c0249c..b34ed4d685 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -416,15 +416,23 @@ jobs:
- name: Install Playwright
run: pnpm install --frozen-lockfile && npx playwright install chromium
+ # Fails on axe violations not present in the checked-in baseline
+ # (.github/a11y-baseline.json). See CONTRIBUTING.md "Accessibility
+ # audits" for the baseline workflow.
- name: Run accessibility audit
run: |
COMPONENTS=$(cat analysis.json | jq -r '(.newComponents + .modifiedComponents) | join(",")')
node .github/scripts/accessibility-audit.js \
--storybook-dir apps/storybook/dist \
--output a11y-report.json \
- --components "$COMPONENTS"
+ --components "$COMPONENTS" \
+ --baseline .github/a11y-baseline.json \
+ --fail-on-new
+ # if: always() so the report artifact (and the PR comment built from it)
+ # still exists when the gate above fails.
- name: Upload a11y artifact
+ if: always()
uses: actions/upload-artifact@v7
with:
name: a11y-report
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 671d73e229..c5c9748e0f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -325,6 +325,42 @@ src/Button/
└── Button.test.tsx # Tests live here
```
+### Accessibility audits
+
+PRs that touch components run an axe-core audit (the `pr-a11y` CI job) over
+the Storybook stories of the changed components. The job **fails** when it
+finds a violation that is not listed in the checked-in baseline,
+`.github/a11y-baseline.json`. Violations are keyed
+`Component::Story::rule-id`, so unrelated markup churn does not invalidate
+baseline entries.
+
+To reproduce and fix a failure locally:
+
+```bash
+# One-time setup
+pnpm storybook:build
+npx playwright install chromium
+
+# Audit specific components against the baseline (what CI does)
+pnpm a11y:audit -- --components Button,Dialog
+```
+
+Fix the violation whenever possible. If it is a known, intentional exception,
+add it to the baseline (scoped to the affected components, and expect
+reviewers to ask why):
+
+```bash
+pnpm a11y:baseline -- --components Button,Dialog
+```
+
+When the audit reports baseline entries as "resolved", delete them from
+`.github/a11y-baseline.json` — the baseline should only shrink over time.
+
+> **Scope caveat:** axe-core automates only a subset of WCAG (roughly a
+> third of the success criteria). A green `pr-a11y` job does not mean a
+> component is accessible — keyboard flows, focus order, screen-reader
+> semantics, and contrast in context still need manual checks.
+
## Versioning & Releases
We use [Changesets](https://github.com/changesets/changesets) for versioning, with a thin Astryx layer on top so changelogs stay categorized, contributor-attributed, and aligned with our pre-1.0 conventions.
diff --git a/package.json b/package.json
index 25cfba2a5b..d82e4a6dd0 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,8 @@
"check:sync": "node scripts/check-sync.js",
"lint": "pnpm check:repo && eslint . --cache",
"lint:strict": "pnpm check:repo && ASTRYX_STRICT_LINT=1 eslint . --cache",
+ "a11y:audit": "node .github/scripts/accessibility-audit.js --baseline .github/a11y-baseline.json --fail-on-new",
+ "a11y:baseline": "node .github/scripts/accessibility-audit.js --baseline .github/a11y-baseline.json --update-baseline",
"storybook": "pnpm -F @astryxdesign/storybook dev",
"storybook:build": "pnpm -F @astryxdesign/storybook build",
"docsite": "pnpm -F @astryxdesign/docsite dev",