Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: weekly full-suite accessibility scan with issue reporting (#4383)

Baraa Hamodi33d4e98a8147
diff --git a/.github/scripts/weekly-a11y-summary.js b/.github/scripts/weekly-a11y-summary.js
new file mode 100644
index 0000000000..bbc9718e0d
--- /dev/null
+++ b/.github/scripts/weekly-a11y-summary.js
@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @description Summarizes a full-suite accessibility audit report for the
+ * weekly a11y scan workflow. Reads the JSON report produced by
+ * accessibility-audit.js, writes a markdown summary (used for both the
+ * GitHub step summary and the tracking issue body), and emits step outputs
+ * (status, total_violations, components_audited) so the workflow can decide
+ * whether to open-or-update the tracking issue and whether to fail the job.
+ * @input --report <file> --audit-outcome <success|failure> --summary-output <file>
+ * [--github-output <file>]
+ * @output Markdown summary file + GitHub Actions step outputs
+ *
+ * Status semantics:
+ * clean — report present, audit step succeeded, zero violations
+ * violations — report present, audit step succeeded, violations found
+ * failed — report present but the audit step exited non-zero (e.g.
+ * --fail-on-new tripped once baseline gating exists)
+ * crashed — no usable report (script crashed before writing one, or the
+ * report itself records a fatal error)
+ */
+
+const fs = require('node:fs');
+const {buildA11ySection} = require('./lib/a11y-format');
+
+// GitHub issue bodies max out at 65536 characters — leave headroom for the
+// run link and footer the workflow appends after this summary.
+const MAX_SUMMARY_CHARS = 60000;
+
+const args = process.argv.slice(2);
+const getArg = name => {
+ const idx = args.indexOf(`--${name}`);
+ return idx !== -1 ? args[idx + 1] : null;
+};
+
+const reportFile = getArg('report') || 'a11y-weekly-report.json';
+const auditOutcome = getArg('audit-outcome') || 'success';
+const summaryFile = getArg('summary-output') || 'a11y-weekly-summary.md';
+const githubOutputFile = getArg('github-output');
+
+function readReport(file) {
+ try {
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+ } catch {
+ return null;
+ }
+}
+
+function computeStatus(report, outcome) {
+ if (!report || report.error) return 'crashed';
+ if (outcome !== 'success') return 'failed';
+ const total = report.summary?.totalViolations ?? 0;
+ return total > 0 ? 'violations' : 'clean';
+}
+
+function countStoryErrors(report) {
+ let errors = 0;
+ for (const comp of Object.values(report?.components || {})) {
+ for (const storyResult of comp.storyDetails || []) {
+ if (storyResult.error) errors++;
+ }
+ }
+ return errors;
+}
+
+function buildSummary(report, status) {
+ const lines = ['## Weekly full-suite accessibility scan', ''];
+
+ if (status === 'crashed') {
+ lines.push(
+ '**Status:** the audit did not produce a usable report — the audit ' +
+ 'script crashed or the Storybook build was missing. See the workflow ' +
+ 'run logs for details.',
+ );
+ if (report?.error) {
+ lines.push('', `Reported error: \`${report.error}\``);
+ }
+ return lines.join('\n') + '\n';
+ }
+
+ const componentsAudited = report.summary?.componentsAudited ?? 0;
+ const auditedAt = report.summary?.auditedAt || 'unknown';
+ const storyErrors = countStoryErrors(report);
+
+ lines.push(
+ `**Scope:** full component library — ${componentsAudited} component(s) audited (${auditedAt}).`,
+ '',
+ );
+
+ if (status === 'failed') {
+ lines.push(
+ '**Status:** the audit step exited non-zero — violations exceeded the ' +
+ 'configured gate (or the run failed after writing the report).',
+ '',
+ );
+ }
+
+ if (storyErrors > 0) {
+ lines.push(
+ `**Warning:** ${storyErrors} ${storyErrors === 1 ? 'story' : 'stories'} failed to load and could not be audited.`,
+ '',
+ );
+ }
+
+ lines.push(buildA11ySection(report));
+
+ let summary = lines.join('\n');
+ if (summary.length > MAX_SUMMARY_CHARS) {
+ summary =
+ summary.slice(0, MAX_SUMMARY_CHARS) +
+ '\n\n_…summary truncated — download the `a11y-weekly-report` artifact for the full report._\n';
+ }
+ return summary;
+}
+
+function writeOutputs(status, report) {
+ const outputs = [
+ `status=${status}`,
+ `total_violations=${report?.summary?.totalViolations ?? 0}`,
+ `components_audited=${report?.summary?.componentsAudited ?? 0}`,
+ ];
+ if (githubOutputFile) {
+ fs.appendFileSync(githubOutputFile, outputs.join('\n') + '\n');
+ }
+ console.log(outputs.join('\n'));
+}
+
+const report = readReport(reportFile);
+const status = computeStatus(report, auditOutcome);
+const summary = buildSummary(report, status);
+
+fs.writeFileSync(summaryFile, summary);
+writeOutputs(status, report);
+console.log(`Summary written to ${summaryFile} (status: ${status})`);
diff --git a/.github/workflows/a11y-weekly.yml b/.github/workflows/a11y-weekly.yml
new file mode 100644
index 0000000000..a49577fa2e
--- /dev/null
+++ b/.github/workflows/a11y-weekly.yml
@@ -0,0 +1,188 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# Weekly full-suite accessibility scan
+#
+# PR CI (ci.yml's pr-a11y job) only audits components a PR touches. Token or
+# shared-hook changes can regress components no PR modified, so this workflow
+# periodically audits EVERY component story: it builds Storybook once, runs
+# .github/scripts/accessibility-audit.js with no --components filter (the
+# script audits all stories when the flag is absent), uploads the JSON report,
+# and opens-or-updates a tracking issue when violations are found or the
+# audit crashes.
+#
+# Baseline gating: if .github/a11y-baseline.json exists (added by the a11y
+# baseline work), the audit runs with --baseline/--fail-on-new so only NEW
+# violations fail the run. Before that file lands, the audit runs ungated and
+# violations are reported via the tracking issue only.
+#
+# Theme coverage: stories are audited in the default light color scheme.
+# Dark mode is a Storybook runtime global (colorMode) that the audit script
+# does not currently plumb through to the story iframe URL — dark-mode
+# coverage is a documented follow-up.
+
+name: Weekly A11y Scan
+
+on:
+ schedule:
+ # Mondays 05:23 UTC — off-peak, avoids the top-of-hour scheduling rush
+ - cron: '23 5 * * 1'
+ workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+ group: a11y-weekly
+ cancel-in-progress: false
+
+jobs:
+ # Build Storybook once and audit the full story index
+ audit:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ outputs:
+ status: ${{ steps.summarize.outputs.status }}
+ total_violations: ${{ steps.summarize.outputs.total_violations }}
+ components_audited: ${{ steps.summarize.outputs.components_audited }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Node and pnpm
+ uses: ./.github/actions/setup
+
+ - name: Build core package
+ run: pnpm build
+
+ - name: Build Storybook
+ run: pnpm -F @astryxdesign/storybook build
+
+ - name: Install Playwright browser
+ run: npx playwright install chromium
+
+ # No --components filter: with the flag absent the audit script includes
+ # every story in the Storybook index (full-suite scan).
+ # continue-on-error so the summarize step still runs (and the report
+ # artifact still uploads) when the audit exits non-zero; the job is
+ # re-failed below based on the summarized status.
+ - name: Run full-suite accessibility audit
+ id: audit
+ continue-on-error: true
+ run: |
+ BASELINE_FLAGS=""
+ if [ -f .github/a11y-baseline.json ]; then
+ BASELINE_FLAGS="--baseline .github/a11y-baseline.json --fail-on-new"
+ fi
+ node .github/scripts/accessibility-audit.js \
+ --storybook-dir apps/storybook/dist \
+ --output a11y-weekly-report.json \
+ $BASELINE_FLAGS
+
+ - name: Summarize report
+ id: summarize
+ env:
+ AUDIT_OUTCOME: ${{ steps.audit.outcome }}
+ run: |
+ node .github/scripts/weekly-a11y-summary.js \
+ --report a11y-weekly-report.json \
+ --audit-outcome "$AUDIT_OUTCOME" \
+ --summary-output a11y-weekly-summary.md \
+ --github-output "$GITHUB_OUTPUT"
+ cat a11y-weekly-summary.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload report artifact
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: a11y-weekly-report
+ path: |
+ a11y-weekly-report.json
+ a11y-weekly-summary.md
+ retention-days: 30
+ if-no-files-found: warn
+
+ # Surface infra breakage (crashed) and gate failures (failed) as a red
+ # run. Plain violations without baseline gating stay green here — the
+ # tracking issue is the reporting channel for those.
+ - name: Fail on audit crash or gate failure
+ if: steps.summarize.outputs.status == 'crashed' || steps.summarize.outputs.status == 'failed'
+ run: |
+ echo "::error::Weekly a11y scan status: ${{ steps.summarize.outputs.status }}"
+ exit 1
+
+ # Open-or-update the tracking issue whenever the scan is not clean.
+ # Modeled on hardening-issue.yml (least-privilege github-script issue
+ # creation with title-based dedup). Runs even when the audit job failed —
+ # a crashed scan should be visible as an issue, not just a red run.
+ report-issue:
+ needs: [audit]
+ if: ${{ !cancelled() && needs.audit.outputs.status != 'clean' }}
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ actions: read # download-artifact from this run
+ steps:
+ - name: Download report artifact
+ continue-on-error: true # artifact may be missing if audit crashed early
+ uses: actions/download-artifact@v8
+ with:
+ name: a11y-weekly-report
+
+ - name: Open or update tracking issue
+ uses: actions/github-script@v9
+ with:
+ script: |
+ const fs = require('node:fs');
+ const { owner, repo } = context.repo;
+ const title = 'Weekly a11y scan: violations found';
+ const label = 'a11y-weekly-scan';
+
+ let summary;
+ try {
+ summary = fs.readFileSync('a11y-weekly-summary.md', 'utf8');
+ } catch {
+ summary =
+ 'The weekly accessibility scan failed before producing a summary. ' +
+ 'See the workflow run logs for details.';
+ }
+
+ const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+ const body = [
+ summary.trim(),
+ '',
+ `Workflow run: ${runUrl}`,
+ '',
+ '_Maintained automatically by `.github/workflows/a11y-weekly.yml`. ' +
+ 'The full JSON report is attached to the run as the `a11y-weekly-report` artifact (30-day retention)._',
+ ].join('\n');
+
+ // Dedup: reuse the open tracking issue if one exists
+ const issues = await github.paginate(github.rest.issues.listForRepo, {
+ owner,
+ repo,
+ state: 'open',
+ labels: label,
+ per_page: 100,
+ });
+ const existing = issues.find(
+ (i) => i.title === title && !i.pull_request
+ );
+
+ if (existing) {
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number: existing.number,
+ body,
+ });
+ console.log(`Updated existing issue #${existing.number}`);
+ } else {
+ const { data: issue } = await github.rest.issues.create({
+ owner,
+ repo,
+ title,
+ body,
+ labels: [label],
+ });
+ console.log(`Created issue #${issue.number}`);
+ }