An open source design system that's fully customizable and agent ready
TypeScript
ci: weekly full-suite accessibility scan with issue reporting (#4383)
Baraa Hamodi33d4e98a8147
diff --git a/.github/scripts/weekly-a11y-summary.js b/.github/scripts/weekly-a11y-summary.jsnew file mode 100644index 0000000000..bbc9718e0d--- /dev/null+++ b/.github/scripts/weekly-a11y-summary.js@@ -0,0 +1,135 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @description Summarizes a full-suite accessibility audit report for the+ * weekly a11y scan workflow. Reads the JSON report produced by+ * accessibility-audit.js, writes a markdown summary (used for both the+ * GitHub step summary and the tracking issue body), and emits step outputs+ * (status, total_violations, components_audited) so the workflow can decide+ * whether to open-or-update the tracking issue and whether to fail the job.+ * @input --report <file> --audit-outcome <success|failure> --summary-output <file>+ * [--github-output <file>]+ * @output Markdown summary file + GitHub Actions step outputs+ *+ * Status semantics:+ * clean — report present, audit step succeeded, zero violations+ * violations — report present, audit step succeeded, violations found+ * failed — report present but the audit step exited non-zero (e.g.+ * --fail-on-new tripped once baseline gating exists)+ * crashed — no usable report (script crashed before writing one, or the+ * report itself records a fatal error)+ */++const fs = require('node:fs');+const {buildA11ySection} = require('./lib/a11y-format');++// GitHub issue bodies max out at 65536 characters — leave headroom for the+// run link and footer the workflow appends after this summary.+const MAX_SUMMARY_CHARS = 60000;++const args = process.argv.slice(2);+const getArg = name => {+ const idx = args.indexOf(`--${name}`);+ return idx !== -1 ? args[idx + 1] : null;+};++const reportFile = getArg('report') || 'a11y-weekly-report.json';+const auditOutcome = getArg('audit-outcome') || 'success';+const summaryFile = getArg('summary-output') || 'a11y-weekly-summary.md';+const githubOutputFile = getArg('github-output');++function readReport(file) {+ try {+ return JSON.parse(fs.readFileSync(file, 'utf8'));+ } catch {+ return null;+ }+}++function computeStatus(report, outcome) {+ if (!report || report.error) return 'crashed';+ if (outcome !== 'success') return 'failed';+ const total = report.summary?.totalViolations ?? 0;+ return total > 0 ? 'violations' : 'clean';+}++function countStoryErrors(report) {+ let errors = 0;+ for (const comp of Object.values(report?.components || {})) {+ for (const storyResult of comp.storyDetails || []) {+ if (storyResult.error) errors++;+ }+ }+ return errors;+}++function buildSummary(report, status) {+ const lines = ['## Weekly full-suite accessibility scan', ''];++ if (status === 'crashed') {+ lines.push(+ '**Status:** the audit did not produce a usable report — the audit ' ++ 'script crashed or the Storybook build was missing. See the workflow ' ++ 'run logs for details.',+ );+ if (report?.error) {+ lines.push('', `Reported error: \`${report.error}\``);+ }+ return lines.join('\n') + '\n';+ }++ const componentsAudited = report.summary?.componentsAudited ?? 0;+ const auditedAt = report.summary?.auditedAt || 'unknown';+ const storyErrors = countStoryErrors(report);++ lines.push(+ `**Scope:** full component library — ${componentsAudited} component(s) audited (${auditedAt}).`,+ '',+ );++ if (status === 'failed') {+ lines.push(+ '**Status:** the audit step exited non-zero — violations exceeded the ' ++ 'configured gate (or the run failed after writing the report).',+ '',+ );+ }++ if (storyErrors > 0) {+ lines.push(+ `**Warning:** ${storyErrors} ${storyErrors === 1 ? 'story' : 'stories'} failed to load and could not be audited.`,+ '',+ );+ }++ lines.push(buildA11ySection(report));++ let summary = lines.join('\n');+ if (summary.length > MAX_SUMMARY_CHARS) {+ summary =+ summary.slice(0, MAX_SUMMARY_CHARS) ++ '\n\n_…summary truncated — download the `a11y-weekly-report` artifact for the full report._\n';+ }+ return summary;+}++function writeOutputs(status, report) {+ const outputs = [+ `status=${status}`,+ `total_violations=${report?.summary?.totalViolations ?? 0}`,+ `components_audited=${report?.summary?.componentsAudited ?? 0}`,+ ];+ if (githubOutputFile) {+ fs.appendFileSync(githubOutputFile, outputs.join('\n') + '\n');+ }+ console.log(outputs.join('\n'));+}++const report = readReport(reportFile);+const status = computeStatus(report, auditOutcome);+const summary = buildSummary(report, status);++fs.writeFileSync(summaryFile, summary);+writeOutputs(status, report);+console.log(`Summary written to ${summaryFile} (status: ${status})`);diff --git a/.github/workflows/a11y-weekly.yml b/.github/workflows/a11y-weekly.ymlnew file mode 100644index 0000000000..a49577fa2e--- /dev/null+++ b/.github/workflows/a11y-weekly.yml@@ -0,0 +1,188 @@+# Copyright (c) Meta Platforms, Inc. and affiliates.++# Weekly full-suite accessibility scan+#+# PR CI (ci.yml's pr-a11y job) only audits components a PR touches. Token or+# shared-hook changes can regress components no PR modified, so this workflow+# periodically audits EVERY component story: it builds Storybook once, runs+# .github/scripts/accessibility-audit.js with no --components filter (the+# script audits all stories when the flag is absent), uploads the JSON report,+# and opens-or-updates a tracking issue when violations are found or the+# audit crashes.+#+# Baseline gating: if .github/a11y-baseline.json exists (added by the a11y+# baseline work), the audit runs with --baseline/--fail-on-new so only NEW+# violations fail the run. Before that file lands, the audit runs ungated and+# violations are reported via the tracking issue only.+#+# Theme coverage: stories are audited in the default light color scheme.+# Dark mode is a Storybook runtime global (colorMode) that the audit script+# does not currently plumb through to the story iframe URL — dark-mode+# coverage is a documented follow-up.++name: Weekly A11y Scan++on:+ schedule:+ # Mondays 05:23 UTC — off-peak, avoids the top-of-hour scheduling rush+ - cron: '23 5 * * 1'+ workflow_dispatch:++permissions: {}++concurrency:+ group: a11y-weekly+ cancel-in-progress: false++jobs:+ # Build Storybook once and audit the full story index+ audit:+ runs-on: ubuntu-latest+ permissions:+ contents: read+ outputs:+ status: ${{ steps.summarize.outputs.status }}+ total_violations: ${{ steps.summarize.outputs.total_violations }}+ components_audited: ${{ steps.summarize.outputs.components_audited }}+ steps:+ - name: Checkout+ uses: actions/checkout@v7++ - name: Setup Node and pnpm+ uses: ./.github/actions/setup++ - name: Build core package+ run: pnpm build++ - name: Build Storybook+ run: pnpm -F @astryxdesign/storybook build++ - name: Install Playwright browser+ run: npx playwright install chromium++ # No --components filter: with the flag absent the audit script includes+ # every story in the Storybook index (full-suite scan).+ # continue-on-error so the summarize step still runs (and the report+ # artifact still uploads) when the audit exits non-zero; the job is+ # re-failed below based on the summarized status.+ - name: Run full-suite accessibility audit+ id: audit+ continue-on-error: true+ run: |+ BASELINE_FLAGS=""+ if [ -f .github/a11y-baseline.json ]; then+ BASELINE_FLAGS="--baseline .github/a11y-baseline.json --fail-on-new"+ fi+ node .github/scripts/accessibility-audit.js \+ --storybook-dir apps/storybook/dist \+ --output a11y-weekly-report.json \+ $BASELINE_FLAGS++ - name: Summarize report+ id: summarize+ env:+ AUDIT_OUTCOME: ${{ steps.audit.outcome }}+ run: |+ node .github/scripts/weekly-a11y-summary.js \+ --report a11y-weekly-report.json \+ --audit-outcome "$AUDIT_OUTCOME" \+ --summary-output a11y-weekly-summary.md \+ --github-output "$GITHUB_OUTPUT"+ cat a11y-weekly-summary.md >> "$GITHUB_STEP_SUMMARY"++ - name: Upload report artifact+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: a11y-weekly-report+ path: |+ a11y-weekly-report.json+ a11y-weekly-summary.md+ retention-days: 30+ if-no-files-found: warn++ # Surface infra breakage (crashed) and gate failures (failed) as a red+ # run. Plain violations without baseline gating stay green here — the+ # tracking issue is the reporting channel for those.+ - name: Fail on audit crash or gate failure+ if: steps.summarize.outputs.status == 'crashed' || steps.summarize.outputs.status == 'failed'+ run: |+ echo "::error::Weekly a11y scan status: ${{ steps.summarize.outputs.status }}"+ exit 1++ # Open-or-update the tracking issue whenever the scan is not clean.+ # Modeled on hardening-issue.yml (least-privilege github-script issue+ # creation with title-based dedup). Runs even when the audit job failed —+ # a crashed scan should be visible as an issue, not just a red run.+ report-issue:+ needs: [audit]+ if: ${{ !cancelled() && needs.audit.outputs.status != 'clean' }}+ runs-on: ubuntu-latest+ permissions:+ issues: write+ actions: read # download-artifact from this run+ steps:+ - name: Download report artifact+ continue-on-error: true # artifact may be missing if audit crashed early+ uses: actions/download-artifact@v8+ with:+ name: a11y-weekly-report++ - name: Open or update tracking issue+ uses: actions/github-script@v9+ with:+ script: |+ const fs = require('node:fs');+ const { owner, repo } = context.repo;+ const title = 'Weekly a11y scan: violations found';+ const label = 'a11y-weekly-scan';++ let summary;+ try {+ summary = fs.readFileSync('a11y-weekly-summary.md', 'utf8');+ } catch {+ summary =+ 'The weekly accessibility scan failed before producing a summary. ' ++ 'See the workflow run logs for details.';+ }++ const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;+ const body = [+ summary.trim(),+ '',+ `Workflow run: ${runUrl}`,+ '',+ '_Maintained automatically by `.github/workflows/a11y-weekly.yml`. ' ++ 'The full JSON report is attached to the run as the `a11y-weekly-report` artifact (30-day retention)._',+ ].join('\n');++ // Dedup: reuse the open tracking issue if one exists+ const issues = await github.paginate(github.rest.issues.listForRepo, {+ owner,+ repo,+ state: 'open',+ labels: label,+ per_page: 100,+ });+ const existing = issues.find(+ (i) => i.title === title && !i.pull_request+ );++ if (existing) {+ await github.rest.issues.createComment({+ owner,+ repo,+ issue_number: existing.number,+ body,+ });+ console.log(`Updated existing issue #${existing.number}`);+ } else {+ const { data: issue } = await github.rest.issues.create({+ owner,+ repo,+ title,+ body,+ labels: [label],+ });+ console.log(`Created issue #${issue.number}`);+ }