The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
test(docs): scan changed pages for WCAG 2.1 A/AA in warn mode (#48727)
Miranda Limonczenko777c02c205b5
diff --git a/apps/docs/DEVELOPERS.md b/apps/docs/DEVELOPERS.mdindex 10ea54cdb32..37f48953ea1 100644--- a/apps/docs/DEVELOPERS.md+++ b/apps/docs/DEVELOPERS.md@@ -35,6 +35,26 @@ This creates Markdown files for all routes under the `public/markdown/guides` diFor production this setup runs as a `prebuild` task to allow Vercel to bundle these files with middleware and functions.+## Accessibility checks++Docs pages are scanned for WCAG 2.1 A/AA issues with axe-core, as part of the+Playwright suite in `e2e/docs`. Pull requests scan the pages your change affects,+limited to the main article.++To scan the pages your current branch changes:++```bash+PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y+```++That resolves which pages to scan from your branch, but reads them from+production, so it won't see your edits and will 404 on a page you just added.+Point `PLAYWRIGHT_BASE_URL` at your pull request's preview to scan your own+content.++See [`e2e/docs/README.md`](https://github.com/supabase/supabase/blob/master/e2e/docs/README.md)+for coverage and skipped rules.+## ContributingFor repo organization and style guide, see the [contributing guide](https://github.com/supabase/supabase/blob/master/apps/docs/CONTRIBUTING.md).diff --git a/e2e/docs/README.md b/e2e/docs/README.mdindex 357f022892e..ea80e37ad60 100644--- a/e2e/docs/README.md+++ b/e2e/docs/README.md@@ -15,6 +15,7 @@ This page covers:- [Override which pages run](#override-which-pages-run) — when the default gitscope is wrong- [What the suite covers](#what-the-suite-covers) — in-scope paths and limits+- [Accessibility scans](#accessibility-scans) — WCAG coverage and skipped rules- [Debug failures](#debug-failures) — reports and traces- [How CI uses this suite](#how-ci-uses-this-suite) — pull request behavior@@ -168,6 +169,30 @@ staged and unstaged working-tree changes.} | pnpm -C e2e/docs resolve-docs-scope```+## Accessibility scans++The `@a11y`-tagged test scans each in-scope page for WCAG 2.1 A/AA violations using+`@axe-core/playwright`, limited to the main article.++```bash+PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y+```++Which pages get scanned comes from your branch, but the content comes from+whatever you point `PLAYWRIGHT_BASE_URL` at. Production won't have your edits and+will 404 on a page you just added, so use your pull request's preview to scan your+own content.++`EXCLUDED_RULES` in `utils/axe-helpers.ts` lists the rules the scan skips.+`color-contrast` is most of the scan time and finds nothing inside an article, since+docs contrast comes from shared tokens and chrome. The rest target `<html>`, `<head>`,+and `<body>`, which an article-scoped scan can't reach.++Cross-origin frames are skipped, so a third-party embed isn't reported as ours.++Not covered: `/docs/reference/*`, shared chrome, and most of WCAG. Keyboard+navigation, focus management, and screen reader behavior need manual testing.+## Debug failures1. Open the HTML report after a run:@@ -186,7 +211,8 @@ owned docs content, partials, or `e2e/docs`.1. Diff the pull request against its base branch and resolve in-scope page paths.2. Skip Playwright when nothing in scope changed.3. When `apps/docs` changed, wait for the Vercel docs preview and set- `PLAYWRIGHT_BASE_URL` to that preview. Otherwise use production.+ `PLAYWRIGHT_BASE_URL` to that preview. When no preview resolves, skip rather than+ test against production.4. Run the suite with `DOCS_E2E_PAGE_PATHS` set to the resolved list.Draft pull requests stay skipped until you mark them ready for review. Manualdiff --git a/e2e/docs/features/docs-pages.spec.ts b/e2e/docs/features/docs-pages.spec.tsindex 3881a0b54d6..3a58f155021 100644--- a/e2e/docs/features/docs-pages.spec.ts+++ b/e2e/docs/features/docs-pages.spec.ts@@ -1,6 +1,17 @@-import { AxeBuilder } from '@axe-core/playwright'import { expect, test } from '@playwright/test'+import type { TestInfo } from '@playwright/test'+import {+ attachScanReport,+ blockingViolations,+ ENFORCED_RULES,+ formatViolations,+ scanArticle,+ scanLooksEmpty,+ settleForAxe,+ shouldEnforceAll,+ unloadedResult,+} from '../utils/axe-helpers.js'import {articleSelectorForPagePath,browserLikeUserAgent,@@ -10,6 +21,11 @@ import {const pagePaths = parseDocsE2EPagePaths(process.env.DOCS_E2E_PAGE_PATHS)+function annotate(testInfo: TestInfo, description: string) {+ testInfo.annotations.push({ type: 'warning', description })+ console.warn(`::warning title=Accessibility::${description}`)+}+test.describe('Docs owned pages', () => {// playwright.config.ts sets fullyParallel: false, and Playwright shards// work by file rather than by test in that mode — without this, every test@@ -63,19 +79,69 @@ test.describe('Docs owned pages', () => {}for (const pagePath of pagePaths) {- test(`${pagePath} has a valid heading hierarchy @a11y`, async ({ page }) => {- const articleSelector = articleSelectorForPagePath(pagePath)- const response = await page.goto(pagePath)- expect(response?.ok(), `Expected a successful response for ${pagePath}`).toBeTruthy()+ test(`${pagePath} has no blocking accessibility violations @a11y`, async ({+ page,+ }, testInfo) => {+ test.setTimeout(120_000)++ const include = articleSelectorForPagePath(pagePath)++ let response+ try {+ response = await page.goto(pagePath)+ } catch (error) {+ await attachScanReport(testInfo, unloadedResult(pagePath, pagePath, null, include))+ throw error+ }++ const status = response?.status() ?? null++ if (!response?.ok()) {+ await attachScanReport(testInfo, unloadedResult(pagePath, page.url(), status, include))+ expect(+ response?.ok(),+ `Expected a successful response for ${pagePath}, got ${status}`+ ).toBeTruthy()+ return+ }++ await settleForAxe(page)++ await expect(+ page.locator(include),+ `No article matching "${include}" on ${pagePath}. This suite covers guides and ` ++ 'troubleshooting entries; other routes have no article element to scan.'+ ).toBeVisible()++ const result = await scanArticle(page, pagePath, include)+ result.status = status++ await attachScanReport(testInfo, result)++ if (scanLooksEmpty(result)) {+ annotate(+ testInfo,+ `${pagePath} scanned only ${result.elementCount} element(s) in ${include}, so a clean ` ++ 'result here proves nothing. Most likely the page had not finished rendering.'+ )+ }++ const blocking = blockingViolations(result)++ const reported = result.violations.filter((violation) => !blocking.includes(violation))+ if (reported.length) {+ annotate(+ testInfo,+ `${pagePath} has ${reported.length} non-blocking accessibility finding(s): ` ++ reported.map((v) => `${v.id} (${v.nodes.length})`).join(', ')+ )+ }- const axeResults = await new AxeBuilder({ page })- .include(articleSelector)- .withRules(['heading-order', 'page-has-heading-one'])- .analyze()+ const enforced = shouldEnforceAll() ? 'all WCAG 2.1 A/AA rules' : ENFORCED_RULES.join(', ')expect(- axeResults.violations,- `Heading hierarchy issues in ${articleSelector}:\n${JSON.stringify(axeResults.violations, null, 2)}`+ blocking,+ `${pagePath} has blocking a11y violations (${enforced}):\n${formatViolations(blocking)}`).toEqual([])})}diff --git a/e2e/docs/package.json b/e2e/docs/package.jsonindex b1e0371f232..c1eb67b9036 100644--- a/e2e/docs/package.json+++ b/e2e/docs/package.json@@ -16,6 +16,7 @@},"devDependencies": {"@axe-core/playwright": "^4.12.1",- "@types/node": "catalog:"+ "@types/node": "catalog:",+ "axe-core": "^4.12.1"}}diff --git a/e2e/docs/scripts/run-e2e-docs.ts b/e2e/docs/scripts/run-e2e-docs.tsindex f13ddb76494..c8883aa90bb 100644--- a/e2e/docs/scripts/run-e2e-docs.ts+++ b/e2e/docs/scripts/run-e2e-docs.ts@@ -122,10 +122,7 @@ async function resolvePagePaths(): Promise<string[] | null> {async function main() {const rawArgs = process.argv.slice(2)const runAll = rawArgs.includes('--all')- const withoutAll = rawArgs.filter((arg) => arg !== '--all')- // pnpm's `--` separator (from `pnpm run ... -- --list`) can land at index 0- // or, once `--all` is stripped, wherever `--all` used to precede it.- const playwrightArgs = withoutAll[0] === '--' ? withoutAll.slice(1) : withoutAll+ const playwrightArgs = rawArgs.filter((arg) => arg !== '--all' && arg !== '--')const pages = runAll ? await resolveAllPagePaths() : await resolvePagePaths()if (pages === null) {diff --git a/e2e/docs/utils/axe-helpers.ts b/e2e/docs/utils/axe-helpers.tsnew file mode 100644index 00000000000..ba82e3d0a96--- /dev/null+++ b/e2e/docs/utils/axe-helpers.ts@@ -0,0 +1,148 @@+import { AxeBuilder } from '@axe-core/playwright'+import type { Page, TestInfo } from '@playwright/test'+import type { Result } from 'axe-core'++export const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']++export const ENFORCED_RULES = ['heading-order', 'page-has-heading-one']++export const EXCLUDED_RULES = [+ 'color-contrast',+ 'html-has-lang',+ 'html-lang-valid',+ 'html-xml-lang-mismatch',+ 'document-title',+ 'aria-hidden-body',+ 'meta-viewport',+ 'meta-refresh',+ 'css-orientation-lock',+]++export interface A11yScanResult {+ surface: string+ url: string+ include: string+ excludedRules: string[]+ loaded: boolean+ status: number | null+ elementCount: number+ violations: Result[]+}++export function shouldEnforceAll(): boolean {+ return !!process.env.A11Y_ENFORCE_ALL+}++export async function settleForAxe(page: Page): Promise<void> {+ await page.waitForLoadState('domcontentloaded')+ await page.waitForLoadState('networkidle', { timeout: 15_000 }).catch(() => {})++ await page+ .evaluate(+ ({ quietMs, capMs }) =>+ new Promise<void>((resolve) => {+ let timer: ReturnType<typeof setTimeout>+ const observer = new MutationObserver(() => {+ clearTimeout(timer)+ timer = setTimeout(finish, quietMs)+ })++ function finish() {+ clearTimeout(cap)+ clearTimeout(timer)+ observer.disconnect()+ resolve()+ }++ const cap = setTimeout(finish, capMs)+ timer = setTimeout(finish, quietMs)+ observer.observe(document.body, { subtree: true, childList: true, attributes: true })+ }),+ { quietMs: 500, capMs: 5_000 }+ )+ .catch(() => {})+}++export async function scanArticle(+ page: Page,+ surface: string,+ include: string+): Promise<A11yScanResult> {+ const scan = () => new AxeBuilder({ page }).setLegacyMode(true).include(include)++ const reported = await scan().withTags(WCAG_TAGS).disableRules(EXCLUDED_RULES).analyze()++ const enforced = await scan().withRules(ENFORCED_RULES).analyze()++ const byRule = new Map(+ [...reported.violations, ...enforced.violations].map((violation) => [violation.id, violation])+ )++ const elementCount = await page.evaluate(+ (selector) => document.querySelector(selector)?.querySelectorAll('*').length ?? 0,+ include+ )++ return {+ surface,+ url: page.url(),+ include,+ excludedRules: EXCLUDED_RULES,+ loaded: true,+ status: null,+ elementCount,+ violations: [...byRule.values()],+ }+}++export function unloadedResult(+ surface: string,+ url: string,+ status: number | null,+ include: string+): A11yScanResult {+ return {+ surface,+ url,+ include,+ excludedRules: EXCLUDED_RULES,+ loaded: false,+ status,+ elementCount: 0,+ violations: [],+ }+}++export const MIN_MEANINGFUL_ELEMENTS = 20++export function scanLooksEmpty(+ result: A11yScanResult,+ minElements: number = MIN_MEANINGFUL_ELEMENTS+): boolean {+ return result.elementCount < minElements+}++export async function attachScanReport(testInfo: TestInfo, result: A11yScanResult): Promise<void> {+ await testInfo.attach('axe-results.json', {+ body: JSON.stringify(result, null, 2),+ contentType: 'application/json',+ })+}++export function blockingViolations(result: A11yScanResult): Result[] {+ if (shouldEnforceAll()) return result.violations+ return result.violations.filter((violation) => ENFORCED_RULES.includes(violation.id))+}++export function formatViolations(violations: Result[]): string {+ return violations+ .map(+ (violation) =>+ `${violation.id} (${violation.impact}, ${violation.nodes.length} node(s)): ${violation.help}\n` ++ violation.nodes+ .slice(0, 5)+ .map((node) => ` ${node.target.join(' ')}\n ${node.html.slice(0, 200)}`)+ .join('\n')+ )+ .join('\n')+}diff --git a/package.json b/package.jsonindex a495efbcbb6..831395fa112 100644--- a/package.json+++ b/package.json@@ -38,6 +38,7 @@"e2e:ui": "pnpm --prefix e2e/studio run e2e:ui","e2e:docs": "pnpm --prefix e2e/docs run e2e:docs","e2e:docs:all": "pnpm --prefix e2e/docs run e2e:docs:all",+ "e2e:docs:a11y": "pnpm --prefix e2e/docs run e2e:docs:a11y","e2e:docs:ui": "pnpm --prefix e2e/docs run e2e:ui","e2e:docs:local-smoke": "pnpm --prefix e2e/docs run e2e:docs:local-smoke","perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex dad4f818cfb..79bce50d667 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -2027,6 +2027,9 @@ importers:'@types/node':specifier: 'catalog:'version: 22.13.14+ axe-core:+ specifier: ^4.12.1+ version: 4.12.1e2e/studio:dependencies:@@ -10253,6 +10256,7 @@ packages:resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}engines: {node: '>=12.0.0'}+ deprecated: v4 is no longer maintained, upgrade to v5resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}