Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

test(docs): scan changed pages for WCAG 2.1 A/AA in warn mode (#48727)

Miranda Limonczenko777c02c205b5
diff --git a/apps/docs/DEVELOPERS.md b/apps/docs/DEVELOPERS.md
index 10ea54cdb32..37f48953ea1 100644
--- a/apps/docs/DEVELOPERS.md
+++ b/apps/docs/DEVELOPERS.md
@@ -35,6 +35,26 @@ This creates Markdown files for all routes under the `public/markdown/guides` di
For production this setup runs as a `prebuild` task to allow Vercel to bundle these files with middleware and functions.
+## Accessibility checks
+
+Docs pages are scanned for WCAG 2.1 A/AA issues with axe-core, as part of the
+Playwright suite in `e2e/docs`. Pull requests scan the pages your change affects,
+limited to the main article.
+
+To scan the pages your current branch changes:
+
+```bash
+PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y
+```
+
+That resolves which pages to scan from your branch, but reads them from
+production, so it won't see your edits and will 404 on a page you just added.
+Point `PLAYWRIGHT_BASE_URL` at your pull request's preview to scan your own
+content.
+
+See [`e2e/docs/README.md`](https://github.com/supabase/supabase/blob/master/e2e/docs/README.md)
+for coverage and skipped rules.
+
## Contributing
For repo organization and style guide, see the [contributing guide](https://github.com/supabase/supabase/blob/master/apps/docs/CONTRIBUTING.md).
diff --git a/e2e/docs/README.md b/e2e/docs/README.md
index 357f022892e..ea80e37ad60 100644
--- a/e2e/docs/README.md
+++ b/e2e/docs/README.md
@@ -15,6 +15,7 @@ This page covers:
- [Override which pages run](#override-which-pages-run) — when the default git
scope is wrong
- [What the suite covers](#what-the-suite-covers) — in-scope paths and limits
+- [Accessibility scans](#accessibility-scans) — WCAG coverage and skipped rules
- [Debug failures](#debug-failures) — reports and traces
- [How CI uses this suite](#how-ci-uses-this-suite) — pull request behavior
@@ -168,6 +169,30 @@ staged and unstaged working-tree changes.
} | pnpm -C e2e/docs resolve-docs-scope
```
+## Accessibility scans
+
+The `@a11y`-tagged test scans each in-scope page for WCAG 2.1 A/AA violations using
+`@axe-core/playwright`, limited to the main article.
+
+```bash
+PLAYWRIGHT_BASE_URL=https://supabase.com pnpm e2e:docs:a11y
+```
+
+Which pages get scanned comes from your branch, but the content comes from
+whatever you point `PLAYWRIGHT_BASE_URL` at. Production won't have your edits and
+will 404 on a page you just added, so use your pull request's preview to scan your
+own content.
+
+`EXCLUDED_RULES` in `utils/axe-helpers.ts` lists the rules the scan skips.
+`color-contrast` is most of the scan time and finds nothing inside an article, since
+docs contrast comes from shared tokens and chrome. The rest target `<html>`, `<head>`,
+and `<body>`, which an article-scoped scan can't reach.
+
+Cross-origin frames are skipped, so a third-party embed isn't reported as ours.
+
+Not covered: `/docs/reference/*`, shared chrome, and most of WCAG. Keyboard
+navigation, focus management, and screen reader behavior need manual testing.
+
## Debug failures
1. Open the HTML report after a run:
@@ -186,7 +211,8 @@ owned docs content, partials, or `e2e/docs`.
1. Diff the pull request against its base branch and resolve in-scope page paths.
2. Skip Playwright when nothing in scope changed.
3. When `apps/docs` changed, wait for the Vercel docs preview and set
- `PLAYWRIGHT_BASE_URL` to that preview. Otherwise use production.
+ `PLAYWRIGHT_BASE_URL` to that preview. When no preview resolves, skip rather than
+ test against production.
4. Run the suite with `DOCS_E2E_PAGE_PATHS` set to the resolved list.
Draft pull requests stay skipped until you mark them ready for review. Manual
diff --git a/e2e/docs/features/docs-pages.spec.ts b/e2e/docs/features/docs-pages.spec.ts
index 3881a0b54d6..3a58f155021 100644
--- a/e2e/docs/features/docs-pages.spec.ts
+++ b/e2e/docs/features/docs-pages.spec.ts
@@ -1,6 +1,17 @@
-import { AxeBuilder } from '@axe-core/playwright'
import { expect, test } from '@playwright/test'
+import type { TestInfo } from '@playwright/test'
+import {
+ attachScanReport,
+ blockingViolations,
+ ENFORCED_RULES,
+ formatViolations,
+ scanArticle,
+ scanLooksEmpty,
+ settleForAxe,
+ shouldEnforceAll,
+ unloadedResult,
+} from '../utils/axe-helpers.js'
import {
articleSelectorForPagePath,
browserLikeUserAgent,
@@ -10,6 +21,11 @@ import {
const pagePaths = parseDocsE2EPagePaths(process.env.DOCS_E2E_PAGE_PATHS)
+function annotate(testInfo: TestInfo, description: string) {
+ testInfo.annotations.push({ type: 'warning', description })
+ console.warn(`::warning title=Accessibility::${description}`)
+}
+
test.describe('Docs owned pages', () => {
// playwright.config.ts sets fullyParallel: false, and Playwright shards
// work by file rather than by test in that mode — without this, every test
@@ -63,19 +79,69 @@ test.describe('Docs owned pages', () => {
}
for (const pagePath of pagePaths) {
- test(`${pagePath} has a valid heading hierarchy @a11y`, async ({ page }) => {
- const articleSelector = articleSelectorForPagePath(pagePath)
- const response = await page.goto(pagePath)
- expect(response?.ok(), `Expected a successful response for ${pagePath}`).toBeTruthy()
+ test(`${pagePath} has no blocking accessibility violations @a11y`, async ({
+ page,
+ }, testInfo) => {
+ test.setTimeout(120_000)
+
+ const include = articleSelectorForPagePath(pagePath)
+
+ let response
+ try {
+ response = await page.goto(pagePath)
+ } catch (error) {
+ await attachScanReport(testInfo, unloadedResult(pagePath, pagePath, null, include))
+ throw error
+ }
+
+ const status = response?.status() ?? null
+
+ if (!response?.ok()) {
+ await attachScanReport(testInfo, unloadedResult(pagePath, page.url(), status, include))
+ expect(
+ response?.ok(),
+ `Expected a successful response for ${pagePath}, got ${status}`
+ ).toBeTruthy()
+ return
+ }
+
+ await settleForAxe(page)
+
+ await expect(
+ page.locator(include),
+ `No article matching "${include}" on ${pagePath}. This suite covers guides and ` +
+ 'troubleshooting entries; other routes have no article element to scan.'
+ ).toBeVisible()
+
+ const result = await scanArticle(page, pagePath, include)
+ result.status = status
+
+ await attachScanReport(testInfo, result)
+
+ if (scanLooksEmpty(result)) {
+ annotate(
+ testInfo,
+ `${pagePath} scanned only ${result.elementCount} element(s) in ${include}, so a clean ` +
+ 'result here proves nothing. Most likely the page had not finished rendering.'
+ )
+ }
+
+ const blocking = blockingViolations(result)
+
+ const reported = result.violations.filter((violation) => !blocking.includes(violation))
+ if (reported.length) {
+ annotate(
+ testInfo,
+ `${pagePath} has ${reported.length} non-blocking accessibility finding(s): ` +
+ reported.map((v) => `${v.id} (${v.nodes.length})`).join(', ')
+ )
+ }
- const axeResults = await new AxeBuilder({ page })
- .include(articleSelector)
- .withRules(['heading-order', 'page-has-heading-one'])
- .analyze()
+ const enforced = shouldEnforceAll() ? 'all WCAG 2.1 A/AA rules' : ENFORCED_RULES.join(', ')
expect(
- axeResults.violations,
- `Heading hierarchy issues in ${articleSelector}:\n${JSON.stringify(axeResults.violations, null, 2)}`
+ blocking,
+ `${pagePath} has blocking a11y violations (${enforced}):\n${formatViolations(blocking)}`
).toEqual([])
})
}
diff --git a/e2e/docs/package.json b/e2e/docs/package.json
index b1e0371f232..c1eb67b9036 100644
--- a/e2e/docs/package.json
+++ b/e2e/docs/package.json
@@ -16,6 +16,7 @@
},
"devDependencies": {
"@axe-core/playwright": "^4.12.1",
- "@types/node": "catalog:"
+ "@types/node": "catalog:",
+ "axe-core": "^4.12.1"
}
}
diff --git a/e2e/docs/scripts/run-e2e-docs.ts b/e2e/docs/scripts/run-e2e-docs.ts
index f13ddb76494..c8883aa90bb 100644
--- a/e2e/docs/scripts/run-e2e-docs.ts
+++ b/e2e/docs/scripts/run-e2e-docs.ts
@@ -122,10 +122,7 @@ async function resolvePagePaths(): Promise<string[] | null> {
async function main() {
const rawArgs = process.argv.slice(2)
const runAll = rawArgs.includes('--all')
- const withoutAll = rawArgs.filter((arg) => arg !== '--all')
- // pnpm's `--` separator (from `pnpm run ... -- --list`) can land at index 0
- // or, once `--all` is stripped, wherever `--all` used to precede it.
- const playwrightArgs = withoutAll[0] === '--' ? withoutAll.slice(1) : withoutAll
+ const playwrightArgs = rawArgs.filter((arg) => arg !== '--all' && arg !== '--')
const pages = runAll ? await resolveAllPagePaths() : await resolvePagePaths()
if (pages === null) {
diff --git a/e2e/docs/utils/axe-helpers.ts b/e2e/docs/utils/axe-helpers.ts
new file mode 100644
index 00000000000..ba82e3d0a96
--- /dev/null
+++ b/e2e/docs/utils/axe-helpers.ts
@@ -0,0 +1,148 @@
+import { AxeBuilder } from '@axe-core/playwright'
+import type { Page, TestInfo } from '@playwright/test'
+import type { Result } from 'axe-core'
+
+export const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']
+
+export const ENFORCED_RULES = ['heading-order', 'page-has-heading-one']
+
+export const EXCLUDED_RULES = [
+ 'color-contrast',
+ 'html-has-lang',
+ 'html-lang-valid',
+ 'html-xml-lang-mismatch',
+ 'document-title',
+ 'aria-hidden-body',
+ 'meta-viewport',
+ 'meta-refresh',
+ 'css-orientation-lock',
+]
+
+export interface A11yScanResult {
+ surface: string
+ url: string
+ include: string
+ excludedRules: string[]
+ loaded: boolean
+ status: number | null
+ elementCount: number
+ violations: Result[]
+}
+
+export function shouldEnforceAll(): boolean {
+ return !!process.env.A11Y_ENFORCE_ALL
+}
+
+export async function settleForAxe(page: Page): Promise<void> {
+ await page.waitForLoadState('domcontentloaded')
+ await page.waitForLoadState('networkidle', { timeout: 15_000 }).catch(() => {})
+
+ await page
+ .evaluate(
+ ({ quietMs, capMs }) =>
+ new Promise<void>((resolve) => {
+ let timer: ReturnType<typeof setTimeout>
+ const observer = new MutationObserver(() => {
+ clearTimeout(timer)
+ timer = setTimeout(finish, quietMs)
+ })
+
+ function finish() {
+ clearTimeout(cap)
+ clearTimeout(timer)
+ observer.disconnect()
+ resolve()
+ }
+
+ const cap = setTimeout(finish, capMs)
+ timer = setTimeout(finish, quietMs)
+ observer.observe(document.body, { subtree: true, childList: true, attributes: true })
+ }),
+ { quietMs: 500, capMs: 5_000 }
+ )
+ .catch(() => {})
+}
+
+export async function scanArticle(
+ page: Page,
+ surface: string,
+ include: string
+): Promise<A11yScanResult> {
+ const scan = () => new AxeBuilder({ page }).setLegacyMode(true).include(include)
+
+ const reported = await scan().withTags(WCAG_TAGS).disableRules(EXCLUDED_RULES).analyze()
+
+ const enforced = await scan().withRules(ENFORCED_RULES).analyze()
+
+ const byRule = new Map(
+ [...reported.violations, ...enforced.violations].map((violation) => [violation.id, violation])
+ )
+
+ const elementCount = await page.evaluate(
+ (selector) => document.querySelector(selector)?.querySelectorAll('*').length ?? 0,
+ include
+ )
+
+ return {
+ surface,
+ url: page.url(),
+ include,
+ excludedRules: EXCLUDED_RULES,
+ loaded: true,
+ status: null,
+ elementCount,
+ violations: [...byRule.values()],
+ }
+}
+
+export function unloadedResult(
+ surface: string,
+ url: string,
+ status: number | null,
+ include: string
+): A11yScanResult {
+ return {
+ surface,
+ url,
+ include,
+ excludedRules: EXCLUDED_RULES,
+ loaded: false,
+ status,
+ elementCount: 0,
+ violations: [],
+ }
+}
+
+export const MIN_MEANINGFUL_ELEMENTS = 20
+
+export function scanLooksEmpty(
+ result: A11yScanResult,
+ minElements: number = MIN_MEANINGFUL_ELEMENTS
+): boolean {
+ return result.elementCount < minElements
+}
+
+export async function attachScanReport(testInfo: TestInfo, result: A11yScanResult): Promise<void> {
+ await testInfo.attach('axe-results.json', {
+ body: JSON.stringify(result, null, 2),
+ contentType: 'application/json',
+ })
+}
+
+export function blockingViolations(result: A11yScanResult): Result[] {
+ if (shouldEnforceAll()) return result.violations
+ return result.violations.filter((violation) => ENFORCED_RULES.includes(violation.id))
+}
+
+export function formatViolations(violations: Result[]): string {
+ return violations
+ .map(
+ (violation) =>
+ `${violation.id} (${violation.impact}, ${violation.nodes.length} node(s)): ${violation.help}\n` +
+ violation.nodes
+ .slice(0, 5)
+ .map((node) => ` ${node.target.join(' ')}\n ${node.html.slice(0, 200)}`)
+ .join('\n')
+ )
+ .join('\n')
+}
diff --git a/package.json b/package.json
index a495efbcbb6..831395fa112 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
"e2e:ui": "pnpm --prefix e2e/studio run e2e:ui",
"e2e:docs": "pnpm --prefix e2e/docs run e2e:docs",
"e2e:docs:all": "pnpm --prefix e2e/docs run e2e:docs:all",
+ "e2e:docs:a11y": "pnpm --prefix e2e/docs run e2e:docs:a11y",
"e2e:docs:ui": "pnpm --prefix e2e/docs run e2e:ui",
"e2e:docs:local-smoke": "pnpm --prefix e2e/docs run e2e:docs:local-smoke",
"perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index dad4f818cfb..79bce50d667 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2027,6 +2027,9 @@ importers:
'@types/node':
specifier: 'catalog:'
version: 22.13.14
+ axe-core:
+ specifier: ^4.12.1
+ version: 4.12.1
e2e/studio:
dependencies:
@@ -10253,6 +10256,7 @@ packages:
resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}
engines: {node: '>=12.0.0'}
+ deprecated: v4 is no longer maintained, upgrade to v5
resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}