Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: a module that lists main's accessibility specs a stale head never ran (#10685)

Cindy Zhangdae3bd912e31
diff --git a/.github/scripts/a11y-scope-overlap.mjs b/.github/scripts/a11y-scope-overlap.mjs
new file mode 100644
index 0000000000..9f4b46b5dd
--- /dev/null
+++ b/.github/scripts/a11y-scope-overlap.mjs
@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file The browser accessibility specs main added or changed since a pull
+ * request's merge base that select the pull request's changes but are not at
+ * its head.
+ * @input `--base` (`git merge-base <main> <head>`), `--main`, `--head`,
+ * `--paths-file` (the pull request's changed paths, one per line) and
+ * `--labelled` when the pull request carries the accessibility run label,
+ * run from a checkout of main.
+ * @output JSON `{"version": 1, "specs": [{"path", "reason", "scope"}]}`, where
+ * `reason` is `added-on-main` or `changed-on-main` and `scope` names what
+ * selects the spec; exit 0. Any other exit means the answer is unknown.
+ * @position merge-guard reads it: a spec listed here never ran on the pull
+ * request's runs, which use the head's specs, so a non-empty list needs a
+ * fresh ref (update the branch) before the merge, as a file overlap does.
+ */
+
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import {fileURLToPath} from 'node:url';
+
+import {planDependentAudits, readPrimitives} from './a11y-dependent-audits.mjs';
+import {
+ componentFolderOf,
+ importerGraph,
+ isRenderedSource,
+ parseRouting,
+ selectSpecs,
+} from './a11y-pr-contracts.mjs';
+import {isA11ySpecFile} from './lib/a11y-spec-files.mjs';
+
+export const VERSION = 1;
+
+/**
+ * The components a pull request's changed paths touch: each changed source
+ * file's component folder, and the component its file is named for.
+ * @param {string[]} changedPaths
+ */
+export function touchedComponents(changedPaths) {
+ const components = new Set();
+ for (const file of changedPaths.filter(isRenderedSource)) {
+ const folder = componentFolderOf(file);
+ if (!folder) continue;
+ components.add(folder);
+ const named = /^([A-Z][A-Za-z0-9]*)\./.exec(file.split('/').pop() ?? '');
+ if (named) components.add(`${folder.split('/')[0]}/${named[1]}`);
+ }
+ return [...components];
+}
+
+/**
+ * Why a spec selects these changed paths, or null. An `audit` spec counts when
+ * the change owes it, by the dependent-audit rule; every other mode by its own
+ * routing lines, as a pull request's labelled run selects it.
+ * @param {string} file
+ * @param {string} source the spec's source on main
+ * @param {string[]} changedPaths
+ * @param {Set<string>} owedAudits the audit specs the change owes
+ * @param {boolean} [labelled] the pull request carries the accessibility run
+ * label
+ */
+export function selectingScope(
+ file,
+ source,
+ changedPaths,
+ owedAudits,
+ labelled = false,
+) {
+ const routing = parseRouting(source);
+ // A pull request's own run runs an unchanged `always` spec only with the
+ // accessibility run label.
+ if (routing.mode === 'always') return labelled ? 'always' : null;
+ if (routing.mode === 'audit') {
+ return owedAudits.has(file)
+ ? `owed audit of ${componentFolderOf(file)}`
+ : null;
+ }
+ const [selected] = selectSpecs({
+ specs: [{file, routing}],
+ isPullRequest: true,
+ changedPaths,
+ components: touchedComponents(changedPaths),
+ title: '',
+ });
+ return selected?.reason ?? null;
+}
+
+/**
+ * The specs that need a fresh ref before the merge.
+ * @param {{path: string, status: 'added' | 'changed', mainSource: string,
+ * headSource: string | null}[]} changedOnMain the spec files main added or
+ * changed since the merge base, with their source on main and at the head
+ * @param {string[]} changedPaths the pull request's changed paths
+ * @param {{primitives: string[], importers: Map<string, Set<string>>,
+ * labelled?: boolean}} graph main's shared primitives and import graph, for
+ * the audits a change owes, and whether the run label is on
+ * @returns {{path: string, reason: string, scope: string}[]}
+ */
+export function overlappingSpecs(
+ changedOnMain,
+ changedPaths,
+ {primitives, importers, labelled = false},
+) {
+ const routings = changedOnMain.map(spec => ({
+ ...spec,
+ routing: parseRouting(spec.mainSource),
+ }));
+ const {owed} = planDependentAudits({
+ changed: changedPaths,
+ primitives,
+ specs: routings.map(({path, routing}) => ({
+ file: path,
+ mode: routing.mode,
+ })),
+ importers,
+ seconds: {},
+ attempt: 2,
+ });
+ const owedAudits = new Set(owed);
+ const out = [];
+ for (const {path, status, mainSource, headSource} of routings) {
+ if (headSource === mainSource) continue;
+ const scope = selectingScope(
+ path,
+ mainSource,
+ changedPaths,
+ owedAudits,
+ labelled,
+ );
+ if (scope == null) continue;
+ out.push({
+ path,
+ reason: status === 'added' ? 'added-on-main' : 'changed-on-main',
+ scope,
+ });
+ }
+ return out.sort((a, b) => (a.path < b.path ? -1 : 1));
+}
+
+function git(args) {
+ return execFileSync('git', args, {
+ encoding: 'utf8',
+ maxBuffer: 1 << 28,
+ stdio: ['ignore', 'pipe', 'ignore'],
+ });
+}
+
+/** A file's source at a commit, or null when the commit lacks it. */
+function sourceAt(sha, file) {
+ try {
+ return git(['show', `${sha}:${file}`]);
+ } catch {
+ return null;
+ }
+}
+
+function main() {
+ const argv = process.argv.slice(2);
+ const value = name => {
+ const at = argv.indexOf(name);
+ return at === -1 ? undefined : argv[at + 1];
+ };
+ const base = value('--base');
+ const mainSha = value('--main');
+ const head = value('--head');
+ const pathsFile = value('--paths-file');
+ if (!base || !mainSha || !head || !pathsFile) {
+ console.error(
+ 'usage: a11y-scope-overlap.mjs --base <git merge-base main head> --main <main> --head <head> --paths-file <file> [--labelled]',
+ );
+ process.exit(2);
+ }
+ const changedPaths = fs
+ .readFileSync(pathsFile, 'utf8')
+ .split('\n')
+ .map(line => line.trim())
+ .filter(Boolean);
+ // An empty list is an unknown change, never "no overlap".
+ if (changedPaths.length === 0) {
+ console.error(`${pathsFile} lists no changed path.`);
+ process.exit(1);
+ }
+ const changedOnMain = git([
+ 'diff',
+ '--name-status',
+ '--no-renames',
+ base,
+ mainSha,
+ ])
+ .split('\n')
+ .map(line => line.split('\t'))
+ .filter(
+ ([status, file]) => file && /^[AM]$/.test(status) && isA11ySpecFile(file),
+ )
+ .map(([status, file]) => ({
+ path: file,
+ status: status === 'A' ? 'added' : 'changed',
+ mainSource: sourceAt(mainSha, file) ?? '',
+ headSource: sourceAt(head, file),
+ }));
+ const root = process.cwd();
+ const specs = overlappingSpecs(changedOnMain, changedPaths, {
+ labelled: argv.includes('--labelled'),
+ primitives: readPrimitives(root),
+ importers: importerGraph(root),
+ });
+ console.log(JSON.stringify({version: VERSION, specs}));
+}
+
+if (
+ process.argv[1] &&
+ fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
+)
+ main();
diff --git a/.github/scripts/a11y-scope-overlap.test.mjs b/.github/scripts/a11y-scope-overlap.test.mjs
new file mode 100644
index 0000000000..b16dc97632
--- /dev/null
+++ b/.github/scripts/a11y-scope-overlap.test.mjs
@@ -0,0 +1,302 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Contract: merge-guard's scope-overlap check lists the browser
+ * accessibility specs main added or changed that select a pull request's
+ * changes but are not at its head.
+ *
+ * A pull request's runs use the head's specs. A table-tree change merged green
+ * while main had added a Table focus-ring spec whose scope covers it, because
+ * no file overlapped and the branch never took main's spec.
+ */
+
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {describe, expect, it} from 'vitest';
+
+import {
+ overlappingSpecs,
+ selectingScope,
+ touchedComponents,
+ VERSION,
+} from './a11y-scope-overlap.mjs';
+
+const SCRIPT = path.join(import.meta.dirname, 'a11y-scope-overlap.mjs');
+
+// The table-tree change, and the focus-ring spec main added meanwhile.
+const TREE_CHANGE = [
+ 'packages/core/src/Table/plugins/tree/useTableTreeData.tsx',
+ 'packages/core/src/Table/plugins/tree/useTableTreeData.test.tsx',
+ 'packages/core/src/Table/useTableTreeData.doc.mjs',
+ 'docs/specs/components/Table/plugins/tree/useTableTreeData.spec.md',
+ '.changeset/table-tree-row-aria-expanded.md',
+];
+const FOCUS_RING =
+ 'packages/core/src/Table/__tests__/TableFocusRing.a11y.chromium.spec.ts';
+// Button is a shared primitive; ChatSendButton renders it.
+const GRAPH = {
+ primitives: ['core/Button', 'core/IconButton'],
+ importers: new Map([
+ [
+ 'packages/core/src/Button/Button.tsx',
+ new Set(['packages/core/src/Chat/ChatSendButton.tsx']),
+ ],
+ ]),
+};
+const FOCUS_RING_SOURCE = `// Copyright (c) Meta Platforms, Inc. and affiliates.
+// @pr-a11y scope ^(packages/core/src/Table/|packages/core/src/utils/focusOutline|apps/storybook/stories/Table(Tree)?\\.stories)
+test('…', () => {});
+`;
+
+describe('overlappingSpecs', () => {
+ it('lists a scoped spec main added that covers the change, and the head lacks', () => {
+ expect(
+ overlappingSpecs(
+ [
+ {
+ path: FOCUS_RING,
+ status: 'added',
+ mainSource: FOCUS_RING_SOURCE,
+ headSource: null,
+ },
+ ],
+ TREE_CHANGE,
+ GRAPH,
+ ),
+ ).toEqual([
+ {
+ path: FOCUS_RING,
+ reason: 'added-on-main',
+ scope:
+ 'changed packages/core/src/Table/plugins/tree/useTableTreeData.tsx',
+ },
+ ]);
+ });
+
+ it('lists a spec main changed, and leaves out one the head already has', () => {
+ const changed = {
+ path: FOCUS_RING,
+ status: 'changed',
+ mainSource: FOCUS_RING_SOURCE,
+ headSource: `${FOCUS_RING_SOURCE}// older\n`,
+ };
+ expect(overlappingSpecs([changed], TREE_CHANGE, GRAPH)).toMatchObject([
+ {path: FOCUS_RING, reason: 'changed-on-main'},
+ ]);
+ expect(
+ overlappingSpecs(
+ [{...changed, headSource: FOCUS_RING_SOURCE}],
+ TREE_CHANGE,
+ GRAPH,
+ ),
+ ).toEqual([]);
+ });
+
+ it('leaves out a spec whose scope does not cover the change', () => {
+ const tabs = 'packages/core/src/Tabs/__tests__/Tabs.a11y.chromium.spec.ts';
+ expect(
+ overlappingSpecs(
+ [
+ {
+ path: tabs,
+ status: 'added',
+ mainSource: 'test();\n',
+ headSource: null,
+ },
+ ],
+ TREE_CHANGE,
+ GRAPH,
+ ),
+ ).toEqual([]);
+ });
+
+ it("lists an audit main added that a shared primitive's change owes", () => {
+ const chat =
+ 'packages/core/src/Chat/__tests__/ChatSendButtonAudit.a11y.chromium.spec.ts';
+ const added = {
+ path: chat,
+ status: 'added',
+ mainSource: '// @pr-a11y audit ChatSendButton\n',
+ headSource: null,
+ };
+ expect(
+ overlappingSpecs([added], ['packages/core/src/Button/Button.tsx'], GRAPH),
+ ).toEqual([
+ {path: chat, reason: 'added-on-main', scope: 'owed audit of core/Chat'},
+ ]);
+ // A test-only change to Button renders nothing, so it owes no audit.
+ expect(
+ overlappingSpecs(
+ [added],
+ ['packages/core/src/Button/Button.test.tsx'],
+ GRAPH,
+ ),
+ ).toEqual([]);
+ });
+
+ it("counts a component's own audit only for a change to its rendered source", () => {
+ const audit =
+ 'packages/core/src/Table/__tests__/TableAudit.a11y.chromium.spec.ts';
+ const added = {
+ path: audit,
+ status: 'added',
+ mainSource: '// @pr-a11y audit Table\n',
+ headSource: null,
+ };
+ expect(overlappingSpecs([added], TREE_CHANGE, GRAPH)).toMatchObject([
+ {path: audit, scope: 'owed audit of core/Table'},
+ ]);
+ expect(
+ overlappingSpecs(
+ [added],
+ [
+ 'packages/core/src/Table/Table.doc.mjs',
+ 'docs/specs/components/Table/Table.spec.md',
+ ],
+ GRAPH,
+ ),
+ ).toEqual([]);
+ expect(
+ selectingScope(
+ audit,
+ '// @pr-a11y audit Table\n',
+ TREE_CHANGE,
+ new Set(),
+ ),
+ ).toBeNull();
+ });
+
+ it('lists a changed always spec only for a pull request with the run label', () => {
+ const always = 'internal/a11y-spec/src/patterns/landmarks.chromium.spec.ts';
+ const changed = [
+ {
+ path: always,
+ status: 'changed',
+ mainSource: '// @pr-a11y always\n',
+ headSource: 'older\n',
+ },
+ ];
+ expect(overlappingSpecs(changed, ['README.md'], GRAPH)).toEqual([]);
+ expect(
+ overlappingSpecs(changed, ['README.md'], {...GRAPH, labelled: true}),
+ ).toEqual([{path: always, reason: 'changed-on-main', scope: 'always'}]);
+ });
+
+ it('reads the touched components from changed source, never from a record', () => {
+ expect(touchedComponents(TREE_CHANGE)).toEqual(['core/Table']);
+ expect(
+ touchedComponents(['packages/core/src/Table/TableRow.tsx']).sort(),
+ ).toEqual(['core/Table', 'core/TableRow']);
+ });
+});
+
+describe('the command merge-guard runs', () => {
+ it('prints version 1 JSON for a merge base, main and a head', () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'scope-overlap-'));
+ const run = (...args) =>
+ execFileSync('git', args, {cwd: dir, encoding: 'utf8'}).trim();
+ try {
+ run('init', '-q', '-b', 'main');
+ run('config', 'user.email', '[email protected]');
+ run('config', 'user.name', 't');
+ const write = (file, text) => {
+ fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true});
+ fs.writeFileSync(path.join(dir, file), text);
+ };
+ write(TREE_CHANGE[0], 'export const a = 1;\n');
+ write(
+ '.github/a11y-shared-primitives.json',
+ JSON.stringify({primitives: GRAPH.primitives}),
+ );
+ run('add', '-A');
+ run('commit', '-qm', 'base');
+ const base = run('rev-parse', 'HEAD');
+ write(FOCUS_RING, FOCUS_RING_SOURCE);
+ run('add', '-A');
+ run('commit', '-qm', 'main adds the spec');
+ const main = run('rev-parse', 'HEAD');
+ run('checkout', '-q', '-b', 'pr', base);
+ write(TREE_CHANGE[0], 'export const a = 2;\n');
+ run('commit', '-qam', 'the change');
+ const head = run('rev-parse', 'HEAD');
+ const paths = path.join(dir, 'paths.txt');
+ fs.writeFileSync(paths, `${TREE_CHANGE[0]}\n`);
+ const out = execFileSync(
+ process.execPath,
+ [
+ SCRIPT,
+ '--base',
+ base,
+ '--main',
+ main,
+ '--head',
+ head,
+ '--paths-file',
+ paths,
+ ],
+ {cwd: dir, encoding: 'utf8'},
+ );
+ expect(JSON.parse(out)).toEqual({
+ version: VERSION,
+ specs: [
+ {
+ path: FOCUS_RING,
+ reason: 'added-on-main',
+ scope: `changed ${TREE_CHANGE[0]}`,
+ },
+ ],
+ });
+ expect(VERSION).toBe(1);
+
+ // An empty list of changed paths is unknown, not "no overlap".
+ const empty = path.join(dir, 'empty.txt');
+ fs.writeFileSync(empty, '\n');
+ expect(() =>
+ execFileSync(
+ process.execPath,
+ [
+ SCRIPT,
+ '--base',
+ base,
+ '--main',
+ main,
+ '--head',
+ head,
+ '--paths-file',
+ empty,
+ ],
+ {cwd: dir, encoding: 'utf8', stdio: 'pipe'},
+ ),
+ ).toThrow();
+
+ // A routing line main cannot read makes the answer unknown: a non-zero
+ // exit, which merge-guard treats as overlap.
+ run('checkout', '-q', 'main');
+ write(FOCUS_RING, '// @pr-a11y nonsense\n');
+ run('commit', '-qam', 'main breaks the routing line');
+ const broken = run('rev-parse', 'HEAD');
+ expect(() =>
+ execFileSync(
+ process.execPath,
+ [
+ SCRIPT,
+ '--base',
+ base,
+ '--main',
+ broken,
+ '--head',
+ head,
+ '--paths-file',
+ paths,
+ ],
+ {cwd: dir, encoding: 'utf8', stdio: 'pipe'},
+ ),
+ ).toThrow();
+ } finally {
+ fs.rmSync(dir, {recursive: true, force: true});
+ }
+ });
+});