An open source design system that's fully customizable and agent ready
TypeScript
ci: run a touched component's own accessibility contracts beside the tests, and block the merge on them (#9707)
Cindy Zhang8a20186e8e24
diff --git a/.github/a11y-contract-durations.json b/.github/a11y-contract-durations.jsonindex 6772af5006..009ecc8243 100644--- a/.github/a11y-contract-durations.json+++ b/.github/a11y-contract-durations.json@@ -1,5 +1,5 @@{- "$comment": "Seconds of test time per accessibility contract spec file and per test (file and title path), from a release check's contract legs (four workers). Only shard balance depends on these numbers; a missing or stale entry never drops or repeats a test. Regenerate with: node .github/scripts/a11y-contract-shard.mjs record <leg logs>…",+ "$comment": "Seconds of test time per accessibility contract spec file and per test (file and title path), from a release check's contract legs (four workers). They balance the release shards, where a missing or stale entry never drops or repeats a test. They also plan the pull-request contract job: a file starts there only when its recorded time fits the job's budget, and a file that does not fit is reported as not run. A missing entry counts as the median file. Regenerate with: node .github/scripts/a11y-contract-shard.mjs record <leg logs>…","source": "release check contract legs at main 2026-11-01","seconds": {"internal/a11y-spec/src/harness/chromium.chromium.spec.ts": 2.6,diff --git a/.github/scripts/a11y-pr-contracts.mjs b/.github/scripts/a11y-pr-contracts.mjsindex 7f240d8748..019bcba623 100644--- a/.github/scripts/a11y-pr-contracts.mjs+++ b/.github/scripts/a11y-pr-contracts.mjs@@ -12,7 +12,9 @@* (`.github/a11y-audit-axe.json` holds the audits that need an open state, a* different component set, or a different receipt folder); the selection,* the JSON report, and the axe report in `pr-a11y-evidence/`.- * @position `pnpm test:a11y-contract:pr`, run by the `pr-a11y` job in ci.yml.+ * @position `pnpm test:a11y-contract:pr`, run by the `pr-a11y` job in ci.yml;+ * with `--changed-only` and `--owned-by`, by its+ * `pr-component-a11y-contracts` job.* Adding a browser spec needs no workflow change: the spec is discovered by* its file name, and its own routing lines decide when a PR runs it. The* release check runs every spec in `a11y-contracts` instead.@@ -22,7 +24,8 @@* @pr-a11y always every PR accessibility run, and release-check* @pr-a11y scope <token> ... also run when one token matches (see below)* @pr-a11y audit <Name>, ... only on a "[Component Audit] <Name>:" PR- * @pr-a11y release-only never on a PR; the release check runs it+ * @pr-a11y release-only the release check; on a PR only for its own+ * component (see below)** A spec with no `always`, `audit`, or `release-only` line is scoped: a PR runs* it when it changes the spec, the spec's component folder, or a component the@@ -34,17 +37,24 @@* the PR's component scope.** Unknown scope (no merge base, no changed-file list) runs every scoped spec,- * never fewer.+ * never fewer. A PR runs a `release-only` spec only when the spec changes, its+ * component folder changes, or a component it is named after is in scope;+ * never for a `scope` token or an unknown scope.*- * `--changed-only <file>` (the default touched-components check, spec:AST-030- * FR12): run only the browser specs the pull request adds or modifies, listed- * one per line in <file>. It reads no component scope or title, runs no- * component audit, and widens to nothing else; a changed `release-only` or- * `audit` spec still waits for the release check or its audit pull request.- * To stay inside the job's 15-minute budget it runs at most- * CHANGED_SPEC_LIMIT files, in path order, and stops after- * CHANGED_SPEC_BUDGET_MS; the rest are reported as not run, never as passed. Locally, `pnpm test:a11y-contract:pr origin/main --dry-run`- * prints what a PR from this branch would run.+ * `--changed-only <specs> --changed-files <files> --owned-by <components>`+ * (the default check of the components a pull request touches): run the+ * browser specs the pull request adds or modifies, listed one per line in+ * <specs>, then the `scoped` and `release-only` specs that the comma-separated+ * touched components own (`ownedSpecs`; <files> lists every changed path, one+ * per line, for the folder-change rule). It reads no title, runs no component+ * audit, and runs no spec of a reverse dependent; a changed `audit` spec still+ * waits for its audit pull request. To stay inside the job's 15-minute budget+ * it takes at most CHANGED_SPEC_LIMIT changed files, starts each file only+ * when the run still fits DEFAULT_CHECK_BUDGET_MS at that file's recorded time+ * (.github/a11y-contract-durations.json), and stops at that budget. Each file+ * it leaves out is reported as not run, never as passed or failed. Locally,+ * `pnpm test:a11y-contract:pr origin/main --dry-run` prints what a PR from+ * this branch would run.** The CI step loads this file from the merge commit its steps come from* (`WORKFLOW_SHA`), and reads each spec's routing lines from that commit too,@@ -262,14 +272,16 @@ export function selectSpecs({for (const {file, routing} of specs) {const reason = (() => {if (routing.mode === 'always') return 'always';- if (!isPullRequest || routing.mode === 'release-only') return null;+ if (!isPullRequest) return null;if (routing.mode === 'audit') {const name = routing.audits.find(n =>audited.some(a => sameComponent(a, n)),);return name ? `component audit ${name}` : null;}- if (changedPaths == null) return 'scope unknown';+ if (changedPaths == null) {+ return routing.mode === 'release-only' ? null : 'scope unknown';+ }if (changedPaths.includes(file)) return 'spec changed';const owners = specOwners(file);for (const prefix of owners.prefixes) {@@ -279,6 +291,7 @@ export function selectSpecs({for (const component of owners.components) {if (components.includes(component)) return `component ${component}`;}+ if (routing.mode === 'release-only') return null;for (const token of routing.scope) {if (token.startsWith('^')) {const pattern = new RegExp(token);@@ -295,11 +308,24 @@ export function selectSpecs({return selected;}-/** The most changed spec files the default check runs (spec:AST-030 FR12). */+/** The most changed spec files the default check runs. */export const CHANGED_SPEC_LIMIT = 4;-/** The changed-spec run's wall-clock bound, inside the job's 15 minutes. */-export const CHANGED_SPEC_BUDGET_MS = 8 * 60 * 1000;+/** The default check's wall-clock bound, inside the job's 15 minutes. */+export const DEFAULT_CHECK_BUDGET_MS = 8 * 60 * 1000;++/**+ * The times come from the release legs. A pull-request runner can be slower,+ * and a file that the budget stops fails the run. Thus the plan adds a margin+ * to each recorded time and keeps time for Playwright to start.+ */+export const RECORDED_TIME_MARGIN = 1.25;+export const STARTUP_MS = 30 * 1000;++/** The expected time of a file that has no recorded time. */+const UNRECORDED_FILE_SECONDS = 60;++const TITLE_DELIMITER = ' \u203A ';/*** Split a changed-only selection at the limit: the first files, in path order,@@ -318,6 +344,156 @@ export function withinBudget(selected, limit = CHANGED_SPEC_LIMIT) {};}+/** Whether a path is inside a component folder (`packages/<pkg>/src/<Name>/`). */+export function isComponentPath(file) {+ return /^packages\/[^/]+\/src\/[A-Z][^/]*\//.test(file);+}++/**+ * The `scoped` and `release-only` specs the touched components own. It is the+ * labelled run's selection (`selectSpecs`) with two limits: only changes inside+ * a component folder count, so a change to a shared module, the contract+ * harness or a story selects nothing; and an `audit` or `always` spec is never+ * owned. With no touched component nothing is owned.+ * @param {object} options+ * @param {{file: string, routing: ReturnType<typeof parseRouting>}[]} options.specs+ * @param {string[]} options.components package-qualified touched components+ * @param {string[]} options.changedPaths+ * @returns {{file: string, reason: string}[]}+ */+export function ownedSpecs({specs, components, changedPaths}) {+ if (components.length === 0) return [];+ return selectSpecs({+ specs: specs.filter(+ ({routing}) =>+ routing.mode === 'scoped' || routing.mode === 'release-only',+ ),+ isPullRequest: true,+ changedPaths: changedPaths.filter(isComponentPath),+ components,+ title: '',+ });+}++/**+ * Recorded seconds per spec file. Test entries (file and title path) are left+ * out.+ * @param {{seconds?: Record<string, number>}} durations+ */+export function fileSeconds(durations) {+ return Object.fromEntries(+ Object.entries(durations?.seconds ?? {}).filter(+ ([key, value]) =>+ !key.includes(TITLE_DELIMITER) && Number.isFinite(value),+ ),+ );+}++/**+ * The wall-clock time of one Playwright run. Playwright gives whole files, in+ * path order, to the first free worker.+ * @param {string[]} files+ * @param {(file: string) => number} msOf+ * @param {number} workers+ */+export function projectedMs(files, msOf, workers) {+ const busyUntil = Array.from({length: Math.max(1, workers)}, () => 0);+ for (const file of [...files].sort()) {+ const next = busyUntil.indexOf(Math.min(...busyUntil));+ busyUntil[next] += msOf(file);+ }+ return Math.max(...busyUntil);+}++function median(values) {+ if (values.length === 0) return UNRECORDED_FILE_SECONDS;+ const sorted = [...values].sort((a, b) => a - b);+ const middle = Math.floor(sorted.length / 2);+ return sorted.length % 2+ ? sorted[middle]+ : (sorted[middle - 1] + sorted[middle]) / 2;+}++/**+ * The default check's run. First the changed specs, at most the limit, in path+ * order. Then the owned specs, in path order. A file starts only when the run+ * still fits the budget with that file at its recorded time (with the margin).+ * The other files are not run, with the label that runs them.+ * @param {object} options+ * @param {{file: string, reason: string}[]} options.changed+ * @param {{file: string, reason: string}[]} options.owned+ * @param {Record<string, number>} options.seconds recorded seconds per file+ * @param {number} options.workers+ * @param {number} [options.budgetMs]+ */+export function planDefaultCheck({+ changed,+ owned,+ seconds,+ workers,+ budgetMs = DEFAULT_CHECK_BUDGET_MS,+}) {+ const {run: changedInLimit, notRun} = withinBudget(changed);+ const fallback = median(Object.values(seconds));+ const secondsOf = file => seconds[file] ?? fallback;+ const msOf = file => secondsOf(file) * 1000 * RECORDED_TIME_MARGIN;+ const planned = new Set(changed.map(({file}) => file));+ const run = [];+ const files = [];+ const ordered = [...owned]+ .filter(({file}) => !planned.has(file))+ .sort((a, b) => a.file.localeCompare(b.file));+ for (const entry of [...changedInLimit, ...ordered]) {+ const withFile = [...files, entry.file];+ if (STARTUP_MS + projectedMs(withFile, msOf, workers) <= budgetMs) {+ run.push(entry);+ files.push(entry.file);+ } else {+ notRun.push({+ file: entry.file,+ reason: `${entry.reason}; its expected ${Math.round(secondsOf(entry.file))} s does not fit the default check's ${budgetMs / 60000}-minute budget; add ci:run-a11y to run it`,+ });+ }+ }+ return {run, notRun};+}++/**+ * The default check's job summary. A file that did not run is listed as not+ * run, so it never reads as passed.+ * @param {{components: string[], selected: {file: string, reason: string}[], notRun: {file: string, reason: string}[]}} plan+ */+export function summaryMarkdown({components, selected, notRun}) {+ const lines = ['### Component accessibility contracts', ''];+ lines.push(+ components.length+ ? `Touched components: ${components.map(c => `\`${c}\``).join(', ')}.`+ : 'No touched component.',+ );+ lines.push('');+ if (selected.length === 0 && notRun.length === 0) {+ lines.push('No accessibility contract to run.');+ }+ if (selected.length) {+ lines.push(`Ran ${selected.length} spec file(s):`, '');+ for (const {file, reason} of selected) {+ lines.push(`- \`${file}\` (${reason})`);+ }+ lines.push('');+ }+ if (notRun.length) {+ lines.push(+ `Not run: ${notRun.length} spec file(s). They did not pass or fail here; the release check runs them.`,+ '',+ );+ for (const {file, reason} of notRun) {+ lines.push(`- \`${file}\`: ${reason}`);+ }+ lines.push('');+ }+ return `${lines.join('\n')}\n`;+}+/*** The changed-only selection: of the specs the routing selects, only those the* pull request itself adds or modifies, each because it changed.@@ -463,6 +639,30 @@ function readAuditConfig(root, file, policySha) {return JSON.parse(source);}+function readDurations(root) {+ const file = path.join(root, '.github/a11y-contract-durations.json');+ if (!fs.existsSync(file)) return {};+ return fileSeconds(JSON.parse(fs.readFileSync(file, 'utf8')));+}++/** The Playwright worker count, as playwright.config.ts reads it. */+function contractWorkers() {+ const workers = Number(process.env.A11Y_CONTRACT_WORKERS || 1);+ return Number.isInteger(workers) && workers > 0 ? workers : 1;+}++/** A file's non-empty lines, or null for no file name. */+function readLines(file) {+ if (!file) return null;+ return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean);+}++/** The value after a flag, or null when the flag is absent. */+function flagValue(argv, flag) {+ const index = argv.indexOf(flag);+ return index === -1 ? null : (argv[index + 1] ?? '');+}+function readComponents(root) {const analysis = path.join(root, 'analysis.json');if (!fs.existsSync(analysis)) {@@ -504,18 +704,34 @@ function listSpecs(root) {function main() {const root = process.cwd();+ const argv = process.argv.slice(2);const isPullRequest = process.env.GITHUB_EVENT_NAME? process.env.GITHUB_EVENT_NAME === 'pull_request': true;+ const changedOnlyFile = flagValue(argv, '--changed-only');+ const changedFilesFile = flagValue(argv, '--changed-files');+ const ownedByValue = flagValue(argv, '--owned-by');+ const ownedBy = (ownedByValue ?? '').split(',').filter(Boolean);+ if (+ (ownedByValue != null || changedFilesFile != null) &&+ changedOnlyFile == null+ ) {+ throw new Error('--owned-by and --changed-files need --changed-only');+ }+ // Without every changed path the folder rule selects nothing, and the+ // dropped contracts would read as passed.+ if (ownedBy.length > 0 && !changedFilesFile) {+ throw new Error(+ '--owned-by needs --changed-files with the list of changed paths',+ );+ }+ const flagValues = new Set([changedOnlyFile, changedFilesFile, ownedByValue]);// Outside CI, a run with no base compares with origin/main.const base =process.env.BASE_SHA ||- process.argv.slice(2).find(arg => !arg.startsWith('-')) ||+ argv.find(arg => !arg.startsWith('-') && !flagValues.has(arg)) ||(process.env.GITHUB_ACTIONS ? undefined : 'origin/main');const policySha = process.env.WORKFLOW_SHA;- const changedOnlyIndex = process.argv.indexOf('--changed-only');- const changedOnlyFile =- changedOnlyIndex === -1 ? null : process.argv[changedOnlyIndex + 1];const specs = listSpecs(root).map(file => ({file,routing: parseRouting(readPolicySource(root, file, policySha)),@@ -542,7 +758,16 @@ function main() {});const budgeted =changedOnlyFile != null- ? withinBudget(changedOnly(routed, changedPaths))+ ? planDefaultCheck({+ changed: changedOnly(routed, changedPaths),+ owned: ownedSpecs({+ specs,+ components: ownedBy,+ changedPaths: readLines(changedFilesFile) ?? changedPaths,+ }),+ seconds: readDurations(root),+ workers: contractWorkers(),+ }): {run: routed, notRun: []};const selected = budgeted.run;const notRun = budgeted.notRun;@@ -568,6 +793,7 @@ function main() {discovered: specs.length,scopeKnown: changedPaths != null,components,+ ownedBy,audit: plan && {component: audited, ...plan},selected,notRun,@@ -583,6 +809,12 @@ function main() {for (const {file, reason} of notRun) {console.log(`::notice title=Not run::${file}: ${reason}`);}+ if (changedOnlyFile != null && process.env.GITHUB_STEP_SUMMARY) {+ fs.appendFileSync(+ process.env.GITHUB_STEP_SUMMARY,+ summaryMarkdown({components: ownedBy, selected, notRun}),+ );+ }if (process.argv.includes('--dry-run')) return 0;let status = 0;@@ -614,10 +846,10 @@ function main() {process.env.GITHUB_ACTIONS ? 'list,json,github' : 'list,json','--output','pr-a11y-output',- // The changed-only run stops at its budget: an interrupted spec fails+ // The default check stops at its budget: an interrupted spec fails// the run, so a spec that did not finish is never read as passed....(changedOnlyFile != null- ? ['--global-timeout', String(CHANGED_SPEC_BUDGET_MS)]+ ? ['--global-timeout', String(DEFAULT_CHECK_BUDGET_MS)]: []),'--project','chromium',diff --git a/.github/scripts/a11y-pr-contracts.test.mjs b/.github/scripts/a11y-pr-contracts.test.mjsindex 6aab4af072..22ffa5e0ee 100644--- a/.github/scripts/a11y-pr-contracts.test.mjs+++ b/.github/scripts/a11y-pr-contracts.test.mjs@@ -23,11 +23,18 @@ import {auditArgs,auditedComponents,auditPlan,- CHANGED_SPEC_BUDGET_MS,CHANGED_SPEC_LIMIT,changedOnly,changedSinceMergeBase,+ DEFAULT_CHECK_BUDGET_MS,+ fileSeconds,kebab,+ ownedSpecs,+ planDefaultCheck,+ projectedMs,+ RECORDED_TIME_MARGIN,+ STARTUP_MS,+ summaryMarkdown,planFor,parseRouting,IGNORED_RECORD_SEGMENTS,@@ -62,8 +69,8 @@ const pr = (specs, overrides = {}) =>const BINDING ='packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts';-// spec:AST-030 FR12: the default touched-components check runs the browser-// specs a pull request adds or modifies, and only those.+// The default check first runs the browser specs a pull request adds or+// modifies.describe('changedOnly (the default check runs changed specs)', () => {const NEW_SPEC ='packages/core/src/Radio/__tests__/RadioKeyboard.a11y.chromium.spec.ts';@@ -101,20 +108,20 @@ describe('changedOnly (the default check runs changed specs)', () => {expect(run([ALWAYS])).toEqual([{file: ALWAYS, reason: 'spec changed'}]);});- it('leaves a changed release-only spec to the release check', () => {- expect(run([RELEASE])).toEqual([]);+ it('runs a changed release-only spec on its own pull request', () => {+ expect(run([RELEASE])).toEqual([{file: RELEASE, reason: 'spec changed'}]);});- it('stops the changed-only run at eight minutes, inside the job budget', () => {+ it('stops the default check at eight minutes, inside the job budget', () => {expect(CHANGED_SPEC_LIMIT).toBe(4);- expect(CHANGED_SPEC_BUDGET_MS).toBe(8 * 60 * 1000);+ expect(DEFAULT_CHECK_BUDGET_MS).toBe(8 * 60 * 1000);const runner = fs.readFileSync(path.join(root, '.github/scripts/a11y-pr-contracts.mjs'),'utf8',);- // Only the changed-only run carries the global timeout.+ // Only the default check carries the global timeout.expect(runner).toMatch(- /changedOnlyFile != null\s*\?\s*\['--global-timeout', String\(CHANGED_SPEC_BUDGET_MS\)\]\s*:\s*\[\]/,+ /changedOnlyFile != null\s*\?\s*\['--global-timeout', String\(DEFAULT_CHECK_BUDGET_MS\)\]\s*:\s*\[\]/,);});@@ -142,6 +149,333 @@ describe('changedOnly (the default check runs changed specs)', () => {});});+// The default check then runs each touched component's own scoped and+// release-only specs, inside its budget.+describe('the default check of the touched components', () => {+ const SWITCH =+ 'packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts';+ const LISTBOX =+ 'packages/core/src/Selector/__tests__/Listbox.a11y.chromium.spec.ts';+ const COMBOBOX_FIELD =+ 'packages/core/src/Selector/__tests__/ComboboxField.a11y.chromium.spec.ts';+ const EMPTY_STATE =+ 'packages/core/src/Selector/__tests__/EmptyStateAnnouncement.a11y.chromium.spec.ts';+ const SELECTOR_AUDIT =+ 'packages/core/src/Selector/__tests__/SelectorAudit.a11y.chromium.spec.ts';+ const specs = [+ spec(SWITCH, ['// @pr-a11y release-only']),+ spec(LISTBOX, ['// @pr-a11y release-only']),+ spec(COMBOBOX_FIELD),+ spec(EMPTY_STATE, ['// @pr-a11y always']),+ spec(SELECTOR_AUDIT, ['// @pr-a11y audit Selector']),+ ];+ const seconds = {+ [SWITCH]: 168.4,+ [LISTBOX]: 325.2,+ [COMBOBOX_FIELD]: 38.2,+ [EMPTY_STATE]: 4,+ 'packages/core/src/TextInput/__tests__/TextInput.a11y.chromium.spec.ts': 478.7,+ };+ const plan = (owned, changed = [], workers = 4) =>+ planDefaultCheck({+ changed: changed.map(file => ({file, reason: 'spec changed'})),+ owned: ownedSpecs({specs, components: owned, changedPaths: []}),+ seconds,+ workers,+ });+ const files = list => list.map(({file}) => file);++ const owns = (components, changedPaths = [], list = specs) =>+ ownedSpecs({specs: list, components, changedPaths});++ it("selects a touched component's own scoped and release-only specs, and nothing else", () => {+ expect(owns(['core/Switch'])).toEqual([+ {file: SWITCH, reason: 'component core/Switch'},+ ]);+ // Not the always spec or the audit spec in the same folder.+ expect(files(owns(['core/Selector']))).toEqual([LISTBOX, COMBOBOX_FIELD]);+ // An untouched component, such as a reverse dependent, owns none.+ expect(owns(['core/Toolbar'])).toEqual([]);+ });++ it('owns a scoped spec whose component scope token names a touched component', () => {+ const pattern = spec(+ 'internal/a11y-spec/src/patterns/switch.chromium.spec.ts',+ ['// @pr-a11y scope core/Switch'],+ );+ expect(owns(['core/Switch'], [], [pattern])).toEqual([+ {file: pattern.file, reason: 'component core/Switch'},+ ]);+ expect(owns(['core/Slider'], [], [pattern])).toEqual([]);+ });++ it('owns nothing without a touched component, so a wide change runs only its changed specs', () => {+ expect(owns([], ['packages/core/src/Switch/Switch.tsx'])).toEqual([]);+ });++ it('owns a binding by its folder component, its own name, or a change in its folder', () => {+ const named = spec(+ 'packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts',+ ['// @pr-a11y release-only'],+ );+ expect(files(owns(['core/ChatSendButton'], [], [named]))).toEqual([+ named.file,+ ]);+ expect(files(owns(['core/Chat'], [], [named]))).toEqual([named.file]);+ expect(owns(['core/ChatComposer'], [], [named])).toEqual([]);+ expect(+ owns(['core/ChatComposer'], ['packages/core/src/Chat/Chat.tsx'], [named]),+ ).toEqual([+ {file: named.file, reason: 'changed packages/core/src/Chat/Chat.tsx'},+ ]);+ // A changed knowledge record is not a change in the folder.+ expect(+ owns(+ ['core/ChatComposer'],+ ['docs/specs/components/Chat/Chat.spec.md'],+ [named],+ ),+ ).toEqual([]);+ });++ it("owns a scoped spec whose path token names a touched component's folder", () => {+ const slider = spec(+ 'internal/a11y-spec/src/patterns/slider.chromium.spec.ts',+ [+ '// @pr-a11y scope ^(internal/a11y-spec/src/patterns/slider|internal/a11y-spec/src/harness|packages/core/src/Slider/)',+ ],+ );+ expect(+ owns(['core/Slider'], ['packages/core/src/Slider/Slider.tsx'], [slider]),+ ).toEqual([+ {+ file: slider.file,+ reason: 'changed packages/core/src/Slider/Slider.tsx',+ },+ ]);+ // A change to the harness the same token names is shared contract code.+ expect(+ owns(+ ['core/Slider'],+ ['internal/a11y-spec/src/harness/chromium.ts'],+ [slider],+ ),+ ).toEqual([]);+ });++ it('never owns a spec through a path token or a shared pattern change', () => {+ const shared = spec(+ 'packages/core/src/Selector/__tests__/SelectorReducedMotion.a11y.chromium.spec.ts',+ ['// @pr-a11y scope ^packages/core/src/motion/'],+ );+ expect(+ owns(+ ['core/Switch'],+ ['packages/core/src/motion/reducedMotion.ts'],+ [shared],+ ),+ ).toEqual([]);+ const pattern = spec(+ 'internal/a11y-spec/src/patterns/menu.chromium.spec.ts',+ );+ expect(+ owns(+ ['core/Switch'],+ ['internal/a11y-spec/src/patterns/menu.ts'],+ [pattern],+ ),+ ).toEqual([]);+ // A release-only spec is never owned through a component token either.+ const tokened = spec(+ 'internal/a11y-spec/src/patterns/switch.chromium.spec.ts',+ ['// @pr-a11y release-only', '// @pr-a11y scope core/Switch'],+ );+ expect(owns(['core/Switch'], [], [tokened])).toEqual([]);+ });++ it('runs every owned file that fits the budget at its recorded time', () => {+ expect(plan(['core/Switch'])).toEqual({+ run: [{file: SWITCH, reason: 'component core/Switch'}],+ notRun: [],+ });+ expect(files(plan(['core/Selector']).run).sort()).toEqual(+ [COMBOBOX_FIELD, LISTBOX].sort(),+ );+ });++ it('reports a file that does not fit as not run, with the label that runs it', () => {+ // On one worker, in path order: ComboboxField fits, Listbox after it does+ // not, Switch after ComboboxField does.+ const {run, notRun} = plan(['core/Selector', 'core/Switch'], [], 1);+ expect(files(run)).toEqual([COMBOBOX_FIELD, SWITCH]);+ expect(files(notRun)).toEqual([LISTBOX]);+ expect(notRun[0].reason).toContain('component core/Selector');+ expect(notRun[0].reason).toContain('ci:run-a11y');+ expect(notRun[0].reason).toContain('8-minute budget');+ // On four workers they run side by side.+ expect(plan(['core/Selector', 'core/Switch']).notRun).toEqual([]);+ });++ it('never starts a file that the recorded time says the stop would cut', () => {+ const textInput = spec(+ 'packages/core/src/TextInput/__tests__/TextInput.a11y.chromium.spec.ts',+ ['// @pr-a11y release-only'],+ );+ const {run, notRun} = planDefaultCheck({+ changed: [],+ owned: owns(['core/TextInput'], [], [textInput]),+ seconds,+ workers: 4,+ });+ expect(run).toEqual([]);+ expect(notRun[0].reason).toContain('expected 479 s');+ expect(STARTUP_MS + 478.7 * 1000 * RECORDED_TIME_MARGIN).toBeGreaterThan(+ DEFAULT_CHECK_BUDGET_MS,+ );+ });++ it('runs changed specs first and counts them against the budget, without running a file twice', () => {+ expect(plan(['core/Selector'], [LISTBOX])).toEqual({+ run: [+ {file: LISTBOX, reason: 'spec changed'},+ {file: COMBOBOX_FIELD, reason: 'component core/Selector'},+ ],+ notRun: [],+ });+ // On one worker, ComboboxField after the changed Listbox passes the+ // budget: 30 + (325.2 + 38.2) * 1.25 s.+ const {run, notRun} = plan(['core/Selector'], [LISTBOX], 1);+ expect(run).toEqual([{file: LISTBOX, reason: 'spec changed'}]);+ expect(files(notRun)).toEqual([COMBOBOX_FIELD]);+ });++ it('reports a changed spec that does not fit as not run, never starts it', () => {+ const tokenizer =+ 'packages/core/src/Tokenizer/__tests__/TokenizerEndLane.a11y.chromium.spec.ts';+ const {run, notRun} = planDefaultCheck({+ changed: [+ {file: SWITCH, reason: 'spec changed'},+ {file: tokenizer, reason: 'spec changed'},+ ],+ owned: [],+ seconds: {...seconds, [tokenizer]: 551.8},+ workers: 4,+ });+ expect(run).toEqual([{file: SWITCH, reason: 'spec changed'}]);+ expect(notRun).toEqual([+ {+ file: tokenizer,+ reason: expect.stringMatching(+ /^spec changed; its expected 552 s does not fit .*ci:run-a11y/,+ ),+ },+ ]);+ });++ it('expects the median recorded time for a file that has none', () => {+ const fresh = spec(+ 'packages/core/src/Switch/__tests__/SwitchFresh.a11y.chromium.spec.ts',+ ['// @pr-a11y release-only'],+ );+ const {notRun} = planDefaultCheck({+ changed: [],+ owned: owns(['core/Switch'], [], [fresh]),+ seconds,+ workers: 4,+ budgetMs: STARTUP_MS,+ });+ expect(notRun[0].reason).toContain('expected 168 s');+ });++ it('projects whole files in path order onto the first free worker', () => {+ const ms = {a: 4, b: 1, c: 1, d: 3};+ expect(projectedMs(['d', 'c', 'b', 'a'], file => ms[file], 2)).toBe(5);+ expect(projectedMs(['a', 'b', 'c', 'd'], file => ms[file], 1)).toBe(9);+ expect(projectedMs([], file => ms[file], 4)).toBe(0);+ });++ it('reads recorded seconds per file, not per test', () => {+ expect(+ fileSeconds({+ seconds: {+ [SWITCH]: 168.4,+ [`${SWITCH} \u203A switch \u203A presses`]: 12,+ },+ }),+ ).toEqual({[SWITCH]: 168.4});+ expect(fileSeconds(undefined)).toEqual({});+ });++ it('summarizes what ran and what did not, never as passed', () => {+ const summary = summaryMarkdown({+ components: ['core/Switch'],+ selected: [{file: SWITCH, reason: 'component core/Switch'}],+ notRun: [{file: LISTBOX, reason: 'does not fit'}],+ });+ expect(summary).toContain('`core/Switch`');+ expect(summary).toContain(`- \`${SWITCH}\` (component core/Switch)`);+ expect(summary).toContain('Not run: 1 spec file(s)');+ expect(summary).toContain('did not pass or fail here');+ expect(+ summaryMarkdown({components: [], selected: [], notRun: []}),+ ).toContain('No accessibility contract to run.');+ });++ it('refuses owners without the changed-file list, so no owned contract is dropped silently', () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'owned-'));+ const specsFile = path.join(dir, 'specs.txt');+ fs.writeFileSync(specsFile, '');+ const run = args => {+ try {+ execFileSync(+ process.execPath,+ [path.join(root, '.github/scripts/a11y-pr-contracts.mjs'), ...args],+ {cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']},+ );+ return {status: 0, output: ''};+ } catch (error) {+ return {status: error.status, output: String(error.stderr)};+ }+ };+ try {+ for (const args of [+ ['--changed-only', specsFile, '--owned-by', 'core/ChatLayout'],+ [+ '--changed-only',+ specsFile,+ '--changed-files',+ '',+ '--owned-by',+ 'core/ChatLayout',+ ],+ ]) {+ const result = run(args);+ expect(result.status).not.toBe(0);+ expect(result.output).toContain(+ '--owned-by needs --changed-files with the list of changed paths',+ );+ }+ } finally {+ fs.rmSync(dir, {recursive: true, force: true});+ }+ });++ it('is wired into the runner: owners, durations and workers reach the plan', () => {+ const runner = fs.readFileSync(+ path.join(root, '.github/scripts/a11y-pr-contracts.mjs'),+ 'utf8',+ );+ expect(runner).toContain("flagValue(argv, '--owned-by')");+ // The folder-change rule reads every changed path, not the spec list.+ expect(runner).toContain("flagValue(argv, '--changed-files')");+ expect(runner).toContain(+ 'changedPaths: readLines(changedFilesFile) ?? changedPaths',+ );+ expect(runner).toContain('seconds: readDurations(root)');+ expect(runner).toContain('workers: contractWorkers()');+ });+});+describe('a11y-spec-files (one spec list for Playwright and the touched check)', () => {it('is the list playwright.config.ts runs', () => {const config = fs.readFileSync(@@ -301,10 +635,10 @@ describe('selectSpecs', () => {).toEqual([always.file]);});- it('runs a scoped spec when its folder changes', () => {+ it("runs the folder's scoped and release-only specs when its folder changes", () => {expect(pr(all, {changedPaths: ['packages/core/src/Chat/ChatLayout.tsx']}),- ).toEqual([always.file, scoped.file]);+ ).toEqual([always.file, scoped.file, releaseOnly.file]);});it('does not run a scoped spec for a sibling folder sharing its prefix', () => {@@ -357,7 +691,7 @@ describe('selectSpecs', () => {pr(all, {changedPaths: [record, 'packages/core/src/Chat/ChatLayout.tsx'],}),- ).toEqual([always.file, scoped.file]);+ ).toEqual([always.file, scoped.file, releaseOnly.file]);});it('still runs a spec for a fixture, test, or generated .spec.md', () => {@@ -369,6 +703,7 @@ describe('selectSpecs', () => {expect(pr(all, {changedPaths: [file]}), file).toEqual([always.file,scoped.file,+ releaseOnly.file,]);}});@@ -403,13 +738,31 @@ describe('selectSpecs', () => {).toEqual([audit.file]);});- it('never runs a release-only spec on a pull request', () => {+ it('runs a release-only spec on a pull request only for its own component', () => {expect(pr([releaseOnly], {changedPaths: null,components: ['core/ChatLayout'],}),).toEqual([]);+ expect(pr([releaseOnly], {components: ['core/ChatLayout']})).toEqual([+ releaseOnly.file,+ ]);+ expect(+ pr([releaseOnly], {changedPaths: ['packages/core/src/Chat/Chat.tsx']}),+ ).toEqual([releaseOnly.file]);+ expect(pr([releaseOnly], {changedPaths: [releaseOnly.file]})).toEqual([+ releaseOnly.file,+ ]);+ expect(pr([releaseOnly], {components: ['core/Selector']})).toEqual([]);+ // A scope token never selects it.+ const tokened = spec(+ 'packages/core/src/Chat/__tests__/ChatLayout.a11y.chromium.spec.ts',+ ['// @pr-a11y release-only', '// @pr-a11y scope core/Selector ^apps/x'],+ );+ expect(+ pr([tokened], {components: ['core/Selector'], changedPaths: ['apps/x']}),+ ).toEqual([]);});it('names the reason for every selection', () => {@@ -648,6 +1001,47 @@ describe('repository specs', () => {).not.toThrow();}});++ it("owns a component folder's contracts for a change in that folder", () => {+ // A folder of several components: an edit to one of them names only that+ // owner, so the folder's hook and surface specs are owned by the change.+ const specs = files.map(file => ({+ file,+ routing: parseRouting(fs.readFileSync(path.join(root, file), 'utf8')),+ }));+ const owned = ownedSpecs({+ specs,+ components: ['core/ChatLayout'],+ changedPaths: ['packages/core/src/Chat/ChatLayout.tsx'],+ }).map(({file}) => file);+ for (const file of [+ 'packages/core/src/Chat/__tests__/ChatLayoutScrollButtonSurface.a11y.chromium.spec.ts',+ 'packages/core/src/Chat/__tests__/useChatStreamScroll.a11y.chromium.spec.ts',+ ]) {+ expect(owned, file).toContain(file);+ }+ expect(+ ownedSpecs({+ specs,+ components: ['core/ChatLayout'],+ changedPaths: [],+ }).map(({file}) => file),+ ).not.toContain(+ 'packages/core/src/Chat/__tests__/useChatStreamScroll.a11y.chromium.spec.ts',+ );+ // Audit bindings in the folder still wait for their audit pull request.+ expect(owned).not.toContain(+ 'packages/core/src/Chat/__tests__/ChatMessageList.a11y.chromium.spec.ts',+ );+ // A pattern contract whose path token names the touched folder.+ expect(+ ownedSpecs({+ specs,+ components: ['core/Slider'],+ changedPaths: ['packages/core/src/Slider/Slider.tsx'],+ }).map(({file}) => file),+ ).toContain('internal/a11y-spec/src/patterns/slider.chromium.spec.ts');+ });});describe('isKnowledgeRecord', () => {diff --git a/.github/scripts/ci-admission.test.mjs b/.github/scripts/ci-admission.test.mjsindex c47f7babd8..aa380911a2 100644--- a/.github/scripts/ci-admission.test.mjs+++ b/.github/scripts/ci-admission.test.mjs@@ -182,6 +182,7 @@ describe('where runs wait', () => {guarded.push(id);}expect(guarded.sort()).toEqual([+ 'pr-component-a11y-contracts','pr-interaction','pr-interaction-webkit','pr-touched-components',diff --git a/.github/scripts/ci-test-join.mjs b/.github/scripts/ci-test-join.mjsindex 2644c0a4ad..af1d26bbeb 100644--- a/.github/scripts/ci-test-join.mjs+++ b/.github/scripts/ci-test-join.mjs@@ -20,6 +20,10 @@ const requiredSuccess = [['test-native-swift', process.env.TEST_NATIVE_SWIFT_RESULT],['pr-interaction', process.env.PR_INTERACTION_RESULT],['pr-interaction-webkit', process.env.PR_INTERACTION_WEBKIT_RESULT],+ [+ 'pr-component-a11y-contracts',+ process.env.PR_COMPONENT_A11Y_CONTRACTS_RESULT,+ ],];function fail(message) {diff --git a/.github/scripts/ci-test-join.test.mjs b/.github/scripts/ci-test-join.test.mjsindex 220fb5bec2..d3cf62eef5 100644--- a/.github/scripts/ci-test-join.test.mjs+++ b/.github/scripts/ci-test-join.test.mjs@@ -21,6 +21,7 @@ const broad = {TEST_NATIVE_SWIFT_RESULT: 'success',PR_INTERACTION_RESULT: 'success',PR_INTERACTION_WEBKIT_RESULT: 'success',+ PR_COMPONENT_A11Y_CONTRACTS_RESULT: 'success',};// An override of `null` removes the variable, as a missing `needs` result would.@@ -61,6 +62,11 @@ describe('pull-request test join', () => {['PR_INTERACTION_WEBKIT_RESULT', 'skipped'],['PR_INTERACTION_WEBKIT_RESULT', 'cancelled'],['PR_INTERACTION_WEBKIT_RESULT', null],+ // A failing component accessibility contract blocks the merge.+ ['PR_COMPONENT_A11Y_CONTRACTS_RESULT', 'failure'],+ ['PR_COMPONENT_A11Y_CONTRACTS_RESULT', 'skipped'],+ ['PR_COMPONENT_A11Y_CONTRACTS_RESULT', 'cancelled'],+ ['PR_COMPONENT_A11Y_CONTRACTS_RESULT', null],]) {expect(run({[name]: value}), name).toMatchObject({status: 1});}diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjsindex 5f836b84f1..889583add1 100644--- a/.github/scripts/ci-test-routing.test.mjs+++ b/.github/scripts/ci-test-routing.test.mjs@@ -467,6 +467,7 @@ describe('ci.yml RTL package sharding', () => {['pr-a11y-guards','pr-a11y-specs',+ 'pr-component-a11y-contracts','pr-rtl','pr-rtl-merge','pr-rtl-shard',diff --git a/.github/scripts/ci-tooling-routing.test.mjs b/.github/scripts/ci-tooling-routing.test.mjsindex 2fc3450f12..64ef335dee 100644--- a/.github/scripts/ci-tooling-routing.test.mjs+++ b/.github/scripts/ci-tooling-routing.test.mjs@@ -455,6 +455,7 @@ describe('Node-tooling CI routing', () => {'test-native-swift','pr-interaction','pr-interaction-webkit',+ 'pr-component-a11y-contracts',]);expect(ci.jobs.build.needs).toEqual(['room',@@ -486,6 +487,8 @@ describe('Node-tooling CI routing', () => {TEST_NATIVE_SWIFT_RESULT: '${{ needs.test-native-swift.result }}',PR_INTERACTION_RESULT: '${{ needs.pr-interaction.result }}',PR_INTERACTION_WEBKIT_RESULT: '${{ needs.pr-interaction-webkit.result }}',+ PR_COMPONENT_A11Y_CONTRACTS_RESULT:+ '${{ needs.pr-component-a11y-contracts.result }}',});const buildJoin = step(ci.jobs.build,diff --git a/.github/scripts/interaction-legs.test.mjs b/.github/scripts/interaction-legs.test.mjsindex f7d9a4e433..8ca6a9e2b9 100644--- a/.github/scripts/interaction-legs.test.mjs+++ b/.github/scripts/interaction-legs.test.mjs@@ -244,20 +244,32 @@ describe('the interaction proof legs in ci.yml', () => {expect(join.env.PR_INTERACTION_WEBKIT_RESULT).toBe('${{ needs.pr-interaction-webkit.result }}',);+ // Build the all-passing environment from the join step itself, so a lane+ // added to the join cannot leave this fixture behind it.+ const laneResults = Object.entries(join.env).flatMap(([name, value]) => {+ const lane = /^\$\{\{ needs\.([\w-]+)\.result \}\}$/.exec(value)?.[1];+ return lane && lane !== 'release-bump' ? [[name, lane]] : [];+ });+ expect(laneResults.map(([, lane]) => lane)).toEqual(+ expect.arrayContaining([+ 'pr-interaction',+ 'pr-interaction-webkit',+ 'pr-component-a11y-contracts',+ ]),+ );+ for (const [name, lane] of laneResults) {+ expect(test.needs, name).toContain(lane);+ }+ const scopes = Object.keys(join.env).filter(name =>+ /^\$\{\{ steps\.scope\.outputs\.\w+_only \}\}$/.test(join.env[name]),+ );const passing = {- TEST_UI_RESULT: 'success',- TEST_NODE_RESULT: 'success',- TEST_BUILD_RESULT: 'success',- TEST_NATIVE_SWIFT_RESULT: 'success',- PR_INTERACTION_RESULT: 'success',- PR_INTERACTION_WEBKIT_RESULT: 'success',+ ...Object.fromEntries(laneResults.map(([name]) => [name, 'success'])),+ ...Object.fromEntries(scopes.map(name => [name, 'false'])),RELEASE_BUMP: 'false',RELEASE_BUMP_RESULT: 'skipped',- DOCSITE_ONLY: 'false',- SPEC_ONLY: 'false',- TOOLING_ONLY: 'false',- LAB_ONLY: 'false',};+ expect(Object.keys(passing).sort()).toEqual(Object.keys(join.env).sort());const joinExit = env =>spawnSync(process.execPath,@@ -266,11 +278,10 @@ describe('the interaction proof legs in ci.yml', () => {).status;expect(joinExit(passing)).toBe(0);// GitHub reports a matrix job's result as success only when every leg- // succeeded; any other aggregate fails the join.- for (const lane of [- 'PR_INTERACTION_RESULT',- 'PR_INTERACTION_WEBKIT_RESULT',- ]) {+ // succeeded; any other aggregate fails the join. A lane that has no work+ // for the pull request still runs and reports success, so a skipped lane+ // fails the join too.+ for (const [lane] of laneResults) {for (const result of ['failure', 'cancelled', 'skipped', '']) {expect(joinExit({...passing, [lane]: result}),diff --git a/.github/scripts/touched-components.test.mjs b/.github/scripts/touched-components.test.mjsindex 31a2cd1e96..e77ac06eb8 100644--- a/.github/scripts/touched-components.test.mjs+++ b/.github/scripts/touched-components.test.mjs@@ -581,40 +581,20 @@ describe('pr-touched-components routing', () => {expect(checkout.with?.ref).toBeUndefined();});- // spec:AST-030 FR12: a browser accessibility spec the change adds or- // modifies is caught on its own pull request, not first on main.- it('runs the changed browser accessibility specs, and only on a pull request that changes one', () => {- const find = step('Find the changed browser accessibility specs');- expect(find.env.FILES_PATH).toBe(- '${{ steps.scope.outputs.changed_files_path }}',- );- // The same spec list playwright.config.ts runs, not a copy of its globs.- expect(find.run).toContain(- 'import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";',- );- expect(find.run).not.toContain('spec\\.ts');- const runSpecs = step('Run the changed browser accessibility specs');- expect(runSpecs.if).toBe(- "always() && steps.playwright.outcome == 'success' && steps.specs.outputs.has_specs == 'true'",- );- expect(runSpecs.run).toBe(- 'node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH"',- );- // A failing spec turns the job red: no continue-on-error.- expect(runSpecs['continue-on-error']).toBeUndefined();- // Bounded inside the job's 15 minutes, past the runner's own 8-minute stop.- expect(runSpecs['timeout-minutes']).toBe(9);- // Setup, Storybook and Playwright run for a spec-only change too.+ it('runs no accessibility contract: the contracts job owns them', () => {+ const names = job.steps.map(({name}) => name);+ expect(names).not.toContain('Find the changed browser accessibility specs');+ expect(JSON.stringify(job)).not.toContain('a11y-pr-contracts.mjs');for (const name of ['Setup Node and pnpm','Download Storybook artifact','Install Playwright',]) {expect(step(name).if).toBe(- "steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'",+ "steps.touched.outputs.has_components == 'true'",);}- expect(step('Upload touched-components results').with.path).toContain(+ expect(step('Upload touched-components results').with.path).not.toContain('pr-a11y-evidence/',);// Still advisory: no required context depends on this job.@@ -663,8 +643,7 @@ describe('pr-touched-components routing', () => {expect(step('Skip when no component changed or the change is wide').if,).toBe("steps.touched.outputs.has_components != 'true'");- // Every later step runs for touched components, or (setup, Storybook,- // Playwright, the spec run and the upload) for a changed browser spec.+ // Every later step runs for touched components only.const skipIndex = job.steps.findIndex(candidate =>candidate.name ===@@ -672,7 +651,7 @@ describe('pr-touched-components routing', () => {);for (const candidate of job.steps.slice(skipIndex + 1)) {expect(candidate.if, candidate.name).toMatch(- /steps\.touched\.outputs\.has_components == 'true'|steps\.specs\.outputs\.has_specs == 'true'/,+ /steps\.touched\.outputs\.has_components == 'true'/,);}});@@ -684,3 +663,126 @@ describe('pr-touched-components routing', () => {expect(upload.if).toContain("steps.touched.outputs.wide_change == 'true'");});});++describe('pr-component-a11y-contracts routing', () => {+ const contracts = ci.jobs['pr-component-a11y-contracts'];+ const contractStep = name =>+ contracts.steps.find(candidate => candidate.name === name);+ const runStep = contractStep(+ "Run the touched components' accessibility contracts",+ );+ const ran =+ "steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'";++ it('starts after the Storybook build, beside the test owners, never after pr-visual or the touched check', () => {+ expect(contracts.needs).toEqual([+ 'room',+ 'admission',+ 'heavy-ci',+ 'build-storybook',+ ]);+ // The same gate as the interaction proofs, which the test join also reads.+ expect(contracts.if).toBe(ci.jobs['pr-interaction'].if);+ expect(contracts['runs-on']).toBe('ubuntu-latest');+ expect(contracts['timeout-minutes']).toBeLessThanOrEqual(15);+ expect(contracts.permissions).toEqual({+ contents: 'read',+ 'pull-requests': 'read',+ });+ });++ it('blocks the merge through the test join, with no new required context', () => {+ expect(ci.jobs.test.needs).toContain('pr-component-a11y-contracts');+ const join = ci.jobs.test.steps.find(+ candidate => candidate.name === 'Require expected test-owner outcomes',+ );+ expect(join.env.PR_COMPONENT_A11Y_CONTRACTS_RESULT).toBe(+ '${{ needs.pr-component-a11y-contracts.result }}',+ );+ expect(runStep['continue-on-error']).toBeUndefined();+ for (const name of Object.keys(ci.jobs)) {+ if (name === 'test') continue;+ expect([ci.jobs[name].needs].flat(), name).not.toContain(+ 'pr-component-a11y-contracts',+ );+ }+ });++ it('skips with success outside a pull request, on a specialized lane, and when nothing it owns changed', () => {+ const scope = contractStep("Decide this job's scope");+ expect(scope.with).toMatchObject({+ paths: '^(packages/[^/]+/src/|internal/a11y-spec/)',+ 'outside-pull-request': 'skip',+ 'component-audit': 'true',+ });+ expect(contractStep('Skip the contracts outside their scope').if).toBe(+ "steps.scope.outputs.applies != 'true'",+ );+ const skipIndex = contracts.steps.findIndex(+ candidate => candidate.name === 'Skip the contracts outside their scope',+ );+ for (const candidate of contracts.steps.slice(skipIndex + 1)) {+ expect(candidate.if, candidate.name).toMatch(+ /steps\.scope\.outputs\.applies == 'true'|steps\.touched\.outputs\.has_components == 'true'|steps\.contracts\.outcome/,+ );+ }+ });++ it('fails, never passes, when the scope step cannot list the changed files', () => {+ const guard = contractStep('Require the changed-file list');+ expect(guard.if).toBe(+ "steps.scope.outputs.applies == 'true' && steps.scope.outputs.changed_files_path == ''",+ );+ expect(guard.run).toContain('exit 1');+ expect(guard['continue-on-error']).toBeUndefined();+ const names = contracts.steps.map(({name}) => name);+ expect(names.indexOf('Require the changed-file list')).toBeLessThan(+ names.indexOf("Run the touched components' accessibility contracts"),+ );+ });++ it('decides the touched components as the touched check does, and finds changed specs from the Playwright list', () => {+ const decide = contractStep('Decide the touched components');+ expect(decide.run).toContain('touched-components.mjs scope');+ expect(decide.run).toContain('--files "$FILES_PATH"');+ const find = contractStep('Find the changed browser accessibility specs');+ expect(find.run).toContain(+ 'import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";',+ );+ const checkout = contractStep('Checkout');+ expect(checkout.with?.ref).toBeUndefined();+ });++ it('runs the changed specs and the touched owners, on four workers, inside the job budget', () => {+ expect(runStep.if).toBe(ran);+ expect(runStep.run).toBe(+ 'node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"',+ );+ // Every changed path, not only the changed specs: a change in a+ // component folder owns that folder's specs.+ expect(runStep.env.FILES_PATH).toBe(+ '${{ steps.scope.outputs.changed_files_path }}',+ );+ // A wide change has no owned specs.+ expect(runStep.env.OWNERS).toBe(+ "${{ steps.touched.outputs.has_components == 'true' && steps.touched.outputs.a11y_owners || '' }}",+ );+ expect(runStep.env.A11Y_CONTRACT_WORKERS).toBe(4);+ expect(runStep.env.WORKFLOW_SHA).toBe('${{ github.sha }}');+ // Past the runner's own eight-minute stop, inside the job's 15 minutes.+ expect(runStep['timeout-minutes']).toBeGreaterThan(8);+ expect(runStep['timeout-minutes']).toBeLessThan(+ contracts['timeout-minutes'],+ );+ for (const name of [+ 'Setup Node and pnpm',+ 'Download Storybook artifact',+ 'Install Playwright',+ ]) {+ expect(contractStep(name).if, name).toBe(ran);+ }+ expect(contractStep('Upload contract evidence').with.path).toContain(+ 'pr-a11y-evidence/',+ );+ });+});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 1b412d4791..bef495025f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1154,6 +1154,7 @@ jobs:test-native-swift,pr-interaction,pr-interaction-webkit,+ pr-component-a11y-contracts,]if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'workflow_dispatch' && always() && !cancelled()) }}runs-on: ubuntu-slim@@ -1235,6 +1236,7 @@ jobs:TEST_NATIVE_SWIFT_RESULT: ${{ needs.test-native-swift.result }}PR_INTERACTION_RESULT: ${{ needs.pr-interaction.result }}PR_INTERACTION_WEBKIT_RESULT: ${{ needs.pr-interaction-webkit.result }}+ PR_COMPONENT_A11Y_CONTRACTS_RESULT: ${{ needs.pr-component-a11y-contracts.result }}run: node .github/scripts/ci-test-join.mjs# Build the packages and Storybook, and analyze components (parallel with@@ -3664,34 +3666,6 @@ jobs:--head-sha "$HEAD_SHA" \--out touched-components.json- # A scoped or always browser accessibility spec the change adds or- # modifies runs here by itself (spec:AST-030 FR12), so it is checked on- # its own pull request; audit and release-only specs still wait. No such- # file, no work.- - name: Find the changed browser accessibility specs- id: specs- env:- FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}- run: |- list="$RUNNER_TEMP/changed-a11y-specs.txt"- : > "$list"- if [ -n "$FILES_PATH" ] && [ -f "$FILES_PATH" ]; then- # The same spec globs playwright.config.ts runs.- node --input-type=module -e '- import fs from "node:fs";- import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";- const paths = fs.readFileSync(process.argv[1], "utf8").split("\n");- for (const file of paths.filter(isA11ySpecFile)) console.log(file);- ' "$FILES_PATH" > "$list"- fi- echo "path=$list" >> "$GITHUB_OUTPUT"- if [ -s "$list" ]; then- echo "has_specs=true" >> "$GITHUB_OUTPUT"- echo "Changed browser accessibility specs:"; cat "$list"- else- echo "has_specs=false" >> "$GITHUB_OUTPUT"- fi-- name: Skip when no component changed or the change is wideif: steps.touched.outputs.has_components != 'true'env:@@ -3705,13 +3679,13 @@ jobs:fi- name: Setup Node and pnpm- if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ if: steps.touched.outputs.has_components == 'true'uses: ./.github/actions/setupwith:install: 'false'- name: Download Storybook artifact- if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ if: steps.touched.outputs.has_components == 'true'uses: actions/download-artifact@v8with:name: storybook-${{ needs.build-storybook.outputs.short_hash }}@@ -3719,7 +3693,7 @@ jobs:- name: Install Playwrightid: playwright- if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ if: steps.touched.outputs.has_components == 'true'run: pnpm install --frozen-lockfile && npx playwright install chromium# The same changed-component filters an opted-in pull request uses.@@ -3765,22 +3739,6 @@ jobs:test -s "rtl-reports/$PACKAGE.json"done- # Only the changed spec files, in Chromium, with their routing lines read- # from this merge commit; a changed release-only or audit spec still- # waits. A failing or unfinished spec turns this advisory job red, and a- # file over the budget is reported as not run.- - name: Run the changed browser accessibility specs- id: changed-specs- if: always() && steps.playwright.outcome == 'success' && steps.specs.outputs.has_specs == 'true'- # The runner runs at most four changed files and stops at eight- # minutes; this bounds a hung browser inside the job's 15 minutes.- timeout-minutes: 9- env:- ASTRYX_HEAD_SHA: ${{ github.event.pull_request.head.sha }}- WORKFLOW_SHA: ${{ github.sha }}- SPECS_PATH: ${{ steps.specs.outputs.path }}- run: node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH"-# The visual result comes from pr-visual, the one visual owner: its# default touched check, or its full check under ci:run-visual.- name: Download pr-visual's verdict@@ -3814,7 +3772,7 @@ jobs:--out touched-components.json- name: Upload touched-components results- if: always() && (steps.touched.outputs.has_components == 'true' || steps.touched.outputs.wide_change == 'true' || steps.specs.outputs.has_specs == 'true')+ if: always() && (steps.touched.outputs.has_components == 'true' || steps.touched.outputs.wide_change == 'true')uses: actions/upload-artifact@v7with:name: touched-components@@ -3822,7 +3780,170 @@ jobs:touched-components.jsona11y-report.jsonrtl-reports/+ if-no-files-found: ignore+ retention-days: 7++ # The accessibility contracts of the components a pull request touches: the+ # browser accessibility specs it changes, then each touched component's own+ # scoped and release-only specs, inside an eight-minute budget. The job needs+ # only the Storybook build, so it runs beside the test owners and ends before+ # the slowest of them. The `test` join reads its result: a failing contract+ # blocks the merge with no new required context. Out of scope (no component+ # or browser accessibility spec changed, or a specialized lane), every step+ # after the scope step skips and the job succeeds. A wide change runs only+ # the changed specs. A file that does not fit the budget is reported as not+ # run, and the job still succeeds.+ pr-component-a11y-contracts:+ name: Component accessibility contracts+ needs: [room, admission, heavy-ci, build-storybook]+ if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.heavy-ci.outputs.deferred != 'true' && (github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.room.outputs.full != 'true' && needs.admission.outputs.full != 'true')) }}+ # One core per Playwright worker, as the release contract legs run.+ runs-on: ubuntu-latest+ timeout-minutes: 15+ permissions:+ contents: read+ pull-requests: read+ steps:+ - name: Check out the scope action+ uses: actions/checkout@v7+ with:+ sparse-checkout: .github/actions/test-scope+ path: .test-scope+ persist-credentials: false++ - name: Decide this job's scope+ id: scope+ uses: ./.test-scope/.github/actions/test-scope+ with:+ paths: '^(packages/[^/]+/src/|internal/a11y-spec/)'+ outside-pull-request: skip+ component-audit: 'true'++ - name: Require this pull request's Storybook build+ if: steps.scope.outputs.applies == 'true' && needs.build-storybook.result != 'success'+ run: |+ echo "The contracts read this pull request's Storybook build, which did not succeed."+ exit 1++ - name: Skip the contracts outside their scope+ if: steps.scope.outputs.applies != 'true'+ run: echo "No component or browser accessibility spec changed; no contract to run."++ # The tested merge commit, as the touched-components job checks out.+ - name: Checkout+ if: steps.scope.outputs.applies == 'true'+ uses: actions/checkout@v7+ with:+ persist-credentials: false++ # The changed-spec list and the folder rule both read this list. Without+ # it, a contract would not run and the job would still pass.+ - name: Require the changed-file list+ if: steps.scope.outputs.applies == 'true' && steps.scope.outputs.changed_files_path == ''+ run: |+ echo "::error::The scope step could not list the changed files, so the contracts to run are unknown. Re-run the job."+ exit 1++ - name: Download analysis artifact+ if: steps.scope.outputs.applies == 'true'+ uses: actions/download-artifact@v8+ with:+ name: pr-analysis++ # The same touched components as the touched-components job. More than+ # five, or a change to shared styles, is a wide change: no owned spec+ # runs, and the comment points to the ci:run-<suite> labels.+ - name: Decide the touched components+ id: touched+ if: steps.scope.outputs.applies == 'true'+ env:+ FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}+ run: |+ GITHUB_STEP_SUMMARY=/dev/null node .github/scripts/touched-components.mjs scope \+ --analysis analysis.json \+ --files "$FILES_PATH" \+ --out "$RUNNER_TEMP/touched-components.json"++ - name: Find the changed browser accessibility specs+ id: specs+ if: steps.scope.outputs.applies == 'true'+ env:+ FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}+ run: |+ list="$RUNNER_TEMP/changed-a11y-specs.txt"+ : > "$list"+ if [ -n "$FILES_PATH" ] && [ -f "$FILES_PATH" ]; then+ # The same spec globs playwright.config.ts runs.+ node --input-type=module -e '+ import fs from "node:fs";+ import {isA11ySpecFile} from "./.github/scripts/lib/a11y-spec-files.mjs";+ const paths = fs.readFileSync(process.argv[1], "utf8").split("\n");+ for (const file of paths.filter(isA11ySpecFile)) console.log(file);+ ' "$FILES_PATH" > "$list"+ fi+ echo "path=$list" >> "$GITHUB_OUTPUT"+ if [ -s "$list" ]; then+ echo "has_specs=true" >> "$GITHUB_OUTPUT"+ echo "Changed browser accessibility specs:"; cat "$list"+ else+ echo "has_specs=false" >> "$GITHUB_OUTPUT"+ fi++ - name: Skip when no contract applies+ if: steps.scope.outputs.applies == 'true' && steps.touched.outputs.has_components != 'true' && steps.specs.outputs.has_specs != 'true'+ env:+ WIDE: ${{ steps.touched.outputs.wide_change }}+ COUNT: ${{ steps.touched.outputs.component_count }}+ run: |+ if [ "$WIDE" = true ]; then+ echo "Wide change ($COUNT components) with no changed spec: add ci:run-a11y to run the scoped accessibility specs."+ else+ echo "No touched component and no changed spec; no contract to run."+ fi++ - name: Setup Node and pnpm+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ uses: ./.github/actions/setup+ with:+ install: 'false'++ - name: Download Storybook artifact+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ uses: actions/download-artifact@v8+ with:+ name: storybook-${{ needs.build-storybook.outputs.short_hash }}+ path: apps/storybook/dist++ - name: Install Playwright+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ run: pnpm install --frozen-lockfile && npx playwright install chromium++ # Routing lines come from this merge commit. A failing or unfinished+ # spec fails the job, and so the test join. A file that does not fit the+ # budget is not run and does not fail the job.+ - name: Run the touched components' accessibility contracts+ id: contracts+ if: steps.touched.outputs.has_components == 'true' || steps.specs.outputs.has_specs == 'true'+ # The runner stops at eight minutes; this bounds a hung browser and+ # the spec listing inside the job's 15 minutes.+ timeout-minutes: 10+ env:+ ASTRYX_HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ WORKFLOW_SHA: ${{ github.sha }}+ SPECS_PATH: ${{ steps.specs.outputs.path }}+ FILES_PATH: ${{ steps.scope.outputs.changed_files_path }}+ OWNERS: ${{ steps.touched.outputs.has_components == 'true' && steps.touched.outputs.a11y_owners || '' }}+ A11Y_CONTRACT_WORKERS: 4+ run: node .github/scripts/a11y-pr-contracts.mjs --changed-only "$SPECS_PATH" --changed-files "$FILES_PATH" --owned-by "$OWNERS"++ - name: Upload contract evidence+ if: always() && (steps.contracts.outcome == 'success' || steps.contracts.outcome == 'failure')+ uses: actions/upload-artifact@v7+ with:+ name: component-a11y-contracts+ path: |pr-a11y-evidence/+ pr-a11y-output/if-no-files-found: ignoreretention-days: 7