An open source design system that's fully customizable and agent ready
TypeScript
ci(a11y): split the contract legs by recorded duration (#7450)
Cindy Zhangf939111e231f
diff --git a/.github/a11y-contract-durations.json b/.github/a11y-contract-durations.jsonnew file mode 100644index 0000000000..7d16a5adba--- /dev/null+++ b/.github/a11y-contract-durations.json@@ -0,0 +1,93 @@+{+ "$comment": "Seconds of test time per accessibility contract spec file, from a release check's contract legs (four workers). Only shard balance depends on these numbers; a missing or stale entry never drops or repeats a file. Regenerate with: node .github/scripts/a11y-contract-shard.mjs record <leg logs>\u2026",+ "source": "release check run 38368145882 (main 06e35f5a)",+ "seconds": {+ "internal/a11y-spec/src/harness/chromium.chromium.spec.ts": 1.4,+ "internal/a11y-spec/src/patterns/breadcrumb.chromium.spec.ts": 2.8,+ "internal/a11y-spec/src/patterns/button.chromium.spec.ts": 9.1,+ "internal/a11y-spec/src/patterns/checkbox.chromium.spec.ts": 29.1,+ "internal/a11y-spec/src/patterns/disclosure.chromium.spec.ts": 5.4,+ "internal/a11y-spec/src/patterns/landmark.chromium.spec.ts": 2.2,+ "internal/a11y-spec/src/patterns/listbox.chromium.spec.ts": 1.2,+ "internal/a11y-spec/src/patterns/modal-dialog.chromium.spec.ts": 10.8,+ "internal/a11y-spec/src/patterns/radio-group.chromium.spec.ts": 18.6,+ "internal/a11y-spec/src/patterns/spinbutton.chromium.spec.ts": 3.6,+ "internal/a11y-spec/src/patterns/status-message.chromium.spec.ts": 10.2,+ "internal/a11y-spec/src/patterns/switch.chromium.spec.ts": 15.0,+ "internal/a11y-spec/src/patterns/tabs.chromium.spec.ts": 18.5,+ "internal/a11y-spec/src/patterns/text-input.chromium.spec.ts": 19.5,+ "internal/a11y-spec/src/patterns/toggle-button.chromium.spec.ts": 15.7,+ "packages/core/src/BottomSheet/__tests__/BottomSheetDrag.a11y.chromium.spec.ts": 3.6,+ "packages/core/src/BottomSheet/__tests__/BottomSheetFocusReturn.a11y.browser.spec.ts": 5.2,+ "packages/core/src/BottomSheet/__tests__/BottomSheetKeyboard.a11y.browser.spec.ts": 48.4,+ "packages/core/src/BottomSheet/__tests__/BottomSheetPinchZoom.a11y.chromium.spec.ts": 27.8,+ "packages/core/src/Breadcrumbs/__tests__/Breadcrumbs.a11y.chromium.spec.ts": 20.5,+ "packages/core/src/Button/__tests__/Button.a11y.chromium.spec.ts": 164.9,+ "packages/core/src/Chat/__tests__/ChatLayoutScrollButton.a11y.chromium.spec.ts": 6.6,+ "packages/core/src/Chat/__tests__/ChatLayoutScrollButtonSurface.a11y.chromium.spec.ts": 19.8,+ "packages/core/src/Chat/__tests__/ChatMessageBubble.a11y.chromium.spec.ts": 10.7,+ "packages/core/src/Chat/__tests__/ChatMessageList.a11y.chromium.spec.ts": 29.8,+ "packages/core/src/Chat/__tests__/ChatMessageMetadata.a11y.chromium.spec.ts": 35.6,+ "packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts": 42.7,+ "packages/core/src/Chat/__tests__/ChatSystemMessage.a11y.chromium.spec.ts": 21.3,+ "packages/core/src/Chat/__tests__/ChatTokenizedText.a11y.chromium.spec.ts": 17.9,+ "packages/core/src/Chat/__tests__/useChatStreamScroll.a11y.chromium.spec.ts": 12.5,+ "packages/core/src/CheckboxInput/__tests__/Checkbox.a11y.chromium.spec.ts": 313.3,+ "packages/core/src/CheckboxList/__tests__/CheckboxList.a11y.chromium.spec.ts": 138.0,+ "packages/core/src/CodeBlock/__tests__/CodeBlockOverflowStop.a11y.chromium.spec.ts": 6.9,+ "packages/core/src/Collapsible/__tests__/Collapsible.a11y.chromium.spec.ts": 82.8,+ "packages/core/src/Collapsible/__tests__/CollapsibleGroup.a11y.chromium.spec.ts": 9.2,+ "packages/core/src/CommandPalette/__tests__/CommandPaletteFooter.a11y.chromium.spec.ts": 17.1,+ "packages/core/src/ContextMenu/__tests__/ContextMenuCursorAnchor.a11y.browser.spec.ts": 5.1,+ "packages/core/src/Dialog/__tests__/Dialog.a11y.browser.spec.ts": 8.7,+ "packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts": 45.5,+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuDrillIn.a11y.chromium.spec.ts": 3.3,+ "packages/core/src/DropdownMenu/__tests__/DropdownMenuFocusReturn.a11y.browser.spec.ts": 3.1,+ "packages/core/src/DropdownMenu/__tests__/MenuHeaderFooter.a11y.chromium.spec.ts": 13.4,+ "packages/core/src/DropdownMenu/__tests__/MenuPinchZoom.a11y.chromium.spec.ts": 8.4,+ "packages/core/src/DropdownMenu/__tests__/MenuPress.a11y.chromium.spec.ts": 25.3,+ "packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts": 59.6,+ "packages/core/src/Item/__tests__/ItemSwipeActions.a11y.chromium.spec.ts": 10.2,+ "packages/core/src/Layout/__tests__/Layout.a11y.chromium.spec.ts": 33.2,+ "packages/core/src/Link/__tests__/Link.navigation.a11y.chromium.spec.ts": 48.6,+ "packages/core/src/Markdown/MarkdownTable.a11y.chromium.spec.ts": 62.9,+ "packages/core/src/Markdown/__tests__/MarkdownNesting.a11y.chromium.spec.ts": 8.9,+ "packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts": 18.7,+ "packages/core/src/MobileNav/MobileNavPinchZoom.a11y.chromium.spec.ts": 8.6,+ "packages/core/src/MultiSelector/CreateRow.a11y.chromium.spec.ts": 2.1,+ "packages/core/src/MultiSelector/OptionActions.a11y.chromium.spec.ts": 10.0,+ "packages/core/src/MultiSelector/__tests__/MultiSelector.a11y.chromium.spec.ts": 2.3,+ "packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts": 63.6,+ "packages/core/src/Outline/OutlineFocus.a11y.chromium.spec.ts": 21.6,+ "packages/core/src/RadioList/__tests__/RadioGroup.a11y.chromium.spec.ts": 214.5,+ "packages/core/src/Selector/__tests__/EmptyStateAnnouncement.a11y.chromium.spec.ts": 3.8,+ "packages/core/src/Selector/__tests__/Listbox.a11y.chromium.spec.ts": 351.8,+ "packages/core/src/Selector/__tests__/Selector.a11y.chromium.spec.ts": 1.6,+ "packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts": 128.3,+ "packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts": 97.3,+ "packages/core/src/Table/__tests__/TableMinWidth.a11y.chromium.spec.ts": 0.7,+ "packages/core/src/Table/__tests__/TableScroll.a11y.chromium.spec.ts": 4.7,+ "packages/core/src/TextInput/__tests__/TextInput.a11y.chromium.spec.ts": 335.4,+ "packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts": 144.1,+ "packages/core/src/Tooltip/__tests__/TooltipFocusReturn.a11y.chromium.spec.ts": 2.2,+ "packages/core/src/__tests__/PressedState.a11y.chromium.spec.ts": 20.2,+ "packages/core/src/reset.a11y.chromium.spec.ts": 1.2,+ "packages/core/src/theme/__tests__/SpacingRootFontSize.a11y.chromium.spec.ts": 15.2,+ "packages/lab/src/Chat/__tests__/ChatTypingIndicator.a11y.chromium.spec.ts": 33.3,+ "packages/lab/src/Schedule/__tests__/ScheduleEventPopover.a11y.chromium.spec.ts": 42.0,+ "packages/lab/src/Schedule/__tests__/ScheduleMonthChip.a11y.chromium.spec.ts": 7.4,+ "packages/lab/src/Schedule/__tests__/ScheduleMonthEventPopover.a11y.chromium.spec.ts": 13.5,+ "packages/lab/src/Schedule/__tests__/ScheduleMonthOverflow.a11y.chromium.spec.ts": 22.7,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeDirection.a11y.chromium.spec.ts": 18.4,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeGridGeometry.a11y.chromium.spec.ts": 13.3,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeGridInitialPosition.a11y.chromium.spec.ts": 16.7,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeGridLongSpan.a11y.chromium.spec.ts": 12.0,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeGridMidnight.a11y.chromium.spec.ts": 5.3,+ "packages/lab/src/Schedule/__tests__/ScheduleTimeGridOverlap.a11y.chromium.spec.ts": 11.4,+ "packages/richtext/src/__tests__/MarkdownParitySandbox.a11y.chromium.spec.ts": 102.2,+ "packages/richtext/src/__tests__/MarkdownPlugins.a11y.chromium.spec.ts": 33.4,+ "packages/richtext/src/__tests__/MarkdownPluginsImportExport.a11y.chromium.spec.ts": 9.8,+ "packages/richtext/src/__tests__/RichTextEditorToolbar.a11y.chromium.spec.ts": 17.7,+ "packages/richtext/src/__tests__/RichTextView.a11y.chromium.spec.ts": 3.9+ }+}diff --git a/.github/scripts/a11y-contract-shard.mjs b/.github/scripts/a11y-contract-shard.mjsnew file mode 100644index 0000000000..74cbaacd7b--- /dev/null+++ b/.github/scripts/a11y-contract-shard.mjs@@ -0,0 +1,193 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Which accessibility contract spec files one release shard runs.+ * @input `--shard <i>/<n>`; the contract suite as `playwright test --list`+ * reports it; recorded seconds per file (.github/a11y-contract-durations.json).+ * @output One file filter per line for `pnpm test:a11y-contract`; with+ * `record <log>…`, a fresh durations file from contract leg logs.+ * @position The a11y-contracts-shard legs in ci.yml. Playwright's own `--shard`+ * splits files in path order by test count, but per-test cost differs by+ * orders of magnitude (fixture checks take milliseconds, binding states+ * seconds), so one leg idled while another ran three times as long. This+ * assigns every listed file to exactly one leg, choosing the split that+ * finishes soonest when each leg runs its files in path order on its+ * workers, using the recorded seconds (a file with none counts as the+ * median). The numbers only steer balance; coverage never depends on them.+ */++import {execFileSync} from 'node:child_process';+import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath, pathToFileURL} from 'node:url';++const ROOT = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ '../..',+);+const DURATIONS = path.join(ROOT, '.github/a11y-contract-durations.json');+const WORKERS = 4;++/** @param {{suites?: object[]}} listing Playwright JSON reporter output (`--list`). */+export function listedFiles(listing) {+ const files = new Set();+ const visit = suite => {+ for (const spec of suite.specs ?? []) files.add(spec.file ?? suite.file);+ for (const child of suite.suites ?? []) visit(child);+ };+ for (const suite of listing.suites ?? []) visit(suite);+ return [...files].filter(Boolean).sort();+}++/** Wall time of files run in path order, each by the first free worker. */+export function legWall(files, seconds, workers = WORKERS) {+ const free = Array(workers).fill(0);+ for (const file of [...files].sort()) {+ const next = free.indexOf(Math.min(...free));+ free[next] += seconds(file);+ }+ return Math.max(...free);+}++/**+ * @param {string[]} files every listed file+ * @param {number} count legs+ * @param {Record<string, number>} recorded seconds per file+ * @returns {string[][]} each leg's files in path order; together exactly `files`+ */+export function partitionFiles(files, count, recorded) {+ const known = files+ .map(file => recorded[file])+ .filter(value => value > 0)+ .sort((a, b) => a - b);+ const median = known.length > 0 ? known[Math.floor(known.length / 2)] : 1;+ const seconds = file => (recorded[file] > 0 ? recorded[file] : median);+ const legs = Array.from({length: count}, () => []);+ // Longest first to the lightest leg, then improve by moving single files.+ for (const file of [...files].sort(+ (a, b) => seconds(b) - seconds(a) || a.localeCompare(b),+ )) {+ const sums = legs.map(leg =>+ leg.reduce((sum, item) => sum + seconds(item), 0),+ );+ legs[sums.indexOf(Math.min(...sums))].push(file);+ }+ const worst = () => Math.max(...legs.map(leg => legWall(leg, seconds)));+ for (let improved = true; improved;) {+ improved = false;+ let best = worst();+ for (let from = 0; from < count && !improved; from += 1) {+ for (const file of [...legs[from]].sort()) {+ for (let to = 0; to < count && !improved; to += 1) {+ if (to === from) continue;+ legs[from] = legs[from].filter(item => item !== file);+ legs[to].push(file);+ const candidate = worst();+ if (candidate < best - 0.5) {+ best = candidate;+ improved = true;+ } else {+ legs[to] = legs[to].filter(item => item !== file);+ legs[from].push(file);+ }+ }+ if (improved) break;+ }+ }+ }+ const result = legs.map(leg => [...leg].sort());+ const assigned = result.flat();+ if (+ assigned.length !== files.length ||+ new Set(assigned).size !== files.length+ ) {+ throw new Error('Contract shard partition lost or repeated a file.');+ }+ return result;+}++/** A Playwright file filter matching exactly this repository-relative path. */+export function fileFilter(file) {+ return `/${file.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`;+}++/** Seconds per file from Playwright list-reporter lines. */+export function recordDurations(text) {+ const seconds = {};+ for (const line of text.split('\n')) {+ const match =+ /[✓✘-]\s+\d+ \[chromium\] › ([^:]+):\d+:\d+ › .*\((\d+(?:\.\d+)?)(ms|s|m)\)\s*$/.exec(+ line,+ );+ if (!match) continue;+ const value = Number(match[2]) * {ms: 0.001, s: 1, m: 60}[match[3]];+ seconds[match[1]] =+ Math.round(((seconds[match[1]] ?? 0) + value) * 10) / 10;+ }+ return seconds;+}++function parseShard(value) {+ const match = /^(\d+)\/(\d+)$/.exec(value ?? '');+ const index = match ? Number(match[1]) : NaN;+ const count = match ? Number(match[2]) : NaN;+ if (+ !Number.isInteger(index) ||+ !Number.isInteger(count) ||+ index < 1 ||+ index > count+ ) {+ throw new Error(+ `--shard must be <index>/<count>, got ${value ?? 'nothing'}`,+ );+ }+ return {index, count};+}++if (+ process.argv[1] &&+ import.meta.url === pathToFileURL(process.argv[1]).href+) {+ const args = process.argv.slice(2);+ if (args[0] === 'record') {+ const seconds = {};+ for (const file of args.slice(1))+ Object.assign(seconds, recordDurations(fs.readFileSync(file, 'utf8')));+ const current = JSON.parse(fs.readFileSync(DURATIONS, 'utf8'));+ const sorted = Object.fromEntries(+ Object.entries(seconds).sort(([a], [b]) => a.localeCompare(b)),+ );+ fs.writeFileSync(+ DURATIONS,+ `${JSON.stringify({...current, source: 'recorded contract leg logs', seconds: sorted}, null, 2)}\n`,+ );+ } else {+ const shard = parseShard(args[args.indexOf('--shard') + 1]);+ const listing = JSON.parse(+ execFileSync(+ 'pnpm',+ ['-s', 'test:a11y-contract', '--list', '--reporter=json'],+ {+ cwd: ROOT,+ encoding: 'utf8',+ maxBuffer: 64 * 1024 * 1024,+ stdio: ['ignore', 'pipe', 'inherit'],+ },+ ),+ );+ const files = listedFiles(listing);+ if (files.length === 0)+ throw new Error('The contract suite lists no spec files.');+ const {seconds} = JSON.parse(fs.readFileSync(DURATIONS, 'utf8'));+ const mine = partitionFiles(files, shard.count, seconds)[shard.index - 1];+ if (mine.length === 0)+ throw new Error(+ `Contract shard ${shard.index}/${shard.count} has no files.`,+ );+ console.error(+ `Contract shard ${shard.index}/${shard.count}: ${mine.length} of ${files.length} files`,+ );+ process.stdout.write(`${mine.map(fileFilter).join('\n')}\n`);+ }+}diff --git a/.github/scripts/a11y-contract-shard.test.mjs b/.github/scripts/a11y-contract-shard.test.mjsnew file mode 100644index 0000000000..278eb5da1d--- /dev/null+++ b/.github/scripts/a11y-contract-shard.test.mjs@@ -0,0 +1,123 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {describe, expect, it} from 'vitest';+import {+ fileFilter,+ legWall,+ listedFiles,+ partitionFiles,+ recordDurations,+} from './a11y-contract-shard.mjs';++const ROOT = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ '../..',+);++describe('a11y contract shards', () => {+ const recorded = {+ 'packages/a.spec.ts': 300,+ 'packages/b.spec.ts': 280,+ 'packages/c.spec.ts': 40,+ 'packages/d.spec.ts': 30,+ 'packages/e.spec.ts': 20,+ 'internal/f.spec.ts': 1,+ };+ const files = [...Object.keys(recorded), 'packages/new.spec.ts'].sort();++ it('assigns every file to exactly one leg, including unrecorded files', () => {+ const legs = partitionFiles(files, 3, recorded);+ expect(legs.flat().sort()).toEqual(files);+ expect(new Set(legs.flat()).size).toBe(files.length);+ for (const leg of legs) expect(leg).toEqual([...leg].sort());+ });++ it('keeps the two heaviest files apart', () => {+ const legs = partitionFiles(files, 3, recorded);+ const leg = file => legs.findIndex(items => items.includes(file));+ expect(leg('packages/a.spec.ts')).not.toBe(leg('packages/b.spec.ts'));+ });++ it('beats a count-based path-order split on the recorded suite', () => {+ const {seconds} = JSON.parse(+ fs.readFileSync(+ path.join(ROOT, '.github/a11y-contract-durations.json'),+ 'utf8',+ ),+ );+ const all = Object.keys(seconds).sort();+ const at = file => seconds[file];+ const balanced = Math.max(+ ...partitionFiles(all, 3, seconds).map(leg => legWall(leg, at)),+ );+ const naive = Math.max(+ ...[0, 1, 2].map(index =>+ legWall(+ all.slice((index * all.length) / 3, ((index + 1) * all.length) / 3),+ at,+ ),+ ),+ );+ expect(balanced).toBeLessThan(naive);+ });++ it('simulates path-order workers', () => {+ expect(+ legWall(['b', 'a', 'c'], file => ({a: 5, b: 1, c: 1})[file], 2),+ ).toBe(5);+ });++ it('lists the files of a Playwright JSON listing', () => {+ expect(+ listedFiles({+ suites: [+ {+ file: 'x.spec.ts',+ specs: [{file: 'x.spec.ts'}],+ suites: [{specs: [{file: 'x.spec.ts'}]}],+ },+ {file: 'a.spec.ts', specs: [{file: 'a.spec.ts'}]},+ ],+ }),+ ).toEqual(['a.spec.ts', 'x.spec.ts']);+ });++ it('builds a filter that matches only that path', () => {+ const filter = new RegExp(+ fileFilter(+ 'packages/core/src/Button/__tests__/Button.a11y.chromium.spec.ts',+ ),+ );+ expect(+ filter.test(+ '/repo/packages/core/src/Button/__tests__/Button.a11y.chromium.spec.ts',+ ),+ ).toBe(true);+ expect(+ filter.test(+ '/repo/packages/core/src/IconButton/__tests__/IconButton.a11y.chromium.spec.ts',+ ),+ ).toBe(false);+ expect(+ filter.test(+ '/repo/xpackages/core/src/Button/__tests__/Button.a11y.chromium.spec.ts',+ ),+ ).toBe(false);+ });++ it('records seconds per file from list-reporter lines', () => {+ expect(+ recordDurations(+ [+ ' ✓ 1 [chromium] › packages/a.spec.ts:10:1 › one (1.5s)',+ ' ✓ 2 [chromium] › packages/a.spec.ts:20:1 › two (500ms)',+ ' ✘ 3 [chromium] › packages/b.spec.ts:5:1 › three (1.2m)',+ 'unrelated line',+ ].join('\n'),+ ),+ ).toEqual({'packages/a.spec.ts': 2, 'packages/b.spec.ts': 72});+ });+});diff --git a/.github/scripts/release-check-workflow.test.mjs b/.github/scripts/release-check-workflow.test.mjsindex ceb7824ef1..f610f1c3e8 100644--- a/.github/scripts/release-check-workflow.test.mjs+++ b/.github/scripts/release-check-workflow.test.mjs@@ -710,9 +710,11 @@ describe('full release scope without PR metadata or maintenance writes', () => {'a11y-contracts-shard','Run accessibility spec-test contracts',);- expect(contracts.run).toBe(- 'pnpm test:a11y-contract --shard=${{ matrix.shard }}/${{ strategy.job-total }}',+ expect(contracts.run).toContain(+ 'a11y-contract-shard.mjs --shard "${{ matrix.shard }}/${{ strategy.job-total }}"',);+ expect(contracts.run).toContain('pnpm test:a11y-contract "${FILES[@]}"');+ expect(contracts.run).toContain('set -euo pipefail');expect(contracts.env.A11Y_CONTRACT_WORKERS).toBe(4);// Shard legs that run four pages or workers get four cores.for (const job of ['visual-scout', 'pr-a11y-shard']) {diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 9a1ef28980..dc4da90540 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -2001,7 +2001,13 @@ jobs:env:ASTRYX_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}A11Y_CONTRACT_WORKERS: 4- run: pnpm test:a11y-contract --shard=${{ matrix.shard }}/${{ strategy.job-total }}+ # Files are split by recorded duration (a11y-contract-shard.mjs), not+ # Playwright's count-based --shard; every listed file runs in one leg.+ run: |+ set -euo pipefail+ node .github/scripts/a11y-contract-shard.mjs --shard "${{ matrix.shard }}/${{ strategy.job-total }}" > contract-files.txt+ mapfile -t FILES < contract-files.txt+ pnpm test:a11y-contract "${FILES[@]}"- name: Upload pressed-state pixel evidenceif: always() && github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'