An open source design system that's fully customizable and agent ready
TypeScript
test(a11y): read each state's result once in six whole-suite contract checks (#7443)
Cindy Zhangec71b55822af
diff --git a/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts b/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.tsindex cca61838dd..36069aadb9 100644--- a/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts+++ b/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts@@ -180,25 +180,17 @@ async function runState(});}-test('every expectation is exercised and every known failure matches once', async ({- page,-}) => {- test.setTimeout(5 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of DIALOG_MODAL_BINDING_STATES) {- results.push(await runState(page, cdp, state));- }- expect(neverExercised(results)).toEqual([]);- expect(unmatchedKnownFailures(DIALOG_MODAL_KNOWN_FAILURES, results)).toEqual(- [],- );-});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();for (const state of DIALOG_MODAL_BINDING_STATES) {test(`Dialog [${state.id}] — ${state.summary}`, async ({page}) => {const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(MODAL_DIALOG_PATTERN, [result]);// eslint-disable-next-line no-console -- the report is this run's artifactconsole.log(formatReport(report));@@ -208,3 +200,18 @@ for (const state of DIALOG_MODAL_BINDING_STATES) {expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised and every known failure matches once', async ({+ page,+}) => {+ test.setTimeout(5 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of DIALOG_MODAL_BINDING_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ expect(neverExercised(results)).toEqual([]);+ expect(unmatchedKnownFailures(DIALOG_MODAL_KNOWN_FAILURES, results)).toEqual(+ [],+ );+});diff --git a/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts b/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.tsindex 9bf9068c09..031b7cef2b 100644--- a/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts+++ b/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts@@ -168,26 +168,18 @@ async function runState(});}-test('every expectation is exercised by at least one Core binding state', async ({- page,-}) => {- test.setTimeout(4 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {- results.push(await runState(page, cdp, state));- }- expect(neverExercised(results)).toEqual([]);- expect(- unmatchedKnownFailures(CORE_STATUS_MESSAGE_KNOWN_FAILURES, results),- ).toEqual([]);-});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {test(`${state.binding} [${state.id}] — ${state.summary}`, async ({page}) => {test.setTimeout(2 * 60 * 1000);const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(STATUS_MESSAGE_PATTERN, [result]);// eslint-disable-next-line no-console -- the report is this run's artifactconsole.log(formatReport(report));@@ -196,3 +188,18 @@ for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised by at least one Core binding state', async ({+ page,+}) => {+ test.setTimeout(4 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ expect(neverExercised(results)).toEqual([]);+ expect(+ unmatchedKnownFailures(CORE_STATUS_MESSAGE_KNOWN_FAILURES, results),+ ).toEqual([]);+});diff --git a/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts b/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.tsindex 98c2ef8b9d..c4588d1626 100644--- a/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts+++ b/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts@@ -71,21 +71,6 @@ async function runState(});}-test('every expectation is exercised by at least one NumberInput state', async ({- page,-}) => {- test.setTimeout(4 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of NUMBER_INPUT_A11Y_STATES) {- results.push(await runState(page, cdp, state));- }- expect(neverExercised(results)).toEqual([]);- expect(unmatchedKnownFailures(NUMBER_INPUT_KNOWN_FAILURES, results)).toEqual(- [],- );-});-test('the Chromium read-only omission remains exact visible debt', async ({page,}) => {@@ -104,13 +89,35 @@ test('the Chromium read-only omission remains exact visible debt', async ({expect(readOnly?.knownFailure).toBe(NUMBER_INPUT_KNOWN_FAILURES[0]);});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();+for (const state of NUMBER_INPUT_A11Y_STATES) {test(`NumberInput [${state.id}] — ${state.summary}`, async ({page}) => {const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(SPINBUTTON_PATTERN, [result]);expect(formatFailures(blockingResults([result]))).toBe('');expect(report.unrunLayers).toEqual([]);expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised by at least one NumberInput state', async ({+ page,+}) => {+ test.setTimeout(4 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of NUMBER_INPUT_A11Y_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ expect(neverExercised(results)).toEqual([]);+ expect(unmatchedKnownFailures(NUMBER_INPUT_KNOWN_FAILURES, results)).toEqual(+ [],+ );+});diff --git a/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts b/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.tsindex 753b3c5f76..7bd3edf3ef 100644--- a/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts+++ b/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts@@ -166,29 +166,17 @@ test('the state inventory describes the labels the page actually renders', asyncexpect(wrong).toEqual([]);});-test('every expectation is exercised by at least one bound state', async ({- page,-}) => {- // A whole-binding sweep: every state, every expectation, a fresh mount each- // time. That is minutes of real browser work, not the seconds a single-state- // test takes.- test.setTimeout(5 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of SWITCH_BINDING_STATES) {- results.push(await runState(page, cdp, state));- }- // An expectation that applies to nothing is not coverage, however green it- // looks. This is the only place that can notice: the contract never sees the- // states, so whether a condition matches a real one is a binding-side fact.- expect(neverExercised(results)).toEqual([]);- expect(unmatchedKnownFailures(SWITCH_KNOWN_FAILURES, results)).toEqual([]);-});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();for (const state of SWITCH_BINDING_STATES) {test(`Switch [${state.id}] — ${state.summary}`, async ({page}) => {const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(SWITCH_PATTERN, [result]);// eslint-disable-next-line no-console -- the report is this run's artifactconsole.log(formatReport(report));@@ -201,3 +189,22 @@ for (const state of SWITCH_BINDING_STATES) {expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised by at least one bound state', async ({+ page,+}) => {+ // A whole-binding sweep: every state's result, read from its own test or,+ // when that test did not run in this worker, measured here (minutes of real+ // browser work in that case, so the budget stays).+ test.setTimeout(5 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of SWITCH_BINDING_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ // An expectation that applies to nothing is not coverage, however green it+ // looks. This is the only place that can notice: the contract never sees the+ // states, so whether a condition matches a real one is a binding-side fact.+ expect(neverExercised(results)).toEqual([]);+ expect(unmatchedKnownFailures(SWITCH_KNOWN_FAILURES, results)).toEqual([]);+});diff --git a/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts b/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.tsindex 384f0b8769..90910f6c5b 100644--- a/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts+++ b/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts@@ -163,24 +163,18 @@ test('every binding fact matches the rendered semantic state', async ({expect(wrong).toEqual([]);});-test('every expectation is exercised by at least one bound state', async ({- page,-}) => {- test.setTimeout(5 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of TABS_BINDING_STATES) {- results.push(await runState(page, cdp, state));- }- expect(neverExercised(results)).toEqual([]);- expect(unmatchedKnownFailures(TABS_KNOWN_FAILURES, results)).toEqual([]);-});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();for (const state of TABS_BINDING_STATES) {test(`${state.binding} [${state.id}] — ${state.summary}`, async ({page}) => {test.setTimeout(3 * 60 * 1000);const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(TABS_PATTERN, [result]);// eslint-disable-next-line no-console -- the report is this run's artifactconsole.log(formatReport(report));@@ -189,3 +183,16 @@ for (const state of TABS_BINDING_STATES) {expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised by at least one bound state', async ({+ page,+}) => {+ test.setTimeout(5 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of TABS_BINDING_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ expect(neverExercised(results)).toEqual([]);+ expect(unmatchedKnownFailures(TABS_KNOWN_FAILURES, results)).toEqual([]);+});diff --git a/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts b/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.tsindex 9c6e7451e8..98af004b30 100644--- a/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts+++ b/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts@@ -143,22 +143,17 @@ test('the state inventory matches rendered labels and applicability facts', asynexpect(wrong).toEqual([]);});-test('every expectation is exercised by at least one ToggleButton state', async ({- page,-}) => {- test.setTimeout(5 * 60 * 1000);- const cdp = await page.context().newCDPSession(page);- const results: BindingResult[] = [];- for (const state of TOGGLE_BUTTON_BINDING_STATES) {- results.push(await runState(page, cdp, state));- }- expect(neverExercised(results)).toEqual([]);-});+// Each state's result, recorded by its own test below so the whole-suite check+// at the end reads it instead of measuring every state a second time. A state+// whose test did not run in this worker (a filtered run, or a worker restarted+// after a failure) is measured by the whole-suite check itself.+const recorded = new Map<string, BindingResult>();for (const state of TOGGLE_BUTTON_BINDING_STATES) {test(`ToggleButton [${state.id}] — ${state.summary}`, async ({page}) => {const cdp = await page.context().newCDPSession(page);const result = await runState(page, cdp, state);+ recorded.set(state.id, result);const report = summarize(TOGGLE_BUTTON_PATTERN, [result]);// eslint-disable-next-line no-console -- the report is this run's evidenceconsole.log(formatReport(report));@@ -167,3 +162,15 @@ for (const state of TOGGLE_BUTTON_BINDING_STATES) {expect(report.counts.unexpectedPass).toBe(0);});}++test('every expectation is exercised by at least one ToggleButton state', async ({+ page,+}) => {+ test.setTimeout(5 * 60 * 1000);+ const cdp = await page.context().newCDPSession(page);+ const results: BindingResult[] = [];+ for (const state of TOGGLE_BUTTON_BINDING_STATES) {+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));+ }+ expect(neverExercised(results)).toEqual([]);+});