Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

test(a11y): read each state's result once in six whole-suite contract checks (#7443)

Cindy Zhangec71b55822af
diff --git a/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts b/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts
index cca61838dd..36069aadb9 100644
--- a/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts
+++ b/packages/core/src/Dialog/__tests__/Dialog.a11y.chromium.spec.ts
@@ -180,25 +180,17 @@ async function runState(
});
}
-test('every expectation is exercised and every known failure matches once', async ({
- page,
-}) => {
- test.setTimeout(5 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of DIALOG_MODAL_BINDING_STATES) {
- results.push(await runState(page, cdp, state));
- }
- expect(neverExercised(results)).toEqual([]);
- expect(unmatchedKnownFailures(DIALOG_MODAL_KNOWN_FAILURES, results)).toEqual(
- [],
- );
-});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
for (const state of DIALOG_MODAL_BINDING_STATES) {
test(`Dialog [${state.id}] — ${state.summary}`, async ({page}) => {
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(MODAL_DIALOG_PATTERN, [result]);
// eslint-disable-next-line no-console -- the report is this run's artifact
console.log(formatReport(report));
@@ -208,3 +200,18 @@ for (const state of DIALOG_MODAL_BINDING_STATES) {
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised and every known failure matches once', async ({
+ page,
+}) => {
+ test.setTimeout(5 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of DIALOG_MODAL_BINDING_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ expect(neverExercised(results)).toEqual([]);
+ expect(unmatchedKnownFailures(DIALOG_MODAL_KNOWN_FAILURES, results)).toEqual(
+ [],
+ );
+});
diff --git a/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts b/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts
index 9bf9068c09..031b7cef2b 100644
--- a/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts
+++ b/packages/core/src/FieldStatus/__tests__/StatusMessage.a11y.chromium.spec.ts
@@ -168,26 +168,18 @@ async function runState(
});
}
-test('every expectation is exercised by at least one Core binding state', async ({
- page,
-}) => {
- test.setTimeout(4 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {
- results.push(await runState(page, cdp, state));
- }
- expect(neverExercised(results)).toEqual([]);
- expect(
- unmatchedKnownFailures(CORE_STATUS_MESSAGE_KNOWN_FAILURES, results),
- ).toEqual([]);
-});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {
test(`${state.binding} [${state.id}] — ${state.summary}`, async ({page}) => {
test.setTimeout(2 * 60 * 1000);
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(STATUS_MESSAGE_PATTERN, [result]);
// eslint-disable-next-line no-console -- the report is this run's artifact
console.log(formatReport(report));
@@ -196,3 +188,18 @@ for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised by at least one Core binding state', async ({
+ page,
+}) => {
+ test.setTimeout(4 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of CORE_STATUS_MESSAGE_BINDING_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ expect(neverExercised(results)).toEqual([]);
+ expect(
+ unmatchedKnownFailures(CORE_STATUS_MESSAGE_KNOWN_FAILURES, results),
+ ).toEqual([]);
+});
diff --git a/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts b/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts
index 98c2ef8b9d..c4588d1626 100644
--- a/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts
+++ b/packages/core/src/NumberInput/__tests__/NumberInput.a11y.chromium.spec.ts
@@ -71,21 +71,6 @@ async function runState(
});
}
-test('every expectation is exercised by at least one NumberInput state', async ({
- page,
-}) => {
- test.setTimeout(4 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of NUMBER_INPUT_A11Y_STATES) {
- results.push(await runState(page, cdp, state));
- }
- expect(neverExercised(results)).toEqual([]);
- expect(unmatchedKnownFailures(NUMBER_INPUT_KNOWN_FAILURES, results)).toEqual(
- [],
- );
-});
-
test('the Chromium read-only omission remains exact visible debt', async ({
page,
}) => {
@@ -104,13 +89,35 @@ test('the Chromium read-only omission remains exact visible debt', async ({
expect(readOnly?.knownFailure).toBe(NUMBER_INPUT_KNOWN_FAILURES[0]);
});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
+
for (const state of NUMBER_INPUT_A11Y_STATES) {
test(`NumberInput [${state.id}] — ${state.summary}`, async ({page}) => {
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(SPINBUTTON_PATTERN, [result]);
expect(formatFailures(blockingResults([result]))).toBe('');
expect(report.unrunLayers).toEqual([]);
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised by at least one NumberInput state', async ({
+ page,
+}) => {
+ test.setTimeout(4 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of NUMBER_INPUT_A11Y_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ expect(neverExercised(results)).toEqual([]);
+ expect(unmatchedKnownFailures(NUMBER_INPUT_KNOWN_FAILURES, results)).toEqual(
+ [],
+ );
+});
diff --git a/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts b/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts
index 753b3c5f76..7bd3edf3ef 100644
--- a/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts
+++ b/packages/core/src/Switch/__tests__/Switch.a11y.chromium.spec.ts
@@ -166,29 +166,17 @@ test('the state inventory describes the labels the page actually renders', async
expect(wrong).toEqual([]);
});
-test('every expectation is exercised by at least one bound state', async ({
- page,
-}) => {
- // A whole-binding sweep: every state, every expectation, a fresh mount each
- // time. That is minutes of real browser work, not the seconds a single-state
- // test takes.
- test.setTimeout(5 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of SWITCH_BINDING_STATES) {
- results.push(await runState(page, cdp, state));
- }
- // An expectation that applies to nothing is not coverage, however green it
- // looks. This is the only place that can notice: the contract never sees the
- // states, so whether a condition matches a real one is a binding-side fact.
- expect(neverExercised(results)).toEqual([]);
- expect(unmatchedKnownFailures(SWITCH_KNOWN_FAILURES, results)).toEqual([]);
-});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
for (const state of SWITCH_BINDING_STATES) {
test(`Switch [${state.id}] — ${state.summary}`, async ({page}) => {
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(SWITCH_PATTERN, [result]);
// eslint-disable-next-line no-console -- the report is this run's artifact
console.log(formatReport(report));
@@ -201,3 +189,22 @@ for (const state of SWITCH_BINDING_STATES) {
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised by at least one bound state', async ({
+ page,
+}) => {
+ // A whole-binding sweep: every state's result, read from its own test or,
+ // when that test did not run in this worker, measured here (minutes of real
+ // browser work in that case, so the budget stays).
+ test.setTimeout(5 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of SWITCH_BINDING_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ // An expectation that applies to nothing is not coverage, however green it
+ // looks. This is the only place that can notice: the contract never sees the
+ // states, so whether a condition matches a real one is a binding-side fact.
+ expect(neverExercised(results)).toEqual([]);
+ expect(unmatchedKnownFailures(SWITCH_KNOWN_FAILURES, results)).toEqual([]);
+});
diff --git a/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts b/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts
index 384f0b8769..90910f6c5b 100644
--- a/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts
+++ b/packages/core/src/TabList/__tests__/Tabs.a11y.chromium.spec.ts
@@ -163,24 +163,18 @@ test('every binding fact matches the rendered semantic state', async ({
expect(wrong).toEqual([]);
});
-test('every expectation is exercised by at least one bound state', async ({
- page,
-}) => {
- test.setTimeout(5 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of TABS_BINDING_STATES) {
- results.push(await runState(page, cdp, state));
- }
- expect(neverExercised(results)).toEqual([]);
- expect(unmatchedKnownFailures(TABS_KNOWN_FAILURES, results)).toEqual([]);
-});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
for (const state of TABS_BINDING_STATES) {
test(`${state.binding} [${state.id}] — ${state.summary}`, async ({page}) => {
test.setTimeout(3 * 60 * 1000);
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(TABS_PATTERN, [result]);
// eslint-disable-next-line no-console -- the report is this run's artifact
console.log(formatReport(report));
@@ -189,3 +183,16 @@ for (const state of TABS_BINDING_STATES) {
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised by at least one bound state', async ({
+ page,
+}) => {
+ test.setTimeout(5 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of TABS_BINDING_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ expect(neverExercised(results)).toEqual([]);
+ expect(unmatchedKnownFailures(TABS_KNOWN_FAILURES, results)).toEqual([]);
+});
diff --git a/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts b/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts
index 9c6e7451e8..98af004b30 100644
--- a/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts
+++ b/packages/core/src/ToggleButton/__tests__/ToggleButton.a11y.chromium.spec.ts
@@ -143,22 +143,17 @@ test('the state inventory matches rendered labels and applicability facts', asyn
expect(wrong).toEqual([]);
});
-test('every expectation is exercised by at least one ToggleButton state', async ({
- page,
-}) => {
- test.setTimeout(5 * 60 * 1000);
- const cdp = await page.context().newCDPSession(page);
- const results: BindingResult[] = [];
- for (const state of TOGGLE_BUTTON_BINDING_STATES) {
- results.push(await runState(page, cdp, state));
- }
- expect(neverExercised(results)).toEqual([]);
-});
+// Each state's result, recorded by its own test below so the whole-suite check
+// at the end reads it instead of measuring every state a second time. A state
+// whose test did not run in this worker (a filtered run, or a worker restarted
+// after a failure) is measured by the whole-suite check itself.
+const recorded = new Map<string, BindingResult>();
for (const state of TOGGLE_BUTTON_BINDING_STATES) {
test(`ToggleButton [${state.id}] — ${state.summary}`, async ({page}) => {
const cdp = await page.context().newCDPSession(page);
const result = await runState(page, cdp, state);
+ recorded.set(state.id, result);
const report = summarize(TOGGLE_BUTTON_PATTERN, [result]);
// eslint-disable-next-line no-console -- the report is this run's evidence
console.log(formatReport(report));
@@ -167,3 +162,15 @@ for (const state of TOGGLE_BUTTON_BINDING_STATES) {
expect(report.counts.unexpectedPass).toBe(0);
});
}
+
+test('every expectation is exercised by at least one ToggleButton state', async ({
+ page,
+}) => {
+ test.setTimeout(5 * 60 * 1000);
+ const cdp = await page.context().newCDPSession(page);
+ const results: BindingResult[] = [];
+ for (const state of TOGGLE_BUTTON_BINDING_STATES) {
+ results.push(recorded.get(state.id) ?? (await runState(page, cdp, state)));
+ }
+ expect(neverExercised(results)).toEqual([]);
+});