An open source design system that's fully customizable and agent ready
TypeScript
test(a11y): separate known failures from issue tracking (#6175)
Cindy Zhang60fc2f23330a
diff --git a/internal/a11y-spec/README.md b/internal/a11y-spec/README.mdindex 16d7c5d6d6..065d676a77 100644--- a/internal/a11y-spec/README.md+++ b/internal/a11y-spec/README.md@@ -156,16 +156,19 @@ and styling stay in the component's own suite (AST-021 FR5).## Known failuresA known failure names one expectation, one binding, one state, one evidence-layer, the exact standards reference, the user impact, a public issue, and why+layer, the exact standards reference, the exact failure, the user impact, and whythe migration is not the place to fix it. It still runs, it still fails, and is-reported as debt. The record matches the complete failure message, not a+reported as debt. Operational ownership stays in the project's durable gap+registry rather than in public source. The record matches the complete failure+message, not asubstring; a different message, another state, or a wider failure remains a`fail` rather than being absorbed by the record. Required failures gate;advisory failures remain report-only under AST-020 FR9. A full binding sweepalso requires every record to match exactly one executed result, so deletedstates and renamed expectations cannot orphan debt. An expectation that startspassing is reported as an unexpected pass so that exact stale record is deleted;-its issue closes only when no remaining records refer to it (AST-021 FR8–FR10).+the separately owned operational gap is reconciled only when no remaining record+refers to it (AST-021 FR8–FR10).## Runningdiff --git a/internal/a11y-spec/src/report.ts b/internal/a11y-spec/src/report.tsindex 73c54cd3dd..05a09e4df9 100644--- a/internal/a11y-spec/src/report.ts+++ b/internal/a11y-spec/src/report.ts@@ -187,16 +187,12 @@ export function formatReport(report: Report): string {for (const binding of report.bindings) {lines.push(`${binding.binding} [${binding.state}] via ${binding.harness}`);for (const result of binding.results) {- const suffix =- result.status === 'known-failure' && result.knownFailure != null- ? ` (${result.knownFailure.issue})`- : '';const failureLike =result.status === 'fail' ||result.status === 'known-failure' ||result.status === 'unexpected-pass';lines.push(- ` ${result.status.padEnd(15)} ${failureLike ? result.description : result.expectation} · ${result.evidenceLayer} · ${result.enforcement}${suffix}`,+ ` ${result.status.padEnd(15)} ${failureLike ? result.description : result.expectation} · ${result.evidenceLayer} · ${result.enforcement}`,);if (failureLike && result.detail != null) {lines.push(` ${result.detail.split('\n').join('\n ')}`);diff --git a/internal/a11y-spec/src/run.test.ts b/internal/a11y-spec/src/run.test.tsindex 69fc064a96..9a12002638 100644--- a/internal/a11y-spec/src/run.test.ts+++ b/internal/a11y-spec/src/run.test.ts@@ -96,7 +96,6 @@ function knownFailure(overrides: Partial<KnownFailure> = {}): KnownFailure {failureEquals: 'the stub outcome is missing entirely',standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact: 'The stub does nothing for the user.',- issue: 'https://github.com/facebook/astryx/issues/1',reason: 'Recorded by the migration; the fix is its own change.',...overrides,};@@ -253,7 +252,13 @@ describe('runBinding', () => {knownFailures: [knownFailure()],});expect(result.results[0]?.status).toBe('known-failure');- expect(result.results[0]?.knownFailure?.issue).toContain('issues/1');+ expect(result.results[0]?.knownFailure).toEqual(+ expect.objectContaining({+ expectation: 'probe.outcome.observed',+ binding: 'Stub',+ state: 'default',+ }),+ );expect(blockingResults([result])).toEqual([]);});@@ -348,7 +353,7 @@ describe('runBinding', () => {'remove this stale known-failure',);expect(result.results[0]?.detail).toContain(- 'close the issue only when no remaining records refer to it',+ 'reconcile its separately owned operational gap record',);expect(blockingResults([result])).toHaveLength(1);});diff --git a/internal/a11y-spec/src/run.ts b/internal/a11y-spec/src/run.tsindex 69b8357b1e..8037cbd526 100644--- a/internal/a11y-spec/src/run.ts+++ b/internal/a11y-spec/src/run.ts@@ -3,17 +3,19 @@/*** @file run.ts* @input Uses ./contract (expectations), ./harness (the runtime seam)- * @output `runBinding` — runs one pattern contract against one component- * binding state — plus the known-failure vocabulary and exact-record- * reconciliation helpers it obeys.+ * @output `runBinding` — runs one pattern contract against one component binding+ * state — plus the known-failure vocabulary and exact-record reconciliation+ * helpers it obeys.* @position The engine between a pattern and a component. Everything a report* later says about a binding is decided here.** The rules come from the accepted migration record,* `docs/specs/AST-021/spec.md`:*- * - FR8 a known failure names an expectation, a binding, a state, an evidence- * layer, the user impact, a public issue, and why it is not fixed here.+ * - FR8 a checked-in known failure names an expectation, binding, state,+ * evidence layer, exact failure, user impact, standards source, and why+ * migration does not fix it. Operational tracking stays outside public+ * source.* - FR9 a known failure changes only its own exact result. A different error,* another state, a new expectation, or a wider failure still fails, and* an expectation that starts passing is reported as an unexpected pass@@ -68,8 +70,6 @@ export interface KnownFailure {readonly standardsReference: string;/** What the person using the component actually experiences. */readonly userImpact: string;- /** Public issue tracking the fix. */- readonly issue: string;/** Why this migration records the gap instead of fixing it. */readonly reason: string;}@@ -291,7 +291,8 @@ export async function runBinding<Facts>(...base,status: 'unexpected-pass',knownFailure: record,- detail: `the recorded failure no longer happens; remove this stale known-failure record and update ${record.issue}; close the issue only when no remaining records refer to it`,+ detail:+ 'the recorded failure no longer happens; remove this stale known-failure record and reconcile its separately owned operational gap record when no other binding still refers to it',},);continue;diff --git a/packages/core/src/Button/__tests__/Button.a11y.known-failures.ts b/packages/core/src/Button/__tests__/Button.a11y.known-failures.tsindex 97cfbc1a53..0f7fe78ed1 100644--- a/packages/core/src/Button/__tests__/Button.a11y.known-failures.ts+++ b/packages/core/src/Button/__tests__/Button.a11y.known-failures.ts@@ -5,25 +5,23 @@* @input Uses KnownFailure from @astryxdesign/a11y-spec* @output BUTTON_KNOWN_FAILURES — the exact outcomes a component that adopts the* button pattern does not deliver yet, each scoped to one expectation, one- * binding, one state, and one public issue.+ * binding, one state, and one exact public-safe failure record.* @position Recorded debt, not permission. `docs/specs/AST-021/spec.md` FR8–FR10* govern this: FR8 requires every record to name the expectation, binding,- * state, user impact, standards reference, evidence layer, issue, and reason;+ * state, user impact, standards reference, evidence layer, exact failure, and+ * reason;* FR9 is the exact gate — the expectation still RUNS and reports* `known-failure`, never `pass`, and "a different error, another state, a new* expectation, or a wider failure MUST fail". When the component is fixed the* record stops matching and the run reports `unexpected-pass`, which gates, so* a fix cannot land while leaving a stale record behind.*- * A record is never a way to make a run green. It is a way to say, in public and- * in one place, precisely what is broken and where the fix is being tracked.+ * A record is never a way to make a run green. It says, in public and in one+ * place, precisely what is broken; operational ownership stays outside source.*/import type {KnownFailure} from '@astryxdesign/a11y-spec';-const BUSY_FOCUS_ISSUE = 'https://github.com/facebook/astryx/issues/4871';-const CARD_POINTER_ISSUE = 'https://github.com/facebook/astryx/issues/6132';-export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [// ---- Button and IconButton go natively disabled while busy --------------// One defect, one record per component.@@ -41,9 +39,8 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = ['WCAG 2.2 2.1.1 Keyboard and 2.1.2 No Keyboard Trap (Level A)',userImpact:'A keyboard user who activates Save and waits is dropped to the top of the document the moment the action starts, and the button leaves the tab sequence entirely — so they cannot get back to it, to see that it is busy or to interrupt it. The next Tab restarts from the beginning of the page.',- issue: BUSY_FOCUS_ISSUE,reason:- 'Button sets the native disabled attribute while a clickAction is pending. A natively disabled element is neither focusable nor a tab stop. Two fixes are already open against the issue; whichever lands turns this record into an unexpected pass.',+ 'Button sets the native disabled attribute while a clickAction is pending. A natively disabled element is neither focusable nor a tab stop. Separate implementation work will turn this record into an unexpected pass.',},{expectation: 'button.focus.reachable-and-escapable',@@ -56,7 +53,6 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = ['WCAG 2.2 2.1.1 Keyboard and 2.1.2 No Keyboard Trap (Level A)',userImpact:'The same drop from an icon button, where it is worse: an icon button is usually one of several in a row, so the user loses their place among them.',- issue: BUSY_FOCUS_ISSUE,reason:'IconButton is a thin wrapper over Button and inherits the behaviour exactly.',},@@ -72,7 +68,6 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = ['Current Astryx family:buttons FR3 and WAI-ARIA APG Button unavailable-state requirement; supports WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:'The third face of the same defect: because the busy button cannot take focus, there is no way to confirm from the keyboard that it declines a second press — the user can neither reach it nor see why it is unavailable.',- issue: BUSY_FOCUS_ISSUE,reason:'Same native disabled attribute. Recorded separately because a record names exactly one outcome, and a fix that restored focus without keeping the button inert would satisfy one of these and not the other.',},@@ -86,7 +81,6 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference:'Current Astryx family:buttons FR3 and WAI-ARIA APG Button unavailable-state requirement; supports WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact: 'The same, from an icon button.',- issue: BUSY_FOCUS_ISSUE,reason: 'Inherited from Button, as above.',},@@ -101,7 +95,6 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 2.5.2 Pointer Cancellation (Level A)',userImpact:'The same defect seen from the other side: pointer cancellation cannot be demonstrated on a control no pointer press can land on. Recorded rather than reported green — a serene pass here would claim an outcome nobody observed.',- issue: CARD_POINTER_ISSUE,reason:'Same cause as the record above. It is a second record rather than a wider one because a known failure names exactly one outcome, and these two would be fixed and verified separately.',},@@ -116,7 +109,6 @@ export const BUTTON_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = ['Current Astryx buttons family FR2 and WAI-ARIA APG Button operability requirement; supports WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:"Anyone whose tooling acts on the accessible object rather than on pixels — a speech-input user saying the card's name, or assistive technology dispatching a click at the element it is told is the button — aims at a 1×1 control the card's own content paints over. A sighted mouse user is unaffected: they click the card surface, which handles it.",- issue: CARD_POINTER_ISSUE,reason:'ClickableCard deliberately splits the surface that takes the click from the hidden button that carries the role and name. Reconciling the two is a design-system decision, not a change this contract should make.',},diff --git a/packages/core/src/CheckboxInput/__tests__/Checkbox.a11y.known-failures.ts b/packages/core/src/CheckboxInput/__tests__/Checkbox.a11y.known-failures.tsindex 020a357a01..ab4b063b00 100644--- a/packages/core/src/CheckboxInput/__tests__/Checkbox.a11y.known-failures.ts+++ b/packages/core/src/CheckboxInput/__tests__/Checkbox.a11y.known-failures.ts@@ -5,21 +5,12 @@* @input Uses KnownFailure from @astryxdesign/a11y-spec* @output CHECKBOX_KNOWN_FAILURES — exact, runnable existing gaps discovered by* the Checkbox pattern migration.- * @position Recorded public debt under AST-021 FR8–FR10, never passing coverage.+ * @position Exact public-safe known-failure records under AST-021 FR8–FR10;+ * operational gap ownership stays outside public source.*/import type {KnownFailure} from '@astryxdesign/a11y-spec';-const CHECKBOX_LIST_DESCRIPTION_ISSUE =- 'https://github.com/facebook/astryx/issues/6154';-const RICH_LABEL_NAME_ISSUE = 'https://github.com/facebook/astryx/issues/6161';-const LIST_HANDLERLESS_READONLY_ISSUE =- 'https://github.com/facebook/astryx/issues/6163';-const INPUT_HANDLERLESS_READONLY_ISSUE =- 'https://github.com/facebook/astryx/issues/6165';-const MENU_HANDLERLESS_UNAVAILABLE_ISSUE =- 'https://github.com/facebook/astryx/issues/6166';-export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [{expectation: 'checkbox.description.resolvable',@@ -31,7 +22,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 1.3.1 Info and Relationships (Level A)',userImpact:'The browser accessibility node exposes no separate description for the visible supporting text, so downstream accessibility consumers cannot distinguish it as the choice explanation.',- issue: CHECKBOX_LIST_DESCRIPTION_ISSUE,reason:'CheckboxListItem renders its description in the ListItem row but does not pass or reference that content from the nested CheckboxInput. The migration records the gap without changing component behavior.',},@@ -45,7 +35,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:'The browser computes no distinct description for the visible explanation; this records browser exposure only, not what any assistive technology announces.',- issue: CHECKBOX_LIST_DESCRIPTION_ISSUE,reason:'This is the accessibility-tree face of the missing relationship recorded above. It is separate because each known failure names exactly one expectation and layer.',},@@ -59,7 +48,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 2.5.3 Label in Name (Level A)',userImpact:'The browser exposes every rich-label item without an aria-label under the generic name "Checkbox", so speech input cannot address the item by the visible words.',- issue: RICH_LABEL_NAME_ISSUE,reason:'The public API permits a ReactNode label without an equivalent plain-text name. The migration records that supported branch without changing the component API.',},@@ -73,7 +61,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:'The browser exposes a controlled handlerless checkbox without a read-only declaration, so accessibility consumers cannot distinguish its static value from an editable setting.',- issue: INPUT_HANDLERLESS_READONLY_ISSUE,reason:'CheckboxInput makes both onChange and changeAction optional. With neither handler, the controlled value cannot persist a user change, but the component does not declare the resulting read-only state.',},@@ -87,7 +74,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:'The browser exposes a handlerless inert checkbox without a read-only declaration, so accessibility consumers cannot distinguish it from an editable control.',- issue: LIST_HANDLERLESS_READONLY_ISSUE,reason:'The public standalone API permits an item with isChecked and no onCheck. The row is inert, but CheckboxInput does not receive isReadOnly.',},@@ -101,7 +87,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = [standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',userImpact:'The menu item looks available to accessibility consumers, but activation cannot change its controlled value because it has no handler.',- issue: MENU_HANDLERLESS_UNAVAILABLE_ISSUE,reason:'DropdownMenuCheckboxItem makes onChange optional. Without it, the controlled value cannot persist a user change, but the role-bearing item remains exposed as available.',},@@ -116,7 +101,6 @@ export const CHECKBOX_KNOWN_FAILURES: ReadonlyArray<KnownFailure> = ['Astryx spec:AST-021 FR7 (preserve existing documented behavior); SelectableCard isDisabled public prop contract',userImpact:'A keyboard user cannot tab to the disabled card to discover that the option exists and is unavailable, despite the public prop contract promising continued focusability.',- issue: 'https://github.com/facebook/astryx/issues/6156',reason:'SelectableCard documents a focusable aria-disabled state, but the implementation applies native disabled to its checkbox and removes it from the tab sequence. This advisory migration check records that public-contract mismatch without presenting disabled focusability as a WCAG requirement.',},diff --git a/packages/core/src/Switch/__tests__/Switch.a11y.known-failures.ts b/packages/core/src/Switch/__tests__/Switch.a11y.known-failures.tsindex 30cc140390..c1ff37100e 100644--- a/packages/core/src/Switch/__tests__/Switch.a11y.known-failures.ts+++ b/packages/core/src/Switch/__tests__/Switch.a11y.known-failures.ts@@ -10,8 +10,8 @@** `docs/specs/AST-021/spec.md` FR8–FR10 govern this file:*- * - a record names the expectation, binding, state, evidence layer, the user- * impact, a public issue, and why the migration does not fix it;+ * - a record names the expectation, binding, state, evidence layer, exact+ * failure, user impact, and why the migration does not fix it;* - it covers only that exact failure — a different message, another state, or* a wider failure still fails the build;* - when the outcome starts passing, the run reports an unexpected pass and the