Skip to content

An open source design system that's fully customizable and agent ready

TypeScript
19.8 KB
Raw Blame
// Copyright (c) Meta Platforms, Inc. and affiliates.

/**
 * @file check.test.ts
 * @input Uses ./contract, ./check, ./report and a stub harness
 * @output Proof that a known failure changes only its own exact result, that an
 *   unobservable layer is reported rather than passed, and that the report
 *   keeps its facts apart.
 * @position Runner self-test. These are the rules a migration depends on, so
 *   they are tested directly rather than through a component.
 *
 * `docs/specs/AST-021/spec.md` FR9 is the rule under test: "A different error,
 * another state, a new expectation, or a wider failure MUST fail. When the
 * expectation starts passing, CI MUST report an unexpected pass."
 */

import {describe, expect, it, vi} from 'vitest';
import {
  definePattern,
  type ExpectationContext,
  type PatternContract,
} from './contract';
import type {EvidenceLayer, Harness, Subject} from './harness';
import {
  blockingResults,
  formatReport,
  neverExercised,
  summarize,
} from './report';
import {
  checkAccessibilitySpec,
  unmatchedKnownFailures,
  type KnownFailure,
} from './check';

interface Facts {
  readonly applicable: boolean;
}

const subject = {} as Subject;

function harness(observes: readonly EvidenceLayer[]): Harness {
  return {
    name: 'stub',
    observes,
    subject: async () => subject,
    related: async () => subject,
    contains: async () => true,
    containsSemantically: async () => true,
    references: async () => true,
    click: async () => {},
    abortedPress: async () => {},
    clickBeside: async () => {},
    hover: async () => {},
    pointerAway: async () => {},
    typeText: async () => {},
    clearText: async () => {},
    press: async () => {},
    resetFocus: async () => {},
  };
}

function contractThat(
  behaviour: (context: ExpectationContext<Facts>) => void | Promise<void>,
  overrides: {
    layer?: EvidenceLayer;
    alsoNeeds?: readonly EvidenceLayer[];
    enforcement?: 'required' | 'advisory';
  } = {},
): PatternContract<Facts> {
  return definePattern<Facts>({
    pattern: 'probe',
    url: 'https://example.invalid/probe',
    scope: 'A stub pattern used to test the runner.',
    exemptions: {},
    expectations: [
      {
        id: 'probe.outcome.observed',
        outcome: 'The stub outcome happens.',
        sources: [
          {
            standard: 'wcag',
            id: '4.1.2',
            name: 'Name, Role, Value',
            level: 'A',
            url: 'https://www.w3.org/WAI/WCAG22/Understanding/name-role-value.html',
          },
        ],
        covers: ['4.1.2-name-role-value'],
        appliesWhen: {
          condition: 'the state is applicable',
          test: facts => facts.applicable,
        },
        evidenceLayer: overrides.layer ?? 'dom',
        alsoNeeds: overrides.alsoNeeds,
        enforcement: overrides.enforcement ?? 'required',
        advisoryBecause:
          overrides.enforcement === 'advisory'
            ? 'Nothing adopts the stub outcome.'
            : undefined,
        run: async context => behaviour(context),
      },
    ],
  });
}

function knownFailure(overrides: Partial<KnownFailure> = {}): KnownFailure {
  return {
    expectation: 'probe.outcome.observed',
    binding: 'Stub',
    state: 'default',
    evidenceLayer: 'dom',
    failureEquals: 'the stub outcome is missing entirely',
    standardsReference: 'WCAG 2.2 4.1.2 Name, Role, Value (Level A)',
    userImpact: 'The stub does nothing for the user.',
    reason: 'Recorded by the migration; the fix is its own change.',
    ...overrides,
  };
}

async function run(
  spec: PatternContract<Facts>,
  options: {
    facts?: Facts;
    knownFailures?: readonly KnownFailure[];
    observes?: readonly EvidenceLayer[];
    state?: string;
    initialFocusEntry?: () => Promise<{readonly subjectWasModal: boolean}>;
    transition?: (name: string) => Promise<void> | void;
  } = {},
) {
  return checkAccessibilitySpec({
    spec,
    binding: 'Stub',
    state: options.state ?? 'default',
    facts: options.facts ?? {applicable: true},
    mount: async () => harness(options.observes ?? ['unit', 'dom']),
    knownFailures: options.knownFailures,
    initialFocusEntry: options.initialFocusEntry,
    transition: options.transition,
  });
}

const missing = () => {
  throw new Error('the stub outcome is missing entirely');
};

describe('checkAccessibilitySpec', () => {
  it('passes when the outcome happens', async () => {
    const result = await run(contractThat(() => {}));
    expect(result.results[0]?.status).toBe('pass');
    expect(blockingResults([result])).toEqual([]);
  });

  it('fails when the outcome is absent, and that blocks', async () => {
    const result = await run(contractThat(missing));
    expect(result.results[0]?.status).toBe('fail');
    expect(result.results[0]?.detail).toContain('the stub outcome is missing');
    expect(blockingResults([result])).toHaveLength(1);
  });

  it('does not run an expectation this state cannot change', async () => {
    const behaviour = vi.fn();
    const result = await run(contractThat(behaviour), {
      facts: {applicable: false},
    });
    expect(behaviour).not.toHaveBeenCalled();
    expect(result.results[0]?.status).toBe('not-applicable');
    expect(result.results[0]?.detail).toContain('applies when');
  });

  it('reports an unobservable evidence layer instead of passing or skipping it', async () => {
    const behaviour = vi.fn();
    const result = await run(
      contractThat(behaviour, {layer: 'accessibility-tree'}),
      {observes: ['unit', 'dom']},
    );
    expect(behaviour).not.toHaveBeenCalled();
    expect(result.results[0]?.status).toBe('unrun');
    expect(result.results[0]?.detail).toContain('cannot observe');
    expect(blockingResults([result])).toEqual([]);
  });

  it('reports unrun when a FURTHER layer the expectation reads is out of reach', async () => {
    const behaviour = vi.fn();
    // The shape of every interaction expectation: the claim is a real-browser
    // one, but the answer is read out of the accessibility tree. A harness with
    // the browser and no tree cannot run it, and must not fail it either.
    const result = await run(
      contractThat(behaviour, {
        layer: 'real-browser',
        alsoNeeds: ['accessibility-tree'],
      }),
      {observes: ['unit', 'dom', 'real-browser']},
    );
    expect(behaviour).not.toHaveBeenCalled();
    expect(result.results[0]?.status).toBe('unrun');
    expect(result.results[0]?.missingLayers).toEqual(['accessibility-tree']);
    expect(result.results[0]?.detail).toContain('accessibility-tree');
    expect(blockingResults([result])).toEqual([]);
  });

  it('runs once every layer it reads is observable', async () => {
    const behaviour = vi.fn();
    const result = await run(
      contractThat(behaviour, {
        layer: 'real-browser',
        alsoNeeds: ['accessibility-tree'],
      }),
      {observes: ['unit', 'dom', 'accessibility-tree', 'real-browser']},
    );
    expect(behaviour).toHaveBeenCalled();
    expect(result.results[0]?.status).toBe('pass');
  });

  it('treats a missing first-focus observation as a binding fault', async () => {
    const spec = contractThat(
      async ({initialFocusEntry}) => {
        await initialFocusEntry();
      },
      {layer: 'real-browser'},
    );

    await expect(
      run(spec, {observes: ['unit', 'dom', 'real-browser']}),
    ).rejects.toThrow('supplies no focus-entry observation');
  });

  it('passes the binding-owned first-focus observation to the expectation', async () => {
    const observed = vi.fn(async () => ({subjectWasModal: true}));
    const spec = contractThat(
      async ({initialFocusEntry}) => {
        expect(await initialFocusEntry()).toEqual({subjectWasModal: true});
      },
      {layer: 'real-browser'},
    );

    const result = await run(spec, {
      observes: ['unit', 'dom', 'real-browser'],
      initialFocusEntry: observed,
    });
    expect(observed).toHaveBeenCalledOnce();
    expect(result.results[0]?.status).toBe('pass');
  });

  it('treats a missing public transition driver as a binding fault', async () => {
    const spec = contractThat(async ({transition}) => transition('show'));

    await expect(run(spec)).rejects.toThrow(
      'requests the "show" transition, but this binding supplies no transition driver',
    );
  });

  it('passes named public transitions to the binding driver', async () => {
    const transition = vi.fn(async () => {});
    const spec = contractThat(async context => context.transition('replace'));

    const result = await run(spec, {transition});
    expect(transition).toHaveBeenCalledOnce();
    expect(transition).toHaveBeenCalledWith('replace');
    expect(result.results[0]?.status).toBe('pass');
  });

  describe('an expectation that discovers mid-run that it does not apply', () => {
    it('reports not-applicable with its reason, never a pass', async () => {
      const contract = contractThat(() => {});
      const withEscape = definePattern<Facts>({
        ...contract,
        expectations: [
          {
            ...contract.expectations[0]!,
            run: async ({notApplicable}) =>
              notApplicable('the page has nothing for this to be about'),
          },
        ],
      });
      const result = await run(withEscape);
      expect(result.results[0]?.status).toBe('not-applicable');
      expect(result.results[0]?.detail).toBe(
        'the page has nothing for this to be about',
      );
      expect(blockingResults([result])).toEqual([]);
    });

    it('does not let a known-failure record turn it into recorded debt', async () => {
      // An expectation that did not run has nothing to record a failure
      // against, so the record must neither match nor go stale.
      const contract = contractThat(() => {});
      const withEscape = definePattern<Facts>({
        ...contract,
        expectations: [
          {
            ...contract.expectations[0]!,
            run: async ({notApplicable}) => notApplicable('nothing to judge'),
          },
        ],
      });
      const result = await run(withEscape, {knownFailures: [knownFailure()]});
      expect(result.results[0]?.status).toBe('not-applicable');
      expect(result.results[0]?.knownFailure).toBeUndefined();
      expect(blockingResults([result])).toEqual([]);
    });

    it('still fails when the expectation throws an ordinary error', async () => {
      // The escape is the only thing that yields not-applicable: an expectation
      // cannot reach it by failing in some other way.
      const result = await run(contractThat(missing));
      expect(result.results[0]?.status).toBe('fail');
    });
  });

  it('reports an advisory failure without blocking', async () => {
    const result = await run(contractThat(missing, {enforcement: 'advisory'}));
    expect(result.results[0]?.status).toBe('fail');
    expect(blockingResults([result])).toEqual([]);
  });

  describe('known failures', () => {
    it('records the exact recorded failure as debt, never as a pass', async () => {
      const result = await run(contractThat(missing), {
        knownFailures: [knownFailure()],
      });
      expect(result.results[0]?.status).toBe('known-failure');
      expect(result.results[0]?.knownFailure).toEqual(
        expect.objectContaining({
          expectation: 'probe.outcome.observed',
          binding: 'Stub',
          state: 'default',
        }),
      );
      expect(blockingResults([result])).toEqual([]);
    });

    it('fails on a different failure, rather than widening to cover it', async () => {
      const result = await run(
        contractThat(() => {
          throw new Error('the stub outcome happened twice');
        }),
        {knownFailures: [knownFailure()]},
      );
      expect(result.results[0]?.status).toBe('fail');
      expect(result.results[0]?.detail).toContain(
        'it covers a different failure',
      );
      expect(blockingResults([result])).toHaveLength(1);
    });

    it('reports an advisory failure in a state the record does not name', async () => {
      const result = await run(
        contractThat(missing, {enforcement: 'advisory'}),
        {
          state: 'another-state',
          knownFailures: [knownFailure()],
        },
      );
      expect(result.results[0]?.status).toBe('fail');
      expect(blockingResults([result])).toEqual([]);
    });

    it('fails at an evidence layer the record does not name', async () => {
      const result = await run(contractThat(missing), {
        knownFailures: [knownFailure({evidenceLayer: 'real-browser'})],
      });
      expect(result.results[0]?.status).toBe('fail');
    });

    it('fails when only a substring of the recorded failure matches', async () => {
      const result = await run(contractThat(missing), {
        knownFailures: [
          knownFailure({failureEquals: 'the stub outcome is missing'}),
        ],
      });
      expect(result.results[0]?.status).toBe('fail');
      expect(blockingResults([result])).toHaveLength(1);
    });

    it('reports a different advisory failure when a known record was consulted', async () => {
      const result = await run(
        contractThat(
          () => {
            throw new Error('the stub outcome broke differently');
          },
          {enforcement: 'advisory'},
        ),
        {knownFailures: [knownFailure()]},
      );
      expect(result.results[0]?.status).toBe('fail');
      expect(result.results[0]?.knownFailure).toBeDefined();
      expect(blockingResults([result])).toEqual([]);
    });

    it('accepts a record matched by exactly one executed result', async () => {
      const record = knownFailure();
      const result = await run(contractThat(missing), {
        knownFailures: [record],
      });
      expect(unmatchedKnownFailures([record], [result])).toEqual([]);
    });

    it('reports a record orphaned by a missing state or expectation', async () => {
      const orphan = knownFailure({state: 'deleted-state'});
      const result = await run(
        contractThat(() => {}),
        {
          knownFailures: [orphan],
        },
      );
      expect(unmatchedKnownFailures([orphan], [result])).toEqual([
        expect.stringContaining('matched 0 results'),
      ]);
    });

    it('reports an unexpected pass when the recorded failure stops happening', async () => {
      const result = await run(
        contractThat(() => {}),
        {
          knownFailures: [knownFailure()],
        },
      );
      expect(result.results[0]?.status).toBe('unexpected-pass');
      expect(result.results[0]?.detail).toContain(
        'remove this stale known-failure',
      );
      expect(result.results[0]?.detail).toContain(
        'reconcile its separately owned operational gap record',
      );
      expect(blockingResults([result])).toHaveLength(1);
    });
  });

  it('lets a broken mount surface instead of absorbing it as a result', async () => {
    await expect(
      checkAccessibilitySpec({
        spec: contractThat(() => {}),
        binding: 'Stub',
        state: 'default',
        facts: {applicable: true},
        mount: async () => {
          throw new Error('the fixture never mounted');
        },
        knownFailures: [knownFailure()],
      }),
    ).rejects.toThrow('the fixture never mounted');
  });
});

describe('the report keeps its facts apart', () => {
  it('prints the source and failure detail for a report-only advisory result', async () => {
    const result = await run(contractThat(missing, {enforcement: 'advisory'}));
    const text = formatReport(
      summarize(contractThat(missing, {enforcement: 'advisory'}), [result]),
    );
    expect(text).toContain('WCAG 2.2 4.1.2 Name, Role, Value (A)');
    expect(text).toContain('the stub outcome is missing entirely');
    expect(blockingResults([result])).toEqual([]);
  });

  it('counts each status separately and quotes no score', async () => {
    const contract = contractThat(missing);
    const failing = await run(contract);
    const known = await run(contract, {
      state: 'known',
      knownFailures: [knownFailure({state: 'known'})],
    });
    const skipped = await run(contract, {facts: {applicable: false}});

    const report = summarize(contract, [failing, known, skipped]);
    expect(report.counts).toEqual({
      pass: 0,
      fail: 1,
      knownFailure: 1,
      unexpectedPass: 0,
      notApplicable: 1,
      unrun: 0,
    });

    const text = formatReport(report);
    expect(text).not.toMatch(/\d+\s*%/);
    expect(text).not.toMatch(/score/i);
    expect(text).toContain('known-failure 1');
  });

  it('names an expectation no state ever exercised', async () => {
    // A gate nothing runs is not a gate, however green the report looks.
    const base = contractThat(() => {});
    const skipped = definePattern<Facts>({
      ...base,
      expectations: [
        {
          ...base.expectations[0]!,
          appliesWhen: {condition: 'never, as it happens', test: () => false},
        },
      ],
    });
    const results = [
      await run(skipped, {state: 'one'}),
      await run(skipped, {state: 'two'}),
    ];
    expect(neverExercised(results)).toEqual(['probe.outcome.observed']);
  });

  it('says nothing when every expectation ran somewhere', async () => {
    const contract = contractThat(() => {});
    const results = [
      await run(contract, {state: 'one', facts: {applicable: false}}),
      await run(contract, {state: 'two'}),
    ];
    expect(neverExercised(results)).toEqual([]);
  });

  it('counts an unrun layer as not exercised', async () => {
    const contract = contractThat(() => {}, {layer: 'real-browser'});
    const results = [await run(contract, {observes: ['unit', 'dom']})];
    expect(neverExercised(results)).toEqual(['probe.outcome.observed']);
  });

  it('tells a part-encoded dimension apart from a fully exempt one', async () => {
    const base = contractThat(() => {});
    const contract = definePattern<Facts>({
      ...base,
      exemptions: {
        // Encoded by the stub expectation, remainder owned elsewhere.
        '4.1.2-name-role-value': {
          owner: 'the caller',
          verifiedBy: 'integration review',
          reason: 'One component cannot see the other half.',
          coversRemainderOnly: true,
        },
        // Not owned here at all.
        '2.4.7-focus-visible': {
          owner: 'the theme',
          verifiedBy: 'the visual gate',
          reason: 'A focus ring is a painted result.',
        },
      },
    });
    const report = summarize(contract, [await run(contract)]);
    const text = formatReport(report);

    expect(text).toContain(
      'part-encoded 4.1.2-name-role-value — encoded by probe.outcome.observed; remainder → the caller',
    );
    // A reader of the report alone can see WHICH half is covered.
    expect(
      report.exemptions.find(e => e.dimension === '4.1.2-name-role-value')
        ?.encodedBy,
    ).toEqual(['probe.outcome.observed']);
    expect(text).toContain('exempt 2.4.7-focus-visible → the theme');
    // The part-encoded one must not read as fully exempt.
    expect(text).not.toContain('exempt 4.1.2-name-role-value →');
    expect(
      report.exemptions.find(e => e.dimension === '4.1.2-name-role-value')
        ?.remainderOnly,
    ).toBe(true);
    expect(
      report.exemptions.find(e => e.dimension === '2.4.7-focus-visible')
        ?.remainderOnly,
    ).toBe(false);
  });

  it('names the layers that were not run, so a green run is not read as full coverage', async () => {
    const contract = contractThat(() => {}, {layer: 'real-browser'});
    const report = summarize(contract, [await run(contract)]);
    expect(report.unrunLayers).toEqual(['real-browser']);
    expect(formatReport(report)).toContain(
      'unrun evidence layers: real-browser',
    );
  });

  it('names every layer that was out of reach, not just the one the expectation is filed under', async () => {
    const contract = contractThat(() => {}, {
      layer: 'real-browser',
      alsoNeeds: ['accessibility-tree'],
    });
    const report = summarize(contract, [await run(contract)]);
    expect(report.unrunLayers).toEqual(['accessibility-tree', 'real-browser']);
  });
});