An open source design system that's fully customizable and agent ready
TypeScript
test: each --repeat-each copy of an audit keeps its own evidence folder (#9515)
Christopher Chedeau35985f2bdd05
diff --git a/internal/a11y-spec/package.json b/internal/a11y-spec/package.jsonindex 185f24aca2..1ae5311bdb 100644--- a/internal/a11y-spec/package.json+++ b/internal/a11y-spec/package.json@@ -8,6 +8,7 @@"exports": {".": "./src/index.ts","./chromium": "./src/harness/chromium.ts",+ "./evidence": "./src/evidence.ts","./playwright": "./src/playwright.ts","./storybook": "./src/storybook.ts"},diff --git a/internal/a11y-spec/src/evidence.test.ts b/internal/a11y-spec/src/evidence.test.tsnew file mode 100644index 0000000000..5d1c2fb769--- /dev/null+++ b/internal/a11y-spec/src/evidence.test.ts@@ -0,0 +1,49 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file evidence.test.ts+ * @input vitest, node:fs, resetAuditEvidence+ * @output Tests that each copy of an audit file gets its own evidence folder+ * and clears only that one+ * @position Unit coverage for evidence.ts+ */++import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {afterEach, beforeEach, describe, expect, it} from 'vitest';+import {resetAuditEvidence} from './evidence';++let root = '';++beforeEach(() => {+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'audit-evidence-'));+});++afterEach(() => {+ fs.rmSync(root, {recursive: true, force: true});+});++describe('resetAuditEvidence', () => {+ it('gives the first copy the folder itself, emptied and created', () => {+ const folder = path.join(root, 'menu-audit-evidence');+ fs.mkdirSync(folder);+ fs.writeFileSync(path.join(folder, 'stale.sensors.json'), '{}');+ expect(resetAuditEvidence(folder, {repeatEachIndex: 0})).toBe(folder);+ expect(fs.readdirSync(folder)).toEqual([]);+ });++ it('gives each further copy its own folder and leaves the others alone', () => {+ const folder = path.join(root, 'menu-audit-evidence');+ const first = resetAuditEvidence(folder, {repeatEachIndex: 0});+ fs.writeFileSync(path.join(first, 'a.sensors.json'), '{}');+ const second = resetAuditEvidence(folder, {repeatEachIndex: 1});+ fs.writeFileSync(path.join(second, 'a.sensors.json'), '{}');+ const third = resetAuditEvidence(folder, {repeatEachIndex: 2});+ expect(second).toBe(`${folder}-repeat-1`);+ expect(third).toBe(`${folder}-repeat-2`);+ expect(fs.readdirSync(first)).toEqual(['a.sensors.json']);+ expect(fs.readdirSync(second)).toEqual(['a.sensors.json']);+ expect(fs.readdirSync(third)).toEqual([]);+ });+});diff --git a/internal/a11y-spec/src/evidence.ts b/internal/a11y-spec/src/evidence.tsnew file mode 100644index 0000000000..dbf6b5903c--- /dev/null+++ b/internal/a11y-spec/src/evidence.ts@@ -0,0 +1,32 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file evidence.ts+ * @input Uses node:fs+ * @output `resetAuditEvidence`, the evidence folder for one copy of an audit+ * file, emptied and created+ * @position Binding infrastructure for the Playwright audit specs: where a+ * spec writes its screenshots and receipts.+ */++import fs from 'node:fs';++/**+ * Empty and create this copy's evidence folder, and return it. The first copy+ * of a file keeps `folder`, which CI uploads and audit pull requests read.+ * Each further `--repeat-each` copy can run at the same time in another+ * worker, so it gets `<folder>-repeat-<n>`. No copy deletes or counts+ * another copy's receipts.+ */+export function resetAuditEvidence(+ folder: string,+ copy: {readonly repeatEachIndex: number},+): string {+ const own =+ copy.repeatEachIndex === 0+ ? folder+ : `${folder}-repeat-${copy.repeatEachIndex}`;+ fs.rmSync(own, {recursive: true, force: true});+ fs.mkdirSync(own, {recursive: true});+ return own;+}diff --git a/packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts b/packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.tsindex f1e6f9ad85..52309a1e8e 100644--- a/packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts+++ b/packages/core/src/Chat/__tests__/ChatSendButton.a11y.chromium.spec.ts@@ -23,6 +23,7 @@ import {} from '@astryxdesign/a11y-spec/playwright';// @ts-expect-error -- pngjs ships no declarations; runtime support is pinned.import {PNG} from 'pngjs';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -30,7 +31,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/chat-send-button-audit-evidence');+const EVIDENCE = path.resolve('test-results/chat-send-button-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const NARROW_STORY = 'core-chatcomposer--disabled-streaming-narrow';const SUBJECT = '.astryx-chat-send-button';@@ -1265,8 +1268,7 @@ test.beforeAll(async () => {if (process.env.ASTRYX_HEAD_SHA && process.env.ASTRYX_HEAD_SHA !== head) {throw new Error('PR head differs from the checked out source');}- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());server = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,);diff --git a/packages/core/src/Chat/__tests__/ChatSystemMessage.a11y.chromium.spec.ts b/packages/core/src/Chat/__tests__/ChatSystemMessage.a11y.chromium.spec.tsindex 5eb2354ac4..f1c9f35fe9 100644--- a/packages/core/src/Chat/__tests__/ChatSystemMessage.a11y.chromium.spec.ts+++ b/packages/core/src/Chat/__tests__/ChatSystemMessage.a11y.chromium.spec.ts@@ -20,6 +20,7 @@ import {} from '@astryxdesign/a11y-spec/playwright';// @ts-expect-error -- pngjs ships no declarations; runtime support is pinned.import {PNG} from 'pngjs';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -27,7 +28,11 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/chat-system-message-audit-evidence');+const EVIDENCE = path.resolve(+ 'test-results/chat-system-message-audit-evidence',+);+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STATES_STORY = 'a11y-chatsystemmessage-audit--states';const NARROW_STORY = 'a11y-chatsystemmessage-audit--narrow';@@ -823,8 +828,7 @@ test.beforeAll(async () => {if (process.env.ASTRYX_HEAD_SHA && process.env.ASTRYX_HEAD_SHA !== head) {throw new Error('PR head differs from the checked out source');}- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());server = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,);diff --git a/packages/core/src/Chat/__tests__/ChatTokenizedText.a11y.chromium.spec.ts b/packages/core/src/Chat/__tests__/ChatTokenizedText.a11y.chromium.spec.tsindex af988e6f15..d5217aa801 100644--- a/packages/core/src/Chat/__tests__/ChatTokenizedText.a11y.chromium.spec.ts+++ b/packages/core/src/Chat/__tests__/ChatTokenizedText.a11y.chromium.spec.ts@@ -15,6 +15,7 @@ import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';// @ts-expect-error -- pngjs ships no declarations; runtime support is pinned.import {PNG} from 'pngjs';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -22,7 +23,11 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/chat-tokenized-text-audit-evidence');+const EVIDENCE = path.resolve(+ 'test-results/chat-tokenized-text-audit-evidence',+);+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const TARGET = '.astryx-chat-tokenized-text';type Mode = 'light' | 'dark';@@ -451,8 +456,7 @@ test.beforeAll(async () => {if (process.env.ASTRYX_HEAD_SHA && process.env.ASTRYX_HEAD_SHA !== head) {throw new Error('PR head differs from the checked out source');}- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());server = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,);diff --git a/packages/core/src/CommandPalette/__tests__/CommandPaletteEmptyAudit.a11y.chromium.spec.ts b/packages/core/src/CommandPalette/__tests__/CommandPaletteEmptyAudit.a11y.chromium.spec.tsindex 0d60b17425..f1514fbc17 100644--- a/packages/core/src/CommandPalette/__tests__/CommandPaletteEmptyAudit.a11y.chromium.spec.ts+++ b/packages/core/src/CommandPalette/__tests__/CommandPaletteEmptyAudit.a11y.chromium.spec.ts@@ -21,6 +21,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -28,7 +29,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('command-palette-empty-audit-evidence');+const EVIDENCE = path.resolve('command-palette-empty-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};const QUERY = 'zzz';@@ -340,8 +343,7 @@ async function readSettledRender(page: Page, subject: Locator) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/CommandPalette/__tests__/CommandPaletteInputAudit.a11y.chromium.spec.ts b/packages/core/src/CommandPalette/__tests__/CommandPaletteInputAudit.a11y.chromium.spec.tsindex 473d972706..cec604ab06 100644--- a/packages/core/src/CommandPalette/__tests__/CommandPaletteInputAudit.a11y.chromium.spec.ts+++ b/packages/core/src/CommandPalette/__tests__/CommandPaletteInputAudit.a11y.chromium.spec.ts@@ -18,6 +18,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -25,7 +26,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('command-palette-input-audit-evidence');+const EVIDENCE = path.resolve('command-palette-input-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -146,8 +149,7 @@ async function focusFromPointer(input: Locator) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/CommandPalette/__tests__/CommandPaletteItemAudit.a11y.chromium.spec.ts b/packages/core/src/CommandPalette/__tests__/CommandPaletteItemAudit.a11y.chromium.spec.tsindex 076167b6b3..4ba0c8975d 100644--- a/packages/core/src/CommandPalette/__tests__/CommandPaletteItemAudit.a11y.chromium.spec.ts+++ b/packages/core/src/CommandPalette/__tests__/CommandPaletteItemAudit.a11y.chromium.spec.ts@@ -21,6 +21,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -28,7 +29,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('command-palette-item-audit-evidence');+const EVIDENCE = path.resolve('command-palette-item-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -202,8 +205,7 @@ async function highlightFromKeyboard(}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/ComplexSelector/__tests__/ComplexSelector.a11y.chromium.spec.ts b/packages/core/src/ComplexSelector/__tests__/ComplexSelector.a11y.chromium.spec.tsindex b6c4389a77..34570d747e 100644--- a/packages/core/src/ComplexSelector/__tests__/ComplexSelector.a11y.chromium.spec.ts+++ b/packages/core/src/ComplexSelector/__tests__/ComplexSelector.a11y.chromium.spec.ts@@ -21,6 +21,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -28,7 +29,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/complex-selector-audit-evidence');+const EVIDENCE = path.resolve('test-results/complex-selector-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 900};const NARROW = {width: 320, height: 720};const SHORT = {width: 1024, height: 360};@@ -492,8 +495,7 @@ async function readPopupScroll(page: Page, contentId: string | null) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/ContextMenu/__tests__/ContextMenuAudit.a11y.chromium.spec.ts b/packages/core/src/ContextMenu/__tests__/ContextMenuAudit.a11y.chromium.spec.tsindex 7d1c0a198a..047f3b5279 100644--- a/packages/core/src/ContextMenu/__tests__/ContextMenuAudit.a11y.chromium.spec.ts+++ b/packages/core/src/ContextMenu/__tests__/ContextMenuAudit.a11y.chromium.spec.ts@@ -18,6 +18,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -25,7 +26,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('context-menu-audit-evidence');+const EVIDENCE = path.resolve('context-menu-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -283,8 +286,7 @@ async function openSurface(}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Drawer/__tests__/DrawerAudit.a11y.chromium.spec.ts b/packages/core/src/Drawer/__tests__/DrawerAudit.a11y.chromium.spec.tsindex 577d36071f..bf248e7a5c 100644--- a/packages/core/src/Drawer/__tests__/DrawerAudit.a11y.chromium.spec.ts+++ b/packages/core/src/Drawer/__tests__/DrawerAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/drawer-audit-evidence');+const EVIDENCE = path.resolve('test-results/drawer-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -193,8 +196,7 @@ async function readClosedTranslateX(page: Page): Promise<number | null> {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Drawer/__tests__/DrawerHeaderAudit.a11y.chromium.spec.ts b/packages/core/src/Drawer/__tests__/DrawerHeaderAudit.a11y.chromium.spec.tsindex 3809a5e120..da0855737c 100644--- a/packages/core/src/Drawer/__tests__/DrawerHeaderAudit.a11y.chromium.spec.ts+++ b/packages/core/src/Drawer/__tests__/DrawerHeaderAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/drawer-header-audit-evidence');+const EVIDENCE = path.resolve('test-results/drawer-header-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -85,8 +88,7 @@ async function settle(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/DropdownMenu/__tests__/DropdownMenuDividerAudit.a11y.chromium.spec.ts b/packages/core/src/DropdownMenu/__tests__/DropdownMenuDividerAudit.a11y.chromium.spec.tsindex 09ddb7e6c7..a1e08d4278 100644--- a/packages/core/src/DropdownMenu/__tests__/DropdownMenuDividerAudit.a11y.chromium.spec.ts+++ b/packages/core/src/DropdownMenu/__tests__/DropdownMenuDividerAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('dropdown-menu-divider-audit-evidence');+const EVIDENCE = path.resolve('dropdown-menu-divider-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-dropdownmenudivider--menu-separator';const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -111,8 +114,7 @@ async function settle(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/DropdownMenu/__tests__/DropdownMenuGroupAudit.a11y.chromium.spec.ts b/packages/core/src/DropdownMenu/__tests__/DropdownMenuGroupAudit.a11y.chromium.spec.tsindex c98221b57e..97ddabdf8d 100644--- a/packages/core/src/DropdownMenu/__tests__/DropdownMenuGroupAudit.a11y.chromium.spec.ts+++ b/packages/core/src/DropdownMenu/__tests__/DropdownMenuGroupAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('dropdown-menu-group-audit-evidence');+const EVIDENCE = path.resolve('dropdown-menu-group-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-dropdownmenugroup--named-and-unnamed-groups';const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -111,8 +114,7 @@ async function settle(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/DropdownMenu/__tests__/DropdownMenuItemAudit.a11y.chromium.spec.ts b/packages/core/src/DropdownMenu/__tests__/DropdownMenuItemAudit.a11y.chromium.spec.tsindex c698d6b37e..25ca7544d6 100644--- a/packages/core/src/DropdownMenu/__tests__/DropdownMenuItemAudit.a11y.chromium.spec.ts+++ b/packages/core/src/DropdownMenu/__tests__/DropdownMenuItemAudit.a11y.chromium.spec.ts@@ -18,6 +18,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -25,7 +26,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('dropdown-menu-item-audit-evidence');+const EVIDENCE = path.resolve('dropdown-menu-item-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-dropdownmenuitem--item-states';/** The overlay a focused or hovered row paints: its `:focus` background in DropdownMenuItem.tsx. */const HIGHLIGHT_TOKEN = '--color-overlay-hover';@@ -207,8 +210,7 @@ async function settlePaint(page: Page, highlighted: Locator | null) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/DropdownMenu/__tests__/DropdownMenuRadioGroupAudit.a11y.chromium.spec.ts b/packages/core/src/DropdownMenu/__tests__/DropdownMenuRadioGroupAudit.a11y.chromium.spec.tsindex be577bf48b..efc150c20e 100644--- a/packages/core/src/DropdownMenu/__tests__/DropdownMenuRadioGroupAudit.a11y.chromium.spec.ts+++ b/packages/core/src/DropdownMenu/__tests__/DropdownMenuRadioGroupAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('dropdown-menu-radio-group-audit-evidence');+const EVIDENCE = path.resolve('dropdown-menu-radio-group-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-dropdownmenuradiogroup--controlled-selection';const WIDE = {width: 1024, height: 768};const NARROW = {width: 320, height: 720};@@ -111,8 +114,7 @@ async function settle(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Field/__tests__/FieldLabelAudit.a11y.chromium.spec.ts b/packages/core/src/Field/__tests__/FieldLabelAudit.a11y.chromium.spec.tsindex b572b4487e..11bbade5cd 100644--- a/packages/core/src/Field/__tests__/FieldLabelAudit.a11y.chromium.spec.ts+++ b/packages/core/src/Field/__tests__/FieldLabelAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('field-label-audit-evidence');+const EVIDENCE = path.resolve('field-label-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const WIDE = {width: 1024, height: 900};const NARROW = {width: 320, height: 900};@@ -78,8 +81,7 @@ async function settle(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/FieldStatus/__tests__/FieldStatusAudit.a11y.chromium.spec.ts b/packages/core/src/FieldStatus/__tests__/FieldStatusAudit.a11y.chromium.spec.tsindex 7d6c9f0044..4d446179cf 100644--- a/packages/core/src/FieldStatus/__tests__/FieldStatusAudit.a11y.chromium.spec.ts+++ b/packages/core/src/FieldStatus/__tests__/FieldStatusAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('field-status-audit-evidence');+const EVIDENCE = path.resolve('field-status-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-fieldstatus--audit-matrix';const WIDE = {width: 1024, height: 900};const NARROW = {width: 320, height: 1400};@@ -265,8 +268,7 @@ async function observe(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/FileInput/__tests__/FileInputAudit.a11y.chromium.spec.ts b/packages/core/src/FileInput/__tests__/FileInputAudit.a11y.chromium.spec.tsindex abf1c1c1cb..4303a100c2 100644--- a/packages/core/src/FileInput/__tests__/FileInputAudit.a11y.chromium.spec.ts+++ b/packages/core/src/FileInput/__tests__/FileInputAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('file-input-audit-evidence');+const EVIDENCE = path.resolve('file-input-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-fileinput--audit-matrix';const WIDE = {width: 1280, height: 1100};const NARROW = {width: 320, height: 2600};@@ -517,8 +520,7 @@ function checkCases(}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Grid/__tests__/GridSpanAudit.a11y.chromium.spec.ts b/packages/core/src/Grid/__tests__/GridSpanAudit.a11y.chromium.spec.tsindex 60b19c5e83..2d028d0137 100644--- a/packages/core/src/Grid/__tests__/GridSpanAudit.a11y.chromium.spec.ts+++ b/packages/core/src/Grid/__tests__/GridSpanAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('grid-span-audit-evidence');+const EVIDENCE = path.resolve('grid-span-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-gridspan--span-matrix';const WIDE = {width: 1024, height: 900};const NARROW = {width: 320, height: 1100};@@ -193,8 +196,7 @@ async function observe(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/HStack/__tests__/HStackAudit.a11y.chromium.spec.ts b/packages/core/src/HStack/__tests__/HStackAudit.a11y.chromium.spec.tsindex 58176c751c..6b8590d656 100644--- a/packages/core/src/HStack/__tests__/HStackAudit.a11y.chromium.spec.ts+++ b/packages/core/src/HStack/__tests__/HStackAudit.a11y.chromium.spec.ts@@ -14,13 +14,16 @@ import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {DEFAULT_STORYBOOK_DIR,serveStorybook,type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('hstack-audit-evidence');+const EVIDENCE = path.resolve('hstack-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-hstack--alignment-matrix';const WIDE = {width: 1024, height: 900};const NARROW = {width: 320, height: 900};@@ -150,8 +153,7 @@ async function observe(page: Page) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Heading/__tests__/HeadingAudit.a11y.chromium.spec.ts b/packages/core/src/Heading/__tests__/HeadingAudit.a11y.chromium.spec.tsindex 6c56e97d75..7f8f396e27 100644--- a/packages/core/src/Heading/__tests__/HeadingAudit.a11y.chromium.spec.ts+++ b/packages/core/src/Heading/__tests__/HeadingAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('heading-audit-evidence');+const EVIDENCE = path.resolve('heading-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const STORY_ID = 'core-heading--audit-matrix';const WIDE = {width: 1024, height: 1100};const NARROW = {width: 320, height: 1400};@@ -299,8 +302,7 @@ async function axHeadingLevels(page: Page): Promise<Record<string, number>> {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/HoverCard/__tests__/HoverCardAudit.a11y.chromium.spec.ts b/packages/core/src/HoverCard/__tests__/HoverCardAudit.a11y.chromium.spec.tsindex d5b3e17f81..5c5c0c72aa 100644--- a/packages/core/src/HoverCard/__tests__/HoverCardAudit.a11y.chromium.spec.ts+++ b/packages/core/src/HoverCard/__tests__/HoverCardAudit.a11y.chromium.spec.ts@@ -13,6 +13,7 @@ import {execFileSync} from 'node:child_process';import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -20,7 +21,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('hover-card-audit-evidence');+const EVIDENCE = path.resolve('hover-card-audit-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const OPEN_STORY_ID = 'core-hovercard--open-states';const INTERACTIVE_STORY_ID = 'core-hovercard--interactive-content';const WIDE = {width: 1024, height: 900};@@ -224,8 +227,7 @@ function writeReceipt(key: string, png: Buffer, receipt: object) {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());checkoutSha = execFileSync('git', ['rev-parse', 'HEAD'], {encoding: 'utf8',}).trim();diff --git a/packages/core/src/Markdown/MarkdownTable.a11y.chromium.spec.ts b/packages/core/src/Markdown/MarkdownTable.a11y.chromium.spec.tsindex 21d382ec37..1a7dfbd215 100644--- a/packages/core/src/Markdown/MarkdownTable.a11y.chromium.spec.ts+++ b/packages/core/src/Markdown/MarkdownTable.a11y.chromium.spec.ts@@ -20,13 +20,16 @@import * as fs from 'node:fs';import * as path from 'node:path';import {expect, test, type Page} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {DEFAULT_STORYBOOK_DIR,serveStorybook,type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/markdown-table-narrow');+const EVIDENCE = path.resolve('test-results/markdown-table-narrow');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;/** Reading widths: two phone-sized columns, a side panel, a roomy document. */const WIDTHS = [320, 390, 528, 1024] as const;@@ -108,8 +111,7 @@ let storybook: StaticServer;const evidence: Record<string, unknown>[] = [];test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());storybook = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,);diff --git a/packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts b/packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.tsindex dedc8dbd65..ef014ccec3 100644--- a/packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts+++ b/packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts@@ -20,6 +20,7 @@ import {type Locator,type Page,} from '@astryxdesign/a11y-spec/playwright';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {DEFAULT_STORYBOOK_DIR,serveStorybook,@@ -27,7 +28,9 @@ import {} from '@astryxdesign/a11y-spec/storybook';const STORY = 'core-markdown--heading-links';-const OUTPUT = path.resolve('test-results/markdown-heading-links');+const EVIDENCE = path.resolve('test-results/markdown-heading-links');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;const GEOMETRY_TOLERANCE_PX = 1;let storybook: StaticServer;const evidence: Record<string, unknown> = {};@@ -56,8 +59,7 @@ interface HeadingGeometry {}test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());evidence.headSha = process.env.ASTRYX_HEAD_SHA ?? null;storybook = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,diff --git a/packages/core/src/__tests__/PressedState.a11y.chromium.spec.ts b/packages/core/src/__tests__/PressedState.a11y.chromium.spec.tsindex 9d723bbac6..2a15a2911e 100644--- a/packages/core/src/__tests__/PressedState.a11y.chromium.spec.ts+++ b/packages/core/src/__tests__/PressedState.a11y.chromium.spec.ts@@ -22,6 +22,7 @@ import {import pixelmatch from 'pixelmatch';// @ts-expect-error -- pngjs ships no declarations; runtime support is pinned.import {PNG} from 'pngjs';+import {resetAuditEvidence} from '@astryxdesign/a11y-spec/evidence';import {holdMotionStill} from '@astryxdesign/a11y-spec/chromium';import {DEFAULT_STORYBOOK_DIR,@@ -29,7 +30,9 @@ import {type StaticServer,} from '@astryxdesign/a11y-spec/storybook';-const OUTPUT = path.resolve('test-results/pressed-state-evidence');+const EVIDENCE = path.resolve('test-results/pressed-state-evidence');+// This copy's evidence folder, set in beforeAll (one per --repeat-each copy).+let OUTPUT = EVIDENCE;type MountedCase = {target: Locator;@@ -81,8 +84,7 @@ let browserVersion = 'unknown';let viewport: {width: number; height: number} | null = null;test.beforeAll(async () => {- fs.rmSync(OUTPUT, {recursive: true, force: true});- fs.mkdirSync(OUTPUT, {recursive: true});+ OUTPUT = resetAuditEvidence(EVIDENCE, test.info());storybook = await serveStorybook(process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR,);