Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: use one Storybook visual-regression workflow (#6486)

Cindy Zhang1f7db5ba2f99
diff --git a/.github/scripts/ci-tooling-routing.test.mjs b/.github/scripts/ci-tooling-routing.test.mjs
index 715a530380..099f4bd241 100644
--- a/.github/scripts/ci-tooling-routing.test.mjs
+++ b/.github/scripts/ci-tooling-routing.test.mjs
@@ -98,7 +98,9 @@ describe('Node-tooling CI routing', () => {
).run;
expect(componentCheck).toContain('needs.check-scope.outputs.tooling_only');
expect(componentCheck).toContain('has_components=false');
- expect(ci.jobs['test-build'].if).toBe(BROAD_ONLY);
+ expect(ci.jobs['test-build'].if).toBe(
+ `github.event_name != 'workflow_dispatch' && ${BROAD_ONLY}`,
+ );
expect(step(ci.jobs['test-build'], 'Build the Vite plugin').run).toBe(
'pnpm -F @astryxdesign/build build',
);
@@ -194,15 +196,17 @@ describe('Node-tooling CI routing', () => {
expect(prComment.jobs.resolve.outputs.tooling_only).toContain(
'steps.identity.outputs.tooling_only',
);
- for (const name of ['invalidate', 'deploy-preview', 'comment']) {
+ for (const name of ['deploy-preview', 'comment']) {
expect(prComment.jobs[name].if, name).toContain(
"needs.resolve.outputs.tooling_only != 'true'",
);
}
- for (const name of ['spec-only-visual', 'spec-only-reconcile']) {
- expect(prComment.jobs[name].if, name).toContain(
- "needs.resolve.outputs.tooling_only == 'true'",
- );
- }
+ expect(prComment.jobs['spec-only-reconcile'].if).toContain(
+ "needs.resolve.outputs.tooling_only == 'true'",
+ );
+ expect(
+ step(prComment.jobs['spec-only-reconcile'], 'Report no visual scope').run,
+ ).toContain('canonical visual checks are not applicable');
+ expect(prComment.jobs.invalidate).toBeUndefined();
});
});
diff --git a/.github/scripts/lib/gh-pages-publisher.mjs b/.github/scripts/lib/gh-pages-publisher.mjs
index b050b81629..b8ba72a93c 100755
--- a/.github/scripts/lib/gh-pages-publisher.mjs
+++ b/.github/scripts/lib/gh-pages-publisher.mjs
@@ -1248,226 +1248,6 @@ export async function publishImmutablePath({
refuse(`could not publish ${destinationRel} after ${maxAttempts} attempts`);
}
-export async function publishVisualAcceptanceRecord({
- repository,
- token,
- tempRoot,
- remoteURL,
- pr,
- head,
- acceptedRunId,
- acceptedRunAttempt,
- approver,
- approverId,
- permission,
- effectivePermission,
- roleName,
- commentId,
- reason,
- maxAttempts = 5,
- beforePush,
-}) {
- const destinationRel = safeRelativePath(
- `visual-gate/acceptances/${pr}/${head}`,
- 'acceptance destination',
- );
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
- const checkout = checkoutPages({
- repository,
- token,
- tempRoot,
- remoteURL,
- sparsePaths: [
- 'visual-gate/baseline',
- `pr/${pr}/visual/${head}/${acceptedRunId}/${acceptedRunAttempt}`,
- destinationRel,
- ],
- prefix: 'gh-pages-acceptance-',
- });
- try {
- runNodeScript(
- ['visual-gate', 'visual-acceptance.mjs'],
- [
- 'accept',
- '--pages',
- checkout,
- '--pr',
- String(pr),
- '--head',
- String(head),
- '--run-id',
- String(acceptedRunId),
- '--run-attempt',
- String(acceptedRunAttempt),
- '--approver',
- String(approver),
- '--approver-id',
- String(approverId),
- '--permission',
- String(permission),
- '--effective-permission',
- String(effectivePermission),
- '--role-name',
- String(roleName ?? ''),
- '--comment-id',
- String(commentId),
- '--reason',
- String(reason),
- ],
- );
- const commit = commitIfNeeded(
- checkout,
- `visual acceptance: PR #${pr} at ${head}`,
- [destinationRel],
- );
- if (commit === null) {
- process.stdout.write(
- 'The current visual bundle was already accepted.\n',
- );
- return {published: false};
- }
- await beforePush?.({attempt, checkout, commit});
- const pushed = tryRun('git', [
- '-C',
- checkout,
- 'push',
- 'origin',
- PAGES_BRANCH,
- ]);
- if (pushOrRetry(pushed)) return {published: true, commit};
- } finally {
- fs.rmSync(checkout, {recursive: true, force: true});
- }
- process.stdout.write(
- `Push rejected; retrying acceptance publish (${attempt}/${maxAttempts}).\n`,
- );
- await sleep(attempt * 2000);
- }
- refuse(`could not archive visual acceptance after ${maxAttempts} attempts`);
-}
-
-export async function publishAcceptedVisualBaseline({
- repository,
- runId,
- token,
- tempRoot,
- remoteURL,
- pr,
- head,
- mergeSha,
- expectedRecordRel,
- capture,
- maxAttempts = 5,
- beforePush,
-}) {
- const scope = 'visual-gate/baseline';
- assertHoldingPublicationTurn({
- repository,
- runId,
- scope,
- token,
- tempRoot,
- remoteURL,
- });
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
- const checkout = checkoutPages({
- repository,
- token,
- tempRoot,
- remoteURL,
- sparsePaths: [
- 'visual-gate/baseline',
- `visual-gate/acceptances/${pr}/${head}`,
- ],
- prefix: 'gh-pages-baseline-',
- });
- try {
- const resolved = JSON.parse(
- runNodeScript(
- ['visual-gate', 'lib', 'promotion-identity.mjs'],
- [
- 'resolve-acceptance',
- '--pages',
- checkout,
- '--pr',
- String(pr),
- '--head',
- String(head),
- '--missing-ok',
- 'false',
- '--expected-record-rel',
- String(expectedRecordRel),
- ],
- ),
- );
- if (resolved.ok !== true) {
- output('failure_description', resolved.description);
- refuse(
- resolved.message ??
- resolved.description ??
- 'acceptance validation failed',
- );
- }
- if (resolved.deferred === true) {
- output('deferred', 'true');
- output('deferred_description', resolved.deferredDescription);
- return {deferred: true};
- }
- runNodeScript(
- ['visual-gate', 'visual-acceptance.mjs'],
- [
- 'promote',
- '--pages',
- checkout,
- '--acceptance',
- resolved.recordPath,
- '--capture',
- path.resolve(capture),
- '--merge-sha',
- String(mergeSha),
- ],
- );
- const commit = commitIfNeeded(
- checkout,
- `visual baseline: accepted PR #${pr}`,
- ['-A', 'visual-gate/baseline'],
- );
- if (commit === null) {
- process.stdout.write(
- 'Baseline already contains the accepted pixels.\n',
- );
- output('publication_confirmed', 'true');
- return {published: false};
- }
- await beforePush?.({attempt, checkout, commit});
- const pushed = tryRun('git', [
- '-C',
- checkout,
- 'push',
- 'origin',
- PAGES_BRANCH,
- ]);
- if (pushOrRetry(pushed)) {
- output('publication_confirmed', 'true');
- return {published: true, commit};
- }
- } finally {
- fs.rmSync(checkout, {recursive: true, force: true});
- }
- process.stdout.write(
- `Push rejected; retrying accepted baseline publish (${attempt}/${maxAttempts}).\n`,
- );
- await sleep(attempt * 2000);
- }
- output(
- 'failure_description',
- 'Merged pixels did not promote to the visual baseline.',
- );
- refuse(
- `could not promote the accepted baseline after ${maxAttempts} attempts`,
- );
-}
-
export async function publishManualVisualBaseline({
repository,
runId,
@@ -2293,84 +2073,6 @@ export async function compactGhPages({
refuse(`could not compact ${PAGES_BRANCH} after ${maxAttempts} attempts`);
}
-export async function publishReleaseGateReport({
- repository,
- token,
- tempRoot,
- remoteURL,
- source,
- runId,
- maxAttempts = 5,
- beforePush,
-}) {
- const sourceDir = path.resolve(source);
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
- const checkout = checkoutPages({
- repository,
- token,
- tempRoot,
- remoteURL,
- sparsePaths: [path.join('visual-gate')],
- prefix: 'gh-pages-release-gate-',
- });
- try {
- const runDir = path.join(checkout, 'visual-gate', String(runId));
- const latestDir = path.join(checkout, 'visual-gate', 'latest');
- fs.rmSync(runDir, {recursive: true, force: true});
- fs.rmSync(latestDir, {recursive: true, force: true});
- copyContents(sourceDir, runDir);
- copyContents(sourceDir, latestDir);
- pruneReleaseGateRuns(path.join(checkout, 'visual-gate'));
- const commit = commitIfNeeded(checkout, `release gate: ${runId}`, [
- '-A',
- 'visual-gate',
- ]);
- if (commit === null) {
- process.stdout.write('Nothing to publish.\n');
- return {published: false};
- }
- await beforePush?.({attempt, checkout, commit});
- const pushed = tryRun('git', [
- '-C',
- checkout,
- 'push',
- 'origin',
- PAGES_BRANCH,
- ]);
- if (pushOrRetry(pushed)) {
- process.stdout.write(
- `Published https://facebook.github.io/astryx/visual-gate/${runId}/\n`,
- );
- return {published: true, commit};
- }
- } finally {
- fs.rmSync(checkout, {recursive: true, force: true});
- }
- process.stdout.write(
- `Push rejected; retrying release-gate publish (${attempt}/${maxAttempts}).\n`,
- );
- await sleep(attempt * 2000);
- }
- refuse(
- `could not publish the release-gate report after ${maxAttempts} attempts`,
- );
-}
-
-function pruneReleaseGateRuns(visualGateRoot) {
- if (!fs.existsSync(visualGateRoot)) return;
- const runs = fs
- .readdirSync(visualGateRoot, {withFileTypes: true})
- .filter(entry => entry.isDirectory() && RUN_ID.test(entry.name))
- .map(entry => Number(entry.name))
- .sort((a, b) => a - b);
- for (const runId of runs.slice(0, Math.max(0, runs.length - 20))) {
- fs.rmSync(path.join(visualGateRoot, String(runId)), {
- recursive: true,
- force: true,
- });
- }
-}
-
export async function publishStableSite({
repository,
token,
@@ -2538,14 +2240,6 @@ export async function main(argv = process.argv.slice(2)) {
scope: 'whole-tree',
publish: () => publishStableSite({...context, source, sha}),
});
- } else if (command === 'release-gate') {
- const source = flag(argv, '--source');
- if (!source) refuse('--source is required');
- await withPublicationTurn({
- ...context,
- scope: 'visual-gate/reports',
- publish: () => publishReleaseGateReport({...context, source}),
- });
} else if (command === 'immutable-path') {
const source = flag(argv, '--source');
const destination = flag(argv, '--destination');
@@ -2564,35 +2258,6 @@ export async function main(argv = process.argv.slice(2)) {
message: flag(argv, '--message'),
}),
});
- } else if (command === 'visual-acceptance-record') {
- await withPublicationTurn({
- ...context,
- scope: 'visual-gate/acceptances',
- publish: () =>
- publishVisualAcceptanceRecord({
- ...context,
- pr: flag(argv, '--pr'),
- head: flag(argv, '--head'),
- acceptedRunId: flag(argv, '--accepted-run-id'),
- acceptedRunAttempt: flag(argv, '--accepted-run-attempt'),
- approver: flag(argv, '--approver'),
- approverId: flag(argv, '--approver-id'),
- permission: flag(argv, '--permission'),
- effectivePermission: flag(argv, '--effective-permission'),
- roleName: flag(argv, '--role-name', ''),
- commentId: flag(argv, '--comment-id'),
- reason: flag(argv, '--reason'),
- }),
- });
- } else if (command === 'visual-baseline-accepted') {
- await publishAcceptedVisualBaseline({
- ...context,
- pr: flag(argv, '--pr'),
- head: flag(argv, '--head'),
- mergeSha: flag(argv, '--merge-sha'),
- expectedRecordRel: flag(argv, '--expected-record-rel'),
- capture: flag(argv, '--capture'),
- });
} else if (command === 'visual-baseline-manual') {
await publishManualVisualBaseline({
...context,
@@ -2671,7 +2336,7 @@ export async function main(argv = process.argv.slice(2)) {
}
} else {
refuse(
- 'usage: gh-pages-publisher.mjs <enqueue|wait|release|stable-site|release-gate|immutable-path|visual-acceptance-record|visual-baseline-accepted|visual-baseline-manual|pr-preview|cleanup-previews|vibe-report|vibe-screenshots|compact>',
+ 'usage: gh-pages-publisher.mjs <enqueue|wait|release|stable-site|immutable-path|visual-baseline-manual|pr-preview|cleanup-previews|vibe-report|vibe-screenshots|compact>',
);
}
}
diff --git a/.github/scripts/lib/gh-pages-publisher.test.mjs b/.github/scripts/lib/gh-pages-publisher.test.mjs
index 20f715e0bf..5de85a2d01 100644
--- a/.github/scripts/lib/gh-pages-publisher.test.mjs
+++ b/.github/scripts/lib/gh-pages-publisher.test.mjs
@@ -14,14 +14,11 @@ import {
cleanupPreviews,
compactGhPages,
enqueuePublication,
- publishAcceptedVisualBaseline,
publishImmutablePath,
publishManualVisualBaseline,
publishPrPreview,
- publishReleaseGateReport,
publishVibeReport,
publishVibeScreenshots,
- publishVisualAcceptanceRecord,
publishStableSite,
releasePublication,
waitForPublicationTurn,
@@ -37,7 +34,6 @@ const LEGACY_LOCK = fileURLToPath(
const HEAD = 'a'.repeat(40);
const TESTED = 'b'.repeat(40);
const BASE = 'd'.repeat(40);
-const MERGE = 'c'.repeat(40);
const KEY = 'core-button--default__neutral-light';
const SHARED_HOLDER = path.join(
@@ -611,122 +607,6 @@ describe('gh-pages publisher', () => {
expect(git(final, 'rev-list', '--count', 'HEAD')).toBe('1');
});
- it('publishes release-gate reports without touching the stable site or baseline', async () => {
- const fx = fixture();
- const report = path.join(fx.root, 'report');
- writeFile(path.join(report, 'index.html'), 'new report');
- writeFile(
- path.join(report, 'release-gate.json'),
- '{"visual":{"status":"passed"}}\n',
- );
-
- await publishReleaseGateReport({
- ...context(fx, 901, 'visual-gate/reports'),
- source: report,
- runId: 901,
- });
-
- const final = cloneRemote(fx.remote, fx.root);
- expect(
- fs.readFileSync(
- path.join(final, 'visual-gate', '901', 'index.html'),
- 'utf8',
- ),
- ).toBe('new report');
- expect(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'latest', 'index.html'),
- 'utf8',
- ),
- ).toBe('new report');
- expect(
- JSON.parse(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- ).version,
- ).toBe(1);
- expect(
- fs.readFileSync(path.join(final, 'storybook', 'old.html'), 'utf8'),
- ).toBe('old storybook');
- });
-
- it('retries from the source checkout after deleting the rejected gh-pages checkout', async () => {
- const fx = fixture();
- const report = path.join(fx.root, 'report');
- writeFile(path.join(report, 'index.html'), 'retry report');
- let raced = false;
-
- await publishReleaseGateReport({
- ...context(fx, 902, 'visual-gate/reports'),
- source: report,
- runId: 902,
- beforePush: async ({attempt}) => {
- if (attempt !== 1 || raced) return;
- raced = true;
- const writer = cloneRemote(fx.remote, fx.root, 'race-writer');
- git(writer, 'config', 'user.name', 'Test');
- git(writer, 'config', 'user.email', '[email protected]');
- writeFile(path.join(writer, 'visual-gate', 'race.txt'), 'race');
- git(writer, 'add', '.');
- git(writer, 'commit', '-qm', 'race');
- git(writer, 'push', '-q', 'origin', 'gh-pages');
- },
- });
-
- expect(raced).toBe(true);
- const final = cloneRemote(fx.remote, fx.root);
- expect(
- fs.readFileSync(path.join(final, 'visual-gate', 'race.txt'), 'utf8'),
- ).toBe('race');
- expect(
- fs.readFileSync(
- path.join(final, 'visual-gate', '902', 'index.html'),
- 'utf8',
- ),
- ).toBe('retry report');
- });
-
- it('prunes only old release-gate run directories', async () => {
- const fx = fixture();
- const seed = cloneRemote(fx.remote, fx.root, 'seed-more-runs');
- git(seed, 'config', 'user.name', 'Test');
- git(seed, 'config', 'user.email', '[email protected]');
- for (let runId = 100; runId <= 125; runId += 1) {
- writeFile(
- path.join(seed, 'visual-gate', String(runId), 'index.html'),
- String(runId),
- );
- }
- writeFile(
- path.join(seed, '.astryx-gh-pages', 'publication-queue', '700.json'),
- '{"version":1,"repository":"facebook/astryx","runId":700,"scope":"whole-tree"}\n',
- );
- git(seed, 'add', '.');
- git(seed, 'commit', '-qm', 'seed runs');
- git(seed, 'push', '-q', 'origin', 'gh-pages');
- const report = path.join(fx.root, 'report-prune');
- writeFile(path.join(report, 'index.html'), 'new');
-
- await publishReleaseGateReport({
- ...context(fx, 126, 'visual-gate/reports'),
- source: report,
- runId: 126,
- });
-
- const final = cloneRemote(fx.remote, fx.root);
- expect(fs.existsSync(path.join(final, 'visual-gate', '100'))).toBe(false);
- expect(fs.existsSync(path.join(final, 'visual-gate', '107'))).toBe(true);
- expect(fs.existsSync(path.join(final, 'visual-gate', 'baseline'))).toBe(
- true,
- );
- expect(fs.existsSync(path.join(final, 'visual-gate', 'latest'))).toBe(true);
- expect(
- fs.existsSync(path.join(final, '.astryx-gh-pages', 'publication-queue')),
- ).toBe(true);
- });
-
it('does not claim either holder while waiting for an older legacy baseline ticket', async () => {
const fx = fixture();
writeLegacyTicket(fx.remote, fx.root, 899);
@@ -1031,78 +911,9 @@ describe('gh-pages publisher', () => {
).rejects.toThrow(/immutable destination already exists/);
});
- it('archives acceptance records through the shared queue', async () => {
+ it('publishes reviewed baseline bytes only while holding the shared turn', async () => {
const fx = fixture();
- await queuedPublish(fx, 920, 'visual-gate/acceptances', () =>
- publishVisualAcceptanceRecord({
- ...context(fx, 920, 'visual-gate/acceptances'),
- pr: 42,
- head: HEAD,
- acceptedRunId: 123,
- acceptedRunAttempt: 1,
- approver: 'maintainer',
- approverId: 99,
- permission: 'maintain',
- effectivePermission: 'maintain',
- roleName: '',
- commentId: 1234,
- reason: 'The new radius matches the approved component design.',
- }),
- );
-
- const final = cloneRemote(fx.remote, fx.root);
- const acceptance = JSON.parse(
- fs.readFileSync(
- path.join(
- final,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- '123',
- '1',
- 'acceptance.json',
- ),
- 'utf8',
- ),
- );
- expect(acceptance.keys.map(entry => entry.key)).toEqual([KEY]);
- expect(
- fs.existsSync(
- path.join(
- final,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- '123',
- '1',
- 'after',
- `${KEY}.png`,
- ),
- ),
- ).toBe(true);
- });
-
- it('promotes accepted baseline pixels only while holding the shared turn', async () => {
- const fx = fixture();
- await queuedPublish(fx, 930, 'visual-gate/acceptances', () =>
- publishVisualAcceptanceRecord({
- ...context(fx, 930, 'visual-gate/acceptances'),
- pr: 42,
- head: HEAD,
- acceptedRunId: 123,
- acceptedRunAttempt: 1,
- approver: 'maintainer',
- approverId: 99,
- permission: 'maintain',
- effectivePermission: 'maintain',
- roleName: '',
- commentId: 1234,
- reason: 'The new radius matches the approved component design.',
- }),
- );
- const capture = path.join(fx.root, 'merged-capture');
+ const capture = path.join(fx.root, 'canonical-capture');
const after = png(0, 0, 255);
writeFile(path.join(capture, 'shots', `${KEY}.png`), after);
writeJSON(path.join(capture, 'manifest.json'), {
@@ -1110,43 +921,40 @@ describe('gh-pages publisher', () => {
platform: 'linux-arm64',
browser: 'chromium-140.0',
viewport: {width: 1280, height: 900},
- context: {sha: MERGE},
+ context: {sha: TESTED},
shots: {
[KEY]: {...SHOT, key: KEY, sha256: digest(after), width: 2, height: 2},
},
});
- const turn = context(fx, 931, 'visual-gate/baseline');
- const previousPath = process.env.PATH;
- process.env.PATH = `${fx.bin}:${previousPath}`;
- try {
- await enqueuePublication(turn);
- await waitForPublicationTurn(turn);
- await publishAcceptedVisualBaseline({
- ...turn,
- pr: 42,
- head: HEAD,
- mergeSha: MERGE,
- expectedRecordRel: '123/1/acceptance.json',
- capture,
- });
- await releasePublication(turn);
- } finally {
- process.env.PATH = previousPath;
- }
-
+ writeJSON(path.join(capture, 'verdict.json'), {status: 'changed'});
+ const options = {
+ ...context(fx, 931, 'visual-gate/baseline'),
+ capture,
+ keys: KEY,
+ reason: 'The reviewed frame matches the intended component rendering.',
+ actor: 'maintainer',
+ };
+ await expect(publishManualVisualBaseline(options)).rejects.toThrow();
+ await queuedPublish(fx, 931, 'visual-gate/baseline', () =>
+ publishManualVisualBaseline(options),
+ );
const final = cloneRemote(fx.remote, fx.root);
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate/baseline/shots', `${KEY}.png`),
+ ),
+ ).toEqual(after);
const manifest = JSON.parse(
fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
+ path.join(final, 'visual-gate/baseline/manifest.json'),
'utf8',
),
);
expect(manifest.shots[KEY].sha256).toBe(digest(after));
- expect(manifest.decisions.at(-1)).toMatchObject({
- pr: 42,
- headSha: HEAD,
- mergeSha: MERGE,
- });
+ expect(manifest.decisions.at(-1).reason).toBe(options.reason);
+ expect(
+ fs.readFileSync(path.join(final, 'storybook/old.html'), 'utf8'),
+ ).toBe('old storybook');
});
it('publishes PR previews without losing reports or visual state', async () => {
diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs
index b9300ae0bd..a8bb4382a5 100644
--- a/.github/scripts/lib/pr-preview.mjs
+++ b/.github/scripts/lib/pr-preview.mjs
@@ -90,8 +90,21 @@ function identityFromPullAndRun({pull, run, baseRepository}) {
refuse('pull request targets another repository');
}
+ const sourcePull = run.pull_requests?.find(
+ candidate =>
+ Number(candidate.number) === Number(pull.number) &&
+ candidate.head?.sha === headSha,
+ );
+ // The source run records the tested base. Live main may have advanced by the
+ // time its report is published; that is not a new CI attempt. Fork runs may
+ // omit this association, so absence is explicit rather than replaced by main.
+ const testedBaseSha = sourcePull?.base?.sha
+ ? fullSha(sourcePull.base.sha, 'source pull request base')
+ : null;
+
return {
prNumber: positiveInteger(pull.number, 'pull request number'),
+ testedBaseSha,
headSha,
headRef,
headRepository,
diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs
index 6280c8ae2f..00ee450a96 100644
--- a/.github/scripts/lib/pr-preview.test.mjs
+++ b/.github/scripts/lib/pr-preview.test.mjs
@@ -46,6 +46,7 @@ function identity(overrides = {}) {
headRepositoryId: '321',
baseRepository: 'facebook/astryx',
baseSha: BASE,
+ testedBaseSha: null,
sourceRunId: 33321033727,
sourceRunAttempt: 1,
sourceConclusion: 'success',
@@ -231,6 +232,42 @@ afterEach(() => {
});
describe('trusted PR preview identity', () => {
+ it('keeps the source-run base when main advances before publication', async () => {
+ const current = identity({baseSha: 'f'.repeat(40)});
+ const {github} = githubFixture({value: current});
+ const run = {
+ ...sourceRun(current),
+ pull_requests: [
+ {number: current.prNumber, head: {sha: HEAD}, base: {sha: BASE}},
+ ],
+ };
+ const resolved = await resolveWorkflowRunPullRequest({
+ github,
+ owner: 'facebook',
+ repo: 'astryx',
+ run,
+ });
+ expect(resolved.testedBaseSha).toBe(BASE);
+ expect(resolved.baseSha).toBe(current.baseSha);
+ const prepare = PR_COMMENT_WORKFLOW.jobs.comment.steps.find(
+ step => step.name === 'Prepare canonical visual report',
+ );
+ expect(prepare.env.BASE_SHA).toBe(
+ '${{ steps.identity.outputs.tested_base_sha }}',
+ );
+ });
+
+ it('does not substitute live main when a fork run omits the base association', async () => {
+ const {github} = githubFixture();
+ const resolved = await resolveWorkflowRunPullRequest({
+ github,
+ owner: 'facebook',
+ repo: 'astryx',
+ run: sourceRun(),
+ });
+ expect(resolved.testedBaseSha).toBeNull();
+ });
+
it('resolves a fork run through the trusted owner and branch fallback', async () => {
const value = identity();
const {github} = githubFixture({value});
@@ -343,8 +380,6 @@ describe('trusted PR preview identity', () => {
.map(([name]) => name);
expect(writeCapableJobs).toEqual([
- 'invalidate',
- 'spec-only-visual',
'spec-only-reconcile',
'deploy-preview',
'comment',
diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.js
index 92185078f1..54501059c4 100644
--- a/.github/scripts/lib/visual-format.js
+++ b/.github/scripts/lib/visual-format.js
@@ -12,8 +12,7 @@ const {inline, num, safeUrl} = require('./report-text');
*
* The section has to survive four states, and the awkward one is `skipped`:
* a check that quietly vanishes on the PRs most likely to break something
- * visually is worse than no check, so a skip states its reason and points at
- * the daily gate.
+ * visually is worse than no check, so a skip states its reason explicitly.
*
* @param {object|null} verdict - the gate's verdict.json, or null when the job did not run
* @param {string} [reportUrl] - immutable published report for this run
@@ -29,10 +28,6 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
const link = safeReportUrl
? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`
: '';
- const acceptanceCommand =
- safeReportUrl && verdict.context?.runId && verdict.context?.runAttempt
- ? `\n\nA repository maintainer can accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <why every changed frame is correct>\``
- : '';
const reportBase = safeReportUrl
? `${safeReportUrl.replace(/\/+$/, '')}/`
: null;
@@ -66,7 +61,7 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
})
.join('\n')
: '';
- return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}${acceptanceCommand}\n\n${frames}\n\n`;
+ return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}\n\n${frames}\n\n`;
}
const compared = num(verdict.counts?.total);
return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).\n\n`;
@@ -115,11 +110,11 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
return `### Visual Regression
-**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}${acceptanceCommand}
+**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}
A change here is a question, not a failure: check whether the *after* is the
-picture you intended. If it is, say so in the PR — the release gate's baseline
-is updated deliberately, and this check never rewrites it.
+picture you intended. Record that review on the PR. Baseline maintenance is an
+explicit dispatch of CI; this report never rewrites the baseline or adds a release gate.
| component | story | theme | mode | pixels |
|---|---|---|---|---|
diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjs
index 26c47d4a53..c27451fec8 100644
--- a/.github/scripts/lib/visual-format.test.mjs
+++ b/.github/scripts/lib/visual-format.test.mjs
@@ -260,12 +260,8 @@ describe('buildVisualSection', () => {
'raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png',
);
expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');
- expect(section).toContain(
- 'A repository maintainer can accept these exact frames',
- );
- expect(section).toContain(
- '/accept-visual 123456/2 <why every changed frame is correct>',
- );
+ expect(section).toContain('Record that review on the PR');
+ expect(section).not.toContain('/accept-visual');
});
it('does not offer acceptance before immutable evidence is published', () => {
diff --git a/.github/scripts/spec-owner-reconcile.cjs b/.github/scripts/spec-owner-reconcile.cjs
index da7b9a4365..6552fcd9de 100644
--- a/.github/scripts/spec-owner-reconcile.cjs
+++ b/.github/scripts/spec-owner-reconcile.cjs
@@ -519,15 +519,6 @@ async function reconcileSpecOwnerGate({
return;
}
- if (scope.specOnly) {
- await createStatus({
- sha: initialHead,
- context: 'visual-acceptance',
- state: 'success',
- description: 'Spec-only change — no visual scope.',
- });
- }
-
const requiredGroups = requiredApprovalGroups(records, {
complete: scope.complete,
touchesDesignAssets: scope.touchesDesignAssets,
diff --git a/.github/scripts/spec-owner-workflow.test.mjs b/.github/scripts/spec-owner-workflow.test.mjs
index eb75337379..0241b723ed 100644
--- a/.github/scripts/spec-owner-workflow.test.mjs
+++ b/.github/scripts/spec-owner-workflow.test.mjs
@@ -217,14 +217,16 @@ describe('spec-only workflow contract', () => {
it('fails closed when file APIs are truncated or scope classification fails', () => {
const ci = read('.github/workflows/ci.yml');
- expect(ci).toContain('if: ${{ always() && !cancelled() }}');
+ expect(ci).toContain(
+ "if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() }}",
+ );
expect(ci).toContain("if: needs.check-scope.result != 'success'");
const reviewSignal = read('.github/workflows/review-signal.yml');
expect(reviewSignal).toContain('allFiles.length !== pr.changed_files');
const prComment = read('.github/workflows/pr-comment.yml');
- expect(prComment).toContain('files.length !== pr.changed_files');
+ expect(prComment).toContain('expectedCount: pull.changed_files');
});
it('runs the tested exact-head reconciler from the trusted default branch', () => {
@@ -380,8 +382,6 @@ describe('spec-only workflow contract', () => {
const workflow = read('.github/workflows/pr-comment.yml');
expect(workflow).toContain("needs.resolve.outputs.spec_only != 'true'");
expect(workflow).toContain("needs.resolve.outputs.spec_only == 'true'");
- expect(workflow).toContain(
- "context: 'visual-acceptance', state: 'success'",
- );
+ expect(workflow).not.toContain("context: 'visual-acceptance'");
});
});
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index ce6f1b6621..e3f2538d81 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -1,97 +1,25 @@
-# Visual gate
+# Storybook visual regression
-Screenshots every theming target the design system claims to support, compares
-them against an accepted baseline, and answers one question before a release
-goes out: **did anything change on screen, and did we mean it?**
+Every visual-regression test uses the shared Storybook framework and
+`.github/workflows/ci.yml` → `pr-visual` (**Stable visual regression**).
+[AST-030 FR12](../../../docs/specs/AST-030/spec.md) owns this requirement.
+Add coverage to that owner, not to a new component, package, theme, scheduled,
+post-CI, or post-merge workflow.
-Not part of PR CI — it costs minutes, not seconds. It runs once a day
-(`.github/workflows/release-gate.yml`, 06:00 PT) ahead of the 08:00 PT release
-cut, and on demand. The two-hour margin is for the a11y leg, which audits every
-story and has taken ~57 minutes; the visual leg is ~15.
+Release evidence is the constituent pull request's exact-head `pr-visual` and
+`pr-a11y` checks, as defined in the public
+[Release Process](https://github.com/facebook/astryx/wiki/Release-Process).
+There is no separate daily release gate or retroactive visual-acceptance gate.
-## Why it exists
+## Coverage and results
-A theme binds to components through theming targets: a stable class plus the
-variant and state data on it. `astryx theme build` validates that an override
-KEY exists, so a renamed target is caught at build time. Nothing catches an
-override that silently stops _painting_ — the element moves behind a wrapper,
-a state stops being reflected, the cascade order shifts. The theme still
-compiles, the component still renders, and the theme quietly stops applying.
-That failure is only visible in pixels.
+Stable Core components use representative stories and explicitly tagged stories.
+Shipped theme changes use their relevant accepted matrix. Shared stable scope
+uses the full canonical plan in the same CI owner, rather than delegating to a
+daily workflow. Packages with `astryx.canaryOnly: true` have no stable visual
+baseline; their existing unit, Storybook, accessibility, and RTL checks remain.
-## The three outcomes
-
-| status | meaning | what to do |
-| --------- | --------------------------------------------------------------- | --------------------------------------- |
-| `pass` | nothing moved | ship |
-| `changed` | pixels moved somewhere | look at the report and decide, per shot |
-| `failed` | a shot could not be captured, or the baseline is not comparable | fix the gate before trusting it |
-
-`changed` is a question, not a failure. **"The after is correct" is a valid
-answer** — a deliberate restyle _should_ move pixels. Recording that answer is
-what `visual-baseline.yml` does, and it writes who decided, when, against which
-run, and why, into the baseline's decision log.
-
-## What gets photographed
-
-Both tiers are derived, never hand-listed, so coverage tracks the system:
-
-- **surface** — one story per component in the default theme, light and dark.
- The broad regression net.
-- **theme-matrix** — for every component override a theme actually authors, a
- story that _renders that state_. A scout pass loads the candidate stories
- and reads the DOM first, so `badge` in its `warning` variant is photographed
- in a story that has one, rather than assuming the default story covers it.
- On this repo that is the difference between 200 overrides no shot could
- verify and 49.
-
-`--tiers full` widens `surface` to every story in the index (~1,770 shots).
-
-Anything the matrix still cannot reach is named in the report under **theme
-overrides that bound to nothing** — either the plan cannot get there, or the
-component stopped rendering what the theme aims at. Both are worth knowing.
-
-## Running it locally
-
-```bash
-pnpm build && pnpm -F @astryxdesign/storybook build
-
-# what would be captured, and why
-node .github/scripts/visual-gate/gate.mjs plan
-
-# capture, compare, write .visual-run/{verdict.json,report/index.html}
-node .github/scripts/visual-gate/gate.mjs check \
- --baseline .visual-baseline --out .visual-run
-
-open .visual-run/report/index.html
-```
-
-Exit codes are the contract: `0` pass, `1` crashed, `2` changed.
-
-### PR scope follows the release channel
-
-`pr-visual` compares only the stable published visual surface:
-
-- Core component change → the representative story and any explicitly opted-in
- story, but only for frame keys already present in the accepted baseline.
- `visual-baseline` opts into the default theme and `visual-theme-matrix` opts
- into every accepted theme; neither tag lets an ordinary PR create a new
- baseline key. Behavioral and audit-only fixtures stay in their dedicated
- checks without multiplying the pixel baseline.
-- Published theme change → every currently accepted visual story rendered in that
- theme. This catches a theme beginning to override a component it did not
- previously target. Theme-only plans are not charged against the focused
- component review ceiling.
-- Shared stable theming/token infrastructure → the full plan, which declines
- visibly at the 240-shot review budget and defers to the daily gate.
-- A package with `package.json.astryx.canaryOnly: true` → no visual comparison,
- no visual baseline, no capture-only smoke job. It still typechecks, unit-tests,
- builds Storybook, and publishes to canary. Experimental pixels are not a
- stable release decision and should not create a red check people learn to
- ignore.
-
-Visual scope is declared directly on the Storybook story object, next to the
-example it controls—not in a separate registry:
+Story tags declare additional coverage next to the example:
```tsx
export const CustomSeparator: Story = {
@@ -100,174 +28,93 @@ export const CustomSeparator: Story = {
};
```
-The representative story (`Default`, `Primary`, and similar conventional names)
-is selected automatically and needs no tag. Use `visual-baseline` for an
-additional default-theme contract, `visual-theme-matrix` only when that story
-must be judged in every accepted theme, and no visual tag for behavioral or
-audit-only fixtures. Ordinary PRs can update existing contracts but cannot add
-or remove baseline keys; seed or prune coverage through the manual baseline
-workflow. The existing `no-visual` tag excludes an unstable story from visual
-capture entirely.
-
-The daily gate uses the same boundary: `stableStoryPackages` in
-`visual-gate.config.json` is currently `["Core"]`, so Lab/canary stories cannot
-hold a stable release. Pass `--story-packages '*'` only for an explicit
-non-release audit.
-
-Ownership decides which stories may own canonical frames, and a Storybook title
-is only evidence of it. When the index records the source file of the component
-a story declares, that source decides: a Core component keeps its frames even
-when its story is titled under another group, and a composed demo cannot earn
-one by importing Core. `stableStoryGroups` is the fallback for the stories that
-declare no component at all, where the title group is the only signal there is.
-
-An empty plan is refused rather than reported clean, in every lane. A plan with
-no shots compares nothing, so a scope whose keys are missing from the accepted
-baseline fails closed and says so; seed the missing frames through the manual
-baseline workflow.
-
-`.github/scripts/visual-scope.mjs` owns that classification from package
-metadata; workflow YAML does not hard-code today's package names.
-
-`--sample 24` takes an even slice of the plan for a quick smoke test.
-`--observations <file>` caches the scout pass between runs.
-
-**A local baseline is for local work only.** Rendering differs by platform and
-by browser build, so a baseline captured on a Mac and a capture from an Ubuntu
-runner would read as "everything changed". The gate refuses that comparison
-instead of showing you 500 false diffs, and the shared baseline is only ever
-written by CI, from the pinned runner label.
-
-## Two different questions
-
-The gate asks two things, and only one of them is a screenshot.
-
-**Did the canonical visual contract move?** — representative and explicitly
-tagged Core stories in Neutral, plus generated Probe coverage, compared against
-an accepted baseline. Theme-specific design still renders in Storybook, but it
-does not multiply the permanent screenshot baseline.
-
-**Did each theming target's override actually reach the pixels?** — `gate.mjs
-reach`, and no baseline is involved. A pixel diff cannot answer this: when an
-override stops applying, the frame is captured broken and promoted as correct,
-and every later run agrees with it forever. The probe theme gives every
-selector a unique deterministic colour, so this is an equality test — compute
-the colour that selector should have produced, read the element, compare. It
-names the target instead of a rectangle, and it cannot flake.
-
-Three outcomes, because the difference matters:
-
-| | meaning |
-| -------- | -------------------------------------------------------------------------------- |
-| reached | the override painted |
-| shadowed | another target on the _same element_ won — a fact about the markup, not a defect |
-| missed | nothing probe-coloured won |
+The representative story needs no tag. `visual-baseline` adds a default-theme
+contract; `visual-theme-matrix` opts the story into every accepted theme.
+Behavioral/audit-only fixtures need no visual tag. `no-visual` excludes unstable
+stories. Ordinary focused PR plans use existing baseline keys; adding or pruning
+keys requires explicit full-plan baseline maintenance.
+
+- `pass`: the compared frames did not change.
+- `changed`: review the before/after/diff report and record whether the rendering
+ is intentional. A change is not automatically a regression.
+- `failed`: capture or comparison failed. Missing evidence is not passing evidence.
+- `skipped`: no comparison was performed; the reason is explicit. It does not
+ imply a separate workflow covered the change.
+
+The `visual-pr-report` artifact belongs to the exact CI run and attempt.
+`pr-comment.yml` only publishes its report: it checks identity, bounds JSON and
+image sizes, re-encodes PNGs, and renders escaped HTML from default-branch code.
+It does not execute artifact HTML, run a browser, compare pixels again, change
+a baseline, or create another visual status.
+
+Deployment previews, `a11y-weekly.yml`, `rtl-weekly.yml`, and
+`vibe-screenshots.yml` remain separate because they do not run visual regression.
+The baseline-free `gate.mjs reach` diagnostic lives with the existing `pr-a11y`
+browser checks; it tests whether Probe overrides reach their targets, not whether
+screenshots match a golden image.
+
+## Explicit baseline maintenance
+
+Maintenance uses **CI**, not another workflow. Dispatch from `main`:
+
+1. Choose `operation=capture`. The same `pr-visual` owner builds Storybook on the
+ pinned Linux runner and captures the full canonical plan: representative and
+ tagged Core stories in Neutral plus generated Probe coverage.
+2. Review that run's report and capture. A browser-version mismatch can make the
+ comparison fail while still producing a complete candidate capture; inspect
+ the rendering rather than accepting it merely to clear a check.
+3. Dispatch CI again with `operation=promote`, the reviewed `run_id` and
+ `run_attempt`, selected `keys` (or `all`), and an explicit `reason`. Choose
+ `prune` only when the full-plan removal checks permit it.
+
+Promotion verifies the source maintenance run, attempt, and artifact identity,
+then uses the shared serialized gh-pages publisher. It never recaptures or
+promotes automatically after merge. The existing accept/prune validation and
+decision log remain the write boundary. Partial captures cannot become complete
+baseline candidates. Bootstrap, coverage changes, and browser refreshes use this
+same path; none is a release gate.
+
+The shared promotion boundary accepts `pass` or `changed`. A failed comparison
+is eligible only for the validated browser-only refresh: every baseline and
+captured key is selected, platform and viewport match, every PNG matches its
+manifest hash, and there are no capture failures or removals. Missing, unreadable,
+skipped, unknown, partial, or otherwise failed evidence is refused before writes.
+
+## Local debugging
```bash
-node .github/scripts/visual-gate/gate.mjs reach
-```
-
-It runs in the daily gate and is **reported, not enforced**. Today 50 targets
-miss, from one systemic cause: StyleX emits into `@layer priority1-4`, which
-sort _after_ `astryx-theme`, so wherever a component sets a property the theme
-override loses. Failing the gate on a known systemic issue would only teach
-everyone to ignore it — enforce it once that is fixed and the count is zero.
-
-## How the release cut uses it
-
-The daily cut (08:00 PT) reads the gate's verdict before it merges the version
-bump:
-
-```
-https://facebook.github.io/astryx/visual-gate/latest/release-gate.json
+pnpm build && pnpm -F @astryxdesign/storybook build
+pnpm visual:plan
+pnpm visual:check --baseline .visual-baseline --out .visual-run
+open .visual-run/report/index.html
```
-| verdict | the cut does |
-| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| visual `pass`, a11y `clean` | cut, no comment |
-| visual `changed` | open the report and judge every change. Intentional → dispatch **Visual Baseline** with those keys and a reason, then cut, naming the accepted changes in the release note. Not intentional → hold the cut, say what regressed in chat, fix it first |
-| a11y `failed` (new violations against the baseline) | hold the cut and fix, or baseline the violation deliberately |
-| either `crashed` | hold. A gate that cannot run is not a green gate |
-
-The one thing that must not happen is a release going out through a diff nobody
-looked at — including a "changed" nobody read.
-
-## Accepting a change
-
-From the report, note the shot keys, then dispatch **Visual Baseline** with the
-gate run id, the keys (or `all`), and a reason. The promoted PNGs come out of
-that run's capture artifact, so the picture that becomes the baseline is
-exactly the picture that was reviewed.
-
-**Bootstrapping**: the first gate run has nothing to compare against — it
-reports every shot as `added` and passes. Dispatch **Visual Baseline** against
-that run with `keys=all` to make it the reference. Do the same after a
-deliberate system-wide restyle.
-
-A browser bump moves antialiasing everywhere at once; the gate detects it and
-keeps the verdict `failed` because those pixels are not comparable. After
-reviewing the capture, use `keys=all` and a browser-bump reason to refresh it.
-`accept` permits this only when the sole failure is the exact browser mismatch
-against the current baseline, the platform and viewport match, every baseline
-and captured shot is selected, and every captured PNG matches its manifest hash.
-Partial refreshes, pruning, missing captures, and any other failure are refused
-before baseline files change. If coverage changed too, restore full coverage
-and rerun the gate before refreshing; removals need a separate comparable run.
-The workflow uses this same validation inside its baseline publication turn,
-not a separate status-only check. Rerun the gate after the refresh to get a
-comparable verdict; refreshing does not turn the original failed run green.
-
-## Determinism
-
-Every shot is taken with animations and transitions forced to their end state,
-carets hidden, a fixed viewport at device scale 1, fonts awaited, **the clock
-frozen at a fixed instant in a fixed timezone**, and **all off-origin requests
-blocked** — nothing that renders may depend on a CDN being up.
-
-The frozen clock is why a story built from `new Date()` — Schedule, Calendar,
-DateRangeInput — does not report `changed` every day. Shots of those stories
-show 13 May 2026, not today; that is the capture's clock, not stale data. The
-instant lives in `lib/capture.mjs` and is recorded in every manifest. Changing
-it changes those shots, so it is a deliberate rebaseline, not a tweak.
-
-Theme and colour mode are switched over Storybook's own channel rather than
-by reloading. Two workers scout and capture independent story groups
-concurrently. Browser execution is canonicalized by story, and every story
-mounts in the default light theme before any fast-global update, so accepted
-exact plans and full release plans cannot seed mount-time state differently.
-The manifest still follows the authoritative plan order. After capture closes
-the browser, two CPU workers decode and compare the baseline PNGs.
-`--no-fast-globals` forces a reload per shot if a story's state ever turns out
-to survive the re-render.
-
-## Where the images live
-
-| what | where | size / retention |
-| ------------------------------------------- | ---------------------------------------------- | -------------------- |
-| baseline PNGs | `gh-pages:visual-gate/baseline/` | ~10 MB, permanent |
-| reports (changed shots only) | `gh-pages:visual-gate/<run_id>/` and `latest/` | last 20 runs, pruned |
-| full capture (what a promotion copies from) | Actions artifact `visual-capture` | 14 days |
-
-Nothing lands in `main`.
-
-**gh-pages is shared, and it is rebuilt as an orphan commit.** `deploy.yml`
-materializes only `storybook`, `sandbox` and `assets`, and carries every other
-path forward by SHA — `visual-gate/` survives by that mechanism, exactly as
-`pr/` and `reports/` do. **Never add `visual-gate` to that sparse-checkout
-set**: paths inside the cone are wiped and republished on every deploy, which
-would destroy the baseline and silently disarm the gate. `cleanup-previews.yml`
-only removes `pr/<number>/` and legacy 7-hex directories, so it does not touch
-it either.
-
-## Files
-
-| file | role |
-| ------------------------- | ------------------------------------------------------------------------ |
-| `gate.mjs` | CLI: `plan`, `capture`, `check`, `accept` |
-| `lib/plan.mjs` | which shots exist, and why each one does |
-| `lib/capture.mjs` | Playwright capture and the scout pass |
-| `lib/compare.mjs` | pixel comparison, targeting analysis, the verdict |
-| `lib/report.mjs` | the before / after / diff report |
-| `lib/baseline.mjs` | the baseline store and the one operation that writes it |
-| `lib/sources.mjs` | theming targets and theme overrides, from the product's own enumerations |
-| `visual-gate.config.json` | viewport, tolerances, and story exclusions with reasons |
+CLI exit codes: `0` clean, `1` crashed, `2` changed. `gate.mjs release` is the
+legacy command name for the closed full canonical plan, not a release gate.
+`gate.mjs accept` is an explicit local write; CI baseline publication uses it
+inside the serialized publication turn. Local baselines must stay local:
+platform and browser differences make them incomparable with the pinned CI
+baseline.
+
+## Determinism and storage
+
+Capture fixes the viewport, scale, clock, timezone, fonts, and animation end
+state, and blocks off-origin requests. Storybook's channel selects theme and
+color mode. Each story starts in the default light theme before switching
+variants. Canonical PNG encoding avoids treating encoding metadata as pixels.
+
+The baseline remains at `gh-pages:visual-gate/baseline/`. Immutable PR reports
+remain beside the preview at `pr/<number>/visual/<head>/<run>/<attempt>/`.
+The shared publisher, deployment, cleanup, and compaction must preserve the
+baseline and publication queue. Do not delete the `visual-gate/` subtree when
+retiring workflows; it still contains live baseline data and queue state.
+
+## Drift guard
+
+`pnpm check:visual-owner` runs inside `pnpm check:repo` and the existing lint lane.
+It rejects independent visual-regression commands, known visual runners,
+package-script aliases, named visual-regression jobs, and visual artifact
+producers outside `ci.yml`'s owner. It deliberately permits report-only downloads,
+Storybook deployment artifacts, accessibility/RTL reports, and vibe screenshots.
+This is a narrow ownership tripwire, not a shell interpreter or a blanket ban on
+Playwright, screenshots, or artifact actions.
diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.mjs
deleted file mode 100644
index 8692e10bc1..0000000000
--- a/.github/scripts/visual-gate/activate-visual-acceptance.mjs
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env node
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-/**
- * @file Activate the required visual-acceptance status without stranding PRs.
- *
- * Dry-run by default. `--apply` derives each open head's scope, backfills
- * no-scope heads as success and stable-visual heads as pending, then requires
- * the context. Re-running only replaces missing or legacy grandfathered states;
- * it never overwrites a verdict produced by the live workflow.
- */
-
-import {execFileSync} from 'node:child_process';
-import fs from 'node:fs';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-import {classifyVisualScope} from '../visual-scope.mjs';
-
-const REPO = 'facebook/astryx';
-const BRANCH = 'main';
-const CONTEXT = 'visual-acceptance';
-const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
-
-export function buildActivationPlan(openPulls, requiredContexts) {
- const unique = [...new Map(openPulls.map(pr => [pr.head.sha, pr])).values()];
- const backfill = unique
- .filter(pr => !pr.visualStatus || pr.visualStatus.description.startsWith('Grandfathered during'))
- .map(pr => ({
- number: pr.number,
- sha: pr.head.sha,
- state: pr.hasStableVisual ? 'pending' : 'success',
- description: pr.hasStableVisual
- ? 'Stable visual result pending; rerun CI after activation.'
- : 'No stable visual scope.',
- }));
- return {backfill, addRequiredContext: !requiredContexts.includes(CONTEXT)};
-}
-
-function gh(args, input) {
- return execFileSync('gh', ['api', ...args], {
- encoding: 'utf8',
- input: input === undefined ? undefined : JSON.stringify(input),
- maxBuffer: 16 * 1024 * 1024,
- });
-}
-
-function paged(endpoint) {
- return JSON.parse(gh([endpoint, '--paginate', '--slurp'])).flat();
-}
-
-function headManifests(pr, files) {
- const paths = new Set();
- for (const file of files) {
- const theme = file.match(/^packages\/themes\/([^/]+)\//);
- const pkg = file.match(/^packages\/([^/]+)\//);
- if (theme) paths.add(`packages/themes/${theme[1]}/package.json`);
- else if (pkg) paths.add(`packages/${pkg[1]}/package.json`);
- }
- const manifests = {};
- for (const manifestPath of paths) {
- try {
- const encoded = manifestPath.split('/').map(encodeURIComponent).join('/');
- const response = JSON.parse(
- gh([`repos/${pr.head.repo.full_name}/contents/${encoded}?ref=${pr.head.sha}`]),
- );
- if (!Array.isArray(response) && response.content) {
- manifests[manifestPath] = JSON.parse(Buffer.from(response.content, 'base64').toString());
- }
- } catch (error) {
- if (!String(error.stderr ?? error.message).includes('404')) throw error;
- }
- }
- return manifests;
-}
-
-function readState() {
- const pullPages = JSON.parse(
- gh([`repos/${REPO}/pulls?state=open&per_page=100`, '--paginate', '--slurp']),
- );
- const pulls = [];
- for (const pr of pullPages.flat()) {
- const combined = JSON.parse(gh([`repos/${REPO}/commits/${pr.head.sha}/status`]));
- const visualStatus = combined.statuses.find(status => status.context === CONTEXT) ?? null;
- if (visualStatus && !visualStatus.description.startsWith('Grandfathered during')) {
- pulls.push({number: pr.number, head: pr.head, visualStatus, hasStableVisual: false});
- continue;
- }
- const files = paged(`repos/${REPO}/pulls/${pr.number}/files?per_page=100`).map(
- file => file.filename,
- );
- const scope = classifyVisualScope(files, ROOT, headManifests(pr, files));
- pulls.push({...pr, visualStatus, hasStableVisual: scope.hasStableVisual});
- }
- const protection = JSON.parse(
- gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),
- );
- return {pulls, contexts: protection.contexts ?? []};
-}
-
-function apply(plan) {
- for (const {sha, number, state, description} of plan.backfill) {
- gh(
- ['--method', 'POST', `repos/${REPO}/statuses/${sha}`, '--input', '-'],
- {state, context: CONTEXT, description: `${description} (PR #${number})`},
- );
- }
- if (plan.addRequiredContext) {
- gh([
- '--method',
- 'POST',
- `repos/${REPO}/branches/${BRANCH}/protection/required_status_checks/contexts`,
- '-f',
- `contexts[]=${CONTEXT}`,
- ]);
- }
- const protection = JSON.parse(
- gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),
- );
- if (!(protection.contexts ?? []).includes(CONTEXT)) {
- throw new Error(`${CONTEXT} is still not required on ${BRANCH}`);
- }
-}
-
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const state = readState();
- const plan = buildActivationPlan(state.pulls, state.contexts);
- process.stdout.write(
- `${JSON.stringify(
- {
- repo: REPO,
- branch: BRANCH,
- context: CONTEXT,
- headsToReconcile: plan.backfill.length,
- pending: plan.backfill.filter(item => item.state === 'pending').length,
- success: plan.backfill.filter(item => item.state === 'success').length,
- addRequiredContext: plan.addRequiredContext,
- },
- null,
- 2,
- )}\n`,
- );
- if (process.argv.includes('--apply')) {
- apply(plan);
- process.stdout.write(`Activated required status ${CONTEXT} on ${BRANCH}.\n`);
- } else {
- process.stdout.write('Dry run only; pass --apply to reconcile and activate.\n');
- }
-}
diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs
deleted file mode 100644
index 63d9d07334..0000000000
--- a/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import {describe, expect, it} from 'vitest';
-
-import {buildActivationPlan} from './activate-visual-acceptance.mjs';
-
-describe('visual-acceptance activation', () => {
- it('derives pending for stable heads and success only for no-scope heads', () => {
- const plan = buildActivationPlan(
- [
- {number: 1, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},
- {number: 2, head: {sha: 'b'.repeat(40)}, hasStableVisual: false, visualStatus: null},
- {number: 3, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},
- ],
- ['build'],
- );
- expect(plan).toEqual({
- backfill: [
- {
- number: 3,
- sha: 'a'.repeat(40),
- state: 'pending',
- description: 'Stable visual result pending; rerun CI after activation.',
- },
- {
- number: 2,
- sha: 'b'.repeat(40),
- state: 'success',
- description: 'No stable visual scope.',
- },
- ],
- addRequiredContext: true,
- });
- });
-
- it('repairs legacy grandfathered states but preserves live workflow verdicts', () => {
- const plan = buildActivationPlan(
- [
- {
- number: 1,
- head: {sha: 'a'.repeat(40)},
- hasStableVisual: true,
- visualStatus: {description: 'Grandfathered during visual-acceptance activation'},
- },
- {
- number: 2,
- head: {sha: 'b'.repeat(40)},
- hasStableVisual: true,
- visualStatus: {description: 'Current visual bundle was explicitly accepted.'},
- },
- ],
- ['build', 'visual-acceptance'],
- );
- expect(plan.backfill).toEqual([
- {
- number: 1,
- sha: 'a'.repeat(40),
- state: 'pending',
- description: 'Stable visual result pending; rerun CI after activation.',
- },
- ]);
- expect(plan.addRequiredContext).toBe(false);
- });
-});
diff --git a/.github/scripts/visual-gate/authorization.mjs b/.github/scripts/visual-gate/authorization.mjs
deleted file mode 100644
index 0da5497ac7..0000000000
--- a/.github/scripts/visual-gate/authorization.mjs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-const MAINTAINER_PERMISSIONS = new Set(['maintain', 'admin']);
-const FULL_SHA = /^[0-9a-f]{40}$/;
-
-export function visualAcceptanceEvidencePath({pr, head, run, attempt}) {
- for (const [name, value] of Object.entries({pr, run, attempt})) {
- if (!Number.isSafeInteger(Number(value)) || Number(value) <= 0) {
- throw new Error(`${name} must be a positive integer`);
- }
- }
- if (!FULL_SHA.test(String(head ?? ''))) {
- throw new Error('head must be a full lowercase SHA');
- }
- return `pr/${Number(pr)}/visual/${head}/${Number(run)}/${Number(attempt)}/evidence.json`;
-}
-
-function isCapabilityObject(value) {
- return value !== null && typeof value === 'object' && !Array.isArray(value);
-}
-
-export function visualAcceptanceIdentity(permissionLevel = {}) {
- const capabilities = permissionLevel.user?.permissions;
- const effectivePermission = isCapabilityObject(capabilities)
- ? capabilities.admin === true
- ? 'admin'
- : capabilities.maintain === true
- ? 'maintain'
- : 'none'
- : 'none';
- return {
- permission:
- typeof permissionLevel.permission === 'string'
- ? permissionLevel.permission
- : 'none',
- roleName:
- typeof permissionLevel.role_name === 'string'
- ? permissionLevel.role_name
- : null,
- effectivePermission,
- };
-}
-
-export function isVisualAcceptanceEndpointMaintainer({
- effectivePermission,
-} = {}) {
- return MAINTAINER_PERMISSIONS.has(effectivePermission);
-}
-
-export function isVisualAcceptanceRecordMaintainer({
- permission,
- effectivePermission,
-} = {}) {
- return MAINTAINER_PERMISSIONS.has(
- effectivePermission === undefined ? permission : effectivePermission,
- );
-}
diff --git a/.github/scripts/visual-gate/authorization.test.mjs b/.github/scripts/visual-gate/authorization.test.mjs
deleted file mode 100644
index b20856f9cf..0000000000
--- a/.github/scripts/visual-gate/authorization.test.mjs
+++ /dev/null
@@ -1,119 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import {describe, expect, it} from 'vitest';
-
-import {
- isVisualAcceptanceEndpointMaintainer,
- isVisualAcceptanceRecordMaintainer,
- visualAcceptanceEvidencePath,
- visualAcceptanceIdentity,
-} from './authorization.mjs';
-
-const endpointAllows = payload =>
- isVisualAcceptanceEndpointMaintainer(visualAcceptanceIdentity(payload));
-
-const payload = (permission, permissions, roleName = 'Custom role') => ({
- permission,
- role_name: roleName,
- user: {permissions},
-});
-
-describe('visual acceptance evidence path', () => {
- it('addresses the exact immutable run attempt', () => {
- expect(
- visualAcceptanceEvidencePath({
- pr: 42,
- head: 'a'.repeat(40),
- run: 123456,
- attempt: 2,
- }),
- ).toBe(`pr/42/visual/${'a'.repeat(40)}/123456/2/evidence.json`);
- });
-
- it.each([
- [{pr: 0, head: 'a'.repeat(40), run: 1, attempt: 1}, /pr/],
- [{pr: 1, head: 'short', run: 1, attempt: 1}, /head/],
- [{pr: 1, head: 'a'.repeat(40), run: -1, attempt: 1}, /run/],
- [{pr: 1, head: 'a'.repeat(40), run: 1, attempt: 0}, /attempt/],
- ])('refuses an invalid identity', (identity, error) => {
- expect(() => visualAcceptanceEvidencePath(identity)).toThrow(error);
- });
-});
-
-describe('visual acceptance endpoint authorization', () => {
- it.each([
- [
- 'repository owner with effective maintain capability',
- payload('write', {maintain: true}, 'Repo Owner'),
- true,
- ],
- ['effective maintainer', payload('write', {maintain: true}), true],
- ['effective administrator', payload('write', {admin: true}), true],
- [
- 'ordinary writer',
- payload('write', {admin: false, maintain: false}),
- false,
- ],
- [
- 'top-level admin overridden by nested false capabilities',
- payload('admin', {admin: false, maintain: false}),
- false,
- ],
- [
- 'top-level maintain overridden by nested false capabilities',
- payload('maintain', {admin: false, maintain: false}),
- false,
- ],
- [
- 'repository owner role without effective capability',
- payload('write', {admin: false, maintain: false}, 'Repo Owner'),
- false,
- ],
- ['string true capability', payload('write', {maintain: 'true'}), false],
- ['string false capability', payload('write', {maintain: 'false'}), false],
- ['null capabilities', payload('admin', null), false],
- ['array capabilities', payload('admin', [true]), false],
- ['missing capability fields', payload('admin', {}), false],
- ['missing capability object', {permission: 'admin', user: {}}, false],
- ['unknown collaborator', {}, false],
- ])('%s', (_label, endpointPayload, expected) => {
- expect(endpointAllows(endpointPayload)).toBe(expected);
- });
-
- it('keeps reported permission and role as provenance', () => {
- expect(
- visualAcceptanceIdentity(
- payload('write', {maintain: true}, 'Repo Owner'),
- ),
- ).toEqual({
- permission: 'write',
- roleName: 'Repo Owner',
- effectivePermission: 'maintain',
- });
- });
-});
-
-describe('visual acceptance record compatibility', () => {
- it.each([
- ['legacy maintainer', {permission: 'maintain'}, true],
- ['legacy administrator', {permission: 'admin'}, true],
- ['legacy writer', {permission: 'write'}, false],
- [
- 'new record never falls back to raw admin',
- {permission: 'admin', effectivePermission: 'none'},
- false,
- ],
- [
- 'new effective maintainer',
- {permission: 'write', effectivePermission: 'maintain'},
- true,
- ],
- [
- 'new effective administrator',
- {permission: 'write', effectivePermission: 'admin'},
- true,
- ],
- ])('%s', (_label, record, expected) => {
- expect(isVisualAcceptanceRecordMaintainer(record)).toBe(expected);
- });
-});
diff --git a/.github/scripts/visual-gate/gate-accept.test.mjs b/.github/scripts/visual-gate/gate-accept.test.mjs
index ca0551a006..e15cebc7db 100644
--- a/.github/scripts/visual-gate/gate-accept.test.mjs
+++ b/.github/scripts/visual-gate/gate-accept.test.mjs
@@ -2,7 +2,7 @@
// `gate.mjs accept` is the promotion boundary: the one command that writes
// baseline files. These tests drive the real command against a scratch capture
-// and baseline, so what they prove is what the Visual Baseline workflow runs.
+// and baseline, so what they prove is what CI's baseline-publication job runs.
import {spawnSync} from 'node:child_process';
import {createHash} from 'node:crypto';
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index 11df2e74a1..9c23220d1f 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -422,7 +422,7 @@ async function check() {
version: 1,
status: 'skipped',
generatedAt: new Date().toISOString(),
- reason: `${measuredShots} ${scope} exceeds the ${budget}-shot budget — too broad to review shot by shot here. The daily release gate covers this change against the full baseline.`,
+ reason: `${measuredShots} ${scope} exceeds the ${budget}-shot budget — no visual comparison was performed. Run the full canonical plan through CI before relying on this scope.`,
context: {
...captureIdentity(),
headSha: process.env.ASTRYX_PR_HEAD_SHA ?? null,
@@ -530,7 +530,11 @@ async function check() {
stageReportImages({
reportDir,
- keys: comparison.changes.map(change => change.key),
+ keys: [
+ ...comparison.changes.map(change => change.key),
+ ...comparison.added,
+ ...comparison.removed,
+ ],
currentDir: outDir,
baselinePath: path.join(baselineDir, 'shots'),
});
diff --git a/.github/scripts/visual-gate/lib/promotion-identity.mjs b/.github/scripts/visual-gate/lib/promotion-identity.mjs
deleted file mode 100644
index 9b1a0b6095..0000000000
--- a/.github/scripts/visual-gate/lib/promotion-identity.mjs
+++ /dev/null
@@ -1,430 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import {execFileSync} from 'node:child_process';
-import fs from 'node:fs';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-const SHA = /^[0-9a-f]{40}$/;
-const REPO = 'facebook/astryx';
-
-const FAILURE_DESCRIPTIONS = {
- 'actions-response-invalid':
- 'Recovery refused: the CI run response was invalid.',
- 'invalid-head': 'Recovery refused: the resolved PR head was invalid.',
- 'invalid-merge': 'Recovery refused: the resolved merge commit was invalid.',
- 'invalid-pr': 'Recovery refused: the PR number was invalid.',
- 'latest-ci-mismatch':
- 'Recovery refused: acceptance is not from the latest completed CI attempt.',
- 'active-ci-retry':
- 'Visual promotion deferred: a newer CI retry is still active.',
- 'merge-not-main':
- 'Recovery refused: the merge commit is not reachable from main.',
- 'pointer-invalid':
- 'Recovery refused: the current acceptance pointer was invalid.',
- 'pointer-missing':
- 'Recovery refused: the current acceptance pointer was missing.',
- 'pr-not-main': 'Recovery refused: the PR was not merged into main.',
- 'pr-not-merged': 'Recovery refused: the PR is not merged.',
- 'record-identity-mismatch':
- 'Recovery refused: the acceptance record identity did not match.',
- 'record-missing':
- 'Recovery refused: the immutable acceptance record was missing.',
- superseded:
- 'Recovery refused: a newer acceptance superseded the reviewed record.',
- 'wrong-pr-response':
- 'Recovery refused: GitHub returned the wrong PR identity.',
-};
-
-function refuse(code, message) {
- const error = new Error(`visual promotion refused: ${message}`);
- error.code = code;
- throw error;
-}
-
-export function promotionFailure(error) {
- const code =
- typeof error?.code === 'string' && FAILURE_DESCRIPTIONS[error.code]
- ? error.code
- : 'infrastructure-failure';
- return {
- code,
- description:
- FAILURE_DESCRIPTIONS[code] ??
- 'Visual promotion validation failed before publication.',
- };
-}
-
-export function recoveryComplete({
- publicationConfirmed,
- gateOutcome,
- releaseOutcome,
-}) {
- return (
- publicationConfirmed === true &&
- gateOutcome === 'success' &&
- releaseOutcome === 'success'
- );
-}
-
-export function recoveryOperationResult({
- jobResult,
- recoveryComplete: complete,
- mutationDeferred = false,
- failureDescription = null,
-}) {
- if (
- jobResult === 'failure' ||
- jobResult === 'cancelled' ||
- Boolean(failureDescription)
- ) {
- return 'failure';
- }
- if (jobResult === 'success' && complete === true) return 'success';
- if (mutationDeferred === true && jobResult === 'success') {
- return 'deferred';
- }
- return 'failure';
-}
-
-export function promotionStatusProjection({
- headKnown,
- validationOk,
- acceptanceFound,
- promotionResult,
- mutationDeferred = false,
- deferredDescription = null,
- failureDescription,
-}) {
- if (!headKnown) return null;
- if (validationOk && !acceptanceFound) return null;
- // Terminal trusted-operation outcomes always outrank deferral. A late retry
- // must never turn a failed/cancelled publisher into a pending status.
- if (promotionResult === 'failure' || promotionResult === 'cancelled') {
- return {
- state: 'failure',
- description:
- String(failureDescription ?? '').slice(0, 140) ||
- 'Visual promotion failed; inspect the linked workflow run.',
- };
- }
- if (validationOk && acceptanceFound && promotionResult === 'success') {
- return {
- state: 'success',
- description: 'Accepted pixels promoted to the visual baseline.',
- };
- }
- if (
- validationOk &&
- acceptanceFound &&
- mutationDeferred &&
- promotionResult === 'deferred' &&
- !failureDescription
- ) {
- return {
- state: 'pending',
- description:
- String(deferredDescription ?? '').slice(0, 140) ||
- FAILURE_DESCRIPTIONS['active-ci-retry'],
- };
- }
- return {
- state: 'failure',
- description:
- String(failureDescription ?? '').slice(0, 140) ||
- 'Visual promotion failed; inspect the linked workflow run.',
- };
-}
-
-function readJSON(file) {
- try {
- return JSON.parse(fs.readFileSync(file, 'utf8'));
- } catch (error) {
- refuse('infrastructure-failure', `cannot read ${file}: ${error.message}`);
- }
-}
-
-export function resolvePullRequestIdentity({
- pull,
- requestedPr,
- repository = REPO,
- baseRef = 'main',
- compareStatus,
-}) {
- const pr = Number(requestedPr);
- if (!Number.isSafeInteger(pr) || pr <= 0)
- refuse('invalid-pr', 'invalid PR number');
- if (pull?.number !== pr)
- refuse('wrong-pr-response', `GitHub returned the wrong PR for #${pr}`);
- if (pull.state !== 'closed' || pull.merged !== true || !pull.merged_at) {
- refuse('pr-not-merged', `PR #${pr} is not merged`);
- }
- if (pull.base?.ref !== baseRef || pull.base?.repo?.full_name !== repository) {
- refuse(
- 'pr-not-main',
- `PR #${pr} was not merged into ${repository}:${baseRef}`,
- );
- }
- if (!SHA.test(pull.head?.sha ?? ''))
- refuse('invalid-head', 'resolved head SHA is invalid');
- if (!SHA.test(pull.merge_commit_sha ?? '')) {
- refuse('invalid-merge', 'resolved merge SHA is invalid');
- }
- if (!['ahead', 'identical'].includes(compareStatus)) {
- refuse(
- 'merge-not-main',
- `merge ${pull.merge_commit_sha} is not reachable from ${baseRef}`,
- );
- }
- return {
- pr,
- headSha: pull.head.sha,
- mergeSha: pull.merge_commit_sha,
- };
-}
-
-export function resolveAcceptanceIdentity({
- pages,
- pr,
- head,
- missingOk = false,
- expectedRecordRel = null,
- repository = REPO,
-}) {
- if (!Number.isSafeInteger(pr) || pr <= 0 || !SHA.test(head)) {
- refuse('invalid-head', 'invalid PR/head');
- }
- const root = path.join(pages, 'visual-gate', 'acceptances', String(pr), head);
- const pointerFile = path.join(root, 'current.json');
- if (!fs.existsSync(pointerFile)) {
- if (missingOk) return {found: false};
- refuse('pointer-missing', 'current visual acceptance pointer is missing');
- }
- const pointer = readJSON(pointerFile);
- const expectedRel = `${pointer?.run}/${pointer?.attempt}/acceptance.json`;
- if (
- pointer?.version !== 1 ||
- !Number.isSafeInteger(pointer.run) ||
- pointer.run <= 0 ||
- !Number.isSafeInteger(pointer.attempt) ||
- pointer.attempt <= 0 ||
- pointer.record !== expectedRel
- ) {
- refuse('pointer-invalid', 'current visual acceptance pointer is invalid');
- }
- if (expectedRecordRel && pointer.record !== expectedRecordRel) {
- refuse(
- 'superseded',
- 'a newer visual acceptance superseded the reviewed record',
- );
- }
- const recordPath = path.join(root, pointer.record);
- if (!fs.existsSync(recordPath))
- refuse('record-missing', 'visual acceptance record is missing');
- const record = readJSON(recordPath);
- if (
- record?.repo !== repository ||
- record.pr !== pr ||
- record.headSha !== head ||
- record.run?.id !== pointer.run ||
- record.run?.attempt !== pointer.attempt
- ) {
- refuse(
- 'record-identity-mismatch',
- 'visual acceptance record identity does not match this merged head',
- );
- }
- return {
- found: true,
- recordPath,
- recordRel: pointer.record,
- runId: pointer.run,
- runAttempt: pointer.attempt,
- };
-}
-
-export function resolveCIState(workflowRuns, accepted) {
- if (!Array.isArray(workflowRuns)) {
- refuse('actions-response-invalid', 'GitHub Actions response is invalid');
- }
- const identity = run => {
- if (
- run?.name !== 'CI' ||
- !Number.isSafeInteger(run.id) ||
- run.id <= 0 ||
- !Number.isSafeInteger(run.run_attempt) ||
- run.run_attempt <= 0
- ) {
- return null;
- }
- return {
- id: run.id,
- attempt: run.run_attempt,
- status: run.status,
- conclusion: run.conclusion ?? null,
- };
- };
- const compare = (first, second) =>
- first.id - second.id || first.attempt - second.attempt;
- const ci = workflowRuns.map(identity).filter(Boolean);
- // A cancelled retry produces no new evidence. The last non-cancelled terminal
- // attempt remains the completed identity until another attempt completes.
- const latestCompleted = ci
- .filter(run => run.status === 'completed' && run.conclusion !== 'cancelled')
- .sort(compare)
- .at(-1);
- if (
- !latestCompleted ||
- latestCompleted.id !== accepted.runId ||
- latestCompleted.attempt !== accepted.runAttempt
- ) {
- refuse(
- 'latest-ci-mismatch',
- 'accepted evidence is not from the latest completed CI attempt',
- );
- }
- const newerActive = ci
- .filter(
- run => run.status !== 'completed' && compare(run, latestCompleted) > 0,
- )
- .sort(compare)
- .at(-1);
- return {
- latestCompleted: {
- id: latestCompleted.id,
- attempt: latestCompleted.attempt,
- },
- newerActive: newerActive
- ? {
- id: newerActive.id,
- attempt: newerActive.attempt,
- status: newerActive.status,
- }
- : null,
- };
-}
-
-export function assertLatestCompletedCI(workflowRuns, accepted) {
- return resolveCIState(workflowRuns, accepted);
-}
-
-function flag(args, name) {
- const index = args.indexOf(`--${name}`);
- return index === -1 ? null : args[index + 1];
-}
-
-function githubJSON(endpoint) {
- return JSON.parse(execFileSync('gh', ['api', endpoint], {encoding: 'utf8'}));
-}
-
-export function expandCIHistory(workflowRuns, loadAttempt) {
- if (!Array.isArray(workflowRuns)) {
- refuse('actions-response-invalid', 'GitHub Actions response is invalid');
- }
- const byRun = new Map();
- for (const run of workflowRuns) {
- if (
- run?.name !== 'CI' ||
- !Number.isSafeInteger(run.id) ||
- !Number.isSafeInteger(run.run_attempt)
- ) {
- continue;
- }
- const prior = byRun.get(run.id);
- if (!prior || run.run_attempt > prior.run_attempt) byRun.set(run.id, run);
- }
- const currentRuns = [...byRun.values()].sort((a, b) => b.id - a.id);
- const history = [];
- let foundCompleted = false;
- for (const current of currentRuns) {
- history.push(current);
- if (current.status === 'completed' && current.conclusion !== 'cancelled') {
- break;
- }
- for (let attempt = current.run_attempt - 1; attempt >= 1; attempt -= 1) {
- const prior = loadAttempt(current.id, attempt);
- history.push(prior);
- if (prior.status === 'completed' && prior.conclusion !== 'cancelled') {
- foundCompleted = true;
- break;
- }
- }
- if (foundCompleted) break;
- }
- return history;
-}
-
-function loadCIHistory(head) {
- const response = githubJSON(
- `repos/${REPO}/actions/runs?event=pull_request&head_sha=${head}&per_page=100`,
- );
- return expandCIHistory(response.workflow_runs, (runId, attempt) =>
- githubJSON(`repos/${REPO}/actions/runs/${runId}/attempts/${attempt}`),
- );
-}
-
-function resolveFromGitHub(args) {
- const pages = path.resolve(flag(args, 'pages'));
- const pr = Number(flag(args, 'pr'));
- const head = flag(args, 'head') ?? '';
- const missingOkValue = flag(args, 'missing-ok') ?? 'false';
- if (!['true', 'false'].includes(missingOkValue)) {
- refuse('infrastructure-failure', 'missing-ok must be true or false');
- }
- const resolved = resolveAcceptanceIdentity({
- pages,
- pr,
- head,
- missingOk: missingOkValue === 'true',
- expectedRecordRel: flag(args, 'expected-record-rel'),
- });
- if (!resolved.found) return resolved;
-
- let runs;
- try {
- runs = loadCIHistory(head);
- } catch (error) {
- refuse(
- 'infrastructure-failure',
- `cannot resolve the latest completed CI run: ${error.message}`,
- );
- }
- const ciState = resolveCIState(runs, resolved);
- return {
- ...resolved,
- latestCompleted: ciState.latestCompleted,
- newerActive: ciState.newerActive,
- deferred: ciState.newerActive !== null,
- deferredDescription:
- ciState.newerActive === null
- ? null
- : FAILURE_DESCRIPTIONS['active-ci-retry'],
- };
-}
-
-if (
- process.argv[1] &&
- fs.realpathSync(process.argv[1]) ===
- fs.realpathSync(fileURLToPath(import.meta.url))
-) {
- try {
- const [command, ...args] = process.argv.slice(2);
- if (command !== 'resolve-acceptance') {
- refuse(
- 'infrastructure-failure',
- 'usage: promotion-identity.mjs resolve-acceptance [flags]',
- );
- }
- process.stdout.write(
- `${JSON.stringify({ok: true, ...resolveFromGitHub(args)})}\n`,
- );
- } catch (error) {
- process.stdout.write(
- `${JSON.stringify({
- ok: false,
- ...promotionFailure(error),
- message: String(error?.message ?? error).slice(0, 500),
- })}\n`,
- );
- }
-}
diff --git a/.github/scripts/visual-gate/lib/promotion-identity.test.mjs b/.github/scripts/visual-gate/lib/promotion-identity.test.mjs
deleted file mode 100644
index 33ac7033fc..0000000000
--- a/.github/scripts/visual-gate/lib/promotion-identity.test.mjs
+++ /dev/null
@@ -1,655 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-
-import {afterEach, describe, expect, it} from 'vitest';
-
-import {
- assertLatestCompletedCI,
- expandCIHistory,
- promotionFailure,
- promotionStatusProjection,
- recoveryComplete,
- recoveryOperationResult,
- resolveAcceptanceIdentity,
- resolveCIState,
- resolvePullRequestIdentity,
-} from './promotion-identity.mjs';
-
-const HEAD = 'a'.repeat(40);
-const MERGE = 'b'.repeat(40);
-const RECORD_REL = '123/2/acceptance.json';
-const roots = [];
-
-afterEach(() => {
- for (const root of roots.splice(0)) {
- fs.rmSync(root, {recursive: true, force: true});
- }
-});
-
-function pull(overrides = {}) {
- const value = {
- number: 42,
- state: 'closed',
- merged: true,
- merged_at: '2026-08-27T07:50:43Z',
- base: {ref: 'main', repo: {full_name: 'facebook/astryx'}},
- head: {sha: HEAD, ref: 'feature', repo: {id: 99}},
- merge_commit_sha: MERGE,
- };
- return {
- ...value,
- ...overrides,
- base: {...value.base, ...overrides.base},
- head: {...value.head, ...overrides.head},
- };
-}
-
-function resolvePull(value = pull(), overrides = {}) {
- return resolvePullRequestIdentity({
- pull: value,
- requestedPr: 42,
- repository: 'facebook/astryx',
- baseRef: 'main',
- compareStatus: 'ahead',
- ...overrides,
- });
-}
-
-function acceptanceFixture() {
- const pages = fs.mkdtempSync(path.join(os.tmpdir(), 'promotion-identity-'));
- roots.push(pages);
- const root = path.join(pages, 'visual-gate', 'acceptances', '42', HEAD);
- fs.mkdirSync(path.join(root, '123', '2'), {recursive: true});
- fs.writeFileSync(
- path.join(root, 'current.json'),
- `${JSON.stringify({
- version: 1,
- run: 123,
- attempt: 2,
- record: RECORD_REL,
- })}\n`,
- );
- fs.writeFileSync(
- path.join(root, RECORD_REL),
- `${JSON.stringify({
- version: 1,
- repo: 'facebook/astryx',
- pr: 42,
- headSha: HEAD,
- run: {id: 123, attempt: 2},
- })}\n`,
- );
- return {pages, root};
-}
-
-describe('promotion PR identity', () => {
- it('returns only server-resolved identity for a merge reachable from main', () => {
- expect(resolvePull()).toEqual({
- pr: 42,
- headSha: HEAD,
- mergeSha: MERGE,
- });
- });
-
- it('does not depend on a merged PR head branch still existing', () => {
- expect(resolvePull(pull({head: {repo: null, ref: null}}))).toMatchObject({
- pr: 42,
- headSha: HEAD,
- mergeSha: MERGE,
- });
- });
-
- it.each([
- [
- 'an invalid PR number',
- pull(),
- {requestedPr: 'not-a-number'},
- /invalid PR number/,
- ],
- ['the wrong PR response', pull({number: 43}), {}, /wrong PR/],
- [
- 'an open PR',
- pull({state: 'open', merged: false, merged_at: null}),
- {},
- /not merged/,
- ],
- [
- 'an unmerged closed PR',
- pull({merged: false, merged_at: null}),
- {},
- /not merged/,
- ],
- ['a non-main base', pull({base: {ref: 'release'}}), {}, /not merged into/],
- [
- 'a different base repository',
- pull({base: {repo: {full_name: 'fork/astryx'}}}),
- {},
- /not merged into/,
- ],
- [
- 'an invalid head SHA',
- pull({head: {sha: 'bad'}}),
- {},
- /head SHA is invalid/,
- ],
- [
- 'an invalid merge SHA',
- pull({merge_commit_sha: 'bad'}),
- {},
- /merge SHA is invalid/,
- ],
- [
- 'a merge not in main',
- pull(),
- {compareStatus: 'diverged'},
- /not reachable from main/,
- ],
- ])('fails closed for %s', (_name, value, overrides, error) => {
- expect(() => resolvePull(value, overrides)).toThrow(error);
- });
-});
-
-describe('promotion acceptance identity', () => {
- it('binds the current pointer to its immutable record', () => {
- const {pages} = acceptanceFixture();
- expect(
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD}),
- ).toMatchObject({
- found: true,
- recordRel: RECORD_REL,
- runId: 123,
- runAttempt: 2,
- });
- });
-
- it('allows no acceptance only for normal closed-PR promotion', () => {
- const pages = fs.mkdtempSync(path.join(os.tmpdir(), 'promotion-identity-'));
- roots.push(pages);
- expect(
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD, missingOk: true}),
- ).toEqual({found: false});
- expect(() =>
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD}),
- ).toThrow(/pointer is missing/);
- });
-
- it('rejects a stale current pointer', () => {
- const {pages} = acceptanceFixture();
- expect(() =>
- resolveAcceptanceIdentity({
- pages,
- pr: 42,
- head: HEAD,
- expectedRecordRel: '124/1/acceptance.json',
- }),
- ).toThrow(/superseded/);
- });
-
- it('rejects a malformed current pointer', () => {
- const {pages, root} = acceptanceFixture();
- fs.writeFileSync(
- path.join(root, 'current.json'),
- `${JSON.stringify({
- version: 1,
- run: 123,
- attempt: 2,
- record: '../acceptance.json',
- })}\n`,
- );
- expect(() =>
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD}),
- ).toThrow(/pointer is invalid/);
- });
-
- it('rejects a missing immutable record', () => {
- const {pages, root} = acceptanceFixture();
- fs.rmSync(path.join(root, RECORD_REL));
- expect(() =>
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD}),
- ).toThrow(/record is missing/);
- });
-
- it.each([
- ['repository', record => (record.repo = 'fork/astryx')],
- ['PR', record => (record.pr = 43)],
- ['head', record => (record.headSha = 'c'.repeat(40))],
- ['run', record => (record.run.id = 124)],
- ['attempt', record => (record.run.attempt = 1)],
- ])('rejects a record %s mismatch', (_name, mutate) => {
- const {pages, root} = acceptanceFixture();
- const file = path.join(root, RECORD_REL);
- const record = JSON.parse(fs.readFileSync(file, 'utf8'));
- mutate(record);
- fs.writeFileSync(file, `${JSON.stringify(record)}\n`);
- expect(() =>
- resolveAcceptanceIdentity({pages, pr: 42, head: HEAD}),
- ).toThrow(/record identity does not match/);
- });
-
- describe('CI attempt resolution', () => {
- const ci = (id, attempt, status, conclusion = null) => ({
- name: 'CI',
- id,
- run_attempt: attempt,
- status,
- conclusion,
- });
-
- it('loads prior attempts when the list exposes only active R/3', () => {
- const loaded = [];
- const history = expandCIHistory(
- [ci(123, 3, 'in_progress')],
- (runId, attempt) => {
- loaded.push([runId, attempt]);
- return ci(runId, attempt, 'completed', 'success');
- },
- );
- expect(loaded).toEqual([[123, 2]]);
- expect(resolveCIState(history, {runId: 123, runAttempt: 2})).toEqual({
- latestCompleted: {id: 123, attempt: 2},
- newerActive: {id: 123, attempt: 3, status: 'in_progress'},
- });
- });
-
- it('keeps completed R/2 eligible while active R/3 defers mutation', () => {
- expect(
- resolveCIState(
- [ci(123, 3, 'in_progress'), ci(123, 2, 'completed', 'success')],
- {runId: 123, runAttempt: 2},
- ),
- ).toEqual({
- latestCompleted: {id: 123, attempt: 2},
- newerActive: {id: 123, attempt: 3, status: 'in_progress'},
- });
- });
-
- it('supersedes R/2 after R/3 completes', () => {
- expect(() =>
- resolveCIState(
- [
- ci(123, 2, 'completed', 'success'),
- ci(123, 3, 'completed', 'success'),
- ],
- {runId: 123, runAttempt: 2},
- ),
- ).toThrow(/latest completed CI attempt/);
- });
-
- it('keeps R/2 eligible after R/3 is cancelled', () => {
- expect(
- resolveCIState(
- [
- ci(123, 3, 'completed', 'cancelled'),
- ci(123, 2, 'completed', 'success'),
- ],
- {runId: 123, runAttempt: 2},
- ),
- ).toEqual({
- latestCompleted: {id: 123, attempt: 2},
- newerActive: null,
- });
- });
-
- it('orders different run ids and attempts independently of API order', () => {
- const runs = [
- ci(125, 1, 'queued'),
- ci(123, 2, 'completed', 'success'),
- ci(124, 1, 'completed', 'cancelled'),
- ci(122, 9, 'completed', 'success'),
- ci(999, 1, 'completed', 'success'),
- ];
- // A non-CI workflow with a larger id must not participate.
- runs.at(-1).name = 'Lint';
- expect(
- resolveCIState(runs.reverse(), {runId: 123, runAttempt: 2}),
- ).toEqual({
- latestCompleted: {id: 123, attempt: 2},
- newerActive: {id: 125, attempt: 1, status: 'queued'},
- });
- });
-
- it('does not use the first API element as latest completed evidence', () => {
- expect(
- assertLatestCompletedCI(
- [
- ci(120, 1, 'completed', 'success'),
- ci(123, 2, 'completed', 'success'),
- ci(121, 8, 'completed', 'success'),
- ],
- {runId: 123, runAttempt: 2},
- ),
- ).toEqual({
- latestCompleted: {id: 123, attempt: 2},
- newerActive: null,
- });
- });
- });
-});
-
-function projectedFailure(run) {
- try {
- run();
- throw new Error('expected resolver refusal');
- } catch (error) {
- const failure = promotionFailure(error);
- return {
- failure,
- projection: promotionStatusProjection({
- headKnown: true,
- validationOk: false,
- acceptanceFound: false,
- promotionResult: 'skipped',
- failureDescription: failure.description,
- }),
- };
- }
-}
-
-describe('promotion status projection', () => {
- it.each([
- ['wrong PR response', () => resolvePull(pull({number: 43}))],
- [
- 'open or unmerged PR',
- () => resolvePull(pull({state: 'open', merged: false, merged_at: null})),
- ],
- ['non-main PR', () => resolvePull(pull({base: {ref: 'release'}}))],
- [
- 'merge outside main',
- () => resolvePull(pull(), {compareStatus: 'diverged'}),
- ],
- ])('keeps %s status-silent while the workflow fails', (_name, run) => {
- let failure;
- try {
- run();
- } catch (error) {
- failure = promotionFailure(error);
- }
- expect(failure).toBeDefined();
- expect(
- promotionStatusProjection({
- headKnown: false,
- validationOk: false,
- acceptanceFound: false,
- promotionResult: 'skipped',
- failureDescription: failure.description,
- }),
- ).toBeNull();
- });
-
- it.each([
- [
- 'missing pointer',
- () => {
- const pages = fs.mkdtempSync(
- path.join(os.tmpdir(), 'promotion-identity-'),
- );
- roots.push(pages);
- return resolveAcceptanceIdentity({pages, pr: 42, head: HEAD});
- },
- ],
- [
- 'stale pointer',
- () => {
- const {pages} = acceptanceFixture();
- return resolveAcceptanceIdentity({
- pages,
- pr: 42,
- head: HEAD,
- expectedRecordRel: '124/1/acceptance.json',
- });
- },
- ],
- [
- 'missing record',
- () => {
- const {pages, root} = acceptanceFixture();
- fs.rmSync(path.join(root, RECORD_REL));
- return resolveAcceptanceIdentity({pages, pr: 42, head: HEAD});
- },
- ],
- [
- 'record mismatch',
- () => {
- const {pages, root} = acceptanceFixture();
- const file = path.join(root, RECORD_REL);
- const record = JSON.parse(fs.readFileSync(file, 'utf8'));
- record.pr = 43;
- fs.writeFileSync(file, `${JSON.stringify(record)}\n`);
- return resolveAcceptanceIdentity({pages, pr: 42, head: HEAD});
- },
- ],
- [
- 'latest CI mismatch',
- () =>
- assertLatestCompletedCI(
- [{name: 'CI', id: 124, run_attempt: 1, status: 'completed'}],
- {runId: 123, runAttempt: 2},
- ),
- ],
- ])('writes a precise failure for %s', (_name, run) => {
- const {failure, projection} = projectedFailure(run);
- expect(failure.code).not.toBe('infrastructure-failure');
- expect(projection).toEqual({
- state: 'failure',
- description: failure.description,
- });
- expect(projection.description.length).toBeLessThanOrEqual(140);
- });
-
- it.each([
- ['deferred plus lock release failure', 'failure', 'Lock release failed.'],
- ['deferred plus gate dispatch failure', 'failure', 'Gate dispatch failed.'],
- ['deferred plus publication failure', 'failure', 'Publication failed.'],
- ['deferred plus failed job', 'failure', 'Trusted operation failed.'],
- [
- 'deferred plus cancelled job',
- 'cancelled',
- 'Trusted operation cancelled.',
- ],
- ])(
- 'projects %s as failure, never pending',
- (_name, promotionResult, failureDescription) => {
- expect(
- promotionStatusProjection({
- headKnown: true,
- validationOk: true,
- acceptanceFound: true,
- mutationDeferred: true,
- deferredDescription:
- 'Visual promotion deferred: a newer CI retry is still active.',
- promotionResult,
- failureDescription,
- }),
- ).toEqual({state: 'failure', description: failureDescription});
- },
- );
-
- it('composes a clean successful defer job into pending', () => {
- const promotionResult = recoveryOperationResult({
- jobResult: 'success',
- recoveryComplete: false,
- mutationDeferred: true,
- failureDescription: '',
- });
- expect(promotionResult).toBe('deferred');
- expect(
- promotionStatusProjection({
- headKnown: true,
- validationOk: true,
- acceptanceFound: true,
- mutationDeferred: true,
- deferredDescription:
- 'Visual promotion deferred: a newer CI retry is still active.',
- promotionResult,
- failureDescription: '',
- }),
- ).toEqual({
- state: 'pending',
- description:
- 'Visual promotion deferred: a newer CI retry is still active.',
- });
- });
-
- it('projects active CI retries as pending only after trusted defer succeeds', () => {
- expect(
- promotionStatusProjection({
- headKnown: true,
- validationOk: true,
- acceptanceFound: true,
- mutationDeferred: true,
- deferredDescription:
- 'Visual promotion deferred: a newer CI retry is still active.',
- promotionResult: 'deferred',
- failureDescription: '',
- }),
- ).toEqual({
- state: 'pending',
- description:
- 'Visual promotion deferred: a newer CI retry is still active.',
- });
- });
-
- it('treats an unexpected skipped promote job with acceptance as failure', () => {
- expect(
- promotionStatusProjection({
- headKnown: true,
- validationOk: true,
- acceptanceFound: true,
- mutationDeferred: true,
- deferredDescription:
- 'Visual promotion deferred: a newer CI retry is still active.',
- promotionResult: 'skipped',
- failureDescription: '',
- }),
- ).toEqual({
- state: 'failure',
- description: 'Visual promotion failed; inspect the linked workflow run.',
- });
- });
-
- it('requires the entire trusted operation before returning success', () => {
- const scenarios = [
- {
- name: 'push succeeds but gate dispatch fails',
- publicationConfirmed: true,
- gateOutcome: 'failure',
- releaseOutcome: 'success',
- jobResult: 'failure',
- expected: 'failure',
- },
- {
- name: 'push succeeds but lock release fails',
- publicationConfirmed: true,
- gateOutcome: 'success',
- releaseOutcome: 'failure',
- jobResult: 'failure',
- expected: 'failure',
- },
- {
- name: 'clean deferred operation',
- publicationConfirmed: false,
- gateOutcome: 'skipped',
- releaseOutcome: 'skipped',
- jobResult: 'success',
- mutationDeferred: true,
- expected: 'deferred',
- },
- {
- name: 'unexpected skipped promote job with accepted record',
- publicationConfirmed: false,
- gateOutcome: 'skipped',
- releaseOutcome: 'skipped',
- jobResult: 'skipped',
- mutationDeferred: true,
- expected: 'failure',
- },
- {
- name: 'deferred flag cannot hide an operational failure',
- publicationConfirmed: false,
- gateOutcome: 'failure',
- releaseOutcome: 'success',
- jobResult: 'failure',
- mutationDeferred: true,
- failureDescription: 'Gate dispatch failed.',
- expected: 'failure',
- },
- {
- name: 'push or no-op and finalization succeed',
- publicationConfirmed: true,
- gateOutcome: 'success',
- releaseOutcome: 'success',
- jobResult: 'success',
- expected: 'success',
- },
- {
- name: 'job is cancelled after push',
- publicationConfirmed: true,
- gateOutcome: 'success',
- releaseOutcome: 'success',
- jobResult: 'cancelled',
- expected: 'failure',
- },
- ];
- for (const scenario of scenarios) {
- const complete = recoveryComplete(scenario);
- const operation = recoveryOperationResult({
- jobResult: scenario.jobResult,
- recoveryComplete: complete,
- mutationDeferred: scenario.mutationDeferred ?? false,
- failureDescription: scenario.failureDescription ?? null,
- });
- expect(operation, scenario.name).toBe(scenario.expected);
- }
- });
-
- it('writes success only after publication succeeds', () => {
- const base = {
- headKnown: true,
- validationOk: true,
- acceptanceFound: true,
- failureDescription: '',
- };
- expect(
- promotionStatusProjection({...base, promotionResult: 'failure'}),
- ).toEqual({
- state: 'failure',
- description: 'Visual promotion failed; inspect the linked workflow run.',
- });
- expect(
- promotionStatusProjection({...base, promotionResult: 'cancelled'}),
- ).toMatchObject({state: 'failure'});
- expect(
- promotionStatusProjection({...base, promotionResult: 'success'}),
- ).toEqual({
- state: 'success',
- description: 'Accepted pixels promoted to the visual baseline.',
- });
- });
-
- it('preserves normal close behavior when no acceptance exists', () => {
- expect(
- promotionStatusProjection({
- headKnown: true,
- validationOk: true,
- acceptanceFound: false,
- promotionResult: 'skipped',
- failureDescription: '',
- }),
- ).toBeNull();
- });
-
- it('cannot project a status without a server-resolved head', () => {
- expect(
- promotionStatusProjection({
- headKnown: false,
- validationOk: false,
- acceptanceFound: false,
- promotionResult: 'skipped',
- failureDescription: 'Recovery refused: PR was not found.',
- }),
- ).toBeNull();
- });
-});
diff --git a/.github/scripts/visual-gate/lib/report.mjs b/.github/scripts/visual-gate/lib/report.mjs
index 61b5eecb8e..04ff9ab5c5 100644
--- a/.github/scripts/visual-gate/lib/report.mjs
+++ b/.github/scripts/visual-gate/lib/report.mjs
@@ -1,7 +1,7 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
/**
- * @file The report a human (or the release cron) actually looks at.
+ * @file The visual report a pull-request reviewer actually looks at.
*
* @input a verdict and the three image sets it refers to
* @output one self-contained index.html
@@ -152,7 +152,7 @@ export function renderReport(verdict, options = {}) {
</style>
<h1>Visual gate <span class="status">skipped</span></h1>
<p><b>Capture deferred.</b> ${escapeHtml(verdict.reason)}</p>
-<p>${counts.total} trusted baseline shot(s) remain covered by the daily release gate.</p>
+<p>${counts.total} planned shot(s) were not compared. This is not passing visual evidence.</p>
</html>
`;
}
diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjs
index cc7d06124a..cf74c547c2 100644
--- a/.github/scripts/visual-gate/publish-pr-report.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.mjs
@@ -1,359 +1,233 @@
-#!/usr/bin/env node
// Copyright (c) Meta Platforms, Inc. and affiliates.
/**
- * @file Derive publishable PR visual evidence from a trusted recapture.
+ * @input The canonical CI visual-pr-report artifact and trusted run identity.
+ * @output Escaped HTML, bounded verdict JSON, and re-encoded report PNGs.
+ * @position Publication only: never captures, compares, accepts, or promotes.
*
- * The Storybook bundle is PR-produced, but the plan, browser capture, scope,
- * comparison, verdict, PNG validation, and report all run from default-branch
- * code. PR-authored verdicts and job conclusions are never authority for the
- * `visual-acceptance` status.
+ * PR artifacts are report data, not trusted code or baseline-write authority.
+ * Rebuild HTML with the default-branch renderer; never copy artifact HTML.
*/
-
-import {createHash} from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
-import {incomparable} from './lib/baseline.mjs';
import {canonicalizePng} from './lib/canonical-png.mjs';
-import {buildVerdict, compareCaptures} from './lib/compare.mjs';
-import {readThemeCatalog, shotKey} from './lib/plan.mjs';
import {renderReport} from './lib/report.mjs';
-import {loadConfig} from './lib/sources.mjs';
-
-const args = process.argv.slice(2);
-const flag = name => {
- const index = args.indexOf(`--${name}`);
- return index === -1 ? null : args[index + 1];
-};
-const input = path.resolve(flag('input') ?? 'visual');
-const output = path.resolve(flag('output') ?? 'trusted-visual');
-const baseline = path.resolve(flag('baseline') ?? '.visual-baseline');
-const scope = JSON.parse(fs.readFileSync(path.resolve(flag('scope')), 'utf8'));
-const pr = Number(flag('pr'));
-const headSha = flag('head-sha') ?? '';
-const runId = flag('run-id') ?? '';
-const runAttempt = flag('run-attempt') ?? '';
-const REPO_ROOT = path.resolve(
- path.dirname(fileURLToPath(import.meta.url)),
- '../../..',
-);
-const config = loadConfig(REPO_ROOT);
-const themeCatalog = readThemeCatalog(REPO_ROOT, config.baselineThemes);
-
-const KEY = /^[A-Za-z0-9._-]{1,240}$/;
-const NAME = /^[A-Za-z0-9._-]{1,120}$/;
-const PACKAGE_NAME = /^@[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
+const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/;
const SHA = /^[0-9a-f]{40}$/;
-const MAX_PNG_BYTES = 12 * 1024 * 1024;
-const MAX_PNG_WIDTH = 5000;
-const MAX_PNG_HEIGHT = 10000;
-const MAX_PNG_PIXELS = 25_000_000;
-const MAX_SHOTS = config.visualPlanSafetyLimit;
+const LIMIT = 10_000;
+
+function refuse(message) {
+ throw new Error(`visual report rejected: ${message}`);
+}
-function fail(message) {
- throw new Error(`visual evidence rejected: ${message}`);
+function text(value = '') {
+ if (typeof value !== 'string' || value.length > 4000)
+ refuse('invalid report text');
+ return value;
}
-function readJSON(file) {
- try {
- return JSON.parse(fs.readFileSync(file, 'utf8'));
- } catch (error) {
- fail(`cannot read ${file}: ${error.message}`);
+function number(value, max = LIMIT) {
+ if (
+ typeof value !== 'number' ||
+ !Number.isFinite(value) ||
+ value < 0 ||
+ value > max
+ ) {
+ refuse('invalid report number');
}
+ return value;
}
-function shaFile(file) {
- return createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+function list(value = []) {
+ if (!Array.isArray(value) || value.length > LIMIT)
+ refuse('invalid report list');
+ return value;
}
-function pngDimensions(bytes, label) {
+function key(value) {
+ if (typeof value !== 'string' || !KEY.test(value)) refuse('invalid shot key');
+ return value;
+}
+
+function artifactFile(input, relative, maxBytes) {
+ const file = path.join(input, relative);
+ const stat = fs.lstatSync(file);
+ const resolved = fs.realpathSync(file);
+ if (
+ !stat.isFile() ||
+ stat.isSymbolicLink() ||
+ !resolved.startsWith(`${fs.realpathSync(input)}${path.sep}`) ||
+ stat.size === 0 ||
+ stat.size > maxBytes
+ ) {
+ refuse(`invalid artifact file: ${relative}`);
+ }
+ return fs.readFileSync(file);
+}
+
+function copyPng(input, output, kind, shotKey) {
+ const bytes = artifactFile(
+ input,
+ `report/${kind}/${shotKey}.png`,
+ 12 * 1024 * 1024,
+ );
+ // Bound dimensions before decoding an untrusted PNG into memory.
const signature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
if (
bytes.length < 24 ||
!bytes.subarray(0, 8).equals(signature) ||
bytes.toString('ascii', 12, 16) !== 'IHDR'
- ) {
- fail(`${label} is not a valid PNG: invalid header`);
- }
+ )
+ refuse('invalid PNG header');
const width = bytes.readUInt32BE(16);
const height = bytes.readUInt32BE(20);
if (
- width <= 0 ||
- height <= 0 ||
- width > MAX_PNG_WIDTH ||
- height > MAX_PNG_HEIGHT ||
- width * height > MAX_PNG_PIXELS
+ !width ||
+ !height ||
+ width > 5000 ||
+ height > 10000 ||
+ width * height > 25_000_000
) {
- fail(`${label} has invalid dimensions ${width}x${height}`);
+ refuse('invalid PNG dimensions');
}
- return {width, height};
+ const image = canonicalizePng(bytes);
+ fs.mkdirSync(path.join(output, kind), {recursive: true});
+ fs.writeFileSync(path.join(output, kind, `${shotKey}.png`), image.bytes);
}
-function copyPng(source, target, label) {
- if (!fs.existsSync(source)) fail(`${label} is missing`);
- const stat = fs.lstatSync(source);
- if (!stat.isFile() || stat.isSymbolicLink())
- fail(`${label} is not a regular file`);
- if (stat.size <= 0 || stat.size > MAX_PNG_BYTES)
- fail(`${label} has invalid size`);
- const bytes = fs.readFileSync(source);
- const dimensions = pngDimensions(bytes, label);
- let image;
- try {
- image = canonicalizePng(bytes);
- } catch (error) {
- fail(`${label} is not a valid PNG: ${error.message}`);
- }
- if (image.width !== dimensions.width || image.height !== dimensions.height) {
- fail(`${label} dimensions changed while decoding`);
+export function publishPrReport({
+ input,
+ output,
+ headSha,
+ baseSha,
+ runId,
+ runAttempt,
+}) {
+ if (
+ !SHA.test(headSha) ||
+ (baseSha && !SHA.test(baseSha)) ||
+ !/^[1-9]\d*$/.test(String(runId)) ||
+ !/^[1-9]\d*$/.test(String(runAttempt))
+ ) {
+ refuse('invalid trusted run identity');
}
- fs.mkdirSync(path.dirname(target), {recursive: true});
- fs.writeFileSync(target, image.bytes);
- return image;
-}
-
-if (!Number.isSafeInteger(pr) || pr <= 0) fail('invalid PR number');
-if (!SHA.test(headSha)) fail('invalid trusted head SHA');
-if (!/^\d+$/.test(runId) || !/^\d+$/.test(runAttempt))
- fail('invalid run identity');
-if (
- scope?.hasStableVisual !== true ||
- typeof scope.broadStableVisual !== 'boolean' ||
- !Array.isArray(scope.stableComponents) ||
- !Array.isArray(scope.stableThemes) ||
- !scope.stableComponents.every(name => NAME.test(name)) ||
- !scope.stableThemes.every(name => NAME.test(name))
-) {
- fail('trusted stable visual scope is invalid');
-}
-
-const manifestPath = path.join(input, 'manifest.json');
-if (
- !fs.existsSync(manifestPath) ||
- fs.lstatSync(manifestPath).isSymbolicLink()
-) {
- fail('trusted capture manifest is missing or symbolic');
-}
-const manifest = readJSON(manifestPath);
-if (
- manifest.version !== 1 ||
- typeof manifest.platform !== 'string' ||
- !manifest.platform ||
- typeof manifest.browser !== 'string' ||
- !manifest.browser ||
- !Number.isSafeInteger(manifest.viewport?.width) ||
- !Number.isSafeInteger(manifest.viewport?.height) ||
- !manifest.shots ||
- typeof manifest.shots !== 'object' ||
- Array.isArray(manifest.shots)
-) {
- fail('trusted capture manifest is invalid');
-}
-if (
- manifest.context?.headSha !== headSha ||
- String(manifest.context?.runId ?? '') !== runId ||
- String(manifest.context?.runAttempt ?? '') !== runAttempt ||
- !SHA.test(manifest.context?.sha ?? '') ||
- !SHA.test(manifest.context?.baseSha ?? '')
-) {
- fail('trusted capture identity mismatch');
-}
-
-const rawBaselineManifest = readJSON(path.join(baseline, 'manifest.json'));
-const capturedKeys = new Set(Object.keys(manifest.shots));
-const baselineManifest = {
- ...rawBaselineManifest,
- shots: Object.fromEntries(
- Object.entries(rawBaselineManifest.shots ?? {}).filter(([key]) =>
- capturedKeys.has(key),
- ),
- ),
-};
-const blocker = incomparable(baselineManifest, manifest);
-if (blocker) fail(`baseline is not comparable: ${blocker}`);
-
-const entries = Object.entries(manifest.shots);
-if (entries.length > MAX_SHOTS)
- fail(`trusted capture exceeds ${MAX_SHOTS} shots`);
-for (const [key, shot] of entries) {
- const themeMetadata = themeCatalog[shot?.theme];
+ const raw = JSON.parse(artifactFile(input, 'verdict.json', 2 * 1024 * 1024));
if (
- !KEY.test(key) ||
- !shot ||
- typeof shot.storyId !== 'string' ||
- !shot.storyId ||
- typeof shot.theme !== 'string' ||
- !NAME.test(shot.theme) ||
- !PACKAGE_NAME.test(shot.packageName ?? '') ||
- shot.stableVisual !== true ||
- !PACKAGE_NAME.test(shot.themePackageName ?? '') ||
- shot.stableThemeVisual !== true ||
- themeMetadata?.packageName !== shot.themePackageName ||
- themeMetadata?.stableVisual !== true ||
- !['light', 'dark'].includes(shot.mode) ||
- !Array.isArray(shot.reasons) ||
- shotKey(shot) !== key
+ raw.version !== 1 ||
+ !['pass', 'changed', 'failed', 'skipped'].includes(raw.status)
) {
- fail(`capture metadata is invalid for ${key}`);
+ refuse('invalid verdict');
}
-}
-fs.rmSync(output, {recursive: true, force: true});
-for (const dir of ['current', 'before', 'after', 'diff']) {
- fs.mkdirSync(path.join(output, dir), {recursive: true});
-}
-
-const sourceShots = path.join(input, 'shots');
-const sourceNames = fs.existsSync(sourceShots)
- ? fs.readdirSync(sourceShots, {withFileTypes: true})
- : [];
-const sourceKeys = sourceNames.map(entry => {
if (
- !entry.isFile() ||
- entry.isSymbolicLink() ||
- !entry.name.endsWith('.png')
+ raw.context?.headSha !== headSha ||
+ !SHA.test(raw.context?.baseSha) ||
+ (baseSha && raw.context.baseSha !== baseSha) ||
+ String(raw.context?.runId) !== String(runId) ||
+ String(raw.context?.runAttempt) !== String(runAttempt) ||
+ !SHA.test(raw.context?.sha)
) {
- fail(`unexpected capture entry ${entry.name}`);
+ refuse('artifact does not match the exact CI head/base/run/attempt');
}
- return entry.name.slice(0, -4);
-});
-if (
- sourceKeys.length !== entries.length ||
- sourceKeys.some(key => !manifest.shots[key])
-) {
- fail('capture PNG set does not match its manifest');
-}
-
-const trustedManifest = {...manifest, shots: {}};
-for (const [key, shot] of entries) {
- const target = path.join(output, 'current', `${key}.png`);
- const image = copyPng(
- path.join(sourceShots, `${key}.png`),
- target,
- `shots/${key}.png`,
+ const counts = Object.fromEntries(
+ ['total', 'unchanged', 'changed', 'added', 'removed', 'failed'].map(
+ name => [name, number(raw.counts?.[name])],
+ ),
);
- trustedManifest.shots[key] = {
- ...shot,
- sha256: shaFile(target),
- width: image.width,
- height: image.height,
+ const changes = list(raw.changes).map(change => ({
+ key: key(change.key),
+ ...Object.fromEntries(
+ ['component', 'title', 'name', 'theme', 'mode'].map(name => [
+ name,
+ text(change[name]),
+ ]),
+ ),
+ diffPixels: number(change.diffPixels, 25_000_000),
+ diffRatio: number(change.diffRatio, 1),
+ sizeChanged: change.sizeChanged === true,
+ reasons: list(change.reasons).map(text),
+ }));
+ const verdict = {
+ version: 1,
+ status: raw.status,
+ generatedAt: text(raw.generatedAt),
+ platform: text(raw.platform),
+ reason: text(raw.reason),
+ context: {
+ sha: raw.context.sha,
+ headSha,
+ // Fork runs can omit GitHub's PR/base association. That base remains
+ // report data, never acceptance or baseline-write authority.
+ baseSha: baseSha || raw.context.baseSha,
+ runId: String(runId),
+ runAttempt: String(runAttempt),
+ },
+ baseline: {sha: text(raw.baseline?.sha ?? '')},
+ counts,
+ changes,
+ added: list(raw.added).map(key),
+ removed: list(raw.removed).map(key),
+ failures: list(raw.failures).map(failure => ({
+ key: text(failure.key),
+ error: text(failure.error),
+ })),
+ targeting: {
+ unexercisedOverrides: list(raw.targeting?.unexercisedOverrides).map(
+ item => ({
+ theme: text(item.theme),
+ key: text(item.key),
+ selector: text(item.selector),
+ }),
+ ),
+ uncoveredTargets: list(raw.targeting?.uncoveredTargets).map(text),
+ undeclaredTargets: list(raw.targeting?.undeclaredTargets).map(text),
+ },
};
-}
-
-if (entries.length === 0) {
- fail('stable visual scope produced no trusted shots');
-}
-
-const comparison = await compareCaptures({
- baselineDir: path.join(baseline, 'shots'),
- currentDir: path.join(output, 'current'),
- baselineManifest,
- currentManifest: trustedManifest,
- diffDir: path.join(output, 'diff'),
- threshold: config.threshold,
- maxDiffPixels: config.maxDiffPixels,
-});
-
-const verdict = buildVerdict({
- comparison,
- currentManifest: trustedManifest,
- baselineManifest,
- targeting: {
- unexercisedOverrides: [],
- undeclaredTargets: [],
- uncoveredTargets: [],
- },
- failures: [],
- context: {...manifest.context, trustedScope: scope},
-});
-
-const beforeSha256 = {};
-for (const change of verdict.changes) {
- const key = change.key;
- const baselineFile = path.join(baseline, 'shots', `${key}.png`);
- copyPng(
- baselineFile,
- path.join(output, 'before', `${key}.png`),
- `baseline/${key}.png`,
- );
- fs.copyFileSync(
- path.join(output, 'current', `${key}.png`),
- path.join(output, 'after', `${key}.png`),
- );
- beforeSha256[key] = shaFile(baselineFile);
-}
-for (const key of verdict.added) {
- fs.copyFileSync(
- path.join(output, 'current', `${key}.png`),
- path.join(output, 'after', `${key}.png`),
+ if (
+ counts.changed !== changes.length ||
+ counts.added !== verdict.added.length ||
+ counts.removed !== verdict.removed.length ||
+ counts.failed !== verdict.failures.length
+ ) {
+ refuse('verdict counts do not match its report entries');
+ }
+ const images = [
+ ...changes.flatMap(change =>
+ ['before', 'after', 'diff'].map(kind => [kind, change.key]),
+ ),
+ ...verdict.added.map(shotKey => ['after', shotKey]),
+ ...verdict.removed.map(shotKey => ['before', shotKey]),
+ ];
+ if (images.length > LIMIT) refuse('too many report images');
+ fs.rmSync(output, {recursive: true, force: true});
+ fs.mkdirSync(output, {recursive: true});
+ for (const [kind, shotKey] of images) copyPng(input, output, kind, shotKey);
+ fs.writeFileSync(
+ path.join(output, 'verdict.json'),
+ `${JSON.stringify(verdict, null, 2)}\n`,
);
- beforeSha256[key] = null;
-}
-for (const key of verdict.removed) {
- const baselineFile = path.join(baseline, 'shots', `${key}.png`);
- copyPng(
- baselineFile,
- path.join(output, 'before', `${key}.png`),
- `baseline/${key}.png`,
+ fs.writeFileSync(
+ path.join(output, 'index.html'),
+ renderReport(verdict, {
+ acceptHint:
+ 'Record the visual review on the pull request. Baseline maintenance is an explicit CI dispatch, not a release gate.',
+ oneSidedEvidence: true,
+ }),
);
- beforeSha256[key] = shaFile(baselineFile);
+ return verdict;
}
-const evidence = {
- version: 1,
- repo: 'facebook/astryx',
- pr,
- headSha,
- testedSha: manifest.context.sha,
- baseSha: manifest.context.baseSha,
- run: {id: Number(runId), attempt: Number(runAttempt)},
- capture: {
- platform: trustedManifest.platform,
- browser: trustedManifest.browser,
- viewport: trustedManifest.viewport,
- },
- deltas: [
- ...verdict.changes.map(change => ({
- key: change.key,
- kind: 'changed',
- beforeSha256: beforeSha256[change.key],
- shot: trustedManifest.shots[change.key],
- })),
- ...verdict.added.map(key => ({
- key,
- kind: 'added',
- beforeSha256: null,
- shot: trustedManifest.shots[key],
- })),
- ...verdict.removed.map(key => ({
- key,
- kind: 'removed',
- beforeSha256: beforeSha256[key],
- shot: null,
- })),
- ],
- verdict,
-};
-
-fs.rmSync(path.join(output, 'current'), {recursive: true, force: true});
-fs.writeFileSync(
- path.join(output, 'evidence.json'),
- `${JSON.stringify(evidence, null, 2)}\n`,
-);
-fs.writeFileSync(
- path.join(output, 'verdict.json'),
- `${JSON.stringify(verdict, null, 2)}\n`,
-);
-fs.writeFileSync(
- path.join(output, 'index.html'),
- renderReport(verdict, {
- acceptHint: `Repository maintainers only: /accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,
- oneSidedEvidence: true,
- }),
-);
-
-process.stdout.write(
- `Derived trusted ${verdict.status} verdict for ${entries.length} shot(s), PR #${pr}, run ${runId}/${runAttempt}.\n`,
-);
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const args = process.argv.slice(2);
+ const flag = name => args[args.indexOf(`--${name}`) + 1];
+ publishPrReport({
+ input: path.resolve(flag('input')),
+ output: path.resolve(flag('output')),
+ headSha: flag('head-sha'),
+ baseSha: flag('base-sha'),
+ runId: flag('run-id'),
+ runAttempt: flag('run-attempt'),
+ });
+}
diff --git a/.github/scripts/visual-gate/publish-pr-report.test.mjs b/.github/scripts/visual-gate/publish-pr-report.test.mjs
index 04f8c4a7be..8a2ab525a0 100644
--- a/.github/scripts/visual-gate/publish-pr-report.test.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.test.mjs
@@ -1,425 +1,260 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
-import {execFileSync} from 'node:child_process';
-import {createHash} from 'node:crypto';
+/**
+ * @input Canonical CI-shaped reports and hostile artifact mutations.
+ * @output Exact-attempt publication and sanitization contracts, without recapture.
+ * @position Node regression tests for the PR report's data-only trust boundary.
+ */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
-import {fileURLToPath} from 'node:url';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
import {PNG} from 'pngjs';
-import {canonicalizePng} from './lib/canonical-png.mjs';
+import {publishPrReport} from './publish-pr-report.mjs';
-const SCRIPT = path.join(
- path.dirname(fileURLToPath(import.meta.url)),
- 'publish-pr-report.mjs',
-);
-const ACCEPTANCE_SCRIPT = path.join(
- path.dirname(fileURLToPath(import.meta.url)),
- 'visual-acceptance.mjs',
-);
const HEAD = 'a'.repeat(40);
-const RUN = '123456';
+const BASE = 'b'.repeat(40);
+const TESTED = 'c'.repeat(40);
const KEY = 'core-button--default__neutral-light';
-const SHOT = {
- storyId: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- component: 'Button',
- packageName: '@astryxdesign/core',
- packageNames: ['@astryxdesign/core'],
- stableVisual: true,
- theme: 'neutral',
- themePackageName: '@astryxdesign/theme-neutral',
- stableThemeVisual: true,
- mode: 'light',
- reasons: ['trusted:pr-scope'],
-};
-
-let root;
-let input;
-let output;
-let baseline;
-let scope;
+let root, input, output, verdict;
+const json = () =>
+ fs.writeFileSync(path.join(input, 'verdict.json'), JSON.stringify(verdict));
+const run = (overrides = {}) =>
+ publishPrReport({
+ input,
+ output,
+ headSha: HEAD,
+ baseSha: BASE,
+ runId: '123',
+ runAttempt: '2',
+ ...overrides,
+ });
-function pngWithSize(width, height, red = 255, green = 0, blue = 0) {
- const image = new PNG({width, height});
+function png(kind, red) {
+ const image = new PNG({width: 2, height: 2});
for (let i = 0; i < image.data.length; i += 4) {
image.data[i] = red;
- image.data[i + 1] = green;
- image.data[i + 2] = blue;
image.data[i + 3] = 255;
}
- return PNG.sync.write(image);
-}
-
-function png(red = 255, green = 0, blue = 0) {
- return pngWithSize(2, 2, red, green, blue);
-}
-
-function writeJSON(file, value) {
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, JSON.stringify(value));
-}
-
-function writeBaseline(shots = {[KEY]: {shot: SHOT, bytes: png()}}) {
- const manifestShots = {};
- for (const [key, {shot, bytes}] of Object.entries(shots)) {
- const file = path.join(baseline, 'shots', `${key}.png`);
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, bytes);
- manifestShots[key] = {
- ...shot,
- sha256: createHash('sha256').update(bytes).digest('hex'),
- };
- }
- writeJSON(path.join(baseline, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- shots: manifestShots,
- decisions: [],
- });
+ const dir = path.join(input, 'report', kind);
+ fs.mkdirSync(dir, {recursive: true});
+ // Noncanonical encoding and metadata must not escape into the public file.
+ image.gamma = 0.45;
+ fs.writeFileSync(
+ path.join(dir, `${KEY}.png`),
+ PNG.sync.write(image, {deflateLevel: 0}),
+ );
}
-function writeCapture(
- shots = {[KEY]: {shot: SHOT, bytes: png(0, 0, 255)}},
- overrides = {},
-) {
- fs.rmSync(input, {recursive: true, force: true});
- const manifestShots = {};
- const shotsDir = path.join(input, 'shots');
- fs.mkdirSync(shotsDir, {recursive: true});
- for (const [key, {shot, bytes}] of Object.entries(shots)) {
- fs.writeFileSync(path.join(shotsDir, `${key}.png`), bytes);
- manifestShots[key] = {...shot, sha256: '0'.repeat(64)};
- }
- writeJSON(path.join(input, 'manifest.json'), {
+beforeEach(() => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'canonical-visual-report-'));
+ input = path.join(root, 'input');
+ output = path.join(root, 'output');
+ fs.mkdirSync(input);
+ verdict = {
version: 1,
+ status: 'changed',
+ generatedAt: '2026-09-23T00:00:00Z',
platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
context: {
- sha: 'b'.repeat(40),
+ sha: TESTED,
headSha: HEAD,
- baseSha: 'c'.repeat(40),
- runId: RUN,
- runAttempt: '1',
+ baseSha: BASE,
+ runId: '123',
+ runAttempt: '2',
},
- shots: manifestShots,
- ...overrides,
- });
- // A forged PR verdict is deliberately present: the trusted publisher ignores it.
- writeJSON(path.join(input, 'verdict.json'), {
- version: 1,
- status: 'pass',
- changes: [],
+ counts: {
+ total: 1,
+ unchanged: 0,
+ changed: 1,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
+ changes: [
+ {
+ key: KEY,
+ component: 'Button',
+ title: 'Core/Button',
+ name: 'Default',
+ theme: 'neutral',
+ mode: 'light',
+ diffPixels: 3,
+ diffRatio: 0.75,
+ },
+ ],
added: [],
removed: [],
failures: [],
- });
-}
-
-function run() {
- return execFileSync(
- process.execPath,
- [
- SCRIPT,
- '--input',
- input,
- '--output',
- output,
- '--baseline',
- baseline,
- '--scope',
- scope,
- '--pr',
- '42',
- '--head-sha',
- HEAD,
- '--run-id',
- RUN,
- '--run-attempt',
- '1',
- ],
- {encoding: 'utf8'},
+ };
+ json();
+ for (const [kind, red] of [
+ ['before', 0],
+ ['after', 255],
+ ['diff', 128],
+ ])
+ png(kind, red);
+ fs.writeFileSync(
+ path.join(input, 'report', 'index.html'),
+ '<script>throw new Error("PR HTML executed")</script>',
);
-}
-
-function runAcceptance(command, flags) {
- const args = [ACCEPTANCE_SCRIPT, command];
- for (const [name, value] of Object.entries(flags)) {
- args.push(`--${name}`, String(value));
- }
- return execFileSync(process.execPath, args, {encoding: 'utf8'});
-}
-
-beforeEach(() => {
- root = fs.mkdtempSync(path.join(os.tmpdir(), 'trusted-visual-'));
- input = path.join(root, 'input');
- output = path.join(root, 'output');
- baseline = path.join(root, 'baseline');
- scope = path.join(root, 'scope.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Button'],
- stableThemes: [],
- });
- writeBaseline();
- writeCapture();
});
-
afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
-describe('trusted PR visual publisher', () => {
- it('canonicalizes raw capture bytes through acceptance and promotion', () => {
- const rawImage = PNG.sync.read(png(0, 0, 255));
- rawImage.gamma = 0.45455;
- const rawAfter = PNG.sync.write(rawImage, {
- deflateLevel: 0,
- deflateStrategy: 0,
- filterType: 0,
- });
- writeCapture({[KEY]: {shot: SHOT, bytes: rawAfter}});
- run();
-
- const publishedAfter = fs.readFileSync(
- path.join(output, 'after', `${KEY}.png`),
- );
- expect(publishedAfter).toEqual(canonicalizePng(rawAfter).bytes);
- expect(publishedAfter).not.toEqual(rawAfter);
- const evidence = JSON.parse(
- fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'),
- );
- expect(evidence.deltas[0].shot.sha256).toBe(
- createHash('sha256').update(publishedAfter).digest('hex'),
- );
-
- const pages = path.join(root, 'pages');
- fs.mkdirSync(path.join(pages, 'visual-gate'), {recursive: true});
- fs.cpSync(baseline, path.join(pages, 'visual-gate', 'baseline'), {
- recursive: true,
- });
- const evidenceDir = path.join(pages, 'pr', '42', 'visual', HEAD, RUN, '1');
- fs.mkdirSync(path.dirname(evidenceDir), {recursive: true});
- fs.cpSync(output, evidenceDir, {recursive: true});
- execFileSync('git', ['init', '-q', '-b', 'gh-pages'], {cwd: pages});
- execFileSync('git', ['config', 'user.name', 'Test'], {cwd: pages});
- execFileSync('git', ['config', 'user.email', '[email protected]'], {
- cwd: pages,
- });
- execFileSync('git', ['add', '.'], {cwd: pages});
- execFileSync('git', ['commit', '-qm', 'trusted evidence'], {cwd: pages});
-
- runAcceptance('accept', {
- pages,
- pr: 42,
- head: HEAD,
- 'run-id': RUN,
- 'run-attempt': 1,
- approver: 'maintainer',
- 'approver-id': 99,
- permission: 'maintain',
- 'effective-permission': 'maintain',
- 'comment-id': 1234,
- reason: 'The changed frame matches the approved component design.',
- });
- const acceptance = path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- RUN,
- '1',
- 'acceptance.json',
- );
- expect(
- JSON.parse(fs.readFileSync(acceptance, 'utf8')).keys[0].afterSha256,
- ).toBe(createHash('sha256').update(publishedAfter).digest('hex'));
-
- const mergeImage = PNG.sync.read(rawAfter);
- mergeImage.gamma = 0.8;
- const rawMerged = PNG.sync.write(mergeImage, {
- deflateLevel: 1,
- deflateStrategy: 1,
- filterType: 4,
- });
- expect(rawMerged).not.toEqual(rawAfter);
- expect(canonicalizePng(rawMerged).bytes).toEqual(publishedAfter);
- const merged = path.join(root, 'merged');
- fs.mkdirSync(path.join(merged, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(merged, 'shots', `${KEY}.png`), rawMerged);
- writeJSON(path.join(merged, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-27T09:00:00.000Z',
- context: {sha: 'd'.repeat(40)},
- shots: {
- [KEY]: {
- ...SHOT,
- sha256: createHash('sha256').update(rawMerged).digest('hex'),
- width: 2,
- height: 2,
- },
- },
- });
-
- runAcceptance('promote', {
- pages,
- acceptance,
- capture: merged,
- 'merge-sha': 'd'.repeat(40),
- });
- expect(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- ),
- ).toEqual(publishedAfter);
+describe('canonical visual report publication', () => {
+ it('publishes fork-run report data without substituting a newer live base', () => {
+ expect(run({baseSha: ''}).context.baseSha).toBe(BASE);
+ verdict.context.baseSha = 'not-a-sha';
+ json();
+ expect(() => run({baseSha: ''})).toThrow('exact CI');
});
- it('derives a changed verdict even when PR data claims pass', () => {
- expect(run()).toContain('trusted changed verdict');
- const verdict = JSON.parse(
- fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'),
+ it('preserves the CI verdict without recapturing or comparing again', () => {
+ const result = run();
+ expect(result.status).toBe('changed');
+ // These 2x2 frames differ at four pixels, but the report must retain the
+ // CI-reported three. A publisher is not another comparison authority.
+ expect(result.changes[0].diffPixels).toBe(3);
+ expect(result.context).toEqual(verdict.context);
+ const html = fs.readFileSync(path.join(output, 'index.html'), 'utf8');
+ expect(html).toContain(`before/${KEY}.png`);
+ expect(html).not.toContain('PR HTML executed');
+ expect(html).not.toContain('/accept-visual');
+ expect(fs.existsSync(path.join(output, 'evidence.json'))).toBe(false);
+ const bytes = fs.readFileSync(path.join(output, 'after', `${KEY}.png`));
+ expect(bytes).not.toEqual(
+ fs.readFileSync(path.join(input, 'report', 'after', `${KEY}.png`)),
);
- expect(verdict).toMatchObject({status: 'changed', counts: {changed: 1}});
- expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);
- expect(fs.existsSync(path.join(output, 'after', `${KEY}.png`))).toBe(true);
- expect(fs.existsSync(path.join(output, 'diff', `${KEY}.png`))).toBe(true);
+ expect([...PNG.sync.read(bytes).data]).toEqual([
+ 255, 0, 0, 255, 255, 0, 0, 255, 255, 0, 0, 255, 255, 0, 0, 255,
+ ]);
});
- it('derives pass only from a trusted capture matching the baseline', () => {
- writeCapture({[KEY]: {shot: SHOT, bytes: png()}});
- run();
- const verdict = JSON.parse(
- fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'),
- );
- expect(verdict.status).toBe('pass');
- });
+ it.each(['headSha', 'baseSha', 'runId', 'runAttempt'])(
+ 'refuses stale or forged %s',
+ field => {
+ verdict.context[field] = field.endsWith('Sha') ? 'd'.repeat(40) : '9';
+ json();
+ expect(() => run()).toThrow('exact CI head/base/run/attempt');
+ },
+ );
- it('derives an added shot and preserves its after image', () => {
- writeBaseline({});
- const key = 'core-new--default__neutral-light';
- const shot = {...SHOT, storyId: 'core-new--default', component: 'New'};
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['New'],
- stableThemes: [],
- });
- writeCapture({[key]: {shot, bytes: png(0, 255, 0)}});
- run();
- const verdict = JSON.parse(
- fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'),
+ it('rejects a malformed tested-tree SHA and invalid trusted identity', () => {
+ verdict.context.sha = 'main';
+ json();
+ expect(() => run()).toThrow('exact CI');
+ expect(() => run({runAttempt: '0'})).toThrow(
+ 'invalid trusted run identity',
);
- expect(verdict).toMatchObject({status: 'changed', added: [key]});
- expect(fs.existsSync(path.join(output, 'after', `${key}.png`))).toBe(true);
});
- it('publishes trusted evidence for a theme-only bundle above the component limit', () => {
- writeBaseline({});
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: [],
- stableThemes: ['neutral'],
- });
- const shots = {};
- const bytes = png(0, 255, 0);
- for (let index = 0; index < 242; index += 1) {
- const key = `core-theme-${index}--default__neutral-light`;
- shots[key] = {
- shot: {
- ...SHOT,
- storyId: `core-theme-${index}--default`,
- component: `Theme${index}`,
- },
- bytes,
- };
- }
- writeCapture(shots);
+ it.each(['pass', 'skipped', 'failed'])(
+ 'reports %s without manufacturing comparison evidence',
+ status => {
+ verdict.status = status;
+ verdict.changes = [];
+ verdict.counts.changed = 0;
+ verdict.counts.unchanged = status === 'pass' ? 1 : 0;
+ if (status === 'failed') {
+ verdict.failures = [{key: KEY, error: 'Capture failed'}];
+ verdict.counts.failed = 1;
+ }
+ if (status === 'skipped') verdict.reason = 'Over review budget';
+ json();
+ expect(run().status).toBe(status);
+ expect(fs.existsSync(path.join(output, 'after'))).toBe(false);
+ const html = fs.readFileSync(path.join(output, 'index.html'), 'utf8');
+ expect(html).not.toContain('daily release gate');
+ if (status === 'skipped')
+ expect(html).toContain('not passing visual evidence');
+ },
+ );
- run();
+ it.each(['added', 'removed'])(
+ 'publishes the canonical %s frame, without inferring new keys',
+ kind => {
+ verdict.changes = [];
+ verdict.counts.changed = 0;
+ verdict[kind] = [KEY];
+ verdict.counts[kind] = 1;
+ json();
+ const result = run();
+ expect(result[kind]).toEqual([KEY]);
+ const imageKind = kind === 'added' ? 'after' : 'before';
+ expect(fs.existsSync(path.join(output, imageKind, `${KEY}.png`))).toBe(
+ true,
+ );
+ },
+ );
- const evidence = JSON.parse(
- fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'),
- );
- expect(evidence.verdict).toMatchObject({
- status: 'changed',
- counts: {total: 242, added: 242, failed: 0},
- });
- expect(evidence.deltas).toHaveLength(242);
- expect(fs.readdirSync(path.join(output, 'after'))).toHaveLength(242);
- });
+ it.each(['../escape', '..', 'frame/escape', 'bad" onerror="x'])(
+ 'rejects unsafe image key %s',
+ value => {
+ verdict.changes[0].key = value;
+ json();
+ expect(() => run()).toThrow('invalid shot key');
+ },
+ );
- it('does not derive removals from a scoped PR capture', () => {
- const other = 'core-card--default__neutral-light';
- writeBaseline({
- [KEY]: {shot: SHOT, bytes: png()},
- [other]: {
- shot: {...SHOT, storyId: 'core-card--default', component: 'Card'},
- bytes: png(),
- },
- });
+ it('escapes text and rejects markup in numeric fields', () => {
+ verdict.changes[0].name = '<img src=x onerror=alert(1)>';
+ json();
run();
- const verdict = JSON.parse(
- fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'),
+ expect(fs.readFileSync(path.join(output, 'index.html'), 'utf8')).toContain(
+ '&lt;img src=x onerror=alert(1)&gt;',
);
- expect(verdict.removed).toEqual([]);
+ verdict.counts.total = '<script>alert(1)</script>';
+ json();
+ expect(() => run()).toThrow('invalid report number');
});
- it('rejects a trusted capture that claims another run', () => {
- const manifest = JSON.parse(
- fs.readFileSync(path.join(input, 'manifest.json'), 'utf8'),
- );
- manifest.context.runId = '999';
- writeJSON(path.join(input, 'manifest.json'), manifest);
- expect(() => run()).toThrow(/identity mismatch/);
+ it('refuses inconsistent counts and unknown statuses', () => {
+ verdict.counts.changed = 0;
+ json();
+ expect(() => run()).toThrow('counts do not match');
+ verdict.status = 'not-a-verdict';
+ json();
+ expect(() => run()).toThrow('invalid verdict');
});
- it('rejects traversal-shaped manifest keys', () => {
- writeCapture({
- '../bad': {shot: {...SHOT, storyId: '../bad'}, bytes: png()},
- });
- expect(() => run()).toThrow(/metadata is invalid/);
- });
-
- it('rejects capture bytes that are not a PNG', () => {
- fs.writeFileSync(path.join(input, 'shots', `${KEY}.png`), 'not png');
- expect(() => run()).toThrow(/not a valid PNG/);
- });
-
- it('accepts tall sheets that stay within the existing pixel budget', () => {
- const tall = pngWithSize(2, 7673, 0, 0, 255);
- writeCapture({[KEY]: {shot: SHOT, bytes: tall}});
-
- run();
-
- const evidence = JSON.parse(
- fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'),
+ it('bounds JSON and image allocations', () => {
+ fs.writeFileSync(
+ path.join(input, 'verdict.json'),
+ ' '.repeat(2 * 1024 * 1024 + 1),
);
- expect(evidence.deltas[0].shot).toMatchObject({width: 2, height: 7673});
+ expect(() => run()).toThrow('invalid artifact file');
+ json();
+ const file = path.join(input, 'report', 'before', `${KEY}.png`);
+ const bytes = fs.readFileSync(file);
+ bytes.writeUInt32BE(100_000, 16);
+ fs.writeFileSync(file, bytes);
+ expect(() => run()).toThrow('invalid PNG dimensions');
});
- it('rejects PNG headers above the pixel budget before decoding', () => {
- const oversized = Buffer.from(png());
- oversized.writeUInt32BE(5000, 16);
- oversized.writeUInt32BE(5001, 20);
- fs.writeFileSync(path.join(input, 'shots', `${KEY}.png`), oversized);
-
- expect(() => run()).toThrow(/invalid dimensions 5000x5001/);
+ it('rejects missing/corrupt frames instead of publishing broken evidence', () => {
+ fs.writeFileSync(
+ path.join(input, 'report', 'before', `${KEY}.png`),
+ 'not an image',
+ );
+ expect(() => run()).toThrow('invalid PNG header');
+ fs.rmSync(path.join(input, 'report', 'after', `${KEY}.png`));
+ png('before', 0);
+ expect(() => run()).toThrow();
});
- it('rejects an empty capture for an unbaselined stable scope', () => {
- writeBaseline({});
- writeCapture({});
- expect(() => run()).toThrow(/produced no trusted shots/);
+ it('refuses symlink files and parent directories outside the artifact', () => {
+ const original = path.join(input, 'report', 'before', `${KEY}.png`);
+ const external = path.join(root, 'external.png');
+ fs.renameSync(original, external);
+ fs.symlinkSync(external, original);
+ expect(() => run()).toThrow('invalid artifact file');
+ fs.rmSync(path.join(input, 'report', 'before'), {recursive: true});
+ const outside = path.join(root, 'outside');
+ fs.mkdirSync(outside);
+ fs.copyFileSync(external, path.join(outside, `${KEY}.png`));
+ fs.symlinkSync(outside, path.join(input, 'report', 'before'));
+ expect(() => run()).toThrow('invalid artifact file');
});
});
diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjs
deleted file mode 100644
index e4ab52fd16..0000000000
--- a/.github/scripts/visual-gate/visual-acceptance.mjs
+++ /dev/null
@@ -1,1037 +0,0 @@
-#!/usr/bin/env node
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-/**
- * @file Record, evaluate, and promote explicit visual acceptance.
- *
- * Labels, comments, and statuses are projections. The immutable JSON record
- * under gh-pages:visual-gate/acceptances/<pr>/<head>/ is the source of truth.
- */
-
-import {execFileSync} from 'node:child_process';
-import {createHash} from 'node:crypto';
-import fs from 'node:fs';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-import {
- isVisualAcceptanceEndpointMaintainer,
- isVisualAcceptanceRecordMaintainer,
-} from './authorization.mjs';
-import {canonicalizePng} from './lib/canonical-png.mjs';
-import {
- acceptedVisualThemes,
- baselineVisualStories,
- canonicalBaselineStories,
- componentVisualStories,
- emptyVisualPlanMessage,
- exceedsPrVisualShotLimit,
- readStoryIndex,
- readThemeCatalog,
- representativeStories,
- shotKey,
- stableBaseline,
- storiesInPackages,
- VISUAL_BASELINE_TAG,
- withThemeMetadata,
-} from './lib/plan.mjs';
-import {renderReport} from './lib/report.mjs';
-import {loadConfig} from './lib/sources.mjs';
-
-const REPO_ROOT = path.resolve(
- path.dirname(fileURLToPath(import.meta.url)),
- '../../..',
-);
-const config = loadConfig(REPO_ROOT);
-const themeCatalog = readThemeCatalog(REPO_ROOT, config.baselineThemes);
-
-const args = process.argv.slice(2);
-const command = args[0];
-const flag = name => {
- const index = args.indexOf(`--${name}`);
- return index === -1 ? null : args[index + 1];
-};
-const SHA = /^[0-9a-f]{40}$/;
-const KEY = /^[A-Za-z0-9._-]{1,240}$/;
-const NAME = /^[A-Za-z0-9._-]{1,120}$/;
-const REPO = 'facebook/astryx';
-
-function fail(message) {
- throw new Error(`visual acceptance refused: ${message}`);
-}
-
-function shaBytes(bytes) {
- return createHash('sha256').update(bytes).digest('hex');
-}
-
-function shaFile(file) {
- return shaBytes(fs.readFileSync(file));
-}
-
-function shaJSON(value) {
- return shaBytes(JSON.stringify(value));
-}
-
-function readJSON(file) {
- try {
- return JSON.parse(fs.readFileSync(file, 'utf8'));
- } catch (error) {
- fail(`cannot read ${file}: ${error.message}`);
- }
-}
-
-function writeJSON(file, value) {
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
-}
-
-function pagesHead(pages) {
- return execFileSync('git', ['rev-parse', 'HEAD'], {
- cwd: pages,
- encoding: 'utf8',
- }).trim();
-}
-
-function validateIdentity(pr, head) {
- if (!Number.isSafeInteger(pr) || pr <= 0 || !SHA.test(head))
- fail('invalid PR/head');
-}
-
-function evidenceRoot(pages, pr, head) {
- return path.join(pages, 'pr', String(pr), 'visual', head);
-}
-
-function evidenceAt(pages, pr, head, run, attempt) {
- if (
- !Number.isSafeInteger(run) ||
- run <= 0 ||
- !Number.isSafeInteger(attempt) ||
- attempt <= 0
- ) {
- fail('invalid evidence run/attempt');
- }
- const dir = path.join(
- evidenceRoot(pages, pr, head),
- String(run),
- String(attempt),
- );
- if (!fs.existsSync(dir))
- fail(`no published evidence for run ${run}/${attempt}`);
- const evidence = readJSON(path.join(dir, 'evidence.json'));
- validateEvidence(evidence, {pr, head, run, attempt});
- return {dir, evidence};
-}
-
-function latestEvidence(pages, pr, head, {required = true} = {}) {
- const root = evidenceRoot(pages, pr, head);
- if (!fs.existsSync(root)) {
- if (!required) return null;
- fail(`no published evidence for PR #${pr}, head ${head}`);
- }
- const runs = fs
- .readdirSync(root, {withFileTypes: true})
- .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))
- .map(entry => Number(entry.name))
- .sort((a, b) => b - a);
- if (runs.length === 0) {
- if (!required) return null;
- fail('published evidence has no run directory');
- }
- const run = runs[0];
- const runDir = path.join(root, String(run));
- const attempts = fs
- .readdirSync(runDir, {withFileTypes: true})
- .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))
- .map(entry => Number(entry.name))
- .sort((a, b) => b - a);
- const dir =
- attempts.length > 0 ? path.join(runDir, String(attempts[0])) : runDir;
- const evidence = readJSON(path.join(dir, 'evidence.json'));
- validateEvidence(evidence, {pr, head, run});
- if (attempts.length > 0 && evidence.run.attempt !== attempts[0]) {
- fail('evidence attempt identity mismatch');
- }
- return {dir, evidence};
-}
-
-function validateCapture(capture) {
- if (
- !capture ||
- typeof capture.platform !== 'string' ||
- !capture.platform ||
- typeof capture.browser !== 'string' ||
- !capture.browser ||
- !Number.isSafeInteger(capture.viewport?.width) ||
- !Number.isSafeInteger(capture.viewport?.height) ||
- capture.viewport.width <= 0 ||
- capture.viewport.height <= 0
- ) {
- fail('evidence capture identity is incomplete');
- }
-}
-
-function validateShot(shot, key) {
- if (
- !shot ||
- typeof shot.storyId !== 'string' ||
- !shot.storyId ||
- typeof shot.theme !== 'string' ||
- !shot.theme ||
- !['light', 'dark'].includes(shot.mode) ||
- !Array.isArray(shot.reasons)
- ) {
- fail(`accepted shot plan is incomplete for ${key}`);
- }
-}
-
-function validateEvidence(evidence, expected) {
- if (
- evidence?.version !== 1 ||
- evidence.repo !== REPO ||
- evidence.pr !== expected.pr ||
- evidence.headSha !== expected.head ||
- !SHA.test(evidence.testedSha ?? '') ||
- !SHA.test(evidence.baseSha ?? '') ||
- evidence.run?.id !== expected.run ||
- !Number.isSafeInteger(evidence.run?.attempt) ||
- evidence.run.attempt <= 0 ||
- (expected.attempt !== undefined &&
- evidence.run.attempt !== expected.attempt)
- ) {
- fail('evidence identity mismatch');
- }
- if (
- !['pass', 'changed', 'failed', 'skipped'].includes(evidence.verdict?.status)
- ) {
- fail('evidence verdict is invalid');
- }
- if (
- !Array.isArray(evidence.deltas) ||
- evidence.deltas.length > config.visualPlanSafetyLimit
- ) {
- fail('evidence delta list is invalid');
- }
- if (evidence.verdict.status === 'skipped') {
- const counts = evidence.verdict.counts;
- if (evidence.capture !== null || evidence.deltas.length !== 0) {
- fail('skipped evidence must not claim a capture or deltas');
- }
- if (
- typeof evidence.verdict.reason !== 'string' ||
- !evidence.verdict.reason.trim() ||
- !Number.isSafeInteger(counts?.total) ||
- counts.total <= 0 ||
- ['unchanged', 'changed', 'added', 'removed', 'failed'].some(
- name => counts[name] !== 0,
- )
- ) {
- fail('skipped evidence must carry a trusted reason and count');
- }
- if (
- evidence.verdict.context?.sha !== evidence.testedSha ||
- evidence.verdict.context?.headSha !== evidence.headSha ||
- evidence.verdict.context?.baseSha !== evidence.baseSha ||
- String(evidence.verdict.context?.runId ?? '') !==
- String(evidence.run.id) ||
- String(evidence.verdict.context?.runAttempt ?? '') !==
- String(evidence.run.attempt)
- ) {
- fail('skipped evidence run identity mismatch');
- }
- } else {
- validateCapture(evidence.capture);
- }
- const seen = new Set();
- for (const delta of evidence.deltas) {
- if (
- !KEY.test(delta?.key ?? '') ||
- !['changed', 'added', 'removed'].includes(delta.kind)
- ) {
- fail(`invalid evidence delta ${JSON.stringify(delta)}`);
- }
- if (seen.has(delta.key)) fail(`duplicate evidence key ${delta.key}`);
- seen.add(delta.key);
- if (delta.kind === 'removed') {
- if (delta.shot !== null)
- fail(`removed key ${delta.key} must not carry a shot plan`);
- } else {
- validateShot(delta.shot, delta.key);
- }
- }
-}
-
-function validateReason(reason) {
- const clean = String(reason ?? '').trim();
- if (
- clean.length < 12 ||
- clean.length > 500 ||
- /[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(clean)
- ) {
- fail('reason must be 12–500 printable characters');
- }
- if (
- /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(clean)
- ) {
- fail(
- 'reason must explain why the pixels are correct, not merely approve them',
- );
- }
- return clean;
-}
-
-function snapshotBundle({pages, evidenceDir, evidence}) {
- const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');
- return evidence.deltas.map(delta => {
- const beforeFile = path.join(baselineShots, `${delta.key}.png`);
- const afterFile = path.join(evidenceDir, 'after', `${delta.key}.png`);
- const currentBeforeSha256 = fs.existsSync(beforeFile)
- ? shaFile(beforeFile)
- : null;
- const beforeSha256 = delta.beforeSha256 ?? null;
- const afterSha256 = fs.existsSync(afterFile) ? shaFile(afterFile) : null;
- if (beforeSha256 !== null && !/^[0-9a-f]{64}$/.test(beforeSha256)) {
- fail(`invalid reviewed preimage for ${delta.key}`);
- }
- if (currentBeforeSha256 !== beforeSha256) {
- fail(`reviewed baseline is stale for ${delta.key}`);
- }
- if (delta.kind === 'changed' && (!beforeSha256 || !afterSha256)) {
- fail(`changed key ${delta.key} lacks before/after evidence`);
- }
- if (delta.kind === 'added' && (beforeSha256 || !afterSha256)) {
- fail(`added key ${delta.key} has invalid preimage/evidence`);
- }
- if (delta.kind === 'removed' && (!beforeSha256 || afterSha256)) {
- fail(`removed key ${delta.key} has invalid preimage/evidence`);
- }
- return {
- key: delta.key,
- kind: delta.kind,
- beforeSha256,
- afterSha256,
- shot: delta.shot,
- };
- });
-}
-
-function sameBundle(first, second) {
- return shaJSON(first) === shaJSON(second);
-}
-
-function recordPath(pages, pr, head, run, attempt) {
- return path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- String(pr),
- head,
- String(run),
- String(attempt),
- 'acceptance.json',
- );
-}
-
-function pointerPath(pages, pr, head) {
- return path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- String(pr),
- head,
- 'current.json',
- );
-}
-
-function accept() {
- const pages = path.resolve(flag('pages') ?? '.');
- const pr = Number(flag('pr'));
- const head = flag('head') ?? '';
- const runId = Number(flag('run-id'));
- const runAttempt = Number(flag('run-attempt'));
- const approver = flag('approver') ?? '';
- const approverId = Number(flag('approver-id'));
- const permission = flag('permission') ?? '';
- const effectivePermission = flag('effective-permission') ?? 'none';
- const roleName = flag('role-name');
- const commentId = Number(flag('comment-id'));
- const reason = validateReason(flag('reason'));
-
- validateIdentity(pr, head);
- if (!approver || !Number.isSafeInteger(approverId) || approverId <= 0)
- fail('invalid approver');
- if (!isVisualAcceptanceEndpointMaintainer({effectivePermission})) {
- fail('approver must have effective maintain/admin permission');
- }
- if (!Number.isSafeInteger(commentId) || commentId <= 0)
- fail('invalid comment id');
-
- const {dir: evidenceDir, evidence} = evidenceAt(
- pages,
- pr,
- head,
- runId,
- runAttempt,
- );
- if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {
- fail('current visual bundle has no delta to accept');
- }
- if (evidence.deltas.some(delta => delta.kind !== 'changed')) {
- fail(
- 'scoped PR acceptance can only authorize changes to existing baseline frames',
- );
- }
- const manifestFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const manifest = readJSON(manifestFile);
- const keys = snapshotBundle({pages, evidenceDir, evidence});
- const output = recordPath(pages, pr, head, runId, runAttempt);
- const destination = path.dirname(output);
-
- const pointer = {
- version: 1,
- run: runId,
- attempt: runAttempt,
- record: `${runId}/${runAttempt}/acceptance.json`,
- };
- if (fs.existsSync(output)) {
- const existing = readJSON(output);
- validateAcceptance(existing, {
- pr,
- head,
- run: runId,
- attempt: runAttempt,
- dir: destination,
- });
- if (sameBundle(existing.keys, keys)) {
- writeJSON(pointerPath(pages, pr, head), pointer);
- process.stdout.write(`Acceptance already recorded at ${output}\n`);
- return;
- }
- fail('this head already has a different acceptance record');
- }
-
- for (const entry of keys) {
- if (!entry.afterSha256) continue;
- const source = path.join(evidenceDir, 'after', `${entry.key}.png`);
- const target = path.join(destination, 'after', `${entry.key}.png`);
- fs.mkdirSync(path.dirname(target), {recursive: true});
- fs.copyFileSync(source, target);
- }
-
- writeJSON(output, {
- version: 1,
- repo: REPO,
- pr,
- headSha: head,
- testedSha: evidence.testedSha,
- baseSha: evidence.baseSha,
- run: evidence.run,
- capture: evidence.capture,
- baseline: {
- ghPagesCommit: pagesHead(pages),
- manifestDigest: shaJSON(manifest),
- },
- keys,
- decision: {
- approver,
- approverId,
- permission,
- effectivePermission,
- roleName,
- reason,
- commentId,
- at: new Date().toISOString(),
- },
- });
- writeJSON(pointerPath(pages, pr, head), pointer);
- process.stdout.write(
- `Accepted ${keys.length} visual delta(s) for PR #${pr}, head ${head}, run ${runId}/${runAttempt}.\n`,
- );
-}
-
-function state() {
- const pages = path.resolve(flag('pages') ?? '.');
- const pr = Number(flag('pr'));
- const head = flag('head') ?? '';
- validateIdentity(pr, head);
-
- const found = latestEvidence(pages, pr, head, {required: false});
- let result;
- if (!found) {
- result = {
- state: 'pending',
- reason: 'capture',
- description: 'Stable visual capture is still running.',
- };
- } else if (found.evidence.verdict.status === 'pass') {
- result = {
- state: 'success',
- reason: 'clean',
- description: 'Stable visual capture is clean.',
- };
- } else if (found.evidence.verdict.status === 'failed') {
- result = {
- state: 'failure',
- reason: 'capture',
- description: 'Stable visual capture failed.',
- };
- } else if (found.evidence.verdict.status === 'skipped') {
- result = {
- state: 'success',
- reason: 'deferred',
- description: 'Broad stable scope is deferred to the release gate.',
- };
- } else {
- let current;
- try {
- current = snapshotBundle({
- pages,
- evidenceDir: found.dir,
- evidence: found.evidence,
- });
- } catch (error) {
- if (!String(error.message).includes('reviewed baseline is stale'))
- throw error;
- result = {
- state: 'pending',
- reason: 'capture',
- description: 'The visual baseline changed; recapture this head.',
- };
- }
- if (!result) {
- const file = recordPath(
- pages,
- pr,
- head,
- found.evidence.run.id,
- found.evidence.run.attempt,
- );
- const accepted = fs.existsSync(file) ? readJSON(file) : null;
- if (accepted)
- validateAcceptance(accepted, {
- pr,
- head,
- run: found.evidence.run.id,
- attempt: found.evidence.run.attempt,
- dir: path.dirname(file),
- });
- result =
- accepted && sameBundle(accepted.keys, current)
- ? {
- state: 'success',
- reason: 'accepted',
- description: 'Current visual bundle was explicitly accepted.',
- }
- : {
- state: 'pending',
- reason: 'decision',
- description: 'Stable visual changes await /accept-visual.',
- };
- }
- }
- process.stdout.write(`${JSON.stringify(result)}\n`);
-}
-
-function validateAcceptance(value, expected = {}) {
- if (
- value?.version !== 1 ||
- value.repo !== REPO ||
- !Number.isSafeInteger(value.pr) ||
- value.pr <= 0 ||
- (expected.pr && value.pr !== expected.pr) ||
- !SHA.test(value.headSha ?? '') ||
- (expected.head && value.headSha !== expected.head) ||
- !SHA.test(value.testedSha ?? '') ||
- !SHA.test(value.baseSha ?? '') ||
- !Number.isSafeInteger(value.run?.id) ||
- value.run.id <= 0 ||
- (expected.run && value.run.id !== expected.run) ||
- !Number.isSafeInteger(value.run?.attempt) ||
- value.run.attempt <= 0 ||
- (expected.attempt && value.run.attempt !== expected.attempt) ||
- !Array.isArray(value.keys) ||
- value.keys.length === 0 ||
- !value.decision?.approver ||
- !Number.isSafeInteger(value.decision?.approverId) ||
- value.decision.approverId <= 0 ||
- !isVisualAcceptanceRecordMaintainer({
- permission: value.decision?.permission,
- effectivePermission: value.decision?.effectivePermission,
- }) ||
- !Number.isSafeInteger(value.decision?.commentId) ||
- value.decision.commentId <= 0 ||
- !value.decision?.reason
- ) {
- fail('acceptance record is invalid');
- }
- validateCapture(value.capture);
- const seen = new Set();
- for (const entry of value.keys) {
- if (
- !KEY.test(entry?.key ?? '') ||
- !['changed', 'added', 'removed'].includes(entry.kind)
- ) {
- fail('acceptance record contains an invalid key');
- }
- if (seen.has(entry.key)) fail(`acceptance record repeats ${entry.key}`);
- seen.add(entry.key);
- if (entry.kind === 'removed') {
- if (entry.afterSha256 !== null || entry.shot !== null)
- fail(`removed key ${entry.key} is invalid`);
- } else {
- if (!/^[0-9a-f]{64}$/.test(entry.afterSha256 ?? ''))
- fail(`after hash is invalid for ${entry.key}`);
- validateShot(entry.shot, entry.key);
- if (expected.dir) {
- const archived = path.join(expected.dir, 'after', `${entry.key}.png`);
- if (
- !fs.existsSync(archived) ||
- shaFile(archived) !== entry.afterSha256
- ) {
- fail(`archived AFTER is missing or changed for ${entry.key}`);
- }
- }
- }
- if (
- entry.beforeSha256 !== null &&
- !/^[0-9a-f]{64}$/.test(entry.beforeSha256)
- ) {
- fail(`before hash is invalid for ${entry.key}`);
- }
- }
-}
-
-function readTrustedScope() {
- const scope = readJSON(path.resolve(flag('scope')));
- if (
- scope?.hasStableVisual !== true ||
- typeof scope.broadStableVisual !== 'boolean' ||
- !Array.isArray(scope.stableComponents) ||
- !Array.isArray(scope.stableThemes) ||
- !scope.stableComponents.every(name => NAME.test(name)) ||
- !scope.stableThemes.every(name => NAME.test(name))
- ) {
- fail('trusted visual scope is invalid');
- }
- return scope;
-}
-
-function readTrustedBaseline(stories = null) {
- const raw = readJSON(
- path.join(path.resolve(flag('baseline')), 'manifest.json'),
- );
- const baseline = stories
- ? stableBaseline(raw, stories, themeCatalog, REPO_ROOT)
- : raw;
- if (
- !baseline?.shots ||
- typeof baseline.shots !== 'object' ||
- Array.isArray(baseline.shots)
- ) {
- fail('trusted visual baseline is invalid');
- }
- const entries = Object.entries(baseline.shots);
- if (
- entries.length === 0 ||
- entries.length > config.visualPlanSafetyLimit
- ) {
- fail(
- `trusted visual baseline has invalid size ${entries.length}; safety limit is ${config.visualPlanSafetyLimit}`,
- );
- }
- return {baseline, entries};
-}
-
-function trustedDefer() {
- const scope = readTrustedScope();
- if (!scope.broadStableVisual) fail('trusted visual scope is not broad');
-
- const {entries} = readTrustedBaseline();
- const output = path.resolve(flag('output'));
- const pr = Number(flag('pr'));
- const head = flag('head') ?? '';
- const base = flag('base') ?? '';
- const runId = Number(flag('run-id'));
- const runAttempt = Number(flag('run-attempt'));
- validateIdentity(pr, head);
- if (!SHA.test(base)) fail('invalid base SHA');
- if (
- !Number.isSafeInteger(runId) ||
- runId <= 0 ||
- !Number.isSafeInteger(runAttempt) ||
- runAttempt <= 0
- ) {
- fail('invalid evidence run/attempt');
- }
-
- const total = entries.length;
- const reason =
- 'Broad stable scope is deferred to the daily release gate. ' +
- `It covers ${total} trusted baseline shot${total === 1 ? '' : 's'} ` +
- 'instead of recapturing them for this PR.';
- const verdict = {
- version: 1,
- status: 'skipped',
- generatedAt: new Date().toISOString(),
- reason,
- context: {
- sha: head,
- headSha: head,
- baseSha: base,
- runId: String(runId),
- runAttempt: String(runAttempt),
- trustedScope: scope,
- },
- counts: {
- total,
- unchanged: 0,
- changed: 0,
- added: 0,
- removed: 0,
- failed: 0,
- },
- components: scope.stableComponents,
- changes: [],
- added: [],
- removed: [],
- failures: [],
- };
- const evidence = {
- version: 1,
- repo: REPO,
- pr,
- headSha: head,
- testedSha: head,
- baseSha: base,
- run: {id: runId, attempt: runAttempt},
- capture: null,
- deltas: [],
- verdict,
- };
- validateEvidence(evidence, {
- pr,
- head,
- run: runId,
- attempt: runAttempt,
- });
-
- fs.rmSync(output, {recursive: true, force: true});
- writeJSON(path.join(output, 'evidence.json'), evidence);
- writeJSON(path.join(output, 'verdict.json'), verdict);
- fs.writeFileSync(path.join(output, 'index.html'), renderReport(verdict));
- process.stdout.write(
- `Deferred ${total} trusted baseline shot${total === 1 ? '' : 's'} for PR #${pr}, run ${runId}/${runAttempt}.\n`,
- );
-}
-
-function trustedPlan() {
- const scope = readTrustedScope();
- if (scope.broadStableVisual)
- fail('broad stable scope must be deferred instead of captured');
- const storybookDir = path.resolve(flag('storybook-dir'));
- const output = path.resolve(flag('output'));
- const allIndexed = readStoryIndex(storybookDir, [], REPO_ROOT);
- const packageStories = storiesInPackages(
- allIndexed,
- config.stableStoryPackages,
- );
- const indexed = canonicalBaselineStories(packageStories, {
- groups: config.stableStoryGroups,
- packages: config.stableStoryPackages,
- });
- const {baseline} = readTrustedBaseline(allIndexed);
- const baselineKeys = new Set(Object.keys(baseline.shots));
-
- const baselineThemes = acceptedVisualThemes(
- baseline,
- themeCatalog,
- config.defaultTheme,
- );
- const shots = [];
-
- if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {
- const add = (
- story,
- theme,
- componentKeys = null,
- existingBaselineOnly = false,
- ) => {
- for (const mode of ['light', 'dark']) {
- const shot = {
- storyId: story.storyId ?? story.id,
- title: story.title,
- name: story.name,
- component: story.component,
- packageName: story.packageName,
- packageNames: story.packageNames,
- stableVisual: story.stableVisual,
- theme,
- mode,
- reasons: ['trusted:pr-scope'],
- };
- const keyed = {...shot, key: shotKey(shot)};
- if (existingBaselineOnly && !baselineKeys.has(keyed.key)) continue;
- shots.push(keyed);
- componentKeys?.add(keyed.key);
- }
- };
-
- const componentKeys = new Set();
- for (const {story, useThemeMatrix} of componentVisualStories(
- indexed,
- scope.stableComponents,
- )) {
- add(story, config.defaultTheme, componentKeys, true);
- if (!useThemeMatrix) continue;
- for (const theme of baselineThemes) {
- if (theme !== config.defaultTheme) {
- add(story, theme, componentKeys, true);
- }
- }
- }
- if (
- exceedsPrVisualShotLimit(
- componentKeys.size,
- config.prVisualShotLimit,
- )
- ) {
- fail(
- `trusted component plan has invalid size ${componentKeys.size}; focused PR limit is ${config.prVisualShotLimit}`,
- );
- }
-
- const themeStories = baselineVisualStories(indexed, baseline);
- for (const theme of scope.stableThemes) {
- if (themeStories.length === 0) {
- fail(`changed theme ${theme} has no accepted visual stories`);
- }
- for (const story of themeStories) add(story, theme);
- }
- }
- const unique = withThemeMetadata(
- [...new Map(shots.map(shot => [shot.key, shot])).values()],
- themeCatalog,
- );
- if (unique.some(shot => shot.stableThemeVisual !== true)) {
- fail('trusted stable plan includes a private or canary theme');
- }
- if (unique.length === 0) {
- fail(emptyVisualPlanMessage('The trusted component scope'));
- }
- if (unique.length > config.visualPlanSafetyLimit) {
- fail(
- `trusted visual plan has invalid size ${unique.length}; safety limit is ${config.visualPlanSafetyLimit}`,
- );
- }
- writeJSON(output, unique);
- process.stdout.write(
- `Wrote ${unique.length} trusted PR shot(s) to ${output}.\n`,
- );
-}
-
-function acceptedStableShots(acceptance) {
- if (acceptance.keys.some(entry => entry.kind !== 'changed')) {
- fail(
- 'scoped PR promotion can only update existing baseline frames',
- );
- }
- const shots = withThemeMetadata(
- acceptance.keys.map(entry => ({...entry.shot, key: entry.key})),
- themeCatalog,
- );
- if (shots.some(shot => shot.stableThemeVisual !== true)) {
- fail('accepted stable plan includes a private or canary theme');
- }
- return shots;
-}
-
-function plan() {
- const acceptanceFile = path.resolve(flag('acceptance'));
- const acceptance = readJSON(acceptanceFile);
- validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
- const output = path.resolve(flag('output'));
- const shots = acceptedStableShots(acceptance);
- writeJSON(output, shots);
- process.stdout.write(
- `Wrote ${shots.length} post-merge shot(s) to ${output}.\n`,
- );
-}
-
-function sameViewport(first, second) {
- return first?.width === second?.width && first?.height === second?.height;
-}
-
-function promote() {
- const pages = path.resolve(flag('pages'));
- const acceptanceFile = path.resolve(flag('acceptance'));
- const capture = path.resolve(flag('capture'));
- const mergeSha = flag('merge-sha') ?? '';
- if (!SHA.test(mergeSha)) fail('invalid merge SHA');
-
- const acceptance = readJSON(acceptanceFile);
- validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
- acceptedStableShots(acceptance);
- const manifestFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const manifest = readJSON(manifestFile);
- const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');
- const captureManifest = readJSON(path.join(capture, 'manifest.json'));
- const acceptedDir = path.dirname(acceptanceFile);
- const alreadyRecorded = (manifest.decisions ?? []).some(
- decision =>
- decision.pr === acceptance.pr &&
- decision.headSha === acceptance.headSha &&
- decision.mergeSha === mergeSha,
- );
-
- if (captureManifest.context?.sha !== mergeSha) {
- fail(
- `capture was produced for ${captureManifest.context?.sha ?? 'an unknown commit'}, not ${mergeSha}`,
- );
- }
- if (
- captureManifest.platform !== acceptance.capture.platform ||
- captureManifest.browser !== acceptance.capture.browser ||
- !sameViewport(captureManifest.viewport, acceptance.capture.viewport)
- ) {
- fail('post-merge capture environment does not match the reviewed capture');
- }
-
- const actions = [];
- for (const entry of acceptance.keys) {
- const baselineFile = path.join(baselineShots, `${entry.key}.png`);
- const currentBefore = fs.existsSync(baselineFile)
- ? shaFile(baselineFile)
- : null;
- if (currentBefore === entry.afterSha256 && alreadyRecorded) {
- actions.push({entry, baselineFile, alreadyPromoted: true});
- continue;
- }
- if (currentBefore !== entry.beforeSha256) {
- fail(
- `baseline conflict for ${entry.key}: expected ${entry.beforeSha256}, found ${currentBefore}`,
- );
- }
-
- const acceptedAfter = path.join(acceptedDir, 'after', `${entry.key}.png`);
- const recaptured = path.join(capture, 'shots', `${entry.key}.png`);
- const capturedShot = captureManifest.shots?.[entry.key];
- if (
- !fs.existsSync(acceptedAfter) ||
- shaFile(acceptedAfter) !== entry.afterSha256
- ) {
- fail(
- `archived AFTER does not match its decision record for ${entry.key}`,
- );
- }
- if (!fs.existsSync(recaptured) || !capturedShot) {
- fail(`post-merge capture is missing ${entry.key}`);
- }
- const rawRecapturedSha256 = shaFile(recaptured);
- if (capturedShot.sha256 !== rawRecapturedSha256) {
- fail(`post-merge manifest hash does not match ${entry.key}`);
- }
- let canonical;
- try {
- canonical = canonicalizePng(fs.readFileSync(recaptured));
- } catch (error) {
- fail(
- `post-merge capture is not a valid PNG for ${entry.key}: ${error.message}`,
- );
- }
- const canonicalSha256 = shaBytes(canonical.bytes);
- if (canonicalSha256 !== entry.afterSha256) {
- fail(
- `canonical post-merge hash does not match accepted AFTER for ${entry.key}`,
- );
- }
- for (const field of ['storyId', 'theme', 'mode']) {
- if (capturedShot[field] !== entry.shot[field]) {
- fail(
- `post-merge ${field} does not match accepted plan for ${entry.key}`,
- );
- }
- }
- if (
- capturedShot.width !== canonical.width ||
- capturedShot.height !== canonical.height
- ) {
- fail(`post-merge dimensions do not match ${entry.key}`);
- }
- actions.push({
- entry,
- baselineFile,
- canonicalBytes: canonical.bytes,
- capturedShot: {...capturedShot, sha256: canonicalSha256},
- });
- }
-
- if (actions.every(action => action.alreadyPromoted)) {
- process.stdout.write(
- `Accepted visual bundle for PR #${acceptance.pr} was already promoted.\n`,
- );
- return;
- }
-
- for (const action of actions) {
- if (action.alreadyPromoted) continue;
- fs.writeFileSync(action.baselineFile, action.canonicalBytes);
- manifest.shots[action.entry.key] = action.capturedShot;
- }
-
- manifest.capturedAt = captureManifest.capturedAt;
- manifest.platform = captureManifest.platform;
- manifest.browser = captureManifest.browser;
- manifest.viewport = captureManifest.viewport;
- manifest.context = {sha: mergeSha};
- manifest.decisions = [
- ...(manifest.decisions ?? []),
- {
- at: new Date().toISOString(),
- actor: acceptance.decision.approver,
- reason: acceptance.decision.reason,
- pr: acceptance.pr,
- headSha: acceptance.headSha,
- mergeSha,
- promoted: acceptance.keys.map(entry => entry.key),
- pruned: [],
- },
- ].slice(-200);
- writeJSON(manifestFile, manifest);
- process.stdout.write(
- `Promoted accepted visual bundle for PR #${acceptance.pr}.\n`,
- );
-}
-
-switch (command) {
- case 'accept':
- accept();
- break;
- case 'state':
- state();
- break;
- case 'trusted-defer':
- trustedDefer();
- break;
- case 'trusted-plan':
- trustedPlan();
- break;
- case 'plan':
- plan();
- break;
- case 'promote':
- promote();
- break;
- default:
- fail(
- 'usage: visual-acceptance.mjs <accept|state|trusted-defer|trusted-plan|plan|promote>',
- );
-}
diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjs
deleted file mode 100644
index c47a253516..0000000000
--- a/.github/scripts/visual-gate/visual-acceptance.test.mjs
+++ /dev/null
@@ -1,1323 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import {execFileSync, spawnSync} from 'node:child_process';
-import {createHash} from 'node:crypto';
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-import {afterEach, beforeEach, describe, expect, it} from 'vitest';
-import {PNG} from 'pngjs';
-
-import {canonicalizePng} from './lib/canonical-png.mjs';
-
-const SCRIPT = path.join(
- path.dirname(fileURLToPath(import.meta.url)),
- 'visual-acceptance.mjs',
-);
-const GATE = path.join(
- path.dirname(fileURLToPath(import.meta.url)),
- 'gate.mjs',
-);
-const HEAD = 'a'.repeat(40);
-const TESTED = 'b'.repeat(40);
-const MERGE = 'c'.repeat(40);
-const KEY = 'core-button--default__neutral-light';
-const SHOT = {
- storyId: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- component: 'Button',
- packageName: '@astryxdesign/core',
- packageNames: ['@astryxdesign/core'],
- stableVisual: true,
- theme: 'neutral',
- themePackageName: '@astryxdesign/theme-neutral',
- stableThemeVisual: true,
- mode: 'light',
- reasons: ['component:Button'],
-};
-
-let root;
-let pages;
-
-function png(red, green = 0, blue = 0) {
- const image = new PNG({width: 2, height: 2});
- for (let offset = 0; offset < image.data.length; offset += 4) {
- image.data[offset] = red;
- image.data[offset + 1] = green;
- image.data[offset + 2] = blue;
- image.data[offset + 3] = 255;
- }
- return PNG.sync.write(image);
-}
-
-function digest(bytes) {
- return createHash('sha256').update(bytes).digest('hex');
-}
-
-function writeJSON(file, value) {
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
-}
-
-function run(command, flags = {}) {
- const args = [SCRIPT, command];
- for (const [name, value] of Object.entries(flags))
- args.push(`--${name}`, String(value));
- return execFileSync(process.execPath, args, {encoding: 'utf8'});
-}
-
-function fail(command, flags = {}) {
- const args = [SCRIPT, command];
- for (const [name, value] of Object.entries(flags))
- args.push(`--${name}`, String(value));
- const result = spawnSync(process.execPath, args, {encoding: 'utf8'});
- expect(result.status).not.toBe(0);
- return result.stderr;
-}
-
-function acceptanceFlags(overrides = {}) {
- return {
- pages,
- pr: 42,
- head: HEAD,
- 'run-id': 123,
- 'run-attempt': 1,
- approver: 'maintainer',
- 'approver-id': 99,
- permission: 'maintain',
- 'effective-permission': 'maintain',
- 'comment-id': 1234,
- reason: 'The new radius matches the approved component design.',
- ...overrides,
- };
-}
-
-function acceptanceFile(run = 123, attempt = 1) {
- return path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- String(run),
- String(attempt),
- 'acceptance.json',
- );
-}
-
-function writeBaseline(entries = {[KEY]: png(255)}) {
- const shots = {};
- for (const [key, bytes] of Object.entries(entries)) {
- const file = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'shots',
- `${key}.png`,
- );
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, bytes);
- shots[key] = {...SHOT, key, sha256: digest(bytes), width: 2, height: 2};
- }
- writeJSON(path.join(pages, 'visual-gate', 'baseline', 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- shots,
- decisions: [],
- });
-}
-
-function writeEvidence({
- kind = 'changed',
- after = png(0, 0, 255),
- key = KEY,
- run = 123,
- attempt = 1,
- status = 'changed',
-} = {}) {
- const dir = path.join(
- pages,
- 'pr',
- '42',
- 'visual',
- HEAD,
- String(run),
- String(attempt),
- );
- fs.mkdirSync(path.join(dir, 'after'), {recursive: true});
- if (status === 'changed' && kind !== 'removed') {
- fs.writeFileSync(path.join(dir, 'after', `${key}.png`), after);
- }
- const deltas =
- status === 'changed'
- ? [
- {
- key,
- kind,
- beforeSha256:
- kind === 'added'
- ? null
- : digest(
- fs.readFileSync(
- path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'shots',
- `${key}.png`,
- ),
- ),
- ),
- shot: kind === 'removed' ? null : SHOT,
- },
- ]
- : [];
- writeJSON(path.join(dir, 'evidence.json'), {
- version: 1,
- repo: 'facebook/astryx',
- pr: 42,
- headSha: HEAD,
- testedSha: TESTED,
- baseSha: 'd'.repeat(40),
- run: {id: run, attempt},
- capture:
- status === 'skipped'
- ? null
- : {
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- },
- deltas,
- verdict:
- status === 'skipped'
- ? {
- version: 1,
- status,
- reason: 'Broad stable scope is deferred to the daily release gate.',
- counts: {
- total: 1,
- unchanged: 0,
- changed: 0,
- added: 0,
- removed: 0,
- failed: 0,
- },
- context: {
- sha: TESTED,
- headSha: HEAD,
- baseSha: 'd'.repeat(40),
- runId: String(run),
- runAttempt: String(attempt),
- },
- }
- : {version: 1, status},
- });
- return dir;
-}
-
-function commitPages() {
- execFileSync('git', ['add', '.'], {cwd: pages});
- execFileSync('git', ['commit', '-qm', 'fixture'], {cwd: pages});
-}
-
-beforeEach(() => {
- root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-acceptance-'));
- pages = path.join(root, 'pages');
- fs.mkdirSync(pages);
- execFileSync('git', ['init', '-q', '-b', 'gh-pages'], {cwd: pages});
- execFileSync('git', ['config', 'user.name', 'Test'], {cwd: pages});
- execFileSync('git', ['config', 'user.email', '[email protected]'], {
- cwd: pages,
- });
- writeBaseline();
- writeEvidence();
- commitPages();
-});
-
-afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
-
-describe('visual acceptance', () => {
- it('archives the reviewed pixels and a complete decision record', () => {
- expect(run('accept', acceptanceFlags())).toContain(
- 'Accepted 1 visual delta',
- );
- const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
- expect(record).toMatchObject({
- version: 1,
- pr: 42,
- headSha: HEAD,
- testedSha: TESTED,
- run: {id: 123, attempt: 1},
- capture: {platform: 'linux-arm64', browser: 'chromium-140.0'},
- decision: {
- approver: 'maintainer',
- approverId: 99,
- permission: 'maintain',
- commentId: 1234,
- },
- });
- expect(record.keys[0]).toMatchObject({key: KEY, kind: 'changed'});
- expect(record.keys[0].beforeSha256).toMatch(/^[0-9a-f]{64}$/);
- expect(record.keys[0].afterSha256).toMatch(/^[0-9a-f]{64}$/);
- expect(
- fs.existsSync(
- path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
- ),
- ).toBe(true);
- });
-
- it('accepts without installed packages in the archive job', () => {
- const isolated = path.join(root, 'isolated-visual-gate');
- fs.cpSync(path.dirname(SCRIPT), isolated, {recursive: true});
- const args = [path.join(isolated, 'visual-acceptance.mjs'), 'accept'];
- for (const [name, value] of Object.entries(acceptanceFlags()))
- args.push(`--${name}`, String(value));
- expect(execFileSync(process.execPath, args, {encoding: 'utf8'})).toContain(
- 'Accepted 1 visual delta',
- );
- });
-
- it('accepts effective maintain capability and records owner provenance', () => {
- run(
- 'accept',
- acceptanceFlags({
- approver: 'cixzhang',
- permission: 'write',
- 'effective-permission': 'maintain',
- 'role-name': 'Repo Owner',
- }),
- );
- expect(
- JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8')).decision,
- ).toMatchObject({
- approver: 'cixzhang',
- permission: 'write',
- effectivePermission: 'maintain',
- roleName: 'Repo Owner',
- });
- });
-
- it.each(['maintain', 'admin'])(
- 'accepts effective %s capability while preserving the reported role',
- effectivePermission => {
- run(
- 'accept',
- acceptanceFlags({
- permission: 'write',
- 'effective-permission': effectivePermission,
- 'role-name': 'Custom role',
- }),
- );
- expect(
- JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8')).decision,
- ).toMatchObject({
- permission: 'write',
- effectivePermission,
- roleName: 'Custom role',
- });
- },
- );
-
- it.each(['maintain', 'admin'])(
- 'validates a legacy %s record without role metadata',
- permission => {
- run('accept', acceptanceFlags({permission}));
- const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
- delete record.decision.roleName;
- delete record.decision.effectivePermission;
- writeJSON(acceptanceFile(), record);
- expect(
- JSON.parse(run('state', {pages, pr: 42, head: HEAD})),
- ).toMatchObject({
- state: 'success',
- reason: 'accepted',
- });
- },
- );
-
- it('rejects a legacy write record without effective capability data', () => {
- run(
- 'accept',
- acceptanceFlags({
- permission: 'write',
- 'effective-permission': 'maintain',
- }),
- );
- const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
- delete record.decision.roleName;
- delete record.decision.effectivePermission;
- writeJSON(acceptanceFile(), record);
- expect(fail('state', {pages, pr: 42, head: HEAD})).toMatch(
- /acceptance record is invalid/,
- );
- });
-
- it('requires an explanatory reason and maintainer permission', () => {
- expect(fail('accept', acceptanceFlags({reason: 'intentional...'}))).toMatch(
- /reason must explain/,
- );
- expect(
- fail(
- 'accept',
- acceptanceFlags({
- permission: 'write',
- 'effective-permission': 'write',
- 'role-name': 'Repo Owner',
- }),
- ),
- ).toMatch(/effective maintain\/admin/);
- expect(
- fail(
- 'accept',
- acceptanceFlags({
- permission: 'write',
- 'effective-permission': 'write',
- }),
- ),
- ).toMatch(/effective maintain\/admin/);
- });
-
- it('binds acceptance to the exact reviewed run attempt', () => {
- writeEvidence({run: 123, attempt: 2, after: png(0, 255, 0)});
- run('accept', acceptanceFlags({'run-id': 123, 'run-attempt': 1}));
- const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
- expect(record.run).toEqual({id: 123, attempt: 1});
- expect(record.keys[0].afterSha256).toBe(digest(png(0, 0, 255)));
- });
-
- it('reports whether the current visual bundle is clean, waiting, accepted, or failed', () => {
- const stateFlags = {pages, pr: 42, head: HEAD};
- expect(JSON.parse(run('state', stateFlags))).toMatchObject({
- state: 'pending',
- reason: 'decision',
- });
- run('accept', acceptanceFlags());
- expect(JSON.parse(run('state', stateFlags))).toMatchObject({
- state: 'success',
- reason: 'accepted',
- });
-
- writeEvidence({run: 124, attempt: 1, after: png(0, 0, 255)});
- expect(JSON.parse(run('state', stateFlags))).toMatchObject({
- state: 'pending',
- reason: 'decision',
- });
- writeEvidence({run: 124, attempt: 2, after: png(0, 255, 0)});
- expect(JSON.parse(run('state', stateFlags))).toMatchObject({
- state: 'pending',
- reason: 'decision',
- });
- writeEvidence({run: 125, status: 'skipped'});
- expect(JSON.parse(run('state', stateFlags))).toMatchObject({
- state: 'success',
- reason: 'deferred',
- });
- });
-
- it('rejects skipped evidence that claims a capture or deltas', () => {
- const stateFlags = {pages, pr: 42, head: HEAD};
- const dir = writeEvidence({run: 126, status: 'skipped'});
- const file = path.join(dir, 'evidence.json');
- const evidence = JSON.parse(fs.readFileSync(file, 'utf8'));
-
- evidence.capture = {
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- };
- writeJSON(file, evidence);
- expect(fail('state', stateFlags)).toMatch(
- /skipped evidence must not claim a capture or deltas/,
- );
-
- evidence.capture = null;
- evidence.deltas = [
- {
- key: KEY,
- kind: 'changed',
- beforeSha256: digest(png(255)),
- shot: SHOT,
- },
- ];
- writeJSON(file, evidence);
- expect(fail('state', stateFlags)).toMatch(
- /skipped evidence must not claim a capture or deltas/,
- );
- });
-
- it('requires skipped evidence to carry its trusted reason and shot count', () => {
- const stateFlags = {pages, pr: 42, head: HEAD};
- const dir = writeEvidence({run: 126, status: 'skipped'});
- const file = path.join(dir, 'evidence.json');
- const evidence = JSON.parse(fs.readFileSync(file, 'utf8'));
-
- evidence.verdict.reason = '';
- writeJSON(file, evidence);
- expect(fail('state', stateFlags)).toMatch(
- /skipped evidence must carry a trusted reason and count/,
- );
-
- evidence.verdict.reason = 'Broad stable scope is deferred.';
- evidence.verdict.counts.total = 0;
- writeJSON(file, evidence);
- expect(fail('state', stateFlags)).toMatch(
- /skipped evidence must carry a trusted reason and count/,
- );
- });
-
- it('rejects skipped evidence whose verdict names another run attempt', () => {
- const dir = writeEvidence({run: 126, attempt: 3, status: 'skipped'});
- const file = path.join(dir, 'evidence.json');
- const evidence = JSON.parse(fs.readFileSync(file, 'utf8'));
- evidence.verdict.context.runAttempt = '2';
- writeJSON(file, evidence);
-
- expect(fail('state', {pages, pr: 42, head: HEAD})).toMatch(
- /skipped evidence run identity mismatch/,
- );
- });
-
- it('refuses to accept an added baseline frame from a scoped PR', () => {
- const key = 'core-button--new__neutral-light';
- writeEvidence({kind: 'added', key});
- expect(fail('accept', acceptanceFlags())).toMatch(
- /changes to existing baseline frames/,
- );
- });
-
- it('returns success for a clean trusted capture without acceptance', () => {
- writeEvidence({run: 124, status: 'pass'});
- expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(
- {
- state: 'success',
- reason: 'clean',
- },
- );
- });
-
- it('derives a trusted component plan from baseline themes and the Storybook index', () => {
- const storybook = path.join(root, 'storybook');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- button: {
- type: 'story',
- id: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- },
- separator: {
- type: 'story',
- id: 'core-button--separator',
- title: 'Core/Button',
- name: 'Separator',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: ['visual-baseline'],
- },
- variants: {
- type: 'story',
- id: 'core-button--variants',
- title: 'Core/Button',
- name: 'Variants',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: ['visual-theme-matrix'],
- },
- fixture: {
- type: 'story',
- id: 'core-button--interaction-fixture',
- title: 'Core/Button',
- name: 'Interaction Fixture',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- },
- },
- });
- const baselineFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
- baseline.shots['core-button--default__y2k-light'] = {
- ...baseline.shots[KEY],
- key: 'core-button--default__y2k-light',
- theme: 'y2k',
- mode: 'light',
- };
- writeJSON(baselineFile, baseline);
- const scope = path.join(root, 'scope.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Button'],
- stableThemes: [],
- });
- const output = path.join(root, 'trusted-plan.json');
- run('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output,
- });
- expect(
- JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
- ).toEqual(['core-button--default__neutral-light']);
- });
-
- it('plans a published component whose only story is titled under another group', () => {
- const storybook = path.join(root, 'storybook-foreign-group');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- resizable: {
- type: 'story',
- id: 'lab-resizable--split',
- title: 'Lab/Resizable',
- name: 'Default',
- componentPath: '../../packages/core/src/Resizable/index.ts',
- tags: [],
- },
- },
- });
- const baselineFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
- baseline.shots['lab-resizable--split__neutral-light'] = {
- ...baseline.shots[KEY],
- key: 'lab-resizable--split__neutral-light',
- storyId: 'lab-resizable--split',
- title: 'Lab/Resizable',
- name: 'Default',
- component: 'Resizable',
- };
- writeJSON(baselineFile, baseline);
- const scope = path.join(root, 'scope-foreign-group.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Resizable'],
- stableThemes: [],
- });
- const output = path.join(root, 'foreign-group-plan.json');
- run('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output,
- });
- expect(
- JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
- ).toEqual(['lab-resizable--split__neutral-light']);
- });
-
- it('refuses an empty trusted plan in the words every lane refuses one', () => {
- const storybook = path.join(root, 'storybook-unseeded');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- card: {
- type: 'story',
- id: 'core-card--default',
- title: 'Core/Card',
- name: 'Default',
- componentPath: '../../packages/core/src/Card/index.ts',
- tags: [],
- },
- },
- });
- const scope = path.join(root, 'scope-unseeded.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Card'],
- stableThemes: [],
- });
- expect(
- fail('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output: path.join(root, 'unseeded-plan.json'),
- }),
- ).toMatch(
- /The trusted component scope planned no shots;.*manual baseline workflow\./,
- );
- });
-
- it('fails closed when any touched component has no representative story', () => {
- const storybook = path.join(root, 'storybook-missing-component');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- button: {
- type: 'story',
- id: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- },
- },
- });
- const scope = path.join(root, 'scope-missing-component.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Button', 'Card'],
- stableThemes: [],
- });
- expect(
- fail('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output: path.join(root, 'missing-component-plan.json'),
- }),
- ).toMatch(/Card/);
- });
-
- it('applies the focused PR shot limit to trusted tagged stories', () => {
- const storybook = path.join(root, 'storybook-shot-limit');
- fs.mkdirSync(storybook);
- const entries = {};
- for (let index = 0; index < 61; index += 1) {
- entries[`story-${index}`] = {
- type: 'story',
- id: `core-button--story-${index}`,
- title: 'Core/Button',
- name: index === 0 ? 'Default' : `Story ${index}`,
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: index === 0 ? [] : ['visual-theme-matrix'],
- };
- }
- writeJSON(path.join(storybook, 'index.json'), {entries});
- const baselineFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
- for (let index = 0; index < 61; index += 1) {
- const storyId = `core-button--story-${index}`;
- for (const theme of ['neutral', 'probe']) {
- for (const mode of ['light', 'dark']) {
- const key = `${storyId}__${theme}-${mode}`;
- baseline.shots[key] = {
- ...baseline.shots[KEY],
- key,
- storyId,
- name: entries[`story-${index}`].name,
- theme,
- themePackageName: `@astryxdesign/theme-${theme}`,
- mode,
- };
- }
- }
- }
- writeJSON(baselineFile, baseline);
- const scope = path.join(root, 'scope-shot-limit.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Button'],
- stableThemes: [],
- });
- expect(
- fail('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output: path.join(root, 'over-limit-plan.json'),
- }),
- ).toMatch(/244.*240/);
- });
-
- it('allows a theme-only trusted plan above the component review limit', () => {
- const storybook = path.join(root, 'storybook-large-theme');
- fs.mkdirSync(storybook);
- const entries = {};
- const shots = {};
- for (let index = 0; index < 121; index += 1) {
- const id = `core-button--theme-${index}`;
- const key = `${id}__neutral-light`;
- entries[`story-${index}`] = {
- type: 'story',
- id,
- title: 'Core/Button',
- name: index === 0 ? 'Default' : `Theme ${index}`,
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- };
- shots[key] = {
- ...SHOT,
- key,
- storyId: id,
- name: entries[`story-${index}`].name,
- theme: 'neutral',
- mode: 'light',
- };
- }
- writeJSON(path.join(storybook, 'index.json'), {entries});
- writeJSON(
- path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
- {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- shots,
- decisions: [],
- },
- );
- const scope = path.join(root, 'scope-large-theme.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: [],
- stableThemes: ['neutral'],
- });
- const output = path.join(root, 'large-theme-plan.json');
- run('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output,
- });
- const plan = JSON.parse(fs.readFileSync(output, 'utf8'));
- expect(plan).toHaveLength(242);
- expect(new Set(plan.map(shot => shot.theme))).toEqual(new Set(['neutral']));
- });
-
- it('limits changed themes to stories in the canonical baseline', () => {
- const storybook = path.join(root, 'storybook-existing-theme');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- button: {
- type: 'story',
- id: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- },
- card: {
- type: 'story',
- id: 'core-card--default',
- title: 'Core/Card',
- name: 'Default',
- componentPath: '../../packages/core/src/Card/index.ts',
- tags: [],
- },
- },
- });
- const baselineFile = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'manifest.json',
- );
- const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
- baseline.shots['core-card--default__y2k-light'] = {
- ...baseline.shots[KEY],
- key: 'core-card--default__y2k-light',
- storyId: 'core-card--default',
- title: 'Core/Card',
- component: 'Card',
- theme: 'y2k',
- mode: 'light',
- };
- writeJSON(baselineFile, baseline);
- const scope = path.join(root, 'scope-existing-theme.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: [],
- stableThemes: ['y2k'],
- });
- const output = path.join(root, 'existing-theme-plan.json');
- run('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output,
- });
- expect(
- JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
- ).toEqual([
- 'core-button--default__y2k-light',
- 'core-button--default__y2k-dark',
- ]);
- });
-
- it('plans every baseline story for a newly promoted stable theme', () => {
- const storybook = path.join(root, 'storybook-new-theme');
- fs.mkdirSync(storybook);
- writeJSON(path.join(storybook, 'index.json'), {
- entries: {
- button: {
- type: 'story',
- id: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- componentPath: '../../packages/core/src/Button/index.ts',
- tags: [],
- },
- },
- });
- const scope = path.join(root, 'scope-new-theme.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: [],
- stableThemes: ['y2k'],
- });
- const output = path.join(root, 'trusted-theme-plan.json');
- run('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': storybook,
- output,
- });
- expect(
- JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
- ).toEqual([
- 'core-button--default__y2k-light',
- 'core-button--default__y2k-dark',
- ]);
- });
-
- it('refuses a trusted capture plan for broad stable infrastructure', () => {
- const scope = path.join(root, 'scope-broad.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: true,
- stableComponents: [],
- stableThemes: [],
- });
- expect(
- fail('trusted-plan', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- 'storybook-dir': root,
- output: path.join(root, 'trusted-broad-plan.json'),
- }),
- ).toMatch(/broad stable scope must be deferred/);
- });
-
- it('writes trusted skipped evidence for broad stable scope', () => {
- const scope = path.join(root, 'scope-broad.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: true,
- stableComponents: ['Button', 'Card'],
- stableThemes: [],
- });
- const output = path.join(pages, 'pr', '42', 'visual', HEAD, '126', '3');
- expect(
- run('trusted-defer', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- output,
- pr: 42,
- head: HEAD,
- base: 'd'.repeat(40),
- 'run-id': 126,
- 'run-attempt': 3,
- }),
- ).toContain('Deferred 1 trusted baseline shot');
-
- const evidence = JSON.parse(
- fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'),
- );
- expect(evidence).toMatchObject({
- version: 1,
- repo: 'facebook/astryx',
- pr: 42,
- headSha: HEAD,
- testedSha: HEAD,
- baseSha: 'd'.repeat(40),
- run: {id: 126, attempt: 3},
- capture: null,
- deltas: [],
- verdict: {
- status: 'skipped',
- counts: {total: 1, changed: 0, added: 0, removed: 0, failed: 0},
- context: {
- sha: HEAD,
- headSha: HEAD,
- baseSha: 'd'.repeat(40),
- runId: '126',
- runAttempt: '3',
- trustedScope: {broadStableVisual: true},
- },
- },
- });
- expect(evidence.verdict.reason).toContain(
- 'Broad stable scope is deferred to the daily release gate',
- );
- expect(fs.readFileSync(path.join(output, 'index.html'), 'utf8')).toContain(
- 'Capture deferred',
- );
- expect(
- fs.readFileSync(path.join(output, 'index.html'), 'utf8'),
- ).not.toContain('/accept-visual');
- expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toEqual({
- state: 'success',
- reason: 'deferred',
- description: 'Broad stable scope is deferred to the release gate.',
- });
- });
-
- it('refuses trusted deferral for a narrow component scope', () => {
- const scope = path.join(root, 'scope-narrow.json');
- writeJSON(scope, {
- hasStableVisual: true,
- broadStableVisual: false,
- stableComponents: ['Button'],
- stableThemes: [],
- });
- expect(
- fail('trusted-defer', {
- scope,
- baseline: path.join(pages, 'visual-gate', 'baseline'),
- output: path.join(root, 'trusted-narrow-defer'),
- pr: 42,
- head: HEAD,
- base: 'd'.repeat(40),
- 'run-id': 126,
- 'run-attempt': 1,
- }),
- ).toMatch(/scope is not broad/);
- });
-
- it('allows a trusted 520-shot exact plan through the capture CLI', () => {
- const plan = Array.from({length: 520}, (_, index) => ({
- key: `story-${index}__neutral-light`,
- storyId: `story-${index}`,
- title: `Core/C${index}`,
- name: 'Default',
- component: `C${index}`,
- theme: 'neutral',
- mode: 'light',
- reasons: ['trusted:broad'],
- }));
- const file = path.join(root, 'large-plan.json');
- writeJSON(file, plan);
- const printed = execFileSync(
- process.execPath,
- [GATE, 'plan', '--plan-file', file],
- {
- encoding: 'utf8',
- },
- );
- expect(printed).toContain('520 shots');
- });
-
- it('rejects a trusted 520-delta added bundle', () => {
- const runId = 130;
- const dir = path.join(
- pages,
- 'pr',
- '42',
- 'visual',
- HEAD,
- String(runId),
- '1',
- );
- const deltas = [];
- fs.mkdirSync(path.join(dir, 'after'), {recursive: true});
- for (let index = 0; index < 520; index += 1) {
- const key = `story-${index}__neutral-light`;
- fs.writeFileSync(path.join(dir, 'after', `${key}.png`), png(index % 255));
- deltas.push({
- key,
- kind: 'added',
- beforeSha256: null,
- shot: {...SHOT, storyId: `story-${index}`},
- });
- }
- writeJSON(path.join(dir, 'evidence.json'), {
- version: 1,
- repo: 'facebook/astryx',
- pr: 42,
- headSha: HEAD,
- testedSha: TESTED,
- baseSha: 'd'.repeat(40),
- run: {id: runId, attempt: 1},
- capture: {
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- },
- deltas,
- verdict: {version: 1, status: 'changed'},
- });
- expect(
- fail('accept', acceptanceFlags({'run-id': runId})),
- ).toMatch(/changes to existing baseline frames/);
- expect(fs.existsSync(acceptanceFile(runId))).toBe(false);
- });
-
- it('emits an exact recapture plan without allowing stored metadata to replace the key', () => {
- run('accept', acceptanceFlags());
- const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
- record.keys[0].shot.key = '../wrong';
- writeJSON(acceptanceFile(), record);
- const output = path.join(root, 'plan.json');
- run('plan', {acceptance: acceptanceFile(), output});
- expect(JSON.parse(fs.readFileSync(output, 'utf8'))).toEqual([
- {...SHOT, key: KEY},
- ]);
- });
-
- it('feeds the accepted shots back into the capture CLI unchanged', () => {
- run('accept', acceptanceFlags());
- const output = path.join(root, 'plan.json');
- run('plan', {acceptance: acceptanceFile(), output});
- const printed = execFileSync(
- process.execPath,
- [GATE, 'plan', '--plan-file', output, '--json'],
- {
- encoding: 'utf8',
- },
- );
- expect(JSON.parse(printed)).toEqual([{...SHOT, key: KEY}]);
- });
-
- it('promotes only when merged pixels equal the reviewed after image', () => {
- run('accept', acceptanceFlags());
- const after = fs.readFileSync(
- path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
- );
- const capture = path.join(root, 'capture');
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), after);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-26T22:00:00.000Z',
- context: {sha: MERGE},
- shots: {[KEY]: {...SHOT, sha256: digest(after), width: 2, height: 2}},
- });
-
- run('promote', {
- pages,
- acceptance: acceptanceFile(),
- capture,
- 'merge-sha': MERGE,
- });
- run('promote', {
- pages,
- acceptance: acceptanceFile(),
- capture,
- 'merge-sha': MERGE,
- });
- expect(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- ),
- ).toEqual(after);
- const manifest = JSON.parse(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- );
- expect(manifest.context).toEqual({sha: MERGE});
- expect(manifest.decisions).toHaveLength(1);
- expect(manifest.decisions.at(-1)).toMatchObject({
- pr: 42,
- headSha: HEAD,
- mergeSha: MERGE,
- });
- });
-
- it('canonicalizes equivalent raw PNG bytes when promoting a changed shot', () => {
- const key = KEY;
- const runId = 123;
- run('accept', acceptanceFlags({'run-id': runId}));
- const record = acceptanceFile(runId);
- const accepted = fs.readFileSync(
- path.join(path.dirname(record), 'after', `${key}.png`),
- );
- const decoded = PNG.sync.read(accepted);
- decoded.gamma = 0.45455;
- const rawRecapture = PNG.sync.write(decoded, {deflateLevel: 0});
- expect(rawRecapture).not.toEqual(accepted);
- expect(PNG.sync.read(rawRecapture).data).toEqual(
- PNG.sync.read(accepted).data,
- );
- expect(canonicalizePng(rawRecapture).bytes).toEqual(accepted);
-
- const capture = path.join(root, 'capture-changed');
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${key}.png`), rawRecapture);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-26T22:00:00.000Z',
- context: {sha: MERGE},
- shots: {
- [key]: {
- ...SHOT,
- sha256: digest(rawRecapture),
- width: 2,
- height: 2,
- },
- },
- });
-
- expect(
- run('promote', {
- pages,
- acceptance: record,
- capture,
- 'merge-sha': MERGE,
- }),
- ).toContain('Promoted accepted visual bundle');
- expect(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${key}.png`),
- ),
- ).toEqual(accepted);
- const baselineManifest = JSON.parse(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- );
- expect(baselineManifest.shots[key].sha256).toBe(digest(accepted));
- });
-
- it('rejects removed PR evidence and preserves the baseline', () => {
- writeEvidence({kind: 'removed', run: 125});
- expect(fail('accept', acceptanceFlags({'run-id': 125}))).toMatch(
- /changes to existing baseline frames/,
- );
- expect(fs.existsSync(acceptanceFile(125))).toBe(false);
- expect(fs.existsSync(path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`))).toBe(true);
- });
-
- it('rejects a merged recapture whose canonical pixels changed', () => {
- run('accept', acceptanceFlags());
- const capture = path.join(root, 'capture-changed-pixels');
- const changed = png(0, 255, 0);
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), changed);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-26T22:00:00.000Z',
- context: {sha: MERGE},
- shots: {
- [KEY]: {...SHOT, sha256: digest(changed), width: 2, height: 2},
- },
- });
-
- expect(
- fail('promote', {
- pages,
- acceptance: acceptanceFile(),
- capture,
- 'merge-sha': MERGE,
- }),
- ).toMatch(/canonical post-merge hash does not match accepted AFTER/);
- });
-
- it('rejects a stale merge identity', () => {
- run('accept', acceptanceFlags());
- const capture = path.join(root, 'capture');
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), png(0, 255, 0));
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- context: {sha: 'e'.repeat(40)},
- shots: {
- [KEY]: {...SHOT, sha256: digest(png(0, 255, 0)), width: 2, height: 2},
- },
- });
- expect(
- fail('promote', {
- pages,
- acceptance: acceptanceFile(),
- capture,
- 'merge-sha': MERGE,
- }),
- ).toMatch(/capture was produced for/);
- });
-
- it('rejects a concurrent baseline change instead of overwriting it', () => {
- run('accept', acceptanceFlags());
- fs.writeFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- png(0, 255, 0),
- );
- expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(
- {
- state: 'pending',
- reason: 'capture',
- },
- );
- const capture = path.join(root, 'capture');
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- const accepted = fs.readFileSync(
- path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
- );
- fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), accepted);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- context: {sha: MERGE},
- shots: {[KEY]: {...SHOT, sha256: digest(accepted), width: 2, height: 2}},
- });
- expect(
- fail('promote', {
- pages,
- acceptance: acceptanceFile(),
- capture,
- 'merge-sha': MERGE,
- }),
- ).toMatch(/baseline conflict/);
- });
-
- it('rejects both added and removed scoped PR evidence', () => {
- const added = 'core-new--default__neutral-light';
- writeEvidence({kind: 'added', key: added, run: 124});
- expect(
- fail('accept', acceptanceFlags({'run-id': 124})),
- ).toMatch(/changes to existing baseline frames/);
- expect(fs.existsSync(acceptanceFile(124))).toBe(false);
-
- writeEvidence({kind: 'removed', run: 125});
- expect(
- fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125})),
- ).toMatch(/changes to existing baseline frames/);
- expect(fs.existsSync(acceptanceFile(125))).toBe(false);
- });
-});
diff --git a/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs b/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs
deleted file mode 100644
index 736c178fb9..0000000000
--- a/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-// Exercise the dispatch shell, including the interval that used to reject a
-// browser-refresh verdict before the publisher could validate its capture.
-import {spawnSync} from 'node:child_process';
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-
-import {afterEach, beforeEach, describe, expect, it} from 'vitest';
-import {parse} from 'yaml';
-
-const workflow = parse(
- fs.readFileSync(
- new URL('../../workflows/visual-baseline.yml', import.meta.url),
- 'utf8',
- ),
-);
-const steps = workflow.jobs.promote.steps;
-let root;
-let env;
-beforeEach(() => {
- root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-baseline-workflow-'));
- const bin = path.join(root, 'bin');
- fs.mkdirSync(bin);
- fs.writeFileSync(
- path.join(bin, 'gh'),
- '#!/bin/sh\nprintf "%s" "$RUN_JSON"\n',
- {mode: 0o755},
- );
- // Publication is covered against a local Git remote in the publisher tests.
- fs.writeFileSync(path.join(bin, 'node'), '#!/bin/sh\nexit 0\n', {
- mode: 0o755,
- });
- env = {
- ...process.env,
- PATH: `${bin}:${process.env.PATH}`,
- RUN_ID: '123',
- GITHUB_REPOSITORY: 'facebook/astryx',
- };
-});
-afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
-
-function run(script) {
- return spawnSync('bash', ['-e', '-c', script], {
- cwd: root,
- env,
- encoding: 'utf8',
- });
-}
-
-describe('Visual Baseline dispatch', () => {
- it.each([
- ['completed release', {}, true],
- ['failed terminal release', {conclusion: 'failure'}, true],
- ['wrong workflow', {path: '.github/workflows/ci.yml'}, false],
- ['wrong branch', {head_branch: 'feature'}, false],
- ['unfinished run', {status: 'in_progress'}, false],
- ['missing workflow', {path: null}, false],
- ])('validates the source run: %s', (_label, overrides, allowed) => {
- env.RUN_JSON = JSON.stringify({
- path: '.github/workflows/release-gate.yml',
- head_branch: 'main',
- status: 'completed',
- ...overrides,
- });
- const result = run(
- steps.find(step => step.name === 'Verify the source run').run,
- );
- expect(result.status === 0, result.stderr).toBe(allowed);
- });
-
- it('lets the publisher validate a failed browser-refresh capture inside its publication turn', () => {
- fs.mkdirSync(path.join(root, '.visual-run'));
- fs.writeFileSync(
- path.join(root, '.visual-run/verdict.json'),
- JSON.stringify({status: 'failed'}),
- );
- const download = steps.findIndex(
- step => step.name === 'Download the capture being promoted',
- );
- const promote = steps.findIndex(
- step => step.name === 'Promote and publish the baseline',
- );
- expect(download).toBeGreaterThan(-1);
- expect(promote).toBeGreaterThan(download);
- const beforePromotion = steps
- .slice(download + 1, promote)
- .map(step => step.run)
- .filter(Boolean);
- expect(beforePromotion.join('\n')).toContain(
- 'gh-pages-publisher.mjs wait --scope visual-gate/baseline',
- );
- const result = run(beforePromotion.join('\n'));
- expect(result.status, result.stderr || result.stdout).toBe(0);
- expect(steps[promote].run).toContain(
- 'gh-pages-publisher.mjs visual-baseline-manual',
- );
- expect(
- steps.find(step => step.name === 'Release the baseline publication turn')
- .if,
- ).toBe('always()');
- });
-});
diff --git a/.github/scripts/visual-gate/visual-promotion-workflow.test.mjs b/.github/scripts/visual-gate/visual-promotion-workflow.test.mjs
deleted file mode 100644
index b129f307ee..0000000000
--- a/.github/scripts/visual-gate/visual-promotion-workflow.test.mjs
+++ /dev/null
@@ -1,611 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-import {execFileSync, spawnSync} from 'node:child_process';
-import {createHash} from 'node:crypto';
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-
-import {afterEach, describe, expect, it} from 'vitest';
-import {PNG} from 'pngjs';
-
-const ROOT = path.resolve(
- path.dirname(fileURLToPath(import.meta.url)),
- '../../..',
-);
-const SCRIPT = path.join(
- ROOT,
- '.github',
- 'scripts',
- 'visual-gate',
- 'visual-acceptance.mjs',
-);
-const GATE = path.join(ROOT, '.github', 'scripts', 'visual-gate', 'gate.mjs');
-const PUBLISHER = path.join(
- ROOT,
- '.github',
- 'scripts',
- 'lib',
- 'gh-pages-publisher.mjs',
-);
-const WORKFLOW = path.join(
- ROOT,
- '.github',
- 'workflows',
- 'visual-acceptance-promote.yml',
-);
-const HEAD = 'a'.repeat(40);
-const MERGE = 'c'.repeat(40);
-const KEY = 'core-button--default__neutral-light';
-const RECORD_REL = '123/1/acceptance.json';
-const SHOT = {
- storyId: 'core-button--default',
- title: 'Core/Button',
- name: 'Default',
- component: 'Button',
- theme: 'neutral',
- mode: 'light',
- reasons: ['component:Button'],
-};
-
-const roots = [];
-
-afterEach(() => {
- for (const root of roots.splice(0)) {
- fs.rmSync(root, {recursive: true, force: true});
- }
-});
-
-function png(red, green = 0, blue = 0) {
- const image = new PNG({width: 2, height: 2});
- for (let offset = 0; offset < image.data.length; offset += 4) {
- image.data[offset] = red;
- image.data[offset + 1] = green;
- image.data[offset + 2] = blue;
- image.data[offset + 3] = 255;
- }
- return PNG.sync.write(image);
-}
-
-function digest(bytes) {
- return createHash('sha256').update(bytes).digest('hex');
-}
-
-function writeJSON(file, value) {
- fs.mkdirSync(path.dirname(file), {recursive: true});
- fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
-}
-
-function git(cwd, ...args) {
- return execFileSync('git', args, {cwd, encoding: 'utf8'}).trim();
-}
-
-function runAcceptance(command, flags) {
- const args = [SCRIPT, command];
- for (const [name, value] of Object.entries(flags)) {
- args.push(`--${name}`, String(value));
- }
- return execFileSync(process.execPath, args, {encoding: 'utf8'});
-}
-
-function workflowStep(name) {
- const workflow = fs.readFileSync(WORKFLOW, 'utf8');
- const start = workflow.indexOf(` - name: ${name}\n`);
- expect(start).toBeGreaterThan(-1);
- const next = workflow.indexOf('\n - name:', start + 1);
- return workflow.slice(start, next === -1 ? undefined : next);
-}
-
-function workflowStepScript(name) {
- const step = workflowStep(name);
- for (const [marker, fold] of [
- [' run: |\n', false],
- [' run: >\n', true],
- ]) {
- const runStart = step.indexOf(marker);
- if (runStart === -1) continue;
- const lines = step
- .slice(runStart + marker.length)
- .split('\n')
- .map(line => (line.startsWith(' ') ? line.slice(10) : line));
- if (!fold) return lines.join('\n');
- return `${lines
- .map(line => line.trim())
- .filter(Boolean)
- .join(' ')}\n`;
- }
- throw new Error(`Step ${name} has no runnable script`);
-}
-
-function writeCommandShims(root) {
- const bin = path.join(root, 'bin');
- fs.mkdirSync(bin);
- const gitShim = path.join(bin, 'git');
- fs.writeFileSync(
- gitShim,
- [
- '#!/bin/bash',
- 'set -e',
- 'REAL_GIT="$TEST_REAL_GIT"',
- 'if [ "${1:-}" = "clone" ]; then',
- ' args=("$@")',
- ' for ((i = 0; i < ${#args[@]}; i += 1)); do',
- ' case "${args[$i]}" in',
- ' https://x-access-token:*) args[$i]="file://${TEST_REMOTE}" ;;',
- ' esac',
- ' done',
- ' exec "$REAL_GIT" "${args[@]}"',
- 'fi',
- 'if [ "${1:-}" = "-C" ] && [ "${3:-}" = "push" ] && [ "${TEST_INJECT_RACE:-0}" = "1" ] && [ ! -e "$TEST_RACE_MARKER" ]; then',
- ' CLEANUP="$TEST_ROOT/cleanup-clone"',
- ' "$REAL_GIT" clone -q --branch gh-pages "file://${TEST_REMOTE}" "$CLEANUP"',
- ' "$REAL_GIT" -C "$CLEANUP" config user.name Test',
- ' "$REAL_GIT" -C "$CLEANUP" config user.email [email protected]',
- ' rm -rf "$CLEANUP/pr/${PR_NUMBER}"',
- ' "$REAL_GIT" -C "$CLEANUP" add -A',
- ' "$REAL_GIT" -C "$CLEANUP" commit -qm "cleanup merged preview"',
- ' "$REAL_GIT" -C "$CLEANUP" push -q origin gh-pages',
- ' touch "$TEST_RACE_MARKER"',
- 'fi',
- 'exec "$REAL_GIT" "$@"',
- '',
- ].join('\n'),
- );
- fs.chmodSync(gitShim, 0o755);
-
- const ghShim = path.join(bin, 'gh');
- fs.writeFileSync(ghShim, '#!/bin/sh\nprintf "%s\\n" "$GH_RUNS_JSON"\n');
- fs.chmodSync(ghShim, 0o755);
-
- const sleepShim = path.join(bin, 'sleep');
- fs.writeFileSync(sleepShim, '#!/bin/sh\nexit 0\n');
- fs.chmodSync(sleepShim, 0o755);
- return bin;
-}
-
-function makeFixture(mutate) {
- const root = fs.mkdtempSync(
- path.join(os.tmpdir(), 'visual-promotion-workflow-'),
- );
- roots.push(root);
- const pages = path.join(root, 'pages');
- fs.mkdirSync(pages);
- git(pages, 'init', '-q', '-b', 'gh-pages');
- git(pages, 'config', 'user.name', 'Test');
- git(pages, 'config', 'user.email', '[email protected]');
-
- const before = png(255);
- const after = png(0, 0, 255);
- const baselineShot = path.join(
- pages,
- 'visual-gate',
- 'baseline',
- 'shots',
- `${KEY}.png`,
- );
- fs.mkdirSync(path.dirname(baselineShot), {recursive: true});
- fs.writeFileSync(baselineShot, before);
- writeJSON(path.join(pages, 'visual-gate', 'baseline', 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- shots: {
- [KEY]: {...SHOT, key: KEY, sha256: digest(before), width: 2, height: 2},
- },
- decisions: [],
- });
-
- const evidence = path.join(pages, 'pr', '42', 'visual', HEAD, '123', '1');
- fs.mkdirSync(path.join(evidence, 'after'), {recursive: true});
- fs.writeFileSync(path.join(evidence, 'after', `${KEY}.png`), after);
- writeJSON(path.join(evidence, 'evidence.json'), {
- version: 1,
- repo: 'facebook/astryx',
- pr: 42,
- headSha: HEAD,
- testedSha: 'b'.repeat(40),
- baseSha: 'd'.repeat(40),
- run: {id: 123, attempt: 1},
- capture: {
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- },
- deltas: [
- {
- key: KEY,
- kind: 'changed',
- beforeSha256: digest(before),
- shot: SHOT,
- },
- ],
- verdict: {version: 1, status: 'changed'},
- });
- git(pages, 'add', '.');
- git(pages, 'commit', '-qm', 'fixture');
-
- runAcceptance('accept', {
- pages,
- pr: 42,
- head: HEAD,
- 'run-id': 123,
- 'run-attempt': 1,
- approver: 'maintainer',
- 'approver-id': 99,
- permission: 'maintain',
- 'effective-permission': 'maintain',
- 'comment-id': 1234,
- reason: 'The new radius matches the approved component design.',
- });
-
- const acceptance = path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- RECORD_REL,
- );
- const sandbox = path.join(root, 'sandbox');
- const sandboxScripts = path.join(sandbox, '.github', 'scripts');
- fs.mkdirSync(sandboxScripts, {recursive: true});
- for (const entry of ['visual-gate', 'lib', 'gh-pages-publisher.mjs']) {
- fs.symlinkSync(
- path.join(ROOT, '.github', 'scripts', entry),
- path.join(sandboxScripts, entry),
- );
- }
- const plan = path.join(sandbox, 'accepted-plan.json');
- runAcceptance('plan', {acceptance, output: plan});
-
- const capture = path.join(sandbox, '.visual-merged');
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), after);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-27T08:00:00.000Z',
- context: {sha: MERGE},
- shots: {[KEY]: {...SHOT, sha256: digest(after), width: 2, height: 2}},
- });
-
- mutate?.({acceptance, baselineShot, capture, pages});
- writeJSON(
- path.join(pages, '.astryx-gh-pages', 'publication-queue', '900.json'),
- {
- version: 1,
- repository: 'facebook/astryx',
- runId: 900,
- scope: 'visual-gate/baseline',
- },
- );
- writeJSON(
- path.join(pages, '.astryx-gh-pages', 'publication-queue', 'holder.json'),
- {
- version: 1,
- repository: 'facebook/astryx',
- runId: 900,
- scope: 'visual-gate/baseline',
- },
- );
- writeJSON(path.join(pages, 'visual-gate', 'publication-queue', '900.json'), {
- version: 1,
- repository: 'facebook/astryx',
- runId: 900,
- });
- writeJSON(
- path.join(pages, 'visual-gate', 'publication-queue', 'holder.json'),
- {
- version: 1,
- repository: 'facebook/astryx',
- runId: 900,
- },
- );
- git(pages, 'add', '.');
- git(pages, 'commit', '-qm', 'record acceptance');
-
- const remote = path.join(root, 'remote.git');
- git(root, 'clone', '-q', '--bare', pages, remote);
- const bin = writeCommandShims(root);
- return {root, remote, sandbox, bin, after};
-}
-
-function runPromotion({
- fixture,
- latest = {id: 123, run_attempt: 1, status: 'completed'},
- race = false,
-}) {
- const marker = path.join(fixture.root, 'race-injected');
- const runnerTemp = path.join(fixture.root, 'runner');
- fs.mkdirSync(runnerTemp, {recursive: true});
- const result = spawnSync(
- '/bin/bash',
- ['-c', workflowStepScript('Verify and promote the baseline')],
- {
- cwd: fixture.sandbox,
- encoding: 'utf8',
- env: {
- ...process.env,
- PATH: `${fixture.bin}:${process.env.PATH}`,
- GH_TOKEN: 'test-token',
- GITHUB_REPOSITORY: 'facebook/astryx',
- GITHUB_RUN_ID: '900',
- PR_NUMBER: '42',
- HEAD_SHA: HEAD,
- MERGE_SHA: MERGE,
- RECORD_REL,
- RUNNER_TEMP: runnerTemp,
- GITHUB_OUTPUT: path.join(fixture.root, 'github-output'),
- GH_RUNS_JSON: JSON.stringify({
- workflow_runs: [{name: 'CI', ...latest}],
- }),
- TEST_INJECT_RACE: race ? '1' : '0',
- TEST_RACE_MARKER: marker,
- TEST_REAL_GIT: execFileSync('which', ['git'], {
- encoding: 'utf8',
- }).trim(),
- TEST_REMOTE: fixture.remote,
- TEST_ROOT: fixture.root,
- },
- },
- );
- return {...result, marker};
-}
-
-describe('visual acceptance promotion workflow', () => {
- it('captures the resolved historical merge when workflow main differs', () => {
- const currentMain = 'f'.repeat(40);
- const root = fs.mkdtempSync(
- path.join(os.tmpdir(), 'visual-promotion-capture-identity-'),
- );
- roots.push(root);
- const plan = path.join(root, 'plan.json');
- const output = path.join(root, 'output');
- writeJSON(plan, [{...SHOT, key: KEY}]);
-
- const step = workflowStep('Recapture exactly the accepted shots');
- const gate = fs.readFileSync(GATE, 'utf8');
- expect(step).toContain(
- 'ASTRYX_VISUAL_SHA: ${{ needs.resolve.outputs.merge_sha }}',
- );
- expect(step).not.toContain('GITHUB_SHA:');
- const manifestContext = gate.slice(
- gate.indexOf('manifest.context = {'),
- gate.indexOf('fs.writeFileSync(', gate.indexOf('manifest.context = {')),
- );
- expect(manifestContext).toContain('...captureIdentity(),');
-
- execFileSync(
- process.execPath,
- [GATE, 'check', '--plan-file', plan, '--max-shots', '0', '--out', output],
- {
- encoding: 'utf8',
- env: {
- ...process.env,
- GITHUB_SHA: currentMain,
- ASTRYX_VISUAL_SHA: MERGE,
- },
- },
- );
- expect(
- JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'))
- .context.sha,
- ).toBe(MERGE);
- });
-
- it('shares the merge-bound recapture between recovery and normal close', () => {
- const workflow = fs.readFileSync(WORKFLOW, 'utf8');
- const capture = workflowStep('Recapture exactly the accepted shots');
- const checkout = workflowStep('Checkout the resolved merged result');
-
- expect(workflow).toContain('workflow_dispatch:');
- expect(workflow).toContain('types: [closed]');
- expect(
- workflow.match(/- name: Recapture exactly the accepted shots/g),
- ).toHaveLength(1);
- expect(capture).toContain(
- 'ASTRYX_VISUAL_SHA: ${{ needs.resolve.outputs.merge_sha }}',
- );
- expect(capture).toContain(
- '--storybook-dir .visual-merged-source/apps/storybook/dist',
- );
- expect(capture).toContain(
- 'node .github/scripts/visual-gate/gate.mjs capture',
- );
- expect(checkout).toContain('ref: ${{ needs.resolve.outputs.merge_sha }}');
- expect(checkout).toContain('allow-unsafe-pr-checkout: true');
- });
-
- it('executes the migrated publisher from the workflow sandbox root', () => {
- const fixture = makeFixture();
- const script = workflowStepScript('Verify and promote the baseline');
- expect(script).toContain('gh-pages-publisher.mjs visual-baseline-accepted');
- expect(
- fs.existsSync(
- path.join(
- fixture.sandbox,
- '.github',
- 'scripts',
- 'gh-pages-publisher.mjs',
- ),
- ),
- ).toBe(true);
- expect(
- fs.existsSync(
- path.join(
- fixture.sandbox,
- '.github',
- 'scripts',
- 'lib',
- 'gh-pages-publisher.mjs',
- ),
- ),
- ).toBe(true);
- const result = runPromotion({fixture});
- expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
- });
-
- it('retries from the immutable record after cleanup wins the first push race', () => {
- const fixture = makeFixture();
- const result = runPromotion({fixture, race: true});
- expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
- expect(fs.existsSync(result.marker)).toBe(true);
-
- const final = path.join(fixture.root, 'final');
- git(
- fixture.root,
- 'clone',
- '-q',
- '--branch',
- 'gh-pages',
- fixture.remote,
- final,
- );
- expect(fs.existsSync(path.join(final, 'pr', '42'))).toBe(false);
- expect(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- ),
- ).toEqual(fixture.after);
- const manifest = JSON.parse(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- );
- expect(manifest.decisions.at(-1)).toMatchObject({
- pr: 42,
- headSha: HEAD,
- mergeSha: MERGE,
- });
- });
-
- it('is idempotent when the same recovery is dispatched twice', () => {
- const fixture = makeFixture();
- const first = runPromotion({fixture});
- expect(first.status, `${first.stdout}\n${first.stderr}`).toBe(0);
- const second = runPromotion({fixture});
- expect(second.status, `${second.stdout}\n${second.stderr}`).toBe(0);
- expect(second.stdout).toContain(
- 'Accepted visual bundle for PR #42 was already promoted.',
- );
-
- const final = path.join(fixture.root, 'duplicate-final');
- git(
- fixture.root,
- 'clone',
- '-q',
- '--branch',
- 'gh-pages',
- fixture.remote,
- final,
- );
- const manifest = JSON.parse(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- );
- expect(manifest.decisions).toHaveLength(1);
- expect(
- fs.readFileSync(
- path.join(final, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- ),
- ).toEqual(fixture.after);
- });
-
- it.each([
- {
- name: 'a stale current pointer',
- mutate: ({pages}) =>
- writeJSON(
- path.join(
- pages,
- 'visual-gate',
- 'acceptances',
- '42',
- HEAD,
- 'current.json',
- ),
- {version: 1, run: 124, attempt: 1, record: '124/1/acceptance.json'},
- ),
- error: /newer visual acceptance superseded/,
- },
- {
- name: 'a superseded CI run',
- latest: {id: 124, run_attempt: 1, status: 'completed'},
- error: /not from the latest completed CI attempt/,
- },
- {
- name: 'the wrong head identity',
- mutate: ({acceptance}) => {
- const record = JSON.parse(fs.readFileSync(acceptance, 'utf8'));
- record.headSha = 'e'.repeat(40);
- writeJSON(acceptance, record);
- },
- error: /record identity does not match this merged head/,
- },
- {
- name: 'a run identity that disagrees with the immutable path',
- mutate: ({acceptance}) => {
- const record = JSON.parse(fs.readFileSync(acceptance, 'utf8'));
- record.run.id = 124;
- writeJSON(acceptance, record);
- },
- latest: {id: 124, run_attempt: 1, status: 'completed'},
- error: /record identity does not match this merged head/,
- },
- {
- name: 'the wrong merged capture identity',
- mutate: ({capture}) => {
- const manifestFile = path.join(capture, 'manifest.json');
- const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8'));
- manifest.context.sha = 'e'.repeat(40);
- writeJSON(manifestFile, manifest);
- },
- error: /capture was produced for/,
- },
- {
- name: 'tampered archived pixels',
- mutate: ({acceptance}) =>
- fs.writeFileSync(
- path.join(path.dirname(acceptance), 'after', `${KEY}.png`),
- png(0, 255, 0),
- ),
- error: /archived AFTER is missing or changed/,
- },
- {
- name: 'a changed baseline preimage',
- mutate: ({baselineShot}) =>
- fs.writeFileSync(baselineShot, png(0, 255, 0)),
- error: /baseline conflict/,
- },
- ])('fails closed for $name', ({mutate, latest, error}) => {
- const fixture = makeFixture(mutate);
- const result = runPromotion({fixture, latest});
- expect(result.status).not.toBe(0);
- expect(`${result.stdout}\n${result.stderr}`).toMatch(error);
- });
-
- it('does not consult ephemeral PR evidence during promotion', () => {
- const script = workflowStepScript('Verify and promote the baseline');
- const publisher = fs.readFileSync(PUBLISHER, 'utf8');
- const promotion = publisher.slice(
- publisher.indexOf('export async function publishAcceptedVisualBaseline'),
- publisher.indexOf('export async function publishManualVisualBaseline'),
- );
-
- expect(script).toContain('gh-pages-publisher.mjs visual-baseline-accepted');
- expect(script).toContain('--expected-record-rel "$RECORD_REL"');
- expect(promotion).toContain('promotion-identity.mjs');
- expect(promotion).toContain('resolve-acceptance');
- expect(promotion).toContain('visual-acceptance.mjs');
- expect(promotion).toContain('promote');
- expect(promotion).not.toContain('visual-acceptance.mjs state');
- expect(promotion).not.toContain("'pr'");
- });
-});
diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
index bb95b2751a..68a40f3c09 100644
--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs
+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
@@ -16,38 +16,44 @@ function workflow(name) {
return fs.readFileSync(path.join(WORKFLOWS, name), 'utf8');
}
-describe('visual acceptance workflow concurrency', () => {
+describe('PR report and deployment workflow contracts', () => {
+ it('publishes canonical CI artifacts without another capture, comparison, or status owner', () => {
+ const value = workflow('pr-comment.yml');
+ expect(value).toContain('name: visual-pr-report');
+ expect(value).toContain('run-id: ${{ steps.identity.outputs.run_id }}');
+ expect(value).toContain('--head-sha "$HEAD_SHA"');
+ expect(value).toContain('--base-sha "$BASE_SHA"');
+ expect(value).toContain('--run-attempt "$RUN_ATTEMPT"');
+ expect(value).toContain('publish-pr-report.mjs');
+ expect(value).toContain('types: [completed]');
+ expect(value).not.toMatch(/gate\.mjs (?:check|capture|release)/);
+ expect(value).not.toContain('playwright');
+ expect(value).not.toContain('createCommitStatus');
+ expect(value).not.toContain('visual-approved');
+ const publisher = fs.readFileSync(
+ path.join(ROOT, '.github/scripts/visual-gate/publish-pr-report.mjs'),
+ 'utf8',
+ );
+ expect(publisher).not.toMatch(/from ['"].*(?:capture|compare)\.mjs['"]/);
+ });
+
it('locks PR report resolution before any mutation', () => {
const value = workflow('pr-comment.yml');
const [header, jobs] = value.split('\njobs:\n');
expect(header).toContain(
- 'group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}',
- );
- expect(header).toContain(
- "cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}",
+ 'group: pr-report-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}',
);
+ expect(header).toContain('cancel-in-progress: false');
expect(jobs).not.toContain('visual-acceptance-pr-');
expect(jobs).not.toContain(' concurrency:');
});
- it('keeps status initialization out of pull-request checks', () => {
- const acceptance = workflow('visual-acceptance.yml');
- const publisher = workflow('pr-comment.yml');
-
- expect(acceptance).not.toContain('pull_request_target:');
- expect(acceptance).not.toContain(' initialize:');
- expect(publisher).toContain('types: [requested, in_progress, completed]');
- expect(publisher).toContain(
- 'description: `CI run ${run.id}/${run.run_attempt} is producing fresh visual evidence.`',
- );
- });
-
it('uses the shared metadata validator for exact and legacy PR artifacts', () => {
const value = workflow('pr-comment.yml');
const crossCheck = value.slice(
value.indexOf(' - name: Cross-check artifact identity'),
- value.indexOf(' - name: Fetch the trusted visual baseline'),
+ value.indexOf(' - name: Prepare canonical visual report'),
);
expect(crossCheck).toContain('import {validateAnalysisMetadata}');
@@ -71,29 +77,6 @@ describe('visual acceptance workflow concurrency', () => {
expect(comment).not.toContain('const previewAvailable');
});
- it('requires published evidence before offering or recording acceptance', () => {
- const publisher = workflow('pr-comment.yml');
- const acceptance = workflow('visual-acceptance.yml');
- const authorize = acceptance.slice(
- acceptance.indexOf(' authorize:'),
- acceptance.indexOf(' accept:'),
- );
-
- expect(publisher).toContain(
- "visualPublished: process.env.VISUAL_PUBLISHED === 'true'",
- );
- expect(authorize).toContain('visualAcceptanceEvidencePath({');
- expect(authorize).toContain('github.rest.repos.getContent({');
- expect(authorize).toContain("ref: 'gh-pages'");
- expect(authorize).toContain('trusted visual evidence is not published yet');
- expect(authorize.indexOf("latestCI.status !== 'completed'")).toBeLessThan(
- authorize.indexOf('github.rest.repos.getContent({'),
- );
- expect(authorize.indexOf('github.rest.repos.getContent({')).toBeLessThan(
- authorize.indexOf('isVisualAcceptanceEndpointMaintainer(identity)'),
- );
- });
-
it('keeps spec-only checks lightweight while removing stale links', () => {
const value = workflow('pr-comment.yml');
const reconcile = value.slice(
@@ -112,225 +95,6 @@ describe('visual acceptance workflow concurrency', () => {
expect(deploy).toContain("needs.resolve.outputs.spec_only != 'true'");
});
- it('keeps comment authorization read-only until the shared lock is held', () => {
- const value = workflow('visual-acceptance.yml');
- const authorize = value.slice(
- value.indexOf(' authorize:'),
- value.indexOf(' accept:'),
- );
- const authorizeCheckout = authorize.slice(
- authorize.indexOf(' - name: Checkout trusted default-branch code'),
- authorize.indexOf(' - name: Authorize and resolve the decision'),
- );
- const accept = value.slice(value.indexOf(' accept:'));
-
- expect(authorize).not.toContain(': write');
- expect(authorize).toContain('actions/checkout@v7');
- expect(authorizeCheckout).not.toContain('ref:');
- expect(authorize).toContain('visualAcceptanceIdentity(response.data)');
- expect(authorize).toContain(
- 'isVisualAcceptanceEndpointMaintainer(identity)',
- );
- expect(authorize).toContain(
- "core.setOutput('effective_permission', identity.effectivePermission)",
- );
- expect(authorize).toContain(
- "core.setOutput('role_name', identity.roleName ?? '')",
- );
- expect(authorize).not.toContain('author_association');
- expect(authorize).not.toContain('issues.createComment');
- expect(authorize).not.toContain('createCommitStatus');
- expect(accept).toContain('needs: authorize');
- expect(accept).toContain(
- 'EFFECTIVE_PERMISSION: ${{ needs.authorize.outputs.effective_permission }}',
- );
- expect(accept).toContain(
- 'ROLE_NAME: ${{ needs.authorize.outputs.role_name }}',
- );
- expect(accept).toContain('--effective-permission "$EFFECTIVE_PERMISSION"');
- expect(accept).toContain('--role-name "$ROLE_NAME"');
- expect(accept).toContain(
- 'group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}',
- );
- expect(accept).toContain('cancel-in-progress: false');
- expect(accept).toContain('pull-requests: write');
- expect(accept).not.toContain('issues: write');
- });
-
- it('invalidates the advisory label only in the trusted workflow_run publisher', () => {
- const value = workflow('pr-comment.yml');
- const invalidate = value.slice(
- value.indexOf(' invalidate:'),
- value.indexOf(' comment:'),
- );
-
- expect(invalidate).toContain('pull-requests: write');
- expect(invalidate).toContain('statuses: write');
- expect(invalidate).toContain('createCommitStatus');
- expect(invalidate).toContain('issues.removeLabel');
- expect(invalidate.indexOf('createCommitStatus')).toBeLessThan(
- invalidate.indexOf('issues.removeLabel'),
- );
- const publisher = value.slice(value.indexOf(' comment:'));
- expect(publisher).toContain("if (state.reason === 'accepted')");
- expect(publisher).toContain('issues.addLabels');
- expect(publisher).toContain('issues.removeLabel');
- });
-
- it('passes source CI identity without overriding reserved GitHub variables', () => {
- const value = workflow('pr-comment.yml');
- const capture = value.slice(
- value.indexOf(' - name: Capture the trusted stable visual scope'),
- value.indexOf(' # The Storybook bundle is untrusted.'),
- );
-
- expect(capture).toContain(
- 'ASTRYX_VISUAL_SHA: ${{ steps.identity.outputs.head_sha }}',
- );
- expect(capture).toContain(
- 'ASTRYX_VISUAL_RUN_ID: ${{ steps.identity.outputs.run_id }}',
- );
- expect(capture).toContain(
- 'ASTRYX_VISUAL_RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}',
- );
- expect(capture).not.toContain('GITHUB_SHA:');
- expect(capture).not.toContain('GITHUB_RUN_ID:');
- expect(capture).not.toContain('GITHUB_RUN_ATTEMPT:');
- });
-
- it('defers broad trusted scopes before downloading or capturing Storybook', () => {
- const value = workflow('pr-comment.yml');
- const download = value.slice(
- value.indexOf(
- ' - name: Download Storybook for trusted visual capture',
- ),
- value.indexOf(' - name: Cross-check artifact identity'),
- );
- const capture = value.slice(
- value.indexOf(' - name: Capture the trusted stable visual scope'),
- value.indexOf(' - name: Derive trusted broad visual deferral'),
- );
- const defer = value.slice(
- value.indexOf(' - name: Derive trusted broad visual deferral'),
- value.indexOf(' # The Storybook bundle is untrusted.'),
- );
- const derive = value.slice(
- value.indexOf(' - name: Derive trusted visual evidence and report'),
- value.indexOf(' - name: Resolve trusted visual evidence path'),
- );
-
- const resolve = value.slice(
- value.indexOf(' - name: Resolve trusted visual evidence path'),
- value.indexOf(' - name: Publish immutable visual evidence'),
- );
-
- expect(download).toContain("steps.scope.outputs.broad != 'true'");
- expect(capture).toContain("steps.scope.outputs.broad != 'true'");
- expect(defer).toContain("steps.scope.outputs.broad == 'true'");
- expect(defer).toContain('visual-acceptance.mjs trusted-defer');
- expect(defer).toContain('--run-attempt "$RUN_ATTEMPT"');
- expect(defer).not.toContain('playwright');
- expect(defer).not.toContain('gate.mjs capture');
- expect(derive).toContain("steps.scope.outputs.broad != 'true'");
- expect(resolve).toContain('test -f trusted-visual/evidence.json');
- expect(resolve).toContain(
- 'path=pr/${PR_NUMBER}/visual/${HEAD_SHA}/${RUN_ID}/${RUN_ATTEMPT}',
- );
- });
-
- it('uses the documented collaborator permission response shape', () => {
- const value = workflow('visual-acceptance.yml');
- const authorize = value.slice(
- value.indexOf(' authorize:'),
- value.indexOf(' accept:'),
- );
-
- expect(authorize).toContain('visualAcceptanceIdentity(response.data)');
- expect(authorize).not.toContain('response.data.user.permission');
- });
-
- it('installs the dependencies used by the acceptance archive script', () => {
- const value = workflow('visual-acceptance.yml');
- const accept = value.slice(value.indexOf(' accept:'));
-
- expect(accept.indexOf('uses: ./.github/actions/setup')).toBeGreaterThan(-1);
- expect(accept.indexOf('uses: ./.github/actions/setup')).toBeLessThan(
- accept.indexOf(
- 'node .github/scripts/gh-pages-publisher.mjs visual-acceptance-record',
- ),
- );
- });
-
- it('serializes normal, recovery, and manual publication without Actions cancellation', () => {
- const value = workflow('visual-acceptance-promote.yml');
- const manual = workflow('visual-baseline.yml');
- const helper = 'node .github/scripts/gh-pages-publisher.mjs';
-
- expect(value).toContain('workflow_dispatch:');
- expect(value).toContain("context.ref !== 'refs/heads/main'");
- expect(value).toContain('Checkout trusted current main');
- expect(value).toContain('Checkout the resolved merged result');
- expect(value).toContain('allow-unsafe-pr-checkout: true');
- expect(value.indexOf('compareCommitsWithBasehead')).toBeLessThan(
- value.indexOf('allow-unsafe-pr-checkout: true'),
- );
- expect(value).toContain('ref: main');
- for (const command of ['enqueue', 'wait', 'release']) {
- expect(
- value.match(
- new RegExp(`${helper} ${command} --scope visual-gate/baseline`, 'g'),
- ),
- ).toHaveLength(1);
- expect(
- manual.match(
- new RegExp(`${helper} ${command} --scope visual-gate/baseline`, 'g'),
- ),
- ).toHaveLength(1);
- }
- expect(value).not.toContain('group: visual-baseline');
- expect(manual).not.toContain('group: visual-baseline');
- expect(value.indexOf('Recapture exactly the accepted shots')).toBeLessThan(
- value.indexOf('Wait for the baseline publication turn'),
- );
- expect(
- value.indexOf('Wait for the baseline publication turn'),
- ).toBeLessThan(value.indexOf('Verify and promote the baseline'));
- expect(value).toContain('visual-baseline-accepted');
- expect(value).toContain('--expected-record-rel "$RECORD_REL"');
- expect(workflow('pr-comment.yml')).toContain(
- 'Post-merge promotion does not join this cancellation group',
- );
- expect(
- manual.indexOf('Wait for the baseline publication turn'),
- ).toBeLessThan(manual.indexOf('Promote and publish the baseline'));
- expect(workflow('release-gate.yml')).toContain(
- 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',
- );
- expect(value).toContain(
- "context.eventName === 'workflow_dispatch' ? 'true' : 'false'",
- );
- });
-
- it('publishes the stable site and release gate through the shared gh-pages publisher', () => {
- const deploy = workflow('deploy.yml');
- const releaseGate = workflow('release-gate.yml');
-
- expect(deploy).toContain('Checkout publisher');
- expect(deploy).toContain(
- 'node .github/scripts/gh-pages-publisher.mjs stable-site --source staged',
- );
- expect(deploy).not.toContain('/tmp/ghp');
- expect(releaseGate).toContain('Checkout publisher');
- expect(releaseGate).toContain(
- 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',
- );
- const releasePublisher = releaseGate.slice(
- releaseGate.indexOf(' publish:'),
- );
- expect(releasePublisher).not.toContain('release-gate-publish');
- expect(releasePublisher).not.toContain('/tmp/gh-pages');
- });
-
it('enforces the main-push stable-site chain through the Actions Pages deployer', () => {
const stableSite = workflow('deploy.yml');
const pages = workflow('pages-deploy.yml');
@@ -395,87 +159,6 @@ describe('visual acceptance workflow concurrency', () => {
expect(deploy).toBeGreaterThan(publish);
});
- it('grants queued gh-pages publishers read access to overlapping workflow runs', () => {
- const deploy = workflow('deploy.yml');
- const deployJob = deploy.slice(
- deploy.indexOf(' deploy:'),
- deploy.indexOf(' - name: Checkout publisher'),
- );
- const releaseGate = workflow('release-gate.yml');
- const publishJob = releaseGate.slice(
- releaseGate.indexOf(' publish:'),
- releaseGate.indexOf(' - name: Checkout publisher'),
- );
-
- expect(deployJob).toContain('actions: read');
- expect(deployJob).toContain('contents: write');
- expect(deploy).toContain('gh-pages-publisher.mjs stable-site');
- expect(publishJob).toContain('actions: read');
- expect(publishJob).toContain('contents: write');
- expect(releaseGate).toContain('gh-pages-publisher.mjs release-gate');
- });
-
- it('moves high-conflict visual gh-pages writers behind the shared publisher', () => {
- const comment = workflow('pr-comment.yml');
- const acceptance = workflow('visual-acceptance.yml');
- const promote = workflow('visual-acceptance-promote.yml');
- const manual = workflow('visual-baseline.yml');
-
- expect(comment).toContain('gh-pages-publisher.mjs immutable-path');
- expect(comment).toContain('--scope pr-visual/evidence');
- expect(comment).not.toContain('Immutable evidence already published');
- expect(acceptance).toContain(
- 'gh-pages-publisher.mjs visual-acceptance-record',
- );
- expect(acceptance).not.toContain('DEST="visual-gate/acceptances');
- expect(promote).toContain(
- 'gh-pages-publisher.mjs visual-baseline-accepted',
- );
- expect(promote).not.toContain('baseline-publication-lock.mjs');
- expect(manual).toContain('gh-pages-publisher.mjs visual-baseline-manual');
- expect(manual).not.toContain('baseline-publication-lock.mjs');
- expect(manual).not.toContain('git rebase origin/gh-pages');
- });
-
- it('grants visual publisher jobs read access to overlapping workflow runs', () => {
- const comment = workflow('pr-comment.yml');
- const commentStart = comment.indexOf(' comment:');
- const commentJob = comment.slice(
- commentStart,
- comment.indexOf(
- ' - name: Checkout trusted default-branch code',
- commentStart,
- ),
- );
- const acceptance = workflow('visual-acceptance.yml');
- const acceptJob = acceptance.slice(
- acceptance.indexOf(' accept:'),
- acceptance.indexOf(
- ' - name: Reject an invalid visual acceptance request',
- ),
- );
- const promote = workflow('visual-acceptance-promote.yml');
- const promoteJob = promote.slice(
- promote.indexOf(' promote:'),
- promote.indexOf(
- ' - name: Checkout trusted current main',
- promote.indexOf(' promote:'),
- ),
- );
- const manual = workflow('visual-baseline.yml');
- const manualJob = manual.slice(
- manual.indexOf(' promote:'),
- manual.indexOf(' - name: Checkout'),
- );
-
- for (const job of [commentJob, acceptJob, manualJob]) {
- expect(job).toContain('actions: read');
- expect(job).toContain('contents: write');
- }
- expect(promoteJob).toContain('actions: write');
- expect(promoteJob).toContain('contents: write');
- });
-
it('verifies exact source identity before checking preview artifacts', () => {
const value = workflow('deploy-preview.yml');
const resolve = value.indexOf(
@@ -591,92 +274,4 @@ describe('visual acceptance workflow concurrency', () => {
expect(value).not.toContain('ref: gh-pages');
}
});
-
- it('projects every known validation or publication failure from an always-running job', () => {
- const value = workflow('visual-acceptance-promote.yml');
- const status = value.slice(value.indexOf(' project-status:'));
-
- expect(status).toContain('if: always()');
- expect(status).toContain('statuses: write');
- expect(status).toContain('needs: [resolve, promote]');
- expect(status).toContain('needs.resolve.outputs.failure_description');
- expect(status).toContain('needs.promote.outputs.failure_description');
- expect(status).toContain('promotionStatusProjection');
- expect(status).toContain('target_url: process.env.TARGET_URL');
- expect(status).toContain("if (projection.state === 'failure')");
- expect(status).toContain('core.setFailed(projection.description)');
- expect(value.match(/core\.setOutput\('head_sha'/g)).toHaveLength(1);
- expect(value.indexOf('compareCommitsWithBasehead')).toBeLessThan(
- value.indexOf("core.setOutput('head_sha'"),
- );
- expect(value).toContain('core.setFailed(failure.description)');
- const beforeStatus = value.slice(0, value.indexOf(' project-status:'));
- expect(beforeStatus).toContain("state: 'pending'");
- expect(beforeStatus).not.toContain("state: 'success'");
- expect(beforeStatus).not.toContain("state: 'failure'");
- const promoteJob = value.slice(
- value.indexOf(' promote:'),
- value.indexOf(' project-status:'),
- );
- expect(promoteJob).toContain(
- "needs.resolve.outputs.acceptance_found == 'true'",
- );
- expect(
- promoteJob.slice(0, promoteJob.indexOf(' runs-on:')),
- ).not.toContain('mutation_deferred');
- expect(promoteJob).toContain('Confirm trusted active-retry deferral');
- expect(promoteJob).toContain('deferred=true');
- expect(promoteJob).toContain("if: steps.defer.outputs.deferred != 'true'");
- expect(value).toContain(
- "mutation_deferred: ${{ steps.defer.outputs.deferred == 'true' || steps.acceptance.outputs.deferred == 'true' || steps.promote.outputs.deferred == 'true' }}",
- );
- expect(status).toContain(
- "MUTATION_DEFERRED: ${{ needs.promote.outputs.mutation_deferred == 'true' }}",
- );
- expect(status).not.toContain('needs.resolve.outputs.mutation_deferred');
- const projectionCall = status.slice(
- status.indexOf('const projection = promotionStatusProjection({'),
- );
- expect(projectionCall).toContain(
- 'promotionResult: recoveryOperationResult({',
- );
- expect(projectionCall).toContain(
- 'mutationDeferred: process.env.MUTATION_DEFERRED',
- );
- expect(projectionCall).toContain(
- 'failureDescription: process.env.FAILURE_DESCRIPTION',
- );
- });
-
- it('marks success only after publication, gate dispatch, and lock release finish', () => {
- const value = workflow('visual-acceptance-promote.yml');
- const gate = value.slice(
- value.indexOf(' - name: Run a fresh release gate'),
- value.indexOf(' - name: Release the baseline publication turn'),
- );
- const release = value.slice(
- value.indexOf(' - name: Release the baseline publication turn'),
- value.indexOf(' - name: Mark trusted recovery complete'),
- );
- const complete = value.slice(
- value.indexOf(' - name: Mark trusted recovery complete'),
- value.indexOf(' project-status:'),
- );
-
- expect(value).toContain("publication_confirmed == 'true'");
- expect(value).toContain('visual-baseline-accepted');
- expect(gate).toContain(
- "steps.promote.outputs.publication_confirmed == 'true'",
- );
- expect(gate).toContain('fresh release gate dispatch failed');
- expect(release).toContain('if: always()');
- expect(release).toContain('lock release failed');
- expect(complete).toContain("steps.gate.outcome == 'success'");
- expect(complete).toContain("steps.release.outcome == 'success'");
- expect(complete).toContain('recovery_complete=true');
- expect(value).toContain(
- 'RECOVERY_COMPLETE: ${{ needs.promote.outputs.recovery_complete }}',
- );
- expect(value).toContain('recoveryOperationResult({');
- });
});
diff --git a/.github/scripts/visual-maintenance-source.mjs b/.github/scripts/visual-maintenance-source.mjs
new file mode 100644
index 0000000000..6afd810e62
--- /dev/null
+++ b/.github/scripts/visual-maintenance-source.mjs
@@ -0,0 +1,160 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Resolve a CI-owned baseline maintenance capture before publication.
+ * @input GitHub workflow/run/job/artifact identity and the downloaded manifest
+ * @output An exact source artifact, or a refusal; never a pixel acceptance
+ * @position Trusted source boundary for ci.yml's publication-only maintenance job
+ */
+
+import {validateReleasePlan} from './visual-gate/lib/compare.mjs';
+
+function refuse(message) {
+ throw new Error(`Visual maintenance source refused: ${message}`);
+}
+
+function positiveInteger(value, label) {
+ if (!/^[1-9][0-9]*$/.test(String(value))) refuse(`${label} is invalid`);
+ const result = Number(value);
+ if (!Number.isSafeInteger(result)) refuse(`${label} is invalid`);
+ return result;
+}
+
+/** Resolve only the named attempt's completed canonical capture on main. */
+export async function resolveVisualMaintenanceSource({
+ github,
+ owner,
+ repo,
+ runId,
+ runAttempt,
+}) {
+ const id = positiveInteger(runId, 'run id');
+ const attempt = positiveInteger(runAttempt, 'run attempt');
+ const repository = `${owner}/${repo}`;
+ const [{data: workflow}, {data: run}] = await Promise.all([
+ github.rest.actions.getWorkflow({owner, repo, workflow_id: 'ci.yml'}),
+ github.rest.actions.getWorkflowRunAttempt({
+ owner,
+ repo,
+ run_id: id,
+ attempt_number: attempt,
+ }),
+ ]);
+ if (
+ workflow.path !== '.github/workflows/ci.yml' ||
+ run.workflow_id !== workflow.id ||
+ run.path !== workflow.path ||
+ run.id !== id ||
+ run.run_attempt !== attempt ||
+ run.repository?.full_name !== repository ||
+ run.head_repository?.full_name !== repository ||
+ !Number.isSafeInteger(run.repository?.id) ||
+ run.head_repository?.id !== run.repository.id ||
+ run.event !== 'workflow_dispatch' ||
+ run.head_branch !== 'main' ||
+ !/^[0-9a-f]{40}$/.test(run.head_sha ?? '') ||
+ run.status !== 'completed' ||
+ !['success', 'failure'].includes(run.conclusion)
+ ) {
+ refuse('expected an exact completed main CI maintenance attempt');
+ }
+
+ const {data: comparison} = await github.rest.repos.compareCommitsWithBasehead(
+ {
+ owner,
+ repo,
+ basehead: `${run.head_sha}...main`,
+ },
+ );
+ if (!['identical', 'ahead'].includes(comparison.status)) {
+ refuse('capture commit is not reachable from main');
+ }
+
+ const jobs = await github.paginate(
+ github.rest.actions.listJobsForWorkflowRunAttempt,
+ {owner, repo, run_id: id, attempt_number: attempt, per_page: 100},
+ );
+ const captures = jobs.filter(job => job.name === 'Stable visual regression');
+ if (
+ captures.length !== 1 ||
+ captures[0].status !== 'completed' ||
+ !['success', 'failure'].includes(captures[0].conclusion) ||
+ !captures[0].steps?.some(
+ step =>
+ step.name === 'Validate canonical baseline capture' &&
+ step.conclusion === 'success',
+ )
+ ) {
+ refuse(
+ 'the named attempt did not complete a canonical maintenance capture',
+ );
+ }
+
+ const artifacts = await github.paginate(
+ github.rest.actions.listWorkflowRunArtifacts,
+ {owner, repo, run_id: id, per_page: 100},
+ );
+ const name = `visual-baseline-capture-${id}-${attempt}`;
+ const matches = artifacts.filter(artifact => artifact.name === name);
+ if (matches.length !== 1)
+ refuse('expected one exact maintenance capture artifact');
+ const artifact = matches[0];
+ const source = artifact.workflow_run;
+ if (
+ !Number.isSafeInteger(artifact.id) ||
+ artifact.id <= 0 ||
+ artifact.expired !== false ||
+ source?.id !== id ||
+ source.repository_id !== run.repository.id ||
+ source.head_repository_id !== run.repository.id ||
+ source.head_branch !== 'main' ||
+ source.head_sha !== run.head_sha
+ ) {
+ refuse(
+ 'capture artifact identity does not match the trusted source attempt',
+ );
+ }
+ return {
+ artifactId: artifact.id,
+ runId: id,
+ runAttempt: attempt,
+ sha: run.head_sha,
+ };
+}
+
+/**
+ * A complete capture may have a failed comparison when the browser changes.
+ * Preserve that explicit maintenance recovery without blessing a partial capture
+ * or weakening gate.mjs accept's existing failed-verdict pruning restriction.
+ */
+export function validateVisualMaintenanceCapture({
+ manifest,
+ verdict,
+ sha,
+ runId,
+ runAttempt,
+}) {
+ const context = manifest?.context;
+ if (
+ !/^[0-9a-f]{40}$/.test(sha ?? '') ||
+ context?.sha !== sha ||
+ context.ref !== 'refs/heads/main' ||
+ String(context.runId) !== String(positiveInteger(runId, 'run id')) ||
+ String(context.runAttempt) !==
+ String(positiveInteger(runAttempt, 'run attempt'))
+ ) {
+ refuse('capture manifest does not match the resolved source identity');
+ }
+ const plan = validateReleasePlan(context.releasePlan, manifest);
+ if (!plan.keys.length) refuse('canonical capture is empty');
+ if (
+ verdict?.context?.sha !== context.sha ||
+ verdict.context.ref !== context.ref ||
+ String(verdict.context.runId) !== String(context.runId) ||
+ String(verdict.context.runAttempt) !== String(context.runAttempt) ||
+ JSON.stringify(verdict.context.releasePlan) !== JSON.stringify(plan) ||
+ !['pass', 'changed', 'failed'].includes(verdict.status)
+ ) {
+ refuse('capture verdict does not match its canonical source plan');
+ }
+}
diff --git a/.github/scripts/visual-maintenance-source.test.mjs b/.github/scripts/visual-maintenance-source.test.mjs
new file mode 100644
index 0000000000..51f8e9383e
--- /dev/null
+++ b/.github/scripts/visual-maintenance-source.test.mjs
@@ -0,0 +1,394 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file CI maintenance source-boundary contracts.
+ * @input Independent GitHub response and canonical capture fixtures
+ * @output Refusals for foreign, stale, incomplete, or wrongly routed evidence
+ * @position Node tests for the publication-only baseline maintenance boundary
+ */
+
+import {createHash} from 'node:crypto';
+
+import {describe, expect, it, vi} from 'vitest';
+
+import {
+ resolveVisualMaintenanceSource,
+ validateVisualMaintenanceCapture,
+} from './visual-maintenance-source.mjs';
+
+const SHA = 'a'.repeat(40);
+const KEY = 'core-button--default__neutral-light';
+
+function sourceFixture() {
+ const workflow = {id: 17, path: '.github/workflows/ci.yml'};
+ const run = {
+ id: 101,
+ run_attempt: 2,
+ workflow_id: 17,
+ path: '.github/workflows/ci.yml',
+ repository: {id: 9, full_name: 'facebook/astryx'},
+ head_repository: {id: 9, full_name: 'facebook/astryx'},
+ head_branch: 'main',
+ head_sha: SHA,
+ event: 'workflow_dispatch',
+ status: 'completed',
+ conclusion: 'success',
+ };
+ const comparison = {status: 'ahead'};
+ const jobs = [
+ {
+ name: 'Stable visual regression',
+ status: 'completed',
+ conclusion: 'success',
+ steps: [
+ {name: 'Validate canonical baseline capture', conclusion: 'success'},
+ ],
+ },
+ ];
+ const artifacts = [
+ {
+ id: 901,
+ name: 'visual-baseline-capture-101-2',
+ expired: false,
+ workflow_run: {
+ id: 101,
+ repository_id: 9,
+ head_repository_id: 9,
+ head_branch: 'main',
+ head_sha: SHA,
+ },
+ },
+ ];
+ const actions = {
+ getWorkflow: vi.fn(async () => ({data: workflow})),
+ getWorkflowRunAttempt: vi.fn(async () => ({data: run})),
+ listJobsForWorkflowRunAttempt: vi.fn(),
+ listWorkflowRunArtifacts: vi.fn(),
+ };
+ const github = {
+ rest: {
+ actions,
+ repos: {
+ compareCommitsWithBasehead: vi.fn(async () => ({data: comparison})),
+ },
+ },
+ paginate: vi.fn(async method => {
+ if (method === actions.listJobsForWorkflowRunAttempt) return jobs;
+ if (method === actions.listWorkflowRunArtifacts) return artifacts;
+ throw new Error('Unexpected pagination endpoint');
+ }),
+ };
+ return {workflow, run, comparison, jobs, artifacts, github};
+}
+
+const resolve = fixture =>
+ resolveVisualMaintenanceSource({
+ github: fixture.github,
+ owner: 'facebook',
+ repo: 'astryx',
+ runId: '101',
+ runAttempt: '2',
+ });
+
+function captureFixture() {
+ const keys = [KEY];
+ const context = {
+ sha: SHA,
+ ref: 'refs/heads/main',
+ runId: '101',
+ runAttempt: '2',
+ releasePlan: {
+ version: 1,
+ lane: 'stable-release',
+ authority: 'report-removals',
+ keys,
+ digest: createHash('sha256').update(JSON.stringify(keys)).digest('hex'),
+ },
+ };
+ return {
+ manifest: {
+ context,
+ shots: {[KEY]: {stableVisual: true, stableThemeVisual: true}},
+ },
+ verdict: {context: structuredClone(context), status: 'changed'},
+ sha: SHA,
+ runId: 101,
+ runAttempt: 2,
+ };
+}
+
+describe('CI-owned baseline maintenance source', () => {
+ it('resolves the reviewed attempt and exact artifact, not the latest named run', async () => {
+ const fx = sourceFixture();
+ expect(await resolve(fx)).toEqual({
+ artifactId: 901,
+ runId: 101,
+ runAttempt: 2,
+ sha: SHA,
+ });
+ expect(fx.github.rest.actions.getWorkflow).toHaveBeenCalledWith({
+ owner: 'facebook',
+ repo: 'astryx',
+ workflow_id: 'ci.yml',
+ });
+ expect(fx.github.rest.actions.getWorkflowRunAttempt).toHaveBeenCalledWith({
+ owner: 'facebook',
+ repo: 'astryx',
+ run_id: 101,
+ attempt_number: 2,
+ });
+ expect(fx.github.paginate).toHaveBeenCalledWith(
+ fx.github.rest.actions.listJobsForWorkflowRunAttempt,
+ {
+ owner: 'facebook',
+ repo: 'astryx',
+ run_id: 101,
+ attempt_number: 2,
+ per_page: 100,
+ },
+ );
+ });
+
+ it('allows an explicit complete capture whose comparison failed during browser refresh', async () => {
+ const fx = sourceFixture();
+ fx.run.conclusion = 'failure';
+ fx.jobs[0].conclusion = 'failure';
+ expect(await resolve(fx)).toMatchObject({artifactId: 901});
+ const capture = captureFixture();
+ capture.verdict.status = 'failed';
+ capture.verdict.failures = [{key: 'baseline', error: 'browser differs'}];
+ expect(() => validateVisualMaintenanceCapture(capture)).not.toThrow();
+ });
+
+ it.each([
+ [
+ 'foreign workflow',
+ fx => {
+ fx.run.workflow_id = 18;
+ },
+ ],
+ [
+ 'renamed workflow path',
+ fx => {
+ fx.workflow.path = '.github/workflows/other.yml';
+ },
+ ],
+ [
+ 'wrong run',
+ fx => {
+ fx.run.id = 102;
+ },
+ ],
+ [
+ 'wrong attempt',
+ fx => {
+ fx.run.run_attempt = 1;
+ },
+ ],
+ [
+ 'foreign repository',
+ fx => {
+ fx.run.repository.full_name = 'other/astryx';
+ },
+ ],
+ [
+ 'fork head',
+ fx => {
+ fx.run.head_repository.full_name = 'fork/astryx';
+ },
+ ],
+ [
+ 'mismatched repository id',
+ fx => {
+ fx.run.head_repository.id = 10;
+ },
+ ],
+ [
+ 'PR source',
+ fx => {
+ fx.run.event = 'pull_request';
+ },
+ ],
+ [
+ 'non-main dispatch',
+ fx => {
+ fx.run.head_branch = 'feature';
+ },
+ ],
+ [
+ 'unfinished source',
+ fx => {
+ fx.run.status = 'in_progress';
+ },
+ ],
+ [
+ 'canceled source',
+ fx => {
+ fx.run.conclusion = 'cancelled';
+ },
+ ],
+ [
+ 'unreachable commit',
+ fx => {
+ fx.comparison.status = 'diverged';
+ },
+ ],
+ [
+ 'wrong visual job',
+ fx => {
+ fx.jobs[0].name = 'Other capture';
+ },
+ ],
+ [
+ 'promote-only source',
+ fx => {
+ fx.jobs[0].steps = [];
+ },
+ ],
+ [
+ 'failed capture validation',
+ fx => {
+ fx.jobs[0].steps[0].conclusion = 'failure';
+ },
+ ],
+ [
+ 'artifact from previous attempt',
+ fx => {
+ fx.artifacts[0].name = 'visual-baseline-capture-101-1';
+ },
+ ],
+ [
+ 'expired artifact',
+ fx => {
+ fx.artifacts[0].expired = true;
+ },
+ ],
+ [
+ 'artifact from foreign run',
+ fx => {
+ fx.artifacts[0].workflow_run.id = 102;
+ },
+ ],
+ [
+ 'artifact from foreign repo',
+ fx => {
+ fx.artifacts[0].workflow_run.repository_id = 10;
+ },
+ ],
+ [
+ 'artifact from foreign head',
+ fx => {
+ fx.artifacts[0].workflow_run.head_sha = 'b'.repeat(40);
+ },
+ ],
+ [
+ 'missing artifact',
+ fx => {
+ fx.artifacts.length = 0;
+ },
+ ],
+ [
+ 'ambiguous artifact',
+ fx => {
+ fx.artifacts.push(structuredClone(fx.artifacts[0]));
+ },
+ ],
+ ])('refuses %s before downloading bytes', async (_label, mutate) => {
+ const fx = sourceFixture();
+ mutate(fx);
+ await expect(resolve(fx)).rejects.toThrow(/refused/);
+ });
+
+ it.each(['', '0', '-1', '1.2', '1/2', '9007199254740992'])(
+ 'rejects invalid source id %j',
+ async runId => {
+ const fx = sourceFixture();
+ await expect(
+ resolveVisualMaintenanceSource({
+ github: fx.github,
+ owner: 'facebook',
+ repo: 'astryx',
+ runId,
+ runAttempt: 2,
+ }),
+ ).rejects.toThrow(/run id is invalid/);
+ expect(fx.github.rest.actions.getWorkflow).not.toHaveBeenCalled();
+ },
+ );
+});
+
+describe('downloaded canonical capture identity', () => {
+ it('accepts the full canonical source without changing the baseline or verdict', () => {
+ const fx = captureFixture();
+ const before = structuredClone(fx);
+ validateVisualMaintenanceCapture(fx);
+ expect(fx).toEqual(before);
+ });
+
+ it.each([
+ [
+ 'wrong captured commit',
+ fx => {
+ fx.manifest.context.sha = 'b'.repeat(40);
+ },
+ ],
+ [
+ 'wrong captured ref',
+ fx => {
+ fx.manifest.context.ref = 'refs/heads/feature';
+ },
+ ],
+ [
+ 'wrong captured run',
+ fx => {
+ fx.manifest.context.runId = '102';
+ },
+ ],
+ [
+ 'wrong captured attempt',
+ fx => {
+ fx.manifest.context.runAttempt = '1';
+ },
+ ],
+ [
+ 'missing captured shot',
+ fx => {
+ fx.manifest.shots = {};
+ },
+ ],
+ [
+ 'scoped PR capture',
+ fx => {
+ delete fx.manifest.context.releasePlan;
+ },
+ ],
+ [
+ 'mismatched plan digest',
+ fx => {
+ fx.manifest.context.releasePlan.digest = 'invalid';
+ },
+ ],
+ [
+ 'mismatched verdict identity',
+ fx => {
+ fx.verdict.context.sha = 'b'.repeat(40);
+ },
+ ],
+ [
+ 'mismatched verdict plan',
+ fx => {
+ fx.verdict.context.releasePlan.keys = [];
+ },
+ ],
+ [
+ 'skipped capture',
+ fx => {
+ fx.verdict.status = 'skipped';
+ },
+ ],
+ ])('refuses %s', (_label, mutate) => {
+ const fx = captureFixture();
+ mutate(fx);
+ expect(() => validateVisualMaintenanceCapture(fx)).toThrow();
+ });
+});
diff --git a/.github/scripts/visual-maintenance-workflow.test.mjs b/.github/scripts/visual-maintenance-workflow.test.mjs
new file mode 100644
index 0000000000..ce0b1fa67f
--- /dev/null
+++ b/.github/scripts/visual-maintenance-workflow.test.mjs
@@ -0,0 +1,406 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Baseline maintenance workflow ownership contracts.
+ * @input ci.yml and independent PR/dispatch states
+ * @output Routing, trust-boundary, and publication ordering regression checks
+ * @position Node contracts for the shared visual owner, not another CI lane
+ */
+
+import fs from 'node:fs';
+import {spawnSync} from 'node:child_process';
+import path from 'node:path';
+
+import {describe, expect, it} from 'vitest';
+import yaml from 'yaml';
+
+const root = path.resolve(import.meta.dirname, '../..');
+const workflow = yaml.parse(
+ fs.readFileSync(path.join(root, '.github/workflows/ci.yml'), 'utf8'),
+);
+const visual = workflow.jobs['pr-visual'];
+const publication = workflow.jobs['baseline-publication'];
+const step = (job, name) =>
+ job.steps.find(candidate => candidate.name === name);
+const index = (job, name) =>
+ job.steps.findIndex(candidate => candidate.name === name);
+
+// Only trusted workflow expressions are evaluated. The fixtures below provide
+// outcomes independently, including skipped dependencies on maintenance dispatch.
+function runs(
+ job,
+ {
+ event = 'workflow_dispatch',
+ operation = 'capture',
+ needs = {},
+ cancelled = false,
+ } = {},
+) {
+ const expression = (job.if ?? 'true')
+ .replace(/^\$\{\{\s*|\s*\}\}$/g, '')
+ .replace(/needs\.([a-z][a-z0-9-]*)/g, 'needs["$1"]');
+ if (!/\b(always|cancelled|success|failure)\(/.test(expression)) {
+ if (
+ cancelled ||
+ Object.values(needs).some(value => value.result !== 'success')
+ )
+ return false;
+ }
+ return Function(
+ 'github',
+ 'inputs',
+ 'needs',
+ 'always',
+ 'cancelled',
+ `return Boolean(${expression});`,
+ )(
+ {event_name: event},
+ {operation},
+ needs,
+ () => true,
+ () => cancelled,
+ );
+}
+
+function dependencies(job, result = 'success') {
+ const names = Array.isArray(job.needs)
+ ? job.needs
+ : job.needs
+ ? [job.needs]
+ : [];
+ return Object.fromEntries(
+ names.map(name => [
+ name,
+ {
+ result,
+ outputs: {has_stable_visual: 'true', has_components: 'true'},
+ },
+ ]),
+ );
+}
+
+describe('CI baseline maintenance routing', () => {
+ it('has explicit capture/promote dispatch without a new workflow or test owner', () => {
+ expect(workflow.on.workflow_dispatch.inputs.operation.options).toEqual([
+ 'capture',
+ 'promote',
+ ]);
+ expect(visual.name).toBe('Stable visual regression');
+ expect(visual['runs-on']).toBe('2-core-ubuntu-arm');
+ expect(publication['runs-on']).toBe('ubuntu-slim');
+ expect(workflow.on).not.toHaveProperty('schedule');
+ });
+
+ it.each(['capture', 'promote'])(
+ 'does not run non-maintenance CI jobs for %s',
+ operation => {
+ for (const [name, job] of Object.entries(workflow.jobs)) {
+ if (
+ ['maintenance-request', 'pr-visual', 'baseline-publication'].includes(
+ name,
+ )
+ )
+ continue;
+ expect(runs(job, {operation, needs: dependencies(job)}), name).toBe(
+ false,
+ );
+ }
+ },
+ );
+
+ it('runs capture despite skipped PR dependencies, only after the main guard succeeds', () => {
+ const needs = dependencies(visual, 'skipped');
+ needs['maintenance-request'].result = 'success';
+ expect(runs(visual, {needs})).toBe(true);
+ expect(runs(visual, {needs, operation: 'promote'})).toBe(false);
+ needs['maintenance-request'].result = 'failure';
+ expect(runs(visual, {needs})).toBe(false);
+ });
+
+ it('preserves focused PR routing when the maintenance guard is skipped', () => {
+ const needs = dependencies(visual);
+ needs['maintenance-request'].result = 'skipped';
+ expect(runs(visual, {event: 'pull_request', needs})).toBe(true);
+ needs['check-components'].outputs.has_stable_visual = 'false';
+ expect(runs(visual, {event: 'pull_request', needs})).toBe(false);
+ needs['check-components'].outputs.has_stable_visual = 'true';
+ needs['build-storybook'].result = 'failure';
+ expect(runs(visual, {event: 'pull_request', needs})).toBe(false);
+ expect(runs(visual, {event: 'merge_group', needs})).toBe(false);
+ });
+
+ it('publishes only an explicit promote dispatch after the main guard', () => {
+ expect(
+ runs(publication, {
+ operation: 'promote',
+ needs: dependencies(publication),
+ }),
+ ).toBe(true);
+ expect(
+ runs(publication, {
+ operation: 'capture',
+ needs: dependencies(publication),
+ }),
+ ).toBe(false);
+ expect(
+ runs(publication, {
+ operation: 'promote',
+ needs: dependencies(publication, 'failure'),
+ }),
+ ).toBe(false);
+ expect(
+ runs(publication, {
+ event: 'pull_request',
+ operation: 'promote',
+ needs: dependencies(publication),
+ }),
+ ).toBe(false);
+ });
+
+ it.each([
+ ['refs/heads/main', 'capture', 0],
+ ['refs/heads/main', 'promote', 0],
+ ['refs/heads/feature', 'capture', 1],
+ ['refs/tags/main', 'promote', 1],
+ ['refs/heads/main', 'other', 1],
+ ])(
+ 'guards maintenance ref %s and operation %s before checkout',
+ (ref, operation, expected) => {
+ const guard = workflow.jobs['maintenance-request'];
+ expect(guard.steps).toHaveLength(1);
+ const result = spawnSync('bash', ['-c', guard.steps[0].run], {
+ env: {
+ ...process.env,
+ GITHUB_REF: ref,
+ OPERATION: operation,
+ RUN_ID: '101',
+ RUN_ATTEMPT: '2',
+ KEYS: 'all',
+ REASON: 'Reviewed browser refresh',
+ },
+ encoding: 'utf8',
+ });
+ expect(result.status, result.stderr).toBe(expected);
+ },
+ );
+
+ it('isolates dispatch concurrency from PRs and does not cancel queued maintenance', () => {
+ expect(workflow.concurrency.group).toContain('|| github.run_id');
+ expect(workflow.concurrency['cancel-in-progress']).toBe(
+ "${{ github.event_name != 'workflow_dispatch' }}",
+ );
+ });
+});
+
+describe('canonical capture and publication separation', () => {
+ it('builds full maintenance Storybook only inside the existing visual owner', () => {
+ const build = step(visual, 'Build canonical maintenance Storybook');
+ expect(build.if).toBe("github.event_name == 'workflow_dispatch'");
+ expect(build.run).toBe(
+ 'pnpm build && pnpm -F @astryxdesign/storybook build',
+ );
+ const capture = step(visual, 'Capture canonical visual baseline');
+ expect(capture.if).toBe("github.event_name == 'workflow_dispatch'");
+ expect(capture.run).toContain('gate.mjs release');
+ expect(capture.run).not.toMatch(
+ /--(?:sample|only|components|themes|max-shots|tiers)\b/,
+ );
+ expect(visual.permissions).toEqual({contents: 'read'});
+ });
+
+ it('keeps browser-refresh capture reachable, but only uploads validated full captures', () => {
+ expect(
+ step(visual, 'Capture canonical visual baseline')['continue-on-error'],
+ ).toBe(true);
+ const validation = step(visual, 'Validate canonical baseline capture');
+ expect(validation.if).toContain('always()');
+ expect(validation.run).toContain('validateVisualMaintenanceCapture');
+ expect(validation.env.CAPTURE_SHA).toBe('${{ github.sha }}');
+ expect(validation.env.CAPTURE_RUN_ATTEMPT).toBe(
+ '${{ github.run_attempt }}',
+ );
+ const artifact = step(visual, 'Upload canonical baseline candidate');
+ expect(artifact.if).toContain(
+ "steps.maintenance-validation.outcome == 'success'",
+ );
+ expect(artifact.with.name).toBe(
+ 'visual-baseline-capture-${{ github.run_id }}-${{ github.run_attempt }}',
+ );
+ expect(artifact.with.path).toContain('.visual-run/shots/');
+ expect(artifact.with.path).toContain('.visual-run/manifest.json');
+ expect(artifact.with.path).toContain('.visual-run/verdict.json');
+ });
+
+ it('uses exact validated artifact identity before any downloaded bytes or publication', () => {
+ const resolve = step(publication, 'Resolve the reviewed canonical capture');
+ const download = step(publication, 'Download the reviewed capture');
+ expect(resolve.with.script).toContain('resolveVisualMaintenanceSource');
+ expect(download.with['artifact-ids']).toBe(
+ '${{ steps.source.outputs.artifact_id }}',
+ );
+ expect(download.with['run-id']).toBe('${{ steps.source.outputs.run_id }}');
+ expect(download.with['merge-multiple']).toBe(true);
+ const verify = step(publication, 'Verify downloaded capture identity');
+ expect(verify.run).toContain('validateVisualMaintenanceCapture');
+ expect(verify.env.CAPTURE_SHA).toBe('${{ steps.source.outputs.sha }}');
+ expect(index(publication, resolve.name)).toBeLessThan(
+ index(publication, download.name),
+ );
+ expect(index(publication, download.name)).toBeLessThan(
+ index(publication, verify.name),
+ );
+ expect(index(publication, verify.name)).toBeLessThan(
+ index(publication, 'Promote the explicitly reviewed baseline'),
+ );
+ });
+
+ it('retains ordered manual publication with explicit keys, reason, actor, and prune', () => {
+ const names = [
+ 'Queue baseline publication',
+ 'Wait for the baseline publication turn',
+ 'Promote the explicitly reviewed baseline',
+ 'Release the baseline publication turn',
+ ];
+ const positions = names.map(name => index(publication, name));
+ expect(positions.every(position => position >= 0)).toBe(true);
+ expect(positions).toEqual([...positions].sort((a, b) => a - b));
+ for (const [name, command] of [
+ [names[0], 'enqueue'],
+ [names[1], 'wait'],
+ [names[3], 'release'],
+ ]) {
+ expect(step(publication, name).run).toContain(
+ `gh-pages-publisher.mjs ${command} --scope visual-gate/baseline`,
+ );
+ }
+ expect(step(publication, names[3]).if).toBe(
+ "always() && steps.queue.outcome == 'success'",
+ );
+ const promote = step(publication, names[2]);
+ expect(promote.run).toContain(
+ 'gh-pages-publisher.mjs visual-baseline-manual',
+ );
+ for (const [flag, variable] of [
+ ['keys', 'KEYS'],
+ ['reason', 'REASON'],
+ ['actor', 'PROMOTER'],
+ ['prune', 'PRUNE'],
+ ]) {
+ expect(promote.run).toContain(`--${flag} "$${variable}"`);
+ }
+ const commands = publication.steps
+ .map(item => item.run ?? item.with?.script ?? '')
+ .join('\n');
+ expect(commands).not.toMatch(
+ /playwright|gate\.mjs (?:release|capture|check)|createCommitStatus|createWorkflowDispatch/,
+ );
+ });
+
+ it.each([
+ ['focused component', 'Button', '', 'false', 'check', 'pass'],
+ ['focused theme', '', 'stone', 'false', 'check', 'pass'],
+ ['mixed broad and component', 'Button', '', 'true', 'release', 'pass'],
+ ['broad without components', '', '', 'true', 'release', 'pass'],
+ ['over-budget focused plan', 'Button', '', 'false', 'check', 'skipped'],
+ ])(
+ 'keeps %s in the canonical visual job',
+ (_label, components, themes, broad, command, status) => {
+ const script = step(
+ visual,
+ 'Run the visual gate for the touched components',
+ ).run;
+ const result = spawnSync(
+ 'bash',
+ [
+ '-c',
+ `
+ jq() {
+ case "$2" in
+ *changedStableThemes*) printf '%s' "$TEST_THEMES" ;;
+ *newComponents*) printf '%s' "$TEST_COMPONENTS" ;;
+ *) printf '%s' "$TEST_STATUS" ;;
+ esac
+ }
+ node() {
+ printf 'gate-command:%s\\n' "$2"
+ if [ "$2" = 'release' ]; then TEST_STATUS=pass; fi
+ }
+ ${script}
+ `,
+ ],
+ {
+ env: {
+ ...process.env,
+ TEST_COMPONENTS: components,
+ TEST_THEMES: themes,
+ BROAD_STABLE_VISUAL: broad,
+ TEST_STATUS: status,
+ GITHUB_STEP_SUMMARY: '/dev/null',
+ },
+ encoding: 'utf8',
+ },
+ );
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain(`gate-command:${command}`);
+ if (status === 'skipped') {
+ expect(result.stdout).toContain('gate-command:release');
+ }
+ expect(visual['continue-on-error']).toBeUndefined();
+ },
+ );
+
+ it.each([
+ [1, 'failed', 1],
+ [2, 'changed', 0],
+ [0, 'skipped', 1],
+ ])(
+ 'handles gate exit %s and final verdict %s without hiding incomplete evidence',
+ (code, status, expected) => {
+ const script = step(
+ visual,
+ 'Run the visual gate for the touched components',
+ ).run;
+ const result = spawnSync(
+ 'bash',
+ [
+ '-c',
+ `
+ jq() {
+ case "$2" in
+ *changedStableThemes*) printf '' ;;
+ *newComponents*) printf 'Button' ;;
+ *) printf '%s' "$TEST_STATUS" ;;
+ esac
+ }
+ node() { return "$TEST_CODE"; }
+ ${script}
+ `,
+ ],
+ {
+ env: {
+ ...process.env,
+ TEST_CODE: String(code),
+ TEST_STATUS: status,
+ BROAD_STABLE_VISUAL: 'false',
+ GITHUB_STEP_SUMMARY: '/dev/null',
+ },
+ encoding: 'utf8',
+ },
+ );
+ expect(result.status, result.stderr).toBe(expected);
+ },
+ );
+
+ it('retains non-regression Probe reach diagnostics in the existing accessibility owner', () => {
+ const a11y = workflow.jobs['pr-a11y'];
+ expect(step(a11y, 'Check Probe theme reach').run).toContain(
+ 'gate.mjs reach',
+ );
+ expect(step(a11y, 'Check Probe theme reach')['continue-on-error']).toBe(
+ true,
+ );
+ expect(step(a11y, 'Upload Probe reach report').with.path).toBe(
+ '.visual-reach/reach.json',
+ );
+ });
+});
diff --git a/.github/scripts/visual-scope.mjs b/.github/scripts/visual-scope.mjs
index 5c43b7be31..9995dfcfe7 100644
--- a/.github/scripts/visual-scope.mjs
+++ b/.github/scripts/visual-scope.mjs
@@ -22,6 +22,8 @@ import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
+import changeScope from './change-scope.cjs';
+
const ROOT = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../..',
@@ -55,7 +57,18 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {
const stableThemes = new Set();
const stableComponents = new Set();
const canaryPackages = new Set();
- let broadStableVisual = false;
+ // Reuse the accepted surface taxonomy: unknown/shared inputs and the build
+ // pipeline cannot rely on a daily backup once this is the only visual owner.
+ const {surfaces} = changeScope.classifyChanges(paths);
+ const stableVisualInfrastructure =
+ paths.length === 0 ||
+ surfaces.some(surface =>
+ ['shared-or-unknown', 'storybook-visual', 'runtime:build'].includes(
+ surface,
+ ),
+ ) ||
+ paths.includes('packages/core/package.json');
+ let broadStableVisual = stableVisualInfrastructure;
for (const file of stableCoreFiles) {
const match = file.match(/^packages\/core\/src\/([^/]+)\//);
@@ -117,7 +130,10 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {
}
return {
- hasStableVisual: stableCoreFiles.length > 0 || stableThemes.size > 0,
+ hasStableVisual:
+ stableVisualInfrastructure ||
+ stableCoreFiles.length > 0 ||
+ stableThemes.size > 0,
broadStableVisual,
stableComponents: [...stableComponents].sort(),
stableCoreFiles,
@@ -150,6 +166,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
`stable_themes=${outputValue(result.stableThemes)}`,
`canary_packages=${outputValue(result.canaryPackages)}`,
`has_stable_visual=${result.hasStableVisual}`,
+ `broad_stable_visual=${result.broadStableVisual}`,
].join('\n') + '\n',
);
}
diff --git a/.github/scripts/visual-scope.test.mjs b/.github/scripts/visual-scope.test.mjs
index 70e7a441ce..af3a8e9fd2 100644
--- a/.github/scripts/visual-scope.test.mjs
+++ b/.github/scripts/visual-scope.test.mjs
@@ -48,6 +48,52 @@ beforeEach(() => {
afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
describe('classifyVisualScope', () => {
+ it.each([
+ 'pnpm-lock.yaml',
+ 'package.json',
+ 'packages/core/package.json',
+ 'apps/storybook/vite.config.ts',
+ 'packages/build/src/vite.ts',
+ 'unknown-rendering-input.toml',
+ ])(
+ 'keeps shared input %s broad alone and beside a focused component',
+ file => {
+ for (const files of [
+ [file],
+ ['packages/core/src/Button/Button.tsx', file],
+ ]) {
+ expect(classifyVisualScope(files, root)).toMatchObject({
+ hasStableVisual: true,
+ broadStableVisual: true,
+ });
+ }
+ },
+ );
+
+ it('fails closed for an empty path set', () => {
+ expect(classifyVisualScope([], root)).toMatchObject({
+ hasStableVisual: true,
+ broadStableVisual: true,
+ });
+ });
+
+ it.each([
+ 'apps/storybook/.storybook/preview.tsx',
+ 'apps/storybook/stories/Button.stories.tsx',
+ '.github/scripts/visual-gate/gate.mjs',
+ '.github/scripts/visual-gate/visual-gate.config.json',
+ '.github/scripts/visual-scope.mjs',
+ '.github/workflows/ci.yml',
+ ])(
+ 'routes shared visual infrastructure through the canonical owner: %s',
+ file => {
+ expect(classifyVisualScope([file], root)).toMatchObject({
+ hasStableVisual: true,
+ broadStableVisual: true,
+ });
+ },
+ );
+
it('includes stable Core runtime changes', () => {
const result = classifyVisualScope(
['packages/core/src/Button/Button.tsx'],
@@ -99,12 +145,7 @@ describe('classifyVisualScope', () => {
it('ignores build-only metadata in a stable theme package', () => {
const result = classifyVisualScope(
- [
- 'packages/themes/neutral/package.json',
- 'packages/core/package.json',
- 'scripts/clean-dist.mjs',
- 'pnpm-lock.yaml',
- ],
+ ['packages/themes/neutral/package.json'],
root,
{
'packages/themes/neutral/package.json': {
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4c4cac878a..05eb777d5f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,16 +11,72 @@ on:
pull_request:
branches: ['main']
merge_group:
+ # Explicit baseline maintenance shares the PR visual owner. It never runs as
+ # a release gate and never changes the baseline without a reviewed promotion.
+ workflow_dispatch:
+ inputs:
+ operation:
+ description: 'Capture canonical baseline candidates or promote a reviewed capture'
+ type: choice
+ options: [capture, promote]
+ default: capture
+ required: true
+ run_id:
+ description: 'For promote: the reviewed CI maintenance capture run id'
+ required: false
+ run_attempt:
+ description: 'For promote: the reviewed capture attempt'
+ required: false
+ keys:
+ description: 'For promote: comma-separated shot keys, or "all"'
+ required: false
+ reason:
+ description: 'For promote: why the reviewed rendering is correct'
+ required: false
+ prune:
+ description: 'Also drop reviewed baseline keys absent from the canonical plan'
+ type: boolean
+ default: true
+ required: false
permissions: {}
concurrency:
- group: 'ci-${{ github.event.merge_group.head_sha || github.head_ref }}'
- cancel-in-progress: true
+ group: 'ci-${{ github.event.merge_group.head_sha || github.head_ref || github.run_id }}'
+ cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
jobs:
+ maintenance-request:
+ if: github.event_name == 'workflow_dispatch'
+ runs-on: ubuntu-slim
+ steps:
+ - name: Require explicit main-branch maintenance
+ env:
+ OPERATION: ${{ inputs.operation }}
+ RUN_ID: ${{ inputs.run_id }}
+ RUN_ATTEMPT: ${{ inputs.run_attempt }}
+ KEYS: ${{ inputs.keys }}
+ REASON: ${{ inputs.reason }}
+ run: |
+ set -eu
+ if [ "$GITHUB_REF" != 'refs/heads/main' ]; then
+ echo 'Baseline maintenance must be dispatched from main.'
+ exit 1
+ fi
+ case "$OPERATION" in
+ capture) ;;
+ promote)
+ test -n "$RUN_ID"
+ test -n "$RUN_ATTEMPT"
+ test -n "$KEYS"
+ test -n "$REASON"
+ ;;
+ *) echo 'Choose capture or promote.'; exit 1 ;;
+ esac
+
# Detect lightweight changes. Missing merge-base data fails closed.
check-scope:
+ if: github.event_name != 'workflow_dispatch'
runs-on: ubuntu-slim
permissions:
contents: read
@@ -70,7 +126,7 @@ jobs:
# Docsite data extraction tests — always runs, fast (~2s)
docsite-test:
needs: [check-scope]
- if: ${{ always() && !cancelled() }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() }}
runs-on: 2-core-ubuntu-arm
permissions:
contents: read
@@ -142,6 +198,7 @@ jobs:
# Stable visual scope is separate from component scope: Core and the
# non-private shipped themes participate; canaryOnly packages do not.
has_stable_visual: ${{ steps.check.outputs.has_stable_visual }}
+ broad_stable_visual: ${{ steps.check.outputs.broad_stable_visual }}
steps:
- name: Checkout
uses: actions/checkout@v7
@@ -169,6 +226,7 @@ jobs:
echo "::warning::No merge base between HEAD and origin/${{ github.base_ref }} — running full component audits"
printf 'has_components=true\nhas_rtl_components=true\nhas_rtl_harness=true\nforce_full_component_audits=true\n' >> "$GITHUB_OUTPUT"
echo "has_stable_visual=true" >> "$GITHUB_OUTPUT"
+ echo "broad_stable_visual=true" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -216,7 +274,7 @@ jobs:
# build-storybook + build-sandbox below.
test-ui:
needs: [check-scope]
- if: ${{ always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: 4-core-ubuntu
permissions:
contents: read
@@ -279,7 +337,7 @@ jobs:
test-node:
needs: [check-scope]
- if: ${{ always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: 2-core-ubuntu-arm
permissions:
contents: read
@@ -327,7 +385,7 @@ jobs:
# item through the exact pinned ShadCN client and compiles every written source.
registry-contract:
needs: [check-scope]
- if: ${{ always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: 4-core-ubuntu
timeout-minutes: 30
permissions:
@@ -391,7 +449,7 @@ jobs:
# projects their combined result without creating another surface lane.
test:
needs: [check-scope, test-ui, test-node, test-build, registry-contract]
- if: ${{ always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: ubuntu-slim
permissions:
contents: read
@@ -418,7 +476,7 @@ jobs:
# `build` join below folds their results into the historical check name.
build-storybook:
needs: [check-scope]
- if: ${{ always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: 2-core-ubuntu-arm
permissions:
contents: read
@@ -685,7 +743,7 @@ jobs:
# behavior.
build:
needs: [build-storybook, build-sandbox]
- if: ${{ !cancelled() && needs.build-storybook.result != 'skipped' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && !cancelled() && needs.build-storybook.result != 'skipped' }}
runs-on: ubuntu-slim
permissions: {}
steps:
@@ -834,6 +892,24 @@ jobs:
retention-days: 30
if-no-files-found: error
+ # Baseline-free target/axis diagnostics are not a visual-regression lane.
+ # Keep their existing report-only behavior with the other browser guards.
+ - name: Check Probe theme reach
+ continue-on-error: true
+ run: |
+ node .github/scripts/visual-gate/gate.mjs reach \
+ --storybook-dir apps/storybook/dist \
+ --out .visual-reach
+
+ - name: Upload Probe reach report
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: visual-reach
+ path: .visual-reach/reach.json
+ retention-days: 30
+ if-no-files-found: warn
+
# Build package test owner. The plugin's production CSS partition is a Build
# integration contract: source inspection cannot prove which declaration
# paints after Vite emits the final bundle. The `test-build` name makes that
@@ -841,7 +917,7 @@ jobs:
# result. Until runtime:build has a specialized route, broad CI dispatches it.
test-build:
needs: [check-scope]
- if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
+ if: github.event_name != 'workflow_dispatch' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
runs-on: 2-core-ubuntu-arm
permissions:
contents: read
@@ -866,7 +942,7 @@ jobs:
# which custom properties paint the portaled surfaces.
fixture-contrast:
needs: [check-scope]
- if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
+ if: github.event_name != 'workflow_dispatch' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true'
runs-on: 2-core-ubuntu-arm
permissions:
contents: read
@@ -886,34 +962,23 @@ jobs:
- name: Verify guest fixture contrast
run: pnpm -F @astryxdesign/vibe-tests fixtures:contrast
- # Stable-package visual regression. Core component changes compare only
- # representative or explicitly tagged story keys already present in the
- # accepted baseline. A changed shipped theme gets its relevant matrix, but PR
- # acceptance cannot add or remove baseline keys. Packages marked
- # `astryx.canaryOnly` (Lab, richtext, vega) do not compare pixels and do not
- # own a baseline.
- #
- # Deeper than the daily gate only where the component declares it: audit and
- # interaction fixtures stay available to their dedicated checks without
- # multiplying the pixel baseline.
- #
- # SKIPS ITSELF WHEN COMPONENT OR SHARED-INFRASTRUCTURE SCOPE IS TOO BROAD.
- # Theme-only matrices are release evidence and are not charged against the
- # focused component review ceiling. A token or shared-hook change puts
- # hundreds of diffs in front of a reviewer who cannot judge them
- # one by one, and a per-PR check is the wrong instrument for that change —
- # the daily gate reviews it against the whole baseline instead. The skip
- # states its reason in the PR comment rather than vanishing.
- #
- # SOFT / NON-BLOCKING (continue-on-error) while the suite earns trust: the
- # verdict surfaces in the job summary and the PR comment, and the release
- # gate is the blocking one. To promote: drop continue-on-error.
+ # Stable-package visual regression and explicit baseline maintenance share
+ # this owner. Focused PRs compare existing canonical frames; broad PRs use the
+ # full canonical plan. Maintenance captures candidates on the same runner,
+ # but only a separate explicit promotion may publish them as the baseline.
+ # Audit and interaction fixtures stay with their existing browser checks.
pr-visual:
name: Stable visual regression
- needs: [build-storybook, check-components]
- if: github.event_name == 'pull_request' && needs.check-components.outputs.has_stable_visual == 'true'
+ needs: [build-storybook, check-components, maintenance-request]
+ if: >-
+ !cancelled() &&
+ ((github.event_name == 'pull_request' &&
+ needs.build-storybook.result == 'success' &&
+ needs.check-components.result == 'success' &&
+ needs.check-components.outputs.has_stable_visual == 'true') ||
+ (github.event_name == 'workflow_dispatch' &&
+ needs.maintenance-request.result == 'success' && inputs.operation == 'capture'))
runs-on: 2-core-ubuntu-arm
- continue-on-error: true
permissions:
contents: read
steps:
@@ -926,21 +991,21 @@ jobs:
install: 'false'
- name: Download analysis artifact
+ if: github.event_name == 'pull_request'
uses: actions/download-artifact@v8
with:
name: pr-analysis
- name: Download Storybook artifact
+ if: github.event_name == 'pull_request'
uses: actions/download-artifact@v8
with:
name: storybook-${{ needs.build-storybook.outputs.short_hash }}
path: apps/storybook/dist
- # Non-fatal: if this artifact is ever missing, renamed, or produced under
- # a different key, the gate builds the themes itself. A crash here would
- # be before the gate ever runs, which is how an artifact-wiring change
- # reddened every open component PR twice in one day.
+ # If the optional theme artifact is missing, the gate builds the themes.
- name: Download built themes and core
+ if: github.event_name == 'pull_request'
continue-on-error: true
uses: actions/download-artifact@v8
with:
@@ -950,12 +1015,15 @@ jobs:
- name: Install Playwright
run: pnpm install --frozen-lockfile && npx playwright install chromium
- # The downloaded theme builds import Core's consumer-facing dist files.
- # Rebuild only Core here so those imports resolve without repeating the
- # full workspace build that produced the theme artifact.
+ # Downloaded theme builds import Core's consumer-facing dist files.
- name: Build Core dependency for theme inspection
+ if: github.event_name == 'pull_request'
run: pnpm -F @astryxdesign/core build
+ - name: Build canonical maintenance Storybook
+ if: github.event_name == 'workflow_dispatch'
+ run: pnpm build && pnpm -F @astryxdesign/storybook build
+
- name: Fetch visual baseline from gh-pages
run: |
set -eu
@@ -970,52 +1038,70 @@ jobs:
- name: Run the visual gate for the touched components
id: visual
+ if: github.event_name == 'pull_request'
env:
+ BROAD_STABLE_VISUAL: ${{ needs.check-components.outputs.broad_stable_visual }}
ASTRYX_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
ASTRYX_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
set -eu
- # Component names are not globally unique, so filter by the package
- # recorded in componentStats — never infer stability from a name.
+ # Component names are not globally unique; use their package identity.
COMPONENTS=$(jq -r '. as $root | [(.newComponents + .modifiedComponents)[] | select($root.componentStats[.].package == "@astryxdesign/core")] | unique | join(",")' analysis.json)
THEMES=$(jq -r '(.changedStableThemes // []) | join(",")' analysis.json)
-
- if [ -n "$COMPONENTS" ] && [ -n "$THEMES" ]; then
- TIERS="component,theme-matrix"
- SCOPE_ARGS=(--components "$COMPONENTS" --themes "$THEMES")
+ COMMAND=check
+ if [ "$BROAD_STABLE_VISUAL" = 'true' ]; then
+ COMMAND=release
+ SCOPE_ARGS=()
+ elif [ -n "$COMPONENTS" ] && [ -n "$THEMES" ]; then
+ SCOPE_ARGS=(--tiers 'component,theme-matrix' --components "$COMPONENTS" --themes "$THEMES")
elif [ -n "$COMPONENTS" ]; then
- TIERS="component"
- SCOPE_ARGS=(--components "$COMPONENTS" --no-scout)
+ SCOPE_ARGS=(--tiers component --components "$COMPONENTS" --no-scout)
elif [ -n "$THEMES" ]; then
- TIERS="theme-matrix"
- SCOPE_ARGS=(--themes "$THEMES")
+ SCOPE_ARGS=(--tiers theme-matrix --themes "$THEMES")
else
- # Shared Core theming/tokens/hooks are stable but broad. Let the
- # gate plan the full stable surface and decline visibly at the shot
- # budget; the daily stable gate owns the exhaustive comparison.
- TIERS="surface,theme-matrix,probe"
+ # Broad changes stay with this owner: the canonical full plan has
+ # no focused-component review ceiling and no deferred backup gate.
+ COMMAND=release
SCOPE_ARGS=()
fi
echo "Stable Core components: ${COMPONENTS:-none}"
echo "Stable themes: ${THEMES:-none}"
set +e
- node .github/scripts/visual-gate/gate.mjs check \
+ node .github/scripts/visual-gate/gate.mjs "$COMMAND" \
--storybook-dir apps/storybook/dist \
--baseline .visual-baseline \
--out .visual-run \
- --tiers "$TIERS" \
"${SCOPE_ARGS[@]}" \
--summary-output visual-summary.md
code=$?
+ set -e
+ # Exit 2 is a visual change for review; every other nonzero exit is
+ # an infrastructure failure, not a successful evidence receipt.
+ case "$code" in 0|2) ;; *) exit "$code" ;; esac
+ STATUS=$(jq -er '.status' .visual-run/verdict.json)
+ if [ "$STATUS" = 'skipped' ]; then
+ # The focused budget refused before capture. The same owner now
+ # captures its bounded canonical full plan rather than leaving a gap.
+ set +e
+ node .github/scripts/visual-gate/gate.mjs release \
+ --storybook-dir apps/storybook/dist \
+ --baseline .visual-baseline \
+ --out .visual-run \
+ --summary-output visual-summary.md
+ code=$?
+ set -e
+ case "$code" in 0|2) ;; *) exit "$code" ;; esac
+ STATUS=$(jq -er '.status' .visual-run/verdict.json)
+ fi
[ -f visual-summary.md ] && cat visual-summary.md >> "$GITHUB_STEP_SUMMARY"
- # Exit 2 is "changed", which is a question for the reviewer, not a
- # broken job. Only a crash (1) is a failure.
- [ "$code" -eq 1 ] && exit 1
- exit 0
+ case "$STATUS" in
+ pass|changed) ;;
+ *) echo "Visual evidence is incomplete: $STATUS"; exit 1 ;;
+ esac
- name: Upload visual verdict and report
- if: always()
+ if: always() && github.event_name == 'pull_request'
uses: actions/upload-artifact@v7
with:
name: visual-pr-report
@@ -1027,6 +1113,154 @@ jobs:
retention-days: 7
if-no-files-found: ignore
+ - name: Capture canonical visual baseline
+ id: maintenance-capture
+ if: github.event_name == 'workflow_dispatch'
+ # A complete browser-refresh capture can be incomparable with the old
+ # baseline. The next step requires full capture, not a clean comparison.
+ continue-on-error: true
+ run: |
+ node .github/scripts/visual-gate/gate.mjs release \
+ --storybook-dir apps/storybook/dist \
+ --baseline .visual-baseline \
+ --out .visual-run \
+ --summary-output visual-summary.md
+
+ - name: Validate canonical baseline capture
+ id: maintenance-validation
+ if: >-
+ always() && github.event_name == 'workflow_dispatch' &&
+ steps.maintenance-capture.outcome != 'skipped'
+ env:
+ CAPTURE_SHA: ${{ github.sha }}
+ CAPTURE_RUN_ID: ${{ github.run_id }}
+ CAPTURE_RUN_ATTEMPT: ${{ github.run_attempt }}
+ run: |
+ node --input-type=module - <<'NODE'
+ import fs from 'node:fs';
+ import {validateVisualMaintenanceCapture} from './.github/scripts/visual-maintenance-source.mjs';
+ validateVisualMaintenanceCapture({
+ manifest: JSON.parse(fs.readFileSync('.visual-run/manifest.json', 'utf8')),
+ verdict: JSON.parse(fs.readFileSync('.visual-run/verdict.json', 'utf8')),
+ sha: process.env.CAPTURE_SHA,
+ runId: process.env.CAPTURE_RUN_ID,
+ runAttempt: process.env.CAPTURE_RUN_ATTEMPT,
+ });
+ NODE
+ cat visual-summary.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload canonical baseline candidate
+ if: always() && steps.maintenance-validation.outcome == 'success'
+ uses: actions/upload-artifact@v7
+ with:
+ name: visual-baseline-capture-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ .visual-run/shots/
+ .visual-run/manifest.json
+ .visual-run/verdict.json
+ .visual-run/report/
+ retention-days: 14
+ if-no-files-found: error
+
+ # Publication only: no browser, comparison, automatic acceptance, or release
+ # status. The explicitly reviewed CI capture supplies the baseline bytes.
+ baseline-publication:
+ needs: maintenance-request
+ if: github.event_name == 'workflow_dispatch' && inputs.operation == 'promote'
+ runs-on: ubuntu-slim
+ timeout-minutes: 90
+ permissions:
+ actions: read
+ contents: write
+ steps:
+ - name: Checkout trusted maintenance implementation
+ uses: actions/checkout@v7
+
+ - name: Setup Node and pnpm
+ uses: ./.github/actions/setup
+
+ - name: Resolve the reviewed canonical capture
+ id: source
+ uses: actions/github-script@v9
+ env:
+ SOURCE_RUN_ID: ${{ inputs.run_id }}
+ SOURCE_RUN_ATTEMPT: ${{ inputs.run_attempt }}
+ with:
+ retries: 3
+ script: |
+ const {pathToFileURL} = require('node:url');
+ const {resolveVisualMaintenanceSource} = await import(
+ pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-maintenance-source.mjs`).href
+ );
+ const source = await resolveVisualMaintenanceSource({
+ github, ...context.repo,
+ runId: process.env.SOURCE_RUN_ID,
+ runAttempt: process.env.SOURCE_RUN_ATTEMPT,
+ });
+ core.setOutput('artifact_id', String(source.artifactId));
+ core.setOutput('run_id', String(source.runId));
+ core.setOutput('run_attempt', String(source.runAttempt));
+ core.setOutput('sha', source.sha);
+
+ - name: Queue baseline publication
+ id: queue
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: node .github/scripts/gh-pages-publisher.mjs enqueue --scope visual-gate/baseline
+
+ - name: Download the reviewed capture
+ uses: actions/download-artifact@v8
+ with:
+ artifact-ids: ${{ steps.source.outputs.artifact_id }}
+ run-id: ${{ steps.source.outputs.run_id }}
+ github-token: ${{ github.token }}
+ path: .visual-run
+ merge-multiple: true
+
+ - name: Verify downloaded capture identity
+ env:
+ CAPTURE_SHA: ${{ steps.source.outputs.sha }}
+ CAPTURE_RUN_ID: ${{ steps.source.outputs.run_id }}
+ CAPTURE_RUN_ATTEMPT: ${{ steps.source.outputs.run_attempt }}
+ run: |
+ node --input-type=module - <<'NODE'
+ import fs from 'node:fs';
+ import {validateVisualMaintenanceCapture} from './.github/scripts/visual-maintenance-source.mjs';
+ validateVisualMaintenanceCapture({
+ manifest: JSON.parse(fs.readFileSync('.visual-run/manifest.json', 'utf8')),
+ verdict: JSON.parse(fs.readFileSync('.visual-run/verdict.json', 'utf8')),
+ sha: process.env.CAPTURE_SHA,
+ runId: process.env.CAPTURE_RUN_ID,
+ runAttempt: process.env.CAPTURE_RUN_ATTEMPT,
+ });
+ NODE
+
+ - name: Wait for the baseline publication turn
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: node .github/scripts/gh-pages-publisher.mjs wait --scope visual-gate/baseline
+
+ - name: Promote the explicitly reviewed baseline
+ env:
+ KEYS: ${{ inputs.keys }}
+ REASON: ${{ inputs.reason }}
+ PROMOTER: ${{ github.actor }}
+ PRUNE: ${{ inputs.prune && 'true' || 'false' }}
+ GH_TOKEN: ${{ github.token }}
+ run: >
+ node .github/scripts/gh-pages-publisher.mjs visual-baseline-manual
+ --capture .visual-run
+ --keys "$KEYS"
+ --reason "$REASON"
+ --actor "$PROMOTER"
+ --prune "$PRUNE"
+
+ - name: Release the baseline publication turn
+ if: always() && steps.queue.outcome == 'success'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: node .github/scripts/gh-pages-publisher.mjs release --scope visual-gate/baseline
+
# RTL semantic audit. Sibling to pr-a11y: same build-storybook artifact, same
# analysis.json scoping, so ordinary PRs only pay for the components they
# touched. A harness-only PR runs the fixed Chart/ChartLegend smoke scope so
@@ -1270,7 +1504,7 @@ jobs:
# Stable required context joining the five package shards above.
pr-rtl:
needs: [check-scope, check-components, pr-rtl-shard]
- if: ${{ always() && !cancelled() && github.event_name == 'pull_request' && needs.check-scope.outputs.spec_only != 'true' }}
+ if: ${{ github.event_name != 'workflow_dispatch' && always() && !cancelled() && github.event_name == 'pull_request' && needs.check-scope.outputs.spec_only != 'true' }}
runs-on: ubuntu-slim
timeout-minutes: 5
steps:
diff --git a/.github/workflows/pages-deploy.yml b/.github/workflows/pages-deploy.yml
index f0dc0bc8d1..78c5f0695c 100644
--- a/.github/workflows/pages-deploy.yml
+++ b/.github/workflows/pages-deploy.yml
@@ -21,10 +21,7 @@ on:
- 'PR Comment'
- 'Re-deploy Preview'
- 'Cleanup Preview Deployments'
- - 'Release Gate'
- - 'Visual Baseline'
- - 'Visual acceptance'
- - 'Visual acceptance promotion'
+ - 'CI'
- 'Vibe Test Screenshots'
- 'Compact GitHub Pages History'
types: [completed]
diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml
index dd7b1271db..f61534e1d9 100644
--- a/.github/workflows/pr-comment.yml
+++ b/.github/workflows/pr-comment.yml
@@ -19,19 +19,14 @@ name: PR Comment
on:
workflow_run:
workflows: ['CI']
- types: [requested, in_progress, completed]
+ types: [completed]
permissions: {}
-# The event already carries a stable head-repository/branch identity. Acquire
-# this shared lock before resolving the PR so a rerun can cancel older
-# invalidation, report-publication, or acceptance mutations for the same head.
-# Post-merge promotion does not join this cancellation group: it serializes
-# durably with every baseline publisher, then recaptures the merged commit and
-# requires its canonical pixels to match the latest completed accepted CI.
+# Serialize report publication for a head without cancelling a Pages writer.
concurrency:
- group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}
- cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}
+ group: pr-report-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}
+ cancel-in-progress: false
jobs:
resolve:
@@ -106,6 +101,7 @@ jobs:
core.setOutput('head_repo_id', identity.headRepositoryId);
core.setOutput('head_ref', identity.headRef);
core.setOutput('base_repo', identity.baseRepository);
+ core.setOutput('tested_base_sha', identity.testedBaseSha ?? '');
core.setOutput('base_sha', identity.baseSha);
core.setOutput('run_id', String(identity.sourceRunId));
core.setOutput('run_attempt', String(identity.sourceRunAttempt));
@@ -113,68 +109,6 @@ jobs:
core.setOutput('spec_only', String(changeScope.specOnly));
core.setOutput('tooling_only', String(changeScope.toolingOnly));
- invalidate:
- needs: resolve
- if: >-
- (github.event.action == 'requested' || github.event.action == 'in_progress') &&
- needs.resolve.outputs.valid == 'true' &&
- needs.resolve.outputs.spec_only != 'true' &&
- needs.resolve.outputs.tooling_only != 'true'
- runs-on: ubuntu-slim
- permissions:
- pull-requests: write
- statuses: write
- steps:
- - name: Invalidate visible approval for the new evidence run
- uses: actions/github-script@v9
- with:
- retries: 3
- script: |
- const {owner, repo} = context.repo;
- const run = context.payload.workflow_run;
- const pr = Number('${{ needs.resolve.outputs.pr_number }}');
- const head = '${{ needs.resolve.outputs.head_sha }}';
- await github.rest.repos.createCommitStatus({
- owner,
- repo,
- sha: head,
- context: 'visual-acceptance',
- state: 'pending',
- description: `CI run ${run.id}/${run.run_attempt} is producing fresh visual evidence.`,
- });
- try {
- await github.rest.issues.removeLabel({
- owner, repo, issue_number: pr, name: 'visual-approved',
- });
- } catch (error) {
- if (error.status !== 404) throw error;
- }
-
- spec-only-visual:
- needs: resolve
- if: >-
- github.event.action == 'completed' &&
- needs.resolve.outputs.valid == 'true' &&
- (needs.resolve.outputs.spec_only == 'true' ||
- needs.resolve.outputs.tooling_only == 'true')
- runs-on: ubuntu-slim
- permissions:
- statuses: write
- steps:
- - name: Mark spec-only visual scope complete
- uses: actions/github-script@v9
- env:
- HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
- with:
- retries: 3
- script: |
- const {owner, repo} = context.repo;
- await github.rest.repos.createCommitStatus({
- owner, repo, sha: process.env.HEAD_SHA,
- context: 'visual-acceptance', state: 'success',
- description: 'Classified change — no visual scope.',
- });
-
spec-only-reconcile:
needs: resolve
if: >-
@@ -188,6 +122,9 @@ jobs:
contents: read
pull-requests: write
steps:
+ - name: Report no visual scope
+ run: echo 'No visual scope applies; canonical visual checks are not applicable.' >> "$GITHUB_STEP_SUMMARY"
+
- name: Checkout trusted default-branch code
uses: actions/checkout@v7
with:
@@ -231,7 +168,7 @@ jobs:
},
createIfMissing: false,
fallbackMessage:
- 'The current change has no preview-producing surface. Preview links are not shown.',
+ 'The current change has no visual or preview-producing scope. Visual checks are not applicable; preview links are not shown.',
});
deploy-preview:
@@ -271,7 +208,6 @@ jobs:
runs-on: 2-core-ubuntu-arm
permissions:
pull-requests: write
- statuses: write
actions: read
contents: write # immutable validated visual evidence on gh-pages
steps:
@@ -327,67 +263,12 @@ jobs:
core.setOutput('head_repo_id', identity.headRepositoryId);
core.setOutput('head_ref', identity.headRef);
core.setOutput('base_repo', identity.baseRepository);
+ core.setOutput('tested_base_sha', identity.testedBaseSha ?? '');
core.setOutput('base_sha', identity.baseSha);
core.setOutput('run_id', String(identity.sourceRunId));
core.setOutput('run_attempt', String(identity.sourceRunAttempt));
core.setOutput('source_conclusion', identity.sourceConclusion);
- - name: Derive trusted stable visual scope
- id: scope
- if: steps.identity.outputs.valid == 'true'
- uses: actions/github-script@v9
- with:
- script: |
- const fs = require('node:fs');
- const {spawnSync} = require('node:child_process');
- const {owner, repo} = context.repo;
- const pullNumber = Number('${{ steps.identity.outputs.pr_number }}');
- const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: pullNumber});
- const files = await github.paginate(github.rest.pulls.listFiles, {
- owner, repo, pull_number: pullNumber, per_page: 100,
- });
- if (files.length !== pr.changed_files) {
- core.setFailed(
- `GitHub returned ${files.length} of ${pr.changed_files} changed files; refusing visual-scope classification.`,
- );
- return;
- }
- const manifestPaths = new Set();
- for (const {filename} of files) {
- const theme = filename.match(/^packages\/themes\/([^/]+)\//);
- const pkg = filename.match(/^packages\/([^/]+)\//);
- if (theme) manifestPaths.add(`packages/themes/${theme[1]}/package.json`);
- else if (pkg) manifestPaths.add(`packages/${pkg[1]}/package.json`);
- }
- const manifests = {};
- const [headOwner, headRepo] = pr.head.repo.full_name.split('/');
- for (const manifestPath of manifestPaths) {
- try {
- const {data} = await github.rest.repos.getContent({
- owner: headOwner, repo: headRepo, path: manifestPath, ref: pr.head.sha,
- });
- if (!Array.isArray(data) && data.content) {
- manifests[manifestPath] = JSON.parse(Buffer.from(data.content, 'base64').toString());
- }
- } catch (error) {
- if (error.status !== 404) throw error;
- }
- }
- fs.writeFileSync('visual-scope-manifests.json', JSON.stringify(manifests));
- const result = spawnSync(
- process.execPath,
- ['.github/scripts/visual-scope.mjs', '--manifests', 'visual-scope-manifests.json'],
- {input: `${files.map((file) => file.filename).join('\n')}\n`, encoding: 'utf8'},
- );
- if (result.status !== 0) {
- core.setFailed(result.stderr || 'Stable visual scope classification failed.');
- return;
- }
- const scope = JSON.parse(result.stdout);
- fs.writeFileSync('trusted-scope.json', JSON.stringify(scope));
- core.setOutput('stable', String(scope.hasStableVisual));
- core.setOutput('broad', String(scope.broadStableVisual));
-
- name: Download analysis artifact from the CI run
if: steps.identity.outputs.valid == 'true'
uses: actions/download-artifact@v8
@@ -416,18 +297,16 @@ jobs:
path: preview-deployment
continue-on-error: true
- - name: Download Storybook for trusted visual capture
- if: >-
- steps.identity.outputs.valid == 'true' &&
- steps.scope.outputs.stable == 'true' &&
- steps.scope.outputs.broad != 'true'
+ - name: Download canonical visual report
+ id: visual-download
+ if: steps.identity.outputs.valid == 'true'
uses: actions/download-artifact@v8
with:
- pattern: storybook-*
- path: apps/storybook/dist
- merge-multiple: true
- run-id: ${{ github.event.workflow_run.id }}
+ name: visual-pr-report
+ path: visual-pr-report
+ run-id: ${{ steps.identity.outputs.run_id }}
github-token: ${{ secrets.GITHUB_TOKEN }}
+ continue-on-error: true
- name: Cross-check artifact identity
if: steps.identity.outputs.valid == 'true' && hashFiles('pr-analysis/pr-meta.json') != ''
@@ -453,80 +332,21 @@ jobs:
});
NODE
- - name: Fetch the trusted visual baseline
- if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'
- run: |
- rm -rf /tmp/gh-pages
- git clone --depth=1 --single-branch --branch gh-pages \
- "https://github.com/${GITHUB_REPOSITORY}.git" /tmp/gh-pages
-
- - name: Capture the trusted stable visual scope
- if: >-
- steps.identity.outputs.valid == 'true' &&
- steps.scope.outputs.stable == 'true' &&
- steps.scope.outputs.broad != 'true'
- env:
- ASTRYX_VISUAL_SHA: ${{ steps.identity.outputs.head_sha }}
- ASTRYX_VISUAL_RUN_ID: ${{ steps.identity.outputs.run_id }}
- ASTRYX_VISUAL_RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
- ASTRYX_PR_HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
- ASTRYX_PR_BASE_SHA: ${{ steps.identity.outputs.base_sha }}
- run: |
- npx playwright install chromium
- node .github/scripts/visual-gate/visual-acceptance.mjs trusted-plan \
- --scope trusted-scope.json \
- --baseline /tmp/gh-pages/visual-gate/baseline \
- --storybook-dir apps/storybook/dist \
- --output trusted-plan.json
- node .github/scripts/visual-gate/gate.mjs capture \
- --storybook-dir apps/storybook/dist \
- --out trusted-capture \
- --plan-file trusted-plan.json
-
- - name: Derive trusted broad visual deferral
- if: >-
- steps.identity.outputs.valid == 'true' &&
- steps.scope.outputs.stable == 'true' &&
- steps.scope.outputs.broad == 'true'
- env:
- PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
- HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
- BASE_SHA: ${{ steps.identity.outputs.base_sha }}
- RUN_ID: ${{ steps.identity.outputs.run_id }}
- RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
- run: |
- node .github/scripts/visual-gate/visual-acceptance.mjs trusted-defer \
- --scope trusted-scope.json \
- --baseline /tmp/gh-pages/visual-gate/baseline \
- --output trusted-visual \
- --pr "$PR_NUMBER" \
- --head "$HEAD_SHA" \
- --base "$BASE_SHA" \
- --run-id "$RUN_ID" \
- --run-attempt "$RUN_ATTEMPT"
-
- # The Storybook bundle is untrusted. Default-branch code chooses the plan,
- # captures it in an off-origin-blocked browser, derives the comparison,
- # validates/re-encodes PNGs, and generates the report. PR-authored verdicts
- # and job conclusions are never authority for the required status.
- - name: Derive trusted visual evidence and report
- if: >-
- steps.identity.outputs.valid == 'true' &&
- steps.scope.outputs.stable == 'true' &&
- steps.scope.outputs.broad != 'true'
+ # Only JSON and re-encoded PNGs from the exact CI attempt are published.
+ # No browser, baseline comparison, acceptance, or status authority lives here.
+ - name: Prepare canonical visual report
+ if: steps.identity.outputs.valid == 'true' && steps.visual-download.outcome == 'success'
env:
- PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
+ BASE_SHA: ${{ steps.identity.outputs.tested_base_sha }}
RUN_ID: ${{ steps.identity.outputs.run_id }}
RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
run: |
node .github/scripts/visual-gate/publish-pr-report.mjs \
- --input trusted-capture \
+ --input visual-pr-report \
--output trusted-visual \
- --baseline /tmp/gh-pages/visual-gate/baseline \
- --scope trusted-scope.json \
- --pr "$PR_NUMBER" \
--head-sha "$HEAD_SHA" \
+ --base-sha "$BASE_SHA" \
--run-id "$RUN_ID" \
--run-attempt "$RUN_ATTEMPT"
@@ -534,14 +354,13 @@ jobs:
id: visual
if: >-
steps.identity.outputs.valid == 'true' &&
- steps.scope.outputs.stable == 'true'
+ steps.visual-download.outcome == 'success'
env:
PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
RUN_ID: ${{ steps.identity.outputs.run_id }}
RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
run: |
- test -f trusted-visual/evidence.json
test -f trusted-visual/verdict.json
test -f trusted-visual/index.html
echo "ready=true" >> "$GITHUB_OUTPUT"
@@ -560,20 +379,6 @@ jobs:
--scope pr-visual/evidence
--message "Visual evidence: ${EVIDENCE_PATH}"
- - name: Refresh trusted visual state checkout
- if: steps.publish.outputs.published == 'true'
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
- HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
- run: >
- rm -rf /tmp/gh-pages &&
- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages
- "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" /tmp/gh-pages &&
- git -C /tmp/gh-pages sparse-checkout set
- visual-gate/baseline
- "pr/${PR_NUMBER}/visual/${HEAD_SHA}"
-
- name: Generate and post PR comment
if: always() && steps.identity.outputs.valid == 'true'
uses: actions/github-script@v9
@@ -616,72 +421,3 @@ jobs:
visualPublished: process.env.VISUAL_PUBLISHED === 'true',
visualReportPath: process.env.VISUAL_PATH,
});
-
- - name: Evaluate visual acceptance state
- if: always() && steps.identity.outputs.valid == 'true'
- env:
- SCOPE_OUTCOME: ${{ steps.scope.outcome }}
- HAS_STABLE_VISUAL: ${{ steps.scope.outputs.stable }}
- VISUAL_READY: ${{ steps.visual.outputs.ready }}
- VISUAL_PUBLISHED: ${{ steps.publish.outputs.published }}
- PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
- HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
- run: |
- if [ "$SCOPE_OUTCOME" != "success" ]; then
- printf '%s\n' '{"state":"failure","reason":"scope","description":"Stable visual scope could not be evaluated."}' > visual-state.json
- elif [ "$HAS_STABLE_VISUAL" != "true" ]; then
- printf '%s\n' '{"state":"success","reason":"scope","description":"No stable visual scope."}' > visual-state.json
- elif [ "$VISUAL_READY" != "true" ] || [ "$VISUAL_PUBLISHED" != "true" ]; then
- printf '%s\n' '{"state":"failure","reason":"capture","description":"Stable visual evidence could not be published."}' > visual-state.json
- else
- node .github/scripts/visual-gate/visual-acceptance.mjs state \
- --pages /tmp/gh-pages --pr "$PR_NUMBER" --head "$HEAD_SHA" > visual-state.json
- fi
-
- - name: Publish visual acceptance status
- if: always() && steps.identity.outputs.valid == 'true'
- uses: actions/github-script@v9
- env:
- PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
- HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
- VISUAL_PATH: ${{ steps.visual.outputs.path }}
- with:
- retries: 3
- script: |
- const fs = require('node:fs');
- const {owner, repo} = context.repo;
- const pr = Number(process.env.PR_NUMBER);
- let state = {state: 'failure', reason: 'state', description: 'Visual acceptance state could not be evaluated.'};
- if (fs.existsSync('visual-state.json')) {
- try {
- state = JSON.parse(fs.readFileSync('visual-state.json', 'utf8'));
- } catch (error) {
- core.warning(`Could not read visual state: ${error.message}`);
- }
- }
- await github.rest.repos.createCommitStatus({
- owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance',
- state: state.state, description: state.description.slice(0, 140),
- ...(process.env.VISUAL_PATH
- ? {target_url: `https://facebook.github.io/astryx/${process.env.VISUAL_PATH}/`}
- : {}),
- });
- const label = 'visual-approved';
- if (state.reason === 'accepted') {
- try {
- await github.rest.issues.getLabel({owner, repo, name: label});
- } catch (error) {
- if (error.status !== 404) throw error;
- await github.rest.issues.createLabel({
- owner, repo, name: label, color: '2da44e',
- description: 'Current visual bundle has an explicit acceptance record',
- });
- }
- await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});
- } else {
- try {
- await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});
- } catch (error) {
- if (error.status !== 404) throw error;
- }
- }
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
deleted file mode 100644
index cae54e0d0d..0000000000
--- a/.github/workflows/release-gate.yml
+++ /dev/null
@@ -1,303 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-
-# Release gate — the checks a release must clear, run before the daily cut.
-#
-# Two of them, and they answer different questions:
-# visual — did the canonical Neutral component surface or generated Probe
-# coverage fixture change since the last accepted baseline?
-# (.github/scripts/visual-gate)
-# a11y — did any NEW accessibility violation appear against the baseline?
-# (the same full-suite audit a11y-weekly runs, run daily here)
-#
-# Why this is not in ci.yml: capturing several hundred screenshots and running
-# axe over every story is the wrong tax on every push to every PR. It runs once
-# a day instead, and on demand for anyone who wants one sooner.
-#
-# TIMING. It fires at 06:00 PT, two hours before the 08:00 PT release cut, and
-# the margin is deliberate: the visual leg takes ~15 min but the a11y leg audits
-# every story in the index and has taken ~57 min (a11y-weekly run 31998548109).
-# At the original 07:10 the verdict would have landed AFTER the cut window
-# opened — a gate the release cannot read in time is not a gate.
-#
-# THE VERDICT IS ADVICE WITH TEETH, NOT A VETO. `changed` is not `failed`: a
-# visual change is only a regression if nobody meant it. The release cut reads
-# verdict.json, opens the report (before / after / diff per shot), and decides
-# per change — "the after is correct" is a valid outcome, recorded by promoting
-# that shot through visual-baseline.yml. What must never happen is a release
-# going out through a diff nobody looked at.
-#
-# Consumed by the release cron as:
-# https://facebook.github.io/astryx/visual-gate/latest/verdict.json (status, changes)
-# https://facebook.github.io/astryx/visual-gate/latest/ (the report)
-
-name: Release Gate
-
-on:
- schedule:
- # 13:00 UTC = 06:00 PT — two hours before the 08:00 PT cut (see TIMING above).
- - cron: '0 13 * * *'
- workflow_dispatch:
-
-permissions: {}
-
-concurrency:
- group: release-gate
- cancel-in-progress: false
-
-jobs:
- visual:
- name: Visual regression
- # Pinned label, deliberately: a baseline is only comparable against the
- # same platform and browser build. The gate refuses the comparison rather
- # than reporting a runner change as hundreds of regressions.
- runs-on: 2-core-ubuntu-arm
- timeout-minutes: 45
- permissions:
- contents: read
- outputs:
- status: ${{ steps.gate.outputs.status }}
- changed: ${{ steps.gate.outputs.changed }}
- added: ${{ steps.gate.outputs.added }}
- total: ${{ steps.gate.outputs.total }}
- steps:
- - name: Checkout
- uses: actions/checkout@v7
-
- - name: Setup Node and pnpm
- uses: ./.github/actions/setup
-
- - name: Build packages
- run: pnpm build
-
- - name: Build Storybook
- run: pnpm -F @astryxdesign/storybook build
-
- - name: Install Playwright browser
- run: npx playwright install chromium
-
- # Only the baseline, not the whole gh-pages tree (which carries every PR
- # preview): sparse + blob:none keeps this to the PNGs actually needed.
- - name: Fetch visual baseline from gh-pages
- run: |
- set -eu
- rm -rf /tmp/gh-pages
- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages \
- "https://github.com/${GITHUB_REPOSITORY}.git" /tmp/gh-pages
- git -C /tmp/gh-pages sparse-checkout set visual-gate/baseline
- mkdir -p .visual-baseline
- if [ -d /tmp/gh-pages/visual-gate/baseline ]; then
- cp -r /tmp/gh-pages/visual-gate/baseline/. .visual-baseline/
- echo "Baseline: $(ls .visual-baseline/shots 2>/dev/null | wc -l) shots"
- else
- echo "::notice::No baseline on gh-pages yet — this run establishes one."
- fi
-
- # The canonical release plan is a closed policy: representative and
- # explicitly tagged Core stories in Neutral, plus generated Probe coverage.
- # Existing keys outside that set are reported as reviewed removals so the
- # manual baseline workflow can prune them deliberately. A caller may not
- # expand, rename, or narrow the plan that owns removal authority.
- # Exit 2 means "changed", which is a release decision; only a crash (1)
- # fails this job mechanically.
- - name: Run the visual gate
- id: gate
- run: |
- set +e
- node .github/scripts/visual-gate/gate.mjs release \
- --storybook-dir apps/storybook/dist \
- --baseline .visual-baseline \
- --out .visual-run \
- --summary-output visual-summary.md
- code=$?
- cat visual-summary.md >> "$GITHUB_STEP_SUMMARY" || true
- [ "$code" -eq 1 ] && exit 1
- exit 0
-
- # The capture is what a promotion copies from, so it has to outlive the
- # run: visual-baseline.yml promotes shots out of this artifact by run id.
- - name: Upload capture
- if: always()
- uses: actions/upload-artifact@v7
- with:
- name: visual-capture
- path: |
- .visual-run/shots/
- .visual-run/manifest.json
- .visual-run/verdict.json
- retention-days: 14
- if-no-files-found: warn
-
- - name: Upload report
- if: always()
- uses: actions/upload-artifact@v7
- with:
- name: visual-report
- path: .visual-run/report/
- retention-days: 30
- if-no-files-found: warn
-
- # The assertion a pixel diff cannot make. A baseline proves a component
- # looks like it did last week — including when the reason is that a theme
- # override stopped applying and the broken frame was promoted as correct.
- # This asserts each declared target's override actually reached the
- # pixels, by equality against the probe theme's unique per-selector
- # colour. No baseline, no images, and it names the target.
- #
- # Reported, not enforced, until the standing failures are worked down:
- # today 50 targets lose to component styles because StyleX emits into
- # @layer priority1-4, which sort AFTER astryx-theme. Failing the gate on
- # a known systemic issue would just teach everyone to ignore it.
- - name: Check theme overrides reach the pixels
- id: reach
- continue-on-error: true
- run: |
- set +e
- node .github/scripts/visual-gate/gate.mjs reach \
- --storybook-dir apps/storybook/dist \
- --out .visual-run > reach-summary.txt 2>&1
- code=$?
- {
- echo "## Theme override reach"
- echo ""
- echo '```'
- head -40 reach-summary.txt
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
- cat reach-summary.txt
- [ "$code" -eq 1 ] && exit 1
- exit 0
-
- - name: Upload reach report
- if: always()
- uses: actions/upload-artifact@v7
- with:
- name: visual-reach
- path: .visual-run/reach.json
- retention-days: 30
- if-no-files-found: warn
-
- a11y:
- name: Accessibility
- runs-on: 2-core-ubuntu-arm
- # The full-suite audit walks every story in the index and has taken ~57 min.
- # A 45-minute budget killed it mid-audit, which reads as `crashed` — and a
- # crashed gate holds the release, so an under-sized timeout would block
- # every cut.
- timeout-minutes: 90
- permissions:
- contents: read
- outputs:
- status: ${{ steps.summarize.outputs.status }}
- total_violations: ${{ steps.summarize.outputs.total_violations }}
- steps:
- - name: Checkout
- uses: actions/checkout@v7
-
- - name: Setup Node and pnpm
- uses: ./.github/actions/setup
-
- - name: Build packages
- run: pnpm build
-
- - name: Build Storybook
- run: pnpm -F @astryxdesign/storybook build
-
- - name: Install Playwright browser
- run: npx playwright install chromium
-
- - name: Run full-suite accessibility audit
- id: audit
- continue-on-error: true
- run: |
- BASELINE_FLAGS=""
- if [ -f .github/a11y-baseline.json ]; then
- BASELINE_FLAGS="--baseline .github/a11y-baseline.json --fail-on-new"
- fi
- node .github/scripts/accessibility-audit.js \
- --storybook-dir apps/storybook/dist \
- --output a11y-report.json \
- $BASELINE_FLAGS
-
- - name: Summarize
- id: summarize
- env:
- AUDIT_OUTCOME: ${{ steps.audit.outcome }}
- run: |
- node .github/scripts/weekly-a11y-summary.js \
- --report a11y-report.json \
- --audit-outcome "$AUDIT_OUTCOME" \
- --summary-output a11y-summary.md \
- --github-output "$GITHUB_OUTPUT"
- cat a11y-summary.md >> "$GITHUB_STEP_SUMMARY"
-
- - name: Upload report
- if: always()
- uses: actions/upload-artifact@v7
- with:
- name: a11y-gate-report
- path: |
- a11y-report.json
- a11y-summary.md
- retention-days: 30
- if-no-files-found: warn
-
- publish:
- name: Publish verdict
- needs: [visual, a11y]
- if: ${{ !cancelled() }}
- runs-on: ubuntu-slim
- permissions:
- actions: read
- contents: write
- steps:
- - name: Checkout publisher
- uses: actions/checkout@v7
-
- - name: Download report
- uses: actions/download-artifact@v8
- with:
- name: visual-report
- path: report
-
- - name: Compose the release-gate verdict
- env:
- VISUAL_STATUS: ${{ needs.visual.result == 'success' && needs.visual.outputs.status || 'crashed' }}
- A11Y_STATUS: ${{ needs.a11y.outputs.status || 'crashed' }}
- A11Y_VIOLATIONS: ${{ needs.a11y.outputs.total_violations || '0' }}
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- run: |
- set -eu
- REPORT_URL="https://facebook.github.io/astryx/visual-gate/${GITHUB_RUN_ID}/"
- cat > report/release-gate.json <<JSON
- {
- "generatedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
- "sha": "${GITHUB_SHA}",
- "runId": "${GITHUB_RUN_ID}",
- "runUrl": "${RUN_URL}",
- "reportUrl": "${REPORT_URL}",
- "visual": {"status": "${VISUAL_STATUS}"},
- "a11y": {"status": "${A11Y_STATUS}", "violations": ${A11Y_VIOLATIONS}}
- }
- JSON
- cat report/release-gate.json
-
- - name: Publish to gh-pages
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: node .github/scripts/gh-pages-publisher.mjs release-gate --source report
-
- - name: Summary
- env:
- VISUAL_STATUS: ${{ needs.visual.outputs.status }}
- A11Y_STATUS: ${{ needs.a11y.outputs.status }}
- run: |
- {
- echo "## Release gate"
- echo ""
- echo "| check | status |"
- echo "|---|---|"
- echo "| visual | ${VISUAL_STATUS:-crashed} |"
- echo "| a11y | ${A11Y_STATUS:-crashed} |"
- echo ""
- echo "Report: https://facebook.github.io/astryx/visual-gate/${GITHUB_RUN_ID}/"
- } >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/visual-acceptance-promote.yml b/.github/workflows/visual-acceptance-promote.yml
deleted file mode 100644
index a0c2086e41..0000000000
--- a/.github/workflows/visual-acceptance-promote.yml
+++ /dev/null
@@ -1,457 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-
-# Promote a PR acceptance only after the merged commit reproduces every reviewed
-# AFTER frame. Normal closed-PR promotion and explicit recovery both resolve
-# identity from GitHub and converge on this one trusted promotion job.
-
-name: Visual acceptance promotion
-
-on:
- pull_request_target:
- branches: ['main']
- types: [closed]
- workflow_dispatch:
- inputs:
- pr_number:
- description: 'Merged PR whose immutable visual acceptance should be retried'
- required: true
-
-permissions: {}
-
-jobs:
- resolve:
- if: >-
- github.event_name == 'workflow_dispatch' ||
- github.event.pull_request.merged == true
- runs-on: ubuntu-slim
- timeout-minutes: 5
- permissions:
- actions: read
- contents: read
- pull-requests: read
- outputs:
- pr: ${{ steps.resolve.outputs.pr }}
- head_sha: ${{ steps.resolve.outputs.head_sha }}
- merge_sha: ${{ steps.resolve.outputs.merge_sha }}
- recovery: ${{ steps.resolve.outputs.recovery }}
- validation_ok: ${{ steps.resolve.outputs.identity_valid == 'true' && steps.acceptance.outputs.validation_ok == 'true' }}
- acceptance_found: ${{ steps.acceptance.outputs.found }}
- mutation_deferred: ${{ steps.acceptance.outputs.deferred }}
- deferred_description: ${{ steps.acceptance.outputs.deferred_description }}
- record_rel: ${{ steps.acceptance.outputs.record_rel }}
- failure_description: ${{ steps.acceptance.outputs.failure_description || steps.resolve.outputs.failure_description }}
- steps:
- - name: Checkout trusted current main
- uses: actions/checkout@v7
- with:
- ref: main
-
- - name: Resolve trusted PR identity
- id: resolve
- uses: actions/github-script@v9
- env:
- REQUESTED_PR: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || github.event.pull_request.number }}
- with:
- retries: 3
- script: |
- const {pathToFileURL} = require('node:url');
- const {
- promotionFailure,
- resolvePullRequestIdentity,
- } = await import(
- pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-gate/lib/promotion-identity.mjs`).href,
- );
- if (
- context.eventName === 'workflow_dispatch' &&
- context.ref !== 'refs/heads/main'
- ) {
- core.setFailed('Recovery must be dispatched from main.');
- return;
- }
- const {owner, repo} = context.repo;
- const requestedPr = Number(process.env.REQUESTED_PR);
- if (!Number.isSafeInteger(requestedPr) || requestedPr <= 0) {
- core.setFailed('Recovery requires a positive integer PR number.');
- return;
- }
- let pull;
- try {
- ({data: pull} = await github.rest.pulls.get({
- owner,
- repo,
- pull_number: requestedPr,
- }));
- } catch (error) {
- core.setFailed(`Could not resolve PR #${requestedPr}: ${error.message}`);
- return;
- }
- try {
- resolvePullRequestIdentity({
- pull,
- requestedPr,
- repository: `${owner}/${repo}`,
- compareStatus: 'identical',
- });
- const {data: comparison} =
- await github.rest.repos.compareCommitsWithBasehead({
- owner,
- repo,
- basehead: `${pull.merge_commit_sha}...main`,
- });
- const identity = resolvePullRequestIdentity({
- pull,
- requestedPr,
- repository: `${owner}/${repo}`,
- compareStatus: comparison.status,
- });
- core.setOutput('pr', String(identity.pr));
- core.setOutput('head_sha', identity.headSha);
- core.setOutput('merge_sha', identity.mergeSha);
- core.setOutput('identity_valid', 'true');
- core.setOutput(
- 'recovery',
- context.eventName === 'workflow_dispatch' ? 'true' : 'false',
- );
- } catch (error) {
- const failure = promotionFailure(error);
- core.setOutput('identity_valid', 'false');
- core.setOutput('failure_code', failure.code);
- core.setOutput('failure_description', failure.description);
- core.setFailed(failure.description);
- }
-
- - name: Setup trusted Node
- uses: actions/setup-node@v7
- with:
- node-version-file: .nvmrc
-
- - name: Resolve the immutable acceptance
- id: acceptance
- if: steps.resolve.outputs.identity_valid == 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- PR_NUMBER: ${{ steps.resolve.outputs.pr }}
- HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}
- RECOVERY: ${{ steps.resolve.outputs.recovery }}
- run: |
- set -eu
- PAGES="$RUNNER_TEMP/visual-acceptance-pages"
- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages \
- "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$PAGES"
- git -C "$PAGES" sparse-checkout set \
- "visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"
- MISSING_OK=false
- if [ "$RECOVERY" != 'true' ]; then MISSING_OK=true; fi
- RESOLVED=$(node .github/scripts/visual-gate/lib/promotion-identity.mjs resolve-acceptance \
- --pages "$PAGES" \
- --pr "$PR_NUMBER" \
- --head "$HEAD_SHA" \
- --missing-ok "$MISSING_OK")
- OK=$(printf '%s' "$RESOLVED" | jq -r '.ok')
- if [ "$OK" != 'true' ]; then
- {
- echo "validation_ok=false"
- echo "found=false"
- echo "failure_description=$(printf '%s' "$RESOLVED" | jq -er '.description')"
- } >> "$GITHUB_OUTPUT"
- exit 0
- fi
- FOUND=$(printf '%s' "$RESOLVED" | jq -r '.found')
- DEFERRED=$(printf '%s' "$RESOLVED" | jq -r '.deferred // false')
- {
- echo "validation_ok=true"
- echo "found=$FOUND"
- echo "deferred=$DEFERRED"
- } >> "$GITHUB_OUTPUT"
- if [ "$DEFERRED" = 'true' ]; then
- echo "deferred_description=$(printf '%s' "$RESOLVED" | jq -er '.deferredDescription')" >> "$GITHUB_OUTPUT"
- fi
- if [ "$FOUND" != 'true' ]; then
- echo "No visual acceptance was recorded for this merged head."
- exit 0
- fi
- echo "record_rel=$(printf '%s' "$RESOLVED" | jq -er '.recordRel')" >> "$GITHUB_OUTPUT"
-
- promote:
- needs: resolve
- if: >-
- needs.resolve.outputs.validation_ok == 'true' &&
- needs.resolve.outputs.acceptance_found == 'true'
- runs-on: 2-core-ubuntu-arm
- timeout-minutes: 90
- permissions:
- actions: write
- contents: write
- statuses: write
- outputs:
- recovery_complete: ${{ steps.complete.outputs.recovery_complete }}
- mutation_deferred: ${{ steps.defer.outputs.deferred == 'true' || steps.acceptance.outputs.deferred == 'true' || steps.promote.outputs.deferred == 'true' }}
- deferred_description: ${{ steps.defer.outputs.deferred_description || steps.acceptance.outputs.deferred_description || steps.promote.outputs.deferred_description }}
- failure_description: ${{ steps.acceptance.outputs.failure_description || steps.promote.outputs.failure_description || steps.gate.outputs.failure_description || steps.release.outputs.failure_description }}
- steps:
- - name: Checkout trusted current main
- uses: actions/checkout@v7
- with:
- ref: main
-
- - name: Setup trusted Node and pnpm
- uses: ./.github/actions/setup
- with:
- install: 'false'
-
- - name: Confirm trusted active-retry deferral
- id: defer
- if: needs.resolve.outputs.mutation_deferred == 'true'
- env:
- DEFERRED_DESCRIPTION: ${{ needs.resolve.outputs.deferred_description }}
- run: |
- set -eu
- test -n "$DEFERRED_DESCRIPTION"
- {
- echo "deferred=true"
- echo "deferred_description=$DEFERRED_DESCRIPTION"
- } >> "$GITHUB_OUTPUT"
-
- - name: Mark trusted promotion pending
- if: steps.defer.outputs.deferred != 'true'
- uses: actions/github-script@v9
- env:
- HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
- with:
- retries: 3
- script: |
- await github.rest.repos.createCommitStatus({
- ...context.repo,
- sha: process.env.HEAD_SHA,
- context: 'visual-acceptance',
- state: 'pending',
- description: 'Trusted post-merge visual promotion is running.',
- target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
- });
-
- - name: Queue baseline publication
- id: queue
- if: steps.defer.outputs.deferred != 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- run: node .github/scripts/gh-pages-publisher.mjs enqueue --scope visual-gate/baseline
-
- - name: Load and verify the immutable acceptance
- id: acceptance
- if: steps.defer.outputs.deferred != 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- PR_NUMBER: ${{ needs.resolve.outputs.pr }}
- HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
- EXPECTED_RECORD_REL: ${{ needs.resolve.outputs.record_rel }}
- run: |
- set -eu
- PAGES="$RUNNER_TEMP/visual-acceptance-pages"
- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages \
- "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$PAGES"
- git -C "$PAGES" sparse-checkout set \
- "visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"
- RESOLVED=$(node .github/scripts/visual-gate/lib/promotion-identity.mjs resolve-acceptance \
- --pages "$PAGES" \
- --pr "$PR_NUMBER" \
- --head "$HEAD_SHA" \
- --missing-ok false \
- --expected-record-rel "$EXPECTED_RECORD_REL")
- OK=$(printf '%s' "$RESOLVED" | jq -r '.ok')
- if [ "$OK" != 'true' ]; then
- DESCRIPTION=$(printf '%s' "$RESOLVED" | jq -er '.description')
- echo "failure_description=$DESCRIPTION" >> "$GITHUB_OUTPUT"
- echo "::error::$(printf '%s' "$RESOLVED" | jq -er '.message')"
- exit 1
- fi
- FOUND=$(printf '%s' "$RESOLVED" | jq -r '.found')
- DEFERRED=$(printf '%s' "$RESOLVED" | jq -r '.deferred // false')
- {
- echo "found=$FOUND"
- echo "deferred=$DEFERRED"
- } >> "$GITHUB_OUTPUT"
- if [ "$DEFERRED" = 'true' ]; then
- {
- echo "found=false"
- echo "deferred_description=$(printf '%s' "$RESOLVED" | jq -er '.deferredDescription')"
- } >> "$GITHUB_OUTPUT"
- exit 0
- fi
- {
- echo "record=$(printf '%s' "$RESOLVED" | jq -er '.recordPath')"
- echo "record_rel=$(printf '%s' "$RESOLVED" | jq -er '.recordRel')"
- } >> "$GITHUB_OUTPUT"
-
- - name: Checkout the resolved merged result
- if: steps.acceptance.outputs.found == 'true'
- uses: actions/checkout@v7
- with:
- ref: ${{ needs.resolve.outputs.merge_sha }}
- path: .visual-merged-source
- # The read-only resolver proved this exact commit is reachable from
- # current main before the privileged job started. Fork PR merges need
- # checkout's explicit opt-in even for that trusted merged commit.
- allow-unsafe-pr-checkout: true
-
- - name: Install trusted visual dependencies
- if: steps.acceptance.outputs.found == 'true'
- run: pnpm install --frozen-lockfile
-
- - name: Install merged-result dependencies
- if: steps.acceptance.outputs.found == 'true'
- working-directory: .visual-merged-source
- run: pnpm install --frozen-lockfile
-
- - name: Build the resolved merged Storybook
- if: steps.acceptance.outputs.found == 'true'
- working-directory: .visual-merged-source
- run: pnpm build && pnpm -F @astryxdesign/storybook build
-
- - name: Install Chromium
- if: steps.acceptance.outputs.found == 'true'
- run: npx playwright install chromium
-
- - name: Recapture exactly the accepted shots
- if: steps.acceptance.outputs.found == 'true'
- env:
- ASTRYX_VISUAL_SHA: ${{ needs.resolve.outputs.merge_sha }}
- ACCEPTANCE: ${{ steps.acceptance.outputs.record }}
- run: |
- set -eu
- node .github/scripts/visual-gate/visual-acceptance.mjs plan \
- --acceptance "$ACCEPTANCE" \
- --output accepted-plan.json
- node .github/scripts/visual-gate/gate.mjs capture \
- --storybook-dir .visual-merged-source/apps/storybook/dist \
- --out .visual-merged \
- --plan-file accepted-plan.json
-
- - name: Wait for the baseline publication turn
- if: steps.acceptance.outputs.found == 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- run: node .github/scripts/gh-pages-publisher.mjs wait --scope visual-gate/baseline
-
- - name: Verify and promote the baseline
- id: promote
- if: steps.acceptance.outputs.found == 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- PR_NUMBER: ${{ needs.resolve.outputs.pr }}
- HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
- MERGE_SHA: ${{ needs.resolve.outputs.merge_sha }}
- RECORD_REL: ${{ steps.acceptance.outputs.record_rel }}
- run: >
- node .github/scripts/gh-pages-publisher.mjs visual-baseline-accepted
- --pr "$PR_NUMBER"
- --head "$HEAD_SHA"
- --merge-sha "$MERGE_SHA"
- --expected-record-rel "$RECORD_REL"
- --capture .visual-merged
-
- - name: Run a fresh release gate
- id: gate
- if: steps.promote.outputs.publication_confirmed == 'true'
- uses: actions/github-script@v9
- with:
- retries: 3
- script: |
- try {
- await github.rest.actions.createWorkflowDispatch({
- ...context.repo,
- workflow_id: 'release-gate.yml',
- ref: context.payload.repository.default_branch,
- });
- } catch (error) {
- core.setOutput(
- 'failure_description',
- 'Baseline published, but the fresh release gate dispatch failed.',
- );
- core.setFailed(error.message);
- }
-
- - name: Release the baseline publication turn
- id: release
- if: always() && steps.defer.outputs.deferred != 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- set -eu
- if ! OUTPUT=$(node .github/scripts/gh-pages-publisher.mjs release --scope visual-gate/baseline 2>&1); then
- echo "failure_description=Baseline publication finished, but lock release failed." >> "$GITHUB_OUTPUT"
- echo "::error::$OUTPUT"
- exit 1
- fi
- printf '%s\n' "$OUTPUT"
-
- - name: Mark trusted recovery complete
- id: complete
- if: >-
- always() &&
- steps.promote.outputs.publication_confirmed == 'true' &&
- steps.gate.outcome == 'success' &&
- steps.release.outcome == 'success'
- run: echo "recovery_complete=true" >> "$GITHUB_OUTPUT"
-
- project-status:
- needs: [resolve, promote]
- if: always() && needs.resolve.outputs.head_sha != ''
- runs-on: ubuntu-slim
- timeout-minutes: 5
- permissions:
- contents: read
- statuses: write
- steps:
- - name: Checkout trusted current main
- uses: actions/checkout@v7
- with:
- ref: main
-
- - name: Project the final visual acceptance status
- uses: actions/github-script@v9
- env:
- HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
- VALIDATION_OK: ${{ needs.resolve.outputs.validation_ok }}
- ACCEPTANCE_FOUND: ${{ needs.resolve.outputs.acceptance_found }}
- JOB_RESULT: ${{ needs.promote.result }}
- RECOVERY_COMPLETE: ${{ needs.promote.outputs.recovery_complete }}
- MUTATION_DEFERRED: ${{ needs.promote.outputs.mutation_deferred == 'true' }}
- DEFERRED_DESCRIPTION: ${{ needs.promote.outputs.deferred_description }}
- FAILURE_DESCRIPTION: ${{ needs.resolve.outputs.failure_description || needs.promote.outputs.failure_description }}
- TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- with:
- retries: 3
- script: |
- const {pathToFileURL} = require('node:url');
- const {
- promotionStatusProjection,
- recoveryOperationResult,
- } = await import(
- pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-gate/lib/promotion-identity.mjs`).href,
- );
- const projection = promotionStatusProjection({
- headKnown: true,
- validationOk: process.env.VALIDATION_OK === 'true',
- acceptanceFound: process.env.ACCEPTANCE_FOUND === 'true',
- promotionResult: recoveryOperationResult({
- jobResult: process.env.JOB_RESULT,
- recoveryComplete: process.env.RECOVERY_COMPLETE === 'true',
- mutationDeferred: process.env.MUTATION_DEFERRED === 'true',
- failureDescription: process.env.FAILURE_DESCRIPTION,
- }),
- mutationDeferred: process.env.MUTATION_DEFERRED === 'true',
- deferredDescription: process.env.DEFERRED_DESCRIPTION,
- failureDescription: process.env.FAILURE_DESCRIPTION,
- });
- if (!projection) return;
- await github.rest.repos.createCommitStatus({
- ...context.repo,
- sha: process.env.HEAD_SHA,
- context: 'visual-acceptance',
- state: projection.state,
- description: projection.description,
- target_url: process.env.TARGET_URL,
- });
- if (projection.state === 'failure') {
- core.setFailed(projection.description);
- }
diff --git a/.github/workflows/visual-acceptance.yml b/.github/workflows/visual-acceptance.yml
deleted file mode 100644
index 53aa50ce74..0000000000
--- a/.github/workflows/visual-acceptance.yml
+++ /dev/null
@@ -1,334 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-
-# Explicit human decisions for stable visual changes. Initial status creation and
-# rerun invalidation live in the trusted PR Comment workflow_run publisher, so
-# they do not create a cancellable pull-request check that GitHub renders red.
-# This workflow never checks out or executes PR-controlled code. It runs the
-# default-branch implementation, resolves the current PR head through GitHub,
-# and stores the signed decision beside trusted, immutable evidence on gh-pages.
-
-name: Visual acceptance
-
-on:
- issue_comment:
- types: [created]
-
-permissions: {}
-
-jobs:
- authorize:
- if: >-
- github.event_name == 'issue_comment' &&
- github.event.issue.pull_request &&
- startsWith(github.event.comment.body, '/accept-visual')
- runs-on: ubuntu-slim
- permissions:
- actions: read
- contents: read
- pull-requests: read
- outputs:
- resolved: ${{ steps.authorize.outputs.resolved }}
- valid: ${{ steps.authorize.outputs.valid }}
- refusal: ${{ steps.authorize.outputs.refusal }}
- pr: ${{ steps.authorize.outputs.pr }}
- head: ${{ steps.authorize.outputs.head }}
- head_repo_id: ${{ steps.authorize.outputs.head_repo_id }}
- head_ref: ${{ steps.authorize.outputs.head_ref }}
- run_id: ${{ steps.authorize.outputs.run_id }}
- run_attempt: ${{ steps.authorize.outputs.run_attempt }}
- approver: ${{ steps.authorize.outputs.approver }}
- approver_id: ${{ steps.authorize.outputs.approver_id }}
- permission: ${{ steps.authorize.outputs.permission }}
- effective_permission: ${{ steps.authorize.outputs.effective_permission }}
- role_name: ${{ steps.authorize.outputs.role_name }}
- comment_id: ${{ steps.authorize.outputs.comment_id }}
- reason: ${{ steps.authorize.outputs.reason }}
- steps:
- # This job is deliberately read-only. The mutation job acquires the shared
- # head-branch lock using these trusted API-derived outputs.
- - name: Checkout trusted default-branch code
- uses: actions/checkout@v7
-
- - name: Authorize and resolve the decision
- id: authorize
- uses: actions/github-script@v9
- with:
- retries: 3
- script: |
- const {pathToFileURL} = require('node:url');
- const {
- isVisualAcceptanceEndpointMaintainer,
- visualAcceptanceEvidencePath,
- visualAcceptanceIdentity,
- } = await import(
- pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-gate/authorization.mjs`).href,
- );
- const {owner, repo} = context.repo;
- const number = context.issue.number;
- const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: number});
- if (!pr.head.repo?.id || !pr.head.ref) {
- core.setFailed('The pull request head repository or branch is unavailable.');
- return;
- }
- core.setOutput('resolved', 'true');
- core.setOutput('pr', String(number));
- core.setOutput('head', pr.head.sha);
- core.setOutput('head_repo_id', String(pr.head.repo.id));
- core.setOutput('head_ref', pr.head.ref);
- const refuse = (message) => {
- core.setOutput('valid', 'false');
- core.setOutput('refusal', message);
- };
- const body = context.payload.comment.body.trim();
- const match = body.match(/^\/accept-visual\s+(\d+)\/(\d+)\s+([\s\S]+)$/);
- if (!match) {
- refuse('copy the exact maintainer-only `/accept-visual <run>/<attempt> <why every changed frame is correct>` command from the latest report.');
- return;
- }
- const runId = Number(match[1]);
- const runAttempt = Number(match[2]);
- const reason = match[3].trim();
- if (reason.length < 12 || reason.length > 500 ||
- /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(reason)) {
- refuse('the reason must explain why the pixels are correct (12–500 characters).');
- return;
- }
- if (pr.state !== 'open' || pr.base.ref !== 'main') {
- refuse('the pull request is not open against `main`.');
- return;
- }
- const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {
- owner, repo, event: 'pull_request', head_sha: pr.head.sha, per_page: 100,
- });
- const latestCI = runs
- .filter((run) => run.name === 'CI')
- .sort((a, b) => b.id - a.id)[0];
- if (!latestCI || latestCI.id !== runId || latestCI.run_attempt !== runAttempt ||
- latestCI.status !== 'completed') {
- refuse('that command is not from the latest completed CI attempt for this head. Wait for the latest report and copy its command.');
- return;
- }
- const evidencePath = visualAcceptanceEvidencePath({
- pr: number,
- head: pr.head.sha,
- run: runId,
- attempt: runAttempt,
- });
- try {
- await github.rest.repos.getContent({
- owner,
- repo,
- path: evidencePath,
- ref: 'gh-pages',
- });
- } catch (error) {
- if (error.status !== 404) throw error;
- refuse('trusted visual evidence is not published yet. Wait for the latest PR report; if it does not appear, rerun CI and use its new command.');
- return;
- }
- let identity = visualAcceptanceIdentity();
- try {
- const response = await github.rest.repos.getCollaboratorPermissionLevel({
- owner, repo, username: context.actor,
- });
- identity = visualAcceptanceIdentity(response.data);
- } catch (error) {
- if (error.status !== 404) throw error;
- }
- if (!isVisualAcceptanceEndpointMaintainer(identity)) {
- refuse('only a repository maintainer may accept stable visual changes.');
- return;
- }
- core.setOutput('valid', 'true');
- core.setOutput('run_id', String(runId));
- core.setOutput('run_attempt', String(runAttempt));
- core.setOutput('approver', context.actor);
- core.setOutput('approver_id', String(context.payload.comment.user.id));
- core.setOutput('permission', identity.permission);
- core.setOutput('effective_permission', identity.effectivePermission);
- core.setOutput('role_name', identity.roleName ?? '');
- core.setOutput('comment_id', String(context.payload.comment.id));
- core.setOutput('reason', reason);
-
- accept:
- needs: authorize
- if: >-
- always() &&
- needs.authorize.result == 'success' &&
- needs.authorize.outputs.resolved == 'true'
- concurrency:
- group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}
- cancel-in-progress: false
- runs-on: ubuntu-slim
- permissions:
- actions: read
- contents: write
- pull-requests: write
- statuses: write
- steps:
- - name: Report a refused decision
- if: needs.authorize.outputs.valid != 'true'
- uses: actions/github-script@v9
- env:
- PR_NUMBER: ${{ needs.authorize.outputs.pr }}
- REFUSAL: ${{ needs.authorize.outputs.refusal }}
- with:
- retries: 3
- script: |
- await github.rest.issues.createComment({
- ...context.repo,
- issue_number: Number(process.env.PR_NUMBER),
- body: `Visual acceptance refused: ${process.env.REFUSAL}`,
- });
-
- - name: Checkout trusted default-branch code
- if: needs.authorize.outputs.valid == 'true'
- uses: actions/checkout@v7
-
- - name: Setup Node and pnpm
- if: needs.authorize.outputs.valid == 'true'
- uses: ./.github/actions/setup
-
- - name: Archive the acceptance decision
- id: record
- if: needs.authorize.outputs.valid == 'true'
- env:
- GH_TOKEN: ${{ github.token }}
- PR_NUMBER: ${{ needs.authorize.outputs.pr }}
- HEAD_SHA: ${{ needs.authorize.outputs.head }}
- RUN_ID: ${{ needs.authorize.outputs.run_id }}
- RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}
- APPROVER: ${{ needs.authorize.outputs.approver }}
- APPROVER_ID: ${{ needs.authorize.outputs.approver_id }}
- PERMISSION: ${{ needs.authorize.outputs.permission }}
- EFFECTIVE_PERMISSION: ${{ needs.authorize.outputs.effective_permission }}
- ROLE_NAME: ${{ needs.authorize.outputs.role_name }}
- COMMENT_ID: ${{ needs.authorize.outputs.comment_id }}
- REASON: ${{ needs.authorize.outputs.reason }}
- run: >
- node .github/scripts/gh-pages-publisher.mjs visual-acceptance-record
- --pr "$PR_NUMBER"
- --head "$HEAD_SHA"
- --accepted-run-id "$RUN_ID"
- --accepted-run-attempt "$RUN_ATTEMPT"
- --approver "$APPROVER"
- --approver-id "$APPROVER_ID"
- --permission "$PERMISSION"
- --effective-permission "$EFFECTIVE_PERMISSION"
- --role-name "$ROLE_NAME"
- --comment-id "$COMMENT_ID"
- --reason "$REASON"
-
- - name: Reconcile acceptance against the latest published evidence
- id: reconcile
- if: steps.record.outcome == 'success'
- env:
- GH_TOKEN: ${{ github.token }}
- PR_NUMBER: ${{ needs.authorize.outputs.pr }}
- HEAD_SHA: ${{ needs.authorize.outputs.head }}
- run: |
- FRESH="$RUNNER_TEMP/visual-acceptance-reconcile"
- rm -rf "$FRESH"
- git clone --depth=1 --single-branch --branch gh-pages \
- "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$FRESH"
- node .github/scripts/visual-gate/visual-acceptance.mjs state \
- --pages "$FRESH" --pr "$PR_NUMBER" --head "$HEAD_SHA" > acceptance-state.json
- node - <<'NODE'
- const fs = require('node:fs');
- const state = JSON.parse(fs.readFileSync('acceptance-state.json', 'utf8'));
- for (const key of ['state', 'reason', 'description']) {
- fs.appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${state[key]}\n`);
- }
- NODE
-
- - name: Project the reconciled decision onto the PR
- if: steps.reconcile.outcome == 'success'
- uses: actions/github-script@v9
- env:
- PR_NUMBER: ${{ needs.authorize.outputs.pr }}
- HEAD_SHA: ${{ needs.authorize.outputs.head }}
- ACCEPTED_RUN_ID: ${{ needs.authorize.outputs.run_id }}
- ACCEPTED_RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}
- ACCEPTANCE_STATE: ${{ steps.reconcile.outputs.state }}
- ACCEPTANCE_REASON: ${{ steps.reconcile.outputs.reason }}
- ACCEPTANCE_DESCRIPTION: ${{ steps.reconcile.outputs.description }}
- with:
- retries: 3
- script: |
- const {owner, repo} = context.repo;
- const pr = Number(process.env.PR_NUMBER);
- const head = process.env.HEAD_SHA;
- let state = process.env.ACCEPTANCE_STATE;
- let reason = process.env.ACCEPTANCE_REASON;
- let description = process.env.ACCEPTANCE_DESCRIPTION;
- const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {
- owner, repo, event: 'pull_request', head_sha: head, per_page: 100,
- });
- const latestCI = runs
- .filter((run) => run.name === 'CI')
- .sort((a, b) => b.id - a.id)[0];
- const exactRun =
- latestCI &&
- latestCI.id === Number(process.env.ACCEPTED_RUN_ID) &&
- latestCI.run_attempt === Number(process.env.ACCEPTED_RUN_ATTEMPT) &&
- latestCI.status === 'completed';
- if (!exactRun) {
- state = 'pending';
- reason = 'capture';
- description = 'A newer CI attempt must publish trusted visual evidence.';
- }
- const accepted = state === 'success' && reason === 'accepted';
- if (!['success', 'pending', 'failure'].includes(state)) {
- core.setFailed('Reconciled visual acceptance state is invalid.');
- return;
- }
- const label = 'visual-approved';
- if (accepted) {
- try {
- await github.rest.issues.getLabel({owner, repo, name: label});
- } catch (error) {
- if (error.status !== 404) throw error;
- await github.rest.issues.createLabel({
- owner, repo, name: label, color: '2da44e',
- description: 'Current visual bundle has an explicit acceptance record',
- });
- }
- await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});
- } else {
- try {
- await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});
- } catch (error) {
- if (error.status !== 404) throw error;
- }
- }
- await github.rest.repos.createCommitStatus({
- owner, repo, sha: head, context: 'visual-acceptance', state,
- description: description.slice(0, 140),
- });
- await github.rest.issues.createComment({
- owner,
- repo,
- issue_number: pr,
- body: accepted
- ? `Visual changes accepted for \`${head.slice(0, 12)}\`. A new push invalidates this approval; after merge, the reviewed AFTER frames must reproduce before the baseline is promoted.`
- : `The requested visual evidence is no longer current. Status remains pending; copy the command from the latest report.`,
- });
-
- - name: Mark an acceptance infrastructure failure
- if: failure() && needs.authorize.outputs.valid == 'true'
- uses: actions/github-script@v9
- env:
- PR_NUMBER: ${{ needs.authorize.outputs.pr }}
- HEAD_SHA: ${{ needs.authorize.outputs.head }}
- with:
- retries: 3
- script: |
- const {owner, repo} = context.repo;
- await github.rest.repos.createCommitStatus({
- owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance', state: 'failure',
- description: 'Visual acceptance hit an infrastructure error.',
- });
- await github.rest.issues.createComment({
- owner, repo, issue_number: Number(process.env.PR_NUMBER),
- body: 'Visual acceptance hit an infrastructure error. Retry only after the latest PR report shows a maintainer command; if no report appears, rerun CI. See this workflow run for details.',
- });
diff --git a/.github/workflows/visual-baseline.yml b/.github/workflows/visual-baseline.yml
deleted file mode 100644
index 3ce5813ff1..0000000000
--- a/.github/workflows/visual-baseline.yml
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-
-# Promote captured shots into the visual baseline — the "the after is correct"
-# outcome of the release gate, made explicit.
-#
-# Separate from release-gate.yml on purpose. The gate only ever READS the
-# baseline; this is the one workflow that writes it, it runs only when a human
-# or the release cut dispatches it, and it refuses to run without a reason.
-# Every promotion appends to the decision log in the baseline manifest (who,
-# when, which run, why, which shots), so any baseline can be read back to the
-# change that justified it.
-#
-# Shots are promoted OUT OF A SPECIFIC GATE RUN's capture artifact, never
-# re-captured here: the picture that becomes the baseline is exactly the
-# picture that was reviewed.
-#
-# Typical uses:
-# keys=core-button--default__y2k-light,core-badge--solid__y2k-light
-# reason="Y2K button radius went square on purpose (#5123)"
-# keys=all reason="Playwright chromium bump — every shot re-rendered"
-
-name: Visual Baseline
-
-on:
- workflow_dispatch:
- inputs:
- run_id:
- description: 'Release Gate run id whose capture to promote from'
- required: true
- keys:
- description: 'Comma-separated shot keys, or "all"'
- required: true
- reason:
- description: 'Why the new rendering is the correct one'
- required: true
- prune:
- description: 'Also drop baseline shots whose story no longer exists'
- required: false
- default: true
- type: boolean
-
-permissions: {}
-
-jobs:
- promote:
- runs-on: ubuntu-slim
- timeout-minutes: 90
- permissions:
- contents: write
- actions: read # download-artifact from another run
- steps:
- - name: Checkout
- uses: actions/checkout@v7
-
- - name: Setup Node and pnpm
- uses: ./.github/actions/setup
-
- - name: Verify the source run
- env:
- GH_TOKEN: ${{ github.token }}
- RUN_ID: ${{ inputs.run_id }}
- run: |
- set -eu
- # Promotion pulls shots out of a specific run's capture artifact. The
- # run id is operator-typed — confirm it names a completed Release Gate
- # run on main before anything from it is treated as baseline material
- # (a typo'd id should fail here, not become the baseline).
- RUN_JSON=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}")
- RUN_PATH=$(printf '%s' "$RUN_JSON" | jq -r '.path')
- RUN_BRANCH=$(printf '%s' "$RUN_JSON" | jq -r '.head_branch')
- RUN_STATUS=$(printf '%s' "$RUN_JSON" | jq -r '.status')
- if [ "$RUN_PATH" != ".github/workflows/release-gate.yml" ] || \
- [ "$RUN_BRANCH" != "main" ] || [ "$RUN_STATUS" != "completed" ]; then
- echo "::error::Run ${RUN_ID} is not a completed Release Gate run on main (path=${RUN_PATH}, branch=${RUN_BRANCH}, status=${RUN_STATUS})."
- exit 1
- fi
-
- - name: Queue baseline publication
- env:
- GH_TOKEN: ${{ github.token }}
- run: node .github/scripts/gh-pages-publisher.mjs enqueue --scope visual-gate/baseline
-
- - name: Download the capture being promoted
- uses: actions/download-artifact@v8
- with:
- name: visual-capture
- run-id: ${{ inputs.run_id }}
- github-token: ${{ github.token }}
- path: .visual-run
-
- - name: Wait for the baseline publication turn
- env:
- GH_TOKEN: ${{ github.token }}
- run: node .github/scripts/gh-pages-publisher.mjs wait --scope visual-gate/baseline
-
- # The publisher calls gate.mjs accept against the current baseline before
- # writing or publishing any shots. Keep verdict validation there: a browser
- # refresh needs both manifests, not a second status-only allowlist here.
- - name: Promote and publish the baseline
- env:
- KEYS: ${{ inputs.keys }}
- REASON: ${{ inputs.reason }}
- PROMOTER: ${{ github.actor }}
- PRUNE: ${{ inputs.prune && 'true' || 'false' }}
- GH_TOKEN: ${{ github.token }}
- run: >
- node .github/scripts/gh-pages-publisher.mjs visual-baseline-manual
- --capture .visual-run
- --keys "$KEYS"
- --reason "$REASON"
- --actor "$PROMOTER"
- --prune "$PRUNE"
-
- - name: Release the baseline publication turn
- if: always()
- env:
- GH_TOKEN: ${{ github.token }}
- run: node .github/scripts/gh-pages-publisher.mjs release --scope visual-gate/baseline
-
- - name: Summary
- env:
- KEYS: ${{ inputs.keys }}
- RUN_ID: ${{ inputs.run_id }}
- REASON: ${{ inputs.reason }}
- run: |
- {
- echo "## Visual baseline updated"
- echo ""
- echo "- promoted: \`${KEYS}\`"
- echo "- from run: ${RUN_ID}"
- echo "- reason: ${REASON}"
- } >> "$GITHUB_STEP_SUMMARY"
diff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.md
index f82bb299a5..8824a9e902 100644
--- a/docs/specs/AST-030/spec.md
+++ b/docs/specs/AST-030/spec.md
@@ -67,7 +67,7 @@ they are separately named.
| `node-tooling` | admitted operational Node tooling whose consumers are covered by Node contract tests | Node Vitest, repository guardrails, and ESLint |
| `runtime:<package>` | public source and package contract for Core, Lab, Charts, Rich Text, Vega, CLI, and Build | the package's unit/type checks plus current broad build and downstream consumer checks; Build's owner includes production CSS-layer browser behavior |
| `theme-build` | shipped theme packages and theme compilation outputs | theme tests, theme package builds, theme-family browser behavior, and stable visual evidence where applicable |
- | `storybook-visual` | Storybook stories/configuration and visual, accessibility, or RTL audit infrastructure | Storybook build, preview/visual-acceptance publication, and the applicable browser, visual, accessibility, and RTL checks |
+ | `storybook-visual` | Storybook stories/configuration and visual, accessibility, or RTL audit infrastructure | Storybook build, preview/visual-report publication, and the applicable browser, visual, accessibility, and RTL checks |
| `shared-or-unknown` | shared configuration, dependency graphs, workflows, classifiers, generated ownership, ambiguous paths, and every unclassified path | all applicable pull-request CI checks |
A category does not earn a specialized lane merely by existing in this table.
@@ -109,9 +109,9 @@ they are separately named.
Node project and the required lint
workflow, including repository guardrails. It skips the UI Vitest project,
component analysis, docsite generation, production package, Storybook and
- Sandbox builds, preview/visual-acceptance publication, and browser/theme/visual/
- a11y/RTL jobs. The trusted post-CI workflow MUST settle the visual status
- explicitly and remove stale preview links without enqueueing the preview
+ Sandbox builds, preview/visual-report publication, and browser/theme/visual/
+ a11y/RTL jobs. The trusted post-CI workflow MUST explicitly report that visual
+ checks are not applicable and remove stale preview links without enqueueing the preview
publisher. The Sandbox score-ledger projection MUST have a Node contract test
for the exports it consumes before this lane can be enabled.
- **FR9 — Routing tests are mutation-sensitive.** Tests MUST prove both the
@@ -138,12 +138,23 @@ they are separately named.
owner includes its production CSS-layer cascade behavior. Theme-family
compilation remains `theme-build` and does not become Build merely because both
behaviors concern CSS.
-- **FR12 — Visual regression has one shared pull-request owner.** Every
+- **FR12 — Visual regression has one shared pull-request workflow.** Every
visual-regression test from every surface MUST use the existing Storybook
- framework and the single shared pull-request visual owner. That owner MAY use
- multiple jobs, workflows, artifacts, and a status projection. A new or existing
- surface, component, package, theme family, or visual suite MUST NOT create
- another independently routed PR visual owner.
+ framework and the single shared owner in `.github/workflows/ci.yml`
+ (`pr-visual`). That workflow MAY use multiple jobs, steps, and artifacts, but
+ visual capture and regression comparison MUST NOT run in another workflow,
+ including scheduled, post-CI, post-merge, or manual replacement owners.
+ Publication of the canonical run's reports is not another test owner: it MUST
+ consume that run's artifacts without recapturing or recomparing pixels.
+ Explicit full-plan baseline capture and reviewed baseline publication MUST
+ remain in `ci.yml`; they are maintenance operations, not release gates.
+ Release evidence remains the constituent PR's exact-head `pr-visual` and
+ `pr-a11y` checks, never a daily gate or retroactive acceptance status.
+ Deployment, accessibility, RTL, and vibe evidence MAY retain their workflows
+ and artifacts when they do not duplicate visual regression. A new or existing
+ surface, component, package, theme family, or visual suite MUST join the
+ canonical workflow. A repository guard MUST reject independent visual owners
+ while preserving legitimate non-visual artifact use.
### Platform support
@@ -172,16 +183,16 @@ the shared Storybook PR owner.
## Verification
-| Contract | Verification | Representative states | Mutation or failure expectation |
-| -------- | ------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| FR1–FR2 | classifier unit tests expose the surface set | knowledge; docsite; admitted tool; Core/Lab/Charts/Rich Text/Vega source; shared config; unknown | an unclassified path receives a specialized lane or a category has no check owner |
-| FR3–FR4 | workflow contract tests execute an isolated trusted-base classifier | valid base; missing merge base; missing matcher/registry; classifier self-change; merge group | PR-controlled policy grants a lane, or missing trust data skips checks |
-| FR5 | mixed-surface and incomplete-owner routing tests | spec+component; tooling+component; named surface without admitted lane; docsite+shared | one surface hides another owner or a named-but-incomplete surface skips broad CI |
-| FR6–FR8 | Node-tooling dependency, CI workflow, and trusted post-CI workflow tests | both admitted score-ledger paths; Sandbox projection imports; test/build joins; preview/visual publisher | UI/browser/build work or preview publication runs for the singleton tool set, an operational consumer is untested, a visual status stays pending, or a required context disappears |
-| FR9 | mutation-sensitive classifier and workflow fixtures | unknown path; rename from unknown; truncated list; package-specific public paths | weakening a fail-closed rule leaves the suite green |
-| FR10 | surface-admission and routing contract tests | existing surface; proposed surface with and without the full admission contract | a new surface adds lanes without its approved paths, commands, ownership, projections, and routing tests |
-| FR11 | surface owner, internal-job, and required-join contract tests | Build package browser behavior; CLI; theme-family broad fallback; mixed Core+CLI | a second owner is routed for one surface, an internal job masquerades as a lane, an owned failure misses its join, or an unrelated owner runs |
-| FR12 | Storybook visual-plan, PR owner, and status-projection contract tests | component, package, story, and theme-family cases | visual coverage bypasses the shared PR owner, creates another independently routed PR visual owner, or conflates a status projection with a lane |
+| Contract | Verification | Representative states | Mutation or failure expectation |
+| -------- | --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| FR1–FR2 | classifier unit tests expose the surface set | knowledge; docsite; admitted tool; Core/Lab/Charts/Rich Text/Vega source; shared config; unknown | an unclassified path receives a specialized lane or a category has no check owner |
+| FR3–FR4 | workflow contract tests execute an isolated trusted-base classifier | valid base; missing merge base; missing matcher/registry; classifier self-change; merge group | PR-controlled policy grants a lane, or missing trust data skips checks |
+| FR5 | mixed-surface and incomplete-owner routing tests | spec+component; tooling+component; named surface without admitted lane; docsite+shared | one surface hides another owner or a named-but-incomplete surface skips broad CI |
+| FR6–FR8 | Node-tooling dependency, CI workflow, and trusted post-CI workflow tests | both admitted score-ledger paths; Sandbox projection imports; test/build joins; preview/visual publisher | UI/browser/build work or preview publication runs for the singleton tool set, an operational consumer is untested, a visual status stays pending, or a required context disappears |
+| FR9 | mutation-sensitive classifier and workflow fixtures | unknown path; rename from unknown; truncated list; package-specific public paths | weakening a fail-closed rule leaves the suite green |
+| FR10 | surface-admission and routing contract tests | existing surface; proposed surface with and without the full admission contract | a new surface adds lanes without its approved paths, commands, ownership, projections, and routing tests |
+| FR11 | surface owner, internal-job, and required-join contract tests | Build package browser behavior; CLI; theme-family broad fallback; mixed Core+CLI | a second owner is routed for one surface, an internal job masquerades as a lane, an owned failure misses its join, or an unrelated owner runs |
+| FR12 | Storybook visual-plan, canonical workflow, and repository-owner guard tests | PR scope; full-plan maintenance; report-only publication; deployment/a11y/RTL/vibe artifacts | another workflow captures or compares pixels, a renamed or aliased owner escapes the guard, or legitimate non-visual artifacts are rejected |
## Decision log
@@ -236,9 +247,11 @@ add owners only after the complete FR10 admission contract is approved. A named
surface without a complete admitted owner remains on broad CI.
Visual regression is the exception to per-surface routing: every surface
-registers its cases with the one shared Storybook-backed PR visual owner. Its
-jobs, workflows, artifacts, and status projection are not additional visual
-lanes.
+registers its cases with the one shared Storybook-backed PR visual owner in
+`ci.yml`. Its jobs, steps, artifacts, and report publication are not additional
+visual lanes; a second workflow that captures or compares those pixels is.
+The single-workflow wording clarifies the existing shared-system requirement;
+it does not admit a new owner.
Build's production CSS-layer cascade is a Build package integration contract, so
it belongs to Build's test owner. Theme-family compilation remains owned by
diff --git a/package.json b/package.json
index 338946bfb9..97467b1104 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,8 @@
"test:coverage": "vitest --coverage",
"test:a11y-contract": "playwright test --config playwright.config.ts --project chromium",
"check:package-boundaries": "node scripts/check-package-boundaries.js",
- "check:repo": "pnpm check:sync && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract",
+ "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract",
+ "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs",
"check:fixtures": "node internal/vibe-tests/scripts/fixture-suite.mjs verify",
"generate:cldr-weekdays": "node scripts/generate-cldr-weekdays.mjs",
"check:cldr-weekdays": "node scripts/generate-cldr-weekdays.mjs --check",
diff --git a/scripts/check-visual-workflow-owner.mjs b/scripts/check-visual-workflow-owner.mjs
new file mode 100644
index 0000000000..7e3333fc89
--- /dev/null
+++ b/scripts/check-visual-workflow-owner.mjs
@@ -0,0 +1,144 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @input GitHub workflow YAML and root package scripts.
+ * @output Independent visual-regression owners that violate AST-030 FR12.
+ * @position Repository guard; runs through check:repo, not a new CI lane.
+ *
+ * This is an ownership tripwire, not a shell interpreter. Inspect executable
+ * commands/actions and visual artifact producers, not comments or every use of
+ * Playwright, Storybook, screenshots, or upload/download-artifact. Those also
+ * serve deployments, a11y, RTL, and vibe evidence.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import YAML from 'yaml';
+
+const ROOT = path.resolve(import.meta.dirname, '..');
+const CANONICAL = 'ci.yml';
+const VISUAL_COMMAND = [
+ /(?:^|[\s/'"`])gate\.mjs[\s\\]+(?:["']?(?:check|capture|release|flaky)\b|["']?\$)/i,
+ /\bvisual:(?:check|flaky)\b/i,
+ /\b(?:chromatic|chromaui|percy|backstop|reg-suit|loki)\b/i,
+ /\b(?:toHaveScreenshot|toMatchImageSnapshot|pixelmatch|compareCaptures)\s*\(/,
+ /\bvisual[-_ ]regression\b/i,
+];
+const VISUAL_ARTIFACT =
+ /^(?:visual-(?:pr-report|capture|report|baseline)|.*visual[-_]regression)(?:$|[-_*])/i;
+
+function strings(value) {
+ if (typeof value === 'string') return [value];
+ if (Array.isArray(value)) return value.flatMap(strings);
+ if (value && typeof value === 'object')
+ return Object.values(value).flatMap(strings);
+ return [];
+}
+
+function visualCommand(command, scripts, seen = new Set()) {
+ // YAML comments are already gone; ignore shell comment-only lines too.
+ const executable = command.replace(/^\s*#.*$/gm, '');
+ if (VISUAL_COMMAND.some(pattern => pattern.test(executable))) return true;
+ // A package-script alias must not hide an otherwise identical second owner.
+ for (const match of executable.matchAll(
+ /\b(?:pnpm|npm|yarn)\s+(?:run\s+)?([\w:.-]+)/g,
+ )) {
+ const name = match[1];
+ if (!scripts[name] || seen.has(name)) continue;
+ seen.add(name);
+ if (visualCommand(scripts[name], scripts, seen)) return true;
+ }
+ return false;
+}
+
+/** Return one actionable error per independently executing workflow job. */
+export function findVisualOwners(workflows, scripts = {}) {
+ const errors = [];
+ for (const [file, source] of Object.entries(workflows)) {
+ let workflow;
+ try {
+ workflow = YAML.parse(source);
+ } catch (error) {
+ errors.push(`${file}: invalid workflow YAML: ${error.message}`);
+ continue;
+ }
+ if (!workflow?.jobs || typeof workflow.jobs !== 'object') {
+ errors.push(`${file}: workflow has no jobs`);
+ continue;
+ }
+ for (const [jobId, job] of Object.entries(workflow.jobs)) {
+ const steps = job.steps ?? [];
+ const commands = [
+ job.uses,
+ ...steps.flatMap(step => [step.run, step.uses, step.with?.script]),
+ ].filter(value => typeof value === 'string');
+ const artifactProducer = steps.some(
+ step =>
+ /^actions\/upload-artifact@/.test(step.uses ?? '') &&
+ strings([step.with?.name, step.with?.path]).some(value =>
+ value.split(/[\s/]+/).some(part => VISUAL_ARTIFACT.test(part)),
+ ),
+ );
+ const namedVisualOwner = strings([
+ workflow.name,
+ jobId,
+ job.name,
+ ...steps.map(step => step.name),
+ ]).some(name => /\bvisual[-_ ]regression\b/i.test(name));
+ if (
+ commands.some(command => visualCommand(command, scripts)) ||
+ artifactProducer ||
+ namedVisualOwner
+ ) {
+ if (file !== CANONICAL || jobId !== 'pr-visual') {
+ errors.push(
+ `${file} → ${jobId}: visual regression belongs in ci.yml → pr-visual (AST-030 FR12)`,
+ );
+ }
+ }
+ }
+ }
+ let canonical;
+ try {
+ canonical = workflows[CANONICAL] && YAML.parse(workflows[CANONICAL]);
+ } catch {
+ // The parse error is already reported above; absence must not pass either.
+ }
+ if (
+ !canonical?.jobs?.['pr-visual']?.steps?.some(step =>
+ /gate\.mjs\s+(?:["']?check\b|["']?\$COMMAND\b)/.test(step.run ?? ''),
+ )
+ ) {
+ errors.push(
+ 'ci.yml → pr-visual: canonical Storybook visual check is missing',
+ );
+ }
+ return errors;
+}
+
+export function checkVisualWorkflowOwner(root = ROOT) {
+ const directory = path.join(root, '.github/workflows');
+ const workflows = Object.fromEntries(
+ fs
+ .readdirSync(directory)
+ .filter(file => /\.ya?ml$/.test(file))
+ .map(file => [file, fs.readFileSync(path.join(directory, file), 'utf8')]),
+ );
+ const {scripts = {}} = JSON.parse(
+ fs.readFileSync(path.join(root, 'package.json'), 'utf8'),
+ );
+ return findVisualOwners(workflows, scripts);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const errors = checkVisualWorkflowOwner();
+ if (errors.length) {
+ console.error(errors.join('\n'));
+ process.exitCode = 1;
+ } else {
+ console.log(
+ 'Visual regression has one workflow owner: ci.yml → pr-visual.',
+ );
+ }
+}
diff --git a/scripts/check-visual-workflow-owner.test.mjs b/scripts/check-visual-workflow-owner.test.mjs
new file mode 100644
index 0000000000..5782bef529
--- /dev/null
+++ b/scripts/check-visual-workflow-owner.test.mjs
@@ -0,0 +1,216 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @input Synthetic independent owners and the repository's real workflows.
+ * @output Red-arm coverage for AST-030 FR12 without banning non-visual evidence.
+ * @position Node tests for the existing repository guardrail lane.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {describe, expect, it} from 'vitest';
+import YAML from 'yaml';
+
+import {
+ checkVisualWorkflowOwner,
+ findVisualOwners,
+} from './check-visual-workflow-owner.mjs';
+
+const root = path.resolve(import.meta.dirname, '..');
+const canonical = {
+ name: 'CI',
+ on: {pull_request: {}},
+ jobs: {
+ 'pr-visual': {
+ steps: [{run: 'node .github/scripts/visual-gate/gate.mjs check'}],
+ },
+ },
+};
+function owners(job, options = {}) {
+ return findVisualOwners(
+ {
+ 'ci.yml': YAML.stringify(canonical),
+ [options.file ?? 'another.yml']: YAML.stringify({
+ name: 'Another check',
+ on: options.on ?? {pull_request: {}},
+ jobs: {check: job},
+ }),
+ },
+ options.scripts,
+ );
+}
+
+describe('single visual workflow owner', () => {
+ it('accepts the canonical Storybook owner', () => {
+ expect(findVisualOwners({'ci.yml': YAML.stringify(canonical)})).toEqual([]);
+ });
+
+ it.each(['check', 'capture', 'release', 'flaky'])(
+ 'rejects another gate.mjs %s owner even under a neutral name',
+ command => {
+ expect(
+ owners({
+ steps: [
+ {run: `node .github/scripts/visual-gate/gate.mjs ${command}`},
+ ],
+ }),
+ ).toEqual([expect.stringContaining('another.yml → check')]);
+ },
+ );
+
+ it.each([
+ {schedule: [{cron: '0 0 * * *'}]},
+ {workflow_dispatch: {}},
+ {workflow_run: {workflows: ['CI'], types: ['completed']}},
+ {pull_request_target: {types: ['closed']}},
+ {workflow_call: {}},
+ ])('rejects duplicate capture regardless of trigger: %j', on => {
+ expect(
+ owners(
+ {steps: [{run: 'node gate.mjs capture'}]},
+ {on, file: 'renamed.yaml'},
+ ),
+ ).toHaveLength(1);
+ });
+
+ it.each([
+ {run: 'pnpm visual:check'},
+ {run: 'npx percy storybook ./dist'},
+ {uses: 'chromaui/action@v1'},
+ {uses: 'percy/exec-action@v1'},
+ {run: 'node -e "expect(page).toHaveScreenshot()"'},
+ {run: 'node -e "pixelmatch(before, after)"'},
+ {
+ uses: 'actions/github-script@v9',
+ with: {script: 'await expect(page).toHaveScreenshot();'},
+ },
+ ])('rejects independent visual runners: %j', step => {
+ // chromaui is the action name for Chromatic, not an artifact publisher.
+ expect(owners({steps: [step]})).toHaveLength(1);
+ });
+
+ it('follows package-script aliases and terminates cycles', () => {
+ expect(
+ owners(
+ {steps: [{run: 'pnpm run screenshots'}]},
+ {
+ scripts: {
+ screenshots: 'pnpm run compare',
+ compare: 'pnpm visual:check',
+ },
+ },
+ ),
+ ).toHaveLength(1);
+ expect(
+ owners(
+ {steps: [{run: 'pnpm run first'}]},
+ {
+ scripts: {first: 'pnpm second', second: 'pnpm first'},
+ },
+ ),
+ ).toEqual([]);
+ });
+
+ it.each([
+ 'visual-capture',
+ 'visual-pr-report',
+ 'visual-report',
+ 'component-visual-regression',
+ ])('rejects a second producer of %s', name => {
+ expect(
+ owners({
+ steps: [
+ {uses: 'actions/upload-artifact@v7', with: {name, path: 'result'}},
+ ],
+ }),
+ ).toHaveLength(1);
+ });
+
+ it('rejects a separately named visual owner and a second canonical-workflow job', () => {
+ expect(
+ owners({name: 'Visual regression', steps: [{run: 'node custom.mjs'}]}),
+ ).toHaveLength(1);
+ const duplicate = structuredClone(canonical);
+ duplicate.jobs.another = {steps: [{run: 'pnpm visual:check'}]};
+ expect(findVisualOwners({'ci.yml': YAML.stringify(duplicate)})).toEqual([
+ expect.stringContaining('ci.yml → another'),
+ ]);
+ });
+
+ it.each([
+ 'deploy-storybook',
+ 'storybook-abc123',
+ 'a11y-report',
+ 'rtl-audit-report-core',
+ 'vibe-test-screenshots',
+ 'screenshot-manifest',
+ ])('permits non-visual-regression artifact %s', name => {
+ expect(
+ owners({
+ steps: [
+ {
+ uses: 'actions/upload-artifact@v7',
+ with: {name, path: 'screenshots'},
+ },
+ ],
+ }),
+ ).toEqual([]);
+ });
+
+ it('permits report-only publication and browser audits', () => {
+ expect(
+ owners({
+ steps: [
+ {
+ uses: 'actions/download-artifact@v8',
+ with: {name: 'visual-pr-report'},
+ },
+ {run: 'node .github/scripts/visual-gate/publish-pr-report.mjs'},
+ {
+ run: 'node .github/scripts/gh-pages-publisher.mjs immutable-path --scope pr-visual/evidence',
+ },
+ {run: 'node .github/scripts/visual-gate/gate.mjs reach'},
+ {run: 'pnpm test:a11y-contract'},
+ {run: 'pnpm rtl:audit'},
+ {run: 'npx playwright install chromium'},
+ {run: 'npx tsx src/screenshot-previews.ts'},
+ ],
+ }),
+ ).toEqual([]);
+ });
+
+ it('ignores YAML and shell comments', () => {
+ const source = YAML.stringify({
+ jobs: {check: {steps: [{run: '# node gate.mjs check\necho audit'}]}},
+ });
+ expect(
+ findVisualOwners({
+ 'ci.yml': YAML.stringify(canonical),
+ 'audit.yml': `# visual regression uses ci.yml\n${source}`,
+ }),
+ ).toEqual([]);
+ });
+
+ it('fails closed on invalid YAML or a missing canonical check', () => {
+ expect(findVisualOwners({'broken.yml': 'jobs: [broken'})).toEqual([
+ expect.stringContaining('invalid workflow YAML'),
+ expect.stringContaining('canonical Storybook visual check is missing'),
+ ]);
+ expect(findVisualOwners({'ci.yml': YAML.stringify({jobs: {}})})).toEqual([
+ expect.stringContaining('canonical Storybook visual check is missing'),
+ ]);
+ });
+
+ it('checks the real repository through the existing check:repo lane', () => {
+ expect(checkVisualWorkflowOwner(root)).toEqual([]);
+ const {scripts} = JSON.parse(
+ fs.readFileSync(path.join(root, 'package.json'), 'utf8'),
+ );
+ expect(scripts['check:repo'].split(' && ')).toContain(
+ 'pnpm check:visual-owner',
+ );
+ expect(scripts['check:visual-owner']).toBe(
+ 'node scripts/check-visual-workflow-owner.mjs',
+ );
+ });
+});