Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci(visual): require explicit reviewed baseline promotion (#5578)

Cindy Zhang08a3758d7a2a
diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.js
index 1b719fe7f5..c2c3cd18db 100644
--- a/.github/scripts/lib/visual-format.js
+++ b/.github/scripts/lib/visual-format.js
@@ -29,23 +29,43 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
const link = safeReportUrl
? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`
: '';
+ const acceptanceCommand =
+ verdict.context?.runId && verdict.context?.runAttempt
+ ? `\n\nTo accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <reason>\``
+ : '';
+ const reportBase = safeReportUrl
+ ? `${safeReportUrl.replace(/\/+$/, '')}/`
+ : null;
+ // Inline images use raw branch URLs so a delayed Pages deployment cannot
+ // break evidence that is already present on gh-pages.
+ const imageBase = safeUrl(imageUrl)
+ ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/`
+ : reportBase;
if (verdict.status === 'skipped') {
- return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}\n\n`;
+ return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}${link}\n\n`;
}
if (verdict.status === 'failed') {
return `### Visual Regression\n\n**Status:** ${num(verdict.counts?.failed)} shot(s) could not be captured.${link}\n\n`;
}
+ const added = verdict.added ?? [];
+ const removed = verdict.removed ?? [];
if (!verdict.changes || verdict.changes.length === 0) {
- const compared = num(verdict.counts?.total) - num(verdict.counts?.added);
- // A PR-scoped run shoots every story of the touched component in every
- // theme that styles it, which is deeper than the daily gate's baseline
- // reaches — so some shots legitimately have nothing to compare against.
- // Saying "added" there reads as a problem; saying it plainly does not.
- const unbaselined = num(verdict.counts?.added)
- ? ` ${num(verdict.counts?.added)} shot(s) have no baseline yet and were not compared.`
- : '';
- return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).${unbaselined}\n\n`;
+ if (added.length > 0 || removed.length > 0) {
+ const frames = imageBase
+ ? [...added.map(key => ({key, kind: 'after'})), ...removed.map(key => ({key, kind: 'before'}))]
+ .slice(0, 3)
+ .map(({key, kind}) => {
+ const safeKey = encodeURIComponent(String(key));
+ const label = kind === 'after' ? 'Added — After' : 'Removed — Before';
+ return `<p><b>${label}</b><br><img src="${imageBase}${kind}/${safeKey}.png" width="300" alt="${label} visual regression frame"></p>`;
+ })
+ .join('\n')
+ : '';
+ return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}${acceptanceCommand}\n\n${frames}\n\n`;
+ }
+ const compared = num(verdict.counts?.total);
+ return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).\n\n`;
}
const rows = verdict.changes
@@ -61,16 +81,6 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
// names before they can answer whether the after is correct. The static
// report is published beside the PR preview, and its first three deltas
// render directly in the comment; the full report carries the rest + wipe UI.
- const reportBase = safeReportUrl
- ? `${safeReportUrl.replace(/\/+$/, '')}/`
- : null;
- // GitHub comments fetch raw branch images directly. The repository's Pages
- // deployment can be delayed or errored independently of the gh-pages push;
- // coupling inline evidence to that deployment produced six broken images in
- // the first live demo even though every PNG existed in the branch.
- const imageBase = safeUrl(imageUrl)
- ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/`
- : reportBase;
const evidence = imageBase
? verdict.changes
.slice(0, 3)
@@ -101,7 +111,7 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
return `### Visual Regression
-**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}
+**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}${acceptanceCommand}
A change here is a question, not a failure: check whether the *after* is the
picture you intended. If it is, say so in the PR — the release gate's baseline
diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjs
index bab8243237..565efc9455 100644
--- a/.github/scripts/lib/visual-format.test.mjs
+++ b/.github/scripts/lib/visual-format.test.mjs
@@ -21,20 +21,46 @@ describe('buildVisualSection', () => {
expect(buildVisualSection(verdict())).toContain('No visual change across 16 compared shot(s)');
});
- it('calls an unbaselined shot exactly that, not an addition', () => {
+ it('shows added-only evidence instead of claiming there was no visual change', () => {
const section = buildVisualSection(
- verdict({counts: {total: 20, unchanged: 14, changed: 0, added: 6, removed: 0, failed: 0}}),
+ verdict({
+ status: 'changed',
+ counts: {total: 20, unchanged: 14, changed: 0, added: 1, removed: 0, failed: 0},
+ added: ['new-shot'],
+ removed: [],
+ }),
+ 'https://example.com/report/',
+ 'https://example.com/images/',
);
- expect(section).toContain('No visual change across 14 compared shot(s)');
- expect(section).toContain('6 shot(s) have no baseline yet');
+ expect(section).toContain('1 added · 0 removed');
+ expect(section).toContain('View the report');
+ expect(section).toContain('https://example.com/images/after/new-shot.png');
+ expect(section).not.toContain('No visual change');
+ });
+
+ it('shows the before image for a removed-only bundle', () => {
+ const section = buildVisualSection(
+ verdict({
+ status: 'changed',
+ counts: {total: 0, unchanged: 0, changed: 0, added: 0, removed: 1, failed: 0},
+ added: [],
+ removed: ['old-shot'],
+ }),
+ 'https://example.com/report/',
+ 'https://example.com/images/',
+ );
+ expect(section).toContain('0 added · 1 removed');
+ expect(section).toContain('https://example.com/images/before/old-shot.png');
});
it('states the reason for a skip, so a broad PR does not look like a pass', () => {
const section = buildVisualSection(
verdict({status: 'skipped', reason: '900 shots exceeds the 240-shot budget'}),
+ 'https://example.com/report/',
);
expect(section).toContain('Skipped');
expect(section).toContain('900 shots exceeds the 240-shot budget');
+ expect(section).toContain('View the report');
});
it('renders verdict strings as one-line literal text', () => {
@@ -110,6 +136,7 @@ describe('buildVisualSection', () => {
verdict({
status: 'changed',
counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},
+ context: {runId: '123456', runAttempt: '2'},
changes: [
{
key: 'core-button--primary__y2k-light',
@@ -130,6 +157,7 @@ describe('buildVisualSection', () => {
expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png');
expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png');
expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');
+ expect(section).toContain('/accept-visual 123456/2 <reason>');
});
it('embeds at most three deltas so the PR comment stays readable', () => {
diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.mjs
new file mode 100644
index 0000000000..8692e10bc1
--- /dev/null
+++ b/.github/scripts/visual-gate/activate-visual-acceptance.mjs
@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Activate the required visual-acceptance status without stranding PRs.
+ *
+ * Dry-run by default. `--apply` derives each open head's scope, backfills
+ * no-scope heads as success and stable-visual heads as pending, then requires
+ * the context. Re-running only replaces missing or legacy grandfathered states;
+ * it never overwrites a verdict produced by the live workflow.
+ */
+
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import {classifyVisualScope} from '../visual-scope.mjs';
+
+const REPO = 'facebook/astryx';
+const BRANCH = 'main';
+const CONTEXT = 'visual-acceptance';
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
+
+export function buildActivationPlan(openPulls, requiredContexts) {
+ const unique = [...new Map(openPulls.map(pr => [pr.head.sha, pr])).values()];
+ const backfill = unique
+ .filter(pr => !pr.visualStatus || pr.visualStatus.description.startsWith('Grandfathered during'))
+ .map(pr => ({
+ number: pr.number,
+ sha: pr.head.sha,
+ state: pr.hasStableVisual ? 'pending' : 'success',
+ description: pr.hasStableVisual
+ ? 'Stable visual result pending; rerun CI after activation.'
+ : 'No stable visual scope.',
+ }));
+ return {backfill, addRequiredContext: !requiredContexts.includes(CONTEXT)};
+}
+
+function gh(args, input) {
+ return execFileSync('gh', ['api', ...args], {
+ encoding: 'utf8',
+ input: input === undefined ? undefined : JSON.stringify(input),
+ maxBuffer: 16 * 1024 * 1024,
+ });
+}
+
+function paged(endpoint) {
+ return JSON.parse(gh([endpoint, '--paginate', '--slurp'])).flat();
+}
+
+function headManifests(pr, files) {
+ const paths = new Set();
+ for (const file of files) {
+ const theme = file.match(/^packages\/themes\/([^/]+)\//);
+ const pkg = file.match(/^packages\/([^/]+)\//);
+ if (theme) paths.add(`packages/themes/${theme[1]}/package.json`);
+ else if (pkg) paths.add(`packages/${pkg[1]}/package.json`);
+ }
+ const manifests = {};
+ for (const manifestPath of paths) {
+ try {
+ const encoded = manifestPath.split('/').map(encodeURIComponent).join('/');
+ const response = JSON.parse(
+ gh([`repos/${pr.head.repo.full_name}/contents/${encoded}?ref=${pr.head.sha}`]),
+ );
+ if (!Array.isArray(response) && response.content) {
+ manifests[manifestPath] = JSON.parse(Buffer.from(response.content, 'base64').toString());
+ }
+ } catch (error) {
+ if (!String(error.stderr ?? error.message).includes('404')) throw error;
+ }
+ }
+ return manifests;
+}
+
+function readState() {
+ const pullPages = JSON.parse(
+ gh([`repos/${REPO}/pulls?state=open&per_page=100`, '--paginate', '--slurp']),
+ );
+ const pulls = [];
+ for (const pr of pullPages.flat()) {
+ const combined = JSON.parse(gh([`repos/${REPO}/commits/${pr.head.sha}/status`]));
+ const visualStatus = combined.statuses.find(status => status.context === CONTEXT) ?? null;
+ if (visualStatus && !visualStatus.description.startsWith('Grandfathered during')) {
+ pulls.push({number: pr.number, head: pr.head, visualStatus, hasStableVisual: false});
+ continue;
+ }
+ const files = paged(`repos/${REPO}/pulls/${pr.number}/files?per_page=100`).map(
+ file => file.filename,
+ );
+ const scope = classifyVisualScope(files, ROOT, headManifests(pr, files));
+ pulls.push({...pr, visualStatus, hasStableVisual: scope.hasStableVisual});
+ }
+ const protection = JSON.parse(
+ gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),
+ );
+ return {pulls, contexts: protection.contexts ?? []};
+}
+
+function apply(plan) {
+ for (const {sha, number, state, description} of plan.backfill) {
+ gh(
+ ['--method', 'POST', `repos/${REPO}/statuses/${sha}`, '--input', '-'],
+ {state, context: CONTEXT, description: `${description} (PR #${number})`},
+ );
+ }
+ if (plan.addRequiredContext) {
+ gh([
+ '--method',
+ 'POST',
+ `repos/${REPO}/branches/${BRANCH}/protection/required_status_checks/contexts`,
+ '-f',
+ `contexts[]=${CONTEXT}`,
+ ]);
+ }
+ const protection = JSON.parse(
+ gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),
+ );
+ if (!(protection.contexts ?? []).includes(CONTEXT)) {
+ throw new Error(`${CONTEXT} is still not required on ${BRANCH}`);
+ }
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const state = readState();
+ const plan = buildActivationPlan(state.pulls, state.contexts);
+ process.stdout.write(
+ `${JSON.stringify(
+ {
+ repo: REPO,
+ branch: BRANCH,
+ context: CONTEXT,
+ headsToReconcile: plan.backfill.length,
+ pending: plan.backfill.filter(item => item.state === 'pending').length,
+ success: plan.backfill.filter(item => item.state === 'success').length,
+ addRequiredContext: plan.addRequiredContext,
+ },
+ null,
+ 2,
+ )}\n`,
+ );
+ if (process.argv.includes('--apply')) {
+ apply(plan);
+ process.stdout.write(`Activated required status ${CONTEXT} on ${BRANCH}.\n`);
+ } else {
+ process.stdout.write('Dry run only; pass --apply to reconcile and activate.\n');
+ }
+}
diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs
new file mode 100644
index 0000000000..63d9d07334
--- /dev/null
+++ b/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs
@@ -0,0 +1,64 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+
+import {buildActivationPlan} from './activate-visual-acceptance.mjs';
+
+describe('visual-acceptance activation', () => {
+ it('derives pending for stable heads and success only for no-scope heads', () => {
+ const plan = buildActivationPlan(
+ [
+ {number: 1, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},
+ {number: 2, head: {sha: 'b'.repeat(40)}, hasStableVisual: false, visualStatus: null},
+ {number: 3, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},
+ ],
+ ['build'],
+ );
+ expect(plan).toEqual({
+ backfill: [
+ {
+ number: 3,
+ sha: 'a'.repeat(40),
+ state: 'pending',
+ description: 'Stable visual result pending; rerun CI after activation.',
+ },
+ {
+ number: 2,
+ sha: 'b'.repeat(40),
+ state: 'success',
+ description: 'No stable visual scope.',
+ },
+ ],
+ addRequiredContext: true,
+ });
+ });
+
+ it('repairs legacy grandfathered states but preserves live workflow verdicts', () => {
+ const plan = buildActivationPlan(
+ [
+ {
+ number: 1,
+ head: {sha: 'a'.repeat(40)},
+ hasStableVisual: true,
+ visualStatus: {description: 'Grandfathered during visual-acceptance activation'},
+ },
+ {
+ number: 2,
+ head: {sha: 'b'.repeat(40)},
+ hasStableVisual: true,
+ visualStatus: {description: 'Current visual bundle was explicitly accepted.'},
+ },
+ ],
+ ['build', 'visual-acceptance'],
+ );
+ expect(plan.backfill).toEqual([
+ {
+ number: 1,
+ sha: 'a'.repeat(40),
+ state: 'pending',
+ description: 'Stable visual result pending; rerun CI after activation.',
+ },
+ ]);
+ expect(plan.addRequiredContext).toBe(false);
+ });
+});
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index a0822e2312..c28d637b27 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -75,6 +75,32 @@ const maxShots = flag('max-shots') ? Number(flag('max-shots')) : Infinity;
const storyPackages = (flag('story-packages') ?? config.stableStoryPackages.join(','))
.split(',')
.filter(Boolean);
+/** A trusted, exact shot list used only to recapture an accepted merged result. */
+const planFile = flag('plan-file');
+
+function readExactPlan(file) {
+ const shots = JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));
+ if (!Array.isArray(shots) || shots.length > 5000) {
+ throw new Error('Exact visual plan must contain at most 5000 trusted shots.');
+ }
+ const keys = new Set();
+ for (const shot of shots) {
+ if (
+ !/^[A-Za-z0-9._-]{1,240}$/.test(shot?.key ?? '') ||
+ typeof shot.storyId !== 'string' ||
+ !shot.storyId ||
+ typeof shot.theme !== 'string' ||
+ !shot.theme ||
+ !['light', 'dark'].includes(shot.mode) ||
+ !Array.isArray(shot.reasons)
+ ) {
+ throw new Error(`Exact visual plan contains an invalid shot: ${JSON.stringify(shot)}`);
+ }
+ if (keys.has(shot.key)) throw new Error(`Exact visual plan repeats ${shot.key}.`);
+ keys.add(shot.key);
+ }
+ return shots;
+}
/**
* Which stories the scout needs to look at: every story of a component some
@@ -96,6 +122,7 @@ function storiesToScout(stories, targets, themeOverrides) {
/** @returns {Promise<import('./lib/plan.mjs').Shot[]>} */
async function plan() {
+ if (planFile) return readExactPlan(planFile);
const [targets, themeOverrides] = await Promise.all([
loadThemingTargets(REPO_ROOT),
loadThemeOverrides(REPO_ROOT, config.probeTheme),
@@ -210,9 +237,23 @@ async function check() {
status: 'skipped',
generatedAt: new Date().toISOString(),
reason: `${shots.length} shots exceeds the ${maxShots}-shot budget${components.length ? ` (${components.length} components touched)` : ''} — too broad to review shot by shot here. The daily release gate covers this change against the full baseline.`,
+ context: {
+ sha: process.env.GITHUB_SHA ?? null,
+ headSha: process.env.ASTRYX_PR_HEAD_SHA ?? null,
+ baseSha: process.env.ASTRYX_PR_BASE_SHA ?? null,
+ ref: process.env.GITHUB_REF ?? null,
+ runId: process.env.GITHUB_RUN_ID ?? null,
+ runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
+ tiers,
+ scoped: components.length > 0,
+ components,
+ },
counts: {total: shots.length, unchanged: 0, changed: 0, added: 0, removed: 0, failed: 0},
components,
changes: [],
+ added: [],
+ removed: [],
+ failures: [],
};
fs.mkdirSync(outDir, {recursive: true});
fs.writeFileSync(path.join(outDir, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`);
diff --git a/.github/scripts/visual-gate/lib/capture.mjs b/.github/scripts/visual-gate/lib/capture.mjs
index e0dc296d0f..97b2e87a5c 100644
--- a/.github/scripts/visual-gate/lib/capture.mjs
+++ b/.github/scripts/visual-gate/lib/capture.mjs
@@ -81,6 +81,19 @@ const SEEDED_RANDOM = `(() => {
};
})();`;
+// BrowserContext routing does not cover network opened from dedicated/shared
+// workers. A static Storybook needs none of these constructors, so remove the
+// channels before any PR-authored script runs. Service workers are separately
+// disabled in the context options below.
+export const BACKGROUND_NETWORK_GUARD = `(() => {
+ const Blocked = class {
+ constructor() { throw new Error('Network channel disabled during visual capture'); }
+ };
+ for (const name of ['WebSocket', 'Worker', 'SharedWorker']) {
+ Object.defineProperty(globalThis, name, {value: Blocked, configurable: false});
+ }
+})();`;
+
/** The instant every capture happens at, and the zone it happens in.
*
* Stories that build their data from `new Date()` photograph a different day
@@ -105,25 +118,66 @@ const TIMEZONE_ID = 'UTC';
* @param {string} dir
* @returns {Promise<{port: number, close: () => Promise<void>}>}
*/
+export function isSameOrigin(url, origin) {
+ try {
+ return new URL(url).origin === origin;
+ } catch {
+ return false;
+ }
+}
+
+/** Install the network boundary before any PR-authored Storybook code runs. */
+export async function blockExternalNetwork(context, origin) {
+ await context.route('**', route =>
+ isSameOrigin(route.request().url(), origin) ? route.continue() : route.abort(),
+ );
+ // A routed WebSocket never connects unless connectToServer() is called. Close
+ // every socket: a static Storybook needs none, including its dev-only HMR.
+ await context.routeWebSocket(/.*/, socket => socket.close({code: 1008, reason: 'blocked'}));
+}
+
+export const CAPTURE_CONTEXT_SECURITY = {serviceWorkers: 'block'};
+
export function serveDirectory(dir) {
+ const root = fs.realpathSync(dir);
const server = http.createServer((req, res) => {
- const requested = path.join(dir, req.url === '/' ? 'index.html' : req.url.split('?')[0]);
- if (!path.resolve(requested).startsWith(path.resolve(dir))) {
+ let requested;
+ try {
+ const pathname = decodeURIComponent(new URL(req.url, 'http://localhost').pathname);
+ requested = path.resolve(root, `.${pathname === '/' ? '/index.html' : pathname}`);
+ } catch {
+ res.writeHead(400);
+ res.end('Bad request');
+ return;
+ }
+ if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {
res.writeHead(403);
res.end('Forbidden');
return;
}
- fs.readFile(requested, (error, data) => {
- if (error) {
+ fs.realpath(requested, (realpathError, realPath) => {
+ if (realpathError) {
res.writeHead(404);
res.end('Not found');
return;
}
- res.writeHead(200, {
- 'Content-Type': CONTENT_TYPES[path.extname(requested)] ?? 'application/octet-stream',
- 'Cache-Control': 'no-store',
+ if (realPath !== root && !realPath.startsWith(`${root}${path.sep}`)) {
+ res.writeHead(403);
+ res.end('Forbidden');
+ return;
+ }
+ fs.readFile(realPath, (error, data) => {
+ if (error) {
+ res.writeHead(404);
+ res.end('Not found');
+ return;
+ }
+ res.writeHead(200, {
+ 'Content-Type': CONTENT_TYPES[path.extname(realPath)] ?? 'application/octet-stream',
+ 'Cache-Control': 'no-store',
+ });
+ res.end(data);
});
- res.end(data);
});
});
return new Promise(resolve => {
@@ -278,12 +332,12 @@ export async function scout({storyIds, storybookDir, theme, viewport, onProgress
viewport,
deviceScaleFactor: 1,
timezoneId: TIMEZONE_ID,
+ ...CAPTURE_CONTEXT_SECURITY,
});
await context.clock.setFixedTime(FROZEN_NOW);
+ await context.addInitScript(BACKGROUND_NETWORK_GUARD);
await context.addInitScript(SEEDED_RANDOM);
- await context.route('**', route =>
- route.request().url().startsWith(origin) ? route.continue() : route.abort(),
- );
+ await blockExternalNetwork(context, origin);
const page = await context.newPage();
/** @type {Record<string, Record<string, string[]>>} */
@@ -344,14 +398,14 @@ export async function capture({
reducedMotion: 'reduce',
colorScheme: 'light',
timezoneId: TIMEZONE_ID,
+ ...CAPTURE_CONTEXT_SECURITY,
});
await context.clock.setFixedTime(FROZEN_NOW);
+ await context.addInitScript(BACKGROUND_NETWORK_GUARD);
await context.addInitScript(SEEDED_RANDOM);
// Anything off-origin is a determinism hazard, and nothing in a component
// story legitimately needs it.
- await context.route('**', route =>
- route.request().url().startsWith(origin) ? route.continue() : route.abort(),
- );
+ await blockExternalNetwork(context, origin);
const page = await context.newPage();
await page.addStyleTag({content: FREEZE_CSS}).catch(() => {});
diff --git a/.github/scripts/visual-gate/lib/capture.test.mjs b/.github/scripts/visual-gate/lib/capture.test.mjs
new file mode 100644
index 0000000000..d52a9be953
--- /dev/null
+++ b/.github/scripts/visual-gate/lib/capture.test.mjs
@@ -0,0 +1,67 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {afterEach, describe, expect, it} from 'vitest';
+
+import {
+ BACKGROUND_NETWORK_GUARD,
+ CAPTURE_CONTEXT_SECURITY,
+ blockExternalNetwork,
+ isSameOrigin,
+ serveDirectory,
+} from './capture.mjs';
+
+const roots = [];
+afterEach(() => {
+ for (const root of roots.splice(0)) fs.rmSync(root, {recursive: true, force: true});
+});
+
+describe('capture network boundary', () => {
+ it('allows the exact Storybook origin', () => {
+ expect(isSameOrigin('http://127.0.0.1:6100/assets/story.js', 'http://127.0.0.1:6100')).toBe(
+ true,
+ );
+ });
+
+ it('rejects URLs that merely share the trusted prefix', () => {
+ expect(
+ isSameOrigin('http://127.0.0.1:[email protected]/collect', 'http://127.0.0.1:6100'),
+ ).toBe(false);
+ });
+
+ it('blocks service workers and every WebSocket before Storybook runs', async () => {
+ let webSocketHandler;
+ const context = {
+ route: async () => {},
+ routeWebSocket: async (_pattern, handler) => {
+ webSocketHandler = handler;
+ },
+ };
+ await blockExternalNetwork(context, 'http://127.0.0.1:6100');
+ const closed = [];
+ await webSocketHandler({close: options => closed.push(options)});
+ expect(CAPTURE_CONTEXT_SECURITY).toEqual({serviceWorkers: 'block'});
+ expect(BACKGROUND_NETWORK_GUARD).toContain("'WebSocket', 'Worker', 'SharedWorker'");
+ expect(closed).toEqual([{code: 1008, reason: 'blocked'}]);
+ });
+
+ it('does not follow a Storybook artifact symlink outside its root', async () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'capture-server-'));
+ roots.push(root);
+ const served = path.join(root, 'served');
+ fs.mkdirSync(served);
+ fs.writeFileSync(path.join(root, 'secret'), 'credential');
+ fs.symlinkSync(path.join(root, 'secret'), path.join(served, 'escape'));
+ const server = await serveDirectory(served);
+ try {
+ const response = await fetch(`http://127.0.0.1:${server.port}/escape`);
+ expect(response.status).toBe(403);
+ expect(await response.text()).not.toContain('credential');
+ } finally {
+ await server.close();
+ }
+ });
+});
diff --git a/.github/scripts/visual-gate/lib/report.mjs b/.github/scripts/visual-gate/lib/report.mjs
index 865cefd25d..172922bcb0 100644
--- a/.github/scripts/visual-gate/lib/report.mjs
+++ b/.github/scripts/visual-gate/lib/report.mjs
@@ -80,6 +80,21 @@ function keyList(keys, title, note) {
</section>`;
}
+/** @param {string[]} keys @param {'added' | 'removed'} kind */
+function oneSidedEvidence(keys, kind) {
+ if (keys.length === 0) return '';
+ const imageKind = kind === 'added' ? 'after' : 'before';
+ const title = kind === 'added' ? 'Added' : 'Removed';
+ return `
+<h2>${title} (${keys.length})</h2>
+${keys
+ .map(key => {
+ const safe = escapeHtml(key);
+ return `<section class="change"><h3><code>${safe}</code></h3><p class="muted">${imageKind}</p><div class="stage"><img src="${imageKind}/${safe}.png" alt="${imageKind}"></div></section>`;
+ })
+ .join('')}`;
+}
+
/** @param {object} targeting */
function targetingSection(targeting) {
const rows = [
@@ -120,7 +135,7 @@ function targetingSection(targeting) {
/**
* @param {object} verdict
- * @param {{acceptHint?: string}} [options]
+ * @param {{acceptHint?: string, oneSidedEvidence?: boolean}} [options]
* @returns {string}
*/
export function renderReport(verdict, options = {}) {
@@ -200,8 +215,8 @@ ${verdict.failures?.length ? `<section class="list"><h2>Failed to capture <span
.join('')}</ul></section>` : ''}
${verdict.changes.length ? `<h2>Changed (${verdict.changes.length})</h2>` : '<p class="ok">No shot changed against the baseline.</p>'}
${verdict.changes.map(changeSection).join('')}
-${keyList(verdict.added ?? [], 'Added', 'New shots with no baseline to regress against — adopted on the next promotion.')}
-${keyList(verdict.removed ?? [], 'Removed', 'Baseline shots whose story no longer exists.')}
+${options.oneSidedEvidence ? oneSidedEvidence(verdict.added ?? [], 'added') : keyList(verdict.added ?? [], 'Added', 'New shots with no baseline to regress against — adopted on the next promotion.')}
+${options.oneSidedEvidence ? oneSidedEvidence(verdict.removed ?? [], 'removed') : keyList(verdict.removed ?? [], 'Removed', 'Baseline shots whose story no longer exists.')}
${targetingSection(verdict.targeting ?? {})}
<script>
for (const views of document.querySelectorAll('.views')) {
diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjs
index eddeff4230..15c10ef824 100644
--- a/.github/scripts/visual-gate/publish-pr-report.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.mjs
@@ -2,22 +2,22 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
/**
- * @file Turn an untrusted pr-visual artifact into publishable evidence.
+ * @file Derive publishable PR visual evidence from a trusted recapture.
*
- * @input verdict.json + before/after/diff PNGs emitted by PR-controlled CI
- * @output schema-validated, decoded/re-encoded PNGs and a report whose HTML is
- * generated by DEFAULT-BRANCH code
- *
- * This script runs only in the trusted `workflow_run` publisher. It never
- * imports or executes a file from the artifact. A fork may choose every byte
- * in its artifact, so paths, counts, identity fields and image bytes are all
- * validated before anything is copied onto the shared GitHub Pages origin.
+ * The Storybook bundle is PR-produced, but the plan, browser capture, scope,
+ * comparison, verdict, PNG validation, and report all run from default-branch
+ * code. PR-authored verdicts and job conclusions are never authority for the
+ * `visual-acceptance` status.
*/
+import {createHash} from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {PNG} from 'pngjs';
+import {incomparable} from './lib/baseline.mjs';
+import {buildVerdict, compareCaptures} from './lib/compare.mjs';
+import {shotKey} from './lib/plan.mjs';
import {renderReport} from './lib/report.mjs';
const args = process.argv.slice(2);
@@ -28,109 +28,202 @@ const flag = name => {
const input = path.resolve(flag('input') ?? 'visual');
const output = path.resolve(flag('output') ?? 'trusted-visual');
+const baseline = path.resolve(flag('baseline') ?? '.visual-baseline');
+const scope = JSON.parse(fs.readFileSync(path.resolve(flag('scope')), 'utf8'));
const pr = Number(flag('pr'));
const headSha = flag('head-sha') ?? '';
const runId = flag('run-id') ?? '';
const runAttempt = flag('run-attempt') ?? '';
+const config = JSON.parse(
+ fs.readFileSync(new URL('./visual-gate.config.json', import.meta.url), 'utf8'),
+);
const KEY = /^[A-Za-z0-9._-]{1,240}$/;
+const NAME = /^[A-Za-z0-9._-]{1,120}$/;
const SHA = /^[0-9a-f]{40}$/;
-const MAX_CHANGES = 240;
const MAX_PNG_BYTES = 12 * 1024 * 1024;
const MAX_EDGE = 5000;
+const MAX_SHOTS = 5000;
function fail(message) {
throw new Error(`visual evidence rejected: ${message}`);
}
-if (!Number.isSafeInteger(pr) || pr <= 0) fail('invalid PR number');
-if (!SHA.test(headSha)) fail('invalid trusted head SHA');
-if (!/^\d+$/.test(runId) || !/^\d+$/.test(runAttempt)) fail('invalid run identity');
-
-const verdictPath = path.join(input, 'verdict.json');
-if (!fs.existsSync(verdictPath)) fail('verdict.json is missing');
-if (fs.lstatSync(verdictPath).isSymbolicLink()) fail('verdict.json is a symlink');
-const verdict = JSON.parse(fs.readFileSync(verdictPath, 'utf8'));
+function readJSON(file) {
+ try {
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+ } catch (error) {
+ fail(`cannot read ${file}: ${error.message}`);
+ }
+}
-if (verdict.version !== 1) fail(`unsupported verdict version ${verdict.version}`);
-if (!['pass', 'changed', 'failed', 'skipped'].includes(verdict.status)) {
- fail(`invalid verdict status ${verdict.status}`);
+function shaFile(file) {
+ return createHash('sha256').update(fs.readFileSync(file)).digest('hex');
}
-if (String(verdict.context?.runId ?? '') !== runId) {
- fail(`verdict run ${verdict.context?.runId ?? 'missing'} does not match ${runId}`);
+
+function copyPng(source, target, label) {
+ if (!fs.existsSync(source)) fail(`${label} is missing`);
+ const stat = fs.lstatSync(source);
+ if (!stat.isFile() || stat.isSymbolicLink()) fail(`${label} is not a regular file`);
+ if (stat.size <= 0 || stat.size > MAX_PNG_BYTES) fail(`${label} has invalid size`);
+ let image;
+ try {
+ image = PNG.sync.read(fs.readFileSync(source), {checkCRC: true});
+ } catch (error) {
+ fail(`${label} is not a valid PNG: ${error.message}`);
+ }
+ if (image.width <= 0 || image.height <= 0 || image.width > MAX_EDGE || image.height > MAX_EDGE) {
+ fail(`${label} has invalid dimensions ${image.width}x${image.height}`);
+ }
+ fs.mkdirSync(path.dirname(target), {recursive: true});
+ fs.writeFileSync(target, PNG.sync.write(image));
+ return image;
}
-if (verdict.context?.runAttempt && String(verdict.context.runAttempt) !== runAttempt) {
- fail(`verdict attempt ${verdict.context.runAttempt} does not match ${runAttempt}`);
+
+if (!Number.isSafeInteger(pr) || pr <= 0) fail('invalid PR number');
+if (!SHA.test(headSha)) fail('invalid trusted head SHA');
+if (!/^\d+$/.test(runId) || !/^\d+$/.test(runAttempt)) fail('invalid run identity');
+if (
+ scope?.hasStableVisual !== true ||
+ typeof scope.broadStableVisual !== 'boolean' ||
+ !Array.isArray(scope.stableComponents) ||
+ !Array.isArray(scope.stableThemes) ||
+ !scope.stableComponents.every(name => NAME.test(name)) ||
+ !scope.stableThemes.every(name => NAME.test(name))
+) {
+ fail('trusted stable visual scope is invalid');
}
-if (!SHA.test(verdict.context?.sha ?? '')) {
- fail('verdict tested SHA is missing or invalid');
+
+const manifestPath = path.join(input, 'manifest.json');
+if (!fs.existsSync(manifestPath) || fs.lstatSync(manifestPath).isSymbolicLink()) {
+ fail('trusted capture manifest is missing or symbolic');
}
-if (verdict.context?.headSha && verdict.context.headSha !== headSha) {
- fail(`verdict head ${verdict.context.headSha} does not match trusted head ${headSha}`);
+const manifest = readJSON(manifestPath);
+if (
+ manifest.version !== 1 ||
+ typeof manifest.platform !== 'string' ||
+ !manifest.platform ||
+ typeof manifest.browser !== 'string' ||
+ !manifest.browser ||
+ !Number.isSafeInteger(manifest.viewport?.width) ||
+ !Number.isSafeInteger(manifest.viewport?.height) ||
+ !manifest.shots ||
+ typeof manifest.shots !== 'object' ||
+ Array.isArray(manifest.shots)
+) {
+ fail('trusted capture manifest is invalid');
}
-if (!Array.isArray(verdict.changes) || verdict.changes.length > MAX_CHANGES) {
- fail(`invalid change count ${verdict.changes?.length ?? 'missing'}`);
+if (
+ manifest.context?.headSha !== headSha ||
+ String(manifest.context?.runId ?? '') !== runId ||
+ String(manifest.context?.runAttempt ?? '') !== runAttempt ||
+ !SHA.test(manifest.context?.sha ?? '') ||
+ !SHA.test(manifest.context?.baseSha ?? '')
+) {
+ fail('trusted capture identity mismatch');
}
-if (!Array.isArray(verdict.added) || !Array.isArray(verdict.removed)) {
- fail('added/removed shot lists are missing');
+
+const baselineManifest = readJSON(path.join(baseline, 'manifest.json'));
+const blocker = incomparable(baselineManifest, manifest);
+if (blocker) fail(`baseline is not comparable: ${blocker}`);
+
+const entries = Object.entries(manifest.shots);
+if (entries.length > MAX_SHOTS) fail(`trusted capture exceeds ${MAX_SHOTS} shots`);
+for (const [key, shot] of entries) {
+ if (
+ !KEY.test(key) ||
+ !shot ||
+ typeof shot.storyId !== 'string' ||
+ !shot.storyId ||
+ typeof shot.theme !== 'string' ||
+ !NAME.test(shot.theme) ||
+ !['light', 'dark'].includes(shot.mode) ||
+ !Array.isArray(shot.reasons) ||
+ shotKey(shot) !== key
+ ) {
+ fail(`capture metadata is invalid for ${key}`);
+ }
}
-if (verdict.changes.length + verdict.added.length + verdict.removed.length > MAX_CHANGES) {
- fail('visual delta bundle exceeds the review budget');
+fs.rmSync(output, {recursive: true, force: true});
+for (const dir of ['current', 'before', 'after', 'diff']) {
+ fs.mkdirSync(path.join(output, dir), {recursive: true});
}
-const seen = new Set();
-const validateKey = key => {
- if (!KEY.test(key ?? '')) fail(`invalid shot key ${JSON.stringify(key)}`);
- if (seen.has(key)) fail(`duplicate shot key ${key}`);
- seen.add(key);
-};
-for (const change of verdict.changes) {
- validateKey(change.key);
- if (!Number.isFinite(Number(change.diffPixels)) || Number(change.diffPixels) < 0) {
- fail(`invalid diff pixel count for ${change.key}`);
+const sourceShots = path.join(input, 'shots');
+const sourceNames = fs.existsSync(sourceShots)
+ ? fs.readdirSync(sourceShots, {withFileTypes: true})
+ : [];
+const sourceKeys = sourceNames.map(entry => {
+ if (!entry.isFile() || entry.isSymbolicLink() || !entry.name.endsWith('.png')) {
+ fail(`unexpected capture entry ${entry.name}`);
}
+ return entry.name.slice(0, -4);
+});
+if (sourceKeys.length !== entries.length || sourceKeys.some(key => !manifest.shots[key])) {
+ fail('capture PNG set does not match its manifest');
}
-for (const key of verdict.added) validateKey(key);
-for (const key of verdict.removed) validateKey(key);
-fs.rmSync(output, {recursive: true, force: true});
-for (const kind of ['before', 'after', 'diff']) {
- fs.mkdirSync(path.join(output, kind), {recursive: true});
+const trustedManifest = {...manifest, shots: {}};
+for (const [key, shot] of entries) {
+ const target = path.join(output, 'current', `${key}.png`);
+ const image = copyPng(path.join(sourceShots, `${key}.png`), target, `shots/${key}.png`);
+ trustedManifest.shots[key] = {
+ ...shot,
+ sha256: shaFile(target),
+ width: image.width,
+ height: image.height,
+ };
}
-/** Decode + re-encode: validates the bytes and strips ancillary metadata. */
-function copyPng(kind, key, sourceDir = path.join(input, 'report', kind)) {
- const source = path.join(sourceDir, `${key}.png`);
- if (!fs.existsSync(source)) fail(`${kind}/${key}.png is missing`);
- const stat = fs.lstatSync(source);
- if (!stat.isFile() || stat.isSymbolicLink()) fail(`${kind}/${key}.png is not a regular file`);
- if (stat.size <= 0 || stat.size > MAX_PNG_BYTES) fail(`${kind}/${key}.png has invalid size`);
-
- let image;
- try {
- image = PNG.sync.read(fs.readFileSync(source), {checkCRC: true});
- } catch (error) {
- fail(`${kind}/${key}.png is not a valid PNG: ${error.message}`);
- }
- if (
- image.width <= 0 ||
- image.height <= 0 ||
- image.width > MAX_EDGE ||
- image.height > MAX_EDGE
- ) {
- fail(`${kind}/${key}.png has invalid dimensions ${image.width}x${image.height}`);
- }
- fs.writeFileSync(path.join(output, kind, `${key}.png`), PNG.sync.write(image));
+const baselineEntries = Object.entries(baselineManifest.shots ?? {});
+const expected = scope.broadStableVisual
+ ? baselineEntries.map(([key]) => key)
+ : baselineEntries
+ .filter(([, shot]) =>
+ scope.stableComponents.includes(shot.component) || scope.stableThemes.includes(shot.theme),
+ )
+ .map(([key]) => key);
+if (entries.length === 0 && expected.length === 0) {
+ fail('stable visual scope produced no trusted shots');
}
-for (const {key} of verdict.changes) {
- for (const kind of ['before', 'after', 'diff']) copyPng(kind, key);
+const comparison = await compareCaptures({
+ baselineDir: path.join(baseline, 'shots'),
+ currentDir: path.join(output, 'current'),
+ baselineManifest,
+ currentManifest: trustedManifest,
+ diffDir: path.join(output, 'diff'),
+ threshold: config.threshold,
+ maxDiffPixels: config.maxDiffPixels,
+ scoped: true,
+});
+comparison.removed = expected.filter(key => !trustedManifest.shots[key]);
+
+const verdict = buildVerdict({
+ comparison,
+ currentManifest: trustedManifest,
+ baselineManifest,
+ targeting: {unexercisedOverrides: [], undeclaredTargets: [], uncoveredTargets: []},
+ failures: [],
+ context: {...manifest.context, trustedScope: scope},
+});
+
+const beforeSha256 = {};
+for (const change of verdict.changes) {
+ const key = change.key;
+ const baselineFile = path.join(baseline, 'shots', `${key}.png`);
+ copyPng(baselineFile, path.join(output, 'before', `${key}.png`), `baseline/${key}.png`);
+ fs.copyFileSync(path.join(output, 'current', `${key}.png`), path.join(output, 'after', `${key}.png`));
+ beforeSha256[key] = shaFile(baselineFile);
}
-// New stable shots have no before/diff, but their AFTER is the image a human
-// accepts and the post-merge verifier must reproduce. Preserve it now; the
-// Actions artifact expires and the per-PR path is deleted when the PR closes.
for (const key of verdict.added) {
- copyPng('after', key, path.join(input, 'shots'));
+ fs.copyFileSync(path.join(output, 'current', `${key}.png`), path.join(output, 'after', `${key}.png`));
+ beforeSha256[key] = null;
+}
+for (const key of verdict.removed) {
+ const baselineFile = path.join(baseline, 'shots', `${key}.png`);
+ copyPng(baselineFile, path.join(output, 'before', `${key}.png`), `baseline/${key}.png`);
+ beforeSha256[key] = shaFile(baselineFile);
}
const evidence = {
@@ -138,23 +231,48 @@ const evidence = {
repo: 'facebook/astryx',
pr,
headSha,
- testedSha: verdict.context?.sha ?? null,
- baseSha: verdict.context?.baseSha ?? null,
+ testedSha: manifest.context.sha,
+ baseSha: manifest.context.baseSha,
run: {id: Number(runId), attempt: Number(runAttempt)},
+ capture: {
+ platform: trustedManifest.platform,
+ browser: trustedManifest.browser,
+ viewport: trustedManifest.viewport,
+ },
deltas: [
- ...verdict.changes.map(change => ({key: change.key, kind: 'changed'})),
- ...verdict.added.map(key => ({key, kind: 'added'})),
- ...verdict.removed.map(key => ({key, kind: 'removed'})),
+ ...verdict.changes.map(change => ({
+ key: change.key,
+ kind: 'changed',
+ beforeSha256: beforeSha256[change.key],
+ shot: trustedManifest.shots[change.key],
+ })),
+ ...verdict.added.map(key => ({
+ key,
+ kind: 'added',
+ beforeSha256: null,
+ shot: trustedManifest.shots[key],
+ })),
+ ...verdict.removed.map(key => ({
+ key,
+ kind: 'removed',
+ beforeSha256: beforeSha256[key],
+ shot: null,
+ })),
],
verdict,
};
+
+fs.rmSync(path.join(output, 'current'), {recursive: true, force: true});
fs.writeFileSync(path.join(output, 'evidence.json'), `${JSON.stringify(evidence, null, 2)}\n`);
fs.writeFileSync(path.join(output, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`);
fs.writeFileSync(
path.join(output, 'index.html'),
- renderReport(verdict, {acceptHint: '/accept-visual <why every changed frame is correct>'}),
+ renderReport(verdict, {
+ acceptHint: `/accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,
+ oneSidedEvidence: true,
+ }),
);
process.stdout.write(
- `Validated ${verdict.changes.length} visual delta(s) for PR #${pr}, head ${headSha.slice(0, 12)}, run ${runId}/${runAttempt}.\n`,
+ `Derived trusted ${verdict.status} verdict for ${entries.length} shot(s), PR #${pr}, run ${runId}/${runAttempt}.\n`,
);
diff --git a/.github/scripts/visual-gate/publish-pr-report.test.mjs b/.github/scripts/visual-gate/publish-pr-report.test.mjs
index 12a95891a0..2a99b05cbb 100644
--- a/.github/scripts/visual-gate/publish-pr-report.test.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.test.mjs
@@ -1,6 +1,7 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
import {execFileSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
@@ -9,134 +10,193 @@ import {fileURLToPath} from 'node:url';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
import {PNG} from 'pngjs';
-const SCRIPT = path.join(
- path.dirname(fileURLToPath(import.meta.url)),
- 'publish-pr-report.mjs',
-);
+const SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), 'publish-pr-report.mjs');
const HEAD = 'a'.repeat(40);
const RUN = '123456';
+const KEY = 'core-button--default__neutral-light';
+const SHOT = {
+ storyId: 'core-button--default',
+ title: 'Core/Button',
+ name: 'Default',
+ component: 'Button',
+ theme: 'neutral',
+ mode: 'light',
+ reasons: ['trusted:pr-scope'],
+};
let root;
let input;
let output;
+let baseline;
+let scope;
-function png() {
+function png(red = 255, green = 0, blue = 0) {
const image = new PNG({width: 2, height: 2});
for (let i = 0; i < image.data.length; i += 4) {
- image.data[i] = 255;
+ image.data[i] = red;
+ image.data[i + 1] = green;
+ image.data[i + 2] = blue;
image.data[i + 3] = 255;
}
return PNG.sync.write(image);
}
-function writeFixture(overrides = {}) {
- const key = overrides.key ?? 'core-button--default__neutral-light';
- const verdict = {
+function writeJSON(file, value) {
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, JSON.stringify(value));
+}
+
+function writeBaseline(shots = {[KEY]: {shot: SHOT, bytes: png()}}) {
+ const manifestShots = {};
+ for (const [key, {shot, bytes}] of Object.entries(shots)) {
+ const file = path.join(baseline, 'shots', `${key}.png`);
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, bytes);
+ manifestShots[key] = {...shot, sha256: createHash('sha256').update(bytes).digest('hex')};
+ }
+ writeJSON(path.join(baseline, 'manifest.json'), {
version: 1,
- status: 'changed',
- context: {runId: RUN, headSha: HEAD, sha: 'b'.repeat(40)},
- counts: {total: 1, changed: 1, added: 0, removed: 0, failed: 0},
- changes: [
- {
- key,
- component: 'Button',
- name: 'Default',
- theme: 'neutral',
- mode: 'light',
- diffPixels: 4,
- },
- ],
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ shots: manifestShots,
+ decisions: [],
+ });
+}
+
+function writeCapture(shots = {[KEY]: {shot: SHOT, bytes: png(0, 0, 255)}}, overrides = {}) {
+ fs.rmSync(input, {recursive: true, force: true});
+ const manifestShots = {};
+ const shotsDir = path.join(input, 'shots');
+ fs.mkdirSync(shotsDir, {recursive: true});
+ for (const [key, {shot, bytes}] of Object.entries(shots)) {
+ fs.writeFileSync(path.join(shotsDir, `${key}.png`), bytes);
+ manifestShots[key] = {...shot, sha256: '0'.repeat(64)};
+ }
+ writeJSON(path.join(input, 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ context: {
+ sha: 'b'.repeat(40),
+ headSha: HEAD,
+ baseSha: 'c'.repeat(40),
+ runId: RUN,
+ runAttempt: '1',
+ },
+ shots: manifestShots,
+ ...overrides,
+ });
+ // A forged PR verdict is deliberately present: the trusted publisher ignores it.
+ writeJSON(path.join(input, 'verdict.json'), {
+ version: 1,
+ status: 'pass',
+ changes: [],
added: [],
removed: [],
- ...overrides.verdict,
- };
- fs.mkdirSync(input, {recursive: true});
- fs.writeFileSync(path.join(input, 'verdict.json'), JSON.stringify(verdict));
- for (const kind of ['before', 'after', 'diff']) {
- const dir = path.join(input, 'report', kind);
- fs.mkdirSync(dir, {recursive: true});
- fs.writeFileSync(path.join(dir, `${key}.png`), overrides.bytes ?? png());
- }
+ failures: [],
+ });
}
-function run(extra = []) {
+function run() {
return execFileSync(
process.execPath,
[
SCRIPT,
- '--input',
- input,
- '--output',
- output,
- '--pr',
- '42',
- '--head-sha',
- HEAD,
- '--run-id',
- RUN,
- '--run-attempt',
- '1',
- ...extra,
+ '--input', input,
+ '--output', output,
+ '--baseline', baseline,
+ '--scope', scope,
+ '--pr', '42',
+ '--head-sha', HEAD,
+ '--run-id', RUN,
+ '--run-attempt', '1',
],
{encoding: 'utf8'},
);
}
beforeEach(() => {
- root = fs.mkdtempSync(path.join(os.tmpdir(), 'publish-visual-'));
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'trusted-visual-'));
input = path.join(root, 'input');
output = path.join(root, 'output');
+ baseline = path.join(root, 'baseline');
+ scope = path.join(root, 'scope.json');
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: false,
+ stableComponents: ['Button'],
+ stableThemes: [],
+ });
+ writeBaseline();
+ writeCapture();
});
+
afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
describe('trusted PR visual publisher', () => {
- it('decodes/re-encodes images and generates the wrapper from trusted code', () => {
- writeFixture();
- expect(run()).toContain('Validated 1 visual delta');
- expect(fs.existsSync(path.join(output, 'index.html'))).toBe(true);
- expect(fs.existsSync(path.join(output, 'before', 'core-button--default__neutral-light.png'))).toBe(true);
- const evidence = JSON.parse(fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'));
- expect(evidence).toMatchObject({pr: 42, headSha: HEAD, testedSha: 'b'.repeat(40)});
+ it('derives a changed verdict even when PR data claims pass', () => {
+ expect(run()).toContain('trusted changed verdict');
+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));
+ expect(verdict).toMatchObject({status: 'changed', counts: {changed: 1}});
+ expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);
+ expect(fs.existsSync(path.join(output, 'after', `${KEY}.png`))).toBe(true);
+ expect(fs.existsSync(path.join(output, 'diff', `${KEY}.png`))).toBe(true);
+ });
+
+ it('derives pass only from a trusted capture matching the baseline', () => {
+ writeCapture({[KEY]: {shot: SHOT, bytes: png()}});
+ run();
+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));
+ expect(verdict.status).toBe('pass');
});
- it('preserves the after image for a newly added stable shot', () => {
+ it('derives an added shot and preserves its after image', () => {
+ writeBaseline({});
const key = 'core-new--default__neutral-light';
- writeFixture({
- key,
- verdict: {
- status: 'changed',
- changes: [],
- added: [key],
- removed: [],
- counts: {total: 1, changed: 0, added: 1, removed: 0, failed: 0},
- },
+ const shot = {...SHOT, storyId: 'core-new--default', component: 'New'};
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: false,
+ stableComponents: ['New'],
+ stableThemes: [],
});
- // Added shots come from capture/shots, not report/after.
- fs.mkdirSync(path.join(input, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(input, 'shots', `${key}.png`), png());
+ writeCapture({[key]: {shot, bytes: png(0, 255, 0)}});
run();
+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));
+ expect(verdict).toMatchObject({status: 'changed', added: [key]});
expect(fs.existsSync(path.join(output, 'after', `${key}.png`))).toBe(true);
- const evidence = JSON.parse(fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'));
- expect(evidence.deltas).toEqual([{key, kind: 'added'}]);
});
- it('rejects a key repeated across changed/added/removed lists', () => {
- writeFixture({verdict: {added: ['core-button--default__neutral-light']}});
- expect(() => run()).toThrow(/duplicate shot key/);
+ it('derives a removed baseline shot when trusted capture omits expected scope', () => {
+ writeCapture({});
+ run();
+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));
+ expect(verdict).toMatchObject({status: 'changed', removed: [KEY]});
+ expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);
});
- it('rejects an artifact that claims another run', () => {
- writeFixture({verdict: {context: {runId: '999', headSha: HEAD}}});
- expect(() => run()).toThrow(/does not match/);
+ it('rejects a trusted capture that claims another run', () => {
+ const manifest = JSON.parse(fs.readFileSync(path.join(input, 'manifest.json'), 'utf8'));
+ manifest.context.runId = '999';
+ writeJSON(path.join(input, 'manifest.json'), manifest);
+ expect(() => run()).toThrow(/identity mismatch/);
});
- it('rejects traversal and shell-shaped shot keys', () => {
- writeFixture({key: '../bad;rm'});
- expect(() => run()).toThrow(/invalid shot key/);
+ it('rejects traversal-shaped manifest keys', () => {
+ writeCapture({'../bad': {shot: {...SHOT, storyId: '../bad'}, bytes: png()}});
+ expect(() => run()).toThrow(/metadata is invalid/);
});
- it('rejects bytes that are not a decodable PNG', () => {
- writeFixture({bytes: Buffer.from('not png')});
+ it('rejects capture bytes that are not a PNG', () => {
+ fs.writeFileSync(path.join(input, 'shots', `${KEY}.png`), 'not png');
expect(() => run()).toThrow(/not a valid PNG/);
});
+
+ it('rejects an empty capture for an unbaselined stable scope', () => {
+ writeBaseline({});
+ writeCapture({});
+ expect(() => run()).toThrow(/produced no trusted shots/);
+ });
});
diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjs
new file mode 100644
index 0000000000..e165024ac4
--- /dev/null
+++ b/.github/scripts/visual-gate/visual-acceptance.mjs
@@ -0,0 +1,752 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Record, evaluate, and promote explicit visual acceptance.
+ *
+ * Labels, comments, and statuses are projections. The immutable JSON record
+ * under gh-pages:visual-gate/acceptances/<pr>/<head>/ is the source of truth.
+ */
+
+import {execFileSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import {PNG} from 'pngjs';
+
+import {readStoryIndex, shotKey, storiesInPackages} from './lib/plan.mjs';
+
+const args = process.argv.slice(2);
+const command = args[0];
+const flag = name => {
+ const index = args.indexOf(`--${name}`);
+ return index === -1 ? null : args[index + 1];
+};
+const SHA = /^[0-9a-f]{40}$/;
+const KEY = /^[A-Za-z0-9._-]{1,240}$/;
+const REPO = 'facebook/astryx';
+
+function fail(message) {
+ throw new Error(`visual acceptance refused: ${message}`);
+}
+
+function shaBytes(bytes) {
+ return createHash('sha256').update(bytes).digest('hex');
+}
+
+function shaFile(file) {
+ return shaBytes(fs.readFileSync(file));
+}
+
+function shaJSON(value) {
+ return shaBytes(JSON.stringify(value));
+}
+
+function readJSON(file) {
+ try {
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+ } catch (error) {
+ fail(`cannot read ${file}: ${error.message}`);
+ }
+}
+
+function writeJSON(file, value) {
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function pagesHead(pages) {
+ return execFileSync('git', ['rev-parse', 'HEAD'], {
+ cwd: pages,
+ encoding: 'utf8',
+ }).trim();
+}
+
+function validateIdentity(pr, head) {
+ if (!Number.isSafeInteger(pr) || pr <= 0 || !SHA.test(head))
+ fail('invalid PR/head');
+}
+
+function evidenceRoot(pages, pr, head) {
+ return path.join(pages, 'pr', String(pr), 'visual', head);
+}
+
+function evidenceAt(pages, pr, head, run, attempt) {
+ if (!Number.isSafeInteger(run) || run <= 0 || !Number.isSafeInteger(attempt) || attempt <= 0) {
+ fail('invalid evidence run/attempt');
+ }
+ const dir = path.join(evidenceRoot(pages, pr, head), String(run), String(attempt));
+ if (!fs.existsSync(dir)) fail(`no published evidence for run ${run}/${attempt}`);
+ const evidence = readJSON(path.join(dir, 'evidence.json'));
+ validateEvidence(evidence, {pr, head, run});
+ if (evidence.run.attempt !== attempt) fail('evidence attempt identity mismatch');
+ return {dir, evidence};
+}
+
+function latestEvidence(pages, pr, head, {required = true} = {}) {
+ const root = evidenceRoot(pages, pr, head);
+ if (!fs.existsSync(root)) {
+ if (!required) return null;
+ fail(`no published evidence for PR #${pr}, head ${head}`);
+ }
+ const runs = fs
+ .readdirSync(root, {withFileTypes: true})
+ .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))
+ .map(entry => Number(entry.name))
+ .sort((a, b) => b - a);
+ if (runs.length === 0) {
+ if (!required) return null;
+ fail('published evidence has no run directory');
+ }
+ const run = runs[0];
+ const runDir = path.join(root, String(run));
+ const attempts = fs
+ .readdirSync(runDir, {withFileTypes: true})
+ .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))
+ .map(entry => Number(entry.name))
+ .sort((a, b) => b - a);
+ const dir = attempts.length > 0 ? path.join(runDir, String(attempts[0])) : runDir;
+ const evidence = readJSON(path.join(dir, 'evidence.json'));
+ validateEvidence(evidence, {pr, head, run});
+ if (attempts.length > 0 && evidence.run.attempt !== attempts[0]) {
+ fail('evidence attempt identity mismatch');
+ }
+ return {dir, evidence};
+}
+
+function validateCapture(capture) {
+ if (
+ !capture ||
+ typeof capture.platform !== 'string' ||
+ !capture.platform ||
+ typeof capture.browser !== 'string' ||
+ !capture.browser ||
+ !Number.isSafeInteger(capture.viewport?.width) ||
+ !Number.isSafeInteger(capture.viewport?.height) ||
+ capture.viewport.width <= 0 ||
+ capture.viewport.height <= 0
+ ) {
+ fail('evidence capture identity is incomplete');
+ }
+}
+
+function validateShot(shot, key) {
+ if (
+ !shot ||
+ typeof shot.storyId !== 'string' ||
+ !shot.storyId ||
+ typeof shot.theme !== 'string' ||
+ !shot.theme ||
+ !['light', 'dark'].includes(shot.mode) ||
+ !Array.isArray(shot.reasons)
+ ) {
+ fail(`accepted shot plan is incomplete for ${key}`);
+ }
+}
+
+function validateEvidence(evidence, expected) {
+ if (
+ evidence?.version !== 1 ||
+ evidence.repo !== REPO ||
+ evidence.pr !== expected.pr ||
+ evidence.headSha !== expected.head ||
+ !SHA.test(evidence.testedSha ?? '') ||
+ !SHA.test(evidence.baseSha ?? '') ||
+ evidence.run?.id !== expected.run ||
+ !Number.isSafeInteger(evidence.run?.attempt) ||
+ evidence.run.attempt <= 0
+ ) {
+ fail('evidence identity mismatch');
+ }
+ if (
+ !['pass', 'changed', 'failed', 'skipped'].includes(evidence.verdict?.status)
+ ) {
+ fail('evidence verdict is invalid');
+ }
+ if (!Array.isArray(evidence.deltas) || evidence.deltas.length > 5000) {
+ fail('evidence delta list is invalid');
+ }
+ if (evidence.verdict.status === 'skipped') {
+ if (evidence.capture !== null || evidence.deltas.length !== 0) {
+ fail('skipped evidence must not claim a capture or deltas');
+ }
+ } else {
+ validateCapture(evidence.capture);
+ }
+ const seen = new Set();
+ for (const delta of evidence.deltas) {
+ if (
+ !KEY.test(delta?.key ?? '') ||
+ !['changed', 'added', 'removed'].includes(delta.kind)
+ ) {
+ fail(`invalid evidence delta ${JSON.stringify(delta)}`);
+ }
+ if (seen.has(delta.key)) fail(`duplicate evidence key ${delta.key}`);
+ seen.add(delta.key);
+ if (delta.kind === 'removed') {
+ if (delta.shot !== null)
+ fail(`removed key ${delta.key} must not carry a shot plan`);
+ } else {
+ validateShot(delta.shot, delta.key);
+ }
+ }
+}
+
+function validateReason(reason) {
+ const clean = String(reason ?? '').trim();
+ if (
+ clean.length < 12 ||
+ clean.length > 500 ||
+ /[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(clean)
+ ) {
+ fail('reason must be 12–500 printable characters');
+ }
+ if (
+ /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(clean)
+ ) {
+ fail(
+ 'reason must explain why the pixels are correct, not merely approve them',
+ );
+ }
+ return clean;
+}
+
+function snapshotBundle({pages, evidenceDir, evidence}) {
+ const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');
+ return evidence.deltas.map(delta => {
+ const beforeFile = path.join(baselineShots, `${delta.key}.png`);
+ const afterFile = path.join(evidenceDir, 'after', `${delta.key}.png`);
+ const currentBeforeSha256 = fs.existsSync(beforeFile)
+ ? shaFile(beforeFile)
+ : null;
+ const beforeSha256 = delta.beforeSha256 ?? null;
+ const afterSha256 = fs.existsSync(afterFile) ? shaFile(afterFile) : null;
+ if (beforeSha256 !== null && !/^[0-9a-f]{64}$/.test(beforeSha256)) {
+ fail(`invalid reviewed preimage for ${delta.key}`);
+ }
+ if (currentBeforeSha256 !== beforeSha256) {
+ fail(`reviewed baseline is stale for ${delta.key}`);
+ }
+ if (delta.kind === 'changed' && (!beforeSha256 || !afterSha256)) {
+ fail(`changed key ${delta.key} lacks before/after evidence`);
+ }
+ if (delta.kind === 'added' && (beforeSha256 || !afterSha256)) {
+ fail(`added key ${delta.key} has invalid preimage/evidence`);
+ }
+ if (delta.kind === 'removed' && (!beforeSha256 || afterSha256)) {
+ fail(`removed key ${delta.key} has invalid preimage/evidence`);
+ }
+ return {
+ key: delta.key,
+ kind: delta.kind,
+ beforeSha256,
+ afterSha256,
+ shot: delta.shot,
+ };
+ });
+}
+
+function sameBundle(first, second) {
+ return shaJSON(first) === shaJSON(second);
+}
+
+function recordPath(pages, pr, head, run, attempt) {
+ return path.join(
+ pages,
+ 'visual-gate',
+ 'acceptances',
+ String(pr),
+ head,
+ String(run),
+ String(attempt),
+ 'acceptance.json',
+ );
+}
+
+function pointerPath(pages, pr, head) {
+ return path.join(pages, 'visual-gate', 'acceptances', String(pr), head, 'current.json');
+}
+
+function accept() {
+ const pages = path.resolve(flag('pages') ?? '.');
+ const pr = Number(flag('pr'));
+ const head = flag('head') ?? '';
+ const runId = Number(flag('run-id'));
+ const runAttempt = Number(flag('run-attempt'));
+ const approver = flag('approver') ?? '';
+ const approverId = Number(flag('approver-id'));
+ const permission = flag('permission') ?? '';
+ const commentId = Number(flag('comment-id'));
+ const reason = validateReason(flag('reason'));
+
+ validateIdentity(pr, head);
+ if (!approver || !Number.isSafeInteger(approverId) || approverId <= 0)
+ fail('invalid approver');
+ if (!['maintain', 'admin'].includes(permission))
+ fail('approver must have maintain/admin permission');
+ if (!Number.isSafeInteger(commentId) || commentId <= 0)
+ fail('invalid comment id');
+
+ const {dir: evidenceDir, evidence} = evidenceAt(pages, pr, head, runId, runAttempt);
+ if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {
+ fail('current visual bundle has no delta to accept');
+ }
+ const manifestFile = path.join(
+ pages,
+ 'visual-gate',
+ 'baseline',
+ 'manifest.json',
+ );
+ const manifest = readJSON(manifestFile);
+ const keys = snapshotBundle({pages, evidenceDir, evidence});
+ const output = recordPath(pages, pr, head, runId, runAttempt);
+ const destination = path.dirname(output);
+
+ const pointer = {
+ version: 1,
+ run: runId,
+ attempt: runAttempt,
+ record: `${runId}/${runAttempt}/acceptance.json`,
+ };
+ if (fs.existsSync(output)) {
+ const existing = readJSON(output);
+ validateAcceptance(existing, {
+ pr,
+ head,
+ run: runId,
+ attempt: runAttempt,
+ dir: destination,
+ });
+ if (sameBundle(existing.keys, keys)) {
+ writeJSON(pointerPath(pages, pr, head), pointer);
+ process.stdout.write(`Acceptance already recorded at ${output}\n`);
+ return;
+ }
+ fail('this head already has a different acceptance record');
+ }
+
+ for (const entry of keys) {
+ if (!entry.afterSha256) continue;
+ const source = path.join(evidenceDir, 'after', `${entry.key}.png`);
+ const target = path.join(destination, 'after', `${entry.key}.png`);
+ fs.mkdirSync(path.dirname(target), {recursive: true});
+ fs.copyFileSync(source, target);
+ }
+
+ writeJSON(output, {
+ version: 1,
+ repo: REPO,
+ pr,
+ headSha: head,
+ testedSha: evidence.testedSha,
+ baseSha: evidence.baseSha,
+ run: evidence.run,
+ capture: evidence.capture,
+ baseline: {
+ ghPagesCommit: pagesHead(pages),
+ manifestDigest: shaJSON(manifest),
+ },
+ keys,
+ decision: {
+ approver,
+ approverId,
+ permission,
+ reason,
+ commentId,
+ at: new Date().toISOString(),
+ },
+ });
+ writeJSON(pointerPath(pages, pr, head), pointer);
+ process.stdout.write(
+ `Accepted ${keys.length} visual delta(s) for PR #${pr}, head ${head}, run ${runId}/${runAttempt}.\n`,
+ );
+}
+
+function state() {
+ const pages = path.resolve(flag('pages') ?? '.');
+ const pr = Number(flag('pr'));
+ const head = flag('head') ?? '';
+ validateIdentity(pr, head);
+
+ const found = latestEvidence(pages, pr, head, {required: false});
+ let result;
+ if (!found) {
+ result = {
+ state: 'pending',
+ reason: 'capture',
+ description: 'Stable visual capture is still running.',
+ };
+ } else if (found.evidence.verdict.status === 'pass') {
+ result = {
+ state: 'success',
+ reason: 'clean',
+ description: 'Stable visual capture is clean.',
+ };
+ } else if (found.evidence.verdict.status === 'failed') {
+ result = {
+ state: 'failure',
+ reason: 'capture',
+ description: 'Stable visual capture failed.',
+ };
+ } else if (found.evidence.verdict.status === 'skipped') {
+ result = {
+ state: 'success',
+ reason: 'deferred',
+ description: 'Broad stable scope is deferred to the release gate.',
+ };
+ } else {
+ let current;
+ try {
+ current = snapshotBundle({
+ pages,
+ evidenceDir: found.dir,
+ evidence: found.evidence,
+ });
+ } catch (error) {
+ if (!String(error.message).includes('reviewed baseline is stale'))
+ throw error;
+ result = {
+ state: 'pending',
+ reason: 'capture',
+ description: 'The visual baseline changed; recapture this head.',
+ };
+ }
+ if (!result) {
+ const file = recordPath(
+ pages,
+ pr,
+ head,
+ found.evidence.run.id,
+ found.evidence.run.attempt,
+ );
+ const accepted = fs.existsSync(file) ? readJSON(file) : null;
+ if (accepted)
+ validateAcceptance(accepted, {
+ pr,
+ head,
+ run: found.evidence.run.id,
+ attempt: found.evidence.run.attempt,
+ dir: path.dirname(file),
+ });
+ result =
+ accepted && sameBundle(accepted.keys, current)
+ ? {
+ state: 'success',
+ reason: 'accepted',
+ description: 'Current visual bundle was explicitly accepted.',
+ }
+ : {
+ state: 'pending',
+ reason: 'decision',
+ description: 'Stable visual changes await /accept-visual.',
+ };
+ }
+ }
+ process.stdout.write(`${JSON.stringify(result)}\n`);
+}
+
+function validateAcceptance(value, expected = {}) {
+ if (
+ value?.version !== 1 ||
+ value.repo !== REPO ||
+ !Number.isSafeInteger(value.pr) ||
+ value.pr <= 0 ||
+ (expected.pr && value.pr !== expected.pr) ||
+ !SHA.test(value.headSha ?? '') ||
+ (expected.head && value.headSha !== expected.head) ||
+ !SHA.test(value.testedSha ?? '') ||
+ !SHA.test(value.baseSha ?? '') ||
+ !Number.isSafeInteger(value.run?.id) ||
+ value.run.id <= 0 ||
+ (expected.run && value.run.id !== expected.run) ||
+ !Number.isSafeInteger(value.run?.attempt) ||
+ value.run.attempt <= 0 ||
+ (expected.attempt && value.run.attempt !== expected.attempt) ||
+ !Array.isArray(value.keys) ||
+ value.keys.length === 0 ||
+ !value.decision?.approver ||
+ !Number.isSafeInteger(value.decision?.approverId) ||
+ value.decision.approverId <= 0 ||
+ !['maintain', 'admin'].includes(value.decision?.permission) ||
+ !Number.isSafeInteger(value.decision?.commentId) ||
+ value.decision.commentId <= 0 ||
+ !value.decision?.reason
+ ) {
+ fail('acceptance record is invalid');
+ }
+ validateCapture(value.capture);
+ const seen = new Set();
+ for (const entry of value.keys) {
+ if (
+ !KEY.test(entry?.key ?? '') ||
+ !['changed', 'added', 'removed'].includes(entry.kind)
+ ) {
+ fail('acceptance record contains an invalid key');
+ }
+ if (seen.has(entry.key)) fail(`acceptance record repeats ${entry.key}`);
+ seen.add(entry.key);
+ if (entry.kind === 'removed') {
+ if (entry.afterSha256 !== null || entry.shot !== null)
+ fail(`removed key ${entry.key} is invalid`);
+ } else {
+ if (!/^[0-9a-f]{64}$/.test(entry.afterSha256 ?? ''))
+ fail(`after hash is invalid for ${entry.key}`);
+ validateShot(entry.shot, entry.key);
+ if (expected.dir) {
+ const archived = path.join(expected.dir, 'after', `${entry.key}.png`);
+ if (!fs.existsSync(archived) || shaFile(archived) !== entry.afterSha256) {
+ fail(`archived AFTER is missing or changed for ${entry.key}`);
+ }
+ }
+ }
+ if (
+ entry.beforeSha256 !== null &&
+ !/^[0-9a-f]{64}$/.test(entry.beforeSha256)
+ ) {
+ fail(`before hash is invalid for ${entry.key}`);
+ }
+ }
+}
+
+function trustedPlan() {
+ const scope = readJSON(path.resolve(flag('scope')));
+ const baseline = readJSON(path.join(path.resolve(flag('baseline')), 'manifest.json'));
+ const storybookDir = path.resolve(flag('storybook-dir'));
+ const output = path.resolve(flag('output'));
+ if (
+ scope?.hasStableVisual !== true ||
+ typeof scope.broadStableVisual !== 'boolean' ||
+ !Array.isArray(scope.stableComponents) ||
+ !Array.isArray(scope.stableThemes)
+ ) {
+ fail('trusted visual scope is invalid');
+ }
+
+ const baselineEntries = Object.entries(baseline.shots ?? {});
+ const baselineThemes = [...new Set(baselineEntries.map(([, shot]) => shot.theme))].filter(Boolean);
+ const shots = scope.broadStableVisual
+ ? baselineEntries.map(([key, shot]) => ({...shot, key, reasons: ['trusted:broad']}))
+ : [];
+
+ if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {
+ const indexed = storiesInPackages(readStoryIndex(storybookDir, []), ['Core']);
+ const stories = new Map();
+ const newTheme = scope.stableThemes.some(theme => !baselineThemes.includes(theme));
+ for (const [, shot] of baselineEntries) {
+ if (
+ newTheme ||
+ scope.stableThemes.includes(shot.theme) ||
+ scope.stableComponents.includes(shot.component)
+ ) {
+ stories.set(shot.storyId, shot);
+ }
+ }
+ for (const story of indexed) {
+ if (scope.stableComponents.includes(story.component)) stories.set(story.id, story);
+ }
+ const themes = scope.stableThemes.length > 0 ? scope.stableThemes : baselineThemes;
+ for (const story of stories.values()) {
+ for (const theme of themes) {
+ for (const mode of ['light', 'dark']) {
+ const shot = {
+ storyId: story.storyId ?? story.id,
+ title: story.title,
+ name: story.name,
+ component: story.component,
+ theme,
+ mode,
+ reasons: ['trusted:pr-scope'],
+ };
+ shots.push({...shot, key: shotKey(shot)});
+ }
+ }
+ }
+ }
+ const unique = [...new Map(shots.map(shot => [shot.key, shot])).values()];
+ if (unique.length === 0 || unique.length > 5000) {
+ fail(`trusted visual plan has invalid size ${unique.length}`);
+ }
+ writeJSON(output, unique);
+ process.stdout.write(`Wrote ${unique.length} trusted PR shot(s) to ${output}.\n`);
+}
+
+function plan() {
+ const acceptanceFile = path.resolve(flag('acceptance'));
+ const acceptance = readJSON(acceptanceFile);
+ validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
+ const output = path.resolve(flag('output'));
+ const shots = acceptance.keys
+ .filter(entry => entry.kind !== 'removed')
+ .map(entry => ({...entry.shot, key: entry.key}));
+ writeJSON(output, shots);
+ process.stdout.write(
+ `Wrote ${shots.length} post-merge shot(s) to ${output}.\n`,
+ );
+}
+
+function samePixels(first, second) {
+ const a = PNG.sync.read(fs.readFileSync(first));
+ const b = PNG.sync.read(fs.readFileSync(second));
+ return (
+ a.width === b.width &&
+ a.height === b.height &&
+ Buffer.compare(a.data, b.data) === 0
+ );
+}
+
+function sameViewport(first, second) {
+ return first?.width === second?.width && first?.height === second?.height;
+}
+
+function promote() {
+ const pages = path.resolve(flag('pages'));
+ const acceptanceFile = path.resolve(flag('acceptance'));
+ const capture = path.resolve(flag('capture'));
+ const mergeSha = flag('merge-sha') ?? '';
+ if (!SHA.test(mergeSha)) fail('invalid merge SHA');
+
+ const acceptance = readJSON(acceptanceFile);
+ validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
+ const manifestFile = path.join(
+ pages,
+ 'visual-gate',
+ 'baseline',
+ 'manifest.json',
+ );
+ const manifest = readJSON(manifestFile);
+ const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');
+ const captureManifest = readJSON(path.join(capture, 'manifest.json'));
+ const acceptedDir = path.dirname(acceptanceFile);
+ const alreadyRecorded = (manifest.decisions ?? []).some(
+ decision =>
+ decision.pr === acceptance.pr &&
+ decision.headSha === acceptance.headSha &&
+ decision.mergeSha === mergeSha,
+ );
+
+ if (captureManifest.context?.sha !== mergeSha) {
+ fail(
+ `capture was produced for ${captureManifest.context?.sha ?? 'an unknown commit'}, not ${mergeSha}`,
+ );
+ }
+ if (
+ captureManifest.platform !== acceptance.capture.platform ||
+ captureManifest.browser !== acceptance.capture.browser ||
+ !sameViewport(captureManifest.viewport, acceptance.capture.viewport)
+ ) {
+ fail('post-merge capture environment does not match the reviewed capture');
+ }
+
+ const actions = [];
+ for (const entry of acceptance.keys) {
+ const baselineFile = path.join(baselineShots, `${entry.key}.png`);
+ const currentBefore = fs.existsSync(baselineFile)
+ ? shaFile(baselineFile)
+ : null;
+ if (currentBefore === entry.afterSha256 && alreadyRecorded) {
+ actions.push({entry, baselineFile, alreadyPromoted: true});
+ continue;
+ }
+ if (currentBefore !== entry.beforeSha256) {
+ fail(
+ `baseline conflict for ${entry.key}: expected ${entry.beforeSha256}, found ${currentBefore}`,
+ );
+ }
+ if (entry.kind === 'removed') {
+ actions.push({entry, baselineFile, remove: true});
+ continue;
+ }
+
+ const acceptedAfter = path.join(acceptedDir, 'after', `${entry.key}.png`);
+ const recaptured = path.join(capture, 'shots', `${entry.key}.png`);
+ const capturedShot = captureManifest.shots?.[entry.key];
+ if (
+ !fs.existsSync(acceptedAfter) ||
+ shaFile(acceptedAfter) !== entry.afterSha256
+ ) {
+ fail(
+ `archived AFTER does not match its decision record for ${entry.key}`,
+ );
+ }
+ if (!fs.existsSync(recaptured) || !capturedShot) {
+ fail(`post-merge capture is missing ${entry.key}`);
+ }
+ if (capturedShot.sha256 !== shaFile(recaptured)) {
+ fail(`post-merge manifest hash does not match ${entry.key}`);
+ }
+ for (const field of ['storyId', 'theme', 'mode']) {
+ if (capturedShot[field] !== entry.shot[field]) {
+ fail(
+ `post-merge ${field} does not match accepted plan for ${entry.key}`,
+ );
+ }
+ }
+ if (!samePixels(acceptedAfter, recaptured)) {
+ fail(`post-merge pixels do not match accepted AFTER for ${entry.key}`);
+ }
+ actions.push({entry, baselineFile, recaptured, capturedShot});
+ }
+
+ if (actions.every(action => action.alreadyPromoted)) {
+ process.stdout.write(`Accepted visual bundle for PR #${acceptance.pr} was already promoted.\n`);
+ return;
+ }
+
+ for (const action of actions) {
+ if (action.alreadyPromoted) continue;
+ if (action.remove) {
+ if (fs.existsSync(action.baselineFile)) fs.rmSync(action.baselineFile);
+ delete manifest.shots[action.entry.key];
+ } else {
+ fs.copyFileSync(action.recaptured, action.baselineFile);
+ manifest.shots[action.entry.key] = action.capturedShot;
+ }
+ }
+
+ manifest.capturedAt = captureManifest.capturedAt;
+ manifest.platform = captureManifest.platform;
+ manifest.browser = captureManifest.browser;
+ manifest.viewport = captureManifest.viewport;
+ manifest.context = {sha: mergeSha};
+ manifest.decisions = [
+ ...(manifest.decisions ?? []),
+ {
+ at: new Date().toISOString(),
+ actor: acceptance.decision.approver,
+ reason: acceptance.decision.reason,
+ pr: acceptance.pr,
+ headSha: acceptance.headSha,
+ mergeSha,
+ promoted: acceptance.keys
+ .filter(entry => entry.kind !== 'removed')
+ .map(entry => entry.key),
+ pruned: acceptance.keys
+ .filter(entry => entry.kind === 'removed')
+ .map(entry => entry.key),
+ },
+ ].slice(-200);
+ writeJSON(manifestFile, manifest);
+ process.stdout.write(
+ `Promoted accepted visual bundle for PR #${acceptance.pr}.\n`,
+ );
+}
+
+switch (command) {
+ case 'accept':
+ accept();
+ break;
+ case 'state':
+ state();
+ break;
+ case 'trusted-plan':
+ trustedPlan();
+ break;
+ case 'plan':
+ plan();
+ break;
+ case 'promote':
+ promote();
+ break;
+ default:
+ fail('usage: visual-acceptance.mjs <accept|state|trusted-plan|plan|promote>');
+}
diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjs
new file mode 100644
index 0000000000..3e3f1f576f
--- /dev/null
+++ b/.github/scripts/visual-gate/visual-acceptance.test.mjs
@@ -0,0 +1,611 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+import {PNG} from 'pngjs';
+
+const SCRIPT = path.join(
+ path.dirname(fileURLToPath(import.meta.url)),
+ 'visual-acceptance.mjs',
+);
+const GATE = path.join(
+ path.dirname(fileURLToPath(import.meta.url)),
+ 'gate.mjs',
+);
+const HEAD = 'a'.repeat(40);
+const TESTED = 'b'.repeat(40);
+const MERGE = 'c'.repeat(40);
+const KEY = 'core-button--default__neutral-light';
+const SHOT = {
+ storyId: 'core-button--default',
+ title: 'Core/Button',
+ name: 'Default',
+ component: 'Button',
+ theme: 'neutral',
+ mode: 'light',
+ reasons: ['component:Button'],
+};
+
+let root;
+let pages;
+
+function png(red, green = 0, blue = 0) {
+ const image = new PNG({width: 2, height: 2});
+ for (let offset = 0; offset < image.data.length; offset += 4) {
+ image.data[offset] = red;
+ image.data[offset + 1] = green;
+ image.data[offset + 2] = blue;
+ image.data[offset + 3] = 255;
+ }
+ return PNG.sync.write(image);
+}
+
+function digest(bytes) {
+ return createHash('sha256').update(bytes).digest('hex');
+}
+
+function writeJSON(file, value) {
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function run(command, flags = {}) {
+ const args = [SCRIPT, command];
+ for (const [name, value] of Object.entries(flags))
+ args.push(`--${name}`, String(value));
+ return execFileSync(process.execPath, args, {encoding: 'utf8'});
+}
+
+function fail(command, flags = {}) {
+ const args = [SCRIPT, command];
+ for (const [name, value] of Object.entries(flags))
+ args.push(`--${name}`, String(value));
+ const result = spawnSync(process.execPath, args, {encoding: 'utf8'});
+ expect(result.status).not.toBe(0);
+ return result.stderr;
+}
+
+function acceptanceFlags(overrides = {}) {
+ return {
+ pages,
+ pr: 42,
+ head: HEAD,
+ 'run-id': 123,
+ 'run-attempt': 1,
+ approver: 'maintainer',
+ 'approver-id': 99,
+ permission: 'maintain',
+ 'comment-id': 1234,
+ reason: 'The new radius matches the approved component design.',
+ ...overrides,
+ };
+}
+
+function acceptanceFile(run = 123, attempt = 1) {
+ return path.join(
+ pages,
+ 'visual-gate',
+ 'acceptances',
+ '42',
+ HEAD,
+ String(run),
+ String(attempt),
+ 'acceptance.json',
+ );
+}
+
+function writeBaseline(entries = {[KEY]: png(255)}) {
+ const shots = {};
+ for (const [key, bytes] of Object.entries(entries)) {
+ const file = path.join(
+ pages,
+ 'visual-gate',
+ 'baseline',
+ 'shots',
+ `${key}.png`,
+ );
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, bytes);
+ shots[key] = {...SHOT, key, sha256: digest(bytes), width: 2, height: 2};
+ }
+ writeJSON(path.join(pages, 'visual-gate', 'baseline', 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ shots,
+ decisions: [],
+ });
+}
+
+function writeEvidence({
+ kind = 'changed',
+ after = png(0, 0, 255),
+ key = KEY,
+ run = 123,
+ attempt = 1,
+ status = 'changed',
+} = {}) {
+ const dir = path.join(pages, 'pr', '42', 'visual', HEAD, String(run), String(attempt));
+ fs.mkdirSync(path.join(dir, 'after'), {recursive: true});
+ if (status === 'changed' && kind !== 'removed') {
+ fs.writeFileSync(path.join(dir, 'after', `${key}.png`), after);
+ }
+ const deltas =
+ status === 'changed'
+ ? [
+ {
+ key,
+ kind,
+ beforeSha256:
+ kind === 'added'
+ ? null
+ : digest(
+ fs.readFileSync(
+ path.join(
+ pages,
+ 'visual-gate',
+ 'baseline',
+ 'shots',
+ `${key}.png`,
+ ),
+ ),
+ ),
+ shot: kind === 'removed' ? null : SHOT,
+ },
+ ]
+ : [];
+ writeJSON(path.join(dir, 'evidence.json'), {
+ version: 1,
+ repo: 'facebook/astryx',
+ pr: 42,
+ headSha: HEAD,
+ testedSha: TESTED,
+ baseSha: 'd'.repeat(40),
+ run: {id: run, attempt},
+ capture:
+ status === 'skipped'
+ ? null
+ : {
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ },
+ deltas,
+ verdict: {version: 1, status},
+ });
+ return dir;
+}
+
+function commitPages() {
+ execFileSync('git', ['add', '.'], {cwd: pages});
+ execFileSync('git', ['commit', '-qm', 'fixture'], {cwd: pages});
+}
+
+beforeEach(() => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-acceptance-'));
+ pages = path.join(root, 'pages');
+ fs.mkdirSync(pages);
+ execFileSync('git', ['init', '-q', '-b', 'gh-pages'], {cwd: pages});
+ execFileSync('git', ['config', 'user.name', 'Test'], {cwd: pages});
+ execFileSync('git', ['config', 'user.email', '[email protected]'], {
+ cwd: pages,
+ });
+ writeBaseline();
+ writeEvidence();
+ commitPages();
+});
+
+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
+
+describe('visual acceptance', () => {
+ it('archives the reviewed pixels and a complete decision record', () => {
+ expect(run('accept', acceptanceFlags())).toContain(
+ 'Accepted 1 visual delta',
+ );
+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
+ expect(record).toMatchObject({
+ version: 1,
+ pr: 42,
+ headSha: HEAD,
+ testedSha: TESTED,
+ run: {id: 123, attempt: 1},
+ capture: {platform: 'linux-arm64', browser: 'chromium-140.0'},
+ decision: {
+ approver: 'maintainer',
+ approverId: 99,
+ permission: 'maintain',
+ commentId: 1234,
+ },
+ });
+ expect(record.keys[0]).toMatchObject({key: KEY, kind: 'changed'});
+ expect(record.keys[0].beforeSha256).toMatch(/^[0-9a-f]{64}$/);
+ expect(record.keys[0].afterSha256).toMatch(/^[0-9a-f]{64}$/);
+ expect(
+ fs.existsSync(
+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
+ ),
+ ).toBe(true);
+ });
+
+ it('requires an explanatory reason and maintainer permission', () => {
+ expect(fail('accept', acceptanceFlags({reason: 'intentional...'}))).toMatch(
+ /reason must explain/,
+ );
+ expect(fail('accept', acceptanceFlags({permission: 'write'}))).toMatch(
+ /maintain\/admin/,
+ );
+ });
+
+ it('binds acceptance to the exact reviewed run attempt', () => {
+ writeEvidence({run: 123, attempt: 2, after: png(0, 255, 0)});
+ run('accept', acceptanceFlags({'run-id': 123, 'run-attempt': 1}));
+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
+ expect(record.run).toEqual({id: 123, attempt: 1});
+ expect(record.keys[0].afterSha256).toBe(digest(png(0, 0, 255)));
+ });
+
+ it('reports whether the current visual bundle is clean, waiting, accepted, or failed', () => {
+ const stateFlags = {pages, pr: 42, head: HEAD};
+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({
+ state: 'pending',
+ reason: 'decision',
+ });
+ run('accept', acceptanceFlags());
+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({
+ state: 'success',
+ reason: 'accepted',
+ });
+
+ writeEvidence({run: 124, attempt: 1, after: png(0, 0, 255)});
+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({
+ state: 'pending',
+ reason: 'decision',
+ });
+ writeEvidence({run: 124, attempt: 2, after: png(0, 255, 0)});
+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({
+ state: 'pending',
+ reason: 'decision',
+ });
+ writeEvidence({run: 125, status: 'skipped'});
+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({
+ state: 'success',
+ reason: 'deferred',
+ });
+ });
+
+ it('derives a trusted component plan from baseline themes and the Storybook index', () => {
+ const storybook = path.join(root, 'storybook');
+ fs.mkdirSync(storybook);
+ writeJSON(path.join(storybook, 'index.json'), {
+ entries: {
+ button: {
+ type: 'story',
+ id: 'core-button--default',
+ title: 'Core/Button',
+ name: 'Default',
+ tags: [],
+ },
+ },
+ });
+ const scope = path.join(root, 'scope.json');
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: false,
+ stableComponents: ['Button'],
+ stableThemes: [],
+ });
+ const output = path.join(root, 'trusted-plan.json');
+ run('trusted-plan', {
+ scope,
+ baseline: path.join(pages, 'visual-gate', 'baseline'),
+ 'storybook-dir': storybook,
+ output,
+ });
+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([
+ 'core-button--default__neutral-light',
+ 'core-button--default__neutral-dark',
+ ]);
+ });
+
+ it('plans every baseline story for a newly promoted stable theme', () => {
+ const storybook = path.join(root, 'storybook-new-theme');
+ fs.mkdirSync(storybook);
+ writeJSON(path.join(storybook, 'index.json'), {entries: {}});
+ const scope = path.join(root, 'scope-new-theme.json');
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: false,
+ stableComponents: [],
+ stableThemes: ['new-theme'],
+ });
+ const output = path.join(root, 'trusted-theme-plan.json');
+ run('trusted-plan', {
+ scope,
+ baseline: path.join(pages, 'visual-gate', 'baseline'),
+ 'storybook-dir': storybook,
+ output,
+ });
+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([
+ 'core-button--default__new-theme-light',
+ 'core-button--default__new-theme-dark',
+ ]);
+ });
+
+ it('uses the full baseline plan for broad stable infrastructure', () => {
+ const scope = path.join(root, 'scope-broad.json');
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: true,
+ stableComponents: [],
+ stableThemes: [],
+ });
+ const output = path.join(root, 'trusted-broad-plan.json');
+ run('trusted-plan', {
+ scope,
+ baseline: path.join(pages, 'visual-gate', 'baseline'),
+ 'storybook-dir': root,
+ output,
+ });
+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([KEY]);
+ });
+
+ it('allows the trusted 520-shot broad plan through the capture CLI', () => {
+ const plan = Array.from({length: 520}, (_, index) => ({
+ key: `story-${index}__neutral-light`,
+ storyId: `story-${index}`,
+ title: `Core/C${index}`,
+ name: 'Default',
+ component: `C${index}`,
+ theme: 'neutral',
+ mode: 'light',
+ reasons: ['trusted:broad'],
+ }));
+ const file = path.join(root, 'large-plan.json');
+ writeJSON(file, plan);
+ const printed = execFileSync(process.execPath, [GATE, 'plan', '--plan-file', file], {
+ encoding: 'utf8',
+ });
+ expect(printed).toContain('520 shots');
+ });
+
+ it('keeps new component stories in a mixed broad plan', () => {
+ const storybook = path.join(root, 'storybook-mixed');
+ fs.mkdirSync(storybook);
+ writeJSON(path.join(storybook, 'index.json'), {
+ entries: {
+ newStory: {
+ type: 'story',
+ id: 'core-new--default',
+ title: 'Core/New',
+ name: 'Default',
+ tags: [],
+ },
+ },
+ });
+ const scope = path.join(root, 'scope-mixed.json');
+ writeJSON(scope, {
+ hasStableVisual: true,
+ broadStableVisual: true,
+ stableComponents: ['New'],
+ stableThemes: [],
+ });
+ const output = path.join(root, 'trusted-mixed-plan.json');
+ run('trusted-plan', {
+ scope,
+ baseline: path.join(pages, 'visual-gate', 'baseline'),
+ 'storybook-dir': storybook,
+ output,
+ });
+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([
+ KEY,
+ 'core-new--default__neutral-light',
+ 'core-new--default__neutral-dark',
+ ]);
+ });
+
+ it('accepts a trusted 520-delta broad bundle', () => {
+ const runId = 130;
+ const dir = path.join(pages, 'pr', '42', 'visual', HEAD, String(runId), '1');
+ const deltas = [];
+ fs.mkdirSync(path.join(dir, 'after'), {recursive: true});
+ for (let index = 0; index < 520; index += 1) {
+ const key = `story-${index}__neutral-light`;
+ fs.writeFileSync(path.join(dir, 'after', `${key}.png`), png(index % 255));
+ deltas.push({
+ key,
+ kind: 'added',
+ beforeSha256: null,
+ shot: {...SHOT, storyId: `story-${index}`},
+ });
+ }
+ writeJSON(path.join(dir, 'evidence.json'), {
+ version: 1,
+ repo: 'facebook/astryx',
+ pr: 42,
+ headSha: HEAD,
+ testedSha: TESTED,
+ baseSha: 'd'.repeat(40),
+ run: {id: runId, attempt: 1},
+ capture: {
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ },
+ deltas,
+ verdict: {version: 1, status: 'changed'},
+ });
+ run('accept', acceptanceFlags({'run-id': runId}));
+ const record = JSON.parse(fs.readFileSync(acceptanceFile(runId), 'utf8'));
+ expect(record.keys).toHaveLength(520);
+ });
+
+ it('emits an exact recapture plan without allowing stored metadata to replace the key', () => {
+ run('accept', acceptanceFlags());
+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));
+ record.keys[0].shot.key = '../wrong';
+ writeJSON(acceptanceFile(), record);
+ const output = path.join(root, 'plan.json');
+ run('plan', {acceptance: acceptanceFile(), output});
+ expect(JSON.parse(fs.readFileSync(output, 'utf8'))).toEqual([
+ {...SHOT, key: KEY},
+ ]);
+ });
+
+ it('feeds the accepted shots back into the capture CLI unchanged', () => {
+ run('accept', acceptanceFlags());
+ const output = path.join(root, 'plan.json');
+ run('plan', {acceptance: acceptanceFile(), output});
+ const printed = execFileSync(
+ process.execPath,
+ [GATE, 'plan', '--plan-file', output, '--json'],
+ {
+ encoding: 'utf8',
+ },
+ );
+ expect(JSON.parse(printed)).toEqual([{...SHOT, key: KEY}]);
+ });
+
+ it('promotes only when merged pixels equal the reviewed after image', () => {
+ run('accept', acceptanceFlags());
+ const after = fs.readFileSync(
+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
+ );
+ const capture = path.join(root, 'capture');
+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), after);
+ writeJSON(path.join(capture, 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ capturedAt: '2026-08-26T22:00:00.000Z',
+ context: {sha: MERGE},
+ shots: {[KEY]: {...SHOT, sha256: digest(after), width: 2, height: 2}},
+ });
+
+ run('promote', {
+ pages,
+ acceptance: acceptanceFile(),
+ capture,
+ 'merge-sha': MERGE,
+ });
+ run('promote', {
+ pages,
+ acceptance: acceptanceFile(),
+ capture,
+ 'merge-sha': MERGE,
+ });
+ expect(
+ fs.readFileSync(
+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
+ ),
+ ).toEqual(after);
+ const manifest = JSON.parse(
+ fs.readFileSync(
+ path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
+ 'utf8',
+ ),
+ );
+ expect(manifest.context).toEqual({sha: MERGE});
+ expect(manifest.decisions).toHaveLength(1);
+ expect(manifest.decisions.at(-1)).toMatchObject({
+ pr: 42,
+ headSha: HEAD,
+ mergeSha: MERGE,
+ });
+ });
+
+ it('rejects changed post-merge pixels and a stale merge identity', () => {
+ run('accept', acceptanceFlags());
+ const capture = path.join(root, 'capture');
+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), png(0, 255, 0));
+ writeJSON(path.join(capture, 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ context: {sha: 'e'.repeat(40)},
+ shots: {
+ [KEY]: {...SHOT, sha256: digest(png(0, 255, 0)), width: 2, height: 2},
+ },
+ });
+ expect(
+ fail('promote', {
+ pages,
+ acceptance: acceptanceFile(),
+ capture,
+ 'merge-sha': MERGE,
+ }),
+ ).toMatch(/capture was produced for/);
+ });
+
+ it('rejects a concurrent baseline change instead of overwriting it', () => {
+ run('accept', acceptanceFlags());
+ fs.writeFileSync(
+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
+ png(0, 255, 0),
+ );
+ expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(
+ {
+ state: 'pending',
+ reason: 'capture',
+ },
+ );
+ const capture = path.join(root, 'capture');
+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
+ const accepted = fs.readFileSync(
+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),
+ );
+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), accepted);
+ writeJSON(path.join(capture, 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ context: {sha: MERGE},
+ shots: {[KEY]: {...SHOT, sha256: digest(accepted), width: 2, height: 2}},
+ });
+ expect(
+ fail('promote', {
+ pages,
+ acceptance: acceptanceFile(),
+ capture,
+ 'merge-sha': MERGE,
+ }),
+ ).toMatch(/baseline conflict/);
+ });
+
+ it('records added and removed shots without inventing missing images', () => {
+ const added = 'core-new--default__neutral-light';
+ writeEvidence({kind: 'added', key: added, run: 124});
+ run('accept', acceptanceFlags({'run-id': 124}));
+ let record = JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8'));
+ expect(record.keys[0]).toMatchObject({
+ key: added,
+ kind: 'added',
+ beforeSha256: null,
+ });
+
+ fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {
+ recursive: true,
+ force: true,
+ });
+ writeEvidence({kind: 'removed', run: 125});
+ run('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}));
+ record = JSON.parse(fs.readFileSync(acceptanceFile(125), 'utf8'));
+ expect(record.keys[0]).toMatchObject({
+ key: KEY,
+ kind: 'removed',
+ afterSha256: null,
+ shot: null,
+ });
+ });
+});
diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
new file mode 100644
index 0000000000..5438e7588a
--- /dev/null
+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
@@ -0,0 +1,69 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import {describe, expect, it} from 'vitest';
+
+const ROOT = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '../../..',
+);
+const WORKFLOWS = path.join(ROOT, '.github', 'workflows');
+
+function workflow(name) {
+ return fs.readFileSync(path.join(WORKFLOWS, name), 'utf8');
+}
+
+describe('visual acceptance workflow concurrency', () => {
+ it('locks PR report resolution before any mutation', () => {
+ const value = workflow('pr-comment.yml');
+ const [header, jobs] = value.split('\njobs:\n');
+
+ expect(header).toContain(
+ 'group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}',
+ );
+ expect(header).toContain(
+ "cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}",
+ );
+ expect(jobs).not.toContain('visual-acceptance-pr-');
+ expect(jobs).not.toContain(' concurrency:');
+ });
+
+ it('keeps comment authorization read-only until the shared lock is held', () => {
+ const value = workflow('visual-acceptance.yml');
+ const initialize = value.slice(
+ value.indexOf(' initialize:'),
+ value.indexOf(' authorize:'),
+ );
+ const authorize = value.slice(
+ value.indexOf(' authorize:'),
+ value.indexOf(' accept:'),
+ );
+ const accept = value.slice(value.indexOf(' accept:'));
+
+ expect(initialize).toContain(
+ 'group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}',
+ );
+ expect(initialize).toContain('cancel-in-progress: true');
+ expect(authorize).not.toContain(': write');
+ expect(authorize).not.toContain('actions/checkout');
+ expect(authorize).not.toContain('issues.createComment');
+ expect(authorize).not.toContain('createCommitStatus');
+ expect(accept).toContain('needs: authorize');
+ expect(accept).toContain(
+ 'group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}',
+ );
+ expect(accept).toContain('cancel-in-progress: false');
+ });
+
+ it('uses the same head identity for post-merge promotion', () => {
+ const value = workflow('visual-acceptance-promote.yml');
+
+ expect(value).toContain(
+ 'group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}',
+ );
+ expect(value).not.toContain('visual-acceptance-pr-');
+ });
+});
diff --git a/.github/scripts/visual-scope.mjs b/.github/scripts/visual-scope.mjs
index 94e3701945..6ccc4ddae9 100644
--- a/.github/scripts/visual-scope.mjs
+++ b/.github/scripts/visual-scope.mjs
@@ -50,7 +50,15 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {
);
const stableThemes = new Set();
+ const stableComponents = new Set();
const canaryPackages = new Set();
+ let broadStableVisual = false;
+
+ for (const file of stableCoreFiles) {
+ const match = file.match(/^packages\/core\/src\/([^/]+)\//);
+ if (match && /^[A-Z]/.test(match[1])) stableComponents.add(match[1]);
+ else broadStableVisual = true;
+ }
for (const file of paths) {
const pkgMatch = file.match(/^packages\/([^/]+)\//);
@@ -70,19 +78,27 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {
themeMatch[1],
'package.json',
);
- const pkg = readPackage(repoRoot, relativeManifest, manifests);
- if (!pkg) continue;
- if (pkg.private === true || pkg.astryx?.canaryOnly === true) {
- if (pkg.astryx?.canaryOnly === true) {
- canaryPackages.add(pkg.name ?? themeMatch[1]);
- }
+ const baseManifest = path.join(repoRoot, relativeManifest);
+ const basePkg = fs.existsSync(baseManifest) ? readJSON(baseManifest) : null;
+ const headPkg = Object.hasOwn(manifests, relativeManifest)
+ ? manifests[relativeManifest]
+ : basePkg;
+ const isStable = pkg => pkg && pkg.private !== true && pkg.astryx?.canaryOnly !== true;
+ // Base metadata is the fail-closed floor: a PR may promote a theme into the
+ // stable lane, but cannot escape review by demoting an existing stable one.
+ if (isStable(basePkg) || isStable(headPkg)) {
+ stableThemes.add(themeMatch[1]);
continue;
}
- stableThemes.add(themeMatch[1]);
+ if (headPkg?.astryx?.canaryOnly === true) {
+ canaryPackages.add(headPkg.name ?? themeMatch[1]);
+ }
}
return {
hasStableVisual: stableCoreFiles.length > 0 || stableThemes.size > 0,
+ broadStableVisual,
+ stableComponents: [...stableComponents].sort(),
stableCoreFiles,
stableThemes: [...stableThemes].sort(),
canaryPackages: [...canaryPackages].sort(),
diff --git a/.github/scripts/visual-scope.test.mjs b/.github/scripts/visual-scope.test.mjs
index bde97e7dfc..dc42e6c175 100644
--- a/.github/scripts/visual-scope.test.mjs
+++ b/.github/scripts/visual-scope.test.mjs
@@ -50,10 +50,23 @@ afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
describe('classifyVisualScope', () => {
it('includes stable Core runtime changes', () => {
const result = classifyVisualScope(['packages/core/src/Button/Button.tsx'], root);
- expect(result.hasStableVisual).toBe(true);
+ expect(result).toMatchObject({
+ hasStableVisual: true,
+ stableComponents: ['Button'],
+ broadStableVisual: false,
+ });
expect(result.stableCoreFiles).toEqual(['packages/core/src/Button/Button.tsx']);
});
+ it('marks shared Core infrastructure as broad stable scope', () => {
+ const result = classifyVisualScope(['packages/core/src/theme/Theme.tsx'], root);
+ expect(result).toMatchObject({
+ hasStableVisual: true,
+ broadStableVisual: true,
+ stableComponents: [],
+ });
+ });
+
it('ignores Core tests and component docs — they do not change pixels', () => {
const result = classifyVisualScope(
[
@@ -73,6 +86,21 @@ describe('classifyVisualScope', () => {
expect(result).toMatchObject({hasStableVisual: true, stableThemes: ['neutral']});
});
+ it('keeps a base-stable theme in scope when the PR marks it private', () => {
+ const result = classifyVisualScope(
+ ['packages/themes/neutral/package.json'],
+ root,
+ {
+ 'packages/themes/neutral/package.json': {
+ name: '@astryxdesign/theme-neutral',
+ private: true,
+ astryx: {canaryOnly: true},
+ },
+ },
+ );
+ expect(result).toMatchObject({hasStableVisual: true, stableThemes: ['neutral']});
+ });
+
it('uses trusted PR-head metadata for a promoted theme', () => {
const result = classifyVisualScope(
['packages/themes/probe/package.json'],
diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml
index c5d85a84a5..6c7b43960f 100644
--- a/.github/workflows/pr-comment.yml
+++ b/.github/workflows/pr-comment.yml
@@ -9,34 +9,102 @@
# write), so it can comment on fork PRs too — one code path for every PR, no
# drift between a fork path and a same-repo path.
#
-# Safety: resolves the PR from the GitHub API, then cross-checks every artifact's
-# claimed PR/head/run identity. Visual evidence is untrusted: default-branch
-# code schema-validates it, rejects unsafe paths/symlinks, decodes and
-# re-encodes PNGs, and generates the HTML wrapper itself. PR-produced HTML/JS
-# is never published or executed. Do not add checkout/execution of the PR head.
+# Safety: resolves scope and PR identity from the GitHub API. Default-branch
+# code builds the capture plan, serves the PR's static Storybook to an isolated
+# browser with off-origin requests blocked, derives the comparison, re-encodes
+# PNGs, and generates the report. PR code is never checked out, imported into
+# Node, or given credentials; PR-produced HTML/JS is never published.
name: PR Comment
on:
workflow_run:
- workflows: ["CI"]
- types: [completed]
+ workflows: ['CI']
+ types: [requested, in_progress, completed]
permissions: {}
-# A newer run for the same branch supersedes an older comment/publication. The
-# trusted identity step also fails a stale run after the PR head moves.
+# The event already carries a stable head-repository/branch identity. Acquire
+# this shared lock before resolving the PR so a rerun can cancel any older
+# acceptance, report publication, or promotion mutation for the same head.
concurrency:
- group: pr-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}
- cancel-in-progress: true
+ group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}
+ cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}
jobs:
- comment:
- # Only for CI runs triggered by a pull request.
+ resolve:
if: github.event.workflow_run.event == 'pull_request'
runs-on: ubuntu-slim
+ permissions:
+ pull-requests: read
+ outputs:
+ valid: ${{ steps.identity.outputs.valid }}
+ pr_number: ${{ steps.identity.outputs.pr_number }}
+ head_sha: ${{ steps.identity.outputs.head_sha }}
+ steps:
+ - name: Resolve trusted PR identity
+ id: identity
+ uses: actions/github-script@v9
+ with:
+ script: |
+ const run = context.payload.workflow_run;
+ const {owner, repo} = context.repo;
+ let numbers = (run.pull_requests || []).map((pr) => pr.number);
+ if (numbers.length === 0) {
+ const associated = await github.paginate(
+ github.rest.repos.listPullRequestsAssociatedWithCommit,
+ {owner, repo, commit_sha: run.head_sha, per_page: 100},
+ );
+ numbers = associated.filter((pr) => pr.state === 'open').map((pr) => pr.number);
+ }
+ numbers = [...new Set(numbers)];
+ if (numbers.length !== 1) {
+ core.setFailed(`Expected exactly one open PR for run ${run.id}; found ${numbers.join(',') || 'none'}`);
+ return;
+ }
+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: numbers[0]});
+ if (pr.head.sha !== run.head_sha) {
+ core.setFailed(`Run head ${run.head_sha} is stale; PR head is ${pr.head.sha}`);
+ return;
+ }
+ core.setOutput('valid', 'true');
+ core.setOutput('pr_number', String(pr.number));
+ core.setOutput('head_sha', pr.head.sha);
+
+ invalidate:
+ needs: resolve
+ if: >-
+ (github.event.action == 'requested' || github.event.action == 'in_progress') &&
+ needs.resolve.outputs.valid == 'true'
+ runs-on: ubuntu-slim
+ permissions:
+ statuses: write
+ steps:
+ - name: Mark same-head reruns pending before evidence can change
+ uses: actions/github-script@v9
+ with:
+ retries: 3
+ script: |
+ const run = context.payload.workflow_run;
+ await github.rest.repos.createCommitStatus({
+ ...context.repo,
+ sha: '${{ needs.resolve.outputs.head_sha }}',
+ context: 'visual-acceptance',
+ state: 'pending',
+ description: `CI run ${run.id}/${run.run_attempt} is producing fresh visual evidence.`,
+ });
+
+ comment:
+ needs: resolve
+ # Only for completed CI runs whose PR identity was resolved above.
+ if: >-
+ github.event.action == 'completed' &&
+ needs.resolve.outputs.valid == 'true'
+ runs-on: 2-core-ubuntu-arm
permissions:
pull-requests: write
+ issues: write
+ statuses: write
actions: read
contents: write # immutable validated visual evidence on gh-pages
steps:
@@ -89,9 +157,60 @@ jobs:
core.setOutput('pr_number', String(pr.number));
core.setOutput('head_sha', pr.head.sha);
core.setOutput('head_repo', pr.head.repo.full_name);
+ core.setOutput('base_sha', pr.base.sha);
core.setOutput('run_id', String(run.id));
core.setOutput('run_attempt', String(run.run_attempt));
+ - name: Derive trusted stable visual scope
+ id: scope
+ if: steps.identity.outputs.valid == 'true'
+ uses: actions/github-script@v9
+ with:
+ script: |
+ const fs = require('node:fs');
+ const {spawnSync} = require('node:child_process');
+ const {owner, repo} = context.repo;
+ const pullNumber = Number('${{ steps.identity.outputs.pr_number }}');
+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: pullNumber});
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ owner, repo, pull_number: pullNumber, per_page: 100,
+ });
+ const manifestPaths = new Set();
+ for (const {filename} of files) {
+ const theme = filename.match(/^packages\/themes\/([^/]+)\//);
+ const pkg = filename.match(/^packages\/([^/]+)\//);
+ if (theme) manifestPaths.add(`packages/themes/${theme[1]}/package.json`);
+ else if (pkg) manifestPaths.add(`packages/${pkg[1]}/package.json`);
+ }
+ const manifests = {};
+ const [headOwner, headRepo] = pr.head.repo.full_name.split('/');
+ for (const manifestPath of manifestPaths) {
+ try {
+ const {data} = await github.rest.repos.getContent({
+ owner: headOwner, repo: headRepo, path: manifestPath, ref: pr.head.sha,
+ });
+ if (!Array.isArray(data) && data.content) {
+ manifests[manifestPath] = JSON.parse(Buffer.from(data.content, 'base64').toString());
+ }
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+ }
+ fs.writeFileSync('visual-scope-manifests.json', JSON.stringify(manifests));
+ const result = spawnSync(
+ process.execPath,
+ ['.github/scripts/visual-scope.mjs', '--manifests', 'visual-scope-manifests.json'],
+ {input: `${files.map((file) => file.filename).join('\n')}\n`, encoding: 'utf8'},
+ );
+ if (result.status !== 0) {
+ core.setFailed(result.stderr || 'Stable visual scope classification failed.');
+ return;
+ }
+ const scope = JSON.parse(result.stdout);
+ fs.writeFileSync('trusted-scope.json', JSON.stringify(scope));
+ core.setOutput('stable', String(scope.hasStableVisual));
+ core.setOutput('broad', String(scope.broadStableVisual));
+
- name: Download analysis artifact from the CI run
if: steps.identity.outputs.valid == 'true'
uses: actions/download-artifact@v8
@@ -112,15 +231,15 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
continue-on-error: true
- - name: Download visual verdict from the CI run
- if: steps.identity.outputs.valid == 'true'
+ - name: Download Storybook for trusted visual capture
+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'
uses: actions/download-artifact@v8
with:
- name: visual-pr-report
- path: visual
+ pattern: storybook-*
+ path: apps/storybook/dist
+ merge-multiple: true
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- continue-on-error: true
- name: Cross-check artifact identity
if: steps.identity.outputs.valid == 'true' && hashFiles('pr-analysis/pr-meta.json') != ''
@@ -140,12 +259,40 @@ jobs:
if (String(meta.runId) !== process.env.EXPECTED_RUN) fail('artifact run id does not match workflow_run');
NODE
- # The artifact is untrusted. Default-branch code validates its schema,
- # keys, identity and PNG bytes, decodes/re-encodes every image, and
- # generates the HTML wrapper. PR-produced HTML/JS is never published.
- - name: Validate visual evidence and generate trusted report
+ - name: Fetch the trusted visual baseline
+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'
+ run: |
+ rm -rf /tmp/gh-pages
+ git clone --depth=1 --single-branch --branch gh-pages \
+ "https://github.com/${GITHUB_REPOSITORY}.git" /tmp/gh-pages
+
+ - name: Capture the trusted stable visual scope
+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'
+ env:
+ GITHUB_SHA: ${{ steps.identity.outputs.head_sha }}
+ GITHUB_RUN_ID: ${{ steps.identity.outputs.run_id }}
+ GITHUB_RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
+ ASTRYX_PR_HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
+ ASTRYX_PR_BASE_SHA: ${{ steps.identity.outputs.base_sha }}
+ run: |
+ npx playwright install chromium
+ node .github/scripts/visual-gate/visual-acceptance.mjs trusted-plan \
+ --scope trusted-scope.json \
+ --baseline /tmp/gh-pages/visual-gate/baseline \
+ --storybook-dir apps/storybook/dist \
+ --output trusted-plan.json
+ node .github/scripts/visual-gate/gate.mjs capture \
+ --storybook-dir apps/storybook/dist \
+ --out trusted-capture \
+ --plan-file trusted-plan.json
+
+ # The Storybook bundle is untrusted. Default-branch code chooses the plan,
+ # captures it in an off-origin-blocked browser, derives the comparison,
+ # validates/re-encodes PNGs, and generates the report. PR-authored verdicts
+ # and job conclusions are never authority for the required status.
+ - name: Derive trusted visual evidence and report
id: visual
- if: steps.identity.outputs.valid == 'true' && hashFiles('visual/verdict.json') != ''
+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'
env:
PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
@@ -153,14 +300,16 @@ jobs:
RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
run: |
node .github/scripts/visual-gate/publish-pr-report.mjs \
- --input visual \
+ --input trusted-capture \
--output trusted-visual \
+ --baseline /tmp/gh-pages/visual-gate/baseline \
+ --scope trusted-scope.json \
--pr "$PR_NUMBER" \
--head-sha "$HEAD_SHA" \
--run-id "$RUN_ID" \
--run-attempt "$RUN_ATTEMPT"
echo "ready=true" >> "$GITHUB_OUTPUT"
- echo "path=pr/${PR_NUMBER}/visual/${HEAD_SHA}/${RUN_ID}" >> "$GITHUB_OUTPUT"
+ echo "path=pr/${PR_NUMBER}/visual/${HEAD_SHA}/${RUN_ID}/${RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- name: Publish immutable visual evidence
id: publish
@@ -247,9 +396,9 @@ jobs:
'.github/scripts/generate-pr-comment.js',
'--analysis', 'pr-analysis/analysis.json',
'--a11y', a11yPath,
- ...(fs.existsSync('visual/verdict.json')
+ ...(fs.existsSync('trusted-visual/verdict.json')
? [
- '--visual', 'visual/verdict.json',
+ '--visual', 'trusted-visual/verdict.json',
...(process.env.VISUAL_PUBLISHED === 'true'
? [
'--visual-report-url',
@@ -288,3 +437,72 @@ jobs:
});
core.info(`Posted PR Analysis Report on #${prNumber}.`);
}
+
+ - name: Evaluate visual acceptance state
+ if: always() && steps.identity.outputs.valid == 'true'
+ env:
+ SCOPE_OUTCOME: ${{ steps.scope.outcome }}
+ HAS_STABLE_VISUAL: ${{ steps.scope.outputs.stable }}
+ VISUAL_READY: ${{ steps.visual.outputs.ready }}
+ VISUAL_PUBLISHED: ${{ steps.publish.outputs.published }}
+ PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
+ HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
+ run: |
+ if [ "$SCOPE_OUTCOME" != "success" ]; then
+ printf '%s\n' '{"state":"failure","reason":"scope","description":"Stable visual scope could not be evaluated."}' > visual-state.json
+ elif [ "$HAS_STABLE_VISUAL" != "true" ]; then
+ printf '%s\n' '{"state":"success","reason":"scope","description":"No stable visual scope."}' > visual-state.json
+ elif [ "$VISUAL_READY" != "true" ] || [ "$VISUAL_PUBLISHED" != "true" ]; then
+ printf '%s\n' '{"state":"failure","reason":"capture","description":"Stable visual evidence could not be published."}' > visual-state.json
+ else
+ node .github/scripts/visual-gate/visual-acceptance.mjs state \
+ --pages /tmp/gh-pages --pr "$PR_NUMBER" --head "$HEAD_SHA" > visual-state.json
+ fi
+
+ - name: Publish visual acceptance status
+ if: always() && steps.identity.outputs.valid == 'true'
+ uses: actions/github-script@v9
+ env:
+ PR_NUMBER: ${{ steps.identity.outputs.pr_number }}
+ HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
+ VISUAL_PATH: ${{ steps.visual.outputs.path }}
+ with:
+ retries: 3
+ script: |
+ const fs = require('node:fs');
+ const {owner, repo} = context.repo;
+ const pr = Number(process.env.PR_NUMBER);
+ let state = {state: 'failure', reason: 'state', description: 'Visual acceptance state could not be evaluated.'};
+ if (fs.existsSync('visual-state.json')) {
+ try {
+ state = JSON.parse(fs.readFileSync('visual-state.json', 'utf8'));
+ } catch (error) {
+ core.warning(`Could not read visual state: ${error.message}`);
+ }
+ }
+ await github.rest.repos.createCommitStatus({
+ owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance',
+ state: state.state, description: state.description.slice(0, 140),
+ ...(process.env.VISUAL_PATH
+ ? {target_url: `https://facebook.github.io/astryx/${process.env.VISUAL_PATH}/`}
+ : {}),
+ });
+ const label = 'visual-approved';
+ if (state.reason === 'accepted') {
+ try {
+ await github.rest.issues.getLabel({owner, repo, name: label});
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ await github.rest.issues.createLabel({
+ owner, repo, name: label, color: '2da44e',
+ description: 'Current visual bundle has an explicit acceptance record',
+ });
+ }
+ await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});
+ } else {
+ try {
+ await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+ }
diff --git a/.github/workflows/visual-acceptance-promote.yml b/.github/workflows/visual-acceptance-promote.yml
new file mode 100644
index 0000000000..faa7053407
--- /dev/null
+++ b/.github/workflows/visual-acceptance-promote.yml
@@ -0,0 +1,188 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# Promote a PR acceptance only after the merged commit reproduces every reviewed
+# AFTER frame. The job is serialized with manual baseline promotion, and each
+# key carries its baseline preimage hash so concurrent decisions cannot overwrite
+# one another silently.
+
+name: Visual acceptance promotion
+
+on:
+ pull_request_target:
+ branches: ['main']
+ types: [closed]
+
+permissions: {}
+
+concurrency:
+ # Shares the head-branch lock with rerun invalidation and acceptance. A new
+ # requested run cancels an older promotion before it can publish stale pixels;
+ # cross-PR baseline races are rejected by preimage hashes and push retries.
+ group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}
+ cancel-in-progress: false
+
+jobs:
+ promote:
+ if: github.event.pull_request.merged == true
+ runs-on: 2-core-ubuntu-arm
+ timeout-minutes: 20
+ permissions:
+ actions: write
+ contents: write
+ statuses: write
+ steps:
+ - name: Checkout the merged result
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ github.event.pull_request.merge_commit_sha }}
+
+ - name: Find an acceptance for the merged head
+ id: acceptance
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: |
+ set -eu
+ PAGES="$RUNNER_TEMP/visual-acceptance-pages"
+ git clone --depth=1 --single-branch --branch gh-pages \
+ "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$PAGES"
+ ROOT="$PAGES/visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"
+ POINTER="$ROOT/current.json"
+ if [ ! -f "$POINTER" ]; then
+ echo "No visual acceptance was recorded for this merged head."
+ echo "found=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ RECORD_REL=$(jq -r '.record // empty' "$POINTER")
+ if ! printf '%s' "$RECORD_REL" | grep -Eq '^[0-9]+/[0-9]+/acceptance\.json$'; then
+ echo "::error::Visual acceptance pointer is invalid"
+ exit 1
+ fi
+ RECORD="$ROOT/$RECORD_REL"
+ if [ ! -f "$RECORD" ]; then
+ echo "::error::Visual acceptance record is missing"
+ exit 1
+ fi
+ {
+ echo "found=true"
+ echo "record=$RECORD"
+ echo "record_rel=$RECORD_REL"
+ } >> "$GITHUB_OUTPUT"
+
+ - name: Setup Node and pnpm
+ if: steps.acceptance.outputs.found == 'true'
+ uses: ./.github/actions/setup
+
+ - name: Build the merged Storybook
+ if: steps.acceptance.outputs.found == 'true'
+ run: pnpm build && pnpm -F @astryxdesign/storybook build
+
+ - name: Install Chromium
+ if: steps.acceptance.outputs.found == 'true'
+ run: npx playwright install chromium
+
+ - name: Recapture exactly the accepted shots
+ if: steps.acceptance.outputs.found == 'true'
+ env:
+ GITHUB_SHA: ${{ github.event.pull_request.merge_commit_sha }}
+ ACCEPTANCE: ${{ steps.acceptance.outputs.record }}
+ run: |
+ set -eu
+ node .github/scripts/visual-gate/visual-acceptance.mjs plan \
+ --acceptance "$ACCEPTANCE" \
+ --output accepted-plan.json
+ node .github/scripts/visual-gate/gate.mjs capture \
+ --storybook-dir apps/storybook/dist \
+ --out .visual-merged \
+ --plan-file accepted-plan.json
+
+ - name: Verify and promote the baseline
+ id: promote
+ if: steps.acceptance.outputs.found == 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}
+ RECORD_REL: ${{ steps.acceptance.outputs.record_rel }}
+ run: |
+ set -eu
+ REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
+ PAGES="$RUNNER_TEMP/visual-promote-pages"
+ RECORD="visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}/${RECORD_REL}"
+ for attempt in 1 2 3 4 5; do
+ rm -rf "$PAGES"
+ git clone --depth=1 --single-branch --branch gh-pages "$REPO_URL" "$PAGES"
+ CURRENT_REL=$(jq -r '.record // empty' \
+ "$PAGES/visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}/current.json")
+ if [ "$CURRENT_REL" != "$RECORD_REL" ]; then
+ echo "::error::A newer visual acceptance superseded the reviewed record"
+ exit 1
+ fi
+ LATEST=$(gh api \
+ "repos/${GITHUB_REPOSITORY}/actions/runs?event=pull_request&head_sha=${HEAD_SHA}&per_page=100" \
+ --jq '[.workflow_runs[] | select(.name == "CI")][0] | [.id, .run_attempt, .status] | @tsv')
+ IFS=$'\t' read -r LATEST_RUN LATEST_ATTEMPT LATEST_STATUS <<< "$LATEST"
+ ACCEPTED_RUN=$(jq -r '.run.id' "$PAGES/$RECORD")
+ ACCEPTED_ATTEMPT=$(jq -r '.run.attempt' "$PAGES/$RECORD")
+ if [ "$LATEST_RUN" != "$ACCEPTED_RUN" ] || \
+ [ "$LATEST_ATTEMPT" != "$ACCEPTED_ATTEMPT" ] || \
+ [ "$LATEST_STATUS" != "completed" ]; then
+ echo "::error::Accepted evidence is not from the latest completed CI attempt"
+ exit 1
+ fi
+ node .github/scripts/visual-gate/visual-acceptance.mjs state \
+ --pages "$PAGES" --pr "$PR_NUMBER" --head "$HEAD_SHA" > "$RUNNER_TEMP/promotion-state.json"
+ if [ "$(jq -r '.state + ":" + .reason' "$RUNNER_TEMP/promotion-state.json")" != "success:accepted" ]; then
+ echo "::error::Accepted evidence is no longer the latest visual result"
+ exit 1
+ fi
+ node .github/scripts/visual-gate/visual-acceptance.mjs promote \
+ --pages "$PAGES" \
+ --acceptance "$PAGES/$RECORD" \
+ --capture .visual-merged \
+ --merge-sha "$MERGE_SHA"
+ git -C "$PAGES" config user.name "github-actions[bot]"
+ git -C "$PAGES" config user.email "github-actions[bot]@users.noreply.github.com"
+ git -C "$PAGES" add visual-gate/baseline
+ if git -C "$PAGES" diff --cached --quiet; then
+ echo "Baseline already contains the accepted pixels."
+ exit 0
+ fi
+ git -C "$PAGES" commit -m "visual baseline: accepted PR #${PR_NUMBER}"
+ if git -C "$PAGES" push origin gh-pages; then
+ exit 0
+ fi
+ sleep $((attempt * 2))
+ done
+ echo "::error::Could not promote the accepted baseline after 5 attempts"
+ exit 1
+
+ - name: Run a fresh release gate
+ if: steps.promote.outcome == 'success'
+ uses: actions/github-script@v9
+ with:
+ retries: 3
+ script: |
+ await github.rest.actions.createWorkflowDispatch({
+ ...context.repo,
+ workflow_id: 'release-gate.yml',
+ ref: context.payload.repository.default_branch,
+ });
+
+ - name: Mark a post-merge verification failure
+ if: failure() && steps.acceptance.outputs.found == 'true'
+ uses: actions/github-script@v9
+ env:
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ with:
+ retries: 3
+ script: |
+ await github.rest.repos.createCommitStatus({
+ ...context.repo,
+ sha: process.env.HEAD_SHA,
+ context: 'visual-acceptance',
+ state: 'failure',
+ description: 'Merged pixels did not promote to the visual baseline.',
+ });
diff --git a/.github/workflows/visual-acceptance.yml b/.github/workflows/visual-acceptance.yml
new file mode 100644
index 0000000000..d9eab24bc1
--- /dev/null
+++ b/.github/workflows/visual-acceptance.yml
@@ -0,0 +1,394 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# Required status and explicit human decision for stable visual changes.
+# This workflow never checks out or executes PR-controlled code. It runs the
+# default-branch implementation, resolves the current PR head through GitHub,
+# and stores the signed decision beside trusted, immutable evidence on gh-pages.
+
+name: Visual acceptance
+
+on:
+ pull_request_target:
+ branches: ['main']
+ types: [opened, reopened, synchronize]
+ issue_comment:
+ types: [created]
+
+permissions: {}
+
+jobs:
+ initialize:
+ if: github.event_name == 'pull_request_target'
+ concurrency:
+ group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}
+ cancel-in-progress: true
+ runs-on: ubuntu-slim
+ permissions:
+ contents: read
+ pull-requests: read
+ issues: write
+ statuses: write
+ steps:
+ - name: Checkout trusted default-branch code
+ uses: actions/checkout@v7
+
+ - name: Classify scope and initialize the required status
+ uses: actions/github-script@v9
+ with:
+ retries: 3
+ script: |
+ const fs = require('node:fs');
+ const {spawnSync} = require('node:child_process');
+ const {owner, repo} = context.repo;
+ const pr = context.payload.pull_request;
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ owner, repo, pull_number: pr.number, per_page: 100,
+ });
+ const manifestPaths = new Set();
+ for (const {filename} of files) {
+ const theme = filename.match(/^packages\/themes\/([^/]+)\//);
+ const pkg = filename.match(/^packages\/([^/]+)\//);
+ if (theme) manifestPaths.add(`packages/themes/${theme[1]}/package.json`);
+ else if (pkg) manifestPaths.add(`packages/${pkg[1]}/package.json`);
+ }
+ const manifests = {};
+ const [headOwner, headRepo] = pr.head.repo.full_name.split('/');
+ for (const manifestPath of manifestPaths) {
+ try {
+ const {data} = await github.rest.repos.getContent({
+ owner: headOwner, repo: headRepo, path: manifestPath, ref: pr.head.sha,
+ });
+ if (!Array.isArray(data) && data.content) {
+ manifests[manifestPath] = JSON.parse(Buffer.from(data.content, 'base64').toString());
+ }
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+ }
+ fs.writeFileSync('visual-scope-manifests.json', JSON.stringify(manifests));
+ const result = spawnSync(
+ process.execPath,
+ ['.github/scripts/visual-scope.mjs', '--manifests', 'visual-scope-manifests.json'],
+ {input: `${files.map((file) => file.filename).join('\n')}\n`, encoding: 'utf8'},
+ );
+ if (result.status !== 0) {
+ core.setFailed(result.stderr || 'Stable visual scope classification failed.');
+ return;
+ }
+ const scope = JSON.parse(result.stdout);
+ await github.rest.repos.createCommitStatus({
+ owner,
+ repo,
+ sha: pr.head.sha,
+ context: 'visual-acceptance',
+ state: scope.hasStableVisual ? 'pending' : 'success',
+ description: scope.hasStableVisual
+ ? 'Stable visual capture is still running.'
+ : 'No stable visual scope.',
+ });
+ try {
+ await github.rest.issues.removeLabel({
+ owner, repo, issue_number: pr.number, name: 'visual-approved',
+ });
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+
+ authorize:
+ if: >-
+ github.event_name == 'issue_comment' &&
+ github.event.issue.pull_request &&
+ startsWith(github.event.comment.body, '/accept-visual')
+ runs-on: ubuntu-slim
+ permissions:
+ actions: read
+ contents: read
+ pull-requests: read
+ outputs:
+ resolved: ${{ steps.authorize.outputs.resolved }}
+ valid: ${{ steps.authorize.outputs.valid }}
+ refusal: ${{ steps.authorize.outputs.refusal }}
+ pr: ${{ steps.authorize.outputs.pr }}
+ head: ${{ steps.authorize.outputs.head }}
+ head_repo_id: ${{ steps.authorize.outputs.head_repo_id }}
+ head_ref: ${{ steps.authorize.outputs.head_ref }}
+ run_id: ${{ steps.authorize.outputs.run_id }}
+ run_attempt: ${{ steps.authorize.outputs.run_attempt }}
+ approver: ${{ steps.authorize.outputs.approver }}
+ approver_id: ${{ steps.authorize.outputs.approver_id }}
+ permission: ${{ steps.authorize.outputs.permission }}
+ comment_id: ${{ steps.authorize.outputs.comment_id }}
+ reason: ${{ steps.authorize.outputs.reason }}
+ steps:
+ # This job is deliberately read-only. The mutation job acquires the shared
+ # head-branch lock using these trusted API-derived outputs.
+ - name: Authorize and resolve the decision
+ id: authorize
+ uses: actions/github-script@v9
+ with:
+ retries: 3
+ script: |
+ const {owner, repo} = context.repo;
+ const number = context.issue.number;
+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: number});
+ if (!pr.head.repo?.id || !pr.head.ref) {
+ core.setFailed('The pull request head repository or branch is unavailable.');
+ return;
+ }
+ core.setOutput('resolved', 'true');
+ core.setOutput('pr', String(number));
+ core.setOutput('head', pr.head.sha);
+ core.setOutput('head_repo_id', String(pr.head.repo.id));
+ core.setOutput('head_ref', pr.head.ref);
+ const refuse = (message) => {
+ core.setOutput('valid', 'false');
+ core.setOutput('refusal', message);
+ };
+ const body = context.payload.comment.body.trim();
+ const match = body.match(/^\/accept-visual\s+(\d+)\/(\d+)\s+([\s\S]+)$/);
+ if (!match) {
+ refuse('copy the exact `/accept-visual <run>/<attempt> <reason>` command from the report.');
+ return;
+ }
+ const runId = Number(match[1]);
+ const runAttempt = Number(match[2]);
+ const reason = match[3].trim();
+ if (reason.length < 12 || reason.length > 500 ||
+ /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(reason)) {
+ refuse('the reason must explain why the pixels are correct (12–500 characters).');
+ return;
+ }
+ if (pr.state !== 'open' || pr.base.ref !== 'main') {
+ refuse('the pull request is not open against `main`.');
+ return;
+ }
+ const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {
+ owner, repo, event: 'pull_request', head_sha: pr.head.sha, per_page: 100,
+ });
+ const latestCI = runs
+ .filter((run) => run.name === 'CI')
+ .sort((a, b) => b.id - a.id)[0];
+ if (!latestCI || latestCI.id !== runId || latestCI.run_attempt !== runAttempt ||
+ latestCI.status !== 'completed') {
+ refuse('that evidence is not from the latest completed CI attempt for this head.');
+ return;
+ }
+ let permission = 'none';
+ try {
+ const response = await github.rest.repos.getCollaboratorPermissionLevel({
+ owner, repo, username: context.actor,
+ });
+ permission = response.data.user.permission;
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+ if (!['maintain', 'admin'].includes(permission)) {
+ refuse('only a repository maintainer may accept stable visual changes.');
+ return;
+ }
+ core.setOutput('valid', 'true');
+ core.setOutput('run_id', String(runId));
+ core.setOutput('run_attempt', String(runAttempt));
+ core.setOutput('approver', context.actor);
+ core.setOutput('approver_id', String(context.payload.comment.user.id));
+ core.setOutput('permission', permission);
+ core.setOutput('comment_id', String(context.payload.comment.id));
+ core.setOutput('reason', reason);
+
+ accept:
+ needs: authorize
+ if: >-
+ always() &&
+ needs.authorize.result == 'success' &&
+ needs.authorize.outputs.resolved == 'true'
+ concurrency:
+ group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}
+ cancel-in-progress: false
+ runs-on: ubuntu-slim
+ permissions:
+ contents: write
+ pull-requests: read
+ issues: write
+ statuses: write
+ steps:
+ - name: Report a refused decision
+ if: needs.authorize.outputs.valid != 'true'
+ uses: actions/github-script@v9
+ env:
+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}
+ REFUSAL: ${{ needs.authorize.outputs.refusal }}
+ with:
+ retries: 3
+ script: |
+ await github.rest.issues.createComment({
+ ...context.repo,
+ issue_number: Number(process.env.PR_NUMBER),
+ body: `Visual acceptance refused: ${process.env.REFUSAL}`,
+ });
+
+ - name: Checkout trusted default-branch code
+ if: needs.authorize.outputs.valid == 'true'
+ uses: actions/checkout@v7
+
+ - name: Archive the acceptance decision
+ id: record
+ if: needs.authorize.outputs.valid == 'true'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}
+ HEAD_SHA: ${{ needs.authorize.outputs.head }}
+ RUN_ID: ${{ needs.authorize.outputs.run_id }}
+ RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}
+ APPROVER: ${{ needs.authorize.outputs.approver }}
+ APPROVER_ID: ${{ needs.authorize.outputs.approver_id }}
+ PERMISSION: ${{ needs.authorize.outputs.permission }}
+ COMMENT_ID: ${{ needs.authorize.outputs.comment_id }}
+ REASON: ${{ needs.authorize.outputs.reason }}
+ run: |
+ set -eu
+ REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
+ PAGES="$RUNNER_TEMP/visual-acceptance-pages"
+ DEST="visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"
+ for attempt in 1 2 3 4 5; do
+ rm -rf "$PAGES"
+ git clone --depth=1 --single-branch --branch gh-pages "$REPO_URL" "$PAGES"
+ node .github/scripts/visual-gate/visual-acceptance.mjs accept \
+ --pages "$PAGES" \
+ --pr "$PR_NUMBER" \
+ --head "$HEAD_SHA" \
+ --run-id "$RUN_ID" \
+ --run-attempt "$RUN_ATTEMPT" \
+ --approver "$APPROVER" \
+ --approver-id "$APPROVER_ID" \
+ --permission "$PERMISSION" \
+ --comment-id "$COMMENT_ID" \
+ --reason "$REASON"
+ if [ -z "$(git -C "$PAGES" status --porcelain -- "$DEST")" ]; then
+ echo "The current visual bundle was already accepted."
+ exit 0
+ fi
+ git -C "$PAGES" config user.name "github-actions[bot]"
+ git -C "$PAGES" config user.email "github-actions[bot]@users.noreply.github.com"
+ git -C "$PAGES" add "$DEST"
+ git -C "$PAGES" commit -m "visual acceptance: PR #${PR_NUMBER} at ${HEAD_SHA}"
+ if git -C "$PAGES" push origin gh-pages; then
+ exit 0
+ fi
+ sleep $((attempt * 2))
+ done
+ echo "::error::Could not archive visual acceptance after 5 attempts"
+ exit 1
+
+ - name: Reconcile acceptance against the latest published evidence
+ id: reconcile
+ if: steps.record.outcome == 'success'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}
+ HEAD_SHA: ${{ needs.authorize.outputs.head }}
+ run: |
+ FRESH="$RUNNER_TEMP/visual-acceptance-reconcile"
+ rm -rf "$FRESH"
+ git clone --depth=1 --single-branch --branch gh-pages \
+ "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$FRESH"
+ node .github/scripts/visual-gate/visual-acceptance.mjs state \
+ --pages "$FRESH" --pr "$PR_NUMBER" --head "$HEAD_SHA" > acceptance-state.json
+ node - <<'NODE'
+ const fs = require('node:fs');
+ const state = JSON.parse(fs.readFileSync('acceptance-state.json', 'utf8'));
+ for (const key of ['state', 'reason', 'description']) {
+ fs.appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${state[key]}\n`);
+ }
+ NODE
+
+ - name: Project the reconciled decision onto the PR
+ if: steps.reconcile.outcome == 'success'
+ uses: actions/github-script@v9
+ env:
+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}
+ HEAD_SHA: ${{ needs.authorize.outputs.head }}
+ ACCEPTED_RUN_ID: ${{ needs.authorize.outputs.run_id }}
+ ACCEPTED_RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}
+ ACCEPTANCE_STATE: ${{ steps.reconcile.outputs.state }}
+ ACCEPTANCE_REASON: ${{ steps.reconcile.outputs.reason }}
+ ACCEPTANCE_DESCRIPTION: ${{ steps.reconcile.outputs.description }}
+ with:
+ retries: 3
+ script: |
+ const {owner, repo} = context.repo;
+ const pr = Number(process.env.PR_NUMBER);
+ const head = process.env.HEAD_SHA;
+ let state = process.env.ACCEPTANCE_STATE;
+ let reason = process.env.ACCEPTANCE_REASON;
+ let description = process.env.ACCEPTANCE_DESCRIPTION;
+ const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {
+ owner, repo, event: 'pull_request', head_sha: head, per_page: 100,
+ });
+ const latestCI = runs
+ .filter((run) => run.name === 'CI')
+ .sort((a, b) => b.id - a.id)[0];
+ const exactRun =
+ latestCI &&
+ latestCI.id === Number(process.env.ACCEPTED_RUN_ID) &&
+ latestCI.run_attempt === Number(process.env.ACCEPTED_RUN_ATTEMPT) &&
+ latestCI.status === 'completed';
+ if (!exactRun) {
+ state = 'pending';
+ reason = 'capture';
+ description = 'A newer CI attempt must publish trusted visual evidence.';
+ }
+ const accepted = state === 'success' && reason === 'accepted';
+ if (!['success', 'pending', 'failure'].includes(state)) {
+ core.setFailed('Reconciled visual acceptance state is invalid.');
+ return;
+ }
+ const label = 'visual-approved';
+ if (accepted) {
+ try {
+ await github.rest.issues.getLabel({owner, repo, name: label});
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ await github.rest.issues.createLabel({
+ owner, repo, name: label, color: '2da44e',
+ description: 'Current visual bundle has an explicit acceptance record',
+ });
+ }
+ await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});
+ } else {
+ try {
+ await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ }
+ }
+ await github.rest.repos.createCommitStatus({
+ owner, repo, sha: head, context: 'visual-acceptance', state,
+ description: description.slice(0, 140),
+ });
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number: pr,
+ body: accepted
+ ? `Visual changes accepted for \`${head.slice(0, 12)}\`. A new push invalidates this approval; after merge, the reviewed AFTER frames must reproduce before the baseline is promoted.`
+ : `The requested visual evidence is no longer current. Status remains pending; copy the command from the latest report.`,
+ });
+
+ - name: Mark an acceptance infrastructure failure
+ if: failure() && needs.authorize.outputs.valid == 'true'
+ uses: actions/github-script@v9
+ env:
+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}
+ HEAD_SHA: ${{ needs.authorize.outputs.head }}
+ with:
+ retries: 3
+ script: |
+ const {owner, repo} = context.repo;
+ await github.rest.repos.createCommitStatus({
+ owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance', state: 'failure',
+ description: 'Visual acceptance could not be recorded.',
+ });
+ await github.rest.issues.createComment({
+ owner, repo, issue_number: Number(process.env.PR_NUMBER),
+ body: 'Visual acceptance could not be recorded. See this workflow run for the rejected invariant or infrastructure error.',
+ });