An open source design system that's fully customizable and agent ready
TypeScript
ci(visual): require explicit reviewed baseline promotion (#5578)
Cindy Zhang08a3758d7a2a
diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.jsindex 1b719fe7f5..c2c3cd18db 100644--- a/.github/scripts/lib/visual-format.js+++ b/.github/scripts/lib/visual-format.js@@ -29,23 +29,43 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {const link = safeReportUrl? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`: '';+ const acceptanceCommand =+ verdict.context?.runId && verdict.context?.runAttempt+ ? `\n\nTo accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <reason>\``+ : '';+ const reportBase = safeReportUrl+ ? `${safeReportUrl.replace(/\/+$/, '')}/`+ : null;+ // Inline images use raw branch URLs so a delayed Pages deployment cannot+ // break evidence that is already present on gh-pages.+ const imageBase = safeUrl(imageUrl)+ ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/`+ : reportBase;if (verdict.status === 'skipped') {- return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}\n\n`;+ return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}${link}\n\n`;}if (verdict.status === 'failed') {return `### Visual Regression\n\n**Status:** ${num(verdict.counts?.failed)} shot(s) could not be captured.${link}\n\n`;}+ const added = verdict.added ?? [];+ const removed = verdict.removed ?? [];if (!verdict.changes || verdict.changes.length === 0) {- const compared = num(verdict.counts?.total) - num(verdict.counts?.added);- // A PR-scoped run shoots every story of the touched component in every- // theme that styles it, which is deeper than the daily gate's baseline- // reaches — so some shots legitimately have nothing to compare against.- // Saying "added" there reads as a problem; saying it plainly does not.- const unbaselined = num(verdict.counts?.added)- ? ` ${num(verdict.counts?.added)} shot(s) have no baseline yet and were not compared.`- : '';- return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).${unbaselined}\n\n`;+ if (added.length > 0 || removed.length > 0) {+ const frames = imageBase+ ? [...added.map(key => ({key, kind: 'after'})), ...removed.map(key => ({key, kind: 'before'}))]+ .slice(0, 3)+ .map(({key, kind}) => {+ const safeKey = encodeURIComponent(String(key));+ const label = kind === 'after' ? 'Added — After' : 'Removed — Before';+ return `<p><b>${label}</b><br><img src="${imageBase}${kind}/${safeKey}.png" width="300" alt="${label} visual regression frame"></p>`;+ })+ .join('\n')+ : '';+ return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}${acceptanceCommand}\n\n${frames}\n\n`;+ }+ const compared = num(verdict.counts?.total);+ return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).\n\n`;}const rows = verdict.changes@@ -61,16 +81,6 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {// names before they can answer whether the after is correct. The static// report is published beside the PR preview, and its first three deltas// render directly in the comment; the full report carries the rest + wipe UI.- const reportBase = safeReportUrl- ? `${safeReportUrl.replace(/\/+$/, '')}/`- : null;- // GitHub comments fetch raw branch images directly. The repository's Pages- // deployment can be delayed or errored independently of the gh-pages push;- // coupling inline evidence to that deployment produced six broken images in- // the first live demo even though every PNG existed in the branch.- const imageBase = safeUrl(imageUrl)- ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/`- : reportBase;const evidence = imageBase? verdict.changes.slice(0, 3)@@ -101,7 +111,7 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {return `### Visual Regression-**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}+**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}${acceptanceCommand}A change here is a question, not a failure: check whether the *after* is thepicture you intended. If it is, say so in the PR — the release gate's baselinediff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjsindex bab8243237..565efc9455 100644--- a/.github/scripts/lib/visual-format.test.mjs+++ b/.github/scripts/lib/visual-format.test.mjs@@ -21,20 +21,46 @@ describe('buildVisualSection', () => {expect(buildVisualSection(verdict())).toContain('No visual change across 16 compared shot(s)');});- it('calls an unbaselined shot exactly that, not an addition', () => {+ it('shows added-only evidence instead of claiming there was no visual change', () => {const section = buildVisualSection(- verdict({counts: {total: 20, unchanged: 14, changed: 0, added: 6, removed: 0, failed: 0}}),+ verdict({+ status: 'changed',+ counts: {total: 20, unchanged: 14, changed: 0, added: 1, removed: 0, failed: 0},+ added: ['new-shot'],+ removed: [],+ }),+ 'https://example.com/report/',+ 'https://example.com/images/',);- expect(section).toContain('No visual change across 14 compared shot(s)');- expect(section).toContain('6 shot(s) have no baseline yet');+ expect(section).toContain('1 added · 0 removed');+ expect(section).toContain('View the report');+ expect(section).toContain('https://example.com/images/after/new-shot.png');+ expect(section).not.toContain('No visual change');+ });++ it('shows the before image for a removed-only bundle', () => {+ const section = buildVisualSection(+ verdict({+ status: 'changed',+ counts: {total: 0, unchanged: 0, changed: 0, added: 0, removed: 1, failed: 0},+ added: [],+ removed: ['old-shot'],+ }),+ 'https://example.com/report/',+ 'https://example.com/images/',+ );+ expect(section).toContain('0 added · 1 removed');+ expect(section).toContain('https://example.com/images/before/old-shot.png');});it('states the reason for a skip, so a broad PR does not look like a pass', () => {const section = buildVisualSection(verdict({status: 'skipped', reason: '900 shots exceeds the 240-shot budget'}),+ 'https://example.com/report/',);expect(section).toContain('Skipped');expect(section).toContain('900 shots exceeds the 240-shot budget');+ expect(section).toContain('View the report');});it('renders verdict strings as one-line literal text', () => {@@ -110,6 +136,7 @@ describe('buildVisualSection', () => {verdict({status: 'changed',counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},+ context: {runId: '123456', runAttempt: '2'},changes: [{key: 'core-button--primary__y2k-light',@@ -130,6 +157,7 @@ describe('buildVisualSection', () => {expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png');expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png');expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');+ expect(section).toContain('/accept-visual 123456/2 <reason>');});it('embeds at most three deltas so the PR comment stays readable', () => {diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.mjsnew file mode 100644index 0000000000..8692e10bc1--- /dev/null+++ b/.github/scripts/visual-gate/activate-visual-acceptance.mjs@@ -0,0 +1,149 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Activate the required visual-acceptance status without stranding PRs.+ *+ * Dry-run by default. `--apply` derives each open head's scope, backfills+ * no-scope heads as success and stable-visual heads as pending, then requires+ * the context. Re-running only replaces missing or legacy grandfathered states;+ * it never overwrites a verdict produced by the live workflow.+ */++import {execFileSync} from 'node:child_process';+import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';++import {classifyVisualScope} from '../visual-scope.mjs';++const REPO = 'facebook/astryx';+const BRANCH = 'main';+const CONTEXT = 'visual-acceptance';+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');++export function buildActivationPlan(openPulls, requiredContexts) {+ const unique = [...new Map(openPulls.map(pr => [pr.head.sha, pr])).values()];+ const backfill = unique+ .filter(pr => !pr.visualStatus || pr.visualStatus.description.startsWith('Grandfathered during'))+ .map(pr => ({+ number: pr.number,+ sha: pr.head.sha,+ state: pr.hasStableVisual ? 'pending' : 'success',+ description: pr.hasStableVisual+ ? 'Stable visual result pending; rerun CI after activation.'+ : 'No stable visual scope.',+ }));+ return {backfill, addRequiredContext: !requiredContexts.includes(CONTEXT)};+}++function gh(args, input) {+ return execFileSync('gh', ['api', ...args], {+ encoding: 'utf8',+ input: input === undefined ? undefined : JSON.stringify(input),+ maxBuffer: 16 * 1024 * 1024,+ });+}++function paged(endpoint) {+ return JSON.parse(gh([endpoint, '--paginate', '--slurp'])).flat();+}++function headManifests(pr, files) {+ const paths = new Set();+ for (const file of files) {+ const theme = file.match(/^packages\/themes\/([^/]+)\//);+ const pkg = file.match(/^packages\/([^/]+)\//);+ if (theme) paths.add(`packages/themes/${theme[1]}/package.json`);+ else if (pkg) paths.add(`packages/${pkg[1]}/package.json`);+ }+ const manifests = {};+ for (const manifestPath of paths) {+ try {+ const encoded = manifestPath.split('/').map(encodeURIComponent).join('/');+ const response = JSON.parse(+ gh([`repos/${pr.head.repo.full_name}/contents/${encoded}?ref=${pr.head.sha}`]),+ );+ if (!Array.isArray(response) && response.content) {+ manifests[manifestPath] = JSON.parse(Buffer.from(response.content, 'base64').toString());+ }+ } catch (error) {+ if (!String(error.stderr ?? error.message).includes('404')) throw error;+ }+ }+ return manifests;+}++function readState() {+ const pullPages = JSON.parse(+ gh([`repos/${REPO}/pulls?state=open&per_page=100`, '--paginate', '--slurp']),+ );+ const pulls = [];+ for (const pr of pullPages.flat()) {+ const combined = JSON.parse(gh([`repos/${REPO}/commits/${pr.head.sha}/status`]));+ const visualStatus = combined.statuses.find(status => status.context === CONTEXT) ?? null;+ if (visualStatus && !visualStatus.description.startsWith('Grandfathered during')) {+ pulls.push({number: pr.number, head: pr.head, visualStatus, hasStableVisual: false});+ continue;+ }+ const files = paged(`repos/${REPO}/pulls/${pr.number}/files?per_page=100`).map(+ file => file.filename,+ );+ const scope = classifyVisualScope(files, ROOT, headManifests(pr, files));+ pulls.push({...pr, visualStatus, hasStableVisual: scope.hasStableVisual});+ }+ const protection = JSON.parse(+ gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),+ );+ return {pulls, contexts: protection.contexts ?? []};+}++function apply(plan) {+ for (const {sha, number, state, description} of plan.backfill) {+ gh(+ ['--method', 'POST', `repos/${REPO}/statuses/${sha}`, '--input', '-'],+ {state, context: CONTEXT, description: `${description} (PR #${number})`},+ );+ }+ if (plan.addRequiredContext) {+ gh([+ '--method',+ 'POST',+ `repos/${REPO}/branches/${BRANCH}/protection/required_status_checks/contexts`,+ '-f',+ `contexts[]=${CONTEXT}`,+ ]);+ }+ const protection = JSON.parse(+ gh([`repos/${REPO}/branches/${BRANCH}/protection/required_status_checks`]),+ );+ if (!(protection.contexts ?? []).includes(CONTEXT)) {+ throw new Error(`${CONTEXT} is still not required on ${BRANCH}`);+ }+}++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const state = readState();+ const plan = buildActivationPlan(state.pulls, state.contexts);+ process.stdout.write(+ `${JSON.stringify(+ {+ repo: REPO,+ branch: BRANCH,+ context: CONTEXT,+ headsToReconcile: plan.backfill.length,+ pending: plan.backfill.filter(item => item.state === 'pending').length,+ success: plan.backfill.filter(item => item.state === 'success').length,+ addRequiredContext: plan.addRequiredContext,+ },+ null,+ 2,+ )}\n`,+ );+ if (process.argv.includes('--apply')) {+ apply(plan);+ process.stdout.write(`Activated required status ${CONTEXT} on ${BRANCH}.\n`);+ } else {+ process.stdout.write('Dry run only; pass --apply to reconcile and activate.\n');+ }+}diff --git a/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs b/.github/scripts/visual-gate/activate-visual-acceptance.test.mjsnew file mode 100644index 0000000000..63d9d07334--- /dev/null+++ b/.github/scripts/visual-gate/activate-visual-acceptance.test.mjs@@ -0,0 +1,64 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';++import {buildActivationPlan} from './activate-visual-acceptance.mjs';++describe('visual-acceptance activation', () => {+ it('derives pending for stable heads and success only for no-scope heads', () => {+ const plan = buildActivationPlan(+ [+ {number: 1, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},+ {number: 2, head: {sha: 'b'.repeat(40)}, hasStableVisual: false, visualStatus: null},+ {number: 3, head: {sha: 'a'.repeat(40)}, hasStableVisual: true, visualStatus: null},+ ],+ ['build'],+ );+ expect(plan).toEqual({+ backfill: [+ {+ number: 3,+ sha: 'a'.repeat(40),+ state: 'pending',+ description: 'Stable visual result pending; rerun CI after activation.',+ },+ {+ number: 2,+ sha: 'b'.repeat(40),+ state: 'success',+ description: 'No stable visual scope.',+ },+ ],+ addRequiredContext: true,+ });+ });++ it('repairs legacy grandfathered states but preserves live workflow verdicts', () => {+ const plan = buildActivationPlan(+ [+ {+ number: 1,+ head: {sha: 'a'.repeat(40)},+ hasStableVisual: true,+ visualStatus: {description: 'Grandfathered during visual-acceptance activation'},+ },+ {+ number: 2,+ head: {sha: 'b'.repeat(40)},+ hasStableVisual: true,+ visualStatus: {description: 'Current visual bundle was explicitly accepted.'},+ },+ ],+ ['build', 'visual-acceptance'],+ );+ expect(plan.backfill).toEqual([+ {+ number: 1,+ sha: 'a'.repeat(40),+ state: 'pending',+ description: 'Stable visual result pending; rerun CI after activation.',+ },+ ]);+ expect(plan.addRequiredContext).toBe(false);+ });+});diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjsindex a0822e2312..c28d637b27 100644--- a/.github/scripts/visual-gate/gate.mjs+++ b/.github/scripts/visual-gate/gate.mjs@@ -75,6 +75,32 @@ const maxShots = flag('max-shots') ? Number(flag('max-shots')) : Infinity;const storyPackages = (flag('story-packages') ?? config.stableStoryPackages.join(',')).split(',').filter(Boolean);+/** A trusted, exact shot list used only to recapture an accepted merged result. */+const planFile = flag('plan-file');++function readExactPlan(file) {+ const shots = JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));+ if (!Array.isArray(shots) || shots.length > 5000) {+ throw new Error('Exact visual plan must contain at most 5000 trusted shots.');+ }+ const keys = new Set();+ for (const shot of shots) {+ if (+ !/^[A-Za-z0-9._-]{1,240}$/.test(shot?.key ?? '') ||+ typeof shot.storyId !== 'string' ||+ !shot.storyId ||+ typeof shot.theme !== 'string' ||+ !shot.theme ||+ !['light', 'dark'].includes(shot.mode) ||+ !Array.isArray(shot.reasons)+ ) {+ throw new Error(`Exact visual plan contains an invalid shot: ${JSON.stringify(shot)}`);+ }+ if (keys.has(shot.key)) throw new Error(`Exact visual plan repeats ${shot.key}.`);+ keys.add(shot.key);+ }+ return shots;+}/*** Which stories the scout needs to look at: every story of a component some@@ -96,6 +122,7 @@ function storiesToScout(stories, targets, themeOverrides) {/** @returns {Promise<import('./lib/plan.mjs').Shot[]>} */async function plan() {+ if (planFile) return readExactPlan(planFile);const [targets, themeOverrides] = await Promise.all([loadThemingTargets(REPO_ROOT),loadThemeOverrides(REPO_ROOT, config.probeTheme),@@ -210,9 +237,23 @@ async function check() {status: 'skipped',generatedAt: new Date().toISOString(),reason: `${shots.length} shots exceeds the ${maxShots}-shot budget${components.length ? ` (${components.length} components touched)` : ''} — too broad to review shot by shot here. The daily release gate covers this change against the full baseline.`,+ context: {+ sha: process.env.GITHUB_SHA ?? null,+ headSha: process.env.ASTRYX_PR_HEAD_SHA ?? null,+ baseSha: process.env.ASTRYX_PR_BASE_SHA ?? null,+ ref: process.env.GITHUB_REF ?? null,+ runId: process.env.GITHUB_RUN_ID ?? null,+ runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,+ tiers,+ scoped: components.length > 0,+ components,+ },counts: {total: shots.length, unchanged: 0, changed: 0, added: 0, removed: 0, failed: 0},components,changes: [],+ added: [],+ removed: [],+ failures: [],};fs.mkdirSync(outDir, {recursive: true});fs.writeFileSync(path.join(outDir, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`);diff --git a/.github/scripts/visual-gate/lib/capture.mjs b/.github/scripts/visual-gate/lib/capture.mjsindex e0dc296d0f..97b2e87a5c 100644--- a/.github/scripts/visual-gate/lib/capture.mjs+++ b/.github/scripts/visual-gate/lib/capture.mjs@@ -81,6 +81,19 @@ const SEEDED_RANDOM = `(() => {};})();`;+// BrowserContext routing does not cover network opened from dedicated/shared+// workers. A static Storybook needs none of these constructors, so remove the+// channels before any PR-authored script runs. Service workers are separately+// disabled in the context options below.+export const BACKGROUND_NETWORK_GUARD = `(() => {+ const Blocked = class {+ constructor() { throw new Error('Network channel disabled during visual capture'); }+ };+ for (const name of ['WebSocket', 'Worker', 'SharedWorker']) {+ Object.defineProperty(globalThis, name, {value: Blocked, configurable: false});+ }+})();`;+/** The instant every capture happens at, and the zone it happens in.** Stories that build their data from `new Date()` photograph a different day@@ -105,25 +118,66 @@ const TIMEZONE_ID = 'UTC';* @param {string} dir* @returns {Promise<{port: number, close: () => Promise<void>}>}*/+export function isSameOrigin(url, origin) {+ try {+ return new URL(url).origin === origin;+ } catch {+ return false;+ }+}++/** Install the network boundary before any PR-authored Storybook code runs. */+export async function blockExternalNetwork(context, origin) {+ await context.route('**', route =>+ isSameOrigin(route.request().url(), origin) ? route.continue() : route.abort(),+ );+ // A routed WebSocket never connects unless connectToServer() is called. Close+ // every socket: a static Storybook needs none, including its dev-only HMR.+ await context.routeWebSocket(/.*/, socket => socket.close({code: 1008, reason: 'blocked'}));+}++export const CAPTURE_CONTEXT_SECURITY = {serviceWorkers: 'block'};+export function serveDirectory(dir) {+ const root = fs.realpathSync(dir);const server = http.createServer((req, res) => {- const requested = path.join(dir, req.url === '/' ? 'index.html' : req.url.split('?')[0]);- if (!path.resolve(requested).startsWith(path.resolve(dir))) {+ let requested;+ try {+ const pathname = decodeURIComponent(new URL(req.url, 'http://localhost').pathname);+ requested = path.resolve(root, `.${pathname === '/' ? '/index.html' : pathname}`);+ } catch {+ res.writeHead(400);+ res.end('Bad request');+ return;+ }+ if (requested !== root && !requested.startsWith(`${root}${path.sep}`)) {res.writeHead(403);res.end('Forbidden');return;}- fs.readFile(requested, (error, data) => {- if (error) {+ fs.realpath(requested, (realpathError, realPath) => {+ if (realpathError) {res.writeHead(404);res.end('Not found');return;}- res.writeHead(200, {- 'Content-Type': CONTENT_TYPES[path.extname(requested)] ?? 'application/octet-stream',- 'Cache-Control': 'no-store',+ if (realPath !== root && !realPath.startsWith(`${root}${path.sep}`)) {+ res.writeHead(403);+ res.end('Forbidden');+ return;+ }+ fs.readFile(realPath, (error, data) => {+ if (error) {+ res.writeHead(404);+ res.end('Not found');+ return;+ }+ res.writeHead(200, {+ 'Content-Type': CONTENT_TYPES[path.extname(realPath)] ?? 'application/octet-stream',+ 'Cache-Control': 'no-store',+ });+ res.end(data);});- res.end(data);});});return new Promise(resolve => {@@ -278,12 +332,12 @@ export async function scout({storyIds, storybookDir, theme, viewport, onProgressviewport,deviceScaleFactor: 1,timezoneId: TIMEZONE_ID,+ ...CAPTURE_CONTEXT_SECURITY,});await context.clock.setFixedTime(FROZEN_NOW);+ await context.addInitScript(BACKGROUND_NETWORK_GUARD);await context.addInitScript(SEEDED_RANDOM);- await context.route('**', route =>- route.request().url().startsWith(origin) ? route.continue() : route.abort(),- );+ await blockExternalNetwork(context, origin);const page = await context.newPage();/** @type {Record<string, Record<string, string[]>>} */@@ -344,14 +398,14 @@ export async function capture({reducedMotion: 'reduce',colorScheme: 'light',timezoneId: TIMEZONE_ID,+ ...CAPTURE_CONTEXT_SECURITY,});await context.clock.setFixedTime(FROZEN_NOW);+ await context.addInitScript(BACKGROUND_NETWORK_GUARD);await context.addInitScript(SEEDED_RANDOM);// Anything off-origin is a determinism hazard, and nothing in a component// story legitimately needs it.- await context.route('**', route =>- route.request().url().startsWith(origin) ? route.continue() : route.abort(),- );+ await blockExternalNetwork(context, origin);const page = await context.newPage();await page.addStyleTag({content: FREEZE_CSS}).catch(() => {});diff --git a/.github/scripts/visual-gate/lib/capture.test.mjs b/.github/scripts/visual-gate/lib/capture.test.mjsnew file mode 100644index 0000000000..d52a9be953--- /dev/null+++ b/.github/scripts/visual-gate/lib/capture.test.mjs@@ -0,0 +1,67 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';++import {afterEach, describe, expect, it} from 'vitest';++import {+ BACKGROUND_NETWORK_GUARD,+ CAPTURE_CONTEXT_SECURITY,+ blockExternalNetwork,+ isSameOrigin,+ serveDirectory,+} from './capture.mjs';++const roots = [];+afterEach(() => {+ for (const root of roots.splice(0)) fs.rmSync(root, {recursive: true, force: true});+});++describe('capture network boundary', () => {+ it('allows the exact Storybook origin', () => {+ expect(isSameOrigin('http://127.0.0.1:6100/assets/story.js', 'http://127.0.0.1:6100')).toBe(+ true,+ );+ });++ it('rejects URLs that merely share the trusted prefix', () => {+ expect(+ isSameOrigin('http://127.0.0.1:[email protected]/collect', 'http://127.0.0.1:6100'),+ ).toBe(false);+ });++ it('blocks service workers and every WebSocket before Storybook runs', async () => {+ let webSocketHandler;+ const context = {+ route: async () => {},+ routeWebSocket: async (_pattern, handler) => {+ webSocketHandler = handler;+ },+ };+ await blockExternalNetwork(context, 'http://127.0.0.1:6100');+ const closed = [];+ await webSocketHandler({close: options => closed.push(options)});+ expect(CAPTURE_CONTEXT_SECURITY).toEqual({serviceWorkers: 'block'});+ expect(BACKGROUND_NETWORK_GUARD).toContain("'WebSocket', 'Worker', 'SharedWorker'");+ expect(closed).toEqual([{code: 1008, reason: 'blocked'}]);+ });++ it('does not follow a Storybook artifact symlink outside its root', async () => {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'capture-server-'));+ roots.push(root);+ const served = path.join(root, 'served');+ fs.mkdirSync(served);+ fs.writeFileSync(path.join(root, 'secret'), 'credential');+ fs.symlinkSync(path.join(root, 'secret'), path.join(served, 'escape'));+ const server = await serveDirectory(served);+ try {+ const response = await fetch(`http://127.0.0.1:${server.port}/escape`);+ expect(response.status).toBe(403);+ expect(await response.text()).not.toContain('credential');+ } finally {+ await server.close();+ }+ });+});diff --git a/.github/scripts/visual-gate/lib/report.mjs b/.github/scripts/visual-gate/lib/report.mjsindex 865cefd25d..172922bcb0 100644--- a/.github/scripts/visual-gate/lib/report.mjs+++ b/.github/scripts/visual-gate/lib/report.mjs@@ -80,6 +80,21 @@ function keyList(keys, title, note) {</section>`;}+/** @param {string[]} keys @param {'added' | 'removed'} kind */+function oneSidedEvidence(keys, kind) {+ if (keys.length === 0) return '';+ const imageKind = kind === 'added' ? 'after' : 'before';+ const title = kind === 'added' ? 'Added' : 'Removed';+ return `+<h2>${title} (${keys.length})</h2>+${keys+ .map(key => {+ const safe = escapeHtml(key);+ return `<section class="change"><h3><code>${safe}</code></h3><p class="muted">${imageKind}</p><div class="stage"><img src="${imageKind}/${safe}.png" alt="${imageKind}"></div></section>`;+ })+ .join('')}`;+}+/** @param {object} targeting */function targetingSection(targeting) {const rows = [@@ -120,7 +135,7 @@ function targetingSection(targeting) {/*** @param {object} verdict- * @param {{acceptHint?: string}} [options]+ * @param {{acceptHint?: string, oneSidedEvidence?: boolean}} [options]* @returns {string}*/export function renderReport(verdict, options = {}) {@@ -200,8 +215,8 @@ ${verdict.failures?.length ? `<section class="list"><h2>Failed to capture <span.join('')}</ul></section>` : ''}${verdict.changes.length ? `<h2>Changed (${verdict.changes.length})</h2>` : '<p class="ok">No shot changed against the baseline.</p>'}${verdict.changes.map(changeSection).join('')}-${keyList(verdict.added ?? [], 'Added', 'New shots with no baseline to regress against — adopted on the next promotion.')}-${keyList(verdict.removed ?? [], 'Removed', 'Baseline shots whose story no longer exists.')}+${options.oneSidedEvidence ? oneSidedEvidence(verdict.added ?? [], 'added') : keyList(verdict.added ?? [], 'Added', 'New shots with no baseline to regress against — adopted on the next promotion.')}+${options.oneSidedEvidence ? oneSidedEvidence(verdict.removed ?? [], 'removed') : keyList(verdict.removed ?? [], 'Removed', 'Baseline shots whose story no longer exists.')}${targetingSection(verdict.targeting ?? {})}<script>for (const views of document.querySelectorAll('.views')) {diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjsindex eddeff4230..15c10ef824 100644--- a/.github/scripts/visual-gate/publish-pr-report.mjs+++ b/.github/scripts/visual-gate/publish-pr-report.mjs@@ -2,22 +2,22 @@// Copyright (c) Meta Platforms, Inc. and affiliates./**- * @file Turn an untrusted pr-visual artifact into publishable evidence.+ * @file Derive publishable PR visual evidence from a trusted recapture.*- * @input verdict.json + before/after/diff PNGs emitted by PR-controlled CI- * @output schema-validated, decoded/re-encoded PNGs and a report whose HTML is- * generated by DEFAULT-BRANCH code- *- * This script runs only in the trusted `workflow_run` publisher. It never- * imports or executes a file from the artifact. A fork may choose every byte- * in its artifact, so paths, counts, identity fields and image bytes are all- * validated before anything is copied onto the shared GitHub Pages origin.+ * The Storybook bundle is PR-produced, but the plan, browser capture, scope,+ * comparison, verdict, PNG validation, and report all run from default-branch+ * code. PR-authored verdicts and job conclusions are never authority for the+ * `visual-acceptance` status.*/+import {createHash} from 'node:crypto';import fs from 'node:fs';import path from 'node:path';import {PNG} from 'pngjs';+import {incomparable} from './lib/baseline.mjs';+import {buildVerdict, compareCaptures} from './lib/compare.mjs';+import {shotKey} from './lib/plan.mjs';import {renderReport} from './lib/report.mjs';const args = process.argv.slice(2);@@ -28,109 +28,202 @@ const flag = name => {const input = path.resolve(flag('input') ?? 'visual');const output = path.resolve(flag('output') ?? 'trusted-visual');+const baseline = path.resolve(flag('baseline') ?? '.visual-baseline');+const scope = JSON.parse(fs.readFileSync(path.resolve(flag('scope')), 'utf8'));const pr = Number(flag('pr'));const headSha = flag('head-sha') ?? '';const runId = flag('run-id') ?? '';const runAttempt = flag('run-attempt') ?? '';+const config = JSON.parse(+ fs.readFileSync(new URL('./visual-gate.config.json', import.meta.url), 'utf8'),+);const KEY = /^[A-Za-z0-9._-]{1,240}$/;+const NAME = /^[A-Za-z0-9._-]{1,120}$/;const SHA = /^[0-9a-f]{40}$/;-const MAX_CHANGES = 240;const MAX_PNG_BYTES = 12 * 1024 * 1024;const MAX_EDGE = 5000;+const MAX_SHOTS = 5000;function fail(message) {throw new Error(`visual evidence rejected: ${message}`);}-if (!Number.isSafeInteger(pr) || pr <= 0) fail('invalid PR number');-if (!SHA.test(headSha)) fail('invalid trusted head SHA');-if (!/^\d+$/.test(runId) || !/^\d+$/.test(runAttempt)) fail('invalid run identity');--const verdictPath = path.join(input, 'verdict.json');-if (!fs.existsSync(verdictPath)) fail('verdict.json is missing');-if (fs.lstatSync(verdictPath).isSymbolicLink()) fail('verdict.json is a symlink');-const verdict = JSON.parse(fs.readFileSync(verdictPath, 'utf8'));+function readJSON(file) {+ try {+ return JSON.parse(fs.readFileSync(file, 'utf8'));+ } catch (error) {+ fail(`cannot read ${file}: ${error.message}`);+ }+}-if (verdict.version !== 1) fail(`unsupported verdict version ${verdict.version}`);-if (!['pass', 'changed', 'failed', 'skipped'].includes(verdict.status)) {- fail(`invalid verdict status ${verdict.status}`);+function shaFile(file) {+ return createHash('sha256').update(fs.readFileSync(file)).digest('hex');}-if (String(verdict.context?.runId ?? '') !== runId) {- fail(`verdict run ${verdict.context?.runId ?? 'missing'} does not match ${runId}`);++function copyPng(source, target, label) {+ if (!fs.existsSync(source)) fail(`${label} is missing`);+ const stat = fs.lstatSync(source);+ if (!stat.isFile() || stat.isSymbolicLink()) fail(`${label} is not a regular file`);+ if (stat.size <= 0 || stat.size > MAX_PNG_BYTES) fail(`${label} has invalid size`);+ let image;+ try {+ image = PNG.sync.read(fs.readFileSync(source), {checkCRC: true});+ } catch (error) {+ fail(`${label} is not a valid PNG: ${error.message}`);+ }+ if (image.width <= 0 || image.height <= 0 || image.width > MAX_EDGE || image.height > MAX_EDGE) {+ fail(`${label} has invalid dimensions ${image.width}x${image.height}`);+ }+ fs.mkdirSync(path.dirname(target), {recursive: true});+ fs.writeFileSync(target, PNG.sync.write(image));+ return image;}-if (verdict.context?.runAttempt && String(verdict.context.runAttempt) !== runAttempt) {- fail(`verdict attempt ${verdict.context.runAttempt} does not match ${runAttempt}`);++if (!Number.isSafeInteger(pr) || pr <= 0) fail('invalid PR number');+if (!SHA.test(headSha)) fail('invalid trusted head SHA');+if (!/^\d+$/.test(runId) || !/^\d+$/.test(runAttempt)) fail('invalid run identity');+if (+ scope?.hasStableVisual !== true ||+ typeof scope.broadStableVisual !== 'boolean' ||+ !Array.isArray(scope.stableComponents) ||+ !Array.isArray(scope.stableThemes) ||+ !scope.stableComponents.every(name => NAME.test(name)) ||+ !scope.stableThemes.every(name => NAME.test(name))+) {+ fail('trusted stable visual scope is invalid');}-if (!SHA.test(verdict.context?.sha ?? '')) {- fail('verdict tested SHA is missing or invalid');++const manifestPath = path.join(input, 'manifest.json');+if (!fs.existsSync(manifestPath) || fs.lstatSync(manifestPath).isSymbolicLink()) {+ fail('trusted capture manifest is missing or symbolic');}-if (verdict.context?.headSha && verdict.context.headSha !== headSha) {- fail(`verdict head ${verdict.context.headSha} does not match trusted head ${headSha}`);+const manifest = readJSON(manifestPath);+if (+ manifest.version !== 1 ||+ typeof manifest.platform !== 'string' ||+ !manifest.platform ||+ typeof manifest.browser !== 'string' ||+ !manifest.browser ||+ !Number.isSafeInteger(manifest.viewport?.width) ||+ !Number.isSafeInteger(manifest.viewport?.height) ||+ !manifest.shots ||+ typeof manifest.shots !== 'object' ||+ Array.isArray(manifest.shots)+) {+ fail('trusted capture manifest is invalid');}-if (!Array.isArray(verdict.changes) || verdict.changes.length > MAX_CHANGES) {- fail(`invalid change count ${verdict.changes?.length ?? 'missing'}`);+if (+ manifest.context?.headSha !== headSha ||+ String(manifest.context?.runId ?? '') !== runId ||+ String(manifest.context?.runAttempt ?? '') !== runAttempt ||+ !SHA.test(manifest.context?.sha ?? '') ||+ !SHA.test(manifest.context?.baseSha ?? '')+) {+ fail('trusted capture identity mismatch');}-if (!Array.isArray(verdict.added) || !Array.isArray(verdict.removed)) {- fail('added/removed shot lists are missing');++const baselineManifest = readJSON(path.join(baseline, 'manifest.json'));+const blocker = incomparable(baselineManifest, manifest);+if (blocker) fail(`baseline is not comparable: ${blocker}`);++const entries = Object.entries(manifest.shots);+if (entries.length > MAX_SHOTS) fail(`trusted capture exceeds ${MAX_SHOTS} shots`);+for (const [key, shot] of entries) {+ if (+ !KEY.test(key) ||+ !shot ||+ typeof shot.storyId !== 'string' ||+ !shot.storyId ||+ typeof shot.theme !== 'string' ||+ !NAME.test(shot.theme) ||+ !['light', 'dark'].includes(shot.mode) ||+ !Array.isArray(shot.reasons) ||+ shotKey(shot) !== key+ ) {+ fail(`capture metadata is invalid for ${key}`);+ }}-if (verdict.changes.length + verdict.added.length + verdict.removed.length > MAX_CHANGES) {- fail('visual delta bundle exceeds the review budget');+fs.rmSync(output, {recursive: true, force: true});+for (const dir of ['current', 'before', 'after', 'diff']) {+ fs.mkdirSync(path.join(output, dir), {recursive: true});}-const seen = new Set();-const validateKey = key => {- if (!KEY.test(key ?? '')) fail(`invalid shot key ${JSON.stringify(key)}`);- if (seen.has(key)) fail(`duplicate shot key ${key}`);- seen.add(key);-};-for (const change of verdict.changes) {- validateKey(change.key);- if (!Number.isFinite(Number(change.diffPixels)) || Number(change.diffPixels) < 0) {- fail(`invalid diff pixel count for ${change.key}`);+const sourceShots = path.join(input, 'shots');+const sourceNames = fs.existsSync(sourceShots)+ ? fs.readdirSync(sourceShots, {withFileTypes: true})+ : [];+const sourceKeys = sourceNames.map(entry => {+ if (!entry.isFile() || entry.isSymbolicLink() || !entry.name.endsWith('.png')) {+ fail(`unexpected capture entry ${entry.name}`);}+ return entry.name.slice(0, -4);+});+if (sourceKeys.length !== entries.length || sourceKeys.some(key => !manifest.shots[key])) {+ fail('capture PNG set does not match its manifest');}-for (const key of verdict.added) validateKey(key);-for (const key of verdict.removed) validateKey(key);-fs.rmSync(output, {recursive: true, force: true});-for (const kind of ['before', 'after', 'diff']) {- fs.mkdirSync(path.join(output, kind), {recursive: true});+const trustedManifest = {...manifest, shots: {}};+for (const [key, shot] of entries) {+ const target = path.join(output, 'current', `${key}.png`);+ const image = copyPng(path.join(sourceShots, `${key}.png`), target, `shots/${key}.png`);+ trustedManifest.shots[key] = {+ ...shot,+ sha256: shaFile(target),+ width: image.width,+ height: image.height,+ };}-/** Decode + re-encode: validates the bytes and strips ancillary metadata. */-function copyPng(kind, key, sourceDir = path.join(input, 'report', kind)) {- const source = path.join(sourceDir, `${key}.png`);- if (!fs.existsSync(source)) fail(`${kind}/${key}.png is missing`);- const stat = fs.lstatSync(source);- if (!stat.isFile() || stat.isSymbolicLink()) fail(`${kind}/${key}.png is not a regular file`);- if (stat.size <= 0 || stat.size > MAX_PNG_BYTES) fail(`${kind}/${key}.png has invalid size`);-- let image;- try {- image = PNG.sync.read(fs.readFileSync(source), {checkCRC: true});- } catch (error) {- fail(`${kind}/${key}.png is not a valid PNG: ${error.message}`);- }- if (- image.width <= 0 ||- image.height <= 0 ||- image.width > MAX_EDGE ||- image.height > MAX_EDGE- ) {- fail(`${kind}/${key}.png has invalid dimensions ${image.width}x${image.height}`);- }- fs.writeFileSync(path.join(output, kind, `${key}.png`), PNG.sync.write(image));+const baselineEntries = Object.entries(baselineManifest.shots ?? {});+const expected = scope.broadStableVisual+ ? baselineEntries.map(([key]) => key)+ : baselineEntries+ .filter(([, shot]) =>+ scope.stableComponents.includes(shot.component) || scope.stableThemes.includes(shot.theme),+ )+ .map(([key]) => key);+if (entries.length === 0 && expected.length === 0) {+ fail('stable visual scope produced no trusted shots');}-for (const {key} of verdict.changes) {- for (const kind of ['before', 'after', 'diff']) copyPng(kind, key);+const comparison = await compareCaptures({+ baselineDir: path.join(baseline, 'shots'),+ currentDir: path.join(output, 'current'),+ baselineManifest,+ currentManifest: trustedManifest,+ diffDir: path.join(output, 'diff'),+ threshold: config.threshold,+ maxDiffPixels: config.maxDiffPixels,+ scoped: true,+});+comparison.removed = expected.filter(key => !trustedManifest.shots[key]);++const verdict = buildVerdict({+ comparison,+ currentManifest: trustedManifest,+ baselineManifest,+ targeting: {unexercisedOverrides: [], undeclaredTargets: [], uncoveredTargets: []},+ failures: [],+ context: {...manifest.context, trustedScope: scope},+});++const beforeSha256 = {};+for (const change of verdict.changes) {+ const key = change.key;+ const baselineFile = path.join(baseline, 'shots', `${key}.png`);+ copyPng(baselineFile, path.join(output, 'before', `${key}.png`), `baseline/${key}.png`);+ fs.copyFileSync(path.join(output, 'current', `${key}.png`), path.join(output, 'after', `${key}.png`));+ beforeSha256[key] = shaFile(baselineFile);}-// New stable shots have no before/diff, but their AFTER is the image a human-// accepts and the post-merge verifier must reproduce. Preserve it now; the-// Actions artifact expires and the per-PR path is deleted when the PR closes.for (const key of verdict.added) {- copyPng('after', key, path.join(input, 'shots'));+ fs.copyFileSync(path.join(output, 'current', `${key}.png`), path.join(output, 'after', `${key}.png`));+ beforeSha256[key] = null;+}+for (const key of verdict.removed) {+ const baselineFile = path.join(baseline, 'shots', `${key}.png`);+ copyPng(baselineFile, path.join(output, 'before', `${key}.png`), `baseline/${key}.png`);+ beforeSha256[key] = shaFile(baselineFile);}const evidence = {@@ -138,23 +231,48 @@ const evidence = {repo: 'facebook/astryx',pr,headSha,- testedSha: verdict.context?.sha ?? null,- baseSha: verdict.context?.baseSha ?? null,+ testedSha: manifest.context.sha,+ baseSha: manifest.context.baseSha,run: {id: Number(runId), attempt: Number(runAttempt)},+ capture: {+ platform: trustedManifest.platform,+ browser: trustedManifest.browser,+ viewport: trustedManifest.viewport,+ },deltas: [- ...verdict.changes.map(change => ({key: change.key, kind: 'changed'})),- ...verdict.added.map(key => ({key, kind: 'added'})),- ...verdict.removed.map(key => ({key, kind: 'removed'})),+ ...verdict.changes.map(change => ({+ key: change.key,+ kind: 'changed',+ beforeSha256: beforeSha256[change.key],+ shot: trustedManifest.shots[change.key],+ })),+ ...verdict.added.map(key => ({+ key,+ kind: 'added',+ beforeSha256: null,+ shot: trustedManifest.shots[key],+ })),+ ...verdict.removed.map(key => ({+ key,+ kind: 'removed',+ beforeSha256: beforeSha256[key],+ shot: null,+ })),],verdict,};++fs.rmSync(path.join(output, 'current'), {recursive: true, force: true});fs.writeFileSync(path.join(output, 'evidence.json'), `${JSON.stringify(evidence, null, 2)}\n`);fs.writeFileSync(path.join(output, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`);fs.writeFileSync(path.join(output, 'index.html'),- renderReport(verdict, {acceptHint: '/accept-visual <why every changed frame is correct>'}),+ renderReport(verdict, {+ acceptHint: `/accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,+ oneSidedEvidence: true,+ }),);process.stdout.write(- `Validated ${verdict.changes.length} visual delta(s) for PR #${pr}, head ${headSha.slice(0, 12)}, run ${runId}/${runAttempt}.\n`,+ `Derived trusted ${verdict.status} verdict for ${entries.length} shot(s), PR #${pr}, run ${runId}/${runAttempt}.\n`,);diff --git a/.github/scripts/visual-gate/publish-pr-report.test.mjs b/.github/scripts/visual-gate/publish-pr-report.test.mjsindex 12a95891a0..2a99b05cbb 100644--- a/.github/scripts/visual-gate/publish-pr-report.test.mjs+++ b/.github/scripts/visual-gate/publish-pr-report.test.mjs@@ -1,6 +1,7 @@// Copyright (c) Meta Platforms, Inc. and affiliates.import {execFileSync} from 'node:child_process';+import {createHash} from 'node:crypto';import fs from 'node:fs';import os from 'node:os';import path from 'node:path';@@ -9,134 +10,193 @@ import {fileURLToPath} from 'node:url';import {afterEach, beforeEach, describe, expect, it} from 'vitest';import {PNG} from 'pngjs';-const SCRIPT = path.join(- path.dirname(fileURLToPath(import.meta.url)),- 'publish-pr-report.mjs',-);+const SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), 'publish-pr-report.mjs');const HEAD = 'a'.repeat(40);const RUN = '123456';+const KEY = 'core-button--default__neutral-light';+const SHOT = {+ storyId: 'core-button--default',+ title: 'Core/Button',+ name: 'Default',+ component: 'Button',+ theme: 'neutral',+ mode: 'light',+ reasons: ['trusted:pr-scope'],+};let root;let input;let output;+let baseline;+let scope;-function png() {+function png(red = 255, green = 0, blue = 0) {const image = new PNG({width: 2, height: 2});for (let i = 0; i < image.data.length; i += 4) {- image.data[i] = 255;+ image.data[i] = red;+ image.data[i + 1] = green;+ image.data[i + 2] = blue;image.data[i + 3] = 255;}return PNG.sync.write(image);}-function writeFixture(overrides = {}) {- const key = overrides.key ?? 'core-button--default__neutral-light';- const verdict = {+function writeJSON(file, value) {+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, JSON.stringify(value));+}++function writeBaseline(shots = {[KEY]: {shot: SHOT, bytes: png()}}) {+ const manifestShots = {};+ for (const [key, {shot, bytes}] of Object.entries(shots)) {+ const file = path.join(baseline, 'shots', `${key}.png`);+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, bytes);+ manifestShots[key] = {...shot, sha256: createHash('sha256').update(bytes).digest('hex')};+ }+ writeJSON(path.join(baseline, 'manifest.json'), {version: 1,- status: 'changed',- context: {runId: RUN, headSha: HEAD, sha: 'b'.repeat(40)},- counts: {total: 1, changed: 1, added: 0, removed: 0, failed: 0},- changes: [- {- key,- component: 'Button',- name: 'Default',- theme: 'neutral',- mode: 'light',- diffPixels: 4,- },- ],+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ shots: manifestShots,+ decisions: [],+ });+}++function writeCapture(shots = {[KEY]: {shot: SHOT, bytes: png(0, 0, 255)}}, overrides = {}) {+ fs.rmSync(input, {recursive: true, force: true});+ const manifestShots = {};+ const shotsDir = path.join(input, 'shots');+ fs.mkdirSync(shotsDir, {recursive: true});+ for (const [key, {shot, bytes}] of Object.entries(shots)) {+ fs.writeFileSync(path.join(shotsDir, `${key}.png`), bytes);+ manifestShots[key] = {...shot, sha256: '0'.repeat(64)};+ }+ writeJSON(path.join(input, 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ context: {+ sha: 'b'.repeat(40),+ headSha: HEAD,+ baseSha: 'c'.repeat(40),+ runId: RUN,+ runAttempt: '1',+ },+ shots: manifestShots,+ ...overrides,+ });+ // A forged PR verdict is deliberately present: the trusted publisher ignores it.+ writeJSON(path.join(input, 'verdict.json'), {+ version: 1,+ status: 'pass',+ changes: [],added: [],removed: [],- ...overrides.verdict,- };- fs.mkdirSync(input, {recursive: true});- fs.writeFileSync(path.join(input, 'verdict.json'), JSON.stringify(verdict));- for (const kind of ['before', 'after', 'diff']) {- const dir = path.join(input, 'report', kind);- fs.mkdirSync(dir, {recursive: true});- fs.writeFileSync(path.join(dir, `${key}.png`), overrides.bytes ?? png());- }+ failures: [],+ });}-function run(extra = []) {+function run() {return execFileSync(process.execPath,[SCRIPT,- '--input',- input,- '--output',- output,- '--pr',- '42',- '--head-sha',- HEAD,- '--run-id',- RUN,- '--run-attempt',- '1',- ...extra,+ '--input', input,+ '--output', output,+ '--baseline', baseline,+ '--scope', scope,+ '--pr', '42',+ '--head-sha', HEAD,+ '--run-id', RUN,+ '--run-attempt', '1',],{encoding: 'utf8'},);}beforeEach(() => {- root = fs.mkdtempSync(path.join(os.tmpdir(), 'publish-visual-'));+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'trusted-visual-'));input = path.join(root, 'input');output = path.join(root, 'output');+ baseline = path.join(root, 'baseline');+ scope = path.join(root, 'scope.json');+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: false,+ stableComponents: ['Button'],+ stableThemes: [],+ });+ writeBaseline();+ writeCapture();});+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));describe('trusted PR visual publisher', () => {- it('decodes/re-encodes images and generates the wrapper from trusted code', () => {- writeFixture();- expect(run()).toContain('Validated 1 visual delta');- expect(fs.existsSync(path.join(output, 'index.html'))).toBe(true);- expect(fs.existsSync(path.join(output, 'before', 'core-button--default__neutral-light.png'))).toBe(true);- const evidence = JSON.parse(fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'));- expect(evidence).toMatchObject({pr: 42, headSha: HEAD, testedSha: 'b'.repeat(40)});+ it('derives a changed verdict even when PR data claims pass', () => {+ expect(run()).toContain('trusted changed verdict');+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));+ expect(verdict).toMatchObject({status: 'changed', counts: {changed: 1}});+ expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);+ expect(fs.existsSync(path.join(output, 'after', `${KEY}.png`))).toBe(true);+ expect(fs.existsSync(path.join(output, 'diff', `${KEY}.png`))).toBe(true);+ });++ it('derives pass only from a trusted capture matching the baseline', () => {+ writeCapture({[KEY]: {shot: SHOT, bytes: png()}});+ run();+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));+ expect(verdict.status).toBe('pass');});- it('preserves the after image for a newly added stable shot', () => {+ it('derives an added shot and preserves its after image', () => {+ writeBaseline({});const key = 'core-new--default__neutral-light';- writeFixture({- key,- verdict: {- status: 'changed',- changes: [],- added: [key],- removed: [],- counts: {total: 1, changed: 0, added: 1, removed: 0, failed: 0},- },+ const shot = {...SHOT, storyId: 'core-new--default', component: 'New'};+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: false,+ stableComponents: ['New'],+ stableThemes: [],});- // Added shots come from capture/shots, not report/after.- fs.mkdirSync(path.join(input, 'shots'), {recursive: true});- fs.writeFileSync(path.join(input, 'shots', `${key}.png`), png());+ writeCapture({[key]: {shot, bytes: png(0, 255, 0)}});run();+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));+ expect(verdict).toMatchObject({status: 'changed', added: [key]});expect(fs.existsSync(path.join(output, 'after', `${key}.png`))).toBe(true);- const evidence = JSON.parse(fs.readFileSync(path.join(output, 'evidence.json'), 'utf8'));- expect(evidence.deltas).toEqual([{key, kind: 'added'}]);});- it('rejects a key repeated across changed/added/removed lists', () => {- writeFixture({verdict: {added: ['core-button--default__neutral-light']}});- expect(() => run()).toThrow(/duplicate shot key/);+ it('derives a removed baseline shot when trusted capture omits expected scope', () => {+ writeCapture({});+ run();+ const verdict = JSON.parse(fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'));+ expect(verdict).toMatchObject({status: 'changed', removed: [KEY]});+ expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);});- it('rejects an artifact that claims another run', () => {- writeFixture({verdict: {context: {runId: '999', headSha: HEAD}}});- expect(() => run()).toThrow(/does not match/);+ it('rejects a trusted capture that claims another run', () => {+ const manifest = JSON.parse(fs.readFileSync(path.join(input, 'manifest.json'), 'utf8'));+ manifest.context.runId = '999';+ writeJSON(path.join(input, 'manifest.json'), manifest);+ expect(() => run()).toThrow(/identity mismatch/);});- it('rejects traversal and shell-shaped shot keys', () => {- writeFixture({key: '../bad;rm'});- expect(() => run()).toThrow(/invalid shot key/);+ it('rejects traversal-shaped manifest keys', () => {+ writeCapture({'../bad': {shot: {...SHOT, storyId: '../bad'}, bytes: png()}});+ expect(() => run()).toThrow(/metadata is invalid/);});- it('rejects bytes that are not a decodable PNG', () => {- writeFixture({bytes: Buffer.from('not png')});+ it('rejects capture bytes that are not a PNG', () => {+ fs.writeFileSync(path.join(input, 'shots', `${KEY}.png`), 'not png');expect(() => run()).toThrow(/not a valid PNG/);});++ it('rejects an empty capture for an unbaselined stable scope', () => {+ writeBaseline({});+ writeCapture({});+ expect(() => run()).toThrow(/produced no trusted shots/);+ });});diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjsnew file mode 100644index 0000000000..e165024ac4--- /dev/null+++ b/.github/scripts/visual-gate/visual-acceptance.mjs@@ -0,0 +1,752 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Record, evaluate, and promote explicit visual acceptance.+ *+ * Labels, comments, and statuses are projections. The immutable JSON record+ * under gh-pages:visual-gate/acceptances/<pr>/<head>/ is the source of truth.+ */++import {execFileSync} from 'node:child_process';+import {createHash} from 'node:crypto';+import fs from 'node:fs';+import path from 'node:path';+import {PNG} from 'pngjs';++import {readStoryIndex, shotKey, storiesInPackages} from './lib/plan.mjs';++const args = process.argv.slice(2);+const command = args[0];+const flag = name => {+ const index = args.indexOf(`--${name}`);+ return index === -1 ? null : args[index + 1];+};+const SHA = /^[0-9a-f]{40}$/;+const KEY = /^[A-Za-z0-9._-]{1,240}$/;+const REPO = 'facebook/astryx';++function fail(message) {+ throw new Error(`visual acceptance refused: ${message}`);+}++function shaBytes(bytes) {+ return createHash('sha256').update(bytes).digest('hex');+}++function shaFile(file) {+ return shaBytes(fs.readFileSync(file));+}++function shaJSON(value) {+ return shaBytes(JSON.stringify(value));+}++function readJSON(file) {+ try {+ return JSON.parse(fs.readFileSync(file, 'utf8'));+ } catch (error) {+ fail(`cannot read ${file}: ${error.message}`);+ }+}++function writeJSON(file, value) {+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);+}++function pagesHead(pages) {+ return execFileSync('git', ['rev-parse', 'HEAD'], {+ cwd: pages,+ encoding: 'utf8',+ }).trim();+}++function validateIdentity(pr, head) {+ if (!Number.isSafeInteger(pr) || pr <= 0 || !SHA.test(head))+ fail('invalid PR/head');+}++function evidenceRoot(pages, pr, head) {+ return path.join(pages, 'pr', String(pr), 'visual', head);+}++function evidenceAt(pages, pr, head, run, attempt) {+ if (!Number.isSafeInteger(run) || run <= 0 || !Number.isSafeInteger(attempt) || attempt <= 0) {+ fail('invalid evidence run/attempt');+ }+ const dir = path.join(evidenceRoot(pages, pr, head), String(run), String(attempt));+ if (!fs.existsSync(dir)) fail(`no published evidence for run ${run}/${attempt}`);+ const evidence = readJSON(path.join(dir, 'evidence.json'));+ validateEvidence(evidence, {pr, head, run});+ if (evidence.run.attempt !== attempt) fail('evidence attempt identity mismatch');+ return {dir, evidence};+}++function latestEvidence(pages, pr, head, {required = true} = {}) {+ const root = evidenceRoot(pages, pr, head);+ if (!fs.existsSync(root)) {+ if (!required) return null;+ fail(`no published evidence for PR #${pr}, head ${head}`);+ }+ const runs = fs+ .readdirSync(root, {withFileTypes: true})+ .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))+ .map(entry => Number(entry.name))+ .sort((a, b) => b - a);+ if (runs.length === 0) {+ if (!required) return null;+ fail('published evidence has no run directory');+ }+ const run = runs[0];+ const runDir = path.join(root, String(run));+ const attempts = fs+ .readdirSync(runDir, {withFileTypes: true})+ .filter(entry => entry.isDirectory() && /^\d+$/.test(entry.name))+ .map(entry => Number(entry.name))+ .sort((a, b) => b - a);+ const dir = attempts.length > 0 ? path.join(runDir, String(attempts[0])) : runDir;+ const evidence = readJSON(path.join(dir, 'evidence.json'));+ validateEvidence(evidence, {pr, head, run});+ if (attempts.length > 0 && evidence.run.attempt !== attempts[0]) {+ fail('evidence attempt identity mismatch');+ }+ return {dir, evidence};+}++function validateCapture(capture) {+ if (+ !capture ||+ typeof capture.platform !== 'string' ||+ !capture.platform ||+ typeof capture.browser !== 'string' ||+ !capture.browser ||+ !Number.isSafeInteger(capture.viewport?.width) ||+ !Number.isSafeInteger(capture.viewport?.height) ||+ capture.viewport.width <= 0 ||+ capture.viewport.height <= 0+ ) {+ fail('evidence capture identity is incomplete');+ }+}++function validateShot(shot, key) {+ if (+ !shot ||+ typeof shot.storyId !== 'string' ||+ !shot.storyId ||+ typeof shot.theme !== 'string' ||+ !shot.theme ||+ !['light', 'dark'].includes(shot.mode) ||+ !Array.isArray(shot.reasons)+ ) {+ fail(`accepted shot plan is incomplete for ${key}`);+ }+}++function validateEvidence(evidence, expected) {+ if (+ evidence?.version !== 1 ||+ evidence.repo !== REPO ||+ evidence.pr !== expected.pr ||+ evidence.headSha !== expected.head ||+ !SHA.test(evidence.testedSha ?? '') ||+ !SHA.test(evidence.baseSha ?? '') ||+ evidence.run?.id !== expected.run ||+ !Number.isSafeInteger(evidence.run?.attempt) ||+ evidence.run.attempt <= 0+ ) {+ fail('evidence identity mismatch');+ }+ if (+ !['pass', 'changed', 'failed', 'skipped'].includes(evidence.verdict?.status)+ ) {+ fail('evidence verdict is invalid');+ }+ if (!Array.isArray(evidence.deltas) || evidence.deltas.length > 5000) {+ fail('evidence delta list is invalid');+ }+ if (evidence.verdict.status === 'skipped') {+ if (evidence.capture !== null || evidence.deltas.length !== 0) {+ fail('skipped evidence must not claim a capture or deltas');+ }+ } else {+ validateCapture(evidence.capture);+ }+ const seen = new Set();+ for (const delta of evidence.deltas) {+ if (+ !KEY.test(delta?.key ?? '') ||+ !['changed', 'added', 'removed'].includes(delta.kind)+ ) {+ fail(`invalid evidence delta ${JSON.stringify(delta)}`);+ }+ if (seen.has(delta.key)) fail(`duplicate evidence key ${delta.key}`);+ seen.add(delta.key);+ if (delta.kind === 'removed') {+ if (delta.shot !== null)+ fail(`removed key ${delta.key} must not carry a shot plan`);+ } else {+ validateShot(delta.shot, delta.key);+ }+ }+}++function validateReason(reason) {+ const clean = String(reason ?? '').trim();+ if (+ clean.length < 12 ||+ clean.length > 500 ||+ /[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(clean)+ ) {+ fail('reason must be 12–500 printable characters');+ }+ if (+ /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(clean)+ ) {+ fail(+ 'reason must explain why the pixels are correct, not merely approve them',+ );+ }+ return clean;+}++function snapshotBundle({pages, evidenceDir, evidence}) {+ const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');+ return evidence.deltas.map(delta => {+ const beforeFile = path.join(baselineShots, `${delta.key}.png`);+ const afterFile = path.join(evidenceDir, 'after', `${delta.key}.png`);+ const currentBeforeSha256 = fs.existsSync(beforeFile)+ ? shaFile(beforeFile)+ : null;+ const beforeSha256 = delta.beforeSha256 ?? null;+ const afterSha256 = fs.existsSync(afterFile) ? shaFile(afterFile) : null;+ if (beforeSha256 !== null && !/^[0-9a-f]{64}$/.test(beforeSha256)) {+ fail(`invalid reviewed preimage for ${delta.key}`);+ }+ if (currentBeforeSha256 !== beforeSha256) {+ fail(`reviewed baseline is stale for ${delta.key}`);+ }+ if (delta.kind === 'changed' && (!beforeSha256 || !afterSha256)) {+ fail(`changed key ${delta.key} lacks before/after evidence`);+ }+ if (delta.kind === 'added' && (beforeSha256 || !afterSha256)) {+ fail(`added key ${delta.key} has invalid preimage/evidence`);+ }+ if (delta.kind === 'removed' && (!beforeSha256 || afterSha256)) {+ fail(`removed key ${delta.key} has invalid preimage/evidence`);+ }+ return {+ key: delta.key,+ kind: delta.kind,+ beforeSha256,+ afterSha256,+ shot: delta.shot,+ };+ });+}++function sameBundle(first, second) {+ return shaJSON(first) === shaJSON(second);+}++function recordPath(pages, pr, head, run, attempt) {+ return path.join(+ pages,+ 'visual-gate',+ 'acceptances',+ String(pr),+ head,+ String(run),+ String(attempt),+ 'acceptance.json',+ );+}++function pointerPath(pages, pr, head) {+ return path.join(pages, 'visual-gate', 'acceptances', String(pr), head, 'current.json');+}++function accept() {+ const pages = path.resolve(flag('pages') ?? '.');+ const pr = Number(flag('pr'));+ const head = flag('head') ?? '';+ const runId = Number(flag('run-id'));+ const runAttempt = Number(flag('run-attempt'));+ const approver = flag('approver') ?? '';+ const approverId = Number(flag('approver-id'));+ const permission = flag('permission') ?? '';+ const commentId = Number(flag('comment-id'));+ const reason = validateReason(flag('reason'));++ validateIdentity(pr, head);+ if (!approver || !Number.isSafeInteger(approverId) || approverId <= 0)+ fail('invalid approver');+ if (!['maintain', 'admin'].includes(permission))+ fail('approver must have maintain/admin permission');+ if (!Number.isSafeInteger(commentId) || commentId <= 0)+ fail('invalid comment id');++ const {dir: evidenceDir, evidence} = evidenceAt(pages, pr, head, runId, runAttempt);+ if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {+ fail('current visual bundle has no delta to accept');+ }+ const manifestFile = path.join(+ pages,+ 'visual-gate',+ 'baseline',+ 'manifest.json',+ );+ const manifest = readJSON(manifestFile);+ const keys = snapshotBundle({pages, evidenceDir, evidence});+ const output = recordPath(pages, pr, head, runId, runAttempt);+ const destination = path.dirname(output);++ const pointer = {+ version: 1,+ run: runId,+ attempt: runAttempt,+ record: `${runId}/${runAttempt}/acceptance.json`,+ };+ if (fs.existsSync(output)) {+ const existing = readJSON(output);+ validateAcceptance(existing, {+ pr,+ head,+ run: runId,+ attempt: runAttempt,+ dir: destination,+ });+ if (sameBundle(existing.keys, keys)) {+ writeJSON(pointerPath(pages, pr, head), pointer);+ process.stdout.write(`Acceptance already recorded at ${output}\n`);+ return;+ }+ fail('this head already has a different acceptance record');+ }++ for (const entry of keys) {+ if (!entry.afterSha256) continue;+ const source = path.join(evidenceDir, 'after', `${entry.key}.png`);+ const target = path.join(destination, 'after', `${entry.key}.png`);+ fs.mkdirSync(path.dirname(target), {recursive: true});+ fs.copyFileSync(source, target);+ }++ writeJSON(output, {+ version: 1,+ repo: REPO,+ pr,+ headSha: head,+ testedSha: evidence.testedSha,+ baseSha: evidence.baseSha,+ run: evidence.run,+ capture: evidence.capture,+ baseline: {+ ghPagesCommit: pagesHead(pages),+ manifestDigest: shaJSON(manifest),+ },+ keys,+ decision: {+ approver,+ approverId,+ permission,+ reason,+ commentId,+ at: new Date().toISOString(),+ },+ });+ writeJSON(pointerPath(pages, pr, head), pointer);+ process.stdout.write(+ `Accepted ${keys.length} visual delta(s) for PR #${pr}, head ${head}, run ${runId}/${runAttempt}.\n`,+ );+}++function state() {+ const pages = path.resolve(flag('pages') ?? '.');+ const pr = Number(flag('pr'));+ const head = flag('head') ?? '';+ validateIdentity(pr, head);++ const found = latestEvidence(pages, pr, head, {required: false});+ let result;+ if (!found) {+ result = {+ state: 'pending',+ reason: 'capture',+ description: 'Stable visual capture is still running.',+ };+ } else if (found.evidence.verdict.status === 'pass') {+ result = {+ state: 'success',+ reason: 'clean',+ description: 'Stable visual capture is clean.',+ };+ } else if (found.evidence.verdict.status === 'failed') {+ result = {+ state: 'failure',+ reason: 'capture',+ description: 'Stable visual capture failed.',+ };+ } else if (found.evidence.verdict.status === 'skipped') {+ result = {+ state: 'success',+ reason: 'deferred',+ description: 'Broad stable scope is deferred to the release gate.',+ };+ } else {+ let current;+ try {+ current = snapshotBundle({+ pages,+ evidenceDir: found.dir,+ evidence: found.evidence,+ });+ } catch (error) {+ if (!String(error.message).includes('reviewed baseline is stale'))+ throw error;+ result = {+ state: 'pending',+ reason: 'capture',+ description: 'The visual baseline changed; recapture this head.',+ };+ }+ if (!result) {+ const file = recordPath(+ pages,+ pr,+ head,+ found.evidence.run.id,+ found.evidence.run.attempt,+ );+ const accepted = fs.existsSync(file) ? readJSON(file) : null;+ if (accepted)+ validateAcceptance(accepted, {+ pr,+ head,+ run: found.evidence.run.id,+ attempt: found.evidence.run.attempt,+ dir: path.dirname(file),+ });+ result =+ accepted && sameBundle(accepted.keys, current)+ ? {+ state: 'success',+ reason: 'accepted',+ description: 'Current visual bundle was explicitly accepted.',+ }+ : {+ state: 'pending',+ reason: 'decision',+ description: 'Stable visual changes await /accept-visual.',+ };+ }+ }+ process.stdout.write(`${JSON.stringify(result)}\n`);+}++function validateAcceptance(value, expected = {}) {+ if (+ value?.version !== 1 ||+ value.repo !== REPO ||+ !Number.isSafeInteger(value.pr) ||+ value.pr <= 0 ||+ (expected.pr && value.pr !== expected.pr) ||+ !SHA.test(value.headSha ?? '') ||+ (expected.head && value.headSha !== expected.head) ||+ !SHA.test(value.testedSha ?? '') ||+ !SHA.test(value.baseSha ?? '') ||+ !Number.isSafeInteger(value.run?.id) ||+ value.run.id <= 0 ||+ (expected.run && value.run.id !== expected.run) ||+ !Number.isSafeInteger(value.run?.attempt) ||+ value.run.attempt <= 0 ||+ (expected.attempt && value.run.attempt !== expected.attempt) ||+ !Array.isArray(value.keys) ||+ value.keys.length === 0 ||+ !value.decision?.approver ||+ !Number.isSafeInteger(value.decision?.approverId) ||+ value.decision.approverId <= 0 ||+ !['maintain', 'admin'].includes(value.decision?.permission) ||+ !Number.isSafeInteger(value.decision?.commentId) ||+ value.decision.commentId <= 0 ||+ !value.decision?.reason+ ) {+ fail('acceptance record is invalid');+ }+ validateCapture(value.capture);+ const seen = new Set();+ for (const entry of value.keys) {+ if (+ !KEY.test(entry?.key ?? '') ||+ !['changed', 'added', 'removed'].includes(entry.kind)+ ) {+ fail('acceptance record contains an invalid key');+ }+ if (seen.has(entry.key)) fail(`acceptance record repeats ${entry.key}`);+ seen.add(entry.key);+ if (entry.kind === 'removed') {+ if (entry.afterSha256 !== null || entry.shot !== null)+ fail(`removed key ${entry.key} is invalid`);+ } else {+ if (!/^[0-9a-f]{64}$/.test(entry.afterSha256 ?? ''))+ fail(`after hash is invalid for ${entry.key}`);+ validateShot(entry.shot, entry.key);+ if (expected.dir) {+ const archived = path.join(expected.dir, 'after', `${entry.key}.png`);+ if (!fs.existsSync(archived) || shaFile(archived) !== entry.afterSha256) {+ fail(`archived AFTER is missing or changed for ${entry.key}`);+ }+ }+ }+ if (+ entry.beforeSha256 !== null &&+ !/^[0-9a-f]{64}$/.test(entry.beforeSha256)+ ) {+ fail(`before hash is invalid for ${entry.key}`);+ }+ }+}++function trustedPlan() {+ const scope = readJSON(path.resolve(flag('scope')));+ const baseline = readJSON(path.join(path.resolve(flag('baseline')), 'manifest.json'));+ const storybookDir = path.resolve(flag('storybook-dir'));+ const output = path.resolve(flag('output'));+ if (+ scope?.hasStableVisual !== true ||+ typeof scope.broadStableVisual !== 'boolean' ||+ !Array.isArray(scope.stableComponents) ||+ !Array.isArray(scope.stableThemes)+ ) {+ fail('trusted visual scope is invalid');+ }++ const baselineEntries = Object.entries(baseline.shots ?? {});+ const baselineThemes = [...new Set(baselineEntries.map(([, shot]) => shot.theme))].filter(Boolean);+ const shots = scope.broadStableVisual+ ? baselineEntries.map(([key, shot]) => ({...shot, key, reasons: ['trusted:broad']}))+ : [];++ if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {+ const indexed = storiesInPackages(readStoryIndex(storybookDir, []), ['Core']);+ const stories = new Map();+ const newTheme = scope.stableThemes.some(theme => !baselineThemes.includes(theme));+ for (const [, shot] of baselineEntries) {+ if (+ newTheme ||+ scope.stableThemes.includes(shot.theme) ||+ scope.stableComponents.includes(shot.component)+ ) {+ stories.set(shot.storyId, shot);+ }+ }+ for (const story of indexed) {+ if (scope.stableComponents.includes(story.component)) stories.set(story.id, story);+ }+ const themes = scope.stableThemes.length > 0 ? scope.stableThemes : baselineThemes;+ for (const story of stories.values()) {+ for (const theme of themes) {+ for (const mode of ['light', 'dark']) {+ const shot = {+ storyId: story.storyId ?? story.id,+ title: story.title,+ name: story.name,+ component: story.component,+ theme,+ mode,+ reasons: ['trusted:pr-scope'],+ };+ shots.push({...shot, key: shotKey(shot)});+ }+ }+ }+ }+ const unique = [...new Map(shots.map(shot => [shot.key, shot])).values()];+ if (unique.length === 0 || unique.length > 5000) {+ fail(`trusted visual plan has invalid size ${unique.length}`);+ }+ writeJSON(output, unique);+ process.stdout.write(`Wrote ${unique.length} trusted PR shot(s) to ${output}.\n`);+}++function plan() {+ const acceptanceFile = path.resolve(flag('acceptance'));+ const acceptance = readJSON(acceptanceFile);+ validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});+ const output = path.resolve(flag('output'));+ const shots = acceptance.keys+ .filter(entry => entry.kind !== 'removed')+ .map(entry => ({...entry.shot, key: entry.key}));+ writeJSON(output, shots);+ process.stdout.write(+ `Wrote ${shots.length} post-merge shot(s) to ${output}.\n`,+ );+}++function samePixels(first, second) {+ const a = PNG.sync.read(fs.readFileSync(first));+ const b = PNG.sync.read(fs.readFileSync(second));+ return (+ a.width === b.width &&+ a.height === b.height &&+ Buffer.compare(a.data, b.data) === 0+ );+}++function sameViewport(first, second) {+ return first?.width === second?.width && first?.height === second?.height;+}++function promote() {+ const pages = path.resolve(flag('pages'));+ const acceptanceFile = path.resolve(flag('acceptance'));+ const capture = path.resolve(flag('capture'));+ const mergeSha = flag('merge-sha') ?? '';+ if (!SHA.test(mergeSha)) fail('invalid merge SHA');++ const acceptance = readJSON(acceptanceFile);+ validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});+ const manifestFile = path.join(+ pages,+ 'visual-gate',+ 'baseline',+ 'manifest.json',+ );+ const manifest = readJSON(manifestFile);+ const baselineShots = path.join(pages, 'visual-gate', 'baseline', 'shots');+ const captureManifest = readJSON(path.join(capture, 'manifest.json'));+ const acceptedDir = path.dirname(acceptanceFile);+ const alreadyRecorded = (manifest.decisions ?? []).some(+ decision =>+ decision.pr === acceptance.pr &&+ decision.headSha === acceptance.headSha &&+ decision.mergeSha === mergeSha,+ );++ if (captureManifest.context?.sha !== mergeSha) {+ fail(+ `capture was produced for ${captureManifest.context?.sha ?? 'an unknown commit'}, not ${mergeSha}`,+ );+ }+ if (+ captureManifest.platform !== acceptance.capture.platform ||+ captureManifest.browser !== acceptance.capture.browser ||+ !sameViewport(captureManifest.viewport, acceptance.capture.viewport)+ ) {+ fail('post-merge capture environment does not match the reviewed capture');+ }++ const actions = [];+ for (const entry of acceptance.keys) {+ const baselineFile = path.join(baselineShots, `${entry.key}.png`);+ const currentBefore = fs.existsSync(baselineFile)+ ? shaFile(baselineFile)+ : null;+ if (currentBefore === entry.afterSha256 && alreadyRecorded) {+ actions.push({entry, baselineFile, alreadyPromoted: true});+ continue;+ }+ if (currentBefore !== entry.beforeSha256) {+ fail(+ `baseline conflict for ${entry.key}: expected ${entry.beforeSha256}, found ${currentBefore}`,+ );+ }+ if (entry.kind === 'removed') {+ actions.push({entry, baselineFile, remove: true});+ continue;+ }++ const acceptedAfter = path.join(acceptedDir, 'after', `${entry.key}.png`);+ const recaptured = path.join(capture, 'shots', `${entry.key}.png`);+ const capturedShot = captureManifest.shots?.[entry.key];+ if (+ !fs.existsSync(acceptedAfter) ||+ shaFile(acceptedAfter) !== entry.afterSha256+ ) {+ fail(+ `archived AFTER does not match its decision record for ${entry.key}`,+ );+ }+ if (!fs.existsSync(recaptured) || !capturedShot) {+ fail(`post-merge capture is missing ${entry.key}`);+ }+ if (capturedShot.sha256 !== shaFile(recaptured)) {+ fail(`post-merge manifest hash does not match ${entry.key}`);+ }+ for (const field of ['storyId', 'theme', 'mode']) {+ if (capturedShot[field] !== entry.shot[field]) {+ fail(+ `post-merge ${field} does not match accepted plan for ${entry.key}`,+ );+ }+ }+ if (!samePixels(acceptedAfter, recaptured)) {+ fail(`post-merge pixels do not match accepted AFTER for ${entry.key}`);+ }+ actions.push({entry, baselineFile, recaptured, capturedShot});+ }++ if (actions.every(action => action.alreadyPromoted)) {+ process.stdout.write(`Accepted visual bundle for PR #${acceptance.pr} was already promoted.\n`);+ return;+ }++ for (const action of actions) {+ if (action.alreadyPromoted) continue;+ if (action.remove) {+ if (fs.existsSync(action.baselineFile)) fs.rmSync(action.baselineFile);+ delete manifest.shots[action.entry.key];+ } else {+ fs.copyFileSync(action.recaptured, action.baselineFile);+ manifest.shots[action.entry.key] = action.capturedShot;+ }+ }++ manifest.capturedAt = captureManifest.capturedAt;+ manifest.platform = captureManifest.platform;+ manifest.browser = captureManifest.browser;+ manifest.viewport = captureManifest.viewport;+ manifest.context = {sha: mergeSha};+ manifest.decisions = [+ ...(manifest.decisions ?? []),+ {+ at: new Date().toISOString(),+ actor: acceptance.decision.approver,+ reason: acceptance.decision.reason,+ pr: acceptance.pr,+ headSha: acceptance.headSha,+ mergeSha,+ promoted: acceptance.keys+ .filter(entry => entry.kind !== 'removed')+ .map(entry => entry.key),+ pruned: acceptance.keys+ .filter(entry => entry.kind === 'removed')+ .map(entry => entry.key),+ },+ ].slice(-200);+ writeJSON(manifestFile, manifest);+ process.stdout.write(+ `Promoted accepted visual bundle for PR #${acceptance.pr}.\n`,+ );+}++switch (command) {+ case 'accept':+ accept();+ break;+ case 'state':+ state();+ break;+ case 'trusted-plan':+ trustedPlan();+ break;+ case 'plan':+ plan();+ break;+ case 'promote':+ promote();+ break;+ default:+ fail('usage: visual-acceptance.mjs <accept|state|trusted-plan|plan|promote>');+}diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjsnew file mode 100644index 0000000000..3e3f1f576f--- /dev/null+++ b/.github/scripts/visual-gate/visual-acceptance.test.mjs@@ -0,0 +1,611 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync, spawnSync} from 'node:child_process';+import {createHash} from 'node:crypto';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';++import {afterEach, beforeEach, describe, expect, it} from 'vitest';+import {PNG} from 'pngjs';++const SCRIPT = path.join(+ path.dirname(fileURLToPath(import.meta.url)),+ 'visual-acceptance.mjs',+);+const GATE = path.join(+ path.dirname(fileURLToPath(import.meta.url)),+ 'gate.mjs',+);+const HEAD = 'a'.repeat(40);+const TESTED = 'b'.repeat(40);+const MERGE = 'c'.repeat(40);+const KEY = 'core-button--default__neutral-light';+const SHOT = {+ storyId: 'core-button--default',+ title: 'Core/Button',+ name: 'Default',+ component: 'Button',+ theme: 'neutral',+ mode: 'light',+ reasons: ['component:Button'],+};++let root;+let pages;++function png(red, green = 0, blue = 0) {+ const image = new PNG({width: 2, height: 2});+ for (let offset = 0; offset < image.data.length; offset += 4) {+ image.data[offset] = red;+ image.data[offset + 1] = green;+ image.data[offset + 2] = blue;+ image.data[offset + 3] = 255;+ }+ return PNG.sync.write(image);+}++function digest(bytes) {+ return createHash('sha256').update(bytes).digest('hex');+}++function writeJSON(file, value) {+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);+}++function run(command, flags = {}) {+ const args = [SCRIPT, command];+ for (const [name, value] of Object.entries(flags))+ args.push(`--${name}`, String(value));+ return execFileSync(process.execPath, args, {encoding: 'utf8'});+}++function fail(command, flags = {}) {+ const args = [SCRIPT, command];+ for (const [name, value] of Object.entries(flags))+ args.push(`--${name}`, String(value));+ const result = spawnSync(process.execPath, args, {encoding: 'utf8'});+ expect(result.status).not.toBe(0);+ return result.stderr;+}++function acceptanceFlags(overrides = {}) {+ return {+ pages,+ pr: 42,+ head: HEAD,+ 'run-id': 123,+ 'run-attempt': 1,+ approver: 'maintainer',+ 'approver-id': 99,+ permission: 'maintain',+ 'comment-id': 1234,+ reason: 'The new radius matches the approved component design.',+ ...overrides,+ };+}++function acceptanceFile(run = 123, attempt = 1) {+ return path.join(+ pages,+ 'visual-gate',+ 'acceptances',+ '42',+ HEAD,+ String(run),+ String(attempt),+ 'acceptance.json',+ );+}++function writeBaseline(entries = {[KEY]: png(255)}) {+ const shots = {};+ for (const [key, bytes] of Object.entries(entries)) {+ const file = path.join(+ pages,+ 'visual-gate',+ 'baseline',+ 'shots',+ `${key}.png`,+ );+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, bytes);+ shots[key] = {...SHOT, key, sha256: digest(bytes), width: 2, height: 2};+ }+ writeJSON(path.join(pages, 'visual-gate', 'baseline', 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ shots,+ decisions: [],+ });+}++function writeEvidence({+ kind = 'changed',+ after = png(0, 0, 255),+ key = KEY,+ run = 123,+ attempt = 1,+ status = 'changed',+} = {}) {+ const dir = path.join(pages, 'pr', '42', 'visual', HEAD, String(run), String(attempt));+ fs.mkdirSync(path.join(dir, 'after'), {recursive: true});+ if (status === 'changed' && kind !== 'removed') {+ fs.writeFileSync(path.join(dir, 'after', `${key}.png`), after);+ }+ const deltas =+ status === 'changed'+ ? [+ {+ key,+ kind,+ beforeSha256:+ kind === 'added'+ ? null+ : digest(+ fs.readFileSync(+ path.join(+ pages,+ 'visual-gate',+ 'baseline',+ 'shots',+ `${key}.png`,+ ),+ ),+ ),+ shot: kind === 'removed' ? null : SHOT,+ },+ ]+ : [];+ writeJSON(path.join(dir, 'evidence.json'), {+ version: 1,+ repo: 'facebook/astryx',+ pr: 42,+ headSha: HEAD,+ testedSha: TESTED,+ baseSha: 'd'.repeat(40),+ run: {id: run, attempt},+ capture:+ status === 'skipped'+ ? null+ : {+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ },+ deltas,+ verdict: {version: 1, status},+ });+ return dir;+}++function commitPages() {+ execFileSync('git', ['add', '.'], {cwd: pages});+ execFileSync('git', ['commit', '-qm', 'fixture'], {cwd: pages});+}++beforeEach(() => {+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-acceptance-'));+ pages = path.join(root, 'pages');+ fs.mkdirSync(pages);+ execFileSync('git', ['init', '-q', '-b', 'gh-pages'], {cwd: pages});+ execFileSync('git', ['config', 'user.name', 'Test'], {cwd: pages});+ execFileSync('git', ['config', 'user.email', '[email protected]'], {+ cwd: pages,+ });+ writeBaseline();+ writeEvidence();+ commitPages();+});++afterEach(() => fs.rmSync(root, {recursive: true, force: true}));++describe('visual acceptance', () => {+ it('archives the reviewed pixels and a complete decision record', () => {+ expect(run('accept', acceptanceFlags())).toContain(+ 'Accepted 1 visual delta',+ );+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));+ expect(record).toMatchObject({+ version: 1,+ pr: 42,+ headSha: HEAD,+ testedSha: TESTED,+ run: {id: 123, attempt: 1},+ capture: {platform: 'linux-arm64', browser: 'chromium-140.0'},+ decision: {+ approver: 'maintainer',+ approverId: 99,+ permission: 'maintain',+ commentId: 1234,+ },+ });+ expect(record.keys[0]).toMatchObject({key: KEY, kind: 'changed'});+ expect(record.keys[0].beforeSha256).toMatch(/^[0-9a-f]{64}$/);+ expect(record.keys[0].afterSha256).toMatch(/^[0-9a-f]{64}$/);+ expect(+ fs.existsSync(+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),+ ),+ ).toBe(true);+ });++ it('requires an explanatory reason and maintainer permission', () => {+ expect(fail('accept', acceptanceFlags({reason: 'intentional...'}))).toMatch(+ /reason must explain/,+ );+ expect(fail('accept', acceptanceFlags({permission: 'write'}))).toMatch(+ /maintain\/admin/,+ );+ });++ it('binds acceptance to the exact reviewed run attempt', () => {+ writeEvidence({run: 123, attempt: 2, after: png(0, 255, 0)});+ run('accept', acceptanceFlags({'run-id': 123, 'run-attempt': 1}));+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));+ expect(record.run).toEqual({id: 123, attempt: 1});+ expect(record.keys[0].afterSha256).toBe(digest(png(0, 0, 255)));+ });++ it('reports whether the current visual bundle is clean, waiting, accepted, or failed', () => {+ const stateFlags = {pages, pr: 42, head: HEAD};+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({+ state: 'pending',+ reason: 'decision',+ });+ run('accept', acceptanceFlags());+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({+ state: 'success',+ reason: 'accepted',+ });++ writeEvidence({run: 124, attempt: 1, after: png(0, 0, 255)});+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({+ state: 'pending',+ reason: 'decision',+ });+ writeEvidence({run: 124, attempt: 2, after: png(0, 255, 0)});+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({+ state: 'pending',+ reason: 'decision',+ });+ writeEvidence({run: 125, status: 'skipped'});+ expect(JSON.parse(run('state', stateFlags))).toMatchObject({+ state: 'success',+ reason: 'deferred',+ });+ });++ it('derives a trusted component plan from baseline themes and the Storybook index', () => {+ const storybook = path.join(root, 'storybook');+ fs.mkdirSync(storybook);+ writeJSON(path.join(storybook, 'index.json'), {+ entries: {+ button: {+ type: 'story',+ id: 'core-button--default',+ title: 'Core/Button',+ name: 'Default',+ tags: [],+ },+ },+ });+ const scope = path.join(root, 'scope.json');+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: false,+ stableComponents: ['Button'],+ stableThemes: [],+ });+ const output = path.join(root, 'trusted-plan.json');+ run('trusted-plan', {+ scope,+ baseline: path.join(pages, 'visual-gate', 'baseline'),+ 'storybook-dir': storybook,+ output,+ });+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([+ 'core-button--default__neutral-light',+ 'core-button--default__neutral-dark',+ ]);+ });++ it('plans every baseline story for a newly promoted stable theme', () => {+ const storybook = path.join(root, 'storybook-new-theme');+ fs.mkdirSync(storybook);+ writeJSON(path.join(storybook, 'index.json'), {entries: {}});+ const scope = path.join(root, 'scope-new-theme.json');+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: false,+ stableComponents: [],+ stableThemes: ['new-theme'],+ });+ const output = path.join(root, 'trusted-theme-plan.json');+ run('trusted-plan', {+ scope,+ baseline: path.join(pages, 'visual-gate', 'baseline'),+ 'storybook-dir': storybook,+ output,+ });+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([+ 'core-button--default__new-theme-light',+ 'core-button--default__new-theme-dark',+ ]);+ });++ it('uses the full baseline plan for broad stable infrastructure', () => {+ const scope = path.join(root, 'scope-broad.json');+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: true,+ stableComponents: [],+ stableThemes: [],+ });+ const output = path.join(root, 'trusted-broad-plan.json');+ run('trusted-plan', {+ scope,+ baseline: path.join(pages, 'visual-gate', 'baseline'),+ 'storybook-dir': root,+ output,+ });+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([KEY]);+ });++ it('allows the trusted 520-shot broad plan through the capture CLI', () => {+ const plan = Array.from({length: 520}, (_, index) => ({+ key: `story-${index}__neutral-light`,+ storyId: `story-${index}`,+ title: `Core/C${index}`,+ name: 'Default',+ component: `C${index}`,+ theme: 'neutral',+ mode: 'light',+ reasons: ['trusted:broad'],+ }));+ const file = path.join(root, 'large-plan.json');+ writeJSON(file, plan);+ const printed = execFileSync(process.execPath, [GATE, 'plan', '--plan-file', file], {+ encoding: 'utf8',+ });+ expect(printed).toContain('520 shots');+ });++ it('keeps new component stories in a mixed broad plan', () => {+ const storybook = path.join(root, 'storybook-mixed');+ fs.mkdirSync(storybook);+ writeJSON(path.join(storybook, 'index.json'), {+ entries: {+ newStory: {+ type: 'story',+ id: 'core-new--default',+ title: 'Core/New',+ name: 'Default',+ tags: [],+ },+ },+ });+ const scope = path.join(root, 'scope-mixed.json');+ writeJSON(scope, {+ hasStableVisual: true,+ broadStableVisual: true,+ stableComponents: ['New'],+ stableThemes: [],+ });+ const output = path.join(root, 'trusted-mixed-plan.json');+ run('trusted-plan', {+ scope,+ baseline: path.join(pages, 'visual-gate', 'baseline'),+ 'storybook-dir': storybook,+ output,+ });+ expect(JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key)).toEqual([+ KEY,+ 'core-new--default__neutral-light',+ 'core-new--default__neutral-dark',+ ]);+ });++ it('accepts a trusted 520-delta broad bundle', () => {+ const runId = 130;+ const dir = path.join(pages, 'pr', '42', 'visual', HEAD, String(runId), '1');+ const deltas = [];+ fs.mkdirSync(path.join(dir, 'after'), {recursive: true});+ for (let index = 0; index < 520; index += 1) {+ const key = `story-${index}__neutral-light`;+ fs.writeFileSync(path.join(dir, 'after', `${key}.png`), png(index % 255));+ deltas.push({+ key,+ kind: 'added',+ beforeSha256: null,+ shot: {...SHOT, storyId: `story-${index}`},+ });+ }+ writeJSON(path.join(dir, 'evidence.json'), {+ version: 1,+ repo: 'facebook/astryx',+ pr: 42,+ headSha: HEAD,+ testedSha: TESTED,+ baseSha: 'd'.repeat(40),+ run: {id: runId, attempt: 1},+ capture: {+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ },+ deltas,+ verdict: {version: 1, status: 'changed'},+ });+ run('accept', acceptanceFlags({'run-id': runId}));+ const record = JSON.parse(fs.readFileSync(acceptanceFile(runId), 'utf8'));+ expect(record.keys).toHaveLength(520);+ });++ it('emits an exact recapture plan without allowing stored metadata to replace the key', () => {+ run('accept', acceptanceFlags());+ const record = JSON.parse(fs.readFileSync(acceptanceFile(), 'utf8'));+ record.keys[0].shot.key = '../wrong';+ writeJSON(acceptanceFile(), record);+ const output = path.join(root, 'plan.json');+ run('plan', {acceptance: acceptanceFile(), output});+ expect(JSON.parse(fs.readFileSync(output, 'utf8'))).toEqual([+ {...SHOT, key: KEY},+ ]);+ });++ it('feeds the accepted shots back into the capture CLI unchanged', () => {+ run('accept', acceptanceFlags());+ const output = path.join(root, 'plan.json');+ run('plan', {acceptance: acceptanceFile(), output});+ const printed = execFileSync(+ process.execPath,+ [GATE, 'plan', '--plan-file', output, '--json'],+ {+ encoding: 'utf8',+ },+ );+ expect(JSON.parse(printed)).toEqual([{...SHOT, key: KEY}]);+ });++ it('promotes only when merged pixels equal the reviewed after image', () => {+ run('accept', acceptanceFlags());+ const after = fs.readFileSync(+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),+ );+ const capture = path.join(root, 'capture');+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), after);+ writeJSON(path.join(capture, 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ capturedAt: '2026-08-26T22:00:00.000Z',+ context: {sha: MERGE},+ shots: {[KEY]: {...SHOT, sha256: digest(after), width: 2, height: 2}},+ });++ run('promote', {+ pages,+ acceptance: acceptanceFile(),+ capture,+ 'merge-sha': MERGE,+ });+ run('promote', {+ pages,+ acceptance: acceptanceFile(),+ capture,+ 'merge-sha': MERGE,+ });+ expect(+ fs.readFileSync(+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),+ ),+ ).toEqual(after);+ const manifest = JSON.parse(+ fs.readFileSync(+ path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),+ 'utf8',+ ),+ );+ expect(manifest.context).toEqual({sha: MERGE});+ expect(manifest.decisions).toHaveLength(1);+ expect(manifest.decisions.at(-1)).toMatchObject({+ pr: 42,+ headSha: HEAD,+ mergeSha: MERGE,+ });+ });++ it('rejects changed post-merge pixels and a stale merge identity', () => {+ run('accept', acceptanceFlags());+ const capture = path.join(root, 'capture');+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), png(0, 255, 0));+ writeJSON(path.join(capture, 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ context: {sha: 'e'.repeat(40)},+ shots: {+ [KEY]: {...SHOT, sha256: digest(png(0, 255, 0)), width: 2, height: 2},+ },+ });+ expect(+ fail('promote', {+ pages,+ acceptance: acceptanceFile(),+ capture,+ 'merge-sha': MERGE,+ }),+ ).toMatch(/capture was produced for/);+ });++ it('rejects a concurrent baseline change instead of overwriting it', () => {+ run('accept', acceptanceFlags());+ fs.writeFileSync(+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),+ png(0, 255, 0),+ );+ expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(+ {+ state: 'pending',+ reason: 'capture',+ },+ );+ const capture = path.join(root, 'capture');+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});+ const accepted = fs.readFileSync(+ path.join(path.dirname(acceptanceFile()), 'after', `${KEY}.png`),+ );+ fs.writeFileSync(path.join(capture, 'shots', `${KEY}.png`), accepted);+ writeJSON(path.join(capture, 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ context: {sha: MERGE},+ shots: {[KEY]: {...SHOT, sha256: digest(accepted), width: 2, height: 2}},+ });+ expect(+ fail('promote', {+ pages,+ acceptance: acceptanceFile(),+ capture,+ 'merge-sha': MERGE,+ }),+ ).toMatch(/baseline conflict/);+ });++ it('records added and removed shots without inventing missing images', () => {+ const added = 'core-new--default__neutral-light';+ writeEvidence({kind: 'added', key: added, run: 124});+ run('accept', acceptanceFlags({'run-id': 124}));+ let record = JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8'));+ expect(record.keys[0]).toMatchObject({+ key: added,+ kind: 'added',+ beforeSha256: null,+ });++ fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {+ recursive: true,+ force: true,+ });+ writeEvidence({kind: 'removed', run: 125});+ run('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}));+ record = JSON.parse(fs.readFileSync(acceptanceFile(125), 'utf8'));+ expect(record.keys[0]).toMatchObject({+ key: KEY,+ kind: 'removed',+ afterSha256: null,+ shot: null,+ });+ });+});diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjsnew file mode 100644index 0000000000..5438e7588a--- /dev/null+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs@@ -0,0 +1,69 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';++import {describe, expect, it} from 'vitest';++const ROOT = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ '../../..',+);+const WORKFLOWS = path.join(ROOT, '.github', 'workflows');++function workflow(name) {+ return fs.readFileSync(path.join(WORKFLOWS, name), 'utf8');+}++describe('visual acceptance workflow concurrency', () => {+ it('locks PR report resolution before any mutation', () => {+ const value = workflow('pr-comment.yml');+ const [header, jobs] = value.split('\njobs:\n');++ expect(header).toContain(+ 'group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}',+ );+ expect(header).toContain(+ "cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}",+ );+ expect(jobs).not.toContain('visual-acceptance-pr-');+ expect(jobs).not.toContain(' concurrency:');+ });++ it('keeps comment authorization read-only until the shared lock is held', () => {+ const value = workflow('visual-acceptance.yml');+ const initialize = value.slice(+ value.indexOf(' initialize:'),+ value.indexOf(' authorize:'),+ );+ const authorize = value.slice(+ value.indexOf(' authorize:'),+ value.indexOf(' accept:'),+ );+ const accept = value.slice(value.indexOf(' accept:'));++ expect(initialize).toContain(+ 'group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}',+ );+ expect(initialize).toContain('cancel-in-progress: true');+ expect(authorize).not.toContain(': write');+ expect(authorize).not.toContain('actions/checkout');+ expect(authorize).not.toContain('issues.createComment');+ expect(authorize).not.toContain('createCommitStatus');+ expect(accept).toContain('needs: authorize');+ expect(accept).toContain(+ 'group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}',+ );+ expect(accept).toContain('cancel-in-progress: false');+ });++ it('uses the same head identity for post-merge promotion', () => {+ const value = workflow('visual-acceptance-promote.yml');++ expect(value).toContain(+ 'group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}',+ );+ expect(value).not.toContain('visual-acceptance-pr-');+ });+});diff --git a/.github/scripts/visual-scope.mjs b/.github/scripts/visual-scope.mjsindex 94e3701945..6ccc4ddae9 100644--- a/.github/scripts/visual-scope.mjs+++ b/.github/scripts/visual-scope.mjs@@ -50,7 +50,15 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {);const stableThemes = new Set();+ const stableComponents = new Set();const canaryPackages = new Set();+ let broadStableVisual = false;++ for (const file of stableCoreFiles) {+ const match = file.match(/^packages\/core\/src\/([^/]+)\//);+ if (match && /^[A-Z]/.test(match[1])) stableComponents.add(match[1]);+ else broadStableVisual = true;+ }for (const file of paths) {const pkgMatch = file.match(/^packages\/([^/]+)\//);@@ -70,19 +78,27 @@ export function classifyVisualScope(files, repoRoot = ROOT, manifests = {}) {themeMatch[1],'package.json',);- const pkg = readPackage(repoRoot, relativeManifest, manifests);- if (!pkg) continue;- if (pkg.private === true || pkg.astryx?.canaryOnly === true) {- if (pkg.astryx?.canaryOnly === true) {- canaryPackages.add(pkg.name ?? themeMatch[1]);- }+ const baseManifest = path.join(repoRoot, relativeManifest);+ const basePkg = fs.existsSync(baseManifest) ? readJSON(baseManifest) : null;+ const headPkg = Object.hasOwn(manifests, relativeManifest)+ ? manifests[relativeManifest]+ : basePkg;+ const isStable = pkg => pkg && pkg.private !== true && pkg.astryx?.canaryOnly !== true;+ // Base metadata is the fail-closed floor: a PR may promote a theme into the+ // stable lane, but cannot escape review by demoting an existing stable one.+ if (isStable(basePkg) || isStable(headPkg)) {+ stableThemes.add(themeMatch[1]);continue;}- stableThemes.add(themeMatch[1]);+ if (headPkg?.astryx?.canaryOnly === true) {+ canaryPackages.add(headPkg.name ?? themeMatch[1]);+ }}return {hasStableVisual: stableCoreFiles.length > 0 || stableThemes.size > 0,+ broadStableVisual,+ stableComponents: [...stableComponents].sort(),stableCoreFiles,stableThemes: [...stableThemes].sort(),canaryPackages: [...canaryPackages].sort(),diff --git a/.github/scripts/visual-scope.test.mjs b/.github/scripts/visual-scope.test.mjsindex bde97e7dfc..dc42e6c175 100644--- a/.github/scripts/visual-scope.test.mjs+++ b/.github/scripts/visual-scope.test.mjs@@ -50,10 +50,23 @@ afterEach(() => fs.rmSync(root, {recursive: true, force: true}));describe('classifyVisualScope', () => {it('includes stable Core runtime changes', () => {const result = classifyVisualScope(['packages/core/src/Button/Button.tsx'], root);- expect(result.hasStableVisual).toBe(true);+ expect(result).toMatchObject({+ hasStableVisual: true,+ stableComponents: ['Button'],+ broadStableVisual: false,+ });expect(result.stableCoreFiles).toEqual(['packages/core/src/Button/Button.tsx']);});+ it('marks shared Core infrastructure as broad stable scope', () => {+ const result = classifyVisualScope(['packages/core/src/theme/Theme.tsx'], root);+ expect(result).toMatchObject({+ hasStableVisual: true,+ broadStableVisual: true,+ stableComponents: [],+ });+ });+it('ignores Core tests and component docs — they do not change pixels', () => {const result = classifyVisualScope([@@ -73,6 +86,21 @@ describe('classifyVisualScope', () => {expect(result).toMatchObject({hasStableVisual: true, stableThemes: ['neutral']});});+ it('keeps a base-stable theme in scope when the PR marks it private', () => {+ const result = classifyVisualScope(+ ['packages/themes/neutral/package.json'],+ root,+ {+ 'packages/themes/neutral/package.json': {+ name: '@astryxdesign/theme-neutral',+ private: true,+ astryx: {canaryOnly: true},+ },+ },+ );+ expect(result).toMatchObject({hasStableVisual: true, stableThemes: ['neutral']});+ });+it('uses trusted PR-head metadata for a promoted theme', () => {const result = classifyVisualScope(['packages/themes/probe/package.json'],diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.ymlindex c5d85a84a5..6c7b43960f 100644--- a/.github/workflows/pr-comment.yml+++ b/.github/workflows/pr-comment.yml@@ -9,34 +9,102 @@# write), so it can comment on fork PRs too — one code path for every PR, no# drift between a fork path and a same-repo path.#-# Safety: resolves the PR from the GitHub API, then cross-checks every artifact's-# claimed PR/head/run identity. Visual evidence is untrusted: default-branch-# code schema-validates it, rejects unsafe paths/symlinks, decodes and-# re-encodes PNGs, and generates the HTML wrapper itself. PR-produced HTML/JS-# is never published or executed. Do not add checkout/execution of the PR head.+# Safety: resolves scope and PR identity from the GitHub API. Default-branch+# code builds the capture plan, serves the PR's static Storybook to an isolated+# browser with off-origin requests blocked, derives the comparison, re-encodes+# PNGs, and generates the report. PR code is never checked out, imported into+# Node, or given credentials; PR-produced HTML/JS is never published.name: PR Commenton:workflow_run:- workflows: ["CI"]- types: [completed]+ workflows: ['CI']+ types: [requested, in_progress, completed]permissions: {}-# A newer run for the same branch supersedes an older comment/publication. The-# trusted identity step also fails a stale run after the PR head moves.+# The event already carries a stable head-repository/branch identity. Acquire+# this shared lock before resolving the PR so a rerun can cancel any older+# acceptance, report publication, or promotion mutation for the same head.concurrency:- group: pr-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}- cancel-in-progress: true+ group: visual-acceptance-head-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }}+ cancel-in-progress: ${{ github.event.action == 'requested' || github.event.action == 'in_progress' }}jobs:- comment:- # Only for CI runs triggered by a pull request.+ resolve:if: github.event.workflow_run.event == 'pull_request'runs-on: ubuntu-slim+ permissions:+ pull-requests: read+ outputs:+ valid: ${{ steps.identity.outputs.valid }}+ pr_number: ${{ steps.identity.outputs.pr_number }}+ head_sha: ${{ steps.identity.outputs.head_sha }}+ steps:+ - name: Resolve trusted PR identity+ id: identity+ uses: actions/github-script@v9+ with:+ script: |+ const run = context.payload.workflow_run;+ const {owner, repo} = context.repo;+ let numbers = (run.pull_requests || []).map((pr) => pr.number);+ if (numbers.length === 0) {+ const associated = await github.paginate(+ github.rest.repos.listPullRequestsAssociatedWithCommit,+ {owner, repo, commit_sha: run.head_sha, per_page: 100},+ );+ numbers = associated.filter((pr) => pr.state === 'open').map((pr) => pr.number);+ }+ numbers = [...new Set(numbers)];+ if (numbers.length !== 1) {+ core.setFailed(`Expected exactly one open PR for run ${run.id}; found ${numbers.join(',') || 'none'}`);+ return;+ }+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: numbers[0]});+ if (pr.head.sha !== run.head_sha) {+ core.setFailed(`Run head ${run.head_sha} is stale; PR head is ${pr.head.sha}`);+ return;+ }+ core.setOutput('valid', 'true');+ core.setOutput('pr_number', String(pr.number));+ core.setOutput('head_sha', pr.head.sha);++ invalidate:+ needs: resolve+ if: >-+ (github.event.action == 'requested' || github.event.action == 'in_progress') &&+ needs.resolve.outputs.valid == 'true'+ runs-on: ubuntu-slim+ permissions:+ statuses: write+ steps:+ - name: Mark same-head reruns pending before evidence can change+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ const run = context.payload.workflow_run;+ await github.rest.repos.createCommitStatus({+ ...context.repo,+ sha: '${{ needs.resolve.outputs.head_sha }}',+ context: 'visual-acceptance',+ state: 'pending',+ description: `CI run ${run.id}/${run.run_attempt} is producing fresh visual evidence.`,+ });++ comment:+ needs: resolve+ # Only for completed CI runs whose PR identity was resolved above.+ if: >-+ github.event.action == 'completed' &&+ needs.resolve.outputs.valid == 'true'+ runs-on: 2-core-ubuntu-armpermissions:pull-requests: write+ issues: write+ statuses: writeactions: readcontents: write # immutable validated visual evidence on gh-pagessteps:@@ -89,9 +157,60 @@ jobs:core.setOutput('pr_number', String(pr.number));core.setOutput('head_sha', pr.head.sha);core.setOutput('head_repo', pr.head.repo.full_name);+ core.setOutput('base_sha', pr.base.sha);core.setOutput('run_id', String(run.id));core.setOutput('run_attempt', String(run.run_attempt));+ - name: Derive trusted stable visual scope+ id: scope+ if: steps.identity.outputs.valid == 'true'+ uses: actions/github-script@v9+ with:+ script: |+ const fs = require('node:fs');+ const {spawnSync} = require('node:child_process');+ const {owner, repo} = context.repo;+ const pullNumber = Number('${{ steps.identity.outputs.pr_number }}');+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: pullNumber});+ const files = await github.paginate(github.rest.pulls.listFiles, {+ owner, repo, pull_number: pullNumber, per_page: 100,+ });+ const manifestPaths = new Set();+ for (const {filename} of files) {+ const theme = filename.match(/^packages\/themes\/([^/]+)\//);+ const pkg = filename.match(/^packages\/([^/]+)\//);+ if (theme) manifestPaths.add(`packages/themes/${theme[1]}/package.json`);+ else if (pkg) manifestPaths.add(`packages/${pkg[1]}/package.json`);+ }+ const manifests = {};+ const [headOwner, headRepo] = pr.head.repo.full_name.split('/');+ for (const manifestPath of manifestPaths) {+ try {+ const {data} = await github.rest.repos.getContent({+ owner: headOwner, repo: headRepo, path: manifestPath, ref: pr.head.sha,+ });+ if (!Array.isArray(data) && data.content) {+ manifests[manifestPath] = JSON.parse(Buffer.from(data.content, 'base64').toString());+ }+ } catch (error) {+ if (error.status !== 404) throw error;+ }+ }+ fs.writeFileSync('visual-scope-manifests.json', JSON.stringify(manifests));+ const result = spawnSync(+ process.execPath,+ ['.github/scripts/visual-scope.mjs', '--manifests', 'visual-scope-manifests.json'],+ {input: `${files.map((file) => file.filename).join('\n')}\n`, encoding: 'utf8'},+ );+ if (result.status !== 0) {+ core.setFailed(result.stderr || 'Stable visual scope classification failed.');+ return;+ }+ const scope = JSON.parse(result.stdout);+ fs.writeFileSync('trusted-scope.json', JSON.stringify(scope));+ core.setOutput('stable', String(scope.hasStableVisual));+ core.setOutput('broad', String(scope.broadStableVisual));+- name: Download analysis artifact from the CI runif: steps.identity.outputs.valid == 'true'uses: actions/download-artifact@v8@@ -112,15 +231,15 @@ jobs:github-token: ${{ secrets.GITHUB_TOKEN }}continue-on-error: true- - name: Download visual verdict from the CI run- if: steps.identity.outputs.valid == 'true'+ - name: Download Storybook for trusted visual capture+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'uses: actions/download-artifact@v8with:- name: visual-pr-report- path: visual+ pattern: storybook-*+ path: apps/storybook/dist+ merge-multiple: truerun-id: ${{ github.event.workflow_run.id }}github-token: ${{ secrets.GITHUB_TOKEN }}- continue-on-error: true- name: Cross-check artifact identityif: steps.identity.outputs.valid == 'true' && hashFiles('pr-analysis/pr-meta.json') != ''@@ -140,12 +259,40 @@ jobs:if (String(meta.runId) !== process.env.EXPECTED_RUN) fail('artifact run id does not match workflow_run');NODE- # The artifact is untrusted. Default-branch code validates its schema,- # keys, identity and PNG bytes, decodes/re-encodes every image, and- # generates the HTML wrapper. PR-produced HTML/JS is never published.- - name: Validate visual evidence and generate trusted report+ - name: Fetch the trusted visual baseline+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'+ run: |+ rm -rf /tmp/gh-pages+ git clone --depth=1 --single-branch --branch gh-pages \+ "https://github.com/${GITHUB_REPOSITORY}.git" /tmp/gh-pages++ - name: Capture the trusted stable visual scope+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'+ env:+ GITHUB_SHA: ${{ steps.identity.outputs.head_sha }}+ GITHUB_RUN_ID: ${{ steps.identity.outputs.run_id }}+ GITHUB_RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}+ ASTRYX_PR_HEAD_SHA: ${{ steps.identity.outputs.head_sha }}+ ASTRYX_PR_BASE_SHA: ${{ steps.identity.outputs.base_sha }}+ run: |+ npx playwright install chromium+ node .github/scripts/visual-gate/visual-acceptance.mjs trusted-plan \+ --scope trusted-scope.json \+ --baseline /tmp/gh-pages/visual-gate/baseline \+ --storybook-dir apps/storybook/dist \+ --output trusted-plan.json+ node .github/scripts/visual-gate/gate.mjs capture \+ --storybook-dir apps/storybook/dist \+ --out trusted-capture \+ --plan-file trusted-plan.json++ # The Storybook bundle is untrusted. Default-branch code chooses the plan,+ # captures it in an off-origin-blocked browser, derives the comparison,+ # validates/re-encodes PNGs, and generates the report. PR-authored verdicts+ # and job conclusions are never authority for the required status.+ - name: Derive trusted visual evidence and reportid: visual- if: steps.identity.outputs.valid == 'true' && hashFiles('visual/verdict.json') != ''+ if: steps.identity.outputs.valid == 'true' && steps.scope.outputs.stable == 'true'env:PR_NUMBER: ${{ steps.identity.outputs.pr_number }}HEAD_SHA: ${{ steps.identity.outputs.head_sha }}@@ -153,14 +300,16 @@ jobs:RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}run: |node .github/scripts/visual-gate/publish-pr-report.mjs \- --input visual \+ --input trusted-capture \--output trusted-visual \+ --baseline /tmp/gh-pages/visual-gate/baseline \+ --scope trusted-scope.json \--pr "$PR_NUMBER" \--head-sha "$HEAD_SHA" \--run-id "$RUN_ID" \--run-attempt "$RUN_ATTEMPT"echo "ready=true" >> "$GITHUB_OUTPUT"- echo "path=pr/${PR_NUMBER}/visual/${HEAD_SHA}/${RUN_ID}" >> "$GITHUB_OUTPUT"+ echo "path=pr/${PR_NUMBER}/visual/${HEAD_SHA}/${RUN_ID}/${RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"- name: Publish immutable visual evidenceid: publish@@ -247,9 +396,9 @@ jobs:'.github/scripts/generate-pr-comment.js','--analysis', 'pr-analysis/analysis.json','--a11y', a11yPath,- ...(fs.existsSync('visual/verdict.json')+ ...(fs.existsSync('trusted-visual/verdict.json')? [- '--visual', 'visual/verdict.json',+ '--visual', 'trusted-visual/verdict.json',...(process.env.VISUAL_PUBLISHED === 'true'? ['--visual-report-url',@@ -288,3 +437,72 @@ jobs:});core.info(`Posted PR Analysis Report on #${prNumber}.`);}++ - name: Evaluate visual acceptance state+ if: always() && steps.identity.outputs.valid == 'true'+ env:+ SCOPE_OUTCOME: ${{ steps.scope.outcome }}+ HAS_STABLE_VISUAL: ${{ steps.scope.outputs.stable }}+ VISUAL_READY: ${{ steps.visual.outputs.ready }}+ VISUAL_PUBLISHED: ${{ steps.publish.outputs.published }}+ PR_NUMBER: ${{ steps.identity.outputs.pr_number }}+ HEAD_SHA: ${{ steps.identity.outputs.head_sha }}+ run: |+ if [ "$SCOPE_OUTCOME" != "success" ]; then+ printf '%s\n' '{"state":"failure","reason":"scope","description":"Stable visual scope could not be evaluated."}' > visual-state.json+ elif [ "$HAS_STABLE_VISUAL" != "true" ]; then+ printf '%s\n' '{"state":"success","reason":"scope","description":"No stable visual scope."}' > visual-state.json+ elif [ "$VISUAL_READY" != "true" ] || [ "$VISUAL_PUBLISHED" != "true" ]; then+ printf '%s\n' '{"state":"failure","reason":"capture","description":"Stable visual evidence could not be published."}' > visual-state.json+ else+ node .github/scripts/visual-gate/visual-acceptance.mjs state \+ --pages /tmp/gh-pages --pr "$PR_NUMBER" --head "$HEAD_SHA" > visual-state.json+ fi++ - name: Publish visual acceptance status+ if: always() && steps.identity.outputs.valid == 'true'+ uses: actions/github-script@v9+ env:+ PR_NUMBER: ${{ steps.identity.outputs.pr_number }}+ HEAD_SHA: ${{ steps.identity.outputs.head_sha }}+ VISUAL_PATH: ${{ steps.visual.outputs.path }}+ with:+ retries: 3+ script: |+ const fs = require('node:fs');+ const {owner, repo} = context.repo;+ const pr = Number(process.env.PR_NUMBER);+ let state = {state: 'failure', reason: 'state', description: 'Visual acceptance state could not be evaluated.'};+ if (fs.existsSync('visual-state.json')) {+ try {+ state = JSON.parse(fs.readFileSync('visual-state.json', 'utf8'));+ } catch (error) {+ core.warning(`Could not read visual state: ${error.message}`);+ }+ }+ await github.rest.repos.createCommitStatus({+ owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance',+ state: state.state, description: state.description.slice(0, 140),+ ...(process.env.VISUAL_PATH+ ? {target_url: `https://facebook.github.io/astryx/${process.env.VISUAL_PATH}/`}+ : {}),+ });+ const label = 'visual-approved';+ if (state.reason === 'accepted') {+ try {+ await github.rest.issues.getLabel({owner, repo, name: label});+ } catch (error) {+ if (error.status !== 404) throw error;+ await github.rest.issues.createLabel({+ owner, repo, name: label, color: '2da44e',+ description: 'Current visual bundle has an explicit acceptance record',+ });+ }+ await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});+ } else {+ try {+ await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});+ } catch (error) {+ if (error.status !== 404) throw error;+ }+ }diff --git a/.github/workflows/visual-acceptance-promote.yml b/.github/workflows/visual-acceptance-promote.ymlnew file mode 100644index 0000000000..faa7053407--- /dev/null+++ b/.github/workflows/visual-acceptance-promote.yml@@ -0,0 +1,188 @@+# Copyright (c) Meta Platforms, Inc. and affiliates.++# Promote a PR acceptance only after the merged commit reproduces every reviewed+# AFTER frame. The job is serialized with manual baseline promotion, and each+# key carries its baseline preimage hash so concurrent decisions cannot overwrite+# one another silently.++name: Visual acceptance promotion++on:+ pull_request_target:+ branches: ['main']+ types: [closed]++permissions: {}++concurrency:+ # Shares the head-branch lock with rerun invalidation and acceptance. A new+ # requested run cancels an older promotion before it can publish stale pixels;+ # cross-PR baseline races are rejected by preimage hashes and push retries.+ group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}+ cancel-in-progress: false++jobs:+ promote:+ if: github.event.pull_request.merged == true+ runs-on: 2-core-ubuntu-arm+ timeout-minutes: 20+ permissions:+ actions: write+ contents: write+ statuses: write+ steps:+ - name: Checkout the merged result+ uses: actions/checkout@v7+ with:+ ref: ${{ github.event.pull_request.merge_commit_sha }}++ - name: Find an acceptance for the merged head+ id: acceptance+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ run: |+ set -eu+ PAGES="$RUNNER_TEMP/visual-acceptance-pages"+ git clone --depth=1 --single-branch --branch gh-pages \+ "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$PAGES"+ ROOT="$PAGES/visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"+ POINTER="$ROOT/current.json"+ if [ ! -f "$POINTER" ]; then+ echo "No visual acceptance was recorded for this merged head."+ echo "found=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ RECORD_REL=$(jq -r '.record // empty' "$POINTER")+ if ! printf '%s' "$RECORD_REL" | grep -Eq '^[0-9]+/[0-9]+/acceptance\.json$'; then+ echo "::error::Visual acceptance pointer is invalid"+ exit 1+ fi+ RECORD="$ROOT/$RECORD_REL"+ if [ ! -f "$RECORD" ]; then+ echo "::error::Visual acceptance record is missing"+ exit 1+ fi+ {+ echo "found=true"+ echo "record=$RECORD"+ echo "record_rel=$RECORD_REL"+ } >> "$GITHUB_OUTPUT"++ - name: Setup Node and pnpm+ if: steps.acceptance.outputs.found == 'true'+ uses: ./.github/actions/setup++ - name: Build the merged Storybook+ if: steps.acceptance.outputs.found == 'true'+ run: pnpm build && pnpm -F @astryxdesign/storybook build++ - name: Install Chromium+ if: steps.acceptance.outputs.found == 'true'+ run: npx playwright install chromium++ - name: Recapture exactly the accepted shots+ if: steps.acceptance.outputs.found == 'true'+ env:+ GITHUB_SHA: ${{ github.event.pull_request.merge_commit_sha }}+ ACCEPTANCE: ${{ steps.acceptance.outputs.record }}+ run: |+ set -eu+ node .github/scripts/visual-gate/visual-acceptance.mjs plan \+ --acceptance "$ACCEPTANCE" \+ --output accepted-plan.json+ node .github/scripts/visual-gate/gate.mjs capture \+ --storybook-dir apps/storybook/dist \+ --out .visual-merged \+ --plan-file accepted-plan.json++ - name: Verify and promote the baseline+ id: promote+ if: steps.acceptance.outputs.found == 'true'+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }}+ RECORD_REL: ${{ steps.acceptance.outputs.record_rel }}+ run: |+ set -eu+ REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"+ PAGES="$RUNNER_TEMP/visual-promote-pages"+ RECORD="visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}/${RECORD_REL}"+ for attempt in 1 2 3 4 5; do+ rm -rf "$PAGES"+ git clone --depth=1 --single-branch --branch gh-pages "$REPO_URL" "$PAGES"+ CURRENT_REL=$(jq -r '.record // empty' \+ "$PAGES/visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}/current.json")+ if [ "$CURRENT_REL" != "$RECORD_REL" ]; then+ echo "::error::A newer visual acceptance superseded the reviewed record"+ exit 1+ fi+ LATEST=$(gh api \+ "repos/${GITHUB_REPOSITORY}/actions/runs?event=pull_request&head_sha=${HEAD_SHA}&per_page=100" \+ --jq '[.workflow_runs[] | select(.name == "CI")][0] | [.id, .run_attempt, .status] | @tsv')+ IFS=$'\t' read -r LATEST_RUN LATEST_ATTEMPT LATEST_STATUS <<< "$LATEST"+ ACCEPTED_RUN=$(jq -r '.run.id' "$PAGES/$RECORD")+ ACCEPTED_ATTEMPT=$(jq -r '.run.attempt' "$PAGES/$RECORD")+ if [ "$LATEST_RUN" != "$ACCEPTED_RUN" ] || \+ [ "$LATEST_ATTEMPT" != "$ACCEPTED_ATTEMPT" ] || \+ [ "$LATEST_STATUS" != "completed" ]; then+ echo "::error::Accepted evidence is not from the latest completed CI attempt"+ exit 1+ fi+ node .github/scripts/visual-gate/visual-acceptance.mjs state \+ --pages "$PAGES" --pr "$PR_NUMBER" --head "$HEAD_SHA" > "$RUNNER_TEMP/promotion-state.json"+ if [ "$(jq -r '.state + ":" + .reason' "$RUNNER_TEMP/promotion-state.json")" != "success:accepted" ]; then+ echo "::error::Accepted evidence is no longer the latest visual result"+ exit 1+ fi+ node .github/scripts/visual-gate/visual-acceptance.mjs promote \+ --pages "$PAGES" \+ --acceptance "$PAGES/$RECORD" \+ --capture .visual-merged \+ --merge-sha "$MERGE_SHA"+ git -C "$PAGES" config user.name "github-actions[bot]"+ git -C "$PAGES" config user.email "github-actions[bot]@users.noreply.github.com"+ git -C "$PAGES" add visual-gate/baseline+ if git -C "$PAGES" diff --cached --quiet; then+ echo "Baseline already contains the accepted pixels."+ exit 0+ fi+ git -C "$PAGES" commit -m "visual baseline: accepted PR #${PR_NUMBER}"+ if git -C "$PAGES" push origin gh-pages; then+ exit 0+ fi+ sleep $((attempt * 2))+ done+ echo "::error::Could not promote the accepted baseline after 5 attempts"+ exit 1++ - name: Run a fresh release gate+ if: steps.promote.outcome == 'success'+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ await github.rest.actions.createWorkflowDispatch({+ ...context.repo,+ workflow_id: 'release-gate.yml',+ ref: context.payload.repository.default_branch,+ });++ - name: Mark a post-merge verification failure+ if: failure() && steps.acceptance.outputs.found == 'true'+ uses: actions/github-script@v9+ env:+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ with:+ retries: 3+ script: |+ await github.rest.repos.createCommitStatus({+ ...context.repo,+ sha: process.env.HEAD_SHA,+ context: 'visual-acceptance',+ state: 'failure',+ description: 'Merged pixels did not promote to the visual baseline.',+ });diff --git a/.github/workflows/visual-acceptance.yml b/.github/workflows/visual-acceptance.ymlnew file mode 100644index 0000000000..d9eab24bc1--- /dev/null+++ b/.github/workflows/visual-acceptance.yml@@ -0,0 +1,394 @@+# Copyright (c) Meta Platforms, Inc. and affiliates.++# Required status and explicit human decision for stable visual changes.+# This workflow never checks out or executes PR-controlled code. It runs the+# default-branch implementation, resolves the current PR head through GitHub,+# and stores the signed decision beside trusted, immutable evidence on gh-pages.++name: Visual acceptance++on:+ pull_request_target:+ branches: ['main']+ types: [opened, reopened, synchronize]+ issue_comment:+ types: [created]++permissions: {}++jobs:+ initialize:+ if: github.event_name == 'pull_request_target'+ concurrency:+ group: visual-acceptance-head-${{ github.event.pull_request.head.repo.id }}-${{ github.event.pull_request.head.ref }}+ cancel-in-progress: true+ runs-on: ubuntu-slim+ permissions:+ contents: read+ pull-requests: read+ issues: write+ statuses: write+ steps:+ - name: Checkout trusted default-branch code+ uses: actions/checkout@v7++ - name: Classify scope and initialize the required status+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ const fs = require('node:fs');+ const {spawnSync} = require('node:child_process');+ const {owner, repo} = context.repo;+ const pr = context.payload.pull_request;+ const files = await github.paginate(github.rest.pulls.listFiles, {+ owner, repo, pull_number: pr.number, per_page: 100,+ });+ const manifestPaths = new Set();+ for (const {filename} of files) {+ const theme = filename.match(/^packages\/themes\/([^/]+)\//);+ const pkg = filename.match(/^packages\/([^/]+)\//);+ if (theme) manifestPaths.add(`packages/themes/${theme[1]}/package.json`);+ else if (pkg) manifestPaths.add(`packages/${pkg[1]}/package.json`);+ }+ const manifests = {};+ const [headOwner, headRepo] = pr.head.repo.full_name.split('/');+ for (const manifestPath of manifestPaths) {+ try {+ const {data} = await github.rest.repos.getContent({+ owner: headOwner, repo: headRepo, path: manifestPath, ref: pr.head.sha,+ });+ if (!Array.isArray(data) && data.content) {+ manifests[manifestPath] = JSON.parse(Buffer.from(data.content, 'base64').toString());+ }+ } catch (error) {+ if (error.status !== 404) throw error;+ }+ }+ fs.writeFileSync('visual-scope-manifests.json', JSON.stringify(manifests));+ const result = spawnSync(+ process.execPath,+ ['.github/scripts/visual-scope.mjs', '--manifests', 'visual-scope-manifests.json'],+ {input: `${files.map((file) => file.filename).join('\n')}\n`, encoding: 'utf8'},+ );+ if (result.status !== 0) {+ core.setFailed(result.stderr || 'Stable visual scope classification failed.');+ return;+ }+ const scope = JSON.parse(result.stdout);+ await github.rest.repos.createCommitStatus({+ owner,+ repo,+ sha: pr.head.sha,+ context: 'visual-acceptance',+ state: scope.hasStableVisual ? 'pending' : 'success',+ description: scope.hasStableVisual+ ? 'Stable visual capture is still running.'+ : 'No stable visual scope.',+ });+ try {+ await github.rest.issues.removeLabel({+ owner, repo, issue_number: pr.number, name: 'visual-approved',+ });+ } catch (error) {+ if (error.status !== 404) throw error;+ }++ authorize:+ if: >-+ github.event_name == 'issue_comment' &&+ github.event.issue.pull_request &&+ startsWith(github.event.comment.body, '/accept-visual')+ runs-on: ubuntu-slim+ permissions:+ actions: read+ contents: read+ pull-requests: read+ outputs:+ resolved: ${{ steps.authorize.outputs.resolved }}+ valid: ${{ steps.authorize.outputs.valid }}+ refusal: ${{ steps.authorize.outputs.refusal }}+ pr: ${{ steps.authorize.outputs.pr }}+ head: ${{ steps.authorize.outputs.head }}+ head_repo_id: ${{ steps.authorize.outputs.head_repo_id }}+ head_ref: ${{ steps.authorize.outputs.head_ref }}+ run_id: ${{ steps.authorize.outputs.run_id }}+ run_attempt: ${{ steps.authorize.outputs.run_attempt }}+ approver: ${{ steps.authorize.outputs.approver }}+ approver_id: ${{ steps.authorize.outputs.approver_id }}+ permission: ${{ steps.authorize.outputs.permission }}+ comment_id: ${{ steps.authorize.outputs.comment_id }}+ reason: ${{ steps.authorize.outputs.reason }}+ steps:+ # This job is deliberately read-only. The mutation job acquires the shared+ # head-branch lock using these trusted API-derived outputs.+ - name: Authorize and resolve the decision+ id: authorize+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ const {owner, repo} = context.repo;+ const number = context.issue.number;+ const {data: pr} = await github.rest.pulls.get({owner, repo, pull_number: number});+ if (!pr.head.repo?.id || !pr.head.ref) {+ core.setFailed('The pull request head repository or branch is unavailable.');+ return;+ }+ core.setOutput('resolved', 'true');+ core.setOutput('pr', String(number));+ core.setOutput('head', pr.head.sha);+ core.setOutput('head_repo_id', String(pr.head.repo.id));+ core.setOutput('head_ref', pr.head.ref);+ const refuse = (message) => {+ core.setOutput('valid', 'false');+ core.setOutput('refusal', message);+ };+ const body = context.payload.comment.body.trim();+ const match = body.match(/^\/accept-visual\s+(\d+)\/(\d+)\s+([\s\S]+)$/);+ if (!match) {+ refuse('copy the exact `/accept-visual <run>/<attempt> <reason>` command from the report.');+ return;+ }+ const runId = Number(match[1]);+ const runAttempt = Number(match[2]);+ const reason = match[3].trim();+ if (reason.length < 12 || reason.length > 500 ||+ /^(looks? good|intentional|approved|accept(ed)?|lgtm)[.! ]*$/i.test(reason)) {+ refuse('the reason must explain why the pixels are correct (12–500 characters).');+ return;+ }+ if (pr.state !== 'open' || pr.base.ref !== 'main') {+ refuse('the pull request is not open against `main`.');+ return;+ }+ const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {+ owner, repo, event: 'pull_request', head_sha: pr.head.sha, per_page: 100,+ });+ const latestCI = runs+ .filter((run) => run.name === 'CI')+ .sort((a, b) => b.id - a.id)[0];+ if (!latestCI || latestCI.id !== runId || latestCI.run_attempt !== runAttempt ||+ latestCI.status !== 'completed') {+ refuse('that evidence is not from the latest completed CI attempt for this head.');+ return;+ }+ let permission = 'none';+ try {+ const response = await github.rest.repos.getCollaboratorPermissionLevel({+ owner, repo, username: context.actor,+ });+ permission = response.data.user.permission;+ } catch (error) {+ if (error.status !== 404) throw error;+ }+ if (!['maintain', 'admin'].includes(permission)) {+ refuse('only a repository maintainer may accept stable visual changes.');+ return;+ }+ core.setOutput('valid', 'true');+ core.setOutput('run_id', String(runId));+ core.setOutput('run_attempt', String(runAttempt));+ core.setOutput('approver', context.actor);+ core.setOutput('approver_id', String(context.payload.comment.user.id));+ core.setOutput('permission', permission);+ core.setOutput('comment_id', String(context.payload.comment.id));+ core.setOutput('reason', reason);++ accept:+ needs: authorize+ if: >-+ always() &&+ needs.authorize.result == 'success' &&+ needs.authorize.outputs.resolved == 'true'+ concurrency:+ group: visual-acceptance-head-${{ needs.authorize.outputs.head_repo_id }}-${{ needs.authorize.outputs.head_ref }}+ cancel-in-progress: false+ runs-on: ubuntu-slim+ permissions:+ contents: write+ pull-requests: read+ issues: write+ statuses: write+ steps:+ - name: Report a refused decision+ if: needs.authorize.outputs.valid != 'true'+ uses: actions/github-script@v9+ env:+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}+ REFUSAL: ${{ needs.authorize.outputs.refusal }}+ with:+ retries: 3+ script: |+ await github.rest.issues.createComment({+ ...context.repo,+ issue_number: Number(process.env.PR_NUMBER),+ body: `Visual acceptance refused: ${process.env.REFUSAL}`,+ });++ - name: Checkout trusted default-branch code+ if: needs.authorize.outputs.valid == 'true'+ uses: actions/checkout@v7++ - name: Archive the acceptance decision+ id: record+ if: needs.authorize.outputs.valid == 'true'+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}+ HEAD_SHA: ${{ needs.authorize.outputs.head }}+ RUN_ID: ${{ needs.authorize.outputs.run_id }}+ RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}+ APPROVER: ${{ needs.authorize.outputs.approver }}+ APPROVER_ID: ${{ needs.authorize.outputs.approver_id }}+ PERMISSION: ${{ needs.authorize.outputs.permission }}+ COMMENT_ID: ${{ needs.authorize.outputs.comment_id }}+ REASON: ${{ needs.authorize.outputs.reason }}+ run: |+ set -eu+ REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"+ PAGES="$RUNNER_TEMP/visual-acceptance-pages"+ DEST="visual-gate/acceptances/${PR_NUMBER}/${HEAD_SHA}"+ for attempt in 1 2 3 4 5; do+ rm -rf "$PAGES"+ git clone --depth=1 --single-branch --branch gh-pages "$REPO_URL" "$PAGES"+ node .github/scripts/visual-gate/visual-acceptance.mjs accept \+ --pages "$PAGES" \+ --pr "$PR_NUMBER" \+ --head "$HEAD_SHA" \+ --run-id "$RUN_ID" \+ --run-attempt "$RUN_ATTEMPT" \+ --approver "$APPROVER" \+ --approver-id "$APPROVER_ID" \+ --permission "$PERMISSION" \+ --comment-id "$COMMENT_ID" \+ --reason "$REASON"+ if [ -z "$(git -C "$PAGES" status --porcelain -- "$DEST")" ]; then+ echo "The current visual bundle was already accepted."+ exit 0+ fi+ git -C "$PAGES" config user.name "github-actions[bot]"+ git -C "$PAGES" config user.email "github-actions[bot]@users.noreply.github.com"+ git -C "$PAGES" add "$DEST"+ git -C "$PAGES" commit -m "visual acceptance: PR #${PR_NUMBER} at ${HEAD_SHA}"+ if git -C "$PAGES" push origin gh-pages; then+ exit 0+ fi+ sleep $((attempt * 2))+ done+ echo "::error::Could not archive visual acceptance after 5 attempts"+ exit 1++ - name: Reconcile acceptance against the latest published evidence+ id: reconcile+ if: steps.record.outcome == 'success'+ env:+ GH_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}+ HEAD_SHA: ${{ needs.authorize.outputs.head }}+ run: |+ FRESH="$RUNNER_TEMP/visual-acceptance-reconcile"+ rm -rf "$FRESH"+ git clone --depth=1 --single-branch --branch gh-pages \+ "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$FRESH"+ node .github/scripts/visual-gate/visual-acceptance.mjs state \+ --pages "$FRESH" --pr "$PR_NUMBER" --head "$HEAD_SHA" > acceptance-state.json+ node - <<'NODE'+ const fs = require('node:fs');+ const state = JSON.parse(fs.readFileSync('acceptance-state.json', 'utf8'));+ for (const key of ['state', 'reason', 'description']) {+ fs.appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${state[key]}\n`);+ }+ NODE++ - name: Project the reconciled decision onto the PR+ if: steps.reconcile.outcome == 'success'+ uses: actions/github-script@v9+ env:+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}+ HEAD_SHA: ${{ needs.authorize.outputs.head }}+ ACCEPTED_RUN_ID: ${{ needs.authorize.outputs.run_id }}+ ACCEPTED_RUN_ATTEMPT: ${{ needs.authorize.outputs.run_attempt }}+ ACCEPTANCE_STATE: ${{ steps.reconcile.outputs.state }}+ ACCEPTANCE_REASON: ${{ steps.reconcile.outputs.reason }}+ ACCEPTANCE_DESCRIPTION: ${{ steps.reconcile.outputs.description }}+ with:+ retries: 3+ script: |+ const {owner, repo} = context.repo;+ const pr = Number(process.env.PR_NUMBER);+ const head = process.env.HEAD_SHA;+ let state = process.env.ACCEPTANCE_STATE;+ let reason = process.env.ACCEPTANCE_REASON;+ let description = process.env.ACCEPTANCE_DESCRIPTION;+ const runs = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, {+ owner, repo, event: 'pull_request', head_sha: head, per_page: 100,+ });+ const latestCI = runs+ .filter((run) => run.name === 'CI')+ .sort((a, b) => b.id - a.id)[0];+ const exactRun =+ latestCI &&+ latestCI.id === Number(process.env.ACCEPTED_RUN_ID) &&+ latestCI.run_attempt === Number(process.env.ACCEPTED_RUN_ATTEMPT) &&+ latestCI.status === 'completed';+ if (!exactRun) {+ state = 'pending';+ reason = 'capture';+ description = 'A newer CI attempt must publish trusted visual evidence.';+ }+ const accepted = state === 'success' && reason === 'accepted';+ if (!['success', 'pending', 'failure'].includes(state)) {+ core.setFailed('Reconciled visual acceptance state is invalid.');+ return;+ }+ const label = 'visual-approved';+ if (accepted) {+ try {+ await github.rest.issues.getLabel({owner, repo, name: label});+ } catch (error) {+ if (error.status !== 404) throw error;+ await github.rest.issues.createLabel({+ owner, repo, name: label, color: '2da44e',+ description: 'Current visual bundle has an explicit acceptance record',+ });+ }+ await github.rest.issues.addLabels({owner, repo, issue_number: pr, labels: [label]});+ } else {+ try {+ await github.rest.issues.removeLabel({owner, repo, issue_number: pr, name: label});+ } catch (error) {+ if (error.status !== 404) throw error;+ }+ }+ await github.rest.repos.createCommitStatus({+ owner, repo, sha: head, context: 'visual-acceptance', state,+ description: description.slice(0, 140),+ });+ await github.rest.issues.createComment({+ owner,+ repo,+ issue_number: pr,+ body: accepted+ ? `Visual changes accepted for \`${head.slice(0, 12)}\`. A new push invalidates this approval; after merge, the reviewed AFTER frames must reproduce before the baseline is promoted.`+ : `The requested visual evidence is no longer current. Status remains pending; copy the command from the latest report.`,+ });++ - name: Mark an acceptance infrastructure failure+ if: failure() && needs.authorize.outputs.valid == 'true'+ uses: actions/github-script@v9+ env:+ PR_NUMBER: ${{ needs.authorize.outputs.pr }}+ HEAD_SHA: ${{ needs.authorize.outputs.head }}+ with:+ retries: 3+ script: |+ const {owner, repo} = context.repo;+ await github.rest.repos.createCommitStatus({+ owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance', state: 'failure',+ description: 'Visual acceptance could not be recorded.',+ });+ await github.rest.issues.createComment({+ owner, repo, issue_number: Number(process.env.PR_NUMBER),+ body: 'Visual acceptance could not be recorded. See this workflow run for the rejected invariant or infrastructure error.',+ });