An open source design system that's fully customizable and agent ready
TypeScript
ci: validate Visual Baseline inputs before promotion (#5520)
Baraa Hamodiea2f048a7480
diff --git a/.github/scripts/lib/gh-pages-publisher.test.mjs b/.github/scripts/lib/gh-pages-publisher.test.mjsindex 764d55770f..20f715e0bf 100644--- a/.github/scripts/lib/gh-pages-publisher.test.mjs+++ b/.github/scripts/lib/gh-pages-publisher.test.mjs@@ -16,6 +16,7 @@ import {enqueuePublication,publishAcceptedVisualBaseline,publishImmutablePath,+ publishManualVisualBaseline,publishPrPreview,publishReleaseGateReport,publishVibeReport,@@ -436,6 +437,88 @@ async function withoutGitIdentity(callback) {}describe('gh-pages publisher', () => {+ it.each(['browser-only', 'capture-failure', 'missing-verdict'])(+ 'validates %s before manual baseline publication',+ async scenario => {+ const fx = fixture();+ const capture = path.join(fx.root, 'capture');+ const after = png(0, 0, 255);+ writeFile(path.join(capture, 'shots', `${KEY}.png`), after);+ writeJSON(path.join(capture, 'manifest.json'), {+ platform: 'linux-arm64',+ browser: 'chromium-141.0',+ viewport: {width: 1280, height: 900},+ shots: {[KEY]: {...SHOT, sha256: digest(after)}},+ });+ if (scenario !== 'missing-verdict') {+ const failures = [+ {+ key: 'baseline',+ error:+ 'baseline was captured with chromium-140.0, this run with chromium-141.0 — refresh the baseline (gate.mjs accept --keys all --reason "browser bump").',+ },+ ];+ if (scenario === 'capture-failure')+ failures.push({key: KEY, error: 'timeout'});+ writeJSON(path.join(capture, 'verdict.json'), {+ status: 'failed',+ counts: {total: 1, failed: failures.length},+ failures,+ removed: [],+ });+ }+ const before = git(+ fx.root,+ '--git-dir',+ fx.remote,+ 'rev-parse',+ 'gh-pages:visual-gate/baseline',+ );+ const publish = () =>+ queuedPublish(fx, 850, 'visual-gate/baseline', () =>+ publishManualVisualBaseline({+ ...context(fx, 850, 'visual-gate/baseline'),+ capture,+ keys: 'all',+ reason: 'browser bump',+ actor: 'maintainer',+ prune: true,+ }),+ );+ if (scenario === 'browser-only') {+ await expect(publish()).resolves.toMatchObject({published: true});+ const final = cloneRemote(fx.remote, fx.root);+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate/baseline/shots', `${KEY}.png`),+ ),+ ).toEqual(after);+ const manifest = JSON.parse(+ fs.readFileSync(+ path.join(final, 'visual-gate/baseline/manifest.json'),+ 'utf8',+ ),+ );+ expect(manifest.browser).toBe('chromium-141.0');+ expect(manifest.decisions.at(-1)).toMatchObject({+ promoted: [KEY],+ reason: 'browser bump',+ });+ } else {+ await expect(publish()).rejects.toThrow(/Refusing to promote/);+ expect(+ git(+ fx.root,+ '--git-dir',+ fx.remote,+ 'rev-parse',+ 'gh-pages:visual-gate/baseline',+ ),+ ).toBe(before);+ }+ },+ );+it('queues whole-tree and scoped writers without Actions pending-run cancellation', async () => {const fx = fixture();const wholeTree = context(fx, 800, 'whole-tree');diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.mdindex 09e3d4c822..ce6f1b6621 100644--- a/.github/scripts/visual-gate/README.md+++ b/.github/scripts/visual-gate/README.md@@ -205,7 +205,17 @@ that run with `keys=all` to make it the reference. Do the same after adeliberate system-wide restyle.A browser bump moves antialiasing everywhere at once; the gate detects it and-tells you to refresh rather than reporting hundreds of regressions.+keeps the verdict `failed` because those pixels are not comparable. After+reviewing the capture, use `keys=all` and a browser-bump reason to refresh it.+`accept` permits this only when the sole failure is the exact browser mismatch+against the current baseline, the platform and viewport match, every baseline+and captured shot is selected, and every captured PNG matches its manifest hash.+Partial refreshes, pruning, missing captures, and any other failure are refused+before baseline files change. If coverage changed too, restore full coverage+and rerun the gate before refreshing; removals need a separate comparable run.+The workflow uses this same validation inside its baseline publication turn,+not a separate status-only check. Rerun the gate after the refresh to get a+comparable verdict; refreshing does not turn the original failed run green.## Determinismdiff --git a/.github/scripts/visual-gate/gate-accept.test.mjs b/.github/scripts/visual-gate/gate-accept.test.mjsnew file mode 100644index 0000000000..ca0551a006--- /dev/null+++ b/.github/scripts/visual-gate/gate-accept.test.mjs@@ -0,0 +1,177 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++// `gate.mjs accept` is the promotion boundary: the one command that writes+// baseline files. These tests drive the real command against a scratch capture+// and baseline, so what they prove is what the Visual Baseline workflow runs.++import {spawnSync} from 'node:child_process';+import {createHash} from 'node:crypto';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';++import {afterEach, beforeEach, describe, expect, it} from 'vitest';++import {incomparable} from './lib/baseline.mjs';+import {buildVerdict} from './lib/compare.mjs';++const SCRIPT = path.join(+ path.dirname(fileURLToPath(import.meta.url)),+ 'gate.mjs',+);++let root;+let baselineDir;+let captureDir;++beforeEach(() => {+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-gate-accept-'));+ baselineDir = path.join(root, 'baseline');+ captureDir = path.join(root, 'capture');+ fs.mkdirSync(path.join(captureDir, 'shots'), {recursive: true});+ fs.writeFileSync(path.join(captureDir, 'shots', 'a.png'), 'new-a');+ fs.writeFileSync(path.join(captureDir, 'shots', 'b.png'), 'new-b');+ fs.writeFileSync(+ path.join(captureDir, 'manifest.json'),+ JSON.stringify({+ platform: 'linux-x64',+ viewport: {width: 1024, height: 768},+ capturedAt: '2026-08-23T00:00:00.000Z',+ shots: {a: {sha256: 'aa'}, b: {sha256: 'bb'}},+ }),+ );+});+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));++function writeVerdict(contents) {+ fs.writeFileSync(+ path.join(captureDir, 'verdict.json'),+ typeof contents === 'string' ? contents : JSON.stringify(contents),+ );+}++function acceptCommand(keys = 'a') {+ return spawnSync(+ process.execPath,+ [+ SCRIPT,+ 'accept',+ '--baseline',+ baselineDir,+ '--out',+ captureDir,+ '--keys',+ keys,+ '--reason',+ 'Button radius changed on purpose',+ '--actor',+ 'tester',+ ],+ {encoding: 'utf8'},+ );+}++describe('gate.mjs accept', () => {+ it.each([+ ['pass', {status: 'pass'}],+ ['changed', {status: 'changed'}],+ ])('promotes the named shot from a %s verdict', (_label, verdict) => {+ writeVerdict(verdict);+ const result = acceptCommand();+ expect(result.status, result.stderr).toBe(0);+ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),+ ).toBe('new-a');+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);+ const manifest = JSON.parse(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ );+ expect(Object.keys(manifest.shots)).toEqual(['a']);+ });++ it.each([+ ['failed', {status: 'failed'}],+ ['skipped', {status: 'skipped'}],+ ['missing', null],+ ['unreadable (not JSON)', 'not json'],+ ['unreadable (no status)', {}],+ ['unknown', {status: 'crashed'}],+ ])('writes no baseline file from a %s verdict', (_label, verdict) => {+ if (verdict !== null) writeVerdict(verdict);+ const result = acceptCommand();+ expect(result.status).not.toBe(0);+ expect(result.stderr).toMatch(/Refusing to promote/);+ // Not a manifest, not a shots directory: the baseline directory itself+ // was never created.+ expect(fs.existsSync(baselineDir)).toBe(false);+ });++ it.each([false, true])(+ 'handles a browser upgrade with additional capture failure=%s',+ captureFailed => {+ const manifestPath = path.join(captureDir, 'manifest.json');+ const baseline = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));+ baseline.browser = 'chromium-139.0';+ for (const key of ['a', 'b']) {+ baseline.shots[key].sha256 = createHash('sha256')+ .update(`new-${key}`)+ .digest('hex');+ }+ fs.writeFileSync(manifestPath, JSON.stringify(baseline));+ writeVerdict({status: 'pass'});+ expect(acceptCommand('all').status).toBe(0);+ const before = fs.readFileSync(+ path.join(baselineDir, 'manifest.json'),+ 'utf8',+ );+ const current = {...baseline, browser: 'chromium-140.0'};+ fs.writeFileSync(manifestPath, JSON.stringify(current));+ // Use the producer's real verdict shape: the existing browser-upgrade+ // guidance must reach accept, but a capture failure still cannot.+ const failures = [+ {key: 'baseline', error: incomparable(baseline, current)},+ ];+ if (captureFailed) failures.push({key: 'a', error: 'timeout'});+ const verdict = buildVerdict({+ comparison: {+ changes: [],+ added: [],+ removed: [],+ unchanged: ['a', 'b'],+ },+ baselineManifest: baseline,+ currentManifest: current,+ failures,+ });+ expect(verdict.status).toBe('failed');+ writeVerdict(verdict);+ const result = acceptCommand('all');+ if (captureFailed) {+ expect(result.status).toBe(1);+ expect(result.stderr).toMatch(/Refusing to promote/);+ expect(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ ).toBe(before);+ } else {+ expect(result.status, result.stderr).toBe(0);+ const refreshed = JSON.parse(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ );+ expect(refreshed.browser).toBe('chromium-140.0');+ expect(refreshed.decisions.at(-1).promoted).toEqual(['a', 'b']);+ expect(incomparable(refreshed, current)).toBeNull();+ }+ },+ );++ it('trims the keys the dispatch form invites: "a, b" promotes both', () => {+ writeVerdict({status: 'changed'});+ const result = acceptCommand('a, b');+ expect(result.status, result.stderr).toBe(0);+ const manifest = JSON.parse(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ );+ expect(Object.keys(manifest.shots).sort()).toEqual(['a', 'b']);+ });+});diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjsindex 727b6109e8..11df2e74a1 100644--- a/.github/scripts/visual-gate/gate.mjs+++ b/.github/scripts/visual-gate/gate.mjs@@ -800,18 +800,27 @@ async function main() {}const currentManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));const verdictPath = path.join(outDir, 'verdict.json');- const verdict = fs.existsSync(verdictPath)- ? JSON.parse(fs.readFileSync(verdictPath, 'utf8'))- : null;+ let verdict = null;+ if (fs.existsSync(verdictPath)) {+ try {+ verdict = JSON.parse(fs.readFileSync(verdictPath, 'utf8'));+ } catch (error) {+ throw new Error(`Refusing to promote: ${verdictPath} is unreadable (${error.message}).`);+ }+ }+ // accept() validates the verdict against the current baseline before+ // writing, including the narrowly allowed full browser refresh.const requested = flag('keys');const removed = new Set(verdict?.removed ?? []);+ // The dispatch form invites "a, b" — trim each key before it meets the+ // key validation in accept().const named =!requested || requested === 'all'? [...Object.keys(currentManifest.shots),...(has('prune') ? removed : []),]- : requested.split(',').filter(Boolean);+ : requested.split(',').map(key => key.trim()).filter(Boolean);if (new Set(named).size !== named.length) throw new Error('accept repeats a shot key.');const prune = has('prune') ? named.filter(key => removed.has(key)) : [];const keys = named.filter(key => currentManifest.shots[key]);@@ -839,6 +848,7 @@ async function main() {baselineDir,captureDir: outDir,currentManifest: baselineManifest,+ verdict,keys,reason: flag('reason') ?? '',actor: flag('actor') ?? process.env.GITHUB_ACTOR ?? process.env.USER ?? 'unknown',diff --git a/.github/scripts/visual-gate/lib/baseline.mjs b/.github/scripts/visual-gate/lib/baseline.mjsindex f17437b57a..9400f63004 100644--- a/.github/scripts/visual-gate/lib/baseline.mjs+++ b/.github/scripts/visual-gate/lib/baseline.mjs@@ -3,8 +3,9 @@/*** @file The baseline store, and the one operation that changes it.*- * @input a baseline directory (manifest + shots) and a capture- * @output an updated baseline, with every promotion recorded+ * @input a baseline directory (manifest + shots), a capture, and its verdict+ * @output an updated baseline, with every promotion recorded; browser refreshes+ * require a complete capture whose only failure is the browser mismatch** "The after is correct" is a decision, not a retry. Promoting a shot writes* a line into the manifest's decision log saying which shots moved, who moved@@ -18,19 +19,93 @@* gate refuses the comparison instead.*/+import {createHash} from 'node:crypto';import * as fs from 'node:fs';import * as path from 'node:path';export const EMPTY_MANIFEST = {version: 1, shots: {}, decisions: []};+/** Verdict statuses that need no browser-refresh recovery validation. */+export const PROMOTABLE_VERDICT_STATUSES = ['pass', 'changed'];++/** A browser bump is recoverable only when it is the sole comparison failure. */+function isBrowserRefresh(verdict, baseline = {}, current = {}) {+ return (+ typeof baseline.platform === 'string' &&+ baseline.platform.length > 0 &&+ baseline.platform === current.platform &&+ typeof baseline.browser === 'string' &&+ baseline.browser.length > 0 &&+ typeof current.browser === 'string' &&+ current.browser.length > 0 &&+ baseline.browser !== current.browser &&+ ['width', 'height'].every(+ axis =>+ Number.isSafeInteger(baseline.viewport?.[axis]) &&+ baseline.viewport[axis] > 0 &&+ baseline.viewport[axis] === current.viewport?.[axis],+ ) &&+ Array.isArray(verdict.failures) &&+ verdict.failures.length === 1 &&+ verdict.failures[0]?.key === 'baseline' &&+ verdict.failures[0].error === incomparable(baseline, current) &&+ verdict.counts?.failed === 1 &&+ Array.isArray(verdict.removed) &&+ verdict.removed.length === 0+ );+}++/**+ * The promotion boundary. Completed runs may still carry failed captures.+ * Pass/changed verdicts are eligible; a failed verdict needs the exact browser+ * mismatch against both manifests. accept() also requires a complete refresh+ * before it writes. Missing, unreadable, skipped, and unknown verdicts fail closed.+ *+ * @param {unknown} verdict - parsed verdict.json, or null when it is missing+ * @param {object} [options]+ * @param {object} [options.baselineManifest]+ * @param {object} [options.currentManifest]+ * @throws {Error} when the capture must not be promoted+ */+export function assertPromotableVerdict(+ verdict,+ {baselineManifest, currentManifest} = {},+) {+ if (verdict == null) {+ throw new Error(+ 'Refusing to promote a capture with no verdict.json — promote from a check or release run whose verdict is pass or changed.',+ );+ }+ const status =+ typeof verdict === 'object' && typeof verdict.status === 'string'+ ? verdict.status+ : null;+ if (PROMOTABLE_VERDICT_STATUSES.includes(status)) return;+ if (+ status === 'failed' &&+ isBrowserRefresh(verdict, baselineManifest, currentManifest)+ )+ return;+ const shown = status == null ? 'unreadable' : JSON.stringify(status);+ throw new Error(+ `Refusing to promote from a gate run whose verdict status is ${shown} — requires pass, changed, or a complete browser-only refresh.`,+ );+}+/*** @param {string} baselineDir* @returns {{manifest: object, exists: boolean}}*/export function readBaseline(baselineDir) {const manifestPath = path.join(baselineDir, 'manifest.json');- if (!fs.existsSync(manifestPath)) return {manifest: {...EMPTY_MANIFEST}, exists: false};- return {manifest: JSON.parse(fs.readFileSync(manifestPath, 'utf8')), exists: true};+ // A deep copy: a shallow spread shared EMPTY_MANIFEST's shots and decisions+ // objects between every fresh baseline in the same process.+ if (!fs.existsSync(manifestPath))+ return {manifest: structuredClone(EMPTY_MANIFEST), exists: false};+ return {+ manifest: JSON.parse(fs.readFileSync(manifestPath, 'utf8')),+ exists: true,+ };}/**@@ -74,12 +149,14 @@ export function incomparable(baselineManifest, currentManifest) {}/**- * Promote captured shots into the baseline.+ * Promote captured shots into the baseline. Nothing is written unless the+ * capture's verdict is one the gate stood behind (see assertPromotableVerdict).** @param {object} options* @param {string} options.baselineDir* @param {string} options.captureDir - directory holding the capture's shots/* @param {object} options.currentManifest+ * @param {unknown} options.verdict - the capture's parsed verdict.json (null when missing)* @param {string[]} options.keys - shot keys to promote* @param {string} options.reason - why the new rendering is the correct one* @param {string} options.actor@@ -91,14 +168,61 @@ export function accept({baselineDir,captureDir,currentManifest,+ verdict,keys,reason,actor,runId = null,prune = [],}) {- if (!reason?.trim()) throw new Error('accept requires a reason — it is the record of the decision');const {manifest} = readBaseline(baselineDir);+ assertPromotableVerdict(verdict, {+ baselineManifest: manifest,+ currentManifest,+ });+ if (!reason?.trim())+ throw new Error(+ 'accept requires a reason — it is the record of the decision',+ );+ // Shot keys name files inside shots/. A real key is shotKey() output+ // ([a-zA-Z0-9._-] only, see plan.mjs), never a path — reject anything else+ // before it is joined into one, whichever manifest or flag it came from.+ const SHOT_KEY = /^(?!\.+$)[a-zA-Z0-9._-]+$/;+ const badKey = [...keys, ...prune].find(key => !SHOT_KEY.test(String(key)));+ if (badKey != null) {+ throw new Error(`Invalid shot key: ${JSON.stringify(badKey)}`);+ }+ if (verdict.status === 'failed') {+ // Browser identity is manifest-wide: a partial refresh would label old+ // pixels as captured by the new browser. Validate every file before copying.+ const captured = Object.keys(currentManifest.shots ?? {});+ const selected = new Set(keys);+ if (+ captured.length === 0 ||+ verdict.counts?.total !== captured.length ||+ selected.size !== captured.length ||+ keys.length !== captured.length ||+ captured.some(key => !selected.has(key)) ||+ Object.keys(manifest.shots).some(key => !selected.has(key)) ||+ prune.length > 0+ ) {+ throw new Error(+ 'Refusing to promote a browser refresh unless every baseline and captured shot is refreshed without pruning.',+ );+ }+ for (const key of keys) {+ const source = path.join(captureDir, 'shots', `${key}.png`);+ if (+ !fs.existsSync(source) ||+ createHash('sha256').update(fs.readFileSync(source)).digest('hex') !==+ currentManifest.shots[key].sha256+ ) {+ throw new Error(+ `Refusing to promote a browser refresh with a missing or changed capture: ${key}`,+ );+ }+ }+ }const shotsDir = path.join(baselineDir, 'shots');fs.mkdirSync(shotsDir, {recursive: true});diff --git a/.github/scripts/visual-gate/lib/baseline.test.mjs b/.github/scripts/visual-gate/lib/baseline.test.mjsindex 38c4da2f1a..3a124aec88 100644--- a/.github/scripts/visual-gate/lib/baseline.test.mjs+++ b/.github/scripts/visual-gate/lib/baseline.test.mjs@@ -1,12 +1,18 @@// Copyright (c) Meta Platforms, Inc. and affiliates.+import * as crypto from 'node:crypto';import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import {afterEach, beforeEach, describe, expect, it} from 'vitest';-import {accept, incomparable, readBaseline} from './baseline.mjs';+import {+ accept,+ assertPromotableVerdict,+ incomparable,+ readBaseline,+} from './baseline.mjs';let root;let baselineDir;@@ -34,28 +40,126 @@ const promote = overrides =>baselineDir,captureDir,currentManifest,+ verdict: {status: 'changed'},keys: ['a'],reason: 'Button radius changed on purpose',actor: 'tester',...overrides,});+// Every way a capture can arrive without a verdict the gate stood behind: the+// gate's own failed and skipped statuses, no verdict.json at all (null), a+// verdict.json that did not parse into an object, and statuses this gate never+// writes.+const UNPROMOTABLE_VERDICTS = [+ ['failed', {status: 'failed'}],+ ['skipped', {status: 'skipped'}],+ ['missing', null],+ ['undefined', undefined],+ ['unreadable (not an object)', 'pass'],+ ['unreadable (no status)', {}],+ ['unreadable (null status)', {status: null}],+ ['unknown (crashed)', {status: 'crashed'}],+ ['unknown (empty)', {status: ''}],+ ['unknown (wrong case)', {status: 'PASS'}],+];++describe('assertPromotableVerdict', () => {+ it('lets pass and changed through', () => {+ expect(() => assertPromotableVerdict({status: 'pass'})).not.toThrow();+ expect(() => assertPromotableVerdict({status: 'changed'})).not.toThrow();+ });++ it.each(UNPROMOTABLE_VERDICTS)('refuses a %s verdict', (_label, verdict) => {+ expect(() => assertPromotableVerdict(verdict)).toThrow(+ /Refusing to promote/,+ );+ });+});+describe('accept', () => {it('refuses to promote without a reason, because the reason is the record', () => {expect(() => promote({reason: ' '})).toThrow(/reason/);});+ it('promotes from a pass verdict as well as a changed one', () => {+ promote({verdict: {status: 'pass'}});+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'a.png'))).toBe(true);+ });++ it.each(UNPROMOTABLE_VERDICTS)(+ 'writes no baseline file from a %s verdict',+ (_label, verdict) => {+ expect(() => promote({verdict})).toThrow(/Refusing to promote/);+ expect(() => promote({verdict, keys: ['a', 'b']})).toThrow(+ /Refusing to promote/,+ );+ // Nothing was created: no manifest, no shots directory, no baseline dir.+ expect(fs.existsSync(baselineDir)).toBe(false);+ },+ );++ it.each(UNPROMOTABLE_VERDICTS)(+ 'neither overwrites nor prunes an existing baseline from a %s verdict',+ (_label, verdict) => {+ promote({keys: ['a', 'b']});+ const before = fs.readFileSync(+ path.join(baselineDir, 'manifest.json'),+ 'utf8',+ );+ fs.writeFileSync(path.join(captureDir, 'shots', 'a.png'), 'broken-a');++ expect(() => promote({verdict, keys: ['a']})).toThrow(+ /Refusing to promote/,+ );+ expect(() =>+ promote({verdict, keys: [], prune: ['b'], reason: 'story deleted'}),+ ).toThrow(/Refusing to promote/);++ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),+ ).toBe('new-a');+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(+ true,+ );+ expect(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ ).toBe(before);+ },+ );++ it('refuses keys that are not shot keys — they name files under shots/', () => {+ // shotKey() output is [a-zA-Z0-9._-] only; anything path-shaped (or+ // dots-only) must be rejected before it is joined into a path, whichever+ // list it arrives in.+ for (const bad of ['../escape', 'a/b', 'a\\b', '..', ' a']) {+ expect(() => promote({keys: [bad]})).toThrow(/Invalid shot key/);+ expect(() => promote({keys: [], prune: [bad]})).toThrow(+ /Invalid shot key/,+ );+ }+ expect(fs.existsSync(baselineDir)).toBe(false);+ });+it('copies only the named shots into the baseline', () => {promote();- expect(fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8')).toBe('new-a');+ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),+ ).toBe('new-a');expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);- expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual(['a']);+ expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual([+ 'a',+ ]);});it('records who promoted what, and why', () => {promote();const [decision] = readBaseline(baselineDir).manifest.decisions;- expect(decision).toMatchObject({actor: 'tester', promoted: ['a'], reason: 'Button radius changed on purpose'});+ expect(decision).toMatchObject({+ actor: 'tester',+ promoted: ['a'],+ reason: 'Button radius changed on purpose',+ });});it('keeps earlier decisions when promoting again', () => {@@ -68,13 +172,146 @@ describe('accept', () => {promote({keys: ['a', 'b']});promote({keys: [], prune: ['b'], reason: 'story deleted'});expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);- expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual(['a']);+ expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual([+ 'a',+ ]);+ });+});++describe('browser-refresh recovery', () => {+ function refresh() {+ const manifest = structuredClone(currentManifest);+ manifest.browser = 'chromium-140.0';+ for (const key of ['a', 'b']) {+ manifest.shots[key].sha256 = crypto+ .createHash('sha256')+ .update(`new-${key}`)+ .digest('hex');+ }+ promote({+ currentManifest: {...manifest, browser: 'chromium-139.0'},+ keys: ['a', 'b'],+ });+ const verdict = {+ status: 'failed',+ counts: {total: 2, failed: 1},+ failures: [+ {+ key: 'baseline',+ error:+ 'baseline was captured with chromium-139.0, this run with chromium-140.0 — refresh the baseline (gate.mjs accept --keys all --reason "browser bump").',+ },+ ],+ removed: [],+ };+ return {+ currentManifest: manifest,+ verdict,+ keys: ['a', 'b'],+ reason: 'browser bump',+ };+ }++ it('refreshes every shot and records the browser bump without changing the failed verdict', () => {+ const options = refresh();+ fs.writeFileSync(path.join(baselineDir, 'shots', 'a.png'), 'old-a');+ const result = promote(options);+ expect(result.promoted).toEqual(['a', 'b']);+ expect(result.manifest.browser).toBe('chromium-140.0');+ expect(result.manifest.decisions.at(-1).reason).toBe('browser bump');+ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),+ ).toBe('new-a');+ expect(options.verdict.status).toBe('failed');+ });++ it.each([+ [+ 'another capture failure',+ options => options.verdict.failures.push({key: 'a', error: 'timeout'}),+ ],+ [+ 'another baseline error',+ options => (options.verdict.failures[0].error = 'baseline is corrupt'),+ ],+ [+ 'an unrelated failure key',+ options => (options.verdict.failures[0].key = 'a'),+ ],+ [+ 'inconsistent failure count',+ options => (options.verdict.counts.failed = 2),+ ],+ [+ 'inconsistent capture count',+ options => (options.verdict.counts.total = 3),+ ],+ ['no failure list', options => delete options.verdict.failures],+ [+ 'a platform change',+ options => (options.currentManifest.platform = 'darwin-arm64'),+ ],+ [+ 'a viewport change',+ options => (options.currentManifest.viewport.width = 800),+ ],+ ['missing viewport', options => delete options.currentManifest.viewport],+ [+ 'an unchanged browser',+ options => (options.currentManifest.browser = 'chromium-139.0'),+ ],+ [+ 'missing browser identity',+ options => delete options.currentManifest.browser,+ ],+ ['a partial selection', options => (options.keys = ['a'])],+ [+ 'an omitted baseline shot',+ options => {+ delete options.currentManifest.shots.b;+ options.keys = ['a'];+ options.verdict.counts.total = 1;+ },+ ],+ ['pruning', options => (options.prune = ['b'])],+ ['reported removals', options => (options.verdict.removed = ['b'])],+ ['a missing PNG', () => fs.rmSync(path.join(captureDir, 'shots', 'b.png'))],+ [+ 'a changed PNG',+ () =>+ fs.writeFileSync(path.join(captureDir, 'shots', 'b.png'), 'corrupt'),+ ],+ [+ 'a missing digest',+ options => delete options.currentManifest.shots.b.sha256,+ ],+ ])('refuses %s without changing any baseline bytes', (_label, mutate) => {+ const options = refresh();+ const before = fs.readFileSync(+ path.join(baselineDir, 'manifest.json'),+ 'utf8',+ );+ fs.writeFileSync(path.join(baselineDir, 'shots', 'a.png'), 'old-a');+ fs.writeFileSync(path.join(baselineDir, 'shots', 'b.png'), 'old-b');+ mutate(options);+ expect(() => promote(options)).toThrow(/Refusing to promote/);+ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),+ ).toBe('old-a');+ expect(+ fs.readFileSync(path.join(baselineDir, 'shots', 'b.png'), 'utf8'),+ ).toBe('old-b');+ expect(+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),+ ).toBe(before);});});describe('incomparable', () => {it('refuses a baseline from another platform', () => {- expect(incomparable({platform: 'darwin-arm64'}, {platform: 'linux-arm64'})).toMatch(/darwin-arm64/);+ expect(+ incomparable({platform: 'darwin-arm64'}, {platform: 'linux-arm64'}),+ ).toMatch(/darwin-arm64/);});it('refuses a baseline captured at another viewport', () => {diff --git a/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs b/.github/scripts/visual-gate/visual-baseline-workflow.test.mjsnew file mode 100644index 0000000000..736c178fb9--- /dev/null+++ b/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs@@ -0,0 +1,104 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++// Exercise the dispatch shell, including the interval that used to reject a+// browser-refresh verdict before the publisher could validate its capture.+import {spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';++import {afterEach, beforeEach, describe, expect, it} from 'vitest';+import {parse} from 'yaml';++const workflow = parse(+ fs.readFileSync(+ new URL('../../workflows/visual-baseline.yml', import.meta.url),+ 'utf8',+ ),+);+const steps = workflow.jobs.promote.steps;+let root;+let env;+beforeEach(() => {+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-baseline-workflow-'));+ const bin = path.join(root, 'bin');+ fs.mkdirSync(bin);+ fs.writeFileSync(+ path.join(bin, 'gh'),+ '#!/bin/sh\nprintf "%s" "$RUN_JSON"\n',+ {mode: 0o755},+ );+ // Publication is covered against a local Git remote in the publisher tests.+ fs.writeFileSync(path.join(bin, 'node'), '#!/bin/sh\nexit 0\n', {+ mode: 0o755,+ });+ env = {+ ...process.env,+ PATH: `${bin}:${process.env.PATH}`,+ RUN_ID: '123',+ GITHUB_REPOSITORY: 'facebook/astryx',+ };+});+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));++function run(script) {+ return spawnSync('bash', ['-e', '-c', script], {+ cwd: root,+ env,+ encoding: 'utf8',+ });+}++describe('Visual Baseline dispatch', () => {+ it.each([+ ['completed release', {}, true],+ ['failed terminal release', {conclusion: 'failure'}, true],+ ['wrong workflow', {path: '.github/workflows/ci.yml'}, false],+ ['wrong branch', {head_branch: 'feature'}, false],+ ['unfinished run', {status: 'in_progress'}, false],+ ['missing workflow', {path: null}, false],+ ])('validates the source run: %s', (_label, overrides, allowed) => {+ env.RUN_JSON = JSON.stringify({+ path: '.github/workflows/release-gate.yml',+ head_branch: 'main',+ status: 'completed',+ ...overrides,+ });+ const result = run(+ steps.find(step => step.name === 'Verify the source run').run,+ );+ expect(result.status === 0, result.stderr).toBe(allowed);+ });++ it('lets the publisher validate a failed browser-refresh capture inside its publication turn', () => {+ fs.mkdirSync(path.join(root, '.visual-run'));+ fs.writeFileSync(+ path.join(root, '.visual-run/verdict.json'),+ JSON.stringify({status: 'failed'}),+ );+ const download = steps.findIndex(+ step => step.name === 'Download the capture being promoted',+ );+ const promote = steps.findIndex(+ step => step.name === 'Promote and publish the baseline',+ );+ expect(download).toBeGreaterThan(-1);+ expect(promote).toBeGreaterThan(download);+ const beforePromotion = steps+ .slice(download + 1, promote)+ .map(step => step.run)+ .filter(Boolean);+ expect(beforePromotion.join('\n')).toContain(+ 'gh-pages-publisher.mjs wait --scope visual-gate/baseline',+ );+ const result = run(beforePromotion.join('\n'));+ expect(result.status, result.stderr || result.stdout).toBe(0);+ expect(steps[promote].run).toContain(+ 'gh-pages-publisher.mjs visual-baseline-manual',+ );+ expect(+ steps.find(step => step.name === 'Release the baseline publication turn')+ .if,+ ).toBe('always()');+ });+});diff --git a/.github/workflows/visual-baseline.yml b/.github/workflows/visual-baseline.ymlindex f9b206bd20..3ce5813ff1 100644--- a/.github/workflows/visual-baseline.yml+++ b/.github/workflows/visual-baseline.yml@@ -55,6 +55,26 @@ jobs:- name: Setup Node and pnpmuses: ./.github/actions/setup+ - name: Verify the source run+ env:+ GH_TOKEN: ${{ github.token }}+ RUN_ID: ${{ inputs.run_id }}+ run: |+ set -eu+ # Promotion pulls shots out of a specific run's capture artifact. The+ # run id is operator-typed — confirm it names a completed Release Gate+ # run on main before anything from it is treated as baseline material+ # (a typo'd id should fail here, not become the baseline).+ RUN_JSON=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}")+ RUN_PATH=$(printf '%s' "$RUN_JSON" | jq -r '.path')+ RUN_BRANCH=$(printf '%s' "$RUN_JSON" | jq -r '.head_branch')+ RUN_STATUS=$(printf '%s' "$RUN_JSON" | jq -r '.status')+ if [ "$RUN_PATH" != ".github/workflows/release-gate.yml" ] || \+ [ "$RUN_BRANCH" != "main" ] || [ "$RUN_STATUS" != "completed" ]; then+ echo "::error::Run ${RUN_ID} is not a completed Release Gate run on main (path=${RUN_PATH}, branch=${RUN_BRANCH}, status=${RUN_STATUS})."+ exit 1+ fi+- name: Queue baseline publicationenv:GH_TOKEN: ${{ github.token }}@@ -73,6 +93,9 @@ jobs:GH_TOKEN: ${{ github.token }}run: node .github/scripts/gh-pages-publisher.mjs wait --scope visual-gate/baseline+ # The publisher calls gate.mjs accept against the current baseline before+ # writing or publishing any shots. Keep verdict validation there: a browser+ # refresh needs both manifests, not a second status-only allowlist here.- name: Promote and publish the baselineenv:KEYS: ${{ inputs.keys }}@@ -95,11 +118,15 @@ jobs:run: node .github/scripts/gh-pages-publisher.mjs release --scope visual-gate/baseline- name: Summary+ env:+ KEYS: ${{ inputs.keys }}+ RUN_ID: ${{ inputs.run_id }}+ REASON: ${{ inputs.reason }}run: |{echo "## Visual baseline updated"echo ""- echo "- promoted: \`${{ inputs.keys }}\`"- echo "- from run: ${{ inputs.run_id }}"- echo "- reason: ${{ inputs.reason }}"+ echo "- promoted: \`${KEYS}\`"+ echo "- from run: ${RUN_ID}"+ echo "- reason: ${REASON}"} >> "$GITHUB_STEP_SUMMARY"