Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: validate Visual Baseline inputs before promotion (#5520)

Baraa Hamodiea2f048a7480
diff --git a/.github/scripts/lib/gh-pages-publisher.test.mjs b/.github/scripts/lib/gh-pages-publisher.test.mjs
index 764d55770f..20f715e0bf 100644
--- a/.github/scripts/lib/gh-pages-publisher.test.mjs
+++ b/.github/scripts/lib/gh-pages-publisher.test.mjs
@@ -16,6 +16,7 @@ import {
enqueuePublication,
publishAcceptedVisualBaseline,
publishImmutablePath,
+ publishManualVisualBaseline,
publishPrPreview,
publishReleaseGateReport,
publishVibeReport,
@@ -436,6 +437,88 @@ async function withoutGitIdentity(callback) {
}
describe('gh-pages publisher', () => {
+ it.each(['browser-only', 'capture-failure', 'missing-verdict'])(
+ 'validates %s before manual baseline publication',
+ async scenario => {
+ const fx = fixture();
+ const capture = path.join(fx.root, 'capture');
+ const after = png(0, 0, 255);
+ writeFile(path.join(capture, 'shots', `${KEY}.png`), after);
+ writeJSON(path.join(capture, 'manifest.json'), {
+ platform: 'linux-arm64',
+ browser: 'chromium-141.0',
+ viewport: {width: 1280, height: 900},
+ shots: {[KEY]: {...SHOT, sha256: digest(after)}},
+ });
+ if (scenario !== 'missing-verdict') {
+ const failures = [
+ {
+ key: 'baseline',
+ error:
+ 'baseline was captured with chromium-140.0, this run with chromium-141.0 — refresh the baseline (gate.mjs accept --keys all --reason "browser bump").',
+ },
+ ];
+ if (scenario === 'capture-failure')
+ failures.push({key: KEY, error: 'timeout'});
+ writeJSON(path.join(capture, 'verdict.json'), {
+ status: 'failed',
+ counts: {total: 1, failed: failures.length},
+ failures,
+ removed: [],
+ });
+ }
+ const before = git(
+ fx.root,
+ '--git-dir',
+ fx.remote,
+ 'rev-parse',
+ 'gh-pages:visual-gate/baseline',
+ );
+ const publish = () =>
+ queuedPublish(fx, 850, 'visual-gate/baseline', () =>
+ publishManualVisualBaseline({
+ ...context(fx, 850, 'visual-gate/baseline'),
+ capture,
+ keys: 'all',
+ reason: 'browser bump',
+ actor: 'maintainer',
+ prune: true,
+ }),
+ );
+ if (scenario === 'browser-only') {
+ await expect(publish()).resolves.toMatchObject({published: true});
+ const final = cloneRemote(fx.remote, fx.root);
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate/baseline/shots', `${KEY}.png`),
+ ),
+ ).toEqual(after);
+ const manifest = JSON.parse(
+ fs.readFileSync(
+ path.join(final, 'visual-gate/baseline/manifest.json'),
+ 'utf8',
+ ),
+ );
+ expect(manifest.browser).toBe('chromium-141.0');
+ expect(manifest.decisions.at(-1)).toMatchObject({
+ promoted: [KEY],
+ reason: 'browser bump',
+ });
+ } else {
+ await expect(publish()).rejects.toThrow(/Refusing to promote/);
+ expect(
+ git(
+ fx.root,
+ '--git-dir',
+ fx.remote,
+ 'rev-parse',
+ 'gh-pages:visual-gate/baseline',
+ ),
+ ).toBe(before);
+ }
+ },
+ );
+
it('queues whole-tree and scoped writers without Actions pending-run cancellation', async () => {
const fx = fixture();
const wholeTree = context(fx, 800, 'whole-tree');
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index 09e3d4c822..ce6f1b6621 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -205,7 +205,17 @@ that run with `keys=all` to make it the reference. Do the same after a
deliberate system-wide restyle.
A browser bump moves antialiasing everywhere at once; the gate detects it and
-tells you to refresh rather than reporting hundreds of regressions.
+keeps the verdict `failed` because those pixels are not comparable. After
+reviewing the capture, use `keys=all` and a browser-bump reason to refresh it.
+`accept` permits this only when the sole failure is the exact browser mismatch
+against the current baseline, the platform and viewport match, every baseline
+and captured shot is selected, and every captured PNG matches its manifest hash.
+Partial refreshes, pruning, missing captures, and any other failure are refused
+before baseline files change. If coverage changed too, restore full coverage
+and rerun the gate before refreshing; removals need a separate comparable run.
+The workflow uses this same validation inside its baseline publication turn,
+not a separate status-only check. Rerun the gate after the refresh to get a
+comparable verdict; refreshing does not turn the original failed run green.
## Determinism
diff --git a/.github/scripts/visual-gate/gate-accept.test.mjs b/.github/scripts/visual-gate/gate-accept.test.mjs
new file mode 100644
index 0000000000..ca0551a006
--- /dev/null
+++ b/.github/scripts/visual-gate/gate-accept.test.mjs
@@ -0,0 +1,177 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+// `gate.mjs accept` is the promotion boundary: the one command that writes
+// baseline files. These tests drive the real command against a scratch capture
+// and baseline, so what they prove is what the Visual Baseline workflow runs.
+
+import {spawnSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+
+import {incomparable} from './lib/baseline.mjs';
+import {buildVerdict} from './lib/compare.mjs';
+
+const SCRIPT = path.join(
+ path.dirname(fileURLToPath(import.meta.url)),
+ 'gate.mjs',
+);
+
+let root;
+let baselineDir;
+let captureDir;
+
+beforeEach(() => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-gate-accept-'));
+ baselineDir = path.join(root, 'baseline');
+ captureDir = path.join(root, 'capture');
+ fs.mkdirSync(path.join(captureDir, 'shots'), {recursive: true});
+ fs.writeFileSync(path.join(captureDir, 'shots', 'a.png'), 'new-a');
+ fs.writeFileSync(path.join(captureDir, 'shots', 'b.png'), 'new-b');
+ fs.writeFileSync(
+ path.join(captureDir, 'manifest.json'),
+ JSON.stringify({
+ platform: 'linux-x64',
+ viewport: {width: 1024, height: 768},
+ capturedAt: '2026-08-23T00:00:00.000Z',
+ shots: {a: {sha256: 'aa'}, b: {sha256: 'bb'}},
+ }),
+ );
+});
+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
+
+function writeVerdict(contents) {
+ fs.writeFileSync(
+ path.join(captureDir, 'verdict.json'),
+ typeof contents === 'string' ? contents : JSON.stringify(contents),
+ );
+}
+
+function acceptCommand(keys = 'a') {
+ return spawnSync(
+ process.execPath,
+ [
+ SCRIPT,
+ 'accept',
+ '--baseline',
+ baselineDir,
+ '--out',
+ captureDir,
+ '--keys',
+ keys,
+ '--reason',
+ 'Button radius changed on purpose',
+ '--actor',
+ 'tester',
+ ],
+ {encoding: 'utf8'},
+ );
+}
+
+describe('gate.mjs accept', () => {
+ it.each([
+ ['pass', {status: 'pass'}],
+ ['changed', {status: 'changed'}],
+ ])('promotes the named shot from a %s verdict', (_label, verdict) => {
+ writeVerdict(verdict);
+ const result = acceptCommand();
+ expect(result.status, result.stderr).toBe(0);
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),
+ ).toBe('new-a');
+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);
+ const manifest = JSON.parse(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ );
+ expect(Object.keys(manifest.shots)).toEqual(['a']);
+ });
+
+ it.each([
+ ['failed', {status: 'failed'}],
+ ['skipped', {status: 'skipped'}],
+ ['missing', null],
+ ['unreadable (not JSON)', 'not json'],
+ ['unreadable (no status)', {}],
+ ['unknown', {status: 'crashed'}],
+ ])('writes no baseline file from a %s verdict', (_label, verdict) => {
+ if (verdict !== null) writeVerdict(verdict);
+ const result = acceptCommand();
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toMatch(/Refusing to promote/);
+ // Not a manifest, not a shots directory: the baseline directory itself
+ // was never created.
+ expect(fs.existsSync(baselineDir)).toBe(false);
+ });
+
+ it.each([false, true])(
+ 'handles a browser upgrade with additional capture failure=%s',
+ captureFailed => {
+ const manifestPath = path.join(captureDir, 'manifest.json');
+ const baseline = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+ baseline.browser = 'chromium-139.0';
+ for (const key of ['a', 'b']) {
+ baseline.shots[key].sha256 = createHash('sha256')
+ .update(`new-${key}`)
+ .digest('hex');
+ }
+ fs.writeFileSync(manifestPath, JSON.stringify(baseline));
+ writeVerdict({status: 'pass'});
+ expect(acceptCommand('all').status).toBe(0);
+ const before = fs.readFileSync(
+ path.join(baselineDir, 'manifest.json'),
+ 'utf8',
+ );
+ const current = {...baseline, browser: 'chromium-140.0'};
+ fs.writeFileSync(manifestPath, JSON.stringify(current));
+ // Use the producer's real verdict shape: the existing browser-upgrade
+ // guidance must reach accept, but a capture failure still cannot.
+ const failures = [
+ {key: 'baseline', error: incomparable(baseline, current)},
+ ];
+ if (captureFailed) failures.push({key: 'a', error: 'timeout'});
+ const verdict = buildVerdict({
+ comparison: {
+ changes: [],
+ added: [],
+ removed: [],
+ unchanged: ['a', 'b'],
+ },
+ baselineManifest: baseline,
+ currentManifest: current,
+ failures,
+ });
+ expect(verdict.status).toBe('failed');
+ writeVerdict(verdict);
+ const result = acceptCommand('all');
+ if (captureFailed) {
+ expect(result.status).toBe(1);
+ expect(result.stderr).toMatch(/Refusing to promote/);
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ ).toBe(before);
+ } else {
+ expect(result.status, result.stderr).toBe(0);
+ const refreshed = JSON.parse(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ );
+ expect(refreshed.browser).toBe('chromium-140.0');
+ expect(refreshed.decisions.at(-1).promoted).toEqual(['a', 'b']);
+ expect(incomparable(refreshed, current)).toBeNull();
+ }
+ },
+ );
+
+ it('trims the keys the dispatch form invites: "a, b" promotes both', () => {
+ writeVerdict({status: 'changed'});
+ const result = acceptCommand('a, b');
+ expect(result.status, result.stderr).toBe(0);
+ const manifest = JSON.parse(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ );
+ expect(Object.keys(manifest.shots).sort()).toEqual(['a', 'b']);
+ });
+});
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index 727b6109e8..11df2e74a1 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -800,18 +800,27 @@ async function main() {
}
const currentManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
const verdictPath = path.join(outDir, 'verdict.json');
- const verdict = fs.existsSync(verdictPath)
- ? JSON.parse(fs.readFileSync(verdictPath, 'utf8'))
- : null;
+ let verdict = null;
+ if (fs.existsSync(verdictPath)) {
+ try {
+ verdict = JSON.parse(fs.readFileSync(verdictPath, 'utf8'));
+ } catch (error) {
+ throw new Error(`Refusing to promote: ${verdictPath} is unreadable (${error.message}).`);
+ }
+ }
+ // accept() validates the verdict against the current baseline before
+ // writing, including the narrowly allowed full browser refresh.
const requested = flag('keys');
const removed = new Set(verdict?.removed ?? []);
+ // The dispatch form invites "a, b" — trim each key before it meets the
+ // key validation in accept().
const named =
!requested || requested === 'all'
? [
...Object.keys(currentManifest.shots),
...(has('prune') ? removed : []),
]
- : requested.split(',').filter(Boolean);
+ : requested.split(',').map(key => key.trim()).filter(Boolean);
if (new Set(named).size !== named.length) throw new Error('accept repeats a shot key.');
const prune = has('prune') ? named.filter(key => removed.has(key)) : [];
const keys = named.filter(key => currentManifest.shots[key]);
@@ -839,6 +848,7 @@ async function main() {
baselineDir,
captureDir: outDir,
currentManifest: baselineManifest,
+ verdict,
keys,
reason: flag('reason') ?? '',
actor: flag('actor') ?? process.env.GITHUB_ACTOR ?? process.env.USER ?? 'unknown',
diff --git a/.github/scripts/visual-gate/lib/baseline.mjs b/.github/scripts/visual-gate/lib/baseline.mjs
index f17437b57a..9400f63004 100644
--- a/.github/scripts/visual-gate/lib/baseline.mjs
+++ b/.github/scripts/visual-gate/lib/baseline.mjs
@@ -3,8 +3,9 @@
/**
* @file The baseline store, and the one operation that changes it.
*
- * @input a baseline directory (manifest + shots) and a capture
- * @output an updated baseline, with every promotion recorded
+ * @input a baseline directory (manifest + shots), a capture, and its verdict
+ * @output an updated baseline, with every promotion recorded; browser refreshes
+ * require a complete capture whose only failure is the browser mismatch
*
* "The after is correct" is a decision, not a retry. Promoting a shot writes
* a line into the manifest's decision log saying which shots moved, who moved
@@ -18,19 +19,93 @@
* gate refuses the comparison instead.
*/
+import {createHash} from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
export const EMPTY_MANIFEST = {version: 1, shots: {}, decisions: []};
+/** Verdict statuses that need no browser-refresh recovery validation. */
+export const PROMOTABLE_VERDICT_STATUSES = ['pass', 'changed'];
+
+/** A browser bump is recoverable only when it is the sole comparison failure. */
+function isBrowserRefresh(verdict, baseline = {}, current = {}) {
+ return (
+ typeof baseline.platform === 'string' &&
+ baseline.platform.length > 0 &&
+ baseline.platform === current.platform &&
+ typeof baseline.browser === 'string' &&
+ baseline.browser.length > 0 &&
+ typeof current.browser === 'string' &&
+ current.browser.length > 0 &&
+ baseline.browser !== current.browser &&
+ ['width', 'height'].every(
+ axis =>
+ Number.isSafeInteger(baseline.viewport?.[axis]) &&
+ baseline.viewport[axis] > 0 &&
+ baseline.viewport[axis] === current.viewport?.[axis],
+ ) &&
+ Array.isArray(verdict.failures) &&
+ verdict.failures.length === 1 &&
+ verdict.failures[0]?.key === 'baseline' &&
+ verdict.failures[0].error === incomparable(baseline, current) &&
+ verdict.counts?.failed === 1 &&
+ Array.isArray(verdict.removed) &&
+ verdict.removed.length === 0
+ );
+}
+
+/**
+ * The promotion boundary. Completed runs may still carry failed captures.
+ * Pass/changed verdicts are eligible; a failed verdict needs the exact browser
+ * mismatch against both manifests. accept() also requires a complete refresh
+ * before it writes. Missing, unreadable, skipped, and unknown verdicts fail closed.
+ *
+ * @param {unknown} verdict - parsed verdict.json, or null when it is missing
+ * @param {object} [options]
+ * @param {object} [options.baselineManifest]
+ * @param {object} [options.currentManifest]
+ * @throws {Error} when the capture must not be promoted
+ */
+export function assertPromotableVerdict(
+ verdict,
+ {baselineManifest, currentManifest} = {},
+) {
+ if (verdict == null) {
+ throw new Error(
+ 'Refusing to promote a capture with no verdict.json — promote from a check or release run whose verdict is pass or changed.',
+ );
+ }
+ const status =
+ typeof verdict === 'object' && typeof verdict.status === 'string'
+ ? verdict.status
+ : null;
+ if (PROMOTABLE_VERDICT_STATUSES.includes(status)) return;
+ if (
+ status === 'failed' &&
+ isBrowserRefresh(verdict, baselineManifest, currentManifest)
+ )
+ return;
+ const shown = status == null ? 'unreadable' : JSON.stringify(status);
+ throw new Error(
+ `Refusing to promote from a gate run whose verdict status is ${shown} — requires pass, changed, or a complete browser-only refresh.`,
+ );
+}
+
/**
* @param {string} baselineDir
* @returns {{manifest: object, exists: boolean}}
*/
export function readBaseline(baselineDir) {
const manifestPath = path.join(baselineDir, 'manifest.json');
- if (!fs.existsSync(manifestPath)) return {manifest: {...EMPTY_MANIFEST}, exists: false};
- return {manifest: JSON.parse(fs.readFileSync(manifestPath, 'utf8')), exists: true};
+ // A deep copy: a shallow spread shared EMPTY_MANIFEST's shots and decisions
+ // objects between every fresh baseline in the same process.
+ if (!fs.existsSync(manifestPath))
+ return {manifest: structuredClone(EMPTY_MANIFEST), exists: false};
+ return {
+ manifest: JSON.parse(fs.readFileSync(manifestPath, 'utf8')),
+ exists: true,
+ };
}
/**
@@ -74,12 +149,14 @@ export function incomparable(baselineManifest, currentManifest) {
}
/**
- * Promote captured shots into the baseline.
+ * Promote captured shots into the baseline. Nothing is written unless the
+ * capture's verdict is one the gate stood behind (see assertPromotableVerdict).
*
* @param {object} options
* @param {string} options.baselineDir
* @param {string} options.captureDir - directory holding the capture's shots/
* @param {object} options.currentManifest
+ * @param {unknown} options.verdict - the capture's parsed verdict.json (null when missing)
* @param {string[]} options.keys - shot keys to promote
* @param {string} options.reason - why the new rendering is the correct one
* @param {string} options.actor
@@ -91,14 +168,61 @@ export function accept({
baselineDir,
captureDir,
currentManifest,
+ verdict,
keys,
reason,
actor,
runId = null,
prune = [],
}) {
- if (!reason?.trim()) throw new Error('accept requires a reason — it is the record of the decision');
const {manifest} = readBaseline(baselineDir);
+ assertPromotableVerdict(verdict, {
+ baselineManifest: manifest,
+ currentManifest,
+ });
+ if (!reason?.trim())
+ throw new Error(
+ 'accept requires a reason — it is the record of the decision',
+ );
+ // Shot keys name files inside shots/. A real key is shotKey() output
+ // ([a-zA-Z0-9._-] only, see plan.mjs), never a path — reject anything else
+ // before it is joined into one, whichever manifest or flag it came from.
+ const SHOT_KEY = /^(?!\.+$)[a-zA-Z0-9._-]+$/;
+ const badKey = [...keys, ...prune].find(key => !SHOT_KEY.test(String(key)));
+ if (badKey != null) {
+ throw new Error(`Invalid shot key: ${JSON.stringify(badKey)}`);
+ }
+ if (verdict.status === 'failed') {
+ // Browser identity is manifest-wide: a partial refresh would label old
+ // pixels as captured by the new browser. Validate every file before copying.
+ const captured = Object.keys(currentManifest.shots ?? {});
+ const selected = new Set(keys);
+ if (
+ captured.length === 0 ||
+ verdict.counts?.total !== captured.length ||
+ selected.size !== captured.length ||
+ keys.length !== captured.length ||
+ captured.some(key => !selected.has(key)) ||
+ Object.keys(manifest.shots).some(key => !selected.has(key)) ||
+ prune.length > 0
+ ) {
+ throw new Error(
+ 'Refusing to promote a browser refresh unless every baseline and captured shot is refreshed without pruning.',
+ );
+ }
+ for (const key of keys) {
+ const source = path.join(captureDir, 'shots', `${key}.png`);
+ if (
+ !fs.existsSync(source) ||
+ createHash('sha256').update(fs.readFileSync(source)).digest('hex') !==
+ currentManifest.shots[key].sha256
+ ) {
+ throw new Error(
+ `Refusing to promote a browser refresh with a missing or changed capture: ${key}`,
+ );
+ }
+ }
+ }
const shotsDir = path.join(baselineDir, 'shots');
fs.mkdirSync(shotsDir, {recursive: true});
diff --git a/.github/scripts/visual-gate/lib/baseline.test.mjs b/.github/scripts/visual-gate/lib/baseline.test.mjs
index 38c4da2f1a..3a124aec88 100644
--- a/.github/scripts/visual-gate/lib/baseline.test.mjs
+++ b/.github/scripts/visual-gate/lib/baseline.test.mjs
@@ -1,12 +1,18 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
+import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
-import {accept, incomparable, readBaseline} from './baseline.mjs';
+import {
+ accept,
+ assertPromotableVerdict,
+ incomparable,
+ readBaseline,
+} from './baseline.mjs';
let root;
let baselineDir;
@@ -34,28 +40,126 @@ const promote = overrides =>
baselineDir,
captureDir,
currentManifest,
+ verdict: {status: 'changed'},
keys: ['a'],
reason: 'Button radius changed on purpose',
actor: 'tester',
...overrides,
});
+// Every way a capture can arrive without a verdict the gate stood behind: the
+// gate's own failed and skipped statuses, no verdict.json at all (null), a
+// verdict.json that did not parse into an object, and statuses this gate never
+// writes.
+const UNPROMOTABLE_VERDICTS = [
+ ['failed', {status: 'failed'}],
+ ['skipped', {status: 'skipped'}],
+ ['missing', null],
+ ['undefined', undefined],
+ ['unreadable (not an object)', 'pass'],
+ ['unreadable (no status)', {}],
+ ['unreadable (null status)', {status: null}],
+ ['unknown (crashed)', {status: 'crashed'}],
+ ['unknown (empty)', {status: ''}],
+ ['unknown (wrong case)', {status: 'PASS'}],
+];
+
+describe('assertPromotableVerdict', () => {
+ it('lets pass and changed through', () => {
+ expect(() => assertPromotableVerdict({status: 'pass'})).not.toThrow();
+ expect(() => assertPromotableVerdict({status: 'changed'})).not.toThrow();
+ });
+
+ it.each(UNPROMOTABLE_VERDICTS)('refuses a %s verdict', (_label, verdict) => {
+ expect(() => assertPromotableVerdict(verdict)).toThrow(
+ /Refusing to promote/,
+ );
+ });
+});
+
describe('accept', () => {
it('refuses to promote without a reason, because the reason is the record', () => {
expect(() => promote({reason: ' '})).toThrow(/reason/);
});
+ it('promotes from a pass verdict as well as a changed one', () => {
+ promote({verdict: {status: 'pass'}});
+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'a.png'))).toBe(true);
+ });
+
+ it.each(UNPROMOTABLE_VERDICTS)(
+ 'writes no baseline file from a %s verdict',
+ (_label, verdict) => {
+ expect(() => promote({verdict})).toThrow(/Refusing to promote/);
+ expect(() => promote({verdict, keys: ['a', 'b']})).toThrow(
+ /Refusing to promote/,
+ );
+ // Nothing was created: no manifest, no shots directory, no baseline dir.
+ expect(fs.existsSync(baselineDir)).toBe(false);
+ },
+ );
+
+ it.each(UNPROMOTABLE_VERDICTS)(
+ 'neither overwrites nor prunes an existing baseline from a %s verdict',
+ (_label, verdict) => {
+ promote({keys: ['a', 'b']});
+ const before = fs.readFileSync(
+ path.join(baselineDir, 'manifest.json'),
+ 'utf8',
+ );
+ fs.writeFileSync(path.join(captureDir, 'shots', 'a.png'), 'broken-a');
+
+ expect(() => promote({verdict, keys: ['a']})).toThrow(
+ /Refusing to promote/,
+ );
+ expect(() =>
+ promote({verdict, keys: [], prune: ['b'], reason: 'story deleted'}),
+ ).toThrow(/Refusing to promote/);
+
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),
+ ).toBe('new-a');
+ expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(
+ true,
+ );
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ ).toBe(before);
+ },
+ );
+
+ it('refuses keys that are not shot keys — they name files under shots/', () => {
+ // shotKey() output is [a-zA-Z0-9._-] only; anything path-shaped (or
+ // dots-only) must be rejected before it is joined into a path, whichever
+ // list it arrives in.
+ for (const bad of ['../escape', 'a/b', 'a\\b', '..', ' a']) {
+ expect(() => promote({keys: [bad]})).toThrow(/Invalid shot key/);
+ expect(() => promote({keys: [], prune: [bad]})).toThrow(
+ /Invalid shot key/,
+ );
+ }
+ expect(fs.existsSync(baselineDir)).toBe(false);
+ });
+
it('copies only the named shots into the baseline', () => {
promote();
- expect(fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8')).toBe('new-a');
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),
+ ).toBe('new-a');
expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);
- expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual(['a']);
+ expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual([
+ 'a',
+ ]);
});
it('records who promoted what, and why', () => {
promote();
const [decision] = readBaseline(baselineDir).manifest.decisions;
- expect(decision).toMatchObject({actor: 'tester', promoted: ['a'], reason: 'Button radius changed on purpose'});
+ expect(decision).toMatchObject({
+ actor: 'tester',
+ promoted: ['a'],
+ reason: 'Button radius changed on purpose',
+ });
});
it('keeps earlier decisions when promoting again', () => {
@@ -68,13 +172,146 @@ describe('accept', () => {
promote({keys: ['a', 'b']});
promote({keys: [], prune: ['b'], reason: 'story deleted'});
expect(fs.existsSync(path.join(baselineDir, 'shots', 'b.png'))).toBe(false);
- expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual(['a']);
+ expect(Object.keys(readBaseline(baselineDir).manifest.shots)).toEqual([
+ 'a',
+ ]);
+ });
+});
+
+describe('browser-refresh recovery', () => {
+ function refresh() {
+ const manifest = structuredClone(currentManifest);
+ manifest.browser = 'chromium-140.0';
+ for (const key of ['a', 'b']) {
+ manifest.shots[key].sha256 = crypto
+ .createHash('sha256')
+ .update(`new-${key}`)
+ .digest('hex');
+ }
+ promote({
+ currentManifest: {...manifest, browser: 'chromium-139.0'},
+ keys: ['a', 'b'],
+ });
+ const verdict = {
+ status: 'failed',
+ counts: {total: 2, failed: 1},
+ failures: [
+ {
+ key: 'baseline',
+ error:
+ 'baseline was captured with chromium-139.0, this run with chromium-140.0 — refresh the baseline (gate.mjs accept --keys all --reason "browser bump").',
+ },
+ ],
+ removed: [],
+ };
+ return {
+ currentManifest: manifest,
+ verdict,
+ keys: ['a', 'b'],
+ reason: 'browser bump',
+ };
+ }
+
+ it('refreshes every shot and records the browser bump without changing the failed verdict', () => {
+ const options = refresh();
+ fs.writeFileSync(path.join(baselineDir, 'shots', 'a.png'), 'old-a');
+ const result = promote(options);
+ expect(result.promoted).toEqual(['a', 'b']);
+ expect(result.manifest.browser).toBe('chromium-140.0');
+ expect(result.manifest.decisions.at(-1).reason).toBe('browser bump');
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),
+ ).toBe('new-a');
+ expect(options.verdict.status).toBe('failed');
+ });
+
+ it.each([
+ [
+ 'another capture failure',
+ options => options.verdict.failures.push({key: 'a', error: 'timeout'}),
+ ],
+ [
+ 'another baseline error',
+ options => (options.verdict.failures[0].error = 'baseline is corrupt'),
+ ],
+ [
+ 'an unrelated failure key',
+ options => (options.verdict.failures[0].key = 'a'),
+ ],
+ [
+ 'inconsistent failure count',
+ options => (options.verdict.counts.failed = 2),
+ ],
+ [
+ 'inconsistent capture count',
+ options => (options.verdict.counts.total = 3),
+ ],
+ ['no failure list', options => delete options.verdict.failures],
+ [
+ 'a platform change',
+ options => (options.currentManifest.platform = 'darwin-arm64'),
+ ],
+ [
+ 'a viewport change',
+ options => (options.currentManifest.viewport.width = 800),
+ ],
+ ['missing viewport', options => delete options.currentManifest.viewport],
+ [
+ 'an unchanged browser',
+ options => (options.currentManifest.browser = 'chromium-139.0'),
+ ],
+ [
+ 'missing browser identity',
+ options => delete options.currentManifest.browser,
+ ],
+ ['a partial selection', options => (options.keys = ['a'])],
+ [
+ 'an omitted baseline shot',
+ options => {
+ delete options.currentManifest.shots.b;
+ options.keys = ['a'];
+ options.verdict.counts.total = 1;
+ },
+ ],
+ ['pruning', options => (options.prune = ['b'])],
+ ['reported removals', options => (options.verdict.removed = ['b'])],
+ ['a missing PNG', () => fs.rmSync(path.join(captureDir, 'shots', 'b.png'))],
+ [
+ 'a changed PNG',
+ () =>
+ fs.writeFileSync(path.join(captureDir, 'shots', 'b.png'), 'corrupt'),
+ ],
+ [
+ 'a missing digest',
+ options => delete options.currentManifest.shots.b.sha256,
+ ],
+ ])('refuses %s without changing any baseline bytes', (_label, mutate) => {
+ const options = refresh();
+ const before = fs.readFileSync(
+ path.join(baselineDir, 'manifest.json'),
+ 'utf8',
+ );
+ fs.writeFileSync(path.join(baselineDir, 'shots', 'a.png'), 'old-a');
+ fs.writeFileSync(path.join(baselineDir, 'shots', 'b.png'), 'old-b');
+ mutate(options);
+ expect(() => promote(options)).toThrow(/Refusing to promote/);
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'a.png'), 'utf8'),
+ ).toBe('old-a');
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'shots', 'b.png'), 'utf8'),
+ ).toBe('old-b');
+ expect(
+ fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'),
+ ).toBe(before);
});
});
describe('incomparable', () => {
it('refuses a baseline from another platform', () => {
- expect(incomparable({platform: 'darwin-arm64'}, {platform: 'linux-arm64'})).toMatch(/darwin-arm64/);
+ expect(
+ incomparable({platform: 'darwin-arm64'}, {platform: 'linux-arm64'}),
+ ).toMatch(/darwin-arm64/);
});
it('refuses a baseline captured at another viewport', () => {
diff --git a/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs b/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs
new file mode 100644
index 0000000000..736c178fb9
--- /dev/null
+++ b/.github/scripts/visual-gate/visual-baseline-workflow.test.mjs
@@ -0,0 +1,104 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+// Exercise the dispatch shell, including the interval that used to reject a
+// browser-refresh verdict before the publisher could validate its capture.
+import {spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+import {parse} from 'yaml';
+
+const workflow = parse(
+ fs.readFileSync(
+ new URL('../../workflows/visual-baseline.yml', import.meta.url),
+ 'utf8',
+ ),
+);
+const steps = workflow.jobs.promote.steps;
+let root;
+let env;
+beforeEach(() => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-baseline-workflow-'));
+ const bin = path.join(root, 'bin');
+ fs.mkdirSync(bin);
+ fs.writeFileSync(
+ path.join(bin, 'gh'),
+ '#!/bin/sh\nprintf "%s" "$RUN_JSON"\n',
+ {mode: 0o755},
+ );
+ // Publication is covered against a local Git remote in the publisher tests.
+ fs.writeFileSync(path.join(bin, 'node'), '#!/bin/sh\nexit 0\n', {
+ mode: 0o755,
+ });
+ env = {
+ ...process.env,
+ PATH: `${bin}:${process.env.PATH}`,
+ RUN_ID: '123',
+ GITHUB_REPOSITORY: 'facebook/astryx',
+ };
+});
+afterEach(() => fs.rmSync(root, {recursive: true, force: true}));
+
+function run(script) {
+ return spawnSync('bash', ['-e', '-c', script], {
+ cwd: root,
+ env,
+ encoding: 'utf8',
+ });
+}
+
+describe('Visual Baseline dispatch', () => {
+ it.each([
+ ['completed release', {}, true],
+ ['failed terminal release', {conclusion: 'failure'}, true],
+ ['wrong workflow', {path: '.github/workflows/ci.yml'}, false],
+ ['wrong branch', {head_branch: 'feature'}, false],
+ ['unfinished run', {status: 'in_progress'}, false],
+ ['missing workflow', {path: null}, false],
+ ])('validates the source run: %s', (_label, overrides, allowed) => {
+ env.RUN_JSON = JSON.stringify({
+ path: '.github/workflows/release-gate.yml',
+ head_branch: 'main',
+ status: 'completed',
+ ...overrides,
+ });
+ const result = run(
+ steps.find(step => step.name === 'Verify the source run').run,
+ );
+ expect(result.status === 0, result.stderr).toBe(allowed);
+ });
+
+ it('lets the publisher validate a failed browser-refresh capture inside its publication turn', () => {
+ fs.mkdirSync(path.join(root, '.visual-run'));
+ fs.writeFileSync(
+ path.join(root, '.visual-run/verdict.json'),
+ JSON.stringify({status: 'failed'}),
+ );
+ const download = steps.findIndex(
+ step => step.name === 'Download the capture being promoted',
+ );
+ const promote = steps.findIndex(
+ step => step.name === 'Promote and publish the baseline',
+ );
+ expect(download).toBeGreaterThan(-1);
+ expect(promote).toBeGreaterThan(download);
+ const beforePromotion = steps
+ .slice(download + 1, promote)
+ .map(step => step.run)
+ .filter(Boolean);
+ expect(beforePromotion.join('\n')).toContain(
+ 'gh-pages-publisher.mjs wait --scope visual-gate/baseline',
+ );
+ const result = run(beforePromotion.join('\n'));
+ expect(result.status, result.stderr || result.stdout).toBe(0);
+ expect(steps[promote].run).toContain(
+ 'gh-pages-publisher.mjs visual-baseline-manual',
+ );
+ expect(
+ steps.find(step => step.name === 'Release the baseline publication turn')
+ .if,
+ ).toBe('always()');
+ });
+});
diff --git a/.github/workflows/visual-baseline.yml b/.github/workflows/visual-baseline.yml
index f9b206bd20..3ce5813ff1 100644
--- a/.github/workflows/visual-baseline.yml
+++ b/.github/workflows/visual-baseline.yml
@@ -55,6 +55,26 @@ jobs:
- name: Setup Node and pnpm
uses: ./.github/actions/setup
+ - name: Verify the source run
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_ID: ${{ inputs.run_id }}
+ run: |
+ set -eu
+ # Promotion pulls shots out of a specific run's capture artifact. The
+ # run id is operator-typed — confirm it names a completed Release Gate
+ # run on main before anything from it is treated as baseline material
+ # (a typo'd id should fail here, not become the baseline).
+ RUN_JSON=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}")
+ RUN_PATH=$(printf '%s' "$RUN_JSON" | jq -r '.path')
+ RUN_BRANCH=$(printf '%s' "$RUN_JSON" | jq -r '.head_branch')
+ RUN_STATUS=$(printf '%s' "$RUN_JSON" | jq -r '.status')
+ if [ "$RUN_PATH" != ".github/workflows/release-gate.yml" ] || \
+ [ "$RUN_BRANCH" != "main" ] || [ "$RUN_STATUS" != "completed" ]; then
+ echo "::error::Run ${RUN_ID} is not a completed Release Gate run on main (path=${RUN_PATH}, branch=${RUN_BRANCH}, status=${RUN_STATUS})."
+ exit 1
+ fi
+
- name: Queue baseline publication
env:
GH_TOKEN: ${{ github.token }}
@@ -73,6 +93,9 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: node .github/scripts/gh-pages-publisher.mjs wait --scope visual-gate/baseline
+ # The publisher calls gate.mjs accept against the current baseline before
+ # writing or publishing any shots. Keep verdict validation there: a browser
+ # refresh needs both manifests, not a second status-only allowlist here.
- name: Promote and publish the baseline
env:
KEYS: ${{ inputs.keys }}
@@ -95,11 +118,15 @@ jobs:
run: node .github/scripts/gh-pages-publisher.mjs release --scope visual-gate/baseline
- name: Summary
+ env:
+ KEYS: ${{ inputs.keys }}
+ RUN_ID: ${{ inputs.run_id }}
+ REASON: ${{ inputs.reason }}
run: |
{
echo "## Visual baseline updated"
echo ""
- echo "- promoted: \`${{ inputs.keys }}\`"
- echo "- from run: ${{ inputs.run_id }}"
- echo "- reason: ${{ inputs.reason }}"
+ echo "- promoted: \`${KEYS}\`"
+ echo "- from run: ${RUN_ID}"
+ echo "- reason: ${REASON}"
} >> "$GITHUB_STEP_SUMMARY"