Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): guide visual acceptance reliably (#5791)

Cindy Zhanga1f0ddebd355
diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.js
index c2c3cd18db..92185078f1 100644
--- a/.github/scripts/lib/visual-format.js
+++ b/.github/scripts/lib/visual-format.js
@@ -30,8 +30,8 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`
: '';
const acceptanceCommand =
- verdict.context?.runId && verdict.context?.runAttempt
- ? `\n\nTo accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <reason>\``
+ safeReportUrl && verdict.context?.runId && verdict.context?.runAttempt
+ ? `\n\nA repository maintainer can accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <why every changed frame is correct>\``
: '';
const reportBase = safeReportUrl
? `${safeReportUrl.replace(/\/+$/, '')}/`
@@ -53,11 +53,15 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {
if (!verdict.changes || verdict.changes.length === 0) {
if (added.length > 0 || removed.length > 0) {
const frames = imageBase
- ? [...added.map(key => ({key, kind: 'after'})), ...removed.map(key => ({key, kind: 'before'}))]
+ ? [
+ ...added.map(key => ({key, kind: 'after'})),
+ ...removed.map(key => ({key, kind: 'before'})),
+ ]
.slice(0, 3)
.map(({key, kind}) => {
const safeKey = encodeURIComponent(String(key));
- const label = kind === 'after' ? 'Added — After' : 'Removed — Before';
+ const label =
+ kind === 'after' ? 'Added — After' : 'Removed — Before';
return `<p><b>${label}</b><br><img src="${imageBase}${kind}/${safeKey}.png" width="300" alt="${label} visual regression frame"></p>`;
})
.join('\n')
diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjs
index c8215b5e98..26c47d4a53 100644
--- a/.github/scripts/lib/visual-format.test.mjs
+++ b/.github/scripts/lib/visual-format.test.mjs
@@ -3,11 +3,20 @@
import {describe, expect, it} from 'vitest';
import {createRequire} from 'node:module';
-const {buildVisualSection} = createRequire(import.meta.url)('./visual-format.js');
+const {buildVisualSection} = createRequire(import.meta.url)(
+ './visual-format.js',
+);
const verdict = (overrides = {}) => ({
status: 'pass',
- counts: {total: 16, unchanged: 16, changed: 0, added: 0, removed: 0, failed: 0},
+ counts: {
+ total: 16,
+ unchanged: 16,
+ changed: 0,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
changes: [],
...overrides,
});
@@ -18,14 +27,23 @@ describe('buildVisualSection', () => {
});
it('says so plainly when nothing moved', () => {
- expect(buildVisualSection(verdict())).toContain('No visual change across 16 compared shot(s)');
+ expect(buildVisualSection(verdict())).toContain(
+ 'No visual change across 16 compared shot(s)',
+ );
});
it('shows added-only evidence instead of claiming there was no visual change', () => {
const section = buildVisualSection(
verdict({
status: 'changed',
- counts: {total: 20, unchanged: 14, changed: 0, added: 1, removed: 0, failed: 0},
+ counts: {
+ total: 20,
+ unchanged: 14,
+ changed: 0,
+ added: 1,
+ removed: 0,
+ failed: 0,
+ },
added: ['new-shot'],
removed: [],
}),
@@ -42,7 +60,14 @@ describe('buildVisualSection', () => {
const section = buildVisualSection(
verdict({
status: 'changed',
- counts: {total: 0, unchanged: 0, changed: 0, added: 0, removed: 1, failed: 0},
+ counts: {
+ total: 0,
+ unchanged: 0,
+ changed: 0,
+ added: 0,
+ removed: 1,
+ failed: 0,
+ },
added: [],
removed: ['old-shot'],
}),
@@ -55,7 +80,10 @@ describe('buildVisualSection', () => {
it('states the reason for a skip, so a broad PR does not look like a pass', () => {
const section = buildVisualSection(
- verdict({status: 'skipped', reason: '900 shots exceeds the 240-shot budget'}),
+ verdict({
+ status: 'skipped',
+ reason: '900 shots exceeds the 240-shot budget',
+ }),
'https://example.com/report/',
);
expect(section).toContain('Skipped');
@@ -66,28 +94,54 @@ describe('buildVisualSection', () => {
it('renders verdict strings as one-line literal text', () => {
const section = buildVisualSection(
- verdict({status: 'skipped', reason: 'first line\n## not a heading, just <em>text</em>'}),
+ verdict({
+ status: 'skipped',
+ reason: 'first line\n## not a heading, just <em>text</em>',
+ }),
+ );
+ expect(section).toContain(
+ 'first line ## not a heading, just &lt;em&gt;text&lt;/em&gt;',
);
- expect(section).toContain('first line ## not a heading, just &lt;em&gt;text&lt;/em&gt;');
expect(section).not.toContain('\n## not a heading');
});
it('keeps a change row a single table row whatever the field contents', () => {
const section = buildVisualSection(
verdict({
- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},
+ counts: {
+ total: 2,
+ unchanged: 1,
+ changed: 1,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
changes: [
- {component: 'Cell | with pipes', name: 'story\nnewline', theme: 'y2k', mode: 'light', diffPixels: '1234'},
+ {
+ component: 'Cell | with pipes',
+ name: 'story\nnewline',
+ theme: 'y2k',
+ mode: 'light',
+ diffPixels: '1234',
+ },
],
}),
);
- expect(section).toContain('| Cell \\| with pipes | story newline | y2k | light | 1,234 |');
+ expect(section).toContain(
+ '| Cell \\| with pipes | story newline | y2k | light | 1,234 |',
+ );
});
it('drops a report link that is not a plain https URL', () => {
- const good = buildVisualSection(verdict({status: 'failed', counts: {total: 1, failed: 1}}), 'https://example.com/report');
+ const good = buildVisualSection(
+ verdict({status: 'failed', counts: {total: 1, failed: 1}}),
+ 'https://example.com/report',
+ );
expect(good).toContain('href="https://example.com/report"');
- const bad = buildVisualSection(verdict({status: 'failed', counts: {total: 1, failed: 1}}), 'report.html" onmouseover="x');
+ const bad = buildVisualSection(
+ verdict({status: 'failed', counts: {total: 1, failed: 1}}),
+ 'report.html" onmouseover="x',
+ );
expect(bad).not.toContain('View the report');
});
@@ -95,10 +149,31 @@ describe('buildVisualSection', () => {
const section = buildVisualSection(
verdict({
status: 'changed',
- counts: {total: 16, unchanged: 14, changed: 2, added: 0, removed: 0, failed: 0},
+ counts: {
+ total: 16,
+ unchanged: 14,
+ changed: 2,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
changes: [
- {key: 'a', component: 'Button', name: 'Primary', theme: 'y2k', mode: 'light', diffPixels: 1126},
- {key: 'b', component: 'Button', name: 'Primary', theme: 'y2k', mode: 'dark', diffPixels: 1401},
+ {
+ key: 'a',
+ component: 'Button',
+ name: 'Primary',
+ theme: 'y2k',
+ mode: 'light',
+ diffPixels: 1126,
+ },
+ {
+ key: 'b',
+ component: 'Button',
+ name: 'Primary',
+ theme: 'y2k',
+ mode: 'dark',
+ diffPixels: 1401,
+ },
],
}),
);
@@ -110,8 +185,24 @@ describe('buildVisualSection', () => {
const section = buildVisualSection(
verdict({
status: 'changed',
- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},
- changes: [{key: 'a', component: 'B', name: 'S', theme: 't', mode: 'light', diffPixels: 5}],
+ counts: {
+ total: 2,
+ unchanged: 1,
+ changed: 1,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
+ changes: [
+ {
+ key: 'a',
+ component: 'B',
+ name: 'S',
+ theme: 't',
+ mode: 'light',
+ diffPixels: 5,
+ },
+ ],
}),
);
expect(section).toMatch(/question, not a failure/);
@@ -136,7 +227,14 @@ describe('buildVisualSection', () => {
const section = buildVisualSection(
verdict({
status: 'changed',
- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},
+ counts: {
+ total: 2,
+ unchanged: 1,
+ changed: 1,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
context: {runId: '123456', runAttempt: '2'},
changes: [
{
@@ -155,10 +253,48 @@ describe('buildVisualSection', () => {
expect(section).toContain(
'https://raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/before/core-button--primary__y2k-light.png',
);
- expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png');
- expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png');
+ expect(section).toContain(
+ 'raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png',
+ );
+ expect(section).toContain(
+ 'raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png',
+ );
expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');
- expect(section).toContain('/accept-visual 123456/2 <reason>');
+ expect(section).toContain(
+ 'A repository maintainer can accept these exact frames',
+ );
+ expect(section).toContain(
+ '/accept-visual 123456/2 <why every changed frame is correct>',
+ );
+ });
+
+ it('does not offer acceptance before immutable evidence is published', () => {
+ const section = buildVisualSection(
+ verdict({
+ status: 'changed',
+ counts: {
+ total: 2,
+ unchanged: 1,
+ changed: 1,
+ added: 0,
+ removed: 0,
+ failed: 0,
+ },
+ context: {runId: '123456', runAttempt: '2'},
+ changes: [
+ {
+ key: 'a',
+ component: 'Button',
+ name: 'Primary',
+ theme: 'neutral',
+ mode: 'light',
+ diffPixels: 12,
+ },
+ ],
+ }),
+ );
+ expect(section).not.toContain('/accept-visual');
+ expect(section).not.toContain('repository maintainer');
});
it('embeds at most three deltas so the PR comment stays readable', () => {
@@ -190,7 +326,16 @@ describe('buildVisualSection', () => {
verdict({
status: 'changed',
counts: {total: 1, changed: 1},
- changes: [{key: 'a', component: 'B', name: 'S', theme: 't', mode: 'light', diffPixels: 1}],
+ changes: [
+ {
+ key: 'a',
+ component: 'B',
+ name: 'S',
+ theme: 't',
+ mode: 'light',
+ diffPixels: 1,
+ },
+ ],
}),
'https://example.com/report/',
);
diff --git a/.github/scripts/visual-gate/authorization.mjs b/.github/scripts/visual-gate/authorization.mjs
index 7940e63f73..0da5497ac7 100644
--- a/.github/scripts/visual-gate/authorization.mjs
+++ b/.github/scripts/visual-gate/authorization.mjs
@@ -1,6 +1,19 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
const MAINTAINER_PERMISSIONS = new Set(['maintain', 'admin']);
+const FULL_SHA = /^[0-9a-f]{40}$/;
+
+export function visualAcceptanceEvidencePath({pr, head, run, attempt}) {
+ for (const [name, value] of Object.entries({pr, run, attempt})) {
+ if (!Number.isSafeInteger(Number(value)) || Number(value) <= 0) {
+ throw new Error(`${name} must be a positive integer`);
+ }
+ }
+ if (!FULL_SHA.test(String(head ?? ''))) {
+ throw new Error('head must be a full lowercase SHA');
+ }
+ return `pr/${Number(pr)}/visual/${head}/${Number(run)}/${Number(attempt)}/evidence.json`;
+}
function isCapabilityObject(value) {
return value !== null && typeof value === 'object' && !Array.isArray(value);
diff --git a/.github/scripts/visual-gate/authorization.test.mjs b/.github/scripts/visual-gate/authorization.test.mjs
index 7fa045060b..b20856f9cf 100644
--- a/.github/scripts/visual-gate/authorization.test.mjs
+++ b/.github/scripts/visual-gate/authorization.test.mjs
@@ -5,6 +5,7 @@ import {describe, expect, it} from 'vitest';
import {
isVisualAcceptanceEndpointMaintainer,
isVisualAcceptanceRecordMaintainer,
+ visualAcceptanceEvidencePath,
visualAcceptanceIdentity,
} from './authorization.mjs';
@@ -17,6 +18,28 @@ const payload = (permission, permissions, roleName = 'Custom role') => ({
user: {permissions},
});
+describe('visual acceptance evidence path', () => {
+ it('addresses the exact immutable run attempt', () => {
+ expect(
+ visualAcceptanceEvidencePath({
+ pr: 42,
+ head: 'a'.repeat(40),
+ run: 123456,
+ attempt: 2,
+ }),
+ ).toBe(`pr/42/visual/${'a'.repeat(40)}/123456/2/evidence.json`);
+ });
+
+ it.each([
+ [{pr: 0, head: 'a'.repeat(40), run: 1, attempt: 1}, /pr/],
+ [{pr: 1, head: 'short', run: 1, attempt: 1}, /head/],
+ [{pr: 1, head: 'a'.repeat(40), run: -1, attempt: 1}, /run/],
+ [{pr: 1, head: 'a'.repeat(40), run: 1, attempt: 0}, /attempt/],
+ ])('refuses an invalid identity', (identity, error) => {
+ expect(() => visualAcceptanceEvidencePath(identity)).toThrow(error);
+ });
+});
+
describe('visual acceptance endpoint authorization', () => {
it.each([
[
@@ -26,7 +49,11 @@ describe('visual acceptance endpoint authorization', () => {
],
['effective maintainer', payload('write', {maintain: true}), true],
['effective administrator', payload('write', {admin: true}), true],
- ['ordinary writer', payload('write', {admin: false, maintain: false}), false],
+ [
+ 'ordinary writer',
+ payload('write', {admin: false, maintain: false}),
+ false,
+ ],
[
'top-level admin overridden by nested false capabilities',
payload('admin', {admin: false, maintain: false}),
diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjs
index 0ddf0f4a71..121bdc414b 100644
--- a/.github/scripts/visual-gate/publish-pr-report.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.mjs
@@ -349,7 +349,7 @@ fs.writeFileSync(
fs.writeFileSync(
path.join(output, 'index.html'),
renderReport(verdict, {
- acceptHint: `/accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,
+ acceptHint: `Repository maintainers only: /accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,
oneSidedEvidence: true,
}),
);
diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
index fb5cdec867..55227f89f0 100644
--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs
+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
@@ -71,6 +71,29 @@ describe('visual acceptance workflow concurrency', () => {
expect(comment).not.toContain('const previewAvailable');
});
+ it('requires published evidence before offering or recording acceptance', () => {
+ const publisher = workflow('pr-comment.yml');
+ const acceptance = workflow('visual-acceptance.yml');
+ const authorize = acceptance.slice(
+ acceptance.indexOf(' authorize:'),
+ acceptance.indexOf(' accept:'),
+ );
+
+ expect(publisher).toContain(
+ "visualPublished: process.env.VISUAL_PUBLISHED === 'true'",
+ );
+ expect(authorize).toContain('visualAcceptanceEvidencePath({');
+ expect(authorize).toContain('github.rest.repos.getContent({');
+ expect(authorize).toContain("ref: 'gh-pages'");
+ expect(authorize).toContain('trusted visual evidence is not published yet');
+ expect(authorize.indexOf("latestCI.status !== 'completed'")).toBeLessThan(
+ authorize.indexOf('github.rest.repos.getContent({'),
+ );
+ expect(authorize.indexOf('github.rest.repos.getContent({')).toBeLessThan(
+ authorize.indexOf('isVisualAcceptanceEndpointMaintainer(identity)'),
+ );
+ });
+
it('keeps spec-only checks lightweight while removing stale links', () => {
const value = workflow('pr-comment.yml');
const reconcile = value.slice(
diff --git a/.github/workflows/visual-acceptance.yml b/.github/workflows/visual-acceptance.yml
index 34441bb4dd..53aa50ce74 100644
--- a/.github/workflows/visual-acceptance.yml
+++ b/.github/workflows/visual-acceptance.yml
@@ -58,6 +58,7 @@ jobs:
const {pathToFileURL} = require('node:url');
const {
isVisualAcceptanceEndpointMaintainer,
+ visualAcceptanceEvidencePath,
visualAcceptanceIdentity,
} = await import(
pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-gate/authorization.mjs`).href,
@@ -81,7 +82,7 @@ jobs:
const body = context.payload.comment.body.trim();
const match = body.match(/^\/accept-visual\s+(\d+)\/(\d+)\s+([\s\S]+)$/);
if (!match) {
- refuse('copy the exact `/accept-visual <run>/<attempt> <reason>` command from the report.');
+ refuse('copy the exact maintainer-only `/accept-visual <run>/<attempt> <why every changed frame is correct>` command from the latest report.');
return;
}
const runId = Number(match[1]);
@@ -104,7 +105,25 @@ jobs:
.sort((a, b) => b.id - a.id)[0];
if (!latestCI || latestCI.id !== runId || latestCI.run_attempt !== runAttempt ||
latestCI.status !== 'completed') {
- refuse('that evidence is not from the latest completed CI attempt for this head.');
+ refuse('that command is not from the latest completed CI attempt for this head. Wait for the latest report and copy its command.');
+ return;
+ }
+ const evidencePath = visualAcceptanceEvidencePath({
+ pr: number,
+ head: pr.head.sha,
+ run: runId,
+ attempt: runAttempt,
+ });
+ try {
+ await github.rest.repos.getContent({
+ owner,
+ repo,
+ path: evidencePath,
+ ref: 'gh-pages',
+ });
+ } catch (error) {
+ if (error.status !== 404) throw error;
+ refuse('trusted visual evidence is not published yet. Wait for the latest PR report; if it does not appear, rerun CI and use its new command.');
return;
}
let identity = visualAcceptanceIdentity();
@@ -307,9 +326,9 @@ jobs:
const {owner, repo} = context.repo;
await github.rest.repos.createCommitStatus({
owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance', state: 'failure',
- description: 'Visual acceptance could not be recorded.',
+ description: 'Visual acceptance hit an infrastructure error.',
});
await github.rest.issues.createComment({
owner, repo, issue_number: Number(process.env.PR_NUMBER),
- body: 'Visual acceptance could not be recorded. See this workflow run for the rejected invariant or infrastructure error.',
+ body: 'Visual acceptance hit an infrastructure error. Retry only after the latest PR report shows a maintainer command; if no report appears, rerun CI. See this workflow run for details.',
});