An open source design system that's fully customizable and agent ready
TypeScript
fix(ci): guide visual acceptance reliably (#5791)
Cindy Zhanga1f0ddebd355
diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.jsindex c2c3cd18db..92185078f1 100644--- a/.github/scripts/lib/visual-format.js+++ b/.github/scripts/lib/visual-format.js@@ -30,8 +30,8 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`: '';const acceptanceCommand =- verdict.context?.runId && verdict.context?.runAttempt- ? `\n\nTo accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <reason>\``+ safeReportUrl && verdict.context?.runId && verdict.context?.runAttempt+ ? `\n\nA repository maintainer can accept these exact frames: \`/accept-visual ${num(verdict.context.runId)}/${num(verdict.context.runAttempt)} <why every changed frame is correct>\``: '';const reportBase = safeReportUrl? `${safeReportUrl.replace(/\/+$/, '')}/`@@ -53,11 +53,15 @@ function buildVisualSection(verdict, reportUrl, imageUrl) {if (!verdict.changes || verdict.changes.length === 0) {if (added.length > 0 || removed.length > 0) {const frames = imageBase- ? [...added.map(key => ({key, kind: 'after'})), ...removed.map(key => ({key, kind: 'before'}))]+ ? [+ ...added.map(key => ({key, kind: 'after'})),+ ...removed.map(key => ({key, kind: 'before'})),+ ].slice(0, 3).map(({key, kind}) => {const safeKey = encodeURIComponent(String(key));- const label = kind === 'after' ? 'Added — After' : 'Removed — Before';+ const label =+ kind === 'after' ? 'Added — After' : 'Removed — Before';return `<p><b>${label}</b><br><img src="${imageBase}${kind}/${safeKey}.png" width="300" alt="${label} visual regression frame"></p>`;}).join('\n')diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjsindex c8215b5e98..26c47d4a53 100644--- a/.github/scripts/lib/visual-format.test.mjs+++ b/.github/scripts/lib/visual-format.test.mjs@@ -3,11 +3,20 @@import {describe, expect, it} from 'vitest';import {createRequire} from 'node:module';-const {buildVisualSection} = createRequire(import.meta.url)('./visual-format.js');+const {buildVisualSection} = createRequire(import.meta.url)(+ './visual-format.js',+);const verdict = (overrides = {}) => ({status: 'pass',- counts: {total: 16, unchanged: 16, changed: 0, added: 0, removed: 0, failed: 0},+ counts: {+ total: 16,+ unchanged: 16,+ changed: 0,+ added: 0,+ removed: 0,+ failed: 0,+ },changes: [],...overrides,});@@ -18,14 +27,23 @@ describe('buildVisualSection', () => {});it('says so plainly when nothing moved', () => {- expect(buildVisualSection(verdict())).toContain('No visual change across 16 compared shot(s)');+ expect(buildVisualSection(verdict())).toContain(+ 'No visual change across 16 compared shot(s)',+ );});it('shows added-only evidence instead of claiming there was no visual change', () => {const section = buildVisualSection(verdict({status: 'changed',- counts: {total: 20, unchanged: 14, changed: 0, added: 1, removed: 0, failed: 0},+ counts: {+ total: 20,+ unchanged: 14,+ changed: 0,+ added: 1,+ removed: 0,+ failed: 0,+ },added: ['new-shot'],removed: [],}),@@ -42,7 +60,14 @@ describe('buildVisualSection', () => {const section = buildVisualSection(verdict({status: 'changed',- counts: {total: 0, unchanged: 0, changed: 0, added: 0, removed: 1, failed: 0},+ counts: {+ total: 0,+ unchanged: 0,+ changed: 0,+ added: 0,+ removed: 1,+ failed: 0,+ },added: [],removed: ['old-shot'],}),@@ -55,7 +80,10 @@ describe('buildVisualSection', () => {it('states the reason for a skip, so a broad PR does not look like a pass', () => {const section = buildVisualSection(- verdict({status: 'skipped', reason: '900 shots exceeds the 240-shot budget'}),+ verdict({+ status: 'skipped',+ reason: '900 shots exceeds the 240-shot budget',+ }),'https://example.com/report/',);expect(section).toContain('Skipped');@@ -66,28 +94,54 @@ describe('buildVisualSection', () => {it('renders verdict strings as one-line literal text', () => {const section = buildVisualSection(- verdict({status: 'skipped', reason: 'first line\n## not a heading, just <em>text</em>'}),+ verdict({+ status: 'skipped',+ reason: 'first line\n## not a heading, just <em>text</em>',+ }),+ );+ expect(section).toContain(+ 'first line ## not a heading, just <em>text</em>',);- expect(section).toContain('first line ## not a heading, just <em>text</em>');expect(section).not.toContain('\n## not a heading');});it('keeps a change row a single table row whatever the field contents', () => {const section = buildVisualSection(verdict({- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},+ counts: {+ total: 2,+ unchanged: 1,+ changed: 1,+ added: 0,+ removed: 0,+ failed: 0,+ },changes: [- {component: 'Cell | with pipes', name: 'story\nnewline', theme: 'y2k', mode: 'light', diffPixels: '1234'},+ {+ component: 'Cell | with pipes',+ name: 'story\nnewline',+ theme: 'y2k',+ mode: 'light',+ diffPixels: '1234',+ },],}),);- expect(section).toContain('| Cell \\| with pipes | story newline | y2k | light | 1,234 |');+ expect(section).toContain(+ '| Cell \\| with pipes | story newline | y2k | light | 1,234 |',+ );});it('drops a report link that is not a plain https URL', () => {- const good = buildVisualSection(verdict({status: 'failed', counts: {total: 1, failed: 1}}), 'https://example.com/report');+ const good = buildVisualSection(+ verdict({status: 'failed', counts: {total: 1, failed: 1}}),+ 'https://example.com/report',+ );expect(good).toContain('href="https://example.com/report"');- const bad = buildVisualSection(verdict({status: 'failed', counts: {total: 1, failed: 1}}), 'report.html" onmouseover="x');+ const bad = buildVisualSection(+ verdict({status: 'failed', counts: {total: 1, failed: 1}}),+ 'report.html" onmouseover="x',+ );expect(bad).not.toContain('View the report');});@@ -95,10 +149,31 @@ describe('buildVisualSection', () => {const section = buildVisualSection(verdict({status: 'changed',- counts: {total: 16, unchanged: 14, changed: 2, added: 0, removed: 0, failed: 0},+ counts: {+ total: 16,+ unchanged: 14,+ changed: 2,+ added: 0,+ removed: 0,+ failed: 0,+ },changes: [- {key: 'a', component: 'Button', name: 'Primary', theme: 'y2k', mode: 'light', diffPixels: 1126},- {key: 'b', component: 'Button', name: 'Primary', theme: 'y2k', mode: 'dark', diffPixels: 1401},+ {+ key: 'a',+ component: 'Button',+ name: 'Primary',+ theme: 'y2k',+ mode: 'light',+ diffPixels: 1126,+ },+ {+ key: 'b',+ component: 'Button',+ name: 'Primary',+ theme: 'y2k',+ mode: 'dark',+ diffPixels: 1401,+ },],}),);@@ -110,8 +185,24 @@ describe('buildVisualSection', () => {const section = buildVisualSection(verdict({status: 'changed',- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},- changes: [{key: 'a', component: 'B', name: 'S', theme: 't', mode: 'light', diffPixels: 5}],+ counts: {+ total: 2,+ unchanged: 1,+ changed: 1,+ added: 0,+ removed: 0,+ failed: 0,+ },+ changes: [+ {+ key: 'a',+ component: 'B',+ name: 'S',+ theme: 't',+ mode: 'light',+ diffPixels: 5,+ },+ ],}),);expect(section).toMatch(/question, not a failure/);@@ -136,7 +227,14 @@ describe('buildVisualSection', () => {const section = buildVisualSection(verdict({status: 'changed',- counts: {total: 2, unchanged: 1, changed: 1, added: 0, removed: 0, failed: 0},+ counts: {+ total: 2,+ unchanged: 1,+ changed: 1,+ added: 0,+ removed: 0,+ failed: 0,+ },context: {runId: '123456', runAttempt: '2'},changes: [{@@ -155,10 +253,48 @@ describe('buildVisualSection', () => {expect(section).toContain('https://raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/before/core-button--primary__y2k-light.png',);- expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png');- expect(section).toContain('raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png');+ expect(section).toContain(+ 'raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/after/core-button--primary__y2k-light.png',+ );+ expect(section).toContain(+ 'raw.githubusercontent.com/facebook/astryx/gh-pages/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png',+ );expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');- expect(section).toContain('/accept-visual 123456/2 <reason>');+ expect(section).toContain(+ 'A repository maintainer can accept these exact frames',+ );+ expect(section).toContain(+ '/accept-visual 123456/2 <why every changed frame is correct>',+ );+ });++ it('does not offer acceptance before immutable evidence is published', () => {+ const section = buildVisualSection(+ verdict({+ status: 'changed',+ counts: {+ total: 2,+ unchanged: 1,+ changed: 1,+ added: 0,+ removed: 0,+ failed: 0,+ },+ context: {runId: '123456', runAttempt: '2'},+ changes: [+ {+ key: 'a',+ component: 'Button',+ name: 'Primary',+ theme: 'neutral',+ mode: 'light',+ diffPixels: 12,+ },+ ],+ }),+ );+ expect(section).not.toContain('/accept-visual');+ expect(section).not.toContain('repository maintainer');});it('embeds at most three deltas so the PR comment stays readable', () => {@@ -190,7 +326,16 @@ describe('buildVisualSection', () => {verdict({status: 'changed',counts: {total: 1, changed: 1},- changes: [{key: 'a', component: 'B', name: 'S', theme: 't', mode: 'light', diffPixels: 1}],+ changes: [+ {+ key: 'a',+ component: 'B',+ name: 'S',+ theme: 't',+ mode: 'light',+ diffPixels: 1,+ },+ ],}),'https://example.com/report/',);diff --git a/.github/scripts/visual-gate/authorization.mjs b/.github/scripts/visual-gate/authorization.mjsindex 7940e63f73..0da5497ac7 100644--- a/.github/scripts/visual-gate/authorization.mjs+++ b/.github/scripts/visual-gate/authorization.mjs@@ -1,6 +1,19 @@// Copyright (c) Meta Platforms, Inc. and affiliates.const MAINTAINER_PERMISSIONS = new Set(['maintain', 'admin']);+const FULL_SHA = /^[0-9a-f]{40}$/;++export function visualAcceptanceEvidencePath({pr, head, run, attempt}) {+ for (const [name, value] of Object.entries({pr, run, attempt})) {+ if (!Number.isSafeInteger(Number(value)) || Number(value) <= 0) {+ throw new Error(`${name} must be a positive integer`);+ }+ }+ if (!FULL_SHA.test(String(head ?? ''))) {+ throw new Error('head must be a full lowercase SHA');+ }+ return `pr/${Number(pr)}/visual/${head}/${Number(run)}/${Number(attempt)}/evidence.json`;+}function isCapabilityObject(value) {return value !== null && typeof value === 'object' && !Array.isArray(value);diff --git a/.github/scripts/visual-gate/authorization.test.mjs b/.github/scripts/visual-gate/authorization.test.mjsindex 7fa045060b..b20856f9cf 100644--- a/.github/scripts/visual-gate/authorization.test.mjs+++ b/.github/scripts/visual-gate/authorization.test.mjs@@ -5,6 +5,7 @@ import {describe, expect, it} from 'vitest';import {isVisualAcceptanceEndpointMaintainer,isVisualAcceptanceRecordMaintainer,+ visualAcceptanceEvidencePath,visualAcceptanceIdentity,} from './authorization.mjs';@@ -17,6 +18,28 @@ const payload = (permission, permissions, roleName = 'Custom role') => ({user: {permissions},});+describe('visual acceptance evidence path', () => {+ it('addresses the exact immutable run attempt', () => {+ expect(+ visualAcceptanceEvidencePath({+ pr: 42,+ head: 'a'.repeat(40),+ run: 123456,+ attempt: 2,+ }),+ ).toBe(`pr/42/visual/${'a'.repeat(40)}/123456/2/evidence.json`);+ });++ it.each([+ [{pr: 0, head: 'a'.repeat(40), run: 1, attempt: 1}, /pr/],+ [{pr: 1, head: 'short', run: 1, attempt: 1}, /head/],+ [{pr: 1, head: 'a'.repeat(40), run: -1, attempt: 1}, /run/],+ [{pr: 1, head: 'a'.repeat(40), run: 1, attempt: 0}, /attempt/],+ ])('refuses an invalid identity', (identity, error) => {+ expect(() => visualAcceptanceEvidencePath(identity)).toThrow(error);+ });+});+describe('visual acceptance endpoint authorization', () => {it.each([[@@ -26,7 +49,11 @@ describe('visual acceptance endpoint authorization', () => {],['effective maintainer', payload('write', {maintain: true}), true],['effective administrator', payload('write', {admin: true}), true],- ['ordinary writer', payload('write', {admin: false, maintain: false}), false],+ [+ 'ordinary writer',+ payload('write', {admin: false, maintain: false}),+ false,+ ],['top-level admin overridden by nested false capabilities',payload('admin', {admin: false, maintain: false}),diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjsindex 0ddf0f4a71..121bdc414b 100644--- a/.github/scripts/visual-gate/publish-pr-report.mjs+++ b/.github/scripts/visual-gate/publish-pr-report.mjs@@ -349,7 +349,7 @@ fs.writeFileSync(fs.writeFileSync(path.join(output, 'index.html'),renderReport(verdict, {- acceptHint: `/accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,+ acceptHint: `Repository maintainers only: /accept-visual ${runId}/${runAttempt} <why every changed frame is correct>`,oneSidedEvidence: true,}),);diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjsindex fb5cdec867..55227f89f0 100644--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs@@ -71,6 +71,29 @@ describe('visual acceptance workflow concurrency', () => {expect(comment).not.toContain('const previewAvailable');});+ it('requires published evidence before offering or recording acceptance', () => {+ const publisher = workflow('pr-comment.yml');+ const acceptance = workflow('visual-acceptance.yml');+ const authorize = acceptance.slice(+ acceptance.indexOf(' authorize:'),+ acceptance.indexOf(' accept:'),+ );++ expect(publisher).toContain(+ "visualPublished: process.env.VISUAL_PUBLISHED === 'true'",+ );+ expect(authorize).toContain('visualAcceptanceEvidencePath({');+ expect(authorize).toContain('github.rest.repos.getContent({');+ expect(authorize).toContain("ref: 'gh-pages'");+ expect(authorize).toContain('trusted visual evidence is not published yet');+ expect(authorize.indexOf("latestCI.status !== 'completed'")).toBeLessThan(+ authorize.indexOf('github.rest.repos.getContent({'),+ );+ expect(authorize.indexOf('github.rest.repos.getContent({')).toBeLessThan(+ authorize.indexOf('isVisualAcceptanceEndpointMaintainer(identity)'),+ );+ });+it('keeps spec-only checks lightweight while removing stale links', () => {const value = workflow('pr-comment.yml');const reconcile = value.slice(diff --git a/.github/workflows/visual-acceptance.yml b/.github/workflows/visual-acceptance.ymlindex 34441bb4dd..53aa50ce74 100644--- a/.github/workflows/visual-acceptance.yml+++ b/.github/workflows/visual-acceptance.yml@@ -58,6 +58,7 @@ jobs:const {pathToFileURL} = require('node:url');const {isVisualAcceptanceEndpointMaintainer,+ visualAcceptanceEvidencePath,visualAcceptanceIdentity,} = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/.github/scripts/visual-gate/authorization.mjs`).href,@@ -81,7 +82,7 @@ jobs:const body = context.payload.comment.body.trim();const match = body.match(/^\/accept-visual\s+(\d+)\/(\d+)\s+([\s\S]+)$/);if (!match) {- refuse('copy the exact `/accept-visual <run>/<attempt> <reason>` command from the report.');+ refuse('copy the exact maintainer-only `/accept-visual <run>/<attempt> <why every changed frame is correct>` command from the latest report.');return;}const runId = Number(match[1]);@@ -104,7 +105,25 @@ jobs:.sort((a, b) => b.id - a.id)[0];if (!latestCI || latestCI.id !== runId || latestCI.run_attempt !== runAttempt ||latestCI.status !== 'completed') {- refuse('that evidence is not from the latest completed CI attempt for this head.');+ refuse('that command is not from the latest completed CI attempt for this head. Wait for the latest report and copy its command.');+ return;+ }+ const evidencePath = visualAcceptanceEvidencePath({+ pr: number,+ head: pr.head.sha,+ run: runId,+ attempt: runAttempt,+ });+ try {+ await github.rest.repos.getContent({+ owner,+ repo,+ path: evidencePath,+ ref: 'gh-pages',+ });+ } catch (error) {+ if (error.status !== 404) throw error;+ refuse('trusted visual evidence is not published yet. Wait for the latest PR report; if it does not appear, rerun CI and use its new command.');return;}let identity = visualAcceptanceIdentity();@@ -307,9 +326,9 @@ jobs:const {owner, repo} = context.repo;await github.rest.repos.createCommitStatus({owner, repo, sha: process.env.HEAD_SHA, context: 'visual-acceptance', state: 'failure',- description: 'Visual acceptance could not be recorded.',+ description: 'Visual acceptance hit an infrastructure error.',});await github.rest.issues.createComment({owner, repo, issue_number: Number(process.env.PR_NUMBER),- body: 'Visual acceptance could not be recorded. See this workflow run for the rejected invariant or infrastructure error.',+ body: 'Visual acceptance hit an infrastructure error. Retry only after the latest PR report shows a maintainer command; if no report appears, rerun CI. See this workflow run for details.',});