An open source design system that's fully customizable and agent ready
TypeScript
ci: the PR report shows before/after/diff thumbnails for visual changes, from frames a frames-only job publishes (#9201)
Cindy Zhang2218dee08f7a
diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.jsindex dfcbf268f6..f0201c2ec6 100644--- a/.github/scripts/generate-pr-comment.js+++ b/.github/scripts/generate-pr-comment.js@@ -3,7 +3,7 @@/*** @description Generates the PR comment from the run's analysis results- * @input --analysis <file> --base-stats <file> --package-size <file> --a11y <file> --visual <file> --preview <file> --jobs <file> --source-conclusion <conclusion> --run-url <url>+ * @input --analysis <file> --base-stats <file> --package-size <file> --a11y <file> --visual <file> --visual-frames <file> --preview <file> --jobs <file> --source-conclusion <conclusion> --run-url <url>* @output Formatted markdown comment body to stdout, compact by design: this* head's preview first, the modified components (preview links, size, lines* of code, a mark per browser suite that ran; three rows visible, the rest@@ -66,6 +66,8 @@ if (baseStats) {const preview = readOptionalJson('preview');const jobs = readOptionalJson('jobs');const packageSize = readOptionalJson('package-size');+// The visual frames publish-frames.mjs put on this PR's assets branch.+const visualFrames = readOptionalJson('visual-frames');const visualVerdict = readOptionalJson('visual');let a11yReport = null;try {@@ -125,6 +127,7 @@ const report = buildCompactReport({a11yReport,visualVerdict,packageSize,+ visualFrames,});const footerLinksStr = runUrl? ` | ${extLink('View full report', runUrl)}`diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjsindex 41b69970a0..03fddfa6eb 100644--- a/.github/scripts/lib/pr-preview.mjs+++ b/.github/scripts/lib/pr-preview.mjs@@ -113,6 +113,35 @@ export async function headPreview({: {state: 'unknown', sha};}+/**+ * The frames publish-frames.mjs published, written for the generator, only+ * when their URL is this pull request's assets branch for this exact head and+ * each frame is a `<before|after|diff>/<key>.png` path; otherwise null.+ */+export function trustedFrames({frames, identity, inputs}) {+ const expected = `https://raw.githubusercontent.com/${identity.baseRepository}/assets/pr-${identity.prNumber}/visual/${identity.headSha}/`;+ let published;+ try {+ published = JSON.parse(frames?.published || '[]');+ } catch {+ return null;+ }+ if (+ frames?.imageBase !== expected ||+ !Array.isArray(published) ||+ published.length === 0 ||+ !published.every(entry =>+ /^(before|after|diff)\/(?!\.+\.png$)[A-Za-z0-9._-]{1,240}\.png$/.test(+ String(entry),+ ),+ )+ )+ return null;+ const file = path.join(inputs, 'visual-frames.json');+ fs.writeFileSync(file, JSON.stringify({imageBase: expected, published}));+ return file;+}+/*** The source run's jobs (latest attempt) with each step's conclusion, so the* report can say which jobs ran and which skipped their work by path. Null@@ -584,6 +613,7 @@ export async function reconcilePrComment({metadataPath = 'pr-analysis/pr-meta.json',baseStatsPath = 'pr-size-base/size-base.json',packageSizePath = 'pr-analysis/package-size.json',+ visualFrames = null,a11yPath = 'a11y/a11y-report.json',visualPath = 'trusted-visual/verdict.json',createIfMissing = true,@@ -648,6 +678,8 @@ export async function reconcilePrComment({repo,runId: identity.sourceRunId,});+ // Frames only from this PR's assets branch for this exact head.+ const framesPath = trustedFrames({frames: visualFrames, identity, inputs});const previewPath = path.join(inputs, 'preview.json');const jobsPath = path.join(inputs, 'jobs.json');if (preview) fs.writeFileSync(previewPath, JSON.stringify(preview));@@ -676,6 +708,7 @@ export async function reconcilePrComment({// checks, or with the ci:run-a11y label); the comment says so....(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []),...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []),+ ...(framesPath ? ['--visual-frames', framesPath] : []),...(preview ? ['--preview', previewPath] : []),...(jobs ? ['--jobs', jobsPath] : []),'--source-conclusion',diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjsindex 4b86724a1f..89d472245b 100644--- a/.github/scripts/lib/pr-preview.test.mjs+++ b/.github/scripts/lib/pr-preview.test.mjs@@ -15,6 +15,7 @@ import {confirmSourceRunIdentity,PR_ANALYSIS_MARKER,reconcilePrComment,+ trustedFrames,resolveWorkflowRunPullRequest,validateAnalysisMetadata,} from './pr-preview.mjs';@@ -492,6 +493,7 @@ describe('trusted PR preview identity', () => {expect(writeCapableJobs).toEqual(['spec-only-reconcile',+ 'frames','comment','report',]);@@ -561,7 +563,13 @@ describe('trusted PR preview identity', () => {describe('PR Comment keeps its outcome off main', () => {// workflow_run attaches these jobs to main's latest commit, so a failing step// here would turn main red for a pull request's problem.- const LANE = ['resolve', 'spec-only-reconcile', 'comment', 'report'];+ const LANE = [+ 'resolve',+ 'spec-only-reconcile',+ 'frames',+ 'comment',+ 'report',+ ];const OPTIONAL = new Set(['actions/download-artifact@v8']);it.each(LANE)('never fails the run from the %s job', name => {@@ -588,10 +596,20 @@ describe('PR Comment keeps its outcome off main', () => {it('reports the outcome on the head of the run the event names', () => {const report = PR_COMMENT_WORKFLOW.jobs.report;- expect(report.needs).toEqual(['resolve', 'spec-only-reconcile', 'comment']);+ expect(report.needs).toEqual([+ 'resolve',+ 'spec-only-reconcile',+ 'frames',+ 'comment',+ ]);const [step] = report.steps;expect(step.env.HEAD_SHA).toBe('${{ github.event.workflow_run.head_sha }}');- for (const name of ['resolve', 'spec-only-reconcile', 'comment']) {+ for (const name of [+ 'resolve',+ 'spec-only-reconcile',+ 'frames',+ 'comment',+ ]) {expect(step.env.FAILED).toContain(`needs.${name}.outputs.failed == 'true'`,);@@ -1358,3 +1376,67 @@ describe('PR comment preview reconciliation', () => {},);});++describe('visual frames', () => {+ const identity = {+ prNumber: 9171,+ headSha: 'a'.repeat(40),+ baseRepository: 'meta/astryx',+ };+ const imageBase = `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${'a'.repeat(40)}/`;+ const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'frames-'));++ it("passes frames only from this pull request's assets branch for this head", () => {+ const published = JSON.stringify(['before/b.png', 'diff/b.png']);+ const file = trustedFrames({+ frames: {imageBase, published},+ identity,+ inputs: tmp(),+ });+ expect(JSON.parse(fs.readFileSync(file, 'utf8'))).toEqual({+ imageBase,+ published: ['before/b.png', 'diff/b.png'],+ });+ for (const bad of [+ imageBase.replace('pr-9171', 'pr-1'),+ imageBase.replace('a'.repeat(40), 'b'.repeat(40)),+ 'https://evil.example/visual/',+ ])+ expect(+ trustedFrames({+ frames: {imageBase: bad, published},+ identity,+ inputs: tmp(),+ }),+ ).toBeNull();+ });++ it('refuses frame paths that are not before/after/diff PNGs', () => {+ for (const published of [+ '["../x.png"]',+ '["other/b.png"]',+ '[]',+ 'not json',+ ])+ expect(+ trustedFrames({+ frames: {imageBase, published},+ identity,+ inputs: tmp(),+ }),+ ).toBeNull();+ });++ it('gives only the frames job contents: write, with no persisted credentials', () => {+ const {jobs} = PR_COMMENT_WORKFLOW;+ for (const [name, job] of Object.entries(jobs))+ expect(job.permissions?.contents === 'write', name).toBe(+ name === 'frames',+ );+ const checkout = jobs.frames.steps.find(step =>+ String(step.uses).startsWith('actions/checkout@'),+ );+ expect(checkout.with).toEqual({ref: 'main', 'persist-credentials': false});+ expect(jobs.comment.needs).toEqual(['resolve', 'frames']);+ });+});diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.jsindex 0111a9d5dc..ba4c358b16 100644--- a/.github/scripts/lib/pr-status.js+++ b/.github/scripts/lib/pr-status.js@@ -283,14 +283,22 @@ function buildComponentTable({analysis, origin, a11yReport, visualVerdict}) {}/**- * The changed shots, largest change first: up to VISUAL_FRAME_LIMIT rows- * naming each shot and its change, inside one disclosure whose summary- * carries the counts. The run's visual report holds the frames.+ * The changed shots, largest change first: up to VISUAL_FRAME_LIMIT rows,+ * inside one disclosure whose summary carries the counts. With `frames`+ * (publish-frames.mjs's `imageBase` and the `<kind>/<key>.png` it published),+ * each row shows its before/after/diff thumbnails, and only published frames;+ * without it, each row names its change.*/-function buildVisualDeltas({verdict}) {+/**+ * The changed shots in report order: changed by pixels changed, largest first,+ * then added, then removed. The report lists the first VISUAL_FRAME_LIMIT,+ * and publish-frames.mjs publishes frames for the same ones, so thumbnails+ * and rows cannot drift apart.+ */+function visualShots(verdict) {if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed')- return '';- const all = [+ return [];+ return [...[...(verdict.changes || [])].sort((a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0)).map(change => ({...change, kinds: ['before', 'after', 'diff']})),@@ -301,6 +309,10 @@ function buildVisualDeltas({verdict}) {kinds: ['before'],})),];+}++function buildVisualDeltas({verdict, frames = null}) {+ const all = visualShots(verdict);if (all.length === 0) return '';const rows = all.slice(0, VISUAL_FRAME_LIMIT);const unshown = all.length - rows.length;@@ -321,8 +333,24 @@ function buildVisualDeltas({verdict}) {.join(', ');return variant ? `${story} (${variant})` : story;};- const header = '| Component · story | Change |\n|---|---|\n';- const line = row => `| ${label(row)} | ${changeText(row)} |`;+ const published = new Set(frames?.published || []);+ const base =+ frames?.imageBase && published.size > 0+ ? `${String(frames.imageBase).replace(/\/+$/, '')}/`+ : null;+ const frame = (row, kind) => {+ if (!row.kinds.includes(kind)) return '—';+ const file = `${kind}/${row.key}.png`;+ if (!published.has(file)) return 'not published';+ return `<img src="${base}${kind}/${encodeURIComponent(String(row.key))}.png" height="120" alt="${kind} frame">`;+ };+ const header = base+ ? '| Component · story | Before | After | Diff |\n|---|---|---|---|\n'+ : '| Component · story | Change |\n|---|---|\n';+ const line = row =>+ base+ ? `| ${label(row)} | ${frame(row, 'before')} | ${frame(row, 'after')} | ${frame(row, 'diff')} |`+ : `| ${label(row)} | ${changeText(row)} |`;// A shot key is `<package>-<component>--<story>__<theme>-<mode>`.const componentOf = row =>String(@@ -432,6 +460,7 @@ function buildCompactReport({a11yReport = null,visualVerdict = null,packageSize = null,+ visualFrames = null,}) {const origin =preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || '';@@ -440,10 +469,11 @@ function buildCompactReport({origin,a11yReport,visualVerdict,- })}${buildVisualDeltas({verdict: visualVerdict})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict})}`;+ })}${buildVisualDeltas({verdict: visualVerdict, frames: visualFrames})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict})}`;}module.exports = {+ visualShots,SUITES,VISUAL_FRAME_LIMIT,baseJobName,diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjsindex 60b13cf5c5..636d9c0365 100644--- a/.github/scripts/lib/pr-status.test.mjs+++ b/.github/scripts/lib/pr-status.test.mjs@@ -247,6 +247,36 @@ describe('buildVisualDeltas', () => {expect(section).toContain("3 more are in the run's visual report.");});+ it('shows thumbnails for the frames publish-frames.mjs published, and only those', () => {+ const section = buildVisualDeltas({+ verdict: {+ status: 'changed',+ changes: [change('b', 50), change('a', 1)],+ added: ['core-dialog--default__neutral-light'],+ },+ frames: {+ imageBase: 'https://raw.example/frames',+ published: [+ 'before/b.png',+ 'after/b.png',+ 'diff/b.png',+ 'after/core-dialog--default__neutral-light.png',+ ],+ },+ });+ expect(section).toContain('| Component · story | Before | After | Diff |');+ expect(section).toContain(+ '<img src="https://raw.example/frames/before/b.png" height="120" alt="before frame">',+ );+ expect(section).toContain(+ '| Card · a (neutral, light) | not published | not published | not published |',+ );+ // An added shot has an after frame only.+ expect(section).toContain(+ '| — | <img src="https://raw.example/frames/after/core-dialog--default__neutral-light.png" height="120" alt="after frame"> | — |',+ );+ });+it('is absent without changes, or when the visual run failed', () => {expect(buildVisualDeltas({verdict: {status: 'ok', changes: []}})).toBe('');expect(buildVisualDeltas({verdict: null})).toBe('');diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.jsdeleted file mode 100644index 54501059c4..0000000000--- a/.github/scripts/lib/visual-format.js+++ /dev/null@@ -1,128 +0,0 @@-// Copyright (c) Meta Platforms, Inc. and affiliates.--/**- * Shared visual-gate formatting for PR enrichment.- * Used by generate-pr-comment.js and test-pr-enrichment.js.- */--const {inline, num, safeUrl} = require('./report-text');--/**- * Build the visual-regression section of the PR comment.- *- * The section has to survive four states, and the awkward one is `skipped`:- * a check that quietly vanishes on the PRs most likely to break something- * visually is worse than no check, so a skip states its reason explicitly.- *- * @param {object|null} verdict - the gate's verdict.json, or null when the job did not run- * @param {string} [reportUrl] - immutable published report for this run- * @param {string} [imageUrl] - immutable raw-image base for inline evidence- * @returns {string} markdown- */-function buildVisualSection(verdict, reportUrl, imageUrl) {- if (!verdict) return '';-- // Verdict fields are report data: render them as literal inline text (and- // counts as numbers), never as markup — see report-text.js.- const safeReportUrl = reportUrl ? safeUrl(reportUrl) : null;- const link = safeReportUrl- ? ` <a href="${safeReportUrl}" target="_blank" rel="noopener noreferrer">View the report</a>`- : '';- const reportBase = safeReportUrl- ? `${safeReportUrl.replace(/\/+$/, '')}/`- : null;- // Inline images use raw branch URLs so a delayed Pages deployment cannot- // break evidence that is already present on gh-pages.- const imageBase = safeUrl(imageUrl)- ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/`- : reportBase;-- if (verdict.status === 'skipped') {- return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}${link}\n\n`;- }- if (verdict.status === 'failed') {- return `### Visual Regression\n\n**Status:** ${num(verdict.counts?.failed)} shot(s) could not be captured.${link}\n\n`;- }- const added = verdict.added ?? [];- const removed = verdict.removed ?? [];- if (!verdict.changes || verdict.changes.length === 0) {- if (added.length > 0 || removed.length > 0) {- const frames = imageBase- ? [- ...added.map(key => ({key, kind: 'after'})),- ...removed.map(key => ({key, kind: 'before'})),- ]- .slice(0, 3)- .map(({key, kind}) => {- const safeKey = encodeURIComponent(String(key));- const label =- kind === 'after' ? 'Added — After' : 'Removed — Before';- return `<p><b>${label}</b><br><img src="${imageBase}${kind}/${safeKey}.png" width="300" alt="${label} visual regression frame"></p>`;- })- .join('\n')- : '';- return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}\n\n${frames}\n\n`;- }- const compared = num(verdict.counts?.total);- return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).\n\n`;- }-- const rows = verdict.changes- .slice(0, 20)- .map(- change =>- `| ${inline(change.component || change.title)} | ${inline(change.name)} | ${inline(change.theme)} | ${inline(change.mode)} | ${num(change.diffPixels).toLocaleString()} |`,- )- .join('\n');-- // Put the evidence where the decision is made. A link to an artifact ZIP is- // not a visual review: it makes the reviewer download, unzip, and match file- // names before they can answer whether the after is correct. The static- // report is published beside the PR preview, and its first three deltas- // render directly in the comment; the full report carries the rest + wipe UI.- const evidence = imageBase- ? verdict.changes- .slice(0, 3)- .map(change => {- const key = encodeURIComponent(String(change.key ?? ''));- if (!key) return '';- const label = `${inline(change.component || change.title)} — ${inline(change.name)} — ${inline(change.theme)} ${inline(change.mode)}`;- return `-<details open>-<summary>${label}</summary>-<table>-<tr><th>Before</th><th>After</th><th>Diff</th></tr>-<tr>-<td><img src="${imageBase}before/${key}.png" width="300" alt="Before visual regression frame"></td>-<td><img src="${imageBase}after/${key}.png" width="300" alt="After visual regression frame"></td>-<td><img src="${imageBase}diff/${key}.png" width="300" alt="Pixel difference frame"></td>-</tr>-</table>-</details>`;- })- .filter(Boolean)- .join('\n')- : '';- const more =- verdict.changes.length > 20- ? `\n\n_and ${verdict.changes.length - 20} more._`- : '';-- return `### Visual Regression--**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link}--A change here is a question, not a failure: check whether the *after* is the-picture you intended. Record that review on the PR. Baseline maintenance is an-explicit dispatch of CI; this report never rewrites the baseline or adds a release gate.--| component | story | theme | mode | pixels |-|---|---|---|---|---|-${rows}${more}--${evidence}--`;-}--module.exports = {buildVisualSection};diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjsdeleted file mode 100644index d5ba7f1719..0000000000--- a/.github/scripts/lib/visual-format.test.mjs+++ /dev/null@@ -1,340 +0,0 @@-// Copyright (c) Meta Platforms, Inc. and affiliates.--import {describe, expect, it} from 'vitest';-import {createRequire} from 'node:module';--const {buildVisualSection} = createRequire(import.meta.url)(- './visual-format.js',-);--const verdict = (overrides = {}) => ({- status: 'pass',- counts: {- total: 16,- unchanged: 16,- changed: 0,- added: 0,- removed: 0,- failed: 0,- },- changes: [],- ...overrides,-});--describe('buildVisualSection', () => {- it('renders nothing when the job did not run', () => {- expect(buildVisualSection(null)).toBe('');- });-- it('says so plainly when nothing moved', () => {- expect(buildVisualSection(verdict())).toContain(- 'No visual change across 16 compared shot(s)',- );- });-- it('shows added-only evidence instead of claiming there was no visual change', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 20,- unchanged: 14,- changed: 0,- added: 1,- removed: 0,- failed: 0,- },- added: ['new-shot'],- removed: [],- }),- 'https://example.com/report/',- 'https://example.com/images/',- );- expect(section).toContain('1 added · 0 removed');- expect(section).toContain('View the report');- expect(section).toContain('https://example.com/images/after/new-shot.png');- expect(section).not.toContain('No visual change');- });-- it('shows the before image for a removed-only bundle', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 0,- unchanged: 0,- changed: 0,- added: 0,- removed: 1,- failed: 0,- },- added: [],- removed: ['old-shot'],- }),- 'https://example.com/report/',- 'https://example.com/images/',- );- expect(section).toContain('0 added · 1 removed');- expect(section).toContain('https://example.com/images/before/old-shot.png');- });-- it('states the reason for a skip, so a broad PR does not look like a pass', () => {- const section = buildVisualSection(- verdict({- status: 'skipped',- reason: '900 shots exceeds the 240-shot budget',- }),- 'https://example.com/report/',- );- expect(section).toContain('Skipped');- expect(section).toContain('900 shots exceeds the 240-shot budget');- expect(section).toContain('View the report');- expect(section).not.toContain('/accept-visual');- });-- it('renders verdict strings as one-line literal text', () => {- const section = buildVisualSection(- verdict({- status: 'skipped',- reason: 'first line\n## not a heading, just <em>text</em>',- }),- );- expect(section).toContain(- 'first line ## not a heading, just <em>text</em>',- );- expect(section).not.toContain('\n## not a heading');- });-- it('keeps a change row a single table row whatever the field contents', () => {- const section = buildVisualSection(- verdict({- counts: {- total: 2,- unchanged: 1,- changed: 1,- added: 0,- removed: 0,- failed: 0,- },- changes: [- {- component: 'Cell | with pipes',- name: 'story\nnewline',- theme: 'y2k',- mode: 'light',- diffPixels: '1234',- },- ],- }),- );- expect(section).toContain(- '| Cell \\| with pipes | story newline | y2k | light | 1,234 |',- );- });-- it('drops a report link that is not a plain https URL', () => {- const good = buildVisualSection(- verdict({status: 'failed', counts: {total: 1, failed: 1}}),- 'https://example.com/report',- );- expect(good).toContain('href="https://example.com/report"');- const bad = buildVisualSection(- verdict({status: 'failed', counts: {total: 1, failed: 1}}),- 'report.html" onmouseover="x',- );- expect(bad).not.toContain('View the report');- });-- it('lists each changed shot with the theme and mode it changed in', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 16,- unchanged: 14,- changed: 2,- added: 0,- removed: 0,- failed: 0,- },- changes: [- {- key: 'a',- component: 'Button',- name: 'Primary',- theme: 'y2k',- mode: 'light',- diffPixels: 1126,- },- {- key: 'b',- component: 'Button',- name: 'Primary',- theme: 'y2k',- mode: 'dark',- diffPixels: 1401,- },- ],- }),- );- expect(section).toContain('2 of 16 shot(s) changed');- expect(section).toContain('| Button | Primary | y2k | light | 1,126 |');- });-- it('frames a change as a question rather than a failure', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 2,- unchanged: 1,- changed: 1,- added: 0,- removed: 0,- failed: 0,- },- changes: [- {- key: 'a',- component: 'B',- name: 'S',- theme: 't',- mode: 'light',- diffPixels: 5,- },- ],- }),- );- expect(section).toMatch(/question, not a failure/);- });-- it('caps the table and says how many were left out', () => {- const changes = Array.from({length: 25}, (_, index) => ({- key: `k${index}`,- component: 'C',- name: 'S',- theme: 't',- mode: 'light',- diffPixels: index,- }));- const section = buildVisualSection(- verdict({status: 'changed', counts: {total: 25, changed: 25}, changes}),- );- expect(section).toContain('and 5 more');- });-- it('embeds before, after, and diff images for changed shots', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 2,- unchanged: 1,- changed: 1,- added: 0,- removed: 0,- failed: 0,- },- context: {runId: '123456', runAttempt: '2'},- changes: [- {- key: 'core-button--primary__y2k-light',- component: 'Button',- name: 'Primary',- theme: 'y2k',- mode: 'light',- diffPixels: 12,- },- ],- }),- 'https://evidence.example/pr/123/visual/head/run/',- 'https://evidence.example/pr/123/visual/head/run/',- );- expect(section).toContain(- 'https://evidence.example/pr/123/visual/head/run/before/core-button--primary__y2k-light.png',- );- expect(section).toContain(- 'https://evidence.example/pr/123/visual/head/run/after/core-button--primary__y2k-light.png',- );- expect(section).toContain(- 'https://evidence.example/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png',- );- expect(section).toContain('<th>Before</th><th>After</th><th>Diff</th>');- expect(section).toContain('Record that review on the PR');- expect(section).not.toContain('/accept-visual');- });-- it('does not offer acceptance before immutable evidence is published', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {- total: 2,- unchanged: 1,- changed: 1,- added: 0,- removed: 0,- failed: 0,- },- context: {runId: '123456', runAttempt: '2'},- changes: [- {- key: 'a',- component: 'Button',- name: 'Primary',- theme: 'neutral',- mode: 'light',- diffPixels: 12,- },- ],- }),- );- expect(section).not.toContain('/accept-visual');- expect(section).not.toContain('repository maintainer');- });-- it('embeds at most three deltas so the PR comment stays readable', () => {- const changes = Array.from({length: 5}, (_, index) => ({- key: `shot-${index}`,- component: 'C',- name: 'S',- theme: 't',- mode: 'light',- diffPixels: 1,- }));- const section = buildVisualSection(- verdict({status: 'changed', counts: {total: 5, changed: 5}, changes}),- 'https://example.com/visual/',- );- expect((section.match(/<details open>/g) ?? []).length).toBe(3);- expect(section).not.toContain('/before/shot-3.png');- });-- it('reports a capture failure distinctly from a change', () => {- const section = buildVisualSection(- verdict({status: 'failed', counts: {total: 4, failed: 2, changed: 0}}),- );- expect(section).toContain('2 shot(s) could not be captured');- });-- it('links the report when one was published', () => {- const section = buildVisualSection(- verdict({- status: 'changed',- counts: {total: 1, changed: 1},- changes: [- {- key: 'a',- component: 'B',- name: 'S',- theme: 't',- mode: 'light',- diffPixels: 1,- },- ],- }),- 'https://example.com/report/',- );- expect(section).toContain('https://example.com/report/');- });-});diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjsnew file mode 100644index 0000000000..123ea201d2--- /dev/null+++ b/.github/scripts/visual-gate/publish-frames.mjs@@ -0,0 +1,227 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Publishes the changed visual shots' frames for the PR report.+ * @input `--input <dir>` (publish-pr-report.mjs's output: verdict.json and+ * re-encoded `before|after|diff/<key>.png`), `--pr <number>`,+ * `--head-sha <sha>`, `--repository <owner/repo>`, `--limit <n>`, and a+ * token in GITHUB_TOKEN.+ * @output Commits up to `limit` (at most 12) changed shots' frames, PNG or+ * WebP of at most 500 KB each by their own bytes, largest change first,+ * to the pull request's `assets/pr-<number>` branch (the path PR screenshots+ * already use) under `visual/<head sha>/`, marking a branch it creates with+ * MARKER so only such branches are pruned, and writes `image_base` (their+ * raw URL prefix) and `published` (each frame's `<kind>/<key>.png`) to+ * GITHUB_OUTPUT. Writes nothing when no shot changed.+ * @position The PR Comment workflow's frames job, after the visual report is+ * prepared; the comment job renders the thumbnails. It writes only that+ * branch, and only frames the trusted publisher re-encoded.+ */++import fs from 'node:fs';+import {createRequire} from 'node:module';+import path from 'node:path';+import process from 'node:process';+import {fileURLToPath} from 'node:url';++// The report's own shot order and limit (lib/pr-status.js).+const {visualShots, VISUAL_FRAME_LIMIT} = createRequire(import.meta.url)(+ '../lib/pr-status.js',+);+const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/;+/** Shots published per head, whatever `--limit` asks for: the report's rows. */+export const MAX_SHOTS = VISUAL_FRAME_LIMIT;+export const MAX_FRAME_BYTES = 500 * 1024;++/** A PNG or WebP no larger than MAX_FRAME_BYTES, by its own bytes. */+export function isPublishableImage(bytes) {+ if (+ !Buffer.isBuffer(bytes) ||+ bytes.length === 0 ||+ bytes.length > MAX_FRAME_BYTES+ )+ return false;+ const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);+ if (bytes.subarray(0, 8).equals(png)) return true;+ return (+ bytes.subarray(0, 4).toString('latin1') === 'RIFF' &&+ bytes.subarray(8, 12).toString('latin1') === 'WEBP'+ );+}++/**+ * The file that marks an assets branch as created by this script. The daily+ * branch pruner deletes only marked branches, once their PR closes; a branch+ * someone pushed by hand never gets the marker.+ */+export const MARKER = '.pr-report-frames';++/** The only ref this script writes: PR #<pr>'s assets branch. */+export function assetsBranch(pr) {+ if (!Number.isSafeInteger(pr) || pr <= 0) throw new Error('not a PR number');+ return `assets/pr-${pr}`;+}++/**+ * The frames to publish: the report's rows (visualShots), the first `limit`+ * of them and at most MAX_SHOTS, and only shots whose key is a safe name.+ */+export function framesToPublish(verdict, limit) {+ return visualShots(verdict)+ .slice(0, Math.min(Number(limit) || 0, MAX_SHOTS))+ .filter(shot => KEY.test(String(shot.key ?? '')))+ .map(shot => ({key: shot.key, kinds: shot.kinds}));+}++async function api(request, method, route, body) {+ const response = await request(method, route, body);+ if (response.status === 404) return null;+ if (response.status >= 300)+ throw new Error(`${method} ${route} failed: ${response.status}`);+ return response.json();+}++/**+ * One commit on `assets/pr-<pr>` holding the frames, on top of the branch when+ * it exists. Returns the raw URL prefix of the published frames and each+ * published frame's `<kind>/<key>.png`, or null when none was published.+ */+export async function publishFrames({+ input,+ pr,+ headSha,+ repository,+ limit,+ request,+}) {+ const verdict = JSON.parse(+ fs.readFileSync(path.join(input, 'verdict.json'), 'utf8'),+ );+ const files = [];+ for (const shot of framesToPublish(verdict, limit)) {+ for (const kind of shot.kinds) {+ const file = path.join(input, kind, `${shot.key}.png`);+ if (!fs.existsSync(file)) continue;+ const bytes = fs.readFileSync(file);+ if (!isPublishableImage(bytes)) continue;+ files.push({+ frame: `${kind}/${shot.key}.png`,+ path: `visual/${headSha}/${kind}/${shot.key}.png`,+ bytes,+ });+ }+ }+ if (files.length === 0) return null;+ if (!/^[0-9a-f]{40}$/.test(String(headSha)))+ throw new Error('not a head SHA');+ const branch = assetsBranch(pr);+ const ref = await api(+ request,+ 'GET',+ `/repos/${repository}/git/ref/heads/${branch}`,+ );+ const parent = ref?.object?.sha ?? null;+ const parentCommit = parent+ ? await api(request, 'GET', `/repos/${repository}/git/commits/${parent}`)+ : null;+ const tree = [];+ if (!parent) {+ const marker = await api(+ request,+ 'POST',+ `/repos/${repository}/git/blobs`,+ {+ content: Buffer.from(+ 'Created by the PR report; pruned when the PR closes.\n',+ ).toString('base64'),+ encoding: 'base64',+ },+ );+ tree.push({path: MARKER, mode: '100644', type: 'blob', sha: marker.sha});+ }+ for (const entry of files) {+ const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, {+ content: entry.bytes.toString('base64'),+ encoding: 'base64',+ });+ tree.push({path: entry.path, mode: '100644', type: 'blob', sha: blob.sha});+ }+ const newTree = await api(request, 'POST', `/repos/${repository}/git/trees`, {+ ...(parentCommit ? {base_tree: parentCommit.tree.sha} : {}),+ tree,+ });+ const commit = await api(+ request,+ 'POST',+ `/repos/${repository}/git/commits`,+ {+ message: `Visual frames for #${pr} at ${headSha.slice(0, 7)}`,+ tree: newTree.sha,+ parents: parent ? [parent] : [],+ },+ );+ if (parent)+ await api(+ request,+ 'PATCH',+ `/repos/${repository}/git/refs/heads/${branch}`,+ {+ sha: commit.sha,+ },+ );+ else+ await api(request, 'POST', `/repos/${repository}/git/refs`, {+ ref: `refs/heads/${branch}`,+ sha: commit.sha,+ });+ return {+ imageBase: `https://raw.githubusercontent.com/${repository}/${branch}/visual/${headSha}/`,+ published: files.map(entry => entry.frame),+ };+}++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const flag = name => {+ const index = process.argv.indexOf(`--${name}`);+ return index === -1 ? null : process.argv[index + 1];+ };+ const pr = Number(flag('pr'));+ const headSha = String(flag('head-sha') ?? '');+ const repository = String(flag('repository') ?? '');+ if (!Number.isSafeInteger(pr) || pr <= 0)+ throw new Error('--pr must be a PR number');+ if (!/^[0-9a-f]{40}$/.test(headSha))+ throw new Error('--head-sha must be a full SHA');+ if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository))+ throw new Error('--repository must be owner/repo');+ const token = process.env.GITHUB_TOKEN;+ const request = (method, route, body) =>+ fetch(`https://api.github.com${route}`, {+ method,+ headers: {+ authorization: `Bearer ${token}`,+ accept: 'application/vnd.github+json',+ 'content-type': 'application/json',+ },+ body: body ? JSON.stringify(body) : undefined,+ });+ const result = await publishFrames({+ input: path.resolve(flag('input')),+ pr,+ headSha,+ repository,+ limit: Number(flag('limit') ?? 12),+ request,+ });+ if (result && process.env.GITHUB_OUTPUT)+ fs.appendFileSync(+ process.env.GITHUB_OUTPUT,+ `image_base=${result.imageBase}\npublished=${JSON.stringify(result.published)}\n`,+ );+ console.log(+ result+ ? `Published ${result.published.length} frames under ${result.imageBase}`+ : 'No changed shots to publish.',+ );+}diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjsnew file mode 100644index 0000000000..4605b8a8ef--- /dev/null+++ b/.github/scripts/visual-gate/publish-frames.test.mjs@@ -0,0 +1,243 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file publish-frames.test.mjs+ * Pins which visual frames the PR report publishes (the largest changes+ * first, then added and removed shots, at most `limit`, safe keys only) and+ * how: one commit on `assets/pr-<n>`, on top of the branch when it exists.+ */++import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {afterEach, describe, expect, it} from 'vitest';+import {+ MAX_FRAME_BYTES,+ MAX_SHOTS,+ MARKER,+ assetsBranch,+ framesToPublish,+ isPublishableImage,+ publishFrames,+} from './publish-frames.mjs';++const PNG = Buffer.concat([+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),+ Buffer.from('rest'),+]);++const HEAD = 'a'.repeat(40);+const roots = [];+afterEach(() => {+ for (const root of roots.splice(0))+ fs.rmSync(root, {recursive: true, force: true});+});++function input(verdict) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'frames-'));+ roots.push(root);+ fs.writeFileSync(path.join(root, 'verdict.json'), JSON.stringify(verdict));+ for (const kind of ['before', 'after', 'diff']) {+ fs.mkdirSync(path.join(root, kind));+ for (const key of ['a', 'b', 'c', 'added'])+ fs.writeFileSync(path.join(root, kind, `${key}.png`), PNG);+ }+ return root;+}++function fakeGitHub({branchExists}) {+ const calls = [];+ const request = async (method, route, body) => {+ calls.push({method, route, body});+ const json = data => ({status: 200, json: async () => data});+ if (method === 'GET' && route.includes('/git/ref/'))+ return branchExists+ ? json({object: {sha: 'parent'}})+ : {status: 404, json: async () => ({})};+ if (method === 'GET' && route.includes('/git/commits/'))+ return json({tree: {sha: 'parent-tree'}});+ if (route.endsWith('/git/blobs'))+ return json({sha: `blob-${calls.length}`});+ if (route.endsWith('/git/trees')) return json({sha: 'tree'});+ if (route.endsWith('/git/commits')) return json({sha: 'commit'});+ return json({});+ };+ return {calls, request};+}++describe('framesToPublish', () => {+ it('takes the largest changes first, then added and removed shots, up to the limit, and never an unsafe key', () => {+ const verdict = {+ changes: [+ {key: 'a', diffPixels: 1},+ {key: 'b', diffPixels: 9},+ {key: '../escape', diffPixels: 99},+ ],+ added: ['added'],+ removed: ['gone'],+ };+ // The unsafe key keeps its report row (shown as not published).+ expect(framesToPublish(verdict, 3).map(shot => shot.key)).toEqual([+ 'b',+ 'a',+ ]);+ expect(framesToPublish(verdict, 5).map(shot => shot.key)).toEqual([+ 'b',+ 'a',+ 'added',+ 'gone',+ ]);+ expect(framesToPublish({...verdict, status: 'skipped'}, 3)).toEqual([]);+ });+});++describe('the same shots as the report', () => {+ it("publishes exactly the report's rows, in its order", async () => {+ const {createRequire} = await import('node:module');+ const {visualShots, VISUAL_FRAME_LIMIT} = createRequire(import.meta.url)(+ '../lib/pr-status.js',+ );+ const verdict = {+ status: 'changed',+ changes: Array.from({length: 20}, (_, i) => ({+ key: `s${i}`,+ diffPixels: i,+ })),+ added: ['new-shot'],+ };+ expect(framesToPublish(verdict, 100).map(shot => shot.key)).toEqual(+ visualShots(verdict)+ .slice(0, VISUAL_FRAME_LIMIT)+ .map(shot => shot.key),+ );+ });+});++describe('image and ref limits', () => {+ it('accepts only PNG or WebP bytes no larger than 500 KB', () => {+ expect(isPublishableImage(PNG)).toBe(true);+ const webp = Buffer.concat([+ Buffer.from('RIFF'),+ Buffer.alloc(4),+ Buffer.from('WEBPVP8 '),+ ]);+ expect(isPublishableImage(webp)).toBe(true);+ expect(isPublishableImage(Buffer.from('<svg onload=alert(1)>'))).toBe(+ false,+ );+ expect(isPublishableImage(Buffer.from('GIF89a'))).toBe(false);+ expect(+ isPublishableImage(Buffer.concat([PNG, Buffer.alloc(MAX_FRAME_BYTES)])),+ ).toBe(false);+ });++ it('publishes at most 12 shots, whatever the limit asks', () => {+ const changes = Array.from({length: 30}, (_, i) => ({+ key: `s${i}`,+ diffPixels: i,+ }));+ expect(framesToPublish({changes}, 100)).toHaveLength(MAX_SHOTS);+ expect(MAX_SHOTS).toBe(12);+ });++ it('writes only assets/pr-<n>', () => {+ expect(assetsBranch(42)).toBe('assets/pr-42');+ for (const bad of [0, -1, 1.5, NaN, '42/../main'])+ expect(() => assetsBranch(bad)).toThrow();+ });+});++describe('publishFrames', () => {+ it("commits the frames on top of the PR's existing assets branch", async () => {+ const {calls, request} = fakeGitHub({branchExists: true});+ const base = await publishFrames({+ input: input({changes: [{key: 'b', diffPixels: 2}]}),+ pr: 9171,+ headSha: HEAD,+ repository: 'meta/astryx',+ limit: 12,+ request,+ });+ expect(base).toEqual({+ imageBase: `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${HEAD}/`,+ published: ['before/b.png', 'after/b.png', 'diff/b.png'],+ });+ const tree = calls.find(call => call.route.endsWith('/git/trees')).body;+ expect(tree.base_tree).toBe('parent-tree');+ expect(tree.tree.map(entry => entry.path)).not.toContain(MARKER);+ expect(tree.tree.map(entry => entry.path)).toEqual([+ `visual/${HEAD}/before/b.png`,+ `visual/${HEAD}/after/b.png`,+ `visual/${HEAD}/diff/b.png`,+ ]);+ expect(+ calls.find(call => call.route.endsWith('/git/commits')).body.parents,+ ).toEqual(['parent']);+ expect(calls.at(-1)).toMatchObject({+ method: 'PATCH',+ route: '/repos/meta/astryx/git/refs/heads/assets/pr-9171',+ body: {sha: 'commit'},+ });+ });++ it('skips a frame that is not an image, and touches no other ref', async () => {+ const root = input({changes: [{key: 'a', diffPixels: 1}]});+ fs.writeFileSync(path.join(root, 'diff', 'a.png'), '<html>');+ const {calls, request} = fakeGitHub({branchExists: true});+ await publishFrames({+ input: root,+ pr: 7,+ headSha: HEAD,+ repository: 'meta/astryx',+ limit: 12,+ request,+ });+ const tree = calls.find(call => call.route.endsWith('/git/trees')).body;+ expect(tree.tree.map(entry => entry.path)).toEqual([+ `visual/${HEAD}/before/a.png`,+ `visual/${HEAD}/after/a.png`,+ ]);+ const refs = calls.filter(call => call.route.includes('/git/ref'));+ expect(+ refs.every(+ call =>+ call.route.includes('assets/pr-7') ||+ call.route.endsWith('/git/refs'),+ ),+ ).toBe(true);+ });++ it('creates the branch when it does not exist, and writes nothing without changes', async () => {+ const created = fakeGitHub({branchExists: false});+ await publishFrames({+ input: input({added: ['added']}),+ pr: 5,+ headSha: HEAD,+ repository: 'meta/astryx',+ limit: 12,+ request: created.request,+ });+ // Only a branch this script creates carries the pruning marker.+ const createdTree = created.calls.find(call =>+ call.route.endsWith('/git/trees'),+ ).body;+ expect(createdTree.tree.map(entry => entry.path)).toContain(MARKER);+ expect(created.calls.at(-1)).toMatchObject({+ method: 'POST',+ route: '/repos/meta/astryx/git/refs',+ body: {ref: 'refs/heads/assets/pr-5'},+ });+ const none = fakeGitHub({branchExists: true});+ await expect(+ publishFrames({+ input: input({changes: []}),+ pr: 5,+ headSha: HEAD,+ repository: 'meta/astryx',+ limit: 12,+ request: none.request,+ }),+ ).resolves.toBeNull();+ expect(none.calls).toEqual([]);+ });+});diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjsindex 35ca0688e2..a83fb51cd4 100644--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs@@ -63,7 +63,10 @@ describe('PR report and deployment workflow contracts', () => {);expect(value.jobs[job].concurrency['cancel-in-progress']).toBe(false);}- expect(value.jobs.comment.needs).toBe('resolve');+ expect(value.jobs.comment.needs).toEqual(['resolve', 'frames']);+ expect(value.jobs.frames.concurrency.group).toBe(+ 'pr-frames-${{ needs.resolve.outputs.pr_number }}',+ );// The comment job reads this head's preview state once; no job runs on a// deployment event, and none waits for one.expect(value.jobs.comment.permissions.deployments).toBe('read');@@ -78,7 +81,7 @@ describe('PR report and deployment workflow contracts', () => {const value = workflow('pr-comment.yml');const reconcile = value.slice(value.indexOf(' spec-only-reconcile:'),- value.indexOf(' comment:'),+ value.indexOf(' frames:'),);expect(reconcile).toContain('reconcilePrComment');expect(reconcile).toContain('createIfMissing: false');diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.ymlindex d16bcf1534..34ed7d92f6 100644--- a/.github/workflows/pr-comment.yml+++ b/.github/workflows/pr-comment.yml@@ -183,8 +183,96 @@ jobs:'The current change has no visual or preview-producing scope. Visual checks are not applicable; preview links are not shown.',});- comment:+ # The changed visual frames behind the report's thumbnails, on the pull+ # request's assets branch. The only job here that writes repository+ # contents: it runs trusted code from main (never pull-request code), keeps+ # no credentials in its checkout, and publish-frames.mjs writes only+ # assets/pr-<n>, and only validated PNG/WebP frames.+ frames:needs: resolve+ concurrency:+ group: pr-frames-${{ needs.resolve.outputs.pr_number }}+ cancel-in-progress: false+ if: >-+ always() &&+ github.event.action == 'completed' &&+ needs.resolve.outputs.valid == 'true' &&+ needs.resolve.outputs.spec_only != 'true' &&+ needs.resolve.outputs.tooling_only != 'true'+ runs-on: ubuntu-24.04-arm+ permissions:+ actions: read+ contents: write+ outputs:+ image_base: ${{ steps.frames.outputs.image_base }}+ published: ${{ steps.frames.outputs.published }}+ failed: >-+ ${{ steps.checkout.outcome == 'failure' ||+ steps.setup.outcome == 'failure' ||+ steps.visual.outcome == 'failure' ||+ steps.frames.outcome == 'failure' }}+ steps:+ - name: Checkout trusted default-branch code+ id: checkout+ continue-on-error: true+ uses: actions/checkout@v7+ with:+ ref: main+ persist-credentials: false++ - name: Setup Node and pnpm+ id: setup+ continue-on-error: true+ if: steps.checkout.outcome == 'success'+ uses: ./.github/actions/setup++ - name: Download canonical visual report+ id: visual-download+ if: steps.setup.outcome == 'success'+ uses: actions/download-artifact@v8+ with:+ name: visual-pr-report+ path: visual-pr-report+ run-id: ${{ needs.resolve.outputs.run_id }}+ github-token: ${{ secrets.GITHUB_TOKEN }}+ continue-on-error: true++ # The same re-encoding the comment job applies: only JSON and+ # re-encoded PNGs from the exact CI attempt.+ - name: Prepare canonical visual report+ id: visual+ continue-on-error: true+ if: steps.visual-download.outcome == 'success'+ env:+ HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}+ RUN_ID: ${{ needs.resolve.outputs.run_id }}+ RUN_ATTEMPT: ${{ needs.resolve.outputs.run_attempt }}+ run: |+ node .github/scripts/visual-gate/publish-pr-report.mjs \+ --input visual-pr-report \+ --output trusted-visual \+ --head-sha "$HEAD_SHA" \+ --run-id "$RUN_ID" \+ --run-attempt "$RUN_ATTEMPT"++ - name: Publish changed visual frames+ id: frames+ continue-on-error: true+ if: steps.visual.outcome == 'success'+ env:+ GITHUB_TOKEN: ${{ github.token }}+ PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}+ HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}+ run: |+ node .github/scripts/visual-gate/publish-frames.mjs \+ --input trusted-visual \+ --pr "$PR_NUMBER" \+ --head-sha "$HEAD_SHA" \+ --repository "$GITHUB_REPOSITORY" \+ --limit 12++ comment:+ needs: [resolve, frames]concurrency:group: pr-report-${{ needs.resolve.outputs.pr_number }}cancel-in-progress: false@@ -385,6 +473,8 @@ jobs:RUN_ID: ${{ steps.identity.outputs.run_id }}RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}SOURCE_CONCLUSION: ${{ steps.identity.outputs.source_conclusion }}+ VISUAL_IMAGE_BASE: ${{ needs.frames.outputs.image_base }}+ VISUAL_PUBLISHED: ${{ needs.frames.outputs.published }}with:retries: 3script: |@@ -408,6 +498,10 @@ jobs:sourceRunAttempt: Number(process.env.RUN_ATTEMPT),sourceConclusion: process.env.SOURCE_CONCLUSION,},+ visualFrames: {+ imageBase: process.env.VISUAL_IMAGE_BASE,+ published: process.env.VISUAL_PUBLISHED,+ },});# The CI-completion lane runs on workflow_run, so GitHub attaches its jobs to@@ -416,7 +510,7 @@ jobs:# continues on error, and this job reports their outcome as the PR Comment# status on the pull request's own head.report:- needs: [resolve, spec-only-reconcile, comment]+ needs: [resolve, spec-only-reconcile, frames, comment]if: >-always() &&github.event_name == 'workflow_run' &&@@ -437,6 +531,7 @@ jobs:FAILED: >-${{ needs.resolve.outputs.failed == 'true' ||needs.spec-only-reconcile.outputs.failed == 'true' ||+ needs.frames.outputs.failed == 'true' ||needs.comment.outputs.failed == 'true' ||contains(needs.*.result, 'failure') ||contains(needs.*.result, 'cancelled') }}diff --git a/scripts/prune-branches.sh b/scripts/prune-branches.shindex f5ad24b161..b20ebdc594 100755--- a/scripts/prune-branches.sh+++ b/scripts/prune-branches.sh@@ -2,6 +2,7 @@# Copyright (c) Meta Platforms, Inc. and affiliates.# scripts/prune-branches.sh — Delete remote branches with merged/closed PRs+# (an `assets/pr-<n>` branch the PR report created goes with PR #<n>)## Usage:# ./scripts/prune-branches.sh # dry run (default)@@ -90,7 +91,15 @@ stale_cutoff="$3"repo="$4"safe_name=$(echo "$branch" | tr '/' '_')-state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error")+# An `assets/pr-<n>` branch the PR report created (it carries the+# `.pr-report-frames` marker) goes with PR #<n>. Every other branch, including+# assets branches pushed by hand, keeps the rule below.+if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]] && git cat-file -e "origin/$branch:.pr-report-frames" 2>/dev/null; then+ state=$(gh pr view "${BASH_REMATCH[1]}" --repo "$repo" --json state --jq '.state' 2>/dev/null || echo "error")+ [ "$state" = "error" ] && state="OPEN"+else+ state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error")+fiif [ "$state" = "MERGED" ] || [ "$state" = "CLOSED" ]; thenecho "$branch" > "$tmp_dir/delete/$safe_name"diff --git a/scripts/prune-branches.test.mjs b/scripts/prune-branches.test.mjsnew file mode 100644index 0000000000..d621e20a95--- /dev/null+++ b/scripts/prune-branches.test.mjs@@ -0,0 +1,79 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file prune-branches.test.mjs+ * Pins that the daily branch pruner deletes an `assets/pr-<n>` branch the PR+ * report created (it carries `.pr-report-frames`) once PR #<n> is merged or+ * closed, keeps it while the PR is open, and does not delete an assets branch+ * without the marker, such as screenshots pushed by hand, for its PR closing.+ */++import {execFileSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterEach, describe, expect, it} from 'vitest';++const SCRIPT = path.join(+ path.dirname(fileURLToPath(import.meta.url)),+ 'prune-branches.sh',+);+const roots = [];+afterEach(() => {+ for (const root of roots.splice(0))+ fs.rmSync(root, {recursive: true, force: true});+});++function setup(prStates) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'prune-'));+ roots.push(root);+ const git = (cwd, ...args) =>+ execFileSync('git', args, {cwd, encoding: 'utf8'});+ const origin = path.join(root, 'origin.git');+ const work = path.join(root, 'work');+ git(root, 'init', '-q', '--bare', origin);+ git(root, 'init', '-q', '-b', 'main', work);+ git(work, 'config', 'user.email', '[email protected]');+ git(work, 'config', 'user.name', 'Test');+ git(work, 'commit', '-q', '--allow-empty', '-m', 'base');+ git(work, 'remote', 'add', 'origin', origin);+ // assets/pr-3 is pushed by hand: no marker.+ git(work, 'push', '-q', 'origin', 'main', 'main:assets/pr-3');+ fs.writeFileSync(path.join(work, '.pr-report-frames'), 'marker\n');+ git(work, 'add', '.pr-report-frames');+ git(work, 'commit', '-q', '-m', 'frames');+ git(work, 'push', '-q', 'origin', 'HEAD:assets/pr-1', 'HEAD:assets/pr-2');+ // A stand-in for gh: `pr view <n>` answers from prStates; `pr list` finds no PR.+ const bin = path.join(root, 'bin');+ fs.mkdirSync(bin);+ fs.writeFileSync(+ path.join(bin, 'gh'),+ `#!/usr/bin/env bash\nif [ "$1 $2" = "pr view" ]; then\n case "$3" in\n${Object.entries(+ prStates,+ )+ .map(([n, state]) => ` ${n}) echo ${state} ;;`)+ .join('\n')}\n esac\nelse\n echo none\nfi\n`,+ {mode: 0o755},+ );+ return {work, bin};+}++describe('prune-branches.sh', () => {+ it("deletes a closed PR's marked assets branch, keeps an open PR's, and never an unmarked one", () => {+ const {work, bin} = setup({1: 'MERGED', 2: 'OPEN', 3: 'CLOSED'});+ const out = execFileSync('bash', [SCRIPT], {+ cwd: work,+ encoding: 'utf8',+ env: {...process.env, PATH: `${bin}:${process.env.PATH}`},+ }).replace(/\u001b\[[0-9;]*m/g, '');+ const willDelete = out.slice(+ out.indexOf('Merged/closed PR (will delete):'),+ );+ expect(willDelete).toContain('assets/pr-1');+ expect(willDelete).not.toContain('assets/pr-2');+ // assets/pr-3 has no marker: its PR is closed, yet it is not deleted.+ expect(out).not.toMatch(/^\s+assets\/pr-3$/m);+ expect(out).toContain('0 branches (no PR');+ });+});