Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): keep accepted stable shots in the release gate (#5608)

Cindy Zhanga83c559a51fb
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index e41d7dd21a..4e4232d742 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -28,8 +28,24 @@ import * as path from 'node:path';
import {fileURLToPath} from 'node:url';
import {capture, scout} from './lib/capture.mjs';
-import {analyzeTargeting, buildVerdict, compareCaptures} from './lib/compare.mjs';
-import {buildPlan, readStoryIndex, storiesInPackages} from './lib/plan.mjs';
+import {
+ analyzeTargeting,
+ buildVerdict,
+ compareCaptures,
+ compareReleaseCaptures,
+ validateReleasePlan,
+} from './lib/compare.mjs';
+import {
+ accountBaseline,
+ buildPlan,
+ createReleasePlan,
+ readStoryIndex,
+ readThemeCatalog,
+ storiesInPackages,
+ summarizeBaselineAccounting,
+ withBaselineCoverage,
+ withThemeMetadata,
+} from './lib/plan.mjs';
import {READ_TARGETS, emptyAccumulator, fold} from './lib/probe-reach.mjs';
import {AXES, PROBE_TOKENS, READ_AXES} from './lib/probe-axes.mjs';
import {renderReport} from './lib/report.mjs';
@@ -39,6 +55,7 @@ import {loadConfig, loadThemeOverrides, loadThemingTargets} from './lib/sources.
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
const EXIT = {clean: 0, crashed: 1, changed: 2};
+const RELEASE_TIERS = ['surface', 'theme-matrix'];
const argv = process.argv.slice(2);
const command = argv[0];
@@ -55,10 +72,14 @@ const captureIdentity = () => ({
});
const config = loadConfig(REPO_ROOT);
+const releaseMode = command === 'release';
+const themeCatalog = readThemeCatalog(REPO_ROOT);
const storybookDir = path.resolve(flag('storybook-dir') ?? 'apps/storybook/dist');
const baselineDir = path.resolve(flag('baseline') ?? '.visual-baseline');
let outDir = path.resolve(flag('out') ?? '.visual-run');
-const tiers = (flag('tiers') ?? config.tiers.join(',')).split(',').filter(Boolean);
+const tiers = releaseMode
+ ? RELEASE_TIERS
+ : (flag('tiers') ?? config.tiers.join(',')).split(',').filter(Boolean);
const sample = flag('sample') ? Number(flag('sample')) : null;
/** Restrict the plan to story ids containing any of these — for debugging a shot, never for a gate run. */
const only = (flag('only') ?? '').split(',').filter(Boolean);
@@ -66,23 +87,27 @@ const only = (flag('only') ?? '').split(',').filter(Boolean);
const components = (flag('components') ?? '').split(',').filter(Boolean);
/** For the `theme-matrix` tier: only these shipped themes changed. */
const matrixThemes = (flag('themes') ?? '').split(',').filter(Boolean);
-/**
- * Above this many shots the run declines instead of capturing.
- *
- * A sweeping PR — a token change, a shared hook, a rename across the system —
- * would put hundreds of diffs in front of a reviewer who has no way to judge
- * them one by one, and the honest answer is that a per-PR check is the wrong
- * instrument for that change: the daily gate reviews it against the whole
- * baseline instead. Declining loudly beats either timing out or dumping a
- * report nobody can read.
- */
const maxShots = flag('max-shots') ? Number(flag('max-shots')) : Infinity;
-/** Storybook package groups that own the stable visual baseline. */
const storyPackages = (flag('story-packages') ?? config.stableStoryPackages.join(','))
.split(',')
.filter(Boolean);
/** A trusted, exact shot list used only to recapture an accepted merged result. */
const planFile = flag('plan-file');
+if (
+ releaseMode &&
+ (flag('tiers') ||
+ flag('story-packages') ||
+ flag('observations') ||
+ flag('max-shots') ||
+ has('no-scout') ||
+ planFile ||
+ components.length ||
+ matrixThemes.length ||
+ only.length ||
+ sample)
+) {
+ throw new Error('The stable release plan is internal and cannot be scoped by CLI flags.');
+}
function readExactPlan(file) {
const shots = JSON.parse(fs.readFileSync(path.resolve(file), 'utf8'));
@@ -126,16 +151,29 @@ function storiesToScout(stories, targets, themeOverrides) {
return stories.filter(story => components.has(story.component)).map(story => story.id);
}
-/** @returns {Promise<import('./lib/plan.mjs').Shot[]>} */
+/** @returns {Promise<{shots: import('./lib/plan.mjs').Shot[], stories: ReturnType<typeof readStoryIndex>}>} */
async function plan() {
- if (planFile) return readExactPlan(planFile);
- const [targets, themeOverrides] = await Promise.all([
+ if (planFile) {
+ return {
+ shots: withThemeMetadata(readExactPlan(planFile), themeCatalog),
+ stories: [],
+ };
+ }
+ const [targets, allThemeOverrides] = await Promise.all([
loadThemingTargets(REPO_ROOT),
loadThemeOverrides(REPO_ROOT, config.probeTheme),
]);
+ const themeOverrides = releaseMode
+ ? Object.fromEntries(
+ Object.entries(allThemeOverrides).filter(
+ ([theme]) => themeCatalog[theme]?.stableVisual === true,
+ ),
+ )
+ : allThemeOverrides;
const indexedStories = readStoryIndex(
storybookDir,
Object.keys(config.excludeStories),
+ REPO_ROOT,
);
const stories = storiesInPackages(indexedStories, storyPackages);
@@ -159,7 +197,7 @@ async function plan() {
}
}
- const shots = buildPlan({
+ let shots = buildPlan({
stories,
targets,
themeOverrides,
@@ -170,17 +208,37 @@ async function plan() {
matrixThemes,
probeTheme: config.probeTheme,
});
+ let baselineAccount = null;
+ if (releaseMode) {
+ const {manifest} = readBaseline(baselineDir);
+ baselineAccount = accountBaseline(manifest, indexedStories, themeCatalog, REPO_ROOT);
+ shots = withBaselineCoverage(shots, {
+ stories,
+ baselineManifest: baselineAccount.manifest,
+ themes: themeCatalog,
+ });
+ }
+ shots = withThemeMetadata(shots, themeCatalog);
// A sample is for trying the rig out, never for a gate run: it is taken
// evenly across the plan so it spans components rather than the first few.
const filtered = only.length
? shots.filter(shot => only.some(fragment => shot.storyId.includes(fragment)))
: shots;
- if (!sample || sample >= filtered.length) return filtered;
+ if (!sample || sample >= filtered.length) {
+ return {shots: filtered, stories: indexedStories, baselineAccount};
+ }
const step = filtered.length / sample;
- return Array.from({length: sample}, (_, index) => filtered[Math.floor(index * step)]);
+ return {
+ shots: Array.from(
+ {length: sample},
+ (_, index) => filtered[Math.floor(index * step)],
+ ),
+ stories: indexedStories,
+ baselineAccount,
+ };
}
-async function runCapture(shots) {
+async function runCapture(shots, releasePlan = null) {
fs.rmSync(outDir, {recursive: true, force: true});
fs.mkdirSync(outDir, {recursive: true});
let last = 0;
@@ -198,6 +256,18 @@ async function runCapture(shots) {
}
},
});
+ const planByKey = new Map(shots.map(shot => [shot.key, shot]));
+ for (const [key, captured] of Object.entries(manifest.shots)) {
+ const planned = planByKey.get(key);
+ Object.assign(captured, {
+ packageName: planned.packageName,
+ packageNames: planned.packageNames,
+ stableVisual: planned.stableVisual,
+ themePackageName: planned.themePackageName,
+ stableThemeVisual: planned.stableThemeVisual,
+ membershipSource: 'accepted-capture',
+ });
+ }
manifest.context = {
// `sha` is the tested synthetic merge tree on pull_request runs. Keep the
// contributor's head and the base separately: acceptance binds to the head
@@ -207,6 +277,7 @@ async function runCapture(shots) {
headSha: process.env.ASTRYX_PR_HEAD_SHA ?? null,
baseSha: process.env.ASTRYX_PR_BASE_SHA ?? null,
ref: process.env.GITHUB_REF ?? null,
+ ...(releasePlan ? {releasePlan} : {}),
};
fs.writeFileSync(
path.join(outDir, 'manifest.json'),
@@ -230,7 +301,19 @@ function stageReportImages({reportDir, keys, currentDir, baselinePath}) {
}
async function check() {
- const shots = await plan();
+ const {shots, stories, baselineAccount} = await plan();
+ const releasePlan = releaseMode ? createReleasePlan(shots) : null;
+ const baselineAccounting = releaseMode
+ ? summarizeBaselineAccounting(baselineAccount, shots)
+ : null;
+ const ineligible = releaseMode
+ ? shots.filter(shot => !shot.stableVisual || !shot.stableThemeVisual)
+ : [];
+ if (ineligible.length) {
+ throw new Error(
+ `Canonical stable release plan contains ${ineligible.length} ineligible shot(s): ${ineligible.slice(0, 5).map(shot => shot.key).join(', ')}`,
+ );
+ }
// Over budget: say so in the verdict rather than capturing. The report and
// the PR comment both render this, so a skipped check is visible as a
@@ -247,8 +330,8 @@ async function check() {
baseSha: process.env.ASTRYX_PR_BASE_SHA ?? null,
ref: process.env.GITHUB_REF ?? null,
tiers,
- scoped: components.length > 0,
components,
+ ...(releasePlan ? {releasePlan} : {}),
},
counts: {total: shots.length, unchanged: 0, changed: 0, added: 0, removed: 0, failed: 0},
components,
@@ -272,17 +355,27 @@ async function check() {
}
process.stderr.write(`Visual gate: ${shots.length} shots (${tiers.join(', ')})\n`);
- const {manifest, failures} = await runCapture(shots);
+ const {manifest, failures: captureFailures} = await runCapture(shots, releasePlan);
+ let failures = [
+ ...captureFailures,
+ ...(baselineAccounting?.unclassifiedKeys ?? []).map(key => ({
+ key,
+ error: 'Baseline membership is unclassified.',
+ })),
+ ];
- const {manifest: baselineManifest, exists} = readBaseline(baselineDir);
+ const {manifest: rawBaseline, exists} = readBaseline(baselineDir);
+ const baselineManifest = releaseMode ? baselineAccount.manifest : rawBaseline;
const blocker = exists ? incomparable(baselineManifest, manifest) : null;
if (blocker) {
- process.stderr.write(`::error::Baseline is not comparable — ${blocker}\n`);
- return EXIT.crashed;
+ failures.push({key: 'baseline', error: blocker});
}
const reportDir = path.join(outDir, 'report');
- const comparison = await compareCaptures({
+ const compare = releaseMode ? compareReleaseCaptures : compareCaptures;
+ let comparison;
+ try {
+ comparison = await compare({
baselineDir: path.join(baselineDir, 'shots'),
currentDir: path.join(outDir, 'shots'),
baselineManifest,
@@ -290,8 +383,24 @@ async function check() {
diffDir: path.join(reportDir, 'diff'),
threshold: config.threshold,
maxDiffPixels: config.maxDiffPixels,
- scoped: components.length > 0,
- });
+ failures,
+ });
+ } catch (error) {
+ if (!releaseMode) throw error;
+ failures = [
+ ...failures,
+ {key: 'release-plan', error: String(error?.message ?? error)},
+ ];
+ comparison = await compareCaptures({
+ baselineDir: path.join(baselineDir, 'shots'),
+ currentDir: path.join(outDir, 'shots'),
+ baselineManifest,
+ currentManifest: manifest,
+ diffDir: path.join(reportDir, 'diff'),
+ threshold: config.threshold,
+ maxDiffPixels: config.maxDiffPixels,
+ });
+ }
const [targets, themeOverrides] = await Promise.all([
loadThemingTargets(REPO_ROOT),
@@ -309,7 +418,13 @@ async function check() {
baselineManifest,
targeting,
failures,
- context: {...manifest.context, tiers, baselineExists: exists, scoped: components.length > 0, components},
+ context: {
+ ...manifest.context,
+ tiers,
+ baselineExists: exists,
+ components,
+ ...(baselineAccounting ? {baselineAccounting} : {}),
+ },
});
stageReportImages({
@@ -357,6 +472,20 @@ function summarize(verdict, baselineExists) {
`| ${verdict.counts.total} | ${verdict.counts.changed} | ${verdict.counts.added} | ${verdict.counts.removed} | ${verdict.counts.failed} |`,
'',
);
+ const accounting = verdict.context?.baselineAccounting;
+ if (accounting) {
+ lines.push(
+ `Baseline accounting: ${accounting.plannedCurrentStable} planned stable · ${accounting.intentionallyExcluded} intentionally excluded · ${accounting.preservedLegacy} preserved legacy · ${accounting.unclassified} unclassified`,
+ '',
+ );
+ if (accounting.unclassifiedKeys) {
+ lines.push(
+ 'Unclassified baseline keys:',
+ ...accounting.unclassifiedKeys.map(key => `- \`${key}\``),
+ '',
+ );
+ }
+ }
if (verdict.changes.length > 0) {
lines.push('### Changed', '', '| shot | theme | mode | diff px | why it is in the plan |', '|---|---|---|---|---|');
for (const change of verdict.changes.slice(0, 40)) {
@@ -384,7 +513,7 @@ function summarize(verdict, baselineExists) {
async function main() {
switch (command) {
case 'plan': {
- const shots = await plan();
+ const {shots} = await plan();
if (has('json')) {
process.stdout.write(`${JSON.stringify(shots, null, 2)}\n`);
return EXIT.clean;
@@ -403,12 +532,13 @@ async function main() {
return EXIT.clean;
}
case 'capture': {
- const shots = await plan();
+ const {shots} = await plan();
const {failures} = await runCapture(shots);
process.stdout.write(`Captured ${shots.length - failures.length}/${shots.length} into ${outDir}\n`);
return failures.length > 0 ? EXIT.crashed : EXIT.clean;
}
case 'check':
+ case 'release':
return check();
case 'reach': {
// The assertion a pixel diff cannot make: did each target's override
@@ -416,7 +546,14 @@ async function main() {
// per-selector colour is the fingerprint, so this is an equality test.
const {chromium} = await import('playwright');
const {serveDirectory} = await import('./lib/capture.mjs');
- const stories = readStoryIndex(storybookDir, Object.keys(config.excludeStories));
+ const stories = storiesInPackages(
+ readStoryIndex(
+ storybookDir,
+ Object.keys(config.excludeStories),
+ REPO_ROOT,
+ ),
+ ['*'],
+ );
const subject = only.length
? stories.filter(story => only.some(f => story.storyId ?? story.id.includes(f)))
: stories;
@@ -523,7 +660,7 @@ async function main() {
// A gate that cries wolf gets ignored, so the exclusion list is
// evidence, not guesswork: capture the same build twice and name every
// shot that could not reproduce itself.
- const shots = await plan();
+ const {shots} = await plan();
const passes = Number(flag('passes') ?? 2);
/** @type {Map<string, Set<string>>} */
const seen = new Map();
@@ -566,19 +703,43 @@ async function main() {
? JSON.parse(fs.readFileSync(verdictPath, 'utf8'))
: null;
const requested = flag('keys');
- const keys =
+ const named =
!requested || requested === 'all'
? Object.keys(currentManifest.shots)
: requested.split(',').filter(Boolean);
+ if (new Set(named).size !== named.length) throw new Error('accept repeats a shot key.');
+ const removed = new Set(verdict?.removed ?? []);
+ const prune = has('prune') ? named.filter(key => removed.has(key)) : [];
+ const keys = named.filter(key => currentManifest.shots[key]);
+ const unknown = named.filter(key => !currentManifest.shots[key] && !removed.has(key));
+ if (unknown.length) throw new Error(`accept names unknown shot(s): ${unknown.join(', ')}`);
+ if (prune.length) {
+ if (verdict?.status === 'failed') {
+ throw new Error('Refusing to prune from a failed visual verdict.');
+ }
+ const plan = validateReleasePlan(
+ currentManifest.context?.releasePlan,
+ currentManifest,
+ );
+ if (JSON.stringify(verdict?.context?.releasePlan) !== JSON.stringify(plan)) {
+ throw new Error('Refusing to prune without the capture’s canonical release plan.');
+ }
+ }
+ const baselineManifest = {
+ ...currentManifest,
+ context: currentManifest.context
+ ? {...currentManifest.context, releasePlan: undefined}
+ : null,
+ };
const result = accept({
baselineDir,
captureDir: outDir,
- currentManifest,
+ currentManifest: baselineManifest,
keys,
reason: flag('reason') ?? '',
actor: flag('actor') ?? process.env.GITHUB_ACTOR ?? process.env.USER ?? 'unknown',
runId: process.env.GITHUB_RUN_ID ?? null,
- prune: has('prune') ? (verdict?.removed ?? []) : [],
+ prune,
});
process.stdout.write(
`Promoted ${result.promoted.length} shot(s), pruned ${result.pruned.length}, into ${baselineDir}\n`,
@@ -587,7 +748,7 @@ async function main() {
}
default:
process.stderr.write(
- 'Usage: gate.mjs <flaky|plan|capture|check|reach|accept> [--storybook-dir dir] [--baseline dir] [--out dir] [--tiers a,b] [--sample n]\n',
+ 'Usage: gate.mjs <flaky|plan|capture|check|release|reach|accept> [--storybook-dir dir] [--baseline dir] [--out dir] [--tiers a,b] [--sample n]\n',
);
return EXIT.crashed;
}
diff --git a/.github/scripts/visual-gate/lib/compare.mjs b/.github/scripts/visual-gate/lib/compare.mjs
index 3add9004f1..d2b5473742 100644
--- a/.github/scripts/visual-gate/lib/compare.mjs
+++ b/.github/scripts/visual-gate/lib/compare.mjs
@@ -15,13 +15,46 @@
* than a red run someone re-runs until it goes away.
*
* A new shot has no before, so it cannot regress: it is reported as `added`
- * and adopted the first time it is seen. A shot whose story disappeared is
- * `removed` and pruned. Neither blocks.
+ * and adopted the first time it is seen. Ordinary comparisons never infer removals. Only a capture carrying a validated
+ * canonical stable-release plan may report baseline keys it did not capture.
*/
+import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
+const RELEASE_LANE = 'stable-release';
+
+export function validateReleasePlan(plan, currentManifest, failures = []) {
+ const keys = plan?.keys;
+ if (
+ plan?.version !== 1 ||
+ plan?.lane !== RELEASE_LANE ||
+ plan?.authority !== 'report-removals' ||
+ !Array.isArray(keys)
+ ) {
+ throw new Error('Capture has no canonical stable-release plan.');
+ }
+ const sorted = [...keys].sort();
+ if (new Set(sorted).size !== sorted.length || sorted.some((key, index) => key !== keys[index])) {
+ throw new Error('Canonical stable-release keys must be sorted and unique.');
+ }
+ const digest = crypto.createHash('sha256').update(JSON.stringify(keys)).digest('hex');
+ if (digest !== plan.digest) throw new Error('Canonical stable-release plan digest does not match its keys.');
+ const captured = Object.keys(currentManifest.shots ?? {}).sort();
+ if (captured.length !== keys.length || captured.some((key, index) => key !== keys[index])) {
+ throw new Error('Capture does not exactly cover the canonical stable-release plan.');
+ }
+ if (failures.length > 0) throw new Error(`Stable release capture has ${failures.length} failure(s).`);
+ if (captured.some(key => {
+ const shot = currentManifest.shots[key];
+ return shot.stableVisual !== true || shot.stableThemeVisual !== true;
+ })) {
+ throw new Error('Stable release capture contains an ineligible story or theme.');
+ }
+ return plan;
+}
+
/**
* @typedef {object} Change
* @property {string} key
@@ -55,7 +88,6 @@ function pad(png, width, height, PNG) {
* @param {string} options.diffDir - where diff PNGs are written
* @param {number} options.threshold - pixelmatch per-pixel colour threshold
* @param {number} options.maxDiffPixels - pixels allowed to differ before a shot counts as changed
- * @param {boolean} [options.scoped] - the plan covers only part of the baseline
* @returns {Promise<{changes: Change[], added: string[], removed: string[], unchanged: string[]}>}
*/
export async function compareCaptures({
@@ -66,7 +98,6 @@ export async function compareCaptures({
diffDir,
threshold,
maxDiffPixels,
- scoped = false,
}) {
const {PNG} = await import('pngjs');
const pixelmatch = (await import('pixelmatch')).default;
@@ -125,15 +156,25 @@ export async function compareCaptures({
});
}
- // A scoped run (a PR shooting only the components it touched) deliberately
- // captures a fraction of the baseline, so "in the baseline, not in this run"
- // means out of scope — not removed. Reporting it as removal would put a
- // five-hundred-shot deletion on every PR.
- const removed = scoped
- ? []
- : [...baselineKeys].filter(key => !currentManifest.shots[key]);
changes.sort((a, b) => b.diffPixels - a.diffPixels);
- return {changes, added, removed, unchanged};
+ return {changes, added, removed: [], unchanged};
+}
+
+export async function compareReleaseCaptures(options) {
+ if (options.failures?.length) {
+ return compareCaptures(options);
+ }
+ validateReleasePlan(
+ options.currentManifest.context?.releasePlan,
+ options.currentManifest,
+ options.failures,
+ );
+ const comparison = await compareCaptures(options);
+ const current = new Set(Object.keys(options.currentManifest.shots ?? {}));
+ return {
+ ...comparison,
+ removed: Object.keys(options.baselineManifest.shots ?? {}).filter(key => !current.has(key)),
+ };
}
/**
diff --git a/.github/scripts/visual-gate/lib/compare.test.mjs b/.github/scripts/visual-gate/lib/compare.test.mjs
index 5a7cfc2ca4..620ce6b4f7 100644
--- a/.github/scripts/visual-gate/lib/compare.test.mjs
+++ b/.github/scripts/visual-gate/lib/compare.test.mjs
@@ -1,13 +1,16 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
+import {execFileSync} from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
+import {fileURLToPath} from 'node:url';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
import {PNG} from 'pngjs';
-import {analyzeTargeting, buildVerdict, compareCaptures} from './compare.mjs';
+import {analyzeTargeting, buildVerdict, compareCaptures, compareReleaseCaptures} from './compare.mjs';
+import {createReleasePlan} from './plan.mjs';
/** A solid rectangle, so a change is unambiguous. */
function png(width, height, [r, g, b]) {
@@ -40,7 +43,7 @@ function manifest(shots) {
shots: Object.fromEntries(
Object.entries(shots).map(([key, value]) => [
key,
- {sha256: value.sha256, width: 10, height: 10, theme: 'neutral', mode: 'light', reasons: ['surface'], ...value},
+ {sha256: value.sha256, width: 10, height: 10, storyId: key, packageName: '@astryxdesign/core', packageNames: ['@astryxdesign/core'], stableVisual: true, theme: 'neutral', themePackageName: '@astryxdesign/theme-neutral', stableThemeVisual: true, mode: 'light', reasons: ['surface'], ...value},
]),
),
};
@@ -87,12 +90,105 @@ describe('compareCaptures', () => {
expect(result).toMatchObject({added: ['new'], changes: []});
});
- it('reports a baseline shot whose story is gone as removed', async () => {
- write('baseline', 'gone', png(10, 10, [0, 255, 0]));
+ it('never reports removals from an ordinary comparison', async () => {
const result = await compare(manifest({gone: {sha256: 'x'}}), manifest({}));
+ expect(result.removed).toEqual([]);
+ });
+
+ it('reports a true removal only from an exact canonical release capture', async () => {
+ const baseline = manifest({kept: {sha256: 'same'}, gone: {sha256: 'old'}});
+ const current = manifest({kept: {sha256: 'same'}});
+ current.context = {releasePlan: createReleasePlan([{key: 'kept'}])};
+ const result = await compareReleaseCaptures({
+ baselineDir: path.join(root, 'baseline'),
+ currentDir: path.join(root, 'current'),
+ baselineManifest: baseline,
+ currentManifest: current,
+ diffDir: path.join(root, 'diff'),
+ threshold: 0.1,
+ maxDiffPixels: 0,
+ failures: [],
+ });
expect(result.removed).toEqual(['gone']);
});
+ it.each([
+ ['subset', ['a']],
+ ['superset', ['a', 'b', 'c']],
+ ])('rejects a canonical %s that differs from captured keys', async (_name, keys) => {
+ const current = manifest({a: {sha256: 'a'}, b: {sha256: 'b'}});
+ current.context = {releasePlan: createReleasePlan(keys.map(key => ({key})))};
+ await expect(compareReleaseCaptures({
+ baselineDir: path.join(root, 'baseline'),
+ currentDir: path.join(root, 'current'),
+ baselineManifest: manifest({}),
+ currentManifest: current,
+ diffDir: path.join(root, 'diff'),
+ threshold: 0.1,
+ maxDiffPixels: 0,
+ failures: [],
+ })).rejects.toThrow(/exactly cover/);
+ });
+
+ it('returns an incomplete comparison for capture failures without removals', async () => {
+ const current = manifest({a: {sha256: 'a'}});
+ current.context = {releasePlan: createReleasePlan([{key: 'a'}])};
+ const failures = [{key: 'a', error: 'capture failed'}];
+ const comparison = await compareReleaseCaptures({
+ baselineDir: path.join(root, 'baseline'),
+ currentDir: path.join(root, 'current'),
+ baselineManifest: manifest({gone: {sha256: 'old'}}),
+ currentManifest: current,
+ diffDir: path.join(root, 'diff'),
+ threshold: 0.1,
+ maxDiffPixels: 0,
+ failures,
+ });
+ expect(comparison.removed).toEqual([]);
+ expect(
+ buildVerdict({
+ comparison,
+ currentManifest: current,
+ baselineManifest: manifest({}),
+ targeting: {},
+ failures,
+ context: current.context,
+ }).status,
+ ).toBe('failed');
+ });
+
+ it('prunes only enumerated reviewed removal keys', () => {
+ const gate = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../gate.mjs');
+ const baselineDir = path.join(root, 'accepted-baseline');
+ const captureDir = path.join(root, 'release-capture');
+ fs.mkdirSync(path.join(baselineDir, 'shots'), {recursive: true});
+ fs.mkdirSync(path.join(captureDir, 'shots'), {recursive: true});
+ fs.writeFileSync(path.join(baselineDir, 'shots/b.png'), png(1, 1, [1, 1, 1]));
+ fs.writeFileSync(path.join(baselineDir, 'shots/c.png'), png(1, 1, [2, 2, 2]));
+ fs.writeFileSync(path.join(baselineDir, 'manifest.json'), JSON.stringify(manifest({b: {sha256: 'b'}, c: {sha256: 'c'}})));
+ const current = manifest({a: {sha256: 'a'}});
+ current.context = {releasePlan: createReleasePlan([{key: 'a'}])};
+ fs.writeFileSync(path.join(captureDir, 'manifest.json'), JSON.stringify(current));
+ fs.writeFileSync(path.join(captureDir, 'verdict.json'), JSON.stringify({
+ status: 'changed',
+ removed: ['b', 'c'],
+ context: current.context,
+ }));
+ execFileSync(process.execPath, [gate, 'accept', '--baseline', baselineDir, '--out', captureDir, '--keys', 'b', '--prune', '--reason', 'Reviewed deletion'], {encoding: 'utf8'});
+ const accepted = JSON.parse(fs.readFileSync(path.join(baselineDir, 'manifest.json'), 'utf8'));
+ expect(accepted.shots.b).toBeUndefined();
+ expect(accepted.shots.c).toBeDefined();
+ expect(accepted.context?.releasePlan).toBeUndefined();
+ expect(fs.existsSync(path.join(baselineDir, 'shots/c.png'))).toBe(true);
+ });
+
+ it('rejects caller-controlled canonical release scope', () => {
+ const gate = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../gate.mjs');
+ expect(() =>
+ execFileSync(process.execPath, [gate, 'release', '--tiers', 'surface'], {encoding: 'utf8'}),
+ ).toThrow(/stable release plan is internal/);
+ });
+
it('ranks the biggest change first', async () => {
write('baseline', 'small', png(10, 10, [255, 0, 0]));
write('current', 'small', png(10, 10, [255, 0, 10]));
diff --git a/.github/scripts/visual-gate/lib/plan.mjs b/.github/scripts/visual-gate/lib/plan.mjs
index 82b8df2eb5..6e32ddfdd1 100644
--- a/.github/scripts/visual-gate/lib/plan.mjs
+++ b/.github/scripts/visual-gate/lib/plan.mjs
@@ -33,6 +33,7 @@
* with the representative-story filter removed.
*/
+import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -45,6 +46,238 @@ const REPRESENTATIVE_NAMES = ['Default', 'Basic', 'Primary', 'Overview', 'Exampl
/** A story carrying this tag is never captured (see visual-gate.config.json for the reasoned list). */
export const SKIP_TAG = 'no-visual';
+
+/** Workspace package manifests are the only package eligibility source. */
+export function readPackageCatalog(repoRoot) {
+ const files = [];
+ for (const parent of [path.join(repoRoot, 'packages'), path.join(repoRoot, 'packages/themes')]) {
+ if (!fs.existsSync(parent)) continue;
+ for (const entry of fs.readdirSync(parent, {withFileTypes: true})) {
+ if (!entry.isDirectory()) continue;
+ const file = path.join(parent, entry.name, 'package.json');
+ if (fs.existsSync(file)) files.push(file);
+ }
+ }
+ return new Map(files.map(file => {
+ const manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
+ return [manifest.name, manifest];
+ }));
+}
+
+export function packageFromComponentPath(componentPath) {
+ if (!componentPath) return null;
+ const normalized = componentPath.replaceAll('\\', '/');
+ const names = new Set();
+ for (const match of normalized.matchAll(/(?:^|\/)node_modules\/(@[^/]+\/[^/]+|[^/]+)(?=\/|$)/g)) {
+ names.add(match[1]);
+ }
+ for (const match of normalized.matchAll(/(?:^|\/)packages\/(themes\/[^/]+|[^/]+)(?=\/|$)/g)) {
+ const [group, leaf] = match[1].split('/');
+ names.add(leaf ? `@astryxdesign/theme-${leaf}` : `@astryxdesign/${group}`);
+ }
+ const bare = normalized.match(/^(@[^/]+\/[^/]+)(?:\/.*)?$/);
+ if (bare) names.add(bare[1]);
+ if (names.size > 1) throw new Error(`Ambiguous Storybook componentPath: ${componentPath}`);
+ if (names.size === 0) throw new Error(`Unsupported Storybook componentPath: ${componentPath}`);
+ return [...names][0];
+}
+
+function eligible(manifest) {
+ return manifest.private !== true && manifest.astryx?.canaryOnly !== true;
+}
+
+function titlePackage(title, catalog, candidates = null) {
+ const parts = String(title ?? '').split('/').filter(Boolean);
+ const wanted = [];
+ if (parts[0]) wanted.push(`@astryxdesign/${parts[0].toLowerCase()}`);
+ if (parts.at(-1)?.endsWith(' Theme')) {
+ const slug = parts.at(-1).slice(0, -6).trim().toLowerCase().replaceAll(/[^a-z0-9]+/g, '-');
+ wanted.unshift(`@astryxdesign/theme-${slug}`);
+ }
+ return wanted.find(name => catalog.has(name) && (!candidates || candidates.includes(name))) ?? null;
+}
+
+function storyPackageNames(entry, storybookDir, repoRoot, catalog) {
+ const fromComponent = packageFromComponentPath(entry.componentPath);
+ let names = fromComponent ? [fromComponent] : [];
+ if (!fromComponent) {
+ const relative = entry.importPath?.replace(/^\.\//, '');
+ const source = relative && path.resolve(path.dirname(storybookDir), relative);
+ if (!source || !fs.existsSync(source)) {
+ throw new Error(`Story ${entry.id} has no resolvable package metadata.`);
+ }
+ const text = fs.readFileSync(source, 'utf8');
+ names = [...new Set([...text.matchAll(/(?:from\s+|import\s*)['"](@astryxdesign\/[^/'"]+)/g)].map(match => match[1]))].sort();
+ }
+ const titleOwner = titlePackage(entry.title, catalog, names);
+ const themeOwner = titlePackage(entry.title, catalog);
+ const owner =
+ fromComponent ??
+ (themeOwner?.startsWith('@astryxdesign/theme-') ? themeOwner : null) ??
+ titleOwner ??
+ (names.length === 1 ? names[0] : null);
+ if (!owner) throw new Error(`Story ${entry.id} has ambiguous package ownership: ${names.join(', ')}.`);
+ for (const name of new Set([...names, owner])) {
+ if (!catalog.has(name)) throw new Error(`Story ${entry.id} names unknown package ${name}.`);
+ }
+ return {
+ packageNames: names.length ? names : [owner],
+ packageName: owner,
+ stableVisual: eligible(catalog.get(owner)),
+ };
+}
+
+export function readThemeCatalog(repoRoot) {
+ const catalog = readPackageCatalog(repoRoot);
+ const themes = {};
+ const parent = path.join(repoRoot, 'packages/themes');
+ if (!fs.existsSync(parent)) return themes;
+ for (const entry of fs.readdirSync(parent, {withFileTypes: true})) {
+ if (!entry.isDirectory()) continue;
+ const manifest = catalog.get(`@astryxdesign/theme-${entry.name}`);
+ if (!manifest) continue;
+ themes[entry.name] = {
+ packageName: manifest.name,
+ stableVisual: manifest.private !== true && manifest.astryx?.canaryOnly !== true,
+ };
+ }
+ return themes;
+}
+
+export function withThemeMetadata(shots, themes) {
+ return shots.map(shot => {
+ const theme = themes[shot.theme];
+ if (!theme) throw new Error(`Shot ${shot.key} names unknown theme ${shot.theme}.`);
+ return {...shot, themePackageName: theme.packageName, stableThemeVisual: theme.stableVisual};
+ });
+}
+
+function baselinePackage(shot, repoRoot, catalog) {
+ if (shot.packageName && catalog.has(shot.packageName)) {
+ return {packageName: shot.packageName, source: 'stored-package'};
+ }
+ const component = String(shot.component ?? '').trim();
+ if (component) {
+ const owners = [...catalog.keys()].filter(name => {
+ if (!name.startsWith('@astryxdesign/') || name.includes('/theme-')) return false;
+ const leaf = name.slice('@astryxdesign/'.length);
+ return fs.existsSync(path.join(repoRoot, 'packages', leaf, 'src', component));
+ });
+ if (owners.length === 1) return {packageName: owners[0], source: 'stored-component'};
+ }
+ const fromTitle = titlePackage(shot.title, catalog);
+ return fromTitle ? {packageName: fromTitle, source: 'stored-title'} : null;
+}
+
+export function accountBaseline(manifest, stories, themes, repoRoot) {
+ const catalog = readPackageCatalog(repoRoot);
+ const byId = new Map(stories.map(story => [story.id, story]));
+ const stable = {};
+ const categories = {
+ currentStable: [],
+ intentionallyExcluded: [],
+ preservedLegacy: [],
+ unclassified: [],
+ };
+ for (const [key, shot] of Object.entries(manifest.shots ?? {})) {
+ const currentStory = byId.get(shot.storyId);
+ const owner = shot.packageName && catalog.has(shot.packageName)
+ ? {
+ packageName: shot.packageName,
+ source: shot.membershipSource ?? 'stored-package',
+ }
+ : currentStory
+ ? {packageName: currentStory.packageName, source: 'current-story'}
+ : baselinePackage(shot, repoRoot, catalog);
+ const storedTheme = shot.themePackageName
+ ? catalog.get(shot.themePackageName)
+ : null;
+ const theme = storedTheme
+ ? {
+ packageName: storedTheme.name,
+ stableVisual: eligible(storedTheme),
+ }
+ : themes[shot.theme];
+ if (!owner || !catalog.has(owner.packageName) || !theme) {
+ categories.unclassified.push(key);
+ continue;
+ }
+ const state = eligible(catalog.get(owner.packageName)) && theme.stableVisual
+ ? 'stable'
+ : 'ineligible';
+ if (state === 'ineligible') {
+ categories.intentionallyExcluded.push(key);
+ continue;
+ }
+ const story = byId.get(shot.storyId);
+ const value = {
+ ...shot,
+ packageName: owner.packageName,
+ packageNames: story?.packageNames ?? [owner.packageName],
+ stableVisual: true,
+ themePackageName: theme.packageName,
+ stableThemeVisual: true,
+ membershipSource: owner.source,
+ };
+ stable[key] = value;
+ (story ? categories.currentStable : categories.preservedLegacy).push(key);
+ }
+ for (const values of Object.values(categories)) values.sort();
+ const total = Object.keys(manifest.shots ?? {}).length;
+ const classified = Object.values(categories).reduce((sum, values) => sum + values.length, 0);
+ if (classified !== total) throw new Error(`Baseline accounting overlap: ${classified} states for ${total} keys.`);
+ return {manifest: {...manifest, shots: stable}, categories, total};
+}
+
+export function summarizeBaselineAccounting(account, releaseShots) {
+ const planned = new Set(releaseShots.map(shot => shot.key));
+ const missing = account.categories.currentStable.filter(key => !planned.has(key));
+ const unclassified = [...new Set([...account.categories.unclassified, ...missing])].sort();
+ const plannedCurrentStable = account.categories.currentStable.length - missing.length;
+ const counts = {
+ total: account.total,
+ plannedCurrentStable,
+ intentionallyExcluded: account.categories.intentionallyExcluded.length,
+ preservedLegacy: account.categories.preservedLegacy.length,
+ unclassified: unclassified.length,
+ };
+ const sum = counts.plannedCurrentStable + counts.intentionallyExcluded + counts.preservedLegacy + counts.unclassified;
+ if (sum !== counts.total) throw new Error(`Baseline accounting overlap: ${sum} states for ${counts.total} keys.`);
+ return {...counts, ...(unclassified.length ? {unclassifiedKeys: unclassified} : {})};
+}
+
+export function stableBaseline(manifest, stories, themes, repoRoot) {
+ return accountBaseline(manifest, stories, themes, repoRoot).manifest;
+}
+
+export function withBaselineCoverage(plan, {stories, baselineManifest, themes}) {
+ const planned = new Map(plan.map(shot => [shot.key, shot]));
+ const byId = new Map(stories.map(story => [story.id, story]));
+ for (const [key, baseline] of Object.entries(baselineManifest.shots ?? {})) {
+ const story = byId.get(baseline.storyId);
+ if (!story || !themes[baseline.theme] || !MODES.includes(baseline.mode)) continue;
+ const shot = {...toShotBase(story), theme: baseline.theme, mode: baseline.mode};
+ if (shotKey(shot) !== key) continue;
+ const existing = planned.get(key);
+ planned.set(key, existing
+ ? {...existing, reasons: [...new Set([...existing.reasons, 'baseline'])]}
+ : {...shot, key, reasons: ['baseline']});
+ }
+ return [...planned.values()].sort((a, b) => a.key.localeCompare(b.key));
+}
+
+export function createReleasePlan(shots) {
+ const keys = shots.map(shot => shot.key).sort();
+ if (new Set(keys).size !== keys.length) throw new Error('Canonical release plan repeats a shot key.');
+ return {
+ version: 1,
+ lane: 'stable-release',
+ authority: 'report-removals',
+ keys,
+ digest: crypto.createHash('sha256').update(JSON.stringify(keys)).digest('hex'),
+ };
+}
+
/**
* @typedef {object} Shot
* @property {string} key - stable identity of the shot; also its file name
@@ -52,7 +285,12 @@ export const SKIP_TAG = 'no-visual';
* @property {string} title - Storybook title, for the report
* @property {string} name - story name, for the report
* @property {string} component - the component this story renders
+ * @property {string} packageName
+ * @property {string[]} packageNames
+ * @property {boolean} stableVisual
* @property {string} theme
+ * @property {string} themePackageName
+ * @property {boolean} stableThemeVisual
* @property {'light'|'dark'} mode
* @property {string[]} reasons - why the shot is in the plan
*/
@@ -70,7 +308,7 @@ export const SKIP_TAG = 'no-visual';
* @param {Iterable<string>} excluded - story ids (or `prefix*`) excluded by config
* @returns {Array<{id: string, title: string, name: string, component: string, tags: string[]}>}
*/
-export function readStoryIndex(storybookDir, excluded = []) {
+export function readStoryIndex(storybookDir, excluded = [], repoRoot) {
const exclusions = [...excluded];
const isExcluded = id =>
exclusions.some(rule =>
@@ -83,6 +321,7 @@ export function readStoryIndex(storybookDir, excluded = []) {
);
}
const index = JSON.parse(fs.readFileSync(indexPath, 'utf8'));
+ const catalog = readPackageCatalog(repoRoot);
return Object.values(index.entries ?? {})
.filter(entry => entry.type === 'story')
.filter(entry => !(entry.tags ?? []).includes(SKIP_TAG))
@@ -93,6 +332,7 @@ export function readStoryIndex(storybookDir, excluded = []) {
name: entry.name ?? '',
component: componentOf(entry),
tags: entry.tags ?? [],
+ ...storyPackageNames(entry, storybookDir, repoRoot, catalog),
}));
}
@@ -121,7 +361,8 @@ function componentOf(entry) {
*/
export function storiesInPackages(stories, packages) {
if (packages.includes('*')) return stories;
- return stories.filter(story => packages.includes(story.title.split('/')[0]));
+ const wanted = new Set(packages.map(name => name.startsWith('@') ? name : `@astryxdesign/${name.toLowerCase()}`));
+ return stories.filter(story => story.stableVisual && story.packageNames.some(name => wanted.has(name)));
}
/**
@@ -411,7 +652,15 @@ function chooseStories({candidates, fallback, key, selectors, observations}) {
/** @param {ReturnType<typeof readStoryIndex>[number]} story */
function toShotBase(story) {
- return {storyId: story.id, title: story.title, name: story.name, component: story.component};
+ return {
+ storyId: story.id,
+ title: story.title,
+ name: story.name,
+ component: story.component,
+ packageName: story.packageName,
+ packageNames: story.packageNames,
+ stableVisual: story.stableVisual,
+ };
}
/**
diff --git a/.github/scripts/visual-gate/lib/plan.test.mjs b/.github/scripts/visual-gate/lib/plan.test.mjs
index 77882933d8..63768d468a 100644
--- a/.github/scripts/visual-gate/lib/plan.test.mjs
+++ b/.github/scripts/visual-gate/lib/plan.test.mjs
@@ -7,18 +7,29 @@ import * as path from 'node:path';
import {describe, expect, it} from 'vitest';
import {
+ accountBaseline,
buildPlan,
+ createReleasePlan,
readStoryIndex,
+ readThemeCatalog,
representativeStories,
shotKey,
storiesInPackages,
+ summarizeBaselineAccounting,
uncoveredTargets,
} from './plan.mjs';
+const story = (value, packageName = '@astryxdesign/core', stableVisual = true) => ({
+ tags: [],
+ packageName,
+ packageNames: [packageName],
+ stableVisual,
+ ...value,
+});
const stories = [
- {id: 'core-button--primary', title: 'Core/Button', name: 'Primary', component: 'Button', tags: []},
- {id: 'core-button--default', title: 'Core/Button', name: 'Default', component: 'Button', tags: []},
- {id: 'core-badge--solid', title: 'Core/Badge', name: 'Solid', component: 'Badge', tags: []},
+ story({id: 'core-button--primary', title: 'Core/Button', name: 'Primary', component: 'Button'}),
+ story({id: 'core-button--default', title: 'Core/Button', name: 'Default', component: 'Button'}),
+ story({id: 'core-badge--solid', title: 'Core/Badge', name: 'Solid', component: 'Badge'}),
];
const targets = [
@@ -35,8 +46,8 @@ const themeOverrides = {
describe('storiesInPackages', () => {
const mixed = [
...stories,
- {id: 'lab-drawer--default', title: 'Lab/Drawer', name: 'Default', component: 'Drawer', tags: []},
- {id: 'charts-bar--default', title: 'Charts/Bar', name: 'Default', component: 'Bar', tags: []},
+ story({id: 'lab-drawer--default', title: 'Lab/Drawer', name: 'Default', component: 'Drawer'}, '@astryxdesign/lab', false),
+ story({id: 'charts-bar--default', title: 'Charts/Bar', name: 'Default', component: 'Bar'}, '@astryxdesign/charts', false),
];
it('keeps only the stable Storybook package groups', () => {
@@ -139,37 +150,195 @@ describe('shotKey', () => {
});
});
-describe('readStoryIndex exclusions', () => {
- const index = {
- entries: {
- a: {type: 'story', id: 'lab-stream--one', title: 'Lab/Stream', name: 'One', tags: []},
- b: {type: 'story', id: 'lab-stream--two', title: 'Lab/Stream', name: 'Two', tags: []},
- c: {type: 'story', id: 'core-button--default', title: 'Core/Button', name: 'Default', tags: []},
- d: {type: 'docs', id: 'core-button--docs', title: 'Core/Button', name: 'Docs', tags: []},
- e: {type: 'story', id: 'core-thing--skipped', title: 'Core/Thing', name: 'S', tags: ['no-visual']},
- },
- };
-
- const read = exclusions => {
- const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-index-'));
- fs.writeFileSync(path.join(dir, 'index.json'), JSON.stringify(index));
+describe('readStoryIndex package metadata', () => {
+ function fixture() {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-index-'));
+ for (const [dir, manifest] of [
+ ['packages/core', {name: '@astryxdesign/core'}],
+ ['packages/charts', {name: '@astryxdesign/charts', private: true, astryx: {canaryOnly: true}}],
+ ['packages/lab', {name: '@astryxdesign/lab', private: true, astryx: {canaryOnly: true}}],
+ ['packages/themes/neutral', {name: '@astryxdesign/theme-neutral', private: false}],
+ ['packages/themes/probe', {name: '@astryxdesign/theme-probe', private: true}],
+ ]) {
+ fs.mkdirSync(path.join(root, dir), {recursive: true});
+ fs.writeFileSync(path.join(root, dir, 'package.json'), JSON.stringify(manifest));
+ }
+ const storybook = path.join(root, 'apps/storybook');
+ const dist = path.join(storybook, 'dist');
+ fs.mkdirSync(path.join(storybook, 'stories'), {recursive: true});
+ fs.mkdirSync(dist);
+ fs.writeFileSync(path.join(storybook, 'stories/Composite.stories.tsx'), "import {Table} from '@astryxdesign/core/Table';");
+ fs.writeFileSync(path.join(storybook, 'stories/Lab.stories.tsx'), "import {Thing} from '@astryxdesign/lab'; import {Button} from '@astryxdesign/core/Button';");
+ fs.writeFileSync(path.join(storybook, 'stories/CoreMixed.stories.tsx'), "import {Thing} from '@astryxdesign/lab'; import {Layer} from '@astryxdesign/core/Layer';");
+ fs.writeFileSync(path.join(storybook, 'stories/Probe.stories.tsx'), "import {Button} from '@astryxdesign/core/Button';");
+ fs.writeFileSync(path.join(dist, 'index.json'), JSON.stringify({entries: {
+ core: {type: 'story', id: 'core-button--default', title: 'Core/Button', name: 'Default', componentPath: '../../packages/core/src/Button/index.ts', importPath: './stories/Button.stories.tsx'},
+ composite: {type: 'story', id: 'core-composite--default', title: 'Core/Composite', name: 'Default', importPath: './stories/Composite.stories.tsx'},
+ charts: {type: 'story', id: 'charts-bar--default', title: 'Charts/Bar', name: 'Default', componentPath: '@astryxdesign/charts/Bar', importPath: './stories/Bar.stories.tsx'},
+ lab: {type: 'story', id: 'lab-thing--default', title: 'Lab/Thing', name: 'Default', importPath: './stories/Lab.stories.tsx'},
+ mixed: {type: 'story', id: 'core-layer--default', title: 'Core/Layer', name: 'Default', importPath: './stories/CoreMixed.stories.tsx'},
+ probe: {type: 'story', id: 'core-probe--default', title: 'Core/Themes/Probe Theme', name: 'Default', importPath: './stories/Probe.stories.tsx'},
+ skipped: {type: 'story', id: 'core-skip--default', title: 'Core/Skip', name: 'Default', tags: ['no-visual']},
+ }}));
+ return {root, dist};
+ }
+
+ it('uses component paths or generated import paths, then package.json eligibility', () => {
+ const {root, dist} = fixture();
+ try {
+ const indexed = readStoryIndex(dist, [], root);
+ expect(indexed.find(value => value.id === 'core-button--default')).toMatchObject({packageName: '@astryxdesign/core', stableVisual: true});
+ expect(indexed.find(value => value.id === 'core-composite--default')).toMatchObject({packageNames: ['@astryxdesign/core'], stableVisual: true});
+ expect(indexed.find(value => value.id === 'charts-bar--default')).toMatchObject({packageName: '@astryxdesign/charts', stableVisual: false});
+ expect(indexed.find(value => value.id === 'lab-thing--default')).toMatchObject({packageName: '@astryxdesign/lab', stableVisual: false});
+ expect(indexed.find(value => value.id === 'core-layer--default')).toMatchObject({packageName: '@astryxdesign/core', stableVisual: true});
+ expect(indexed.find(value => value.id === 'core-probe--default')).toMatchObject({packageName: '@astryxdesign/theme-probe', stableVisual: false});
+ expect(indexed.some(value => value.id === 'core-skip--default')).toBe(false);
+ } finally {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
+ });
+
+ it('accounts for a live-shaped 974-key baseline without dropping legacy keys', () => {
+ const {root} = fixture();
+ for (const [dir, manifest] of [
+ ['packages/vega', {name: '@astryxdesign/vega', private: true, astryx: {canaryOnly: true}}],
+ ['packages/richtext', {name: '@astryxdesign/richtext', private: true, astryx: {canaryOnly: true}}],
+ ]) {
+ fs.mkdirSync(path.join(root, dir), {recursive: true});
+ fs.writeFileSync(path.join(root, dir, 'package.json'), JSON.stringify(manifest));
+ }
+ const baselineShots = {};
+ const current = [];
+ for (let index = 0; index < 882; index += 1) {
+ const storyId = `core-story-${index}`;
+ const key = `${storyId}__neutral-light`;
+ baselineShots[key] = {
+ storyId,
+ title: `Core/Story ${index}`,
+ component: `Story${index}`,
+ theme: 'neutral',
+ mode: 'light',
+ };
+ current.push({
+ id: storyId,
+ packageName: '@astryxdesign/core',
+ packageNames: ['@astryxdesign/core'],
+ stableVisual: true,
+ });
+ }
+ const excluded = [
+ ['Charts', 28],
+ ['Lab', 60],
+ ['Richtext', 2],
+ ['Vega', 2],
+ ];
+ for (const [title, count] of excluded) {
+ for (let index = 0; index < count; index += 1) {
+ const storyId = `${title.toLowerCase()}-story-${index}`;
+ baselineShots[`${storyId}__neutral-light`] = {
+ storyId,
+ title: `${title}/Story ${index}`,
+ component: `Story${index}`,
+ theme: 'neutral',
+ mode: 'light',
+ };
+ }
+ }
try {
- return readStoryIndex(dir, exclusions).map(story => story.id);
+ const account = accountBaseline(
+ {shots: baselineShots},
+ current,
+ readThemeCatalog(root),
+ root,
+ );
+ const summary = summarizeBaselineAccounting(
+ account,
+ current.map(story => ({key: `${story.id}__neutral-light`})),
+ );
+ expect(summary).toEqual({
+ total: 974,
+ plannedCurrentStable: 882,
+ intentionallyExcluded: 92,
+ preservedLegacy: 0,
+ unclassified: 0,
+ });
+ expect(Object.keys(account.manifest.shots)).toHaveLength(882);
} finally {
- fs.rmSync(dir, {recursive: true, force: true});
+ fs.rmSync(root, {recursive: true, force: true});
}
- };
+ });
- it('drops docs entries and stories tagged out', () => {
- expect(read([])).toEqual(['lab-stream--one', 'lab-stream--two', 'core-button--default']);
+ it('preserves a deleted accepted story as a release decision', () => {
+ const {root} = fixture();
+ try {
+ const account = accountBaseline(
+ {
+ shots: {
+ 'core-gone--default__neutral-light': {
+ storyId: 'core-gone--default',
+ title: 'Core/Gone',
+ component: 'Gone',
+ theme: 'neutral',
+ mode: 'light',
+ },
+ },
+ },
+ [],
+ readThemeCatalog(root),
+ root,
+ );
+ expect(summarizeBaselineAccounting(account, [])).toEqual({
+ total: 1,
+ plannedCurrentStable: 0,
+ intentionallyExcluded: 0,
+ preservedLegacy: 1,
+ unclassified: 0,
+ });
+ expect(account.manifest.shots).toHaveProperty(
+ 'core-gone--default__neutral-light',
+ );
+ } finally {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
});
- it('drops one story by id', () => {
- expect(read(['lab-stream--one'])).not.toContain('lab-stream--one');
+ it('reports exact unclassified legacy keys without dropping them silently', () => {
+ const {root} = fixture();
+ try {
+ const account = accountBaseline(
+ {
+ shots: {
+ 'unknown--default__neutral-light': {
+ storyId: 'unknown--default',
+ title: 'Unknown/Thing',
+ component: 'Missing',
+ theme: 'neutral',
+ mode: 'light',
+ },
+ },
+ },
+ [],
+ readThemeCatalog(root),
+ root,
+ );
+ expect(summarizeBaselineAccounting(account, [])).toEqual({
+ total: 1,
+ plannedCurrentStable: 0,
+ intentionallyExcluded: 0,
+ preservedLegacy: 0,
+ unclassified: 1,
+ unclassifiedKeys: ['unknown--default__neutral-light'],
+ });
+ } finally {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
});
- it('drops a whole story file with a trailing star', () => {
- expect(read(['lab-stream--*'])).toEqual(['core-button--default']);
+ it('canonicalizes release keys and rejects duplicates', () => {
+ const shot = {...stories[0], storyId: stories[0].id, key: 'b', theme: 'neutral', mode: 'light', reasons: []};
+ expect(createReleasePlan([{...shot, key: 'b'}, {...shot, key: 'a'}]).keys).toEqual(['a', 'b']);
+ expect(() => createReleasePlan([shot, shot])).toThrow(/repeats/);
});
});
diff --git a/.github/scripts/visual-gate/publish-pr-report.mjs b/.github/scripts/visual-gate/publish-pr-report.mjs
index 36a74ca4b3..e28153e2a0 100644
--- a/.github/scripts/visual-gate/publish-pr-report.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.mjs
@@ -13,12 +13,14 @@
import {createHash} from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
+import {fileURLToPath} from 'node:url';
import {incomparable} from './lib/baseline.mjs';
import {canonicalizePng} from './lib/canonical-png.mjs';
import {buildVerdict, compareCaptures} from './lib/compare.mjs';
-import {shotKey} from './lib/plan.mjs';
+import {readThemeCatalog, shotKey} from './lib/plan.mjs';
import {renderReport} from './lib/report.mjs';
+import {loadConfig} from './lib/sources.mjs';
const args = process.argv.slice(2);
const flag = name => {
@@ -34,15 +36,16 @@ const pr = Number(flag('pr'));
const headSha = flag('head-sha') ?? '';
const runId = flag('run-id') ?? '';
const runAttempt = flag('run-attempt') ?? '';
-const config = JSON.parse(
- fs.readFileSync(
- new URL('./visual-gate.config.json', import.meta.url),
- 'utf8',
- ),
+const REPO_ROOT = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '../../..',
);
+const config = loadConfig(REPO_ROOT);
+const themeCatalog = readThemeCatalog(REPO_ROOT);
const KEY = /^[A-Za-z0-9._-]{1,240}$/;
const NAME = /^[A-Za-z0-9._-]{1,120}$/;
+const PACKAGE_NAME = /^@[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
const SHA = /^[0-9a-f]{40}$/;
const MAX_PNG_BYTES = 12 * 1024 * 1024;
const MAX_EDGE = 5000;
@@ -137,7 +140,16 @@ if (
fail('trusted capture identity mismatch');
}
-const baselineManifest = readJSON(path.join(baseline, 'manifest.json'));
+const rawBaselineManifest = readJSON(path.join(baseline, 'manifest.json'));
+const capturedKeys = new Set(Object.keys(manifest.shots));
+const baselineManifest = {
+ ...rawBaselineManifest,
+ shots: Object.fromEntries(
+ Object.entries(rawBaselineManifest.shots ?? {}).filter(([key]) =>
+ capturedKeys.has(key),
+ ),
+ ),
+};
const blocker = incomparable(baselineManifest, manifest);
if (blocker) fail(`baseline is not comparable: ${blocker}`);
@@ -145,6 +157,7 @@ const entries = Object.entries(manifest.shots);
if (entries.length > MAX_SHOTS)
fail(`trusted capture exceeds ${MAX_SHOTS} shots`);
for (const [key, shot] of entries) {
+ const themeMetadata = themeCatalog[shot?.theme];
if (
!KEY.test(key) ||
!shot ||
@@ -152,6 +165,12 @@ for (const [key, shot] of entries) {
!shot.storyId ||
typeof shot.theme !== 'string' ||
!NAME.test(shot.theme) ||
+ !PACKAGE_NAME.test(shot.packageName ?? '') ||
+ shot.stableVisual !== true ||
+ !PACKAGE_NAME.test(shot.themePackageName ?? '') ||
+ shot.stableThemeVisual !== true ||
+ themeMetadata?.packageName !== shot.themePackageName ||
+ themeMetadata?.stableVisual !== true ||
!['light', 'dark'].includes(shot.mode) ||
!Array.isArray(shot.reasons) ||
shotKey(shot) !== key
@@ -201,17 +220,7 @@ for (const [key, shot] of entries) {
};
}
-const baselineEntries = Object.entries(baselineManifest.shots ?? {});
-const expected = scope.broadStableVisual
- ? baselineEntries.map(([key]) => key)
- : baselineEntries
- .filter(
- ([, shot]) =>
- scope.stableComponents.includes(shot.component) ||
- scope.stableThemes.includes(shot.theme),
- )
- .map(([key]) => key);
-if (entries.length === 0 && expected.length === 0) {
+if (entries.length === 0) {
fail('stable visual scope produced no trusted shots');
}
@@ -223,9 +232,7 @@ const comparison = await compareCaptures({
diffDir: path.join(output, 'diff'),
threshold: config.threshold,
maxDiffPixels: config.maxDiffPixels,
- scoped: true,
});
-comparison.removed = expected.filter(key => !trustedManifest.shots[key]);
const verdict = buildVerdict({
comparison,
diff --git a/.github/scripts/visual-gate/publish-pr-report.test.mjs b/.github/scripts/visual-gate/publish-pr-report.test.mjs
index dfc22c5643..0cb6b55797 100644
--- a/.github/scripts/visual-gate/publish-pr-report.test.mjs
+++ b/.github/scripts/visual-gate/publish-pr-report.test.mjs
@@ -28,7 +28,12 @@ const SHOT = {
title: 'Core/Button',
name: 'Default',
component: 'Button',
+ packageName: '@astryxdesign/core',
+ packageNames: ['@astryxdesign/core'],
+ stableVisual: true,
theme: 'neutral',
+ themePackageName: '@astryxdesign/theme-neutral',
+ stableThemeVisual: true,
mode: 'light',
reasons: ['trusted:pr-scope'],
};
@@ -314,14 +319,17 @@ describe('trusted PR visual publisher', () => {
expect(fs.existsSync(path.join(output, 'after', `${key}.png`))).toBe(true);
});
- it('derives a removed baseline shot when trusted capture omits expected scope', () => {
- writeCapture({});
+ it('does not derive removals from a scoped PR capture', () => {
+ const other = 'core-card--default__neutral-light';
+ writeBaseline({
+ [KEY]: {shot: SHOT, bytes: png()},
+ [other]: {shot: {...SHOT, storyId: 'core-card--default', component: 'Card'}, bytes: png()},
+ });
run();
const verdict = JSON.parse(
fs.readFileSync(path.join(output, 'verdict.json'), 'utf8'),
);
- expect(verdict).toMatchObject({status: 'changed', removed: [KEY]});
- expect(fs.existsSync(path.join(output, 'before', `${KEY}.png`))).toBe(true);
+ expect(verdict.removed).toEqual([]);
});
it('rejects a trusted capture that claims another run', () => {
diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjs
index 79fefc4086..3209bb8018 100644
--- a/.github/scripts/visual-gate/visual-acceptance.mjs
+++ b/.github/scripts/visual-gate/visual-acceptance.mjs
@@ -12,14 +12,30 @@ import {execFileSync} from 'node:child_process';
import {createHash} from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
+import {fileURLToPath} from 'node:url';
import {
isVisualAcceptanceEndpointMaintainer,
isVisualAcceptanceRecordMaintainer,
} from './authorization.mjs';
import {canonicalizePng} from './lib/canonical-png.mjs';
-import {readStoryIndex, shotKey, storiesInPackages} from './lib/plan.mjs';
+import {
+ readStoryIndex,
+ readThemeCatalog,
+ shotKey,
+ stableBaseline,
+ storiesInPackages,
+ withThemeMetadata,
+} from './lib/plan.mjs';
import {renderReport} from './lib/report.mjs';
+import {loadConfig} from './lib/sources.mjs';
+
+const REPO_ROOT = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '../../..',
+);
+const config = loadConfig(REPO_ROOT);
+const themeCatalog = readThemeCatalog(REPO_ROOT);
const args = process.argv.slice(2);
const command = args[0];
@@ -348,6 +364,9 @@ function accept() {
if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {
fail('current visual bundle has no delta to accept');
}
+ if (evidence.deltas.some(delta => delta.kind === 'removed')) {
+ fail('scoped PR acceptance cannot authorize baseline removals');
+ }
const manifestFile = path.join(
pages,
'visual-gate',
@@ -588,10 +607,13 @@ function readTrustedScope() {
return scope;
}
-function readTrustedBaseline() {
- const baseline = readJSON(
+function readTrustedBaseline(stories = null) {
+ const raw = readJSON(
path.join(path.resolve(flag('baseline')), 'manifest.json'),
);
+ const baseline = stories
+ ? stableBaseline(raw, stories, themeCatalog, REPO_ROOT)
+ : raw;
if (
!baseline?.shots ||
typeof baseline.shots !== 'object' ||
@@ -692,9 +714,14 @@ function trustedPlan() {
const scope = readTrustedScope();
if (scope.broadStableVisual)
fail('broad stable scope must be deferred instead of captured');
- const {entries: baselineEntries} = readTrustedBaseline();
const storybookDir = path.resolve(flag('storybook-dir'));
const output = path.resolve(flag('output'));
+ const allIndexed = readStoryIndex(storybookDir, [], REPO_ROOT);
+ const indexed = storiesInPackages(
+ allIndexed,
+ config.stableStoryPackages,
+ );
+ const {entries: baselineEntries} = readTrustedBaseline(allIndexed);
const baselineThemes = [
...new Set(baselineEntries.map(([, shot]) => shot.theme)),
@@ -702,46 +729,54 @@ function trustedPlan() {
const shots = [];
if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {
- const indexed = storiesInPackages(readStoryIndex(storybookDir, []), [
- 'Core',
- ]);
- const stories = new Map();
- const newTheme = scope.stableThemes.some(
- theme => !baselineThemes.includes(theme),
- );
- for (const [, shot] of baselineEntries) {
- if (
- newTheme ||
- scope.stableThemes.includes(shot.theme) ||
- scope.stableComponents.includes(shot.component)
- ) {
- stories.set(shot.storyId, shot);
+ const indexedStories = new Map(indexed.map(story => [story.id, story]));
+ const add = (story, theme) => {
+ for (const mode of ['light', 'dark']) {
+ const shot = {
+ storyId: story.storyId ?? story.id,
+ title: story.title,
+ name: story.name,
+ component: story.component,
+ packageName: story.packageName,
+ packageNames: story.packageNames,
+ stableVisual: story.stableVisual,
+ theme,
+ mode,
+ reasons: ['trusted:pr-scope'],
+ };
+ shots.push({...shot, key: shotKey(shot)});
+ }
+ };
+
+ const componentStories = new Map();
+ for (const story of indexedStories.values()) {
+ if (scope.stableComponents.includes(story.component)) {
+ componentStories.set(story.storyId ?? story.id, story);
}
}
- for (const story of indexed) {
- if (scope.stableComponents.includes(story.component))
- stories.set(story.id, story);
+ for (const story of componentStories.values()) {
+ for (const theme of baselineThemes) add(story, theme);
}
- const themes =
- scope.stableThemes.length > 0 ? scope.stableThemes : baselineThemes;
- for (const story of stories.values()) {
- for (const theme of themes) {
- for (const mode of ['light', 'dark']) {
- const shot = {
- storyId: story.storyId ?? story.id,
- title: story.title,
- name: story.name,
- component: story.component,
- theme,
- mode,
- reasons: ['trusted:pr-scope'],
- };
- shots.push({...shot, key: shotKey(shot)});
+
+ for (const theme of scope.stableThemes) {
+ const isNew = !baselineThemes.includes(theme);
+ const themeStories = new Map();
+ for (const [, shot] of baselineEntries) {
+ const story = indexedStories.get(shot.storyId) ?? shot;
+ if (isNew || shot.theme === theme) {
+ themeStories.set(shot.storyId, story);
}
}
+ for (const story of themeStories.values()) add(story, theme);
}
}
- const unique = [...new Map(shots.map(shot => [shot.key, shot])).values()];
+ const unique = withThemeMetadata(
+ [...new Map(shots.map(shot => [shot.key, shot])).values()],
+ themeCatalog,
+ );
+ if (unique.some(shot => shot.stableThemeVisual !== true)) {
+ fail('trusted stable plan includes a private or canary theme');
+ }
if (unique.length === 0 || unique.length > 5000) {
fail(`trusted visual plan has invalid size ${unique.length}`);
}
@@ -751,14 +786,26 @@ function trustedPlan() {
);
}
+function acceptedStableShots(acceptance) {
+ if (acceptance.keys.some(entry => entry.kind === 'removed')) {
+ fail('scoped PR promotion cannot remove baseline keys');
+ }
+ const shots = withThemeMetadata(
+ acceptance.keys.map(entry => ({...entry.shot, key: entry.key})),
+ themeCatalog,
+ );
+ if (shots.some(shot => shot.stableThemeVisual !== true)) {
+ fail('accepted stable plan includes a private or canary theme');
+ }
+ return shots;
+}
+
function plan() {
const acceptanceFile = path.resolve(flag('acceptance'));
const acceptance = readJSON(acceptanceFile);
validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
const output = path.resolve(flag('output'));
- const shots = acceptance.keys
- .filter(entry => entry.kind !== 'removed')
- .map(entry => ({...entry.shot, key: entry.key}));
+ const shots = acceptedStableShots(acceptance);
writeJSON(output, shots);
process.stdout.write(
`Wrote ${shots.length} post-merge shot(s) to ${output}.\n`,
@@ -778,6 +825,7 @@ function promote() {
const acceptance = readJSON(acceptanceFile);
validateAcceptance(acceptance, {dir: path.dirname(acceptanceFile)});
+ acceptedStableShots(acceptance);
const manifestFile = path.join(
pages,
'visual-gate',
@@ -823,10 +871,6 @@ function promote() {
`baseline conflict for ${entry.key}: expected ${entry.beforeSha256}, found ${currentBefore}`,
);
}
- if (entry.kind === 'removed') {
- actions.push({entry, baselineFile, remove: true});
- continue;
- }
const acceptedAfter = path.join(acceptedDir, 'after', `${entry.key}.png`);
const recaptured = path.join(capture, 'shots', `${entry.key}.png`);
@@ -890,13 +934,8 @@ function promote() {
for (const action of actions) {
if (action.alreadyPromoted) continue;
- if (action.remove) {
- if (fs.existsSync(action.baselineFile)) fs.rmSync(action.baselineFile);
- delete manifest.shots[action.entry.key];
- } else {
- fs.writeFileSync(action.baselineFile, action.canonicalBytes);
- manifest.shots[action.entry.key] = action.capturedShot;
- }
+ fs.writeFileSync(action.baselineFile, action.canonicalBytes);
+ manifest.shots[action.entry.key] = action.capturedShot;
}
manifest.capturedAt = captureManifest.capturedAt;
@@ -913,12 +952,8 @@ function promote() {
pr: acceptance.pr,
headSha: acceptance.headSha,
mergeSha,
- promoted: acceptance.keys
- .filter(entry => entry.kind !== 'removed')
- .map(entry => entry.key),
- pruned: acceptance.keys
- .filter(entry => entry.kind === 'removed')
- .map(entry => entry.key),
+ promoted: acceptance.keys.map(entry => entry.key),
+ pruned: [],
},
].slice(-200);
writeJSON(manifestFile, manifest);
diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjs
index efa1c67f4d..7b96e4da5e 100644
--- a/.github/scripts/visual-gate/visual-acceptance.test.mjs
+++ b/.github/scripts/visual-gate/visual-acceptance.test.mjs
@@ -29,7 +29,12 @@ const SHOT = {
title: 'Core/Button',
name: 'Default',
component: 'Button',
+ packageName: '@astryxdesign/core',
+ packageNames: ['@astryxdesign/core'],
+ stableVisual: true,
theme: 'neutral',
+ themePackageName: '@astryxdesign/theme-neutral',
+ stableThemeVisual: true,
mode: 'light',
reasons: ['component:Button'],
};
@@ -498,6 +503,7 @@ describe('visual acceptance', () => {
id: 'core-button--default',
title: 'Core/Button',
name: 'Default',
+ componentPath: '../../packages/core/src/Button/index.ts',
tags: [],
},
},
@@ -533,7 +539,7 @@ describe('visual acceptance', () => {
hasStableVisual: true,
broadStableVisual: false,
stableComponents: [],
- stableThemes: ['new-theme'],
+ stableThemes: ['y2k'],
});
const output = path.join(root, 'trusted-theme-plan.json');
run('trusted-plan', {
@@ -545,8 +551,8 @@ describe('visual acceptance', () => {
expect(
JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
).toEqual([
- 'core-button--default__new-theme-light',
- 'core-button--default__new-theme-dark',
+ 'core-button--default__y2k-light',
+ 'core-button--default__y2k-dark',
]);
});
@@ -861,34 +867,13 @@ describe('visual acceptance', () => {
},
);
- it('promotes a removed shot without an AFTER hash or recaptured file', () => {
+ it('rejects removed PR evidence and preserves the baseline', () => {
writeEvidence({kind: 'removed', run: 125});
- run('accept', acceptanceFlags({'run-id': 125}));
- const capture = path.join(root, 'capture-removed');
- fs.mkdirSync(capture);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-26T22:00:00.000Z',
- context: {sha: MERGE},
- shots: {},
- });
-
- expect(
- run('promote', {
- pages,
- acceptance: acceptanceFile(125),
- capture,
- 'merge-sha': MERGE,
- }),
- ).toContain('Promoted accepted visual bundle');
- expect(
- fs.existsSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`),
- ),
- ).toBe(false);
+ expect(fail('accept', acceptanceFlags({'run-id': 125}))).toMatch(
+ /scoped PR acceptance cannot authorize baseline removals/,
+ );
+ expect(fs.existsSync(acceptanceFile(125))).toBe(false);
+ expect(fs.existsSync(path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`))).toBe(true);
});
it('rejects a merged recapture whose canonical pixels changed', () => {
@@ -980,29 +965,19 @@ describe('visual acceptance', () => {
).toMatch(/baseline conflict/);
});
- it('records added and removed shots without inventing missing images', () => {
+ it('records added shots but rejects removed evidence', () => {
const added = 'core-new--default__neutral-light';
writeEvidence({kind: 'added', key: added, run: 124});
run('accept', acceptanceFlags({'run-id': 124}));
- let record = JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8'));
- expect(record.keys[0]).toMatchObject({
+ expect(JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8')).keys[0]).toMatchObject({
key: added,
kind: 'added',
beforeSha256: null,
});
-
- fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {
- recursive: true,
- force: true,
- });
+ fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {recursive: true, force: true});
writeEvidence({kind: 'removed', run: 125});
- run('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}));
- record = JSON.parse(fs.readFileSync(acceptanceFile(125), 'utf8'));
- expect(record.keys[0]).toMatchObject({
- key: KEY,
- kind: 'removed',
- afterSha256: null,
- shot: null,
- });
+ expect(fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}))).toMatch(
+ /scoped PR acceptance cannot authorize baseline removals/,
+ );
});
});
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
index 50723d50a8..96ad0bcfe7 100644
--- a/.github/workflows/release-gate.yml
+++ b/.github/workflows/release-gate.yml
@@ -36,11 +36,6 @@ on:
# 13:00 UTC = 06:00 PT — two hours before the 08:00 PT cut (see TIMING above).
- cron: '0 13 * * *'
workflow_dispatch:
- inputs:
- tiers:
- description: 'Shot tiers: surface, theme-matrix, full'
- required: false
- default: 'surface,theme-matrix'
permissions: {}
@@ -96,17 +91,22 @@ jobs:
echo "::notice::No baseline on gh-pages yet — this run establishes one."
fi
- # Exit 2 means "changed", which is a question for the release cut, not a
- # broken run. Exit 1 (crashed) still fails the job.
+ # The derived surface/matrix stays representative, but an accepted
+ # stable key is a durable release promise: recapture every still-valid
+ # baseline key until a reviewed removal prunes it. Package filtering is
+ # applied to the baseline too, so legacy canary keys do not block Core.
+ # The gate constructs the canonical stable tiers internally. A caller may
+ # not expand, rename, or narrow the plan that owns removal authority.
+ # Exit 2 means "changed", which is a release decision; only a crash (1)
+ # fails this job mechanically.
- name: Run the visual gate
id: gate
run: |
set +e
- node .github/scripts/visual-gate/gate.mjs check \
+ node .github/scripts/visual-gate/gate.mjs release \
--storybook-dir apps/storybook/dist \
--baseline .visual-baseline \
--out .visual-run \
- --tiers "${{ inputs.tiers || 'surface,theme-matrix' }}" \
--summary-output visual-summary.md
code=$?
cat visual-summary.md >> "$GITHUB_STEP_SUMMARY" || true