Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): prevent opportunistic visual baselines (#5948)

Cindy Zhangadf002a44717
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index 4cb8401ab8..f9635ba825 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -72,11 +72,12 @@ Exit codes are the contract: `0` pass, `1` crashed, `2` changed.
`pr-visual` compares only the stable published visual surface:
-- Core component change → the representative story in every accepted baseline
- theme. Additional stories may opt into the default theme with
- `visual-baseline`, or the same all-theme matrix with `visual-theme-matrix`;
- all are captured light/dark. Behavioral and audit-only fixtures stay in their
- dedicated checks without multiplying the pixel baseline.
+- Core component change → the representative story and any explicitly opted-in
+ story, but only for frame keys already present in the accepted baseline.
+ `visual-baseline` opts into the default theme and `visual-theme-matrix` opts
+ into every accepted theme; neither tag lets an ordinary PR create a new
+ baseline key. Behavioral and audit-only fixtures stay in their dedicated
+ checks without multiplying the pixel baseline.
- Published theme change → every currently accepted visual story rendered in that
theme. This catches a theme beginning to override a component it did not
previously target. Theme-only plans are not charged against the focused
@@ -103,8 +104,10 @@ The representative story (`Default`, `Primary`, and similar conventional names)
is selected automatically and needs no tag. Use `visual-baseline` for an
additional default-theme contract, `visual-theme-matrix` only when that story
must be judged in every accepted theme, and no visual tag for behavioral or
-audit-only fixtures. The existing `no-visual` tag excludes an unstable story
-from visual capture entirely.
+audit-only fixtures. Ordinary PRs can update existing contracts but cannot add
+or remove baseline keys; seed or prune coverage through the manual baseline
+workflow. The existing `no-visual` tag excludes an unstable story from visual
+capture entirely.
The daily gate uses the same boundary: `stableStoryPackages` in
`visual-gate.config.json` is currently `["Core"]`, so Lab/canary stories cannot
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index 3075f88be7..86fc5405a0 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -42,6 +42,7 @@ import {
buildPlan,
createReleasePlan,
exceedsPrVisualShotLimit,
+ existingComponentBaselinePlan,
readStoryIndex,
readThemeCatalog,
resolvePrVisualTotalShotLimit,
@@ -227,17 +228,20 @@ async function plan() {
}
const componentShotCount = tiers.includes('component')
- ? buildPlan({
- stories,
- targets,
- themeOverrides,
- observations,
- defaultTheme: config.defaultTheme,
- tiers: ['component'],
- components,
- componentThemes,
- probeTheme: config.probeTheme,
- }).length
+ ? existingComponentBaselinePlan(
+ buildPlan({
+ stories,
+ targets,
+ themeOverrides,
+ observations,
+ defaultTheme: config.defaultTheme,
+ tiers: ['component'],
+ components,
+ componentThemes,
+ probeTheme: config.probeTheme,
+ }),
+ baselineManifest,
+ ).length
: 0;
let shots = buildPlan({
stories,
@@ -252,6 +256,9 @@ async function plan() {
themeStories,
probeTheme: config.probeTheme,
});
+ if (tiers.includes('component')) {
+ shots = existingComponentBaselinePlan(shots, baselineManifest);
+ }
let baselineAccount = null;
if (releaseMode) {
baselineAccount = accountBaseline(
diff --git a/.github/scripts/visual-gate/lib/plan.mjs b/.github/scripts/visual-gate/lib/plan.mjs
index 2c5e933355..4f24984687 100644
--- a/.github/scripts/visual-gate/lib/plan.mjs
+++ b/.github/scripts/visual-gate/lib/plan.mjs
@@ -429,6 +429,27 @@ export function componentVisualStories(stories, components) {
}));
}
+function isComponentScopeReason(reason) {
+ return reason === 'component' || /^theme:[^:]+$/.test(reason);
+}
+
+/**
+ * A component edit may compare an accepted baseline contract, but it may not
+ * create one. Keep independently selected theme/probe shots even when the same
+ * key is also part of the component scope.
+ */
+export function existingComponentBaselinePlan(plan, manifest) {
+ const accepted = new Set(Object.keys(manifest?.shots ?? {}));
+ return plan.filter(shot => {
+ const reasons = shot.reasons ?? [];
+ const componentScoped = reasons.some(isComponentScopeReason);
+ const independentlyScoped = reasons.some(
+ reason => !isComponentScopeReason(reason),
+ );
+ return !componentScoped || independentlyScoped || accepted.has(shot.key);
+ });
+}
+
/**
* Themes already represented by the accepted stable baseline. The default
* theme is always present for a focused component capture.
diff --git a/.github/scripts/visual-gate/lib/plan.test.mjs b/.github/scripts/visual-gate/lib/plan.test.mjs
index 63d6edac57..a5a95f1e7f 100644
--- a/.github/scripts/visual-gate/lib/plan.test.mjs
+++ b/.github/scripts/visual-gate/lib/plan.test.mjs
@@ -13,6 +13,7 @@ import {
buildPlan,
createReleasePlan,
exceedsPrVisualShotLimit,
+ existingComponentBaselinePlan,
readStoryIndex,
readThemeCatalog,
representativeStories,
@@ -75,6 +76,22 @@ describe('representativeStories', () => {
});
});
+describe('existingComponentBaselinePlan', () => {
+ it('drops only component-selected keys that have no accepted baseline', () => {
+ const plan = [
+ {key: 'component-existing', reasons: ['component']},
+ {key: 'component-new', reasons: ['component', 'theme:y2k']},
+ {key: 'theme-new', reasons: ['component', 'changed-theme:y2k']},
+ {key: 'probe-new', reasons: ['probe']},
+ ];
+ expect(
+ existingComponentBaselinePlan(plan, {
+ shots: {'component-existing': {}},
+ }).map(shot => shot.key),
+ ).toEqual(['component-existing', 'theme-new', 'probe-new']);
+ });
+});
+
describe('acceptedVisualThemes', () => {
it('uses stable baseline themes and always includes the stable default', () => {
expect(
diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjs
index 690f1ceee3..850d93e028 100644
--- a/.github/scripts/visual-gate/visual-acceptance.mjs
+++ b/.github/scripts/visual-gate/visual-acceptance.mjs
@@ -373,8 +373,10 @@ function accept() {
if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {
fail('current visual bundle has no delta to accept');
}
- if (evidence.deltas.some(delta => delta.kind === 'removed')) {
- fail('scoped PR acceptance cannot authorize baseline removals');
+ if (evidence.deltas.some(delta => delta.kind !== 'changed')) {
+ fail(
+ 'scoped PR acceptance can only authorize changes to existing baseline frames',
+ );
}
const manifestFile = path.join(
pages,
@@ -736,6 +738,7 @@ function trustedPlan() {
config.stableStoryPackages,
);
const {baseline} = readTrustedBaseline(allIndexed);
+ const baselineKeys = new Set(Object.keys(baseline.shots));
const baselineThemes = acceptedVisualThemes(
baseline,
@@ -745,7 +748,12 @@ function trustedPlan() {
const shots = [];
if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {
- const add = (story, theme, componentKeys = null) => {
+ const add = (
+ story,
+ theme,
+ componentKeys = null,
+ existingBaselineOnly = false,
+ ) => {
for (const mode of ['light', 'dark']) {
const shot = {
storyId: story.storyId ?? story.id,
@@ -760,6 +768,7 @@ function trustedPlan() {
reasons: ['trusted:pr-scope'],
};
const keyed = {...shot, key: shotKey(shot)};
+ if (existingBaselineOnly && !baselineKeys.has(keyed.key)) continue;
shots.push(keyed);
componentKeys?.add(keyed.key);
}
@@ -770,10 +779,12 @@ function trustedPlan() {
indexed,
scope.stableComponents,
)) {
- add(story, config.defaultTheme, componentKeys);
+ add(story, config.defaultTheme, componentKeys, true);
if (!useThemeMatrix) continue;
for (const theme of baselineThemes) {
- if (theme !== config.defaultTheme) add(story, theme, componentKeys);
+ if (theme !== config.defaultTheme) {
+ add(story, theme, componentKeys, true);
+ }
}
}
if (
@@ -802,10 +813,12 @@ function trustedPlan() {
if (unique.some(shot => shot.stableThemeVisual !== true)) {
fail('trusted stable plan includes a private or canary theme');
}
- if (
- unique.length === 0 ||
- unique.length > config.visualPlanSafetyLimit
- ) {
+ if (unique.length === 0) {
+ fail(
+ 'trusted component scope has no existing baseline frames; seed coverage through the manual baseline workflow',
+ );
+ }
+ if (unique.length > config.visualPlanSafetyLimit) {
fail(
`trusted visual plan has invalid size ${unique.length}; safety limit is ${config.visualPlanSafetyLimit}`,
);
@@ -817,8 +830,10 @@ function trustedPlan() {
}
function acceptedStableShots(acceptance) {
- if (acceptance.keys.some(entry => entry.kind === 'removed')) {
- fail('scoped PR promotion cannot remove baseline keys');
+ if (acceptance.keys.some(entry => entry.kind !== 'changed')) {
+ fail(
+ 'scoped PR promotion can only update existing baseline frames',
+ );
}
const shots = withThemeMetadata(
acceptance.keys.map(entry => ({...entry.shot, key: entry.key})),
diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjs
index 4ed1086560..2535decc07 100644
--- a/.github/scripts/visual-gate/visual-acceptance.test.mjs
+++ b/.github/scripts/visual-gate/visual-acceptance.test.mjs
@@ -483,6 +483,14 @@ describe('visual acceptance', () => {
);
});
+ it('refuses to accept an added baseline frame from a scoped PR', () => {
+ const key = 'core-button--new__neutral-light';
+ writeEvidence({kind: 'added', key});
+ expect(fail('accept', acceptanceFlags())).toMatch(
+ /changes to existing baseline frames/,
+ );
+ });
+
it('returns success for a clean trusted capture without acceptance', () => {
writeEvidence({run: 124, status: 'pass'});
expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(
@@ -564,15 +572,7 @@ describe('visual acceptance', () => {
JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),
).toEqual([
'core-button--default__neutral-light',
- 'core-button--default__neutral-dark',
'core-button--default__y2k-light',
- 'core-button--default__y2k-dark',
- 'core-button--separator__neutral-light',
- 'core-button--separator__neutral-dark',
- 'core-button--variants__neutral-light',
- 'core-button--variants__neutral-dark',
- 'core-button--variants__y2k-light',
- 'core-button--variants__y2k-dark',
]);
});
@@ -630,12 +630,22 @@ describe('visual acceptance', () => {
'manifest.json',
);
const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));
- baseline.shots['core-button--default__y2k-light'] = {
- ...baseline.shots[KEY],
- key: 'core-button--default__y2k-light',
- theme: 'y2k',
- mode: 'light',
- };
+ for (let index = 0; index < 61; index += 1) {
+ const storyId = `core-button--story-${index}`;
+ for (const theme of ['neutral', 'y2k']) {
+ for (const mode of ['light', 'dark']) {
+ const key = `${storyId}__${theme}-${mode}`;
+ baseline.shots[key] = {
+ ...baseline.shots[KEY],
+ key,
+ storyId,
+ name: entries[`story-${index}`].name,
+ theme,
+ mode,
+ };
+ }
+ }
+ }
writeJSON(baselineFile, baseline);
const scope = path.join(root, 'scope-shot-limit.json');
writeJSON(scope, {
@@ -938,7 +948,7 @@ describe('visual acceptance', () => {
expect(printed).toContain('520 shots');
});
- it('accepts a trusted 520-delta broad bundle', () => {
+ it('rejects a trusted 520-delta added bundle', () => {
const runId = 130;
const dir = path.join(
pages,
@@ -977,9 +987,10 @@ describe('visual acceptance', () => {
deltas,
verdict: {version: 1, status: 'changed'},
});
- run('accept', acceptanceFlags({'run-id': runId}));
- const record = JSON.parse(fs.readFileSync(acceptanceFile(runId), 'utf8'));
- expect(record.keys).toHaveLength(520);
+ expect(
+ fail('accept', acceptanceFlags({'run-id': runId})),
+ ).toMatch(/changes to existing baseline frames/);
+ expect(fs.existsSync(acceptanceFile(runId))).toBe(false);
});
it('emits an exact recapture plan without allowing stored metadata to replace the key', () => {
@@ -1058,74 +1069,69 @@ describe('visual acceptance', () => {
});
});
- it.each([
- {kind: 'changed', key: KEY, runId: 123},
- {kind: 'added', key: 'core-new--default__neutral-light', runId: 124},
- ])(
- 'canonicalizes equivalent raw PNG bytes when promoting a $kind shot',
- ({kind, key, runId}) => {
- if (kind === 'added') writeEvidence({kind, key, run: runId});
- run('accept', acceptanceFlags({'run-id': runId}));
- const record = acceptanceFile(runId);
- const accepted = fs.readFileSync(
- path.join(path.dirname(record), 'after', `${key}.png`),
- );
- const decoded = PNG.sync.read(accepted);
- decoded.gamma = 0.45455;
- const rawRecapture = PNG.sync.write(decoded, {deflateLevel: 0});
- expect(rawRecapture).not.toEqual(accepted);
- expect(PNG.sync.read(rawRecapture).data).toEqual(
- PNG.sync.read(accepted).data,
- );
- expect(canonicalizePng(rawRecapture).bytes).toEqual(accepted);
+ it('canonicalizes equivalent raw PNG bytes when promoting a changed shot', () => {
+ const key = KEY;
+ const runId = 123;
+ run('accept', acceptanceFlags({'run-id': runId}));
+ const record = acceptanceFile(runId);
+ const accepted = fs.readFileSync(
+ path.join(path.dirname(record), 'after', `${key}.png`),
+ );
+ const decoded = PNG.sync.read(accepted);
+ decoded.gamma = 0.45455;
+ const rawRecapture = PNG.sync.write(decoded, {deflateLevel: 0});
+ expect(rawRecapture).not.toEqual(accepted);
+ expect(PNG.sync.read(rawRecapture).data).toEqual(
+ PNG.sync.read(accepted).data,
+ );
+ expect(canonicalizePng(rawRecapture).bytes).toEqual(accepted);
- const capture = path.join(root, `capture-${kind}`);
- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
- fs.writeFileSync(path.join(capture, 'shots', `${key}.png`), rawRecapture);
- writeJSON(path.join(capture, 'manifest.json'), {
- version: 1,
- platform: 'linux-arm64',
- browser: 'chromium-140.0',
- viewport: {width: 1280, height: 900},
- capturedAt: '2026-08-26T22:00:00.000Z',
- context: {sha: MERGE},
- shots: {
- [key]: {
- ...SHOT,
- sha256: digest(rawRecapture),
- width: 2,
- height: 2,
- },
+ const capture = path.join(root, 'capture-changed');
+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});
+ fs.writeFileSync(path.join(capture, 'shots', `${key}.png`), rawRecapture);
+ writeJSON(path.join(capture, 'manifest.json'), {
+ version: 1,
+ platform: 'linux-arm64',
+ browser: 'chromium-140.0',
+ viewport: {width: 1280, height: 900},
+ capturedAt: '2026-08-26T22:00:00.000Z',
+ context: {sha: MERGE},
+ shots: {
+ [key]: {
+ ...SHOT,
+ sha256: digest(rawRecapture),
+ width: 2,
+ height: 2,
},
- });
+ },
+ });
- expect(
- run('promote', {
- pages,
- acceptance: record,
- capture,
- 'merge-sha': MERGE,
- }),
- ).toContain('Promoted accepted visual bundle');
- expect(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'shots', `${key}.png`),
- ),
- ).toEqual(accepted);
- const baselineManifest = JSON.parse(
- fs.readFileSync(
- path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
- 'utf8',
- ),
- );
- expect(baselineManifest.shots[key].sha256).toBe(digest(accepted));
- },
- );
+ expect(
+ run('promote', {
+ pages,
+ acceptance: record,
+ capture,
+ 'merge-sha': MERGE,
+ }),
+ ).toContain('Promoted accepted visual bundle');
+ expect(
+ fs.readFileSync(
+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${key}.png`),
+ ),
+ ).toEqual(accepted);
+ const baselineManifest = JSON.parse(
+ fs.readFileSync(
+ path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),
+ 'utf8',
+ ),
+ );
+ expect(baselineManifest.shots[key].sha256).toBe(digest(accepted));
+ });
it('rejects removed PR evidence and preserves the baseline', () => {
writeEvidence({kind: 'removed', run: 125});
expect(fail('accept', acceptanceFlags({'run-id': 125}))).toMatch(
- /scoped PR acceptance cannot authorize baseline removals/,
+ /changes to existing baseline frames/,
);
expect(fs.existsSync(acceptanceFile(125))).toBe(false);
expect(fs.existsSync(path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`))).toBe(true);
@@ -1220,19 +1226,18 @@ describe('visual acceptance', () => {
).toMatch(/baseline conflict/);
});
- it('records added shots but rejects removed evidence', () => {
+ it('rejects both added and removed scoped PR evidence', () => {
const added = 'core-new--default__neutral-light';
writeEvidence({kind: 'added', key: added, run: 124});
- run('accept', acceptanceFlags({'run-id': 124}));
- expect(JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8')).keys[0]).toMatchObject({
- key: added,
- kind: 'added',
- beforeSha256: null,
- });
- fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {recursive: true, force: true});
+ expect(
+ fail('accept', acceptanceFlags({'run-id': 124})),
+ ).toMatch(/changes to existing baseline frames/);
+ expect(fs.existsSync(acceptanceFile(124))).toBe(false);
+
writeEvidence({kind: 'removed', run: 125});
- expect(fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}))).toMatch(
- /scoped PR acceptance cannot authorize baseline removals/,
- );
+ expect(
+ fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125})),
+ ).toMatch(/changes to existing baseline frames/);
+ expect(fs.existsSync(acceptanceFile(125))).toBe(false);
});
});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 052a3b06a2..752c98e4e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -626,11 +626,12 @@ jobs:
- name: Verify guest fixture contrast
run: pnpm -F @astryxdesign/vibe-tests fixtures:contrast
- # Stable-package visual regression. Core components get one representative
- # story in every accepted baseline theme. `visual-baseline` adds default-theme
- # stories; `visual-theme-matrix` adds all-theme stories. A changed shipped
- # theme gets its relevant matrix. Packages marked `astryx.canaryOnly` (Lab,
- # richtext, vega) do not compare pixels and do not own a baseline.
+ # Stable-package visual regression. Core component changes compare only
+ # representative or explicitly tagged story keys already present in the
+ # accepted baseline. A changed shipped theme gets its relevant matrix, but PR
+ # acceptance cannot add or remove baseline keys. Packages marked
+ # `astryx.canaryOnly` (Lab, richtext, vega) do not compare pixels and do not
+ # own a baseline.
#
# Deeper than the daily gate only where the component declares it: audit and
# interaction fixtures stay available to their dedicated checks without