An open source design system that's fully customizable and agent ready
TypeScript
fix(ci): prevent opportunistic visual baselines (#5948)
Cindy Zhangadf002a44717
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.mdindex 4cb8401ab8..f9635ba825 100644--- a/.github/scripts/visual-gate/README.md+++ b/.github/scripts/visual-gate/README.md@@ -72,11 +72,12 @@ Exit codes are the contract: `0` pass, `1` crashed, `2` changed.`pr-visual` compares only the stable published visual surface:-- Core component change → the representative story in every accepted baseline- theme. Additional stories may opt into the default theme with- `visual-baseline`, or the same all-theme matrix with `visual-theme-matrix`;- all are captured light/dark. Behavioral and audit-only fixtures stay in their- dedicated checks without multiplying the pixel baseline.+- Core component change → the representative story and any explicitly opted-in+ story, but only for frame keys already present in the accepted baseline.+ `visual-baseline` opts into the default theme and `visual-theme-matrix` opts+ into every accepted theme; neither tag lets an ordinary PR create a new+ baseline key. Behavioral and audit-only fixtures stay in their dedicated+ checks without multiplying the pixel baseline.- Published theme change → every currently accepted visual story rendered in thattheme. This catches a theme beginning to override a component it did notpreviously target. Theme-only plans are not charged against the focused@@ -103,8 +104,10 @@ The representative story (`Default`, `Primary`, and similar conventional names)is selected automatically and needs no tag. Use `visual-baseline` for anadditional default-theme contract, `visual-theme-matrix` only when that storymust be judged in every accepted theme, and no visual tag for behavioral or-audit-only fixtures. The existing `no-visual` tag excludes an unstable story-from visual capture entirely.+audit-only fixtures. Ordinary PRs can update existing contracts but cannot add+or remove baseline keys; seed or prune coverage through the manual baseline+workflow. The existing `no-visual` tag excludes an unstable story from visual+capture entirely.The daily gate uses the same boundary: `stableStoryPackages` in`visual-gate.config.json` is currently `["Core"]`, so Lab/canary stories cannotdiff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjsindex 3075f88be7..86fc5405a0 100644--- a/.github/scripts/visual-gate/gate.mjs+++ b/.github/scripts/visual-gate/gate.mjs@@ -42,6 +42,7 @@ import {buildPlan,createReleasePlan,exceedsPrVisualShotLimit,+ existingComponentBaselinePlan,readStoryIndex,readThemeCatalog,resolvePrVisualTotalShotLimit,@@ -227,17 +228,20 @@ async function plan() {}const componentShotCount = tiers.includes('component')- ? buildPlan({- stories,- targets,- themeOverrides,- observations,- defaultTheme: config.defaultTheme,- tiers: ['component'],- components,- componentThemes,- probeTheme: config.probeTheme,- }).length+ ? existingComponentBaselinePlan(+ buildPlan({+ stories,+ targets,+ themeOverrides,+ observations,+ defaultTheme: config.defaultTheme,+ tiers: ['component'],+ components,+ componentThemes,+ probeTheme: config.probeTheme,+ }),+ baselineManifest,+ ).length: 0;let shots = buildPlan({stories,@@ -252,6 +256,9 @@ async function plan() {themeStories,probeTheme: config.probeTheme,});+ if (tiers.includes('component')) {+ shots = existingComponentBaselinePlan(shots, baselineManifest);+ }let baselineAccount = null;if (releaseMode) {baselineAccount = accountBaseline(diff --git a/.github/scripts/visual-gate/lib/plan.mjs b/.github/scripts/visual-gate/lib/plan.mjsindex 2c5e933355..4f24984687 100644--- a/.github/scripts/visual-gate/lib/plan.mjs+++ b/.github/scripts/visual-gate/lib/plan.mjs@@ -429,6 +429,27 @@ export function componentVisualStories(stories, components) {}));}+function isComponentScopeReason(reason) {+ return reason === 'component' || /^theme:[^:]+$/.test(reason);+}++/**+ * A component edit may compare an accepted baseline contract, but it may not+ * create one. Keep independently selected theme/probe shots even when the same+ * key is also part of the component scope.+ */+export function existingComponentBaselinePlan(plan, manifest) {+ const accepted = new Set(Object.keys(manifest?.shots ?? {}));+ return plan.filter(shot => {+ const reasons = shot.reasons ?? [];+ const componentScoped = reasons.some(isComponentScopeReason);+ const independentlyScoped = reasons.some(+ reason => !isComponentScopeReason(reason),+ );+ return !componentScoped || independentlyScoped || accepted.has(shot.key);+ });+}+/*** Themes already represented by the accepted stable baseline. The default* theme is always present for a focused component capture.diff --git a/.github/scripts/visual-gate/lib/plan.test.mjs b/.github/scripts/visual-gate/lib/plan.test.mjsindex 63d6edac57..a5a95f1e7f 100644--- a/.github/scripts/visual-gate/lib/plan.test.mjs+++ b/.github/scripts/visual-gate/lib/plan.test.mjs@@ -13,6 +13,7 @@ import {buildPlan,createReleasePlan,exceedsPrVisualShotLimit,+ existingComponentBaselinePlan,readStoryIndex,readThemeCatalog,representativeStories,@@ -75,6 +76,22 @@ describe('representativeStories', () => {});});+describe('existingComponentBaselinePlan', () => {+ it('drops only component-selected keys that have no accepted baseline', () => {+ const plan = [+ {key: 'component-existing', reasons: ['component']},+ {key: 'component-new', reasons: ['component', 'theme:y2k']},+ {key: 'theme-new', reasons: ['component', 'changed-theme:y2k']},+ {key: 'probe-new', reasons: ['probe']},+ ];+ expect(+ existingComponentBaselinePlan(plan, {+ shots: {'component-existing': {}},+ }).map(shot => shot.key),+ ).toEqual(['component-existing', 'theme-new', 'probe-new']);+ });+});+describe('acceptedVisualThemes', () => {it('uses stable baseline themes and always includes the stable default', () => {expect(diff --git a/.github/scripts/visual-gate/visual-acceptance.mjs b/.github/scripts/visual-gate/visual-acceptance.mjsindex 690f1ceee3..850d93e028 100644--- a/.github/scripts/visual-gate/visual-acceptance.mjs+++ b/.github/scripts/visual-gate/visual-acceptance.mjs@@ -373,8 +373,10 @@ function accept() {if (evidence.verdict.status !== 'changed' || evidence.deltas.length === 0) {fail('current visual bundle has no delta to accept');}- if (evidence.deltas.some(delta => delta.kind === 'removed')) {- fail('scoped PR acceptance cannot authorize baseline removals');+ if (evidence.deltas.some(delta => delta.kind !== 'changed')) {+ fail(+ 'scoped PR acceptance can only authorize changes to existing baseline frames',+ );}const manifestFile = path.join(pages,@@ -736,6 +738,7 @@ function trustedPlan() {config.stableStoryPackages,);const {baseline} = readTrustedBaseline(allIndexed);+ const baselineKeys = new Set(Object.keys(baseline.shots));const baselineThemes = acceptedVisualThemes(baseline,@@ -745,7 +748,12 @@ function trustedPlan() {const shots = [];if (scope.stableComponents.length > 0 || scope.stableThemes.length > 0) {- const add = (story, theme, componentKeys = null) => {+ const add = (+ story,+ theme,+ componentKeys = null,+ existingBaselineOnly = false,+ ) => {for (const mode of ['light', 'dark']) {const shot = {storyId: story.storyId ?? story.id,@@ -760,6 +768,7 @@ function trustedPlan() {reasons: ['trusted:pr-scope'],};const keyed = {...shot, key: shotKey(shot)};+ if (existingBaselineOnly && !baselineKeys.has(keyed.key)) continue;shots.push(keyed);componentKeys?.add(keyed.key);}@@ -770,10 +779,12 @@ function trustedPlan() {indexed,scope.stableComponents,)) {- add(story, config.defaultTheme, componentKeys);+ add(story, config.defaultTheme, componentKeys, true);if (!useThemeMatrix) continue;for (const theme of baselineThemes) {- if (theme !== config.defaultTheme) add(story, theme, componentKeys);+ if (theme !== config.defaultTheme) {+ add(story, theme, componentKeys, true);+ }}}if (@@ -802,10 +813,12 @@ function trustedPlan() {if (unique.some(shot => shot.stableThemeVisual !== true)) {fail('trusted stable plan includes a private or canary theme');}- if (- unique.length === 0 ||- unique.length > config.visualPlanSafetyLimit- ) {+ if (unique.length === 0) {+ fail(+ 'trusted component scope has no existing baseline frames; seed coverage through the manual baseline workflow',+ );+ }+ if (unique.length > config.visualPlanSafetyLimit) {fail(`trusted visual plan has invalid size ${unique.length}; safety limit is ${config.visualPlanSafetyLimit}`,);@@ -817,8 +830,10 @@ function trustedPlan() {}function acceptedStableShots(acceptance) {- if (acceptance.keys.some(entry => entry.kind === 'removed')) {- fail('scoped PR promotion cannot remove baseline keys');+ if (acceptance.keys.some(entry => entry.kind !== 'changed')) {+ fail(+ 'scoped PR promotion can only update existing baseline frames',+ );}const shots = withThemeMetadata(acceptance.keys.map(entry => ({...entry.shot, key: entry.key})),diff --git a/.github/scripts/visual-gate/visual-acceptance.test.mjs b/.github/scripts/visual-gate/visual-acceptance.test.mjsindex 4ed1086560..2535decc07 100644--- a/.github/scripts/visual-gate/visual-acceptance.test.mjs+++ b/.github/scripts/visual-gate/visual-acceptance.test.mjs@@ -483,6 +483,14 @@ describe('visual acceptance', () => {);});+ it('refuses to accept an added baseline frame from a scoped PR', () => {+ const key = 'core-button--new__neutral-light';+ writeEvidence({kind: 'added', key});+ expect(fail('accept', acceptanceFlags())).toMatch(+ /changes to existing baseline frames/,+ );+ });+it('returns success for a clean trusted capture without acceptance', () => {writeEvidence({run: 124, status: 'pass'});expect(JSON.parse(run('state', {pages, pr: 42, head: HEAD}))).toMatchObject(@@ -564,15 +572,7 @@ describe('visual acceptance', () => {JSON.parse(fs.readFileSync(output, 'utf8')).map(shot => shot.key),).toEqual(['core-button--default__neutral-light',- 'core-button--default__neutral-dark','core-button--default__y2k-light',- 'core-button--default__y2k-dark',- 'core-button--separator__neutral-light',- 'core-button--separator__neutral-dark',- 'core-button--variants__neutral-light',- 'core-button--variants__neutral-dark',- 'core-button--variants__y2k-light',- 'core-button--variants__y2k-dark',]);});@@ -630,12 +630,22 @@ describe('visual acceptance', () => {'manifest.json',);const baseline = JSON.parse(fs.readFileSync(baselineFile, 'utf8'));- baseline.shots['core-button--default__y2k-light'] = {- ...baseline.shots[KEY],- key: 'core-button--default__y2k-light',- theme: 'y2k',- mode: 'light',- };+ for (let index = 0; index < 61; index += 1) {+ const storyId = `core-button--story-${index}`;+ for (const theme of ['neutral', 'y2k']) {+ for (const mode of ['light', 'dark']) {+ const key = `${storyId}__${theme}-${mode}`;+ baseline.shots[key] = {+ ...baseline.shots[KEY],+ key,+ storyId,+ name: entries[`story-${index}`].name,+ theme,+ mode,+ };+ }+ }+ }writeJSON(baselineFile, baseline);const scope = path.join(root, 'scope-shot-limit.json');writeJSON(scope, {@@ -938,7 +948,7 @@ describe('visual acceptance', () => {expect(printed).toContain('520 shots');});- it('accepts a trusted 520-delta broad bundle', () => {+ it('rejects a trusted 520-delta added bundle', () => {const runId = 130;const dir = path.join(pages,@@ -977,9 +987,10 @@ describe('visual acceptance', () => {deltas,verdict: {version: 1, status: 'changed'},});- run('accept', acceptanceFlags({'run-id': runId}));- const record = JSON.parse(fs.readFileSync(acceptanceFile(runId), 'utf8'));- expect(record.keys).toHaveLength(520);+ expect(+ fail('accept', acceptanceFlags({'run-id': runId})),+ ).toMatch(/changes to existing baseline frames/);+ expect(fs.existsSync(acceptanceFile(runId))).toBe(false);});it('emits an exact recapture plan without allowing stored metadata to replace the key', () => {@@ -1058,74 +1069,69 @@ describe('visual acceptance', () => {});});- it.each([- {kind: 'changed', key: KEY, runId: 123},- {kind: 'added', key: 'core-new--default__neutral-light', runId: 124},- ])(- 'canonicalizes equivalent raw PNG bytes when promoting a $kind shot',- ({kind, key, runId}) => {- if (kind === 'added') writeEvidence({kind, key, run: runId});- run('accept', acceptanceFlags({'run-id': runId}));- const record = acceptanceFile(runId);- const accepted = fs.readFileSync(- path.join(path.dirname(record), 'after', `${key}.png`),- );- const decoded = PNG.sync.read(accepted);- decoded.gamma = 0.45455;- const rawRecapture = PNG.sync.write(decoded, {deflateLevel: 0});- expect(rawRecapture).not.toEqual(accepted);- expect(PNG.sync.read(rawRecapture).data).toEqual(- PNG.sync.read(accepted).data,- );- expect(canonicalizePng(rawRecapture).bytes).toEqual(accepted);+ it('canonicalizes equivalent raw PNG bytes when promoting a changed shot', () => {+ const key = KEY;+ const runId = 123;+ run('accept', acceptanceFlags({'run-id': runId}));+ const record = acceptanceFile(runId);+ const accepted = fs.readFileSync(+ path.join(path.dirname(record), 'after', `${key}.png`),+ );+ const decoded = PNG.sync.read(accepted);+ decoded.gamma = 0.45455;+ const rawRecapture = PNG.sync.write(decoded, {deflateLevel: 0});+ expect(rawRecapture).not.toEqual(accepted);+ expect(PNG.sync.read(rawRecapture).data).toEqual(+ PNG.sync.read(accepted).data,+ );+ expect(canonicalizePng(rawRecapture).bytes).toEqual(accepted);- const capture = path.join(root, `capture-${kind}`);- fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});- fs.writeFileSync(path.join(capture, 'shots', `${key}.png`), rawRecapture);- writeJSON(path.join(capture, 'manifest.json'), {- version: 1,- platform: 'linux-arm64',- browser: 'chromium-140.0',- viewport: {width: 1280, height: 900},- capturedAt: '2026-08-26T22:00:00.000Z',- context: {sha: MERGE},- shots: {- [key]: {- ...SHOT,- sha256: digest(rawRecapture),- width: 2,- height: 2,- },+ const capture = path.join(root, 'capture-changed');+ fs.mkdirSync(path.join(capture, 'shots'), {recursive: true});+ fs.writeFileSync(path.join(capture, 'shots', `${key}.png`), rawRecapture);+ writeJSON(path.join(capture, 'manifest.json'), {+ version: 1,+ platform: 'linux-arm64',+ browser: 'chromium-140.0',+ viewport: {width: 1280, height: 900},+ capturedAt: '2026-08-26T22:00:00.000Z',+ context: {sha: MERGE},+ shots: {+ [key]: {+ ...SHOT,+ sha256: digest(rawRecapture),+ width: 2,+ height: 2,},- });+ },+ });- expect(- run('promote', {- pages,- acceptance: record,- capture,- 'merge-sha': MERGE,- }),- ).toContain('Promoted accepted visual bundle');- expect(- fs.readFileSync(- path.join(pages, 'visual-gate', 'baseline', 'shots', `${key}.png`),- ),- ).toEqual(accepted);- const baselineManifest = JSON.parse(- fs.readFileSync(- path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),- 'utf8',- ),- );- expect(baselineManifest.shots[key].sha256).toBe(digest(accepted));- },- );+ expect(+ run('promote', {+ pages,+ acceptance: record,+ capture,+ 'merge-sha': MERGE,+ }),+ ).toContain('Promoted accepted visual bundle');+ expect(+ fs.readFileSync(+ path.join(pages, 'visual-gate', 'baseline', 'shots', `${key}.png`),+ ),+ ).toEqual(accepted);+ const baselineManifest = JSON.parse(+ fs.readFileSync(+ path.join(pages, 'visual-gate', 'baseline', 'manifest.json'),+ 'utf8',+ ),+ );+ expect(baselineManifest.shots[key].sha256).toBe(digest(accepted));+ });it('rejects removed PR evidence and preserves the baseline', () => {writeEvidence({kind: 'removed', run: 125});expect(fail('accept', acceptanceFlags({'run-id': 125}))).toMatch(- /scoped PR acceptance cannot authorize baseline removals/,+ /changes to existing baseline frames/,);expect(fs.existsSync(acceptanceFile(125))).toBe(false);expect(fs.existsSync(path.join(pages, 'visual-gate', 'baseline', 'shots', `${KEY}.png`))).toBe(true);@@ -1220,19 +1226,18 @@ describe('visual acceptance', () => {).toMatch(/baseline conflict/);});- it('records added shots but rejects removed evidence', () => {+ it('rejects both added and removed scoped PR evidence', () => {const added = 'core-new--default__neutral-light';writeEvidence({kind: 'added', key: added, run: 124});- run('accept', acceptanceFlags({'run-id': 124}));- expect(JSON.parse(fs.readFileSync(acceptanceFile(124), 'utf8')).keys[0]).toMatchObject({- key: added,- kind: 'added',- beforeSha256: null,- });- fs.rmSync(path.join(pages, 'visual-gate', 'acceptances'), {recursive: true, force: true});+ expect(+ fail('accept', acceptanceFlags({'run-id': 124})),+ ).toMatch(/changes to existing baseline frames/);+ expect(fs.existsSync(acceptanceFile(124))).toBe(false);+writeEvidence({kind: 'removed', run: 125});- expect(fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125}))).toMatch(- /scoped PR acceptance cannot authorize baseline removals/,- );+ expect(+ fail('accept', acceptanceFlags({'comment-id': 1235, 'run-id': 125})),+ ).toMatch(/changes to existing baseline frames/);+ expect(fs.existsSync(acceptanceFile(125))).toBe(false);});});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 052a3b06a2..752c98e4e7 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -626,11 +626,12 @@ jobs:- name: Verify guest fixture contrastrun: pnpm -F @astryxdesign/vibe-tests fixtures:contrast- # Stable-package visual regression. Core components get one representative- # story in every accepted baseline theme. `visual-baseline` adds default-theme- # stories; `visual-theme-matrix` adds all-theme stories. A changed shipped- # theme gets its relevant matrix. Packages marked `astryx.canaryOnly` (Lab,- # richtext, vega) do not compare pixels and do not own a baseline.+ # Stable-package visual regression. Core component changes compare only+ # representative or explicitly tagged story keys already present in the+ # accepted baseline. A changed shipped theme gets its relevant matrix, but PR+ # acceptance cannot add or remove baseline keys. Packages marked+ # `astryx.canaryOnly` (Lab, richtext, vega) do not compare pixels and do not+ # own a baseline.## Deeper than the daily gate only where the component declares it: audit and# interaction fixtures stay available to their dedicated checks without