Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): make visual capture plan-order independent (#5837)

Cindy Zhang237d82a6b012
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index 82e89964f4..ab947a733e 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -185,11 +185,13 @@ it changes those shots, so it is a deliberate rebaseline, not a tweak.
Theme and colour mode are switched over Storybook's own channel rather than
by reloading. Two workers scout and capture independent story groups
-concurrently, but all shots for one story stay together so seeded random state
-and fast-global ordering remain deterministic. After capture closes the browser,
-two CPU workers decode and compare the baseline PNGs. `--no-fast-globals`
-forces a reload per shot if a story's state ever turns out to survive the
-re-render.
+concurrently. Browser execution is canonicalized by story, and every story
+mounts in the default light theme before any fast-global update, so accepted
+exact plans and full release plans cannot seed mount-time state differently.
+The manifest still follows the authoritative plan order. After capture closes
+the browser, two CPU workers decode and compare the baseline PNGs.
+`--no-fast-globals` forces a reload per shot if a story's state ever turns out
+to survive the re-render.
## Where the images live
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index b61b0e23d0..6bc6c913e6 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -250,6 +250,7 @@ async function runCapture(shots, releasePlan = null) {
viewport: config.viewport,
settleMs: config.settleMs,
fastGlobals: !has('no-fast-globals'),
+ bootstrapGlobals: {astryxTheme: config.defaultTheme, colorMode: 'light'},
concurrency: config.captureConcurrency,
onProgress: ({done, total}) => {
if (done === total || done - last >= 50) {
diff --git a/.github/scripts/visual-gate/lib/capture.mjs b/.github/scripts/visual-gate/lib/capture.mjs
index 64dc642461..82460d7e0b 100644
--- a/.github/scripts/visual-gate/lib/capture.mjs
+++ b/.github/scripts/visual-gate/lib/capture.mjs
@@ -24,11 +24,12 @@
* re-render, not a reload: the plan is walked grouped by story, and the theme
* and color mode are switched over Storybook's own channel while the page
* stays put (~130ms per shot against ~1s for a reload). Independent story
- * groups run on a small fixed worker pool; one story never crosses workers, so
- * its seeded random state and fast-global sequence remain deterministic.
- * `--no-fast-globals` forces a reload per shot for the case where a story's own
- * state would survive the re-render and make the shot depend on the one before
- * it.
+ * groups run on a small fixed worker pool. Their browser execution is
+ * canonicalized by story and starts in the default light theme before any fast
+ * global update, so an accepted exact plan and the full release plan cannot
+ * seed mount-time state differently. `--no-fast-globals` forces a reload per
+ * shot for the case where a story's own state would survive the re-render and
+ * make the shot depend on the one before it.
*/
import * as fs from 'node:fs';
@@ -153,7 +154,7 @@ export const CAPTURE_CONTEXT_SECURITY = {serviceWorkers: 'block'};
* @param {number} concurrency
* @returns {import('./plan.mjs').Shot[][]}
*/
-export function partitionCapturePlan(plan, concurrency) {
+export function partitionCapturePlan(plan, concurrency, bootstrapGlobals = null) {
if (plan.length === 0) return [];
const workerCount = Math.max(1, Math.min(Math.floor(concurrency), plan.length));
const groups = [];
@@ -168,6 +169,24 @@ export function partitionCapturePlan(plan, concurrency) {
group.push(shot);
}
+ // An accepted PR plan follows baseline insertion order, while the canonical
+ // release plan is key-sorted. Fast globals preserve mounted story state, so
+ // execution order must come from the shots themselves rather than whichever
+ // caller supplied them. The manifest is still written in authoritative plan
+ // order; this ordering is only for browser work.
+ if (bootstrapGlobals) {
+ const isBootstrap = shot =>
+ shot.theme === bootstrapGlobals.astryxTheme &&
+ shot.mode === bootstrapGlobals.colorMode;
+ groups.sort((a, b) => a[0].storyId.localeCompare(b[0].storyId));
+ for (const group of groups) {
+ group.sort((a, b) => {
+ const bootstrap = Number(isBootstrap(b)) - Number(isBootstrap(a));
+ return bootstrap || a.key.localeCompare(b.key);
+ });
+ }
+ }
+
const partitions = Array.from({length: workerCount}, () => []);
const loads = Array(workerCount).fill(0);
for (const group of groups) {
@@ -181,6 +200,18 @@ export function partitionCapturePlan(plan, concurrency) {
return partitions.filter(partition => partition.length > 0);
}
+export function storyLoadGlobals(shot, fastGlobals, bootstrapGlobals) {
+ const requested = {astryxTheme: shot.theme, colorMode: shot.mode};
+ const initial = fastGlobals ? bootstrapGlobals : requested;
+ return {
+ initial,
+ requested,
+ needsUpdate:
+ initial.astryxTheme !== requested.astryxTheme ||
+ initial.colorMode !== requested.colorMode,
+ };
+}
+
export function serveDirectory(dir) {
const root = fs.realpathSync(dir);
const server = http.createServer((req, res) => {
@@ -447,6 +478,7 @@ async function capturePartition({
viewport,
settleMs,
fastGlobals,
+ bootstrapGlobals,
onShot,
}) {
const context = await browser.newContext({
@@ -472,16 +504,32 @@ async function capturePartition({
try {
for (const shot of plan) {
try {
- const globals = {astryxTheme: shot.theme, colorMode: shot.mode};
+ const {initial, requested: globals, needsUpdate} = storyLoadGlobals(
+ shot,
+ fastGlobals,
+ bootstrapGlobals,
+ );
const needsLoad = !fastGlobals || currentStory !== shot.storyId;
if (needsLoad) {
- const globalsParam = `astryxTheme:${shot.theme};colorMode:${shot.mode}`;
+ const globalsParam = `astryxTheme:${initial.astryxTheme};colorMode:${initial.colorMode}`;
await page.goto(
`${origin}/iframe.html?id=${encodeURIComponent(shot.storyId)}&viewMode=story&globals=${globalsParam}`,
{waitUntil: 'load', timeout: 30000},
);
await page.waitForSelector('#storybook-root > *', {timeout: 30000});
currentStory = shot.storyId;
+ if (needsUpdate) {
+ // Let mount-time state settle in one canonical environment before a
+ // fast global update. Otherwise a default-open layer remembers the
+ // first theme in the caller's plan and every later shot inherits it.
+ await page.addStyleTag({content: FREEZE_CSS});
+ await settle(page, settleMs);
+ await verifyApplied(page, {
+ theme: initial.astryxTheme,
+ mode: initial.colorMode,
+ });
+ await applyGlobals(page, globals);
+ }
} else {
await applyGlobals(page, globals);
}
@@ -535,6 +583,7 @@ async function capturePartition({
* @param {{width: number, height: number}} options.viewport
* @param {number} options.settleMs
* @param {boolean} options.fastGlobals
+ * @param {{astryxTheme: string, colorMode: string}} options.bootstrapGlobals
* @param {number} [options.concurrency]
* @param {(progress: {done: number, total: number, key: string}) => void} [options.onProgress]
* @returns {Promise<{manifest: object, failures: Array<{key: string, error: string}>}>}
@@ -546,6 +595,7 @@ export async function capture({
viewport,
settleMs,
fastGlobals,
+ bootstrapGlobals,
concurrency = 1,
onProgress,
}) {
@@ -561,7 +611,7 @@ export async function capture({
const browserVersion = browser.version();
let done = 0;
const results = await Promise.all(
- partitionCapturePlan(plan, concurrency).map(partition =>
+ partitionCapturePlan(plan, concurrency, bootstrapGlobals).map(partition =>
capturePartition({
plan: partition,
browser,
@@ -570,6 +620,7 @@ export async function capture({
viewport,
settleMs,
fastGlobals,
+ bootstrapGlobals,
onShot: key => onProgress?.({done: ++done, total: plan.length, key}),
}),
),
diff --git a/.github/scripts/visual-gate/lib/capture.test.mjs b/.github/scripts/visual-gate/lib/capture.test.mjs
index 949d443151..794a6b1313 100644
--- a/.github/scripts/visual-gate/lib/capture.test.mjs
+++ b/.github/scripts/visual-gate/lib/capture.test.mjs
@@ -14,6 +14,7 @@ import {
partitionCapturePlan,
partitionScoutStories,
serveDirectory,
+ storyLoadGlobals,
} from './capture.mjs';
const roots = [];
@@ -55,6 +56,56 @@ describe('capture plan partitioning', () => {
expect(new Set(partitions.flat().map(shot => shot.key)).size).toBe(3378);
});
+ it('canonicalizes story and global order without changing the shot set', () => {
+ const bootstrap = {astryxTheme: 'neutral', colorMode: 'light'};
+ const shot = (storyId, theme, mode) => ({
+ key: `${storyId}__${theme}-${mode}`,
+ storyId,
+ theme,
+ mode,
+ });
+ const accepted = [
+ shot('radio', 'neutral', 'light'),
+ shot('radio', 'neutral', 'dark'),
+ shot('default', 'neutral', 'light'),
+ shot('default', 'neutral', 'dark'),
+ shot('default', 'butter', 'light'),
+ shot('default', 'butter', 'dark'),
+ ];
+ const release = [
+ shot('default', 'butter', 'dark'),
+ shot('default', 'butter', 'light'),
+ shot('default', 'neutral', 'dark'),
+ shot('default', 'neutral', 'light'),
+ shot('radio', 'neutral', 'dark'),
+ shot('radio', 'neutral', 'light'),
+ ];
+
+ const sequence = plan =>
+ partitionCapturePlan(plan, 2, bootstrap).map(partition =>
+ partition.map(candidate => candidate.key),
+ );
+ expect(sequence(accepted)).toEqual(sequence(release));
+ expect(new Set(partitionCapturePlan(release, 2, bootstrap).flat())).toEqual(
+ new Set(release),
+ );
+ });
+
+ it('mounts fast-global stories in the canonical environment first', () => {
+ const bootstrap = {astryxTheme: 'neutral', colorMode: 'light'};
+ const shot = {theme: 'butter', mode: 'dark'};
+ expect(storyLoadGlobals(shot, true, bootstrap)).toEqual({
+ initial: bootstrap,
+ requested: {astryxTheme: 'butter', colorMode: 'dark'},
+ needsUpdate: true,
+ });
+ expect(storyLoadGlobals(shot, false, bootstrap)).toEqual({
+ initial: {astryxTheme: 'butter', colorMode: 'dark'},
+ requested: {astryxTheme: 'butter', colorMode: 'dark'},
+ needsUpdate: false,
+ });
+ });
+
it('uses one worker for empty or single-worker plans and never splits interleaved stories', () => {
expect(partitionCapturePlan([], 2)).toEqual([]);
const plan = [