Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): parallelize visual capture by story (#5828)

Cindy Zhang4c302ae0dce0
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index 533e230d2a..3ac48ae4f5 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -181,9 +181,10 @@ instant lives in `lib/capture.mjs` and is recorded in every manifest. Changing
it changes those shots, so it is a deliberate rebaseline, not a tweak.
Theme and colour mode are switched over Storybook's own channel rather than
-by reloading, which is what keeps 514 shots inside a few minutes;
-`--no-fast-globals` forces a reload per shot if a story's state ever turns out
-to survive the re-render.
+by reloading. Two workers capture independent story groups concurrently, but
+all shots for one story stay together so seeded random state and fast-global
+ordering remain deterministic. `--no-fast-globals` forces a reload per shot if
+a story's state ever turns out to survive the re-render.
## Where the images live
diff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjs
index 4e4232d742..abefb54aff 100644
--- a/.github/scripts/visual-gate/gate.mjs
+++ b/.github/scripts/visual-gate/gate.mjs
@@ -249,6 +249,7 @@ async function runCapture(shots, releasePlan = null) {
viewport: config.viewport,
settleMs: config.settleMs,
fastGlobals: !has('no-fast-globals'),
+ concurrency: config.captureConcurrency,
onProgress: ({done, total}) => {
if (done === total || done - last >= 50) {
last = done;
diff --git a/.github/scripts/visual-gate/lib/capture.mjs b/.github/scripts/visual-gate/lib/capture.mjs
index 97b2e87a5c..3764b4e8aa 100644
--- a/.github/scripts/visual-gate/lib/capture.mjs
+++ b/.github/scripts/visual-gate/lib/capture.mjs
@@ -23,9 +23,12 @@
* Speed. Storybook applies the theme through a global, so a theme change is a
* re-render, not a reload: the plan is walked grouped by story, and the theme
* and color mode are switched over Storybook's own channel while the page
- * stays put (~130ms per shot against ~1s for a reload). `--no-fast-globals`
- * forces a reload per shot for the case where a story's own state would
- * survive the re-render and make the shot depend on the one before it.
+ * stays put (~130ms per shot against ~1s for a reload). Independent story
+ * groups run on a small fixed worker pool; one story never crosses workers, so
+ * its seeded random state and fast-global sequence remain deterministic.
+ * `--no-fast-globals` forces a reload per shot for the case where a story's own
+ * state would survive the re-render and make the shot depend on the one before
+ * it.
*/
import * as fs from 'node:fs';
@@ -138,6 +141,46 @@ export async function blockExternalNetwork(context, origin) {
export const CAPTURE_CONTEXT_SECURITY = {serviceWorkers: 'block'};
+/**
+ * Split a capture plan across workers without ever splitting one story.
+ *
+ * Story boundaries are load-bearing: theme and mode changes reuse one loaded
+ * Storybook document, and every navigation resets the seeded PRNG. Keeping all
+ * shots for a story together preserves both fast globals and deterministic
+ * random state while letting independent stories run in parallel.
+ *
+ * @param {import('./plan.mjs').Shot[]} plan
+ * @param {number} concurrency
+ * @returns {import('./plan.mjs').Shot[][]}
+ */
+export function partitionCapturePlan(plan, concurrency) {
+ if (plan.length === 0) return [];
+ const workerCount = Math.max(1, Math.min(Math.floor(concurrency), plan.length));
+ const groups = [];
+ const groupByStory = Object.create(null);
+ for (const shot of plan) {
+ let group = groupByStory[shot.storyId];
+ if (!group) {
+ group = [];
+ groupByStory[shot.storyId] = group;
+ groups.push(group);
+ }
+ group.push(shot);
+ }
+
+ const partitions = Array.from({length: workerCount}, () => []);
+ const loads = Array(workerCount).fill(0);
+ for (const group of groups) {
+ let worker = 0;
+ for (let index = 1; index < workerCount; index += 1) {
+ if (loads[index] < loads[worker]) worker = index;
+ }
+ partitions[worker].push(...group);
+ loads[worker] += group.length;
+ }
+ return partitions.filter(partition => partition.length > 0);
+}
+
export function serveDirectory(dir) {
const root = fs.realpathSync(dir);
const server = http.createServer((req, res) => {
@@ -364,34 +407,16 @@ export async function scout({storyIds, storybookDir, theme, viewport, onProgress
return observations;
}
-/**
- * @param {object} options
- * @param {import('./plan.mjs').Shot[]} options.plan
- * @param {string} options.storybookDir
- * @param {string} options.outDir
- * @param {{width: number, height: number}} options.viewport
- * @param {number} options.settleMs
- * @param {boolean} options.fastGlobals
- * @param {(progress: {done: number, total: number, key: string}) => void} [options.onProgress]
- * @returns {Promise<{manifest: object, failures: Array<{key: string, error: string}>}>}
- */
-export async function capture({
+async function capturePartition({
plan,
- storybookDir,
- outDir,
+ browser,
+ origin,
+ shotsDir,
viewport,
settleMs,
fastGlobals,
- onProgress,
+ onShot,
}) {
- const {chromium} = await import('playwright');
- const shotsDir = path.join(outDir, 'shots');
- fs.mkdirSync(shotsDir, {recursive: true});
-
- const server = await serveDirectory(storybookDir);
- const origin = `http://127.0.0.1:${server.port}`;
- const browser = await chromium.launch();
- const browserVersion = browser.version();
const context = await browser.newContext({
viewport,
deviceScaleFactor: 1,
@@ -403,92 +428,159 @@ export async function capture({
await context.clock.setFixedTime(FROZEN_NOW);
await context.addInitScript(BACKGROUND_NETWORK_GUARD);
await context.addInitScript(SEEDED_RANDOM);
- // Anything off-origin is a determinism hazard, and nothing in a component
- // story legitimately needs it.
await blockExternalNetwork(context, origin);
const page = await context.newPage();
await page.addStyleTag({content: FREEZE_CSS}).catch(() => {});
- /** @type {Record<string, {sha256: string, width: number, height: number, storyId: string, theme: string, mode: string, reasons: string[]}>} */
const shots = {};
- /** @type {Record<string, Set<string>>} */
const observed = {};
- /** @type {Array<{key: string, error: string}>} */
const failures = [];
-
let currentStory = null;
- let done = 0;
- for (const shot of plan) {
- try {
- const globals = {astryxTheme: shot.theme, colorMode: shot.mode};
- const needsLoad = !fastGlobals || currentStory !== shot.storyId;
- if (needsLoad) {
- const globalsParam = `astryxTheme:${shot.theme};colorMode:${shot.mode}`;
- await page.goto(
- `${origin}/iframe.html?id=${encodeURIComponent(shot.storyId)}&viewMode=story&globals=${globalsParam}`,
- {waitUntil: 'load', timeout: 30000},
- );
- await page.waitForSelector('#storybook-root > *', {timeout: 30000});
- currentStory = shot.storyId;
- } else {
- await applyGlobals(page, globals);
+ try {
+ for (const shot of plan) {
+ try {
+ const globals = {astryxTheme: shot.theme, colorMode: shot.mode};
+ const needsLoad = !fastGlobals || currentStory !== shot.storyId;
+ if (needsLoad) {
+ const globalsParam = `astryxTheme:${shot.theme};colorMode:${shot.mode}`;
+ await page.goto(
+ `${origin}/iframe.html?id=${encodeURIComponent(shot.storyId)}&viewMode=story&globals=${globalsParam}`,
+ {waitUntil: 'load', timeout: 30000},
+ );
+ await page.waitForSelector('#storybook-root > *', {timeout: 30000});
+ currentStory = shot.storyId;
+ } else {
+ await applyGlobals(page, globals);
+ }
+ await page.addStyleTag({content: FREEZE_CSS});
+ await settle(page, settleMs);
+ await verifyApplied(page, {theme: shot.theme, mode: shot.mode});
+
+ const png = await page.screenshot({fullPage: true, animations: 'disabled'});
+ fs.writeFileSync(path.join(shotsDir, `${shot.key}.png`), png);
+ const size = await page.evaluate(() => ({
+ width: document.documentElement.scrollWidth,
+ height: document.documentElement.scrollHeight,
+ }));
+ shots[shot.key] = {
+ sha256: crypto.createHash('sha256').update(png).digest('hex'),
+ width: size.width,
+ height: size.height,
+ storyId: shot.storyId,
+ title: shot.title,
+ name: shot.name,
+ component: shot.component,
+ theme: shot.theme,
+ mode: shot.mode,
+ reasons: shot.reasons,
+ };
+
+ for (const [key, values] of Object.entries(await observeTargets(page))) {
+ observed[key] ??= new Set();
+ for (const value of values) observed[key].add(value);
+ }
+ } catch (error) {
+ failures.push({key: shot.key, error: String(error?.message ?? error).slice(0, 400)});
+ // A story that will not render poisons the page for the next shot in the
+ // same story group; force a fresh navigation on the next iteration.
+ currentStory = null;
}
- await page.addStyleTag({content: FREEZE_CSS});
- await settle(page, settleMs);
- await verifyApplied(page, {theme: shot.theme, mode: shot.mode});
-
- const png = await page.screenshot({fullPage: true, animations: 'disabled'});
- fs.writeFileSync(path.join(shotsDir, `${shot.key}.png`), png);
- const size = await page.evaluate(() => ({
- width: document.documentElement.scrollWidth,
- height: document.documentElement.scrollHeight,
- }));
- shots[shot.key] = {
- sha256: crypto.createHash('sha256').update(png).digest('hex'),
- width: size.width,
- height: size.height,
- storyId: shot.storyId,
- title: shot.title,
- name: shot.name,
- component: shot.component,
- theme: shot.theme,
- mode: shot.mode,
- reasons: shot.reasons,
- };
+ onShot(shot.key);
+ }
+ } finally {
+ await context.close();
+ }
+
+ return {shots, observed, failures};
+}
- for (const [key, values] of Object.entries(await observeTargets(page))) {
+/**
+ * @param {object} options
+ * @param {import('./plan.mjs').Shot[]} options.plan
+ * @param {string} options.storybookDir
+ * @param {string} options.outDir
+ * @param {{width: number, height: number}} options.viewport
+ * @param {number} options.settleMs
+ * @param {boolean} options.fastGlobals
+ * @param {number} [options.concurrency]
+ * @param {(progress: {done: number, total: number, key: string}) => void} [options.onProgress]
+ * @returns {Promise<{manifest: object, failures: Array<{key: string, error: string}>}>}
+ */
+export async function capture({
+ plan,
+ storybookDir,
+ outDir,
+ viewport,
+ settleMs,
+ fastGlobals,
+ concurrency = 1,
+ onProgress,
+}) {
+ const {chromium} = await import('playwright');
+ const shotsDir = path.join(outDir, 'shots');
+ fs.mkdirSync(shotsDir, {recursive: true});
+
+ const server = await serveDirectory(storybookDir);
+ const origin = `http://127.0.0.1:${server.port}`;
+ let browser;
+ try {
+ browser = await chromium.launch();
+ const browserVersion = browser.version();
+ let done = 0;
+ const results = await Promise.all(
+ partitionCapturePlan(plan, concurrency).map(partition =>
+ capturePartition({
+ plan: partition,
+ browser,
+ origin,
+ shotsDir,
+ viewport,
+ settleMs,
+ fastGlobals,
+ onShot: key => onProgress?.({done: ++done, total: plan.length, key}),
+ }),
+ ),
+ );
+
+ const captured = Object.assign({}, ...results.map(result => result.shots));
+ const shots = Object.fromEntries(
+ plan.filter(shot => captured[shot.key]).map(shot => [shot.key, captured[shot.key]]),
+ );
+ const observed = {};
+ for (const result of results) {
+ for (const [key, values] of Object.entries(result.observed)) {
observed[key] ??= new Set();
for (const value of values) observed[key].add(value);
}
- } catch (error) {
- failures.push({key: shot.key, error: String(error?.message ?? error).slice(0, 400)});
- // A story that will not render poisons the page for the next shot in the
- // same story group.
- currentStory = null;
}
- done += 1;
- onProgress?.({done, total: plan.length, key: shot.key});
- }
- await browser.close();
- await server.close();
+ const failuresByKey = Object.fromEntries(
+ results.flatMap(result => result.failures).map(failure => [failure.key, failure]),
+ );
+ const failures = plan.flatMap(shot =>
+ failuresByKey[shot.key] ? [failuresByKey[shot.key]] : [],
+ );
- return {
- manifest: {
- version: 1,
- platform: `${process.platform}-${process.arch}`,
- browser: `chromium-${browserVersion}`,
- viewport,
- settleMs,
- frozenClock: FROZEN_NOW.toISOString(),
- timezoneId: TIMEZONE_ID,
- capturedAt: new Date().toISOString(),
- shots,
- observedTargets: Object.fromEntries(
- Object.entries(observed).map(([key, values]) => [key, [...values].sort()]),
- ),
- },
- failures,
- };
+ return {
+ manifest: {
+ version: 1,
+ platform: `${process.platform}-${process.arch}`,
+ browser: `chromium-${browserVersion}`,
+ viewport,
+ settleMs,
+ frozenClock: FROZEN_NOW.toISOString(),
+ timezoneId: TIMEZONE_ID,
+ capturedAt: new Date().toISOString(),
+ shots,
+ observedTargets: Object.fromEntries(
+ Object.entries(observed).map(([key, values]) => [key, [...values].sort()]),
+ ),
+ },
+ failures,
+ };
+ } finally {
+ await browser?.close().catch(() => {});
+ await server.close();
+ }
}
diff --git a/.github/scripts/visual-gate/lib/capture.test.mjs b/.github/scripts/visual-gate/lib/capture.test.mjs
index d52a9be953..34a90a1e6b 100644
--- a/.github/scripts/visual-gate/lib/capture.test.mjs
+++ b/.github/scripts/visual-gate/lib/capture.test.mjs
@@ -11,6 +11,7 @@ import {
CAPTURE_CONTEXT_SECURITY,
blockExternalNetwork,
isSameOrigin,
+ partitionCapturePlan,
serveDirectory,
} from './capture.mjs';
@@ -19,6 +20,49 @@ afterEach(() => {
for (const root of roots.splice(0)) fs.rmSync(root, {recursive: true, force: true});
});
+describe('capture plan partitioning', () => {
+ it('keeps the 3,378-shot release workload below 1,700 shots per worker', () => {
+ const plan = [];
+ for (let story = 0; story < 388; story += 1) {
+ const shots = story < 274 ? 9 : 8;
+ for (let index = 0; index < shots; index += 1) {
+ plan.push({key: `${story}-${index}`, storyId: `story-${story}`});
+ }
+ }
+
+ const partitions = partitionCapturePlan(plan, 2);
+ expect(partitions).toHaveLength(2);
+ expect(partitions.flat()).toHaveLength(3378);
+ expect(Math.max(...partitions.map(partition => partition.length))).toBeLessThan(1700);
+
+ const ownerByStory = {};
+ partitions.forEach((partition, worker) => {
+ for (const shot of partition) {
+ ownerByStory[shot.storyId] ??= worker;
+ expect(ownerByStory[shot.storyId]).toBe(worker);
+ }
+ });
+ expect(new Set(partitions.flat().map(shot => shot.key)).size).toBe(3378);
+ });
+
+ it('uses one worker for empty or single-worker plans and never splits interleaved stories', () => {
+ expect(partitionCapturePlan([], 2)).toEqual([]);
+ const plan = [
+ {key: 'a-light', storyId: 'a'},
+ {key: 'a-dark', storyId: 'a'},
+ {key: 'b-light', storyId: 'b'},
+ ];
+ expect(partitionCapturePlan(plan, 1)).toEqual([plan]);
+
+ const interleaved = [plan[0], plan[2], plan[1]];
+ const partitions = partitionCapturePlan(interleaved, 2);
+ const aWorkers = partitions
+ .map((partition, worker) => (partition.some(shot => shot.storyId === 'a') ? worker : null))
+ .filter(worker => worker !== null);
+ expect(aWorkers).toHaveLength(1);
+ });
+});
+
describe('capture network boundary', () => {
it('allows the exact Storybook origin', () => {
expect(isSameOrigin('http://127.0.0.1:6100/assets/story.js', 'http://127.0.0.1:6100')).toBe(
diff --git a/.github/scripts/visual-gate/lib/sources.mjs b/.github/scripts/visual-gate/lib/sources.mjs
index fe4178b95b..2aee4f2d9e 100644
--- a/.github/scripts/visual-gate/lib/sources.mjs
+++ b/.github/scripts/visual-gate/lib/sources.mjs
@@ -174,13 +174,14 @@ function unreadableTheme(repoRoot, name, built, error, rebuilt) {
/**
* @param {string} repoRoot
- * @returns {{excludeStories: Record<string, string>, viewport: {width: number, height: number}, settleMs: number, threshold: number, maxDiffPixels: number, defaultTheme: string, probeTheme: string, stableStoryPackages: string[], tiers: string[]}}
+ * @returns {{excludeStories: Record<string, string>, viewport: {width: number, height: number}, settleMs: number, captureConcurrency: number, threshold: number, maxDiffPixels: number, defaultTheme: string, probeTheme: string, stableStoryPackages: string[], tiers: string[]}}
*/
export function loadConfig(repoRoot) {
const defaults = {
excludeStories: {},
viewport: {width: 1024, height: 768},
settleMs: 50,
+ captureConcurrency: 2,
threshold: 0.1,
maxDiffPixels: 0,
defaultTheme: 'neutral',
diff --git a/.github/scripts/visual-gate/visual-gate.config.json b/.github/scripts/visual-gate/visual-gate.config.json
index b9c959b987..62ba98e6e8 100644
--- a/.github/scripts/visual-gate/visual-gate.config.json
+++ b/.github/scripts/visual-gate/visual-gate.config.json
@@ -8,6 +8,8 @@
"tiers": ["surface", "theme-matrix", "probe"],
"viewport": {"width": 1024, "height": 768},
"settleMs": 50,
+ "$captureConcurrency": "Independent stories may capture concurrently, but every shot for one story stays on one worker so seeded state and fast theme switches remain deterministic.",
+ "captureConcurrency": 2,
"threshold": 0.1,
"maxDiffPixels": 0,
"excludeStories": {