An open source design system that's fully customizable and agent ready
TypeScript
fix(ci): parallelize visual capture by story (#5828)
Cindy Zhang4c302ae0dce0
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.mdindex 533e230d2a..3ac48ae4f5 100644--- a/.github/scripts/visual-gate/README.md+++ b/.github/scripts/visual-gate/README.md@@ -181,9 +181,10 @@ instant lives in `lib/capture.mjs` and is recorded in every manifest. Changingit changes those shots, so it is a deliberate rebaseline, not a tweak.Theme and colour mode are switched over Storybook's own channel rather than-by reloading, which is what keeps 514 shots inside a few minutes;-`--no-fast-globals` forces a reload per shot if a story's state ever turns out-to survive the re-render.+by reloading. Two workers capture independent story groups concurrently, but+all shots for one story stay together so seeded random state and fast-global+ordering remain deterministic. `--no-fast-globals` forces a reload per shot if+a story's state ever turns out to survive the re-render.## Where the images livediff --git a/.github/scripts/visual-gate/gate.mjs b/.github/scripts/visual-gate/gate.mjsindex 4e4232d742..abefb54aff 100644--- a/.github/scripts/visual-gate/gate.mjs+++ b/.github/scripts/visual-gate/gate.mjs@@ -249,6 +249,7 @@ async function runCapture(shots, releasePlan = null) {viewport: config.viewport,settleMs: config.settleMs,fastGlobals: !has('no-fast-globals'),+ concurrency: config.captureConcurrency,onProgress: ({done, total}) => {if (done === total || done - last >= 50) {last = done;diff --git a/.github/scripts/visual-gate/lib/capture.mjs b/.github/scripts/visual-gate/lib/capture.mjsindex 97b2e87a5c..3764b4e8aa 100644--- a/.github/scripts/visual-gate/lib/capture.mjs+++ b/.github/scripts/visual-gate/lib/capture.mjs@@ -23,9 +23,12 @@* Speed. Storybook applies the theme through a global, so a theme change is a* re-render, not a reload: the plan is walked grouped by story, and the theme* and color mode are switched over Storybook's own channel while the page- * stays put (~130ms per shot against ~1s for a reload). `--no-fast-globals`- * forces a reload per shot for the case where a story's own state would- * survive the re-render and make the shot depend on the one before it.+ * stays put (~130ms per shot against ~1s for a reload). Independent story+ * groups run on a small fixed worker pool; one story never crosses workers, so+ * its seeded random state and fast-global sequence remain deterministic.+ * `--no-fast-globals` forces a reload per shot for the case where a story's own+ * state would survive the re-render and make the shot depend on the one before+ * it.*/import * as fs from 'node:fs';@@ -138,6 +141,46 @@ export async function blockExternalNetwork(context, origin) {export const CAPTURE_CONTEXT_SECURITY = {serviceWorkers: 'block'};+/**+ * Split a capture plan across workers without ever splitting one story.+ *+ * Story boundaries are load-bearing: theme and mode changes reuse one loaded+ * Storybook document, and every navigation resets the seeded PRNG. Keeping all+ * shots for a story together preserves both fast globals and deterministic+ * random state while letting independent stories run in parallel.+ *+ * @param {import('./plan.mjs').Shot[]} plan+ * @param {number} concurrency+ * @returns {import('./plan.mjs').Shot[][]}+ */+export function partitionCapturePlan(plan, concurrency) {+ if (plan.length === 0) return [];+ const workerCount = Math.max(1, Math.min(Math.floor(concurrency), plan.length));+ const groups = [];+ const groupByStory = Object.create(null);+ for (const shot of plan) {+ let group = groupByStory[shot.storyId];+ if (!group) {+ group = [];+ groupByStory[shot.storyId] = group;+ groups.push(group);+ }+ group.push(shot);+ }++ const partitions = Array.from({length: workerCount}, () => []);+ const loads = Array(workerCount).fill(0);+ for (const group of groups) {+ let worker = 0;+ for (let index = 1; index < workerCount; index += 1) {+ if (loads[index] < loads[worker]) worker = index;+ }+ partitions[worker].push(...group);+ loads[worker] += group.length;+ }+ return partitions.filter(partition => partition.length > 0);+}+export function serveDirectory(dir) {const root = fs.realpathSync(dir);const server = http.createServer((req, res) => {@@ -364,34 +407,16 @@ export async function scout({storyIds, storybookDir, theme, viewport, onProgressreturn observations;}-/**- * @param {object} options- * @param {import('./plan.mjs').Shot[]} options.plan- * @param {string} options.storybookDir- * @param {string} options.outDir- * @param {{width: number, height: number}} options.viewport- * @param {number} options.settleMs- * @param {boolean} options.fastGlobals- * @param {(progress: {done: number, total: number, key: string}) => void} [options.onProgress]- * @returns {Promise<{manifest: object, failures: Array<{key: string, error: string}>}>}- */-export async function capture({+async function capturePartition({plan,- storybookDir,- outDir,+ browser,+ origin,+ shotsDir,viewport,settleMs,fastGlobals,- onProgress,+ onShot,}) {- const {chromium} = await import('playwright');- const shotsDir = path.join(outDir, 'shots');- fs.mkdirSync(shotsDir, {recursive: true});-- const server = await serveDirectory(storybookDir);- const origin = `http://127.0.0.1:${server.port}`;- const browser = await chromium.launch();- const browserVersion = browser.version();const context = await browser.newContext({viewport,deviceScaleFactor: 1,@@ -403,92 +428,159 @@ export async function capture({await context.clock.setFixedTime(FROZEN_NOW);await context.addInitScript(BACKGROUND_NETWORK_GUARD);await context.addInitScript(SEEDED_RANDOM);- // Anything off-origin is a determinism hazard, and nothing in a component- // story legitimately needs it.await blockExternalNetwork(context, origin);const page = await context.newPage();await page.addStyleTag({content: FREEZE_CSS}).catch(() => {});- /** @type {Record<string, {sha256: string, width: number, height: number, storyId: string, theme: string, mode: string, reasons: string[]}>} */const shots = {};- /** @type {Record<string, Set<string>>} */const observed = {};- /** @type {Array<{key: string, error: string}>} */const failures = [];-let currentStory = null;- let done = 0;- for (const shot of plan) {- try {- const globals = {astryxTheme: shot.theme, colorMode: shot.mode};- const needsLoad = !fastGlobals || currentStory !== shot.storyId;- if (needsLoad) {- const globalsParam = `astryxTheme:${shot.theme};colorMode:${shot.mode}`;- await page.goto(- `${origin}/iframe.html?id=${encodeURIComponent(shot.storyId)}&viewMode=story&globals=${globalsParam}`,- {waitUntil: 'load', timeout: 30000},- );- await page.waitForSelector('#storybook-root > *', {timeout: 30000});- currentStory = shot.storyId;- } else {- await applyGlobals(page, globals);+ try {+ for (const shot of plan) {+ try {+ const globals = {astryxTheme: shot.theme, colorMode: shot.mode};+ const needsLoad = !fastGlobals || currentStory !== shot.storyId;+ if (needsLoad) {+ const globalsParam = `astryxTheme:${shot.theme};colorMode:${shot.mode}`;+ await page.goto(+ `${origin}/iframe.html?id=${encodeURIComponent(shot.storyId)}&viewMode=story&globals=${globalsParam}`,+ {waitUntil: 'load', timeout: 30000},+ );+ await page.waitForSelector('#storybook-root > *', {timeout: 30000});+ currentStory = shot.storyId;+ } else {+ await applyGlobals(page, globals);+ }+ await page.addStyleTag({content: FREEZE_CSS});+ await settle(page, settleMs);+ await verifyApplied(page, {theme: shot.theme, mode: shot.mode});++ const png = await page.screenshot({fullPage: true, animations: 'disabled'});+ fs.writeFileSync(path.join(shotsDir, `${shot.key}.png`), png);+ const size = await page.evaluate(() => ({+ width: document.documentElement.scrollWidth,+ height: document.documentElement.scrollHeight,+ }));+ shots[shot.key] = {+ sha256: crypto.createHash('sha256').update(png).digest('hex'),+ width: size.width,+ height: size.height,+ storyId: shot.storyId,+ title: shot.title,+ name: shot.name,+ component: shot.component,+ theme: shot.theme,+ mode: shot.mode,+ reasons: shot.reasons,+ };++ for (const [key, values] of Object.entries(await observeTargets(page))) {+ observed[key] ??= new Set();+ for (const value of values) observed[key].add(value);+ }+ } catch (error) {+ failures.push({key: shot.key, error: String(error?.message ?? error).slice(0, 400)});+ // A story that will not render poisons the page for the next shot in the+ // same story group; force a fresh navigation on the next iteration.+ currentStory = null;}- await page.addStyleTag({content: FREEZE_CSS});- await settle(page, settleMs);- await verifyApplied(page, {theme: shot.theme, mode: shot.mode});-- const png = await page.screenshot({fullPage: true, animations: 'disabled'});- fs.writeFileSync(path.join(shotsDir, `${shot.key}.png`), png);- const size = await page.evaluate(() => ({- width: document.documentElement.scrollWidth,- height: document.documentElement.scrollHeight,- }));- shots[shot.key] = {- sha256: crypto.createHash('sha256').update(png).digest('hex'),- width: size.width,- height: size.height,- storyId: shot.storyId,- title: shot.title,- name: shot.name,- component: shot.component,- theme: shot.theme,- mode: shot.mode,- reasons: shot.reasons,- };+ onShot(shot.key);+ }+ } finally {+ await context.close();+ }++ return {shots, observed, failures};+}- for (const [key, values] of Object.entries(await observeTargets(page))) {+/**+ * @param {object} options+ * @param {import('./plan.mjs').Shot[]} options.plan+ * @param {string} options.storybookDir+ * @param {string} options.outDir+ * @param {{width: number, height: number}} options.viewport+ * @param {number} options.settleMs+ * @param {boolean} options.fastGlobals+ * @param {number} [options.concurrency]+ * @param {(progress: {done: number, total: number, key: string}) => void} [options.onProgress]+ * @returns {Promise<{manifest: object, failures: Array<{key: string, error: string}>}>}+ */+export async function capture({+ plan,+ storybookDir,+ outDir,+ viewport,+ settleMs,+ fastGlobals,+ concurrency = 1,+ onProgress,+}) {+ const {chromium} = await import('playwright');+ const shotsDir = path.join(outDir, 'shots');+ fs.mkdirSync(shotsDir, {recursive: true});++ const server = await serveDirectory(storybookDir);+ const origin = `http://127.0.0.1:${server.port}`;+ let browser;+ try {+ browser = await chromium.launch();+ const browserVersion = browser.version();+ let done = 0;+ const results = await Promise.all(+ partitionCapturePlan(plan, concurrency).map(partition =>+ capturePartition({+ plan: partition,+ browser,+ origin,+ shotsDir,+ viewport,+ settleMs,+ fastGlobals,+ onShot: key => onProgress?.({done: ++done, total: plan.length, key}),+ }),+ ),+ );++ const captured = Object.assign({}, ...results.map(result => result.shots));+ const shots = Object.fromEntries(+ plan.filter(shot => captured[shot.key]).map(shot => [shot.key, captured[shot.key]]),+ );+ const observed = {};+ for (const result of results) {+ for (const [key, values] of Object.entries(result.observed)) {observed[key] ??= new Set();for (const value of values) observed[key].add(value);}- } catch (error) {- failures.push({key: shot.key, error: String(error?.message ?? error).slice(0, 400)});- // A story that will not render poisons the page for the next shot in the- // same story group.- currentStory = null;}- done += 1;- onProgress?.({done, total: plan.length, key: shot.key});- }- await browser.close();- await server.close();+ const failuresByKey = Object.fromEntries(+ results.flatMap(result => result.failures).map(failure => [failure.key, failure]),+ );+ const failures = plan.flatMap(shot =>+ failuresByKey[shot.key] ? [failuresByKey[shot.key]] : [],+ );- return {- manifest: {- version: 1,- platform: `${process.platform}-${process.arch}`,- browser: `chromium-${browserVersion}`,- viewport,- settleMs,- frozenClock: FROZEN_NOW.toISOString(),- timezoneId: TIMEZONE_ID,- capturedAt: new Date().toISOString(),- shots,- observedTargets: Object.fromEntries(- Object.entries(observed).map(([key, values]) => [key, [...values].sort()]),- ),- },- failures,- };+ return {+ manifest: {+ version: 1,+ platform: `${process.platform}-${process.arch}`,+ browser: `chromium-${browserVersion}`,+ viewport,+ settleMs,+ frozenClock: FROZEN_NOW.toISOString(),+ timezoneId: TIMEZONE_ID,+ capturedAt: new Date().toISOString(),+ shots,+ observedTargets: Object.fromEntries(+ Object.entries(observed).map(([key, values]) => [key, [...values].sort()]),+ ),+ },+ failures,+ };+ } finally {+ await browser?.close().catch(() => {});+ await server.close();+ }}diff --git a/.github/scripts/visual-gate/lib/capture.test.mjs b/.github/scripts/visual-gate/lib/capture.test.mjsindex d52a9be953..34a90a1e6b 100644--- a/.github/scripts/visual-gate/lib/capture.test.mjs+++ b/.github/scripts/visual-gate/lib/capture.test.mjs@@ -11,6 +11,7 @@ import {CAPTURE_CONTEXT_SECURITY,blockExternalNetwork,isSameOrigin,+ partitionCapturePlan,serveDirectory,} from './capture.mjs';@@ -19,6 +20,49 @@ afterEach(() => {for (const root of roots.splice(0)) fs.rmSync(root, {recursive: true, force: true});});+describe('capture plan partitioning', () => {+ it('keeps the 3,378-shot release workload below 1,700 shots per worker', () => {+ const plan = [];+ for (let story = 0; story < 388; story += 1) {+ const shots = story < 274 ? 9 : 8;+ for (let index = 0; index < shots; index += 1) {+ plan.push({key: `${story}-${index}`, storyId: `story-${story}`});+ }+ }++ const partitions = partitionCapturePlan(plan, 2);+ expect(partitions).toHaveLength(2);+ expect(partitions.flat()).toHaveLength(3378);+ expect(Math.max(...partitions.map(partition => partition.length))).toBeLessThan(1700);++ const ownerByStory = {};+ partitions.forEach((partition, worker) => {+ for (const shot of partition) {+ ownerByStory[shot.storyId] ??= worker;+ expect(ownerByStory[shot.storyId]).toBe(worker);+ }+ });+ expect(new Set(partitions.flat().map(shot => shot.key)).size).toBe(3378);+ });++ it('uses one worker for empty or single-worker plans and never splits interleaved stories', () => {+ expect(partitionCapturePlan([], 2)).toEqual([]);+ const plan = [+ {key: 'a-light', storyId: 'a'},+ {key: 'a-dark', storyId: 'a'},+ {key: 'b-light', storyId: 'b'},+ ];+ expect(partitionCapturePlan(plan, 1)).toEqual([plan]);++ const interleaved = [plan[0], plan[2], plan[1]];+ const partitions = partitionCapturePlan(interleaved, 2);+ const aWorkers = partitions+ .map((partition, worker) => (partition.some(shot => shot.storyId === 'a') ? worker : null))+ .filter(worker => worker !== null);+ expect(aWorkers).toHaveLength(1);+ });+});+describe('capture network boundary', () => {it('allows the exact Storybook origin', () => {expect(isSameOrigin('http://127.0.0.1:6100/assets/story.js', 'http://127.0.0.1:6100')).toBe(diff --git a/.github/scripts/visual-gate/lib/sources.mjs b/.github/scripts/visual-gate/lib/sources.mjsindex fe4178b95b..2aee4f2d9e 100644--- a/.github/scripts/visual-gate/lib/sources.mjs+++ b/.github/scripts/visual-gate/lib/sources.mjs@@ -174,13 +174,14 @@ function unreadableTheme(repoRoot, name, built, error, rebuilt) {/*** @param {string} repoRoot- * @returns {{excludeStories: Record<string, string>, viewport: {width: number, height: number}, settleMs: number, threshold: number, maxDiffPixels: number, defaultTheme: string, probeTheme: string, stableStoryPackages: string[], tiers: string[]}}+ * @returns {{excludeStories: Record<string, string>, viewport: {width: number, height: number}, settleMs: number, captureConcurrency: number, threshold: number, maxDiffPixels: number, defaultTheme: string, probeTheme: string, stableStoryPackages: string[], tiers: string[]}}*/export function loadConfig(repoRoot) {const defaults = {excludeStories: {},viewport: {width: 1024, height: 768},settleMs: 50,+ captureConcurrency: 2,threshold: 0.1,maxDiffPixels: 0,defaultTheme: 'neutral',diff --git a/.github/scripts/visual-gate/visual-gate.config.json b/.github/scripts/visual-gate/visual-gate.config.jsonindex b9c959b987..62ba98e6e8 100644--- a/.github/scripts/visual-gate/visual-gate.config.json+++ b/.github/scripts/visual-gate/visual-gate.config.json@@ -8,6 +8,8 @@"tiers": ["surface", "theme-matrix", "probe"],"viewport": {"width": 1024, "height": 768},"settleMs": 50,+ "$captureConcurrency": "Independent stories may capture concurrently, but every shot for one story stays on one worker so seeded state and fast theme switches remain deterministic.",+ "captureConcurrency": 2,"threshold": 0.1,"maxDiffPixels": 0,"excludeStories": {