Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.6 KB
Raw Blame
#!/usr/bin/env node
/**
 * Compute the set of stories affected by a set of changed files using the
 * module-graph.json emitted by the Storybook build (see the ModuleGraphPlugin
 * in `common/storybook/.storybook/plugins/vite-module-graph-plugin.ts`).
 *
 * Called from `.github/workflows/ci-storybook.yml`'s story-selection job to
 * narrow visual-regression runs to stories affected by a pull request.
 *
 * Usage:
 *   bin/find-affected-stories \
 *       --graph common/storybook/dist/module-graph.json \
 *       --base-ref origin/master
 *
 * The script computes `git diff --name-only BASE_REF...HEAD` itself. The
 * `...` resolves against the merge-base, so files that came in via a merge
 * of BASE_REF into the branch aren't counted — avoids false full-runs when
 * `pull_request.base.sha` is stale relative to the master the branch merged.
 *
 * Output (JSON to stdout):
 *   { mode: "selective", affected_stories: [...], affected_story_count,
 *     total_story_count, suggested_shards, affected_duration_seconds,
 *     total_duration_seconds, unresolved_changed_files }
 *   OR
 *   { mode: "full", reason: "<why>", total_story_count }
 */

import { execFileSync } from 'child_process'
import * as fs from 'fs'
import { pathToFileURL } from 'url'

// -----------------------------------------------------------------------------
// Invalidators the module graph can't see
// -----------------------------------------------------------------------------

// Changes to any file matching these substrings force a full-run: the effect
// isn't captured by the static import graph (global CSS side effects, theme
// providers, MSW mocks, the storybook harness itself, generated schema files).
const FULL_RUN_PATTERNS = [
    '.storybook/',
    'common/storybook/tailwind.config.js',
    'common/storybook/package.json',
    'frontend/src/lib/',
    'frontend/src/styles/',
    'frontend/src/mocks/',
    // A staticDirs source in common/storybook/.storybook/main.ts. The stories fetch it by URL,
    // so no story imports it and a selective run would render zero stories.
    'posthog/templates/replay_player_frame/',
    'common/tailwind/',
    'tailwind.config.js',
    'tailwind.config.ts',
    'frontend/src/queries/schema/',
    'frontend/src/queries/schema.json',
    'frontend/src/products.tsx',
    'frontend/src/products.json',
    'manifest.tsx',
    'playwright.config.ts',
    '.github/workflows/ci-storybook.yml',
    'pnpm-lock.yaml',
    'package.json',
    '.nvmrc',
]

const STORYBOOK_RELEVANT_FILE_RE =
    /^(frontend\/|products\/.*\/(frontend|mcp\/apps)\/|common\/(esbuilder|mosaic|storybook|tailwind)\/|packages\/quill\/|ee\/frontend\/|services\/mcp\/src\/ui-apps\/)/
const NON_RUNTIME_FILE_RE =
    /(^|\/)(__tests__|tests?)\/|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)README\.md$|^(frontend|playwright)\/snapshots\.yml$/

const MAX_CHANGED_FILES = 100
// Fallback per-story render cost, used only when the timing manifest is absent.
// The manifest is the real source; this keeps the estimate in the right order of
// magnitude rather than sizing from zero.
const AVG_STORY_DURATION_SECONDS = 6
// Shards the full chromium matrix uses, i.e. CHROMIUM_SHARDS in ci-storybook.yml.
// Overridden by --max-shards, which the workflow passes.
const DEFAULT_FULL_SHARDS = 28
const STORY_RE = /\.stories\.(tsx?|mdx|jsx?)$/
// Per-story render times, keyed by story path, refreshed nightly by
// ci-storybook-update-test-timing.yml. The same file the test sequencer balances with.
const DEFAULT_TIMINGS_PATH = 'common/storybook/storybook-timings.json'

// -----------------------------------------------------------------------------
// CLI
// -----------------------------------------------------------------------------

function parseArgs(argv) {
    const args = { graph: null, baseRef: null, timings: DEFAULT_TIMINGS_PATH, maxShards: DEFAULT_FULL_SHARDS }
    for (let i = 0; i < argv.length; i++) {
        const a = argv[i]
        if (a === '--graph') args.graph = argv[++i]
        else if (a === '--base-ref') args.baseRef = argv[++i]
        else if (a === '--timings') args.timings = argv[++i]
        else if (a === '--max-shards') {
            // A missing or non-numeric value must not reach the matrix as NaN.
            const value = Number(argv[++i])
            args.maxShards = Number.isFinite(value) && value >= 1 ? Math.floor(value) : DEFAULT_FULL_SHARDS
        }
        else if (a === '--help' || a === '-h') {
            process.stderr.write(
                'Usage: find-affected-stories --graph <path> --base-ref <git-ref> ' +
                    '[--timings <path>] [--max-shards <n>]\n'
            )
            process.exit(0)
        } else {
            process.stderr.write(`Unknown argument: ${a}\n`)
            process.exit(2)
        }
    }
    return args
}

function changedFilesFromGit(baseRef) {
    // `...` resolves against the merge-base so files pulled in via a merge of
    // baseRef into the branch aren't counted (avoids inflated diffs when
    // pull_request.base.sha is stale).
    const stdout = execFileSync('git', ['diff', '--name-only', `${baseRef}...HEAD`], {
        encoding: 'utf-8',
    })
    return stdout
        .split('\n')
        .map((l) => l.trim())
        .filter(Boolean)
}

// -----------------------------------------------------------------------------
// Output helpers
// -----------------------------------------------------------------------------

function emitFull(reason, totalStories) {
    process.stdout.write(JSON.stringify({ mode: 'full', reason, total_story_count: totalStories }) + '\n')
}

export function loadTimings(path) {
    try {
        const data = JSON.parse(fs.readFileSync(path, 'utf-8'))
        if (!data || typeof data !== 'object' || Array.isArray(data)) {
            return null
        }
        const entries = Object.entries(data).filter(([, v]) => v && typeof v.chromium === 'number')
        return entries.length > 0 ? new Map(entries) : null
    } catch {
        return null
    }
}

// Render seconds for a set of stories. A story the nightly refresh has not timed
// yet costs the manifest median, which is far closer than the old flat constant.
export function workloadSeconds(stories, timings) {
    if (!timings) {
        return stories.length * AVG_STORY_DURATION_SECONDS
    }
    const known = [...timings.values()]
        .map((v) => v.chromium)
        .sort((a, b) => a - b)
    const fallback = known[Math.floor(known.length / 2)]
    let total = 0
    for (const story of stories) {
        const entry = timings.get(story)
        total += entry ? entry.chromium : fallback
    }
    return total
}

export function suggestShards(affectedDuration, totalDuration, maxShards) {
    // Give the narrowed run the shard density the full matrix has, so a selective
    // run finishes its tests in about the wall time a full run does. Sizing from a
    // flat per-story constant instead put the full run's story time on a quarter of
    // its runners, and a narrowed run took twice as long per shard as a full one.
    if (!(totalDuration > 0) || !(affectedDuration > 0)) {
        return 1
    }
    const proportional = Math.ceil((affectedDuration / totalDuration) * maxShards)
    return Math.max(1, Math.min(maxShards, proportional))
}

function emitSelective(affected, totalStories, unresolved, allStories, maxShards, timings) {
    const affectedDuration = workloadSeconds(affected, timings)
    const totalDuration = workloadSeconds(allStories, timings)
    const suggestedShards = suggestShards(affectedDuration, totalDuration, maxShards)
    process.stdout.write(
        JSON.stringify({
            mode: 'selective',
            affected_stories: affected,
            affected_story_count: affected.length,
            total_story_count: totalStories,
            suggested_shards: suggestedShards,
            affected_duration_seconds: Math.round(affectedDuration),
            total_duration_seconds: Math.round(totalDuration),
            unresolved_changed_files: unresolved,
        }) + '\n'
    )
}

// -----------------------------------------------------------------------------
// Core logic
// -----------------------------------------------------------------------------

export function requiresFullRun(changedFile) {
    for (const pattern of FULL_RUN_PATTERNS) {
        if (changedFile.includes(pattern)) {
            return `changed file matches full-run pattern '${pattern}': ${changedFile}`
        }
    }
    return null
}

export function normalize(p) {
    if (!p) return p
    // Strip any leading ./ or / so equality works against module names from the
    // module graph plugin (which emits repo-relative names).
    return p.replace(/^\.\//, '').replace(/^\/+/, '')
}

function loadGraph(graphPath) {
    const raw = fs.readFileSync(graphPath, 'utf-8')
    const data = JSON.parse(raw)
    if (!data || !Array.isArray(data.modules)) {
        throw new Error('module-graph.json is missing `modules[]`')
    }
    const byName = new Map()
    for (const m of data.modules) {
        if (!m || typeof m.name !== 'string') continue
        byName.set(normalize(m.name), m)
    }
    return { modules: data.modules, byName }
}

function countStories(modules) {
    let n = 0
    for (const m of modules) if (STORY_RE.test(m.name)) n++
    return n
}

export function affectedStoriesFor(changedFiles, graph) {
    const { byName } = graph
    const unresolved = []
    const start = new Set()
    for (const raw of changedFiles) {
        const normalized = normalize(raw)
        if (byName.has(normalized)) {
            start.add(normalized)
        } else {
            unresolved.push(raw)
        }
    }

    // BFS upward via `reasons` (which are the importers — things that depend on
    // the current module). Any story we pass through is collected.
    const visited = new Set(start)
    const queue = [...start]
    const stories = new Set()
    while (queue.length > 0) {
        const cur = queue.shift()
        const mod = byName.get(cur)
        if (!mod || !Array.isArray(mod.reasons)) continue
        for (const importerRaw of mod.reasons) {
            const importer = normalize(importerRaw)
            if (visited.has(importer)) continue
            visited.add(importer)
            queue.push(importer)
            if (STORY_RE.test(importer)) stories.add(importer)
        }
    }
    // Also count the changed files themselves if they happen to be stories.
    for (const f of start) if (STORY_RE.test(f)) stories.add(f)
    return { affected: [...stories].sort(), unresolved }
}

export function unresolvedRuntimeFiles(unresolvedFiles) {
    return unresolvedFiles.filter(
        (file) => STORYBOOK_RELEVANT_FILE_RE.test(normalize(file)) && !NON_RUNTIME_FILE_RE.test(normalize(file))
    )
}

// -----------------------------------------------------------------------------
// Entrypoint
// -----------------------------------------------------------------------------

export function main() {
    const args = parseArgs(process.argv.slice(2))
    if (!args.graph) {
        process.stderr.write('Error: --graph is required\n')
        process.exit(2)
    }
    if (!args.baseRef) {
        process.stderr.write('Error: --base-ref is required\n')
        process.exit(2)
    }

    let changed
    try {
        changed = changedFilesFromGit(args.baseRef)
    } catch (err) {
        process.stderr.write(`Error: \`git diff\` against ${args.baseRef} failed: ${err.message}\n`)
        process.exit(2)
    }
    process.stderr.write(`Derived ${changed.length} changed file(s) from \`git diff ${args.baseRef}...HEAD\`\n`)

    // Load graph first so we can emit total_story_count even in full-run mode.
    // If the graph is unreadable/missing, fall back to a full-run recommendation
    // without a count (caller will treat 0 as "unknown total").
    let graph
    let totalStories = 0
    try {
        graph = loadGraph(args.graph)
        totalStories = countStories(graph.modules)
    } catch (err) {
        emitFull(`module-graph.json unavailable: ${err.message}`, 0)
        return
    }

    if (process.env.FORCE_FULL_STORIES) {
        emitFull('FORCE_FULL_STORIES env var set', totalStories)
        return
    }
    if (changed.length > MAX_CHANGED_FILES) {
        emitFull(`too many changed files (${changed.length} > ${MAX_CHANGED_FILES})`, totalStories)
        return
    }
    for (const f of changed) {
        const reason = requiresFullRun(f)
        if (reason) {
            emitFull(reason, totalStories)
            return
        }
    }

    const { affected, unresolved } = affectedStoriesFor(changed, graph)
    const unresolvedRuntime = unresolvedRuntimeFiles(unresolved)
    if (unresolvedRuntime.length > 0) {
        emitFull(`changed frontend file missing from module graph: ${unresolvedRuntime[0]}`, totalStories)
        return
    }
    process.stderr.write(`Affected stories: ${affected.length} / ${totalStories} (unresolved: ${unresolved.length})\n`)

    const timings = loadTimings(args.timings)
    if (!timings) {
        process.stderr.write(`No timing manifest at ${args.timings}; sizing shards from the flat estimate\n`)
    } else {
        // A key format drift would send every story to the median without failing,
        // so the hit rate is printed where the job log shows it.
        const timed = affected.filter((story) => timings.has(story)).length
        process.stderr.write(
            `Timed ${timed} of ${affected.length} affected stories from ${args.timings}; the rest cost the median\n`
        )
    }
    const allStories = graph.modules.map((m) => normalize(m.name)).filter((name) => STORY_RE.test(name))
    emitSelective(affected, totalStories, unresolved, allStories, args.maxShards, timings)
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
    main()
}