Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.5 KB
Raw Blame
#!/usr/bin/env node
/**
 * Visual Review CLI
 *
 * Commands:
 *   submit - Scan directory, hash PNGs, submit run, upload artifacts
 *   verify - Compare local screenshots against baseline (no API calls)
 */
import { program } from 'commander'
import { execSync } from 'node:child_process'
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { resolve } from 'node:path'

import { VisualReviewClient, type Run } from './client.js'
import { hashImageWithDimensions } from './hasher.js'
import { log, reportRunOutcome } from './outcome.js'
import { scanDirectory } from './scanner.js'
import { readBaselineHashes, readSnapshotsFile } from './snapshots.js'
import { buildStoryIndex, type StoryIndexMap } from './storyIndex.js'

program.name('vr').description('Visual Review CLI for snapshot testing').version('0.0.1')

// --- submit command ---

program
    .command('submit')
    .description('Scan directory for PNGs, hash them, and submit a run')
    .requiredOption('--dir <path>', 'Directory containing PNG screenshots')
    .requiredOption('--type <type>', 'Run type (e.g., storybook, playwright)')
    .requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
    .option('--api <url>', 'API URL (overrides snapshots.yml config)')
    .option('--team <id>', 'Team ID (overrides snapshots.yml config)')
    .option('--repo <id>', 'Repo ID (UUID, overrides snapshots.yml config)')
    .option('--branch <name>', 'Git branch name')
    .option('--commit <sha>', 'Git commit SHA')
    .option('--pr <number>', 'PR number')
    .option('--token <value>', 'Personal API token (Authorization: Bearer)')
    .option('--cookie <value>', 'Session cookie for authentication')
    .option('--purpose <purpose>', 'Run purpose: review (gating, approvable) or observe (tracking only)', 'review')
    .option('--tolerate-drift', 'Report snapshot drift on an observe run without failing. Default branch only.')
    .option('--auto-approve', 'Auto-approve all changes and write signed baseline')
    .action(async (options: SubmitOptions) => {
        if (!baselineExists(options.baseline)) {
            process.exit(0)
        }
        try {
            const exitCode = await runSubmit(options)
            process.exit(exitCode)
        } catch (error) {
            console.error('Error:', error)
            process.exit(1)
        }
    })

// --- verify command ---

program
    .command('verify')
    .description('Compare local screenshots against baseline (no API calls)')
    .requiredOption('--dir <path>', 'Directory containing PNG screenshots')
    .requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
    .option('--partial', 'Screenshots cover only part of the baseline (a shard); skip the removed check')
    .action(async (options: VerifyOptions) => {
        try {
            const exitCode = await runVerify(options)
            process.exit(exitCode)
        } catch (error) {
            console.error('Error:', error)
            process.exit(1)
        }
    })

// --- run subcommands (shard-based flow) ---

const run = program.command('run').description('Shard-based run management')

run.command('create')
    .description('Create a new run (call once before shards)')
    .requiredOption('--type <type>', 'Run type (e.g., storybook, playwright)')
    .requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
    .option('--api <url>', 'API URL (overrides snapshots.yml config)')
    .option('--team <id>', 'Team ID (overrides snapshots.yml config)')
    .option('--repo <id>', 'Repo ID (UUID, overrides snapshots.yml config)')
    .option('--branch <name>', 'Git branch name')
    .option('--commit <sha>', 'Git commit SHA')
    .option('--pr <number>', 'PR number')
    .option('--token <value>', 'Personal API token')
    .option('--cookie <value>', 'Session cookie')
    .option('--purpose <purpose>', 'Run purpose: review or observe', 'review')
    .option(
        '--partial',
        'Mark the run as a subset. Identifiers in the baseline but not in this run are left untouched instead of flagged as removed.'
    )
    .action(async (options: RunCreateOptions) => {
        if (!baselineExists(options.baseline)) {
            process.exit(0)
        }
        try {
            const runId = await runCreate(options)
            // Output just the run ID so CI can capture it
            process.stdout.write(runId + '\n')
        } catch (error) {
            console.error('Error:', error)
            process.exit(1)
        }
    })

run.command('upload')
    .description('Hash and upload snapshots from a shard')
    .requiredOption('--run-id <id>', 'Run ID from `run create`')
    .requiredOption('--dir <path>', 'Directory containing PNG screenshots')
    .requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
    .option('--api <url>', 'API URL (overrides snapshots.yml config)')
    .option('--team <id>', 'Team ID (overrides snapshots.yml config)')
    .option('--token <value>', 'Personal API token')
    .option('--cookie <value>', 'Session cookie')
    .option(
        '--storybook-index <path>',
        "Storybook index.json of the build these snapshots came from. Sends the story-to-file map Visual Review uses to find each snapshot's owning team."
    )
    .option(
        '--storybook-root <dir>',
        'Directory Storybook ran in, relative to the repository root. Required with --storybook-index.'
    )
    .action(async (options: RunUploadOptions) => {
        if (options.storybookIndex && !options.storybookRoot) {
            console.error('Error: --storybook-root is required with --storybook-index')
            process.exit(2)
        }
        if (!baselineExists(options.baseline)) {
            process.exit(0)
        }
        try {
            await runUpload(options)
        } catch (error) {
            console.error('Error:', error)
            process.exit(1)
        }
    })

run.command('complete')
    .description('Complete a run after all shards have uploaded')
    .requiredOption('--run-id <id>', 'Run ID from `run create`')
    .requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
    .option('--api <url>', 'API URL (overrides snapshots.yml config)')
    .option('--team <id>', 'Team ID (overrides snapshots.yml config)')
    .option('--token <value>', 'Personal API token')
    .option('--cookie <value>', 'Session cookie')
    .option('--purpose <purpose>', 'Run purpose: review or observe. Must match `run create`.', 'review')
    .option('--tolerate-drift', 'Report snapshot drift on an observe run without failing. Default branch only.')
    .option('--auto-approve', 'Auto-approve all changes and write signed baseline')
    .action(async (options: RunCompleteOptions) => {
        if (!baselineExists(options.baseline)) {
            process.exit(0)
        }
        try {
            const exitCode = await runComplete(options)
            process.exit(exitCode)
        } catch (error) {
            console.error('Error:', error)
            // 2, not 1, so callers can tell a failed CLI apart from the exit 1 that
            // runComplete returns for unresolved visual changes on a review run.
            process.exit(2)
        }
    })

program.parse()

// --- Types ---

interface VerifyOptions {
    dir: string
    baseline: string
    partial?: boolean
}

interface SubmitOptions {
    dir: string
    type: string
    baseline: string
    api?: string
    team?: string
    repo?: string
    branch?: string
    commit?: string
    pr?: string
    token?: string
    cookie?: string
    purpose?: string
    tolerateDrift?: boolean
    autoApprove?: boolean
}

interface RunCreateOptions {
    type: string
    baseline: string
    api?: string
    team?: string
    repo?: string
    branch?: string
    commit?: string
    pr?: string
    token?: string
    cookie?: string
    purpose?: string
    partial?: boolean
}

interface RunUploadOptions {
    runId: string
    dir: string
    baseline: string
    api?: string
    team?: string
    token?: string
    cookie?: string
    storybookIndex?: string
    storybookRoot?: string
}

interface RunCompleteOptions {
    runId: string
    baseline: string
    api?: string
    team?: string
    token?: string
    cookie?: string
    purpose?: string
    tolerateDrift?: boolean
    autoApprove?: boolean
}

// --- Helpers ---

function baselineExists(baselinePath: string): boolean {
    const p = resolve(baselinePath)
    if (!existsSync(p)) {
        log(`Baseline file not found: ${p} — skipping (branch may not have snapshots.yml yet)`)
        return false
    }
    return true
}

function extractContentHash(signedHash: string): string {
    const parts = signedHash.split('.')
    return parts.length === 4 ? parts[2] : signedHash
}

function getCurrentBranch(): string {
    try {
        return execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim()
    } catch {
        return 'unknown'
    }
}

function getCurrentCommit(): string {
    try {
        return execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim()
    } catch {
        return 'unknown'
    }
}

function resolveConfig(options: { baseline: string; api?: string; team?: string; repo?: string }): {
    api: string
    team: string
    repo: string
} {
    const baselinePath = resolve(options.baseline)
    const snapshotsFile = readSnapshotsFile(baselinePath)
    const config = snapshotsFile?.config

    const api = options.api ?? config?.api
    if (!api) {
        throw new Error('API URL required: pass --api or set config.api in snapshots.yml')
    }

    const team = options.team ?? config?.team
    if (!team) {
        throw new Error('Team ID required: pass --team or set config.team in snapshots.yml')
    }

    const repo = options.repo ?? config?.repo
    if (!repo) {
        throw new Error('Repo ID required: pass --repo or set config.repo in snapshots.yml')
    }

    return { api, team, repo }
}

function makeClient(options: { baseline: string; api?: string; team?: string; token?: string; cookie?: string }): {
    client: VisualReviewClient
    api: string
    team: string
    repo: string
} {
    const config = resolveConfig(options)
    const client = new VisualReviewClient({
        apiUrl: config.api,
        teamId: config.team,
        token: options.token,
        sessionCookie: options.cookie,
    })
    return { client, ...config }
}

function collectCIMetadata(): Record<string, string> {
    const metadata: Record<string, string> = {}
    const runId = process.env.GITHUB_RUN_ID
    if (runId) {
        metadata.github_run_id = runId
    }
    const jobId = process.env.JOB_CHECK_RUN_ID
    if (jobId) {
        metadata.github_check_run_id = jobId
    }
    return metadata
}

// --- Command implementations ---

// --- Shard command implementations ---

async function runCreate(options: RunCreateOptions): Promise<string> {
    const { client, repo } = makeClient(options)

    const branch = options.branch ?? getCurrentBranch()
    const commit = options.commit ?? getCurrentCommit()

    log(
        `Creating run: type=${options.type}, branch=${branch}, commit=${commit.slice(0, 10)}, purpose=${options.purpose ?? 'review'}${options.partial ? ', partial' : ''}`
    )

    const result = await client.createRun({
        repoId: repo,
        runType: options.type,
        commitSha: commit,
        branch,
        prNumber: options.pr ? parseInt(options.pr, 10) : undefined,
        snapshots: [],
        purpose: options.purpose,
        metadata: collectCIMetadata(),
        isPartial: options.partial,
    })

    log(`Run created: ${result.run_id}`)
    return result.run_id
}

// The map only attributes snapshots to teams, so a map that cannot be read or sent must not fail the
// upload of the snapshots themselves.
function readStoryIndex(
    indexPath: string | undefined,
    storybookRoot: string | undefined,
    runId: string
): StoryIndexMap | undefined {
    if (!indexPath || !storybookRoot) {
        return undefined
    }
    try {
        const map = buildStoryIndex(readFileSync(indexPath, 'utf-8'), storybookRoot)
        log(`[run:${runId}] Story index: ${map.storyCount} stories, ${map.hash.slice(0, 12)}`)
        return map
    } catch (error) {
        log(`[run:${runId}] Could not read the Storybook index, sending snapshots without it: ${error}`)
        return undefined
    }
}

async function runUpload(options: RunUploadOptions): Promise<void> {
    const { client } = makeClient(options)
    const runId = options.runId

    const dirPath = resolve(options.dir)

    log(`[run:${runId}] Scanning ${dirPath} for PNGs...`)
    const scanned = scanDirectory(dirPath)
    if (scanned.length === 0) {
        log(`[run:${runId}] No PNGs found — nothing to upload`)
        return
    }

    const snapshots: Array<{ identifier: string; hash: string; width: number; height: number; data: Buffer }> = []
    log(`[run:${runId}] Found ${scanned.length} snapshots, hashing...`)
    const HASH_CONCURRENCY = 16
    for (let i = 0; i < scanned.length; i += HASH_CONCURRENCY) {
        const batch = scanned.slice(i, i + HASH_CONCURRENCY)
        const results = await Promise.all(
            batch.map(async ({ identifier, filePath }) => {
                const data = await readFile(filePath)
                const { hash, width, height } = await hashImageWithDimensions(data)
                return { identifier, hash, width, height, data }
            })
        )
        snapshots.push(...results)
    }

    log(`[run:${runId}] Sending ${snapshots.length} snapshots to backend`)

    const storyIndex = readStoryIndex(options.storybookIndex, options.storybookRoot, runId)
    const addResult = await client.addSnapshots(runId, {
        snapshots: snapshots.map((s) => ({
            identifier: s.identifier,
            content_hash: s.hash,
            width: s.width,
            height: s.height,
        })),
        storyIndexHash: storyIndex?.hash,
    })

    if (storyIndex && addResult.story_index_upload) {
        try {
            await client.uploadToS3(addResult.story_index_upload, storyIndex.content, 'application/json')
            log(`[run:${runId}] Uploaded story index ${storyIndex.hash.slice(0, 12)}`)
        } catch (error) {
            log(`[run:${runId}] Story index upload failed: ${error}`)
        }
    }

    log(`[run:${runId}] Registered ${addResult.added} snapshot(s), ${addResult.uploads.length} upload(s) needed`)

    if (addResult.uploads.length > 0) {
        const hashToSnapshot = new Map(snapshots.map((s) => [s.hash, s]))
        const CONCURRENCY = 10
        let uploaded = 0
        let failed = 0

        for (let i = 0; i < addResult.uploads.length; i += CONCURRENCY) {
            const batch = addResult.uploads.slice(i, i + CONCURRENCY)
            await Promise.all(
                batch.map(async (upload) => {
                    const snapshot = hashToSnapshot.get(upload.content_hash)
                    if (!snapshot) {
                        return
                    }
                    try {
                        await client.uploadToS3(upload, snapshot.data)
                        uploaded++
                    } catch (error) {
                        failed++
                        log(`[run:${runId}] Upload failed ${snapshot.identifier}: ${error}`)
                    }
                })
            )
        }
        log(`[run:${runId}] Uploaded ${uploaded} artifact(s)${failed > 0 ? `, ${failed} failed` : ''}`)
        if (failed > 0) {
            throw new Error(`[run:${runId}] ${failed} artifact upload(s) failed`)
        }
    }
}

async function runComplete(options: RunCompleteOptions): Promise<number> {
    const { client, api, team } = makeClient(options)
    const baselinePath = resolve(options.baseline)
    const runId = options.runId

    log(`[run:${runId}] Completing run`)

    let run = await client.completeRun(runId, process.env.JOB_CHECK_RUN_ID)

    log(`[run:${runId}] Status: ${run.status}`)

    if (run.status !== 'completed' && run.status !== 'failed') {
        log(`[run:${runId}] Waiting for diff processing...`)
        run = await waitForCompletion(client, runId)
    }

    const s = run.summary
    log(
        `[run:${runId}] Done: ${s.total} snapshots — ${s.unchanged} unchanged, ${s.changed} changed, ${s.new} new, ${s.removed} removed`
    )

    if (options.autoApprove) {
        log(`[run:${runId}] Auto-approving all changes...`)
        const approveResult = await client.autoApproveRun(runId)
        writeFileSync(baselinePath, approveResult.baseline_content, 'utf-8')
        log(`[run:${runId}] Baseline written to ${baselinePath}`)
        return 0
    }

    return reportRunOutcome(
        client,
        run,
        `${api}/project/${team}/visual-review/runs/${runId}`,
        options.purpose ?? 'review',
        options.tolerateDrift ?? false
    )
}

// --- Legacy command implementations ---

async function runVerify(options: VerifyOptions): Promise<number> {
    const dirPath = resolve(options.dir)
    const baselinePath = resolve(options.baseline)

    const scanned = scanDirectory(dirPath)

    const baselineHashes = readBaselineHashes(baselinePath)
    if (Object.keys(baselineHashes).length === 0) {
        console.error('No baseline hashes found — run `vr submit` on a PR first')
        return 1
    }

    if (scanned.length === 0) {
        // A shard whose story files all skip their screenshot writes no PNG, and that is not a mismatch.
        if (options.partial) {
            log('No PNGs found in directory, nothing to verify')
            return 0
        }
        console.error('No PNGs found in directory')
        return 1
    }

    log(`Found ${scanned.length} snapshots, verifying against ${Object.keys(baselineHashes).length} baselines...`)

    const changed: string[] = []
    const added: string[] = []

    for (const { identifier, filePath } of scanned) {
        const data = readFileSync(filePath)
        const { hash } = await hashImageWithDimensions(data)
        const baselineSignedHash = baselineHashes[identifier]

        if (!baselineSignedHash) {
            added.push(identifier)
        } else {
            if (hash !== extractContentHash(baselineSignedHash)) {
                changed.push(identifier)
            }
        }
    }

    const currentIds = new Set(scanned.map((s) => s.identifier))
    const removed = options.partial ? [] : Object.keys(baselineHashes).filter((id) => !currentIds.has(id))

    const unchanged = scanned.length - changed.length - added.length
    log(
        `\n${scanned.length} snapshots — ${unchanged} unchanged, ${changed.length} changed, ${added.length} new, ${removed.length} removed`
    )

    if (changed.length > 0) {
        log('\nChanged:')
        changed.forEach((id) => log(`  - ${id}`))
    }
    if (added.length > 0) {
        log('\nNew:')
        added.forEach((id) => log(`  - ${id}`))
    }
    if (removed.length > 0) {
        log('\nRemoved:')
        removed.forEach((id) => log(`  - ${id}`))
    }

    if (changed.length > 0 || added.length > 0 || removed.length > 0) {
        log('\nBaseline mismatch — submit a PR to update baselines')
        return 1
    }

    log('All snapshots match baseline')
    return 0
}

function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

async function waitForCompletion(client: VisualReviewClient, runId: string): Promise<Run> {
    const maxAttempts = 120
    const intervalMs = 5000

    for (let i = 0; i < maxAttempts; i++) {
        const run = await client.getRun(runId)
        if (run.status === 'completed' || run.status === 'failed') {
            return run
        }
        if (i > 0 && i % 6 === 0) {
            log(`[run:${runId}] Still processing (${(i * intervalMs) / 1000}s elapsed, status=${run.status})`)
        }
        await sleep(intervalMs)
    }

    throw new Error(`[run:${runId}] Run did not complete within ${(maxAttempts * intervalMs) / 1000}s`)
}

async function runSubmit(options: SubmitOptions): Promise<number> {
    const { api, team, repo } = resolveConfig(options)

    const client = new VisualReviewClient({
        apiUrl: api,
        teamId: team,
        token: options.token,
        sessionCookie: options.cookie,
    })

    // 1. Scan directory for PNGs
    const dirPath = resolve(options.dir)

    log(`Scanning ${dirPath} for PNGs...`)
    const scanned = scanDirectory(dirPath)

    if (scanned.length === 0) {
        console.error('No PNGs found in directory')
        return 1
    }

    // 2. Hash each image (decode PNG → RGBA → SHA256)
    const snapshots: Array<{
        identifier: string
        hash: string
        width: number
        height: number
        data: Buffer
    }> = []

    log(`Found ${scanned.length} snapshots, hashing...`)
    const HASH_CONCURRENCY = 16
    for (let i = 0; i < scanned.length; i += HASH_CONCURRENCY) {
        const batch = scanned.slice(i, i + HASH_CONCURRENCY)
        const results = await Promise.all(
            batch.map(async ({ identifier, filePath }) => {
                const data = await readFile(filePath)
                const { hash, width, height } = await hashImageWithDimensions(data)
                return { identifier, hash, width, height, data }
            })
        )
        snapshots.push(...results)
    }

    // 3. Create run with full manifest — backend fetches baseline and classifies
    const branch = options.branch ?? getCurrentBranch()
    const commit = options.commit ?? getCurrentCommit()
    // --auto-approve only makes sense on review runs; observe runs are non-approvable.
    const purpose = options.autoApprove ? 'review' : (options.purpose ?? 'review')
    if (options.autoApprove && options.purpose && options.purpose !== 'review') {
        log(`Warning: --auto-approve forces purpose=review (got --purpose ${options.purpose})`)
    }
    log(
        `Creating run: type=${options.type}, ${snapshots.length} snapshots, branch=${branch}, commit=${commit.slice(0, 10)}, purpose=${purpose}`
    )

    const result = await client.createRun({
        repoId: repo,
        runType: options.type,
        commitSha: commit,
        branch,
        prNumber: options.pr ? parseInt(options.pr, 10) : undefined,
        purpose,
        snapshots: snapshots.map((s) => ({
            identifier: s.identifier,
            content_hash: s.hash,
            width: s.width,
            height: s.height,
        })),
        metadata: collectCIMetadata(),
    })

    const runId = result.run_id
    log(
        `[run:${runId}] Run created, ${result.uploads.length} upload(s) needed, ${snapshots.length - result.uploads.length} already exist`
    )

    if (result.uploads.length > 0) {
        const hashToSnapshot = new Map(snapshots.map((s) => [s.hash, s]))
        const CONCURRENCY = 10
        let uploaded = 0
        let failed = 0

        const uploadOne = async (upload: (typeof result.uploads)[number]): Promise<void> => {
            const snapshot = hashToSnapshot.get(upload.content_hash)
            if (!snapshot) {
                return
            }
            try {
                await client.uploadToS3(upload, snapshot.data)
                uploaded++
            } catch (error) {
                failed++
                log(`[run:${runId}] Upload failed ${snapshot.identifier}: ${error}`)
            }
        }

        for (let i = 0; i < result.uploads.length; i += CONCURRENCY) {
            const batch = result.uploads.slice(i, i + CONCURRENCY)
            await Promise.all(batch.map(uploadOne))
        }
        log(`[run:${runId}] Uploaded ${uploaded} artifact(s)${failed > 0 ? `, ${failed} failed` : ''}`)
        if (failed > 0) {
            throw new Error(`[run:${runId}] ${failed} artifact upload(s) failed`)
        }
    }

    let run = await client.completeRun(runId, process.env.JOB_CHECK_RUN_ID)
    log(`[run:${runId}] Status: ${run.status}`)

    if (run.status !== 'completed' && run.status !== 'failed') {
        log(`[run:${runId}] Waiting for diff processing...`)
        run = await waitForCompletion(client, runId)
    }

    const s = run.summary
    log(
        `[run:${runId}] Done: ${s.total} snapshots — ${s.unchanged} unchanged, ${s.changed} changed, ${s.new} new, ${s.removed} removed`
    )

    if (options.autoApprove) {
        log(`[run:${runId}] Auto-approving all changes...`)
        const approveResult = await client.autoApproveRun(runId)
        const baselinePath = resolve(options.baseline)
        writeFileSync(baselinePath, approveResult.baseline_content, 'utf-8')
        log(`[run:${runId}] Baseline written to ${baselinePath}`)
        return 0
    }

    return reportRunOutcome(
        client,
        run,
        `${api}/project/${team}/visual-review/runs/${runId}`,
        purpose,
        options.tolerateDrift ?? false
    )
}