:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
#!/usr/bin/env node
/**
* Visual Review CLI
*
* Commands:
* submit - Scan directory, hash PNGs, submit run, upload artifacts
* verify - Compare local screenshots against baseline (no API calls)
*/
import { program } from 'commander'
import { execSync } from 'node:child_process'
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { resolve } from 'node:path'
import { VisualReviewClient, type Run } from './client.js'
import { hashImageWithDimensions } from './hasher.js'
import { log, reportRunOutcome } from './outcome.js'
import { scanDirectory } from './scanner.js'
import { readBaselineHashes, readSnapshotsFile } from './snapshots.js'
import { buildStoryIndex, type StoryIndexMap } from './storyIndex.js'
program.name('vr').description('Visual Review CLI for snapshot testing').version('0.0.1')
// --- submit command ---
program
.command('submit')
.description('Scan directory for PNGs, hash them, and submit a run')
.requiredOption('--dir <path>', 'Directory containing PNG screenshots')
.requiredOption('--type <type>', 'Run type (e.g., storybook, playwright)')
.requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
.option('--api <url>', 'API URL (overrides snapshots.yml config)')
.option('--team <id>', 'Team ID (overrides snapshots.yml config)')
.option('--repo <id>', 'Repo ID (UUID, overrides snapshots.yml config)')
.option('--branch <name>', 'Git branch name')
.option('--commit <sha>', 'Git commit SHA')
.option('--pr <number>', 'PR number')
.option('--token <value>', 'Personal API token (Authorization: Bearer)')
.option('--cookie <value>', 'Session cookie for authentication')
.option('--purpose <purpose>', 'Run purpose: review (gating, approvable) or observe (tracking only)', 'review')
.option('--tolerate-drift', 'Report snapshot drift on an observe run without failing. Default branch only.')
.option('--auto-approve', 'Auto-approve all changes and write signed baseline')
.action(async (options: SubmitOptions) => {
if (!baselineExists(options.baseline)) {
process.exit(0)
}
try {
const exitCode = await runSubmit(options)
process.exit(exitCode)
} catch (error) {
console.error('Error:', error)
process.exit(1)
}
})
// --- verify command ---
program
.command('verify')
.description('Compare local screenshots against baseline (no API calls)')
.requiredOption('--dir <path>', 'Directory containing PNG screenshots')
.requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
.option('--partial', 'Screenshots cover only part of the baseline (a shard); skip the removed check')
.action(async (options: VerifyOptions) => {
try {
const exitCode = await runVerify(options)
process.exit(exitCode)
} catch (error) {
console.error('Error:', error)
process.exit(1)
}
})
// --- run subcommands (shard-based flow) ---
const run = program.command('run').description('Shard-based run management')
run.command('create')
.description('Create a new run (call once before shards)')
.requiredOption('--type <type>', 'Run type (e.g., storybook, playwright)')
.requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
.option('--api <url>', 'API URL (overrides snapshots.yml config)')
.option('--team <id>', 'Team ID (overrides snapshots.yml config)')
.option('--repo <id>', 'Repo ID (UUID, overrides snapshots.yml config)')
.option('--branch <name>', 'Git branch name')
.option('--commit <sha>', 'Git commit SHA')
.option('--pr <number>', 'PR number')
.option('--token <value>', 'Personal API token')
.option('--cookie <value>', 'Session cookie')
.option('--purpose <purpose>', 'Run purpose: review or observe', 'review')
.option(
'--partial',
'Mark the run as a subset. Identifiers in the baseline but not in this run are left untouched instead of flagged as removed.'
)
.action(async (options: RunCreateOptions) => {
if (!baselineExists(options.baseline)) {
process.exit(0)
}
try {
const runId = await runCreate(options)
// Output just the run ID so CI can capture it
process.stdout.write(runId + '\n')
} catch (error) {
console.error('Error:', error)
process.exit(1)
}
})
run.command('upload')
.description('Hash and upload snapshots from a shard')
.requiredOption('--run-id <id>', 'Run ID from `run create`')
.requiredOption('--dir <path>', 'Directory containing PNG screenshots')
.requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
.option('--api <url>', 'API URL (overrides snapshots.yml config)')
.option('--team <id>', 'Team ID (overrides snapshots.yml config)')
.option('--token <value>', 'Personal API token')
.option('--cookie <value>', 'Session cookie')
.option(
'--storybook-index <path>',
"Storybook index.json of the build these snapshots came from. Sends the story-to-file map Visual Review uses to find each snapshot's owning team."
)
.option(
'--storybook-root <dir>',
'Directory Storybook ran in, relative to the repository root. Required with --storybook-index.'
)
.action(async (options: RunUploadOptions) => {
if (options.storybookIndex && !options.storybookRoot) {
console.error('Error: --storybook-root is required with --storybook-index')
process.exit(2)
}
if (!baselineExists(options.baseline)) {
process.exit(0)
}
try {
await runUpload(options)
} catch (error) {
console.error('Error:', error)
process.exit(1)
}
})
run.command('complete')
.description('Complete a run after all shards have uploaded')
.requiredOption('--run-id <id>', 'Run ID from `run create`')
.requiredOption('--baseline <path>', 'Path to snapshots.yml baseline file')
.option('--api <url>', 'API URL (overrides snapshots.yml config)')
.option('--team <id>', 'Team ID (overrides snapshots.yml config)')
.option('--token <value>', 'Personal API token')
.option('--cookie <value>', 'Session cookie')
.option('--purpose <purpose>', 'Run purpose: review or observe. Must match `run create`.', 'review')
.option('--tolerate-drift', 'Report snapshot drift on an observe run without failing. Default branch only.')
.option('--auto-approve', 'Auto-approve all changes and write signed baseline')
.action(async (options: RunCompleteOptions) => {
if (!baselineExists(options.baseline)) {
process.exit(0)
}
try {
const exitCode = await runComplete(options)
process.exit(exitCode)
} catch (error) {
console.error('Error:', error)
// 2, not 1, so callers can tell a failed CLI apart from the exit 1 that
// runComplete returns for unresolved visual changes on a review run.
process.exit(2)
}
})
program.parse()
// --- Types ---
interface VerifyOptions {
dir: string
baseline: string
partial?: boolean
}
interface SubmitOptions {
dir: string
type: string
baseline: string
api?: string
team?: string
repo?: string
branch?: string
commit?: string
pr?: string
token?: string
cookie?: string
purpose?: string
tolerateDrift?: boolean
autoApprove?: boolean
}
interface RunCreateOptions {
type: string
baseline: string
api?: string
team?: string
repo?: string
branch?: string
commit?: string
pr?: string
token?: string
cookie?: string
purpose?: string
partial?: boolean
}
interface RunUploadOptions {
runId: string
dir: string
baseline: string
api?: string
team?: string
token?: string
cookie?: string
storybookIndex?: string
storybookRoot?: string
}
interface RunCompleteOptions {
runId: string
baseline: string
api?: string
team?: string
token?: string
cookie?: string
purpose?: string
tolerateDrift?: boolean
autoApprove?: boolean
}
// --- Helpers ---
function baselineExists(baselinePath: string): boolean {
const p = resolve(baselinePath)
if (!existsSync(p)) {
log(`Baseline file not found: ${p} — skipping (branch may not have snapshots.yml yet)`)
return false
}
return true
}
function extractContentHash(signedHash: string): string {
const parts = signedHash.split('.')
return parts.length === 4 ? parts[2] : signedHash
}
function getCurrentBranch(): string {
try {
return execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim()
} catch {
return 'unknown'
}
}
function getCurrentCommit(): string {
try {
return execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim()
} catch {
return 'unknown'
}
}
function resolveConfig(options: { baseline: string; api?: string; team?: string; repo?: string }): {
api: string
team: string
repo: string
} {
const baselinePath = resolve(options.baseline)
const snapshotsFile = readSnapshotsFile(baselinePath)
const config = snapshotsFile?.config
const api = options.api ?? config?.api
if (!api) {
throw new Error('API URL required: pass --api or set config.api in snapshots.yml')
}
const team = options.team ?? config?.team
if (!team) {
throw new Error('Team ID required: pass --team or set config.team in snapshots.yml')
}
const repo = options.repo ?? config?.repo
if (!repo) {
throw new Error('Repo ID required: pass --repo or set config.repo in snapshots.yml')
}
return { api, team, repo }
}
function makeClient(options: { baseline: string; api?: string; team?: string; token?: string; cookie?: string }): {
client: VisualReviewClient
api: string
team: string
repo: string
} {
const config = resolveConfig(options)
const client = new VisualReviewClient({
apiUrl: config.api,
teamId: config.team,
token: options.token,
sessionCookie: options.cookie,
})
return { client, ...config }
}
function collectCIMetadata(): Record<string, string> {
const metadata: Record<string, string> = {}
const runId = process.env.GITHUB_RUN_ID
if (runId) {
metadata.github_run_id = runId
}
const jobId = process.env.JOB_CHECK_RUN_ID
if (jobId) {
metadata.github_check_run_id = jobId
}
return metadata
}
// --- Command implementations ---
// --- Shard command implementations ---
async function runCreate(options: RunCreateOptions): Promise<string> {
const { client, repo } = makeClient(options)
const branch = options.branch ?? getCurrentBranch()
const commit = options.commit ?? getCurrentCommit()
log(
`Creating run: type=${options.type}, branch=${branch}, commit=${commit.slice(0, 10)}, purpose=${options.purpose ?? 'review'}${options.partial ? ', partial' : ''}`
)
const result = await client.createRun({
repoId: repo,
runType: options.type,
commitSha: commit,
branch,
prNumber: options.pr ? parseInt(options.pr, 10) : undefined,
snapshots: [],
purpose: options.purpose,
metadata: collectCIMetadata(),
isPartial: options.partial,
})
log(`Run created: ${result.run_id}`)
return result.run_id
}
// The map only attributes snapshots to teams, so a map that cannot be read or sent must not fail the
// upload of the snapshots themselves.
function readStoryIndex(
indexPath: string | undefined,
storybookRoot: string | undefined,
runId: string
): StoryIndexMap | undefined {
if (!indexPath || !storybookRoot) {
return undefined
}
try {
const map = buildStoryIndex(readFileSync(indexPath, 'utf-8'), storybookRoot)
log(`[run:${runId}] Story index: ${map.storyCount} stories, ${map.hash.slice(0, 12)}`)
return map
} catch (error) {
log(`[run:${runId}] Could not read the Storybook index, sending snapshots without it: ${error}`)
return undefined
}
}
async function runUpload(options: RunUploadOptions): Promise<void> {
const { client } = makeClient(options)
const runId = options.runId
const dirPath = resolve(options.dir)
log(`[run:${runId}] Scanning ${dirPath} for PNGs...`)
const scanned = scanDirectory(dirPath)
if (scanned.length === 0) {
log(`[run:${runId}] No PNGs found — nothing to upload`)
return
}
const snapshots: Array<{ identifier: string; hash: string; width: number; height: number; data: Buffer }> = []
log(`[run:${runId}] Found ${scanned.length} snapshots, hashing...`)
const HASH_CONCURRENCY = 16
for (let i = 0; i < scanned.length; i += HASH_CONCURRENCY) {
const batch = scanned.slice(i, i + HASH_CONCURRENCY)
const results = await Promise.all(
batch.map(async ({ identifier, filePath }) => {
const data = await readFile(filePath)
const { hash, width, height } = await hashImageWithDimensions(data)
return { identifier, hash, width, height, data }
})
)
snapshots.push(...results)
}
log(`[run:${runId}] Sending ${snapshots.length} snapshots to backend`)
const storyIndex = readStoryIndex(options.storybookIndex, options.storybookRoot, runId)
const addResult = await client.addSnapshots(runId, {
snapshots: snapshots.map((s) => ({
identifier: s.identifier,
content_hash: s.hash,
width: s.width,
height: s.height,
})),
storyIndexHash: storyIndex?.hash,
})
if (storyIndex && addResult.story_index_upload) {
try {
await client.uploadToS3(addResult.story_index_upload, storyIndex.content, 'application/json')
log(`[run:${runId}] Uploaded story index ${storyIndex.hash.slice(0, 12)}`)
} catch (error) {
log(`[run:${runId}] Story index upload failed: ${error}`)
}
}
log(`[run:${runId}] Registered ${addResult.added} snapshot(s), ${addResult.uploads.length} upload(s) needed`)
if (addResult.uploads.length > 0) {
const hashToSnapshot = new Map(snapshots.map((s) => [s.hash, s]))
const CONCURRENCY = 10
let uploaded = 0
let failed = 0
for (let i = 0; i < addResult.uploads.length; i += CONCURRENCY) {
const batch = addResult.uploads.slice(i, i + CONCURRENCY)
await Promise.all(
batch.map(async (upload) => {
const snapshot = hashToSnapshot.get(upload.content_hash)
if (!snapshot) {
return
}
try {
await client.uploadToS3(upload, snapshot.data)
uploaded++
} catch (error) {
failed++
log(`[run:${runId}] Upload failed ${snapshot.identifier}: ${error}`)
}
})
)
}
log(`[run:${runId}] Uploaded ${uploaded} artifact(s)${failed > 0 ? `, ${failed} failed` : ''}`)
if (failed > 0) {
throw new Error(`[run:${runId}] ${failed} artifact upload(s) failed`)
}
}
}
async function runComplete(options: RunCompleteOptions): Promise<number> {
const { client, api, team } = makeClient(options)
const baselinePath = resolve(options.baseline)
const runId = options.runId
log(`[run:${runId}] Completing run`)
let run = await client.completeRun(runId, process.env.JOB_CHECK_RUN_ID)
log(`[run:${runId}] Status: ${run.status}`)
if (run.status !== 'completed' && run.status !== 'failed') {
log(`[run:${runId}] Waiting for diff processing...`)
run = await waitForCompletion(client, runId)
}
const s = run.summary
log(
`[run:${runId}] Done: ${s.total} snapshots — ${s.unchanged} unchanged, ${s.changed} changed, ${s.new} new, ${s.removed} removed`
)
if (options.autoApprove) {
log(`[run:${runId}] Auto-approving all changes...`)
const approveResult = await client.autoApproveRun(runId)
writeFileSync(baselinePath, approveResult.baseline_content, 'utf-8')
log(`[run:${runId}] Baseline written to ${baselinePath}`)
return 0
}
return reportRunOutcome(
client,
run,
`${api}/project/${team}/visual-review/runs/${runId}`,
options.purpose ?? 'review',
options.tolerateDrift ?? false
)
}
// --- Legacy command implementations ---
async function runVerify(options: VerifyOptions): Promise<number> {
const dirPath = resolve(options.dir)
const baselinePath = resolve(options.baseline)
const scanned = scanDirectory(dirPath)
const baselineHashes = readBaselineHashes(baselinePath)
if (Object.keys(baselineHashes).length === 0) {
console.error('No baseline hashes found — run `vr submit` on a PR first')
return 1
}
if (scanned.length === 0) {
// A shard whose story files all skip their screenshot writes no PNG, and that is not a mismatch.
if (options.partial) {
log('No PNGs found in directory, nothing to verify')
return 0
}
console.error('No PNGs found in directory')
return 1
}
log(`Found ${scanned.length} snapshots, verifying against ${Object.keys(baselineHashes).length} baselines...`)
const changed: string[] = []
const added: string[] = []
for (const { identifier, filePath } of scanned) {
const data = readFileSync(filePath)
const { hash } = await hashImageWithDimensions(data)
const baselineSignedHash = baselineHashes[identifier]
if (!baselineSignedHash) {
added.push(identifier)
} else {
if (hash !== extractContentHash(baselineSignedHash)) {
changed.push(identifier)
}
}
}
const currentIds = new Set(scanned.map((s) => s.identifier))
const removed = options.partial ? [] : Object.keys(baselineHashes).filter((id) => !currentIds.has(id))
const unchanged = scanned.length - changed.length - added.length
log(
`\n${scanned.length} snapshots — ${unchanged} unchanged, ${changed.length} changed, ${added.length} new, ${removed.length} removed`
)
if (changed.length > 0) {
log('\nChanged:')
changed.forEach((id) => log(` - ${id}`))
}
if (added.length > 0) {
log('\nNew:')
added.forEach((id) => log(` - ${id}`))
}
if (removed.length > 0) {
log('\nRemoved:')
removed.forEach((id) => log(` - ${id}`))
}
if (changed.length > 0 || added.length > 0 || removed.length > 0) {
log('\nBaseline mismatch — submit a PR to update baselines')
return 1
}
log('All snapshots match baseline')
return 0
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}
async function waitForCompletion(client: VisualReviewClient, runId: string): Promise<Run> {
const maxAttempts = 120
const intervalMs = 5000
for (let i = 0; i < maxAttempts; i++) {
const run = await client.getRun(runId)
if (run.status === 'completed' || run.status === 'failed') {
return run
}
if (i > 0 && i % 6 === 0) {
log(`[run:${runId}] Still processing (${(i * intervalMs) / 1000}s elapsed, status=${run.status})`)
}
await sleep(intervalMs)
}
throw new Error(`[run:${runId}] Run did not complete within ${(maxAttempts * intervalMs) / 1000}s`)
}
async function runSubmit(options: SubmitOptions): Promise<number> {
const { api, team, repo } = resolveConfig(options)
const client = new VisualReviewClient({
apiUrl: api,
teamId: team,
token: options.token,
sessionCookie: options.cookie,
})
// 1. Scan directory for PNGs
const dirPath = resolve(options.dir)
log(`Scanning ${dirPath} for PNGs...`)
const scanned = scanDirectory(dirPath)
if (scanned.length === 0) {
console.error('No PNGs found in directory')
return 1
}
// 2. Hash each image (decode PNG → RGBA → SHA256)
const snapshots: Array<{
identifier: string
hash: string
width: number
height: number
data: Buffer
}> = []
log(`Found ${scanned.length} snapshots, hashing...`)
const HASH_CONCURRENCY = 16
for (let i = 0; i < scanned.length; i += HASH_CONCURRENCY) {
const batch = scanned.slice(i, i + HASH_CONCURRENCY)
const results = await Promise.all(
batch.map(async ({ identifier, filePath }) => {
const data = await readFile(filePath)
const { hash, width, height } = await hashImageWithDimensions(data)
return { identifier, hash, width, height, data }
})
)
snapshots.push(...results)
}
// 3. Create run with full manifest — backend fetches baseline and classifies
const branch = options.branch ?? getCurrentBranch()
const commit = options.commit ?? getCurrentCommit()
// --auto-approve only makes sense on review runs; observe runs are non-approvable.
const purpose = options.autoApprove ? 'review' : (options.purpose ?? 'review')
if (options.autoApprove && options.purpose && options.purpose !== 'review') {
log(`Warning: --auto-approve forces purpose=review (got --purpose ${options.purpose})`)
}
log(
`Creating run: type=${options.type}, ${snapshots.length} snapshots, branch=${branch}, commit=${commit.slice(0, 10)}, purpose=${purpose}`
)
const result = await client.createRun({
repoId: repo,
runType: options.type,
commitSha: commit,
branch,
prNumber: options.pr ? parseInt(options.pr, 10) : undefined,
purpose,
snapshots: snapshots.map((s) => ({
identifier: s.identifier,
content_hash: s.hash,
width: s.width,
height: s.height,
})),
metadata: collectCIMetadata(),
})
const runId = result.run_id
log(
`[run:${runId}] Run created, ${result.uploads.length} upload(s) needed, ${snapshots.length - result.uploads.length} already exist`
)
if (result.uploads.length > 0) {
const hashToSnapshot = new Map(snapshots.map((s) => [s.hash, s]))
const CONCURRENCY = 10
let uploaded = 0
let failed = 0
const uploadOne = async (upload: (typeof result.uploads)[number]): Promise<void> => {
const snapshot = hashToSnapshot.get(upload.content_hash)
if (!snapshot) {
return
}
try {
await client.uploadToS3(upload, snapshot.data)
uploaded++
} catch (error) {
failed++
log(`[run:${runId}] Upload failed ${snapshot.identifier}: ${error}`)
}
}
for (let i = 0; i < result.uploads.length; i += CONCURRENCY) {
const batch = result.uploads.slice(i, i + CONCURRENCY)
await Promise.all(batch.map(uploadOne))
}
log(`[run:${runId}] Uploaded ${uploaded} artifact(s)${failed > 0 ? `, ${failed} failed` : ''}`)
if (failed > 0) {
throw new Error(`[run:${runId}] ${failed} artifact upload(s) failed`)
}
}
let run = await client.completeRun(runId, process.env.JOB_CHECK_RUN_ID)
log(`[run:${runId}] Status: ${run.status}`)
if (run.status !== 'completed' && run.status !== 'failed') {
log(`[run:${runId}] Waiting for diff processing...`)
run = await waitForCompletion(client, runId)
}
const s = run.summary
log(
`[run:${runId}] Done: ${s.total} snapshots — ${s.unchanged} unchanged, ${s.changed} changed, ${s.new} new, ${s.removed} removed`
)
if (options.autoApprove) {
log(`[run:${runId}] Auto-approving all changes...`)
const approveResult = await client.autoApproveRun(runId)
const baselinePath = resolve(options.baseline)
writeFileSync(baselinePath, approveResult.baseline_content, 'utf-8')
log(`[run:${runId}] Baseline written to ${baselinePath}`)
return 0
}
return reportRunOutcome(
client,
run,
`${api}/project/${team}/visual-review/runs/${runId}`,
purpose,
options.tolerateDrift ?? false
)
}