Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.3 KB
Raw Blame
import { execFile } from 'child_process'
import * as fs from 'fs/promises'
import * as os from 'os'
import * as path from 'path'
import { promisify } from 'util'

import { toFiniteNumber } from './capture/config'
import { RasterizationError } from './errors'
import { createLogger } from './logger'
import { downloadFromS3, parseS3Uri, uploadToS3 } from './storage'
import type { ExtractThumbnailsInput, ExtractThumbnailsOutput } from './types'

const execFileAsync = promisify(execFile)
const log = createLogger()

// One frame out of an existing MP4 never approaches the render timeouts; a longer wait means ffmpeg
// is wedged on a corrupt file rather than working.
const FFMPEG_TIMEOUT_MS = 60_000

export interface Rect {
    w: number
    h: number
    x: number
    y: number
}

const BORDER_TOLERANCE = 6
// Every 8th row/column is enough to tell a uniform bar from page content, and keeps the scan cheap.
const SCAN_STEP = 8

function matchesBorder(frame: Buffer, offset: number, r: number, g: number, b: number): boolean {
    return (
        Math.abs(frame[offset] - r) <= BORDER_TOLERANCE &&
        Math.abs(frame[offset + 1] - g) <= BORDER_TOLERANCE &&
        Math.abs(frame[offset + 2] - b) <= BORDER_TOLERANCE
    )
}

/** The page inside the analysis canvas, when a phone-sized session leaves a uniform border around it. */
// Corner colour rather than a luma threshold: the background is not black, so cropdetect misses it, and
// a threshold high enough to catch it crops into a dark page.
export function uniformBorderRect(frame: Buffer, width: number, height: number): Rect | null {
    if (frame.length < width * height * 3) {
        return null
    }
    const [r, g, b] = [frame[0], frame[1], frame[2]]
    const at = (x: number, y: number): number => (y * width + x) * 3

    const columnIsBorder = (x: number): boolean => {
        for (let y = 0; y < height; y += SCAN_STEP) {
            if (!matchesBorder(frame, at(x, y), r, g, b)) {
                return false
            }
        }
        return true
    }
    const rowIsBorder = (y: number): boolean => {
        for (let x = 0; x < width; x += SCAN_STEP) {
            if (!matchesBorder(frame, at(x, y), r, g, b)) {
                return false
            }
        }
        return true
    }

    let left = 0
    while (left < width && columnIsBorder(left)) {
        left++
    }
    if (left === width) {
        // A frame of nothing but border: the render is blank, so there is no page to find.
        return null
    }
    let right = width - 1
    while (right > left && columnIsBorder(right)) {
        right--
    }
    let top = 0
    while (top < height && rowIsBorder(top)) {
        top++
    }
    let bottom = height - 1
    while (bottom > top && rowIsBorder(bottom)) {
        bottom--
    }

    const rect = { w: right - left + 1, h: bottom - top + 1, x: left, y: top }
    if (rect.w < width * 0.1 || rect.h < height * 0.1) {
        return null
    }
    // Worth a second ffmpeg pass only when it removes a real share of the frame.
    if (1 - (rect.w * rect.h) / (width * height) < 0.15) {
        return null
    }
    return rect
}

async function frameSize(sourcePath: string): Promise<{ width: number; height: number } | null> {
    try {
        const { stdout } = await execFileAsync(
            'ffprobe',
            [
                '-v',
                'error',
                '-select_streams',
                'v:0',
                '-show_entries',
                'stream=width,height',
                '-of',
                'csv=p=0',
                sourcePath,
            ],
            { timeout: FFMPEG_TIMEOUT_MS }
        )
        const [width, height] = stdout.trim().split(',').map(Number)
        return Number.isFinite(width) && Number.isFinite(height) ? { width, height } : null
    } catch (err) {
        log.warn({ err: (err as Error)?.message }, 'ffprobe could not read the frame size')
        return null
    }
}

async function detectBorder(
    sourcePath: string,
    rawPath: string,
    videoTimeS: number,
    footerCrop: string,
    width: number,
    height: number
): Promise<Rect | null> {
    try {
        await execFileAsync(
            'ffmpeg',
            [
                '-nostdin',
                '-loglevel',
                'error',
                '-ss',
                String(videoTimeS),
                '-i',
                sourcePath,
                '-frames:v',
                '1',
                '-vf',
                footerCrop,
                '-pix_fmt',
                'rgb24',
                '-f',
                'rawvideo',
                '-y',
                rawPath,
            ],
            { timeout: FFMPEG_TIMEOUT_MS }
        )
        return uniformBorderRect(await fs.readFile(rawPath), width, height)
    } catch (err) {
        // Best-effort: a failed probe means the full frame, never a failed thumbnail.
        log.warn({ err: (err as Error)?.message }, 'border detection failed, keeping the full frame')
        return null
    }
}

interface FrameSource {
    workDir: string
    sourcePath: string
    footerCrop: string
    size: { width: number; height: number } | null
    footer: number
    width: number
    smallWidth: number | null
    format: 'png' | 'webp'
}

// Lossy, which is what keeps a poster of a text-heavy page small; the large copy keeps more detail for the detail view.
const WEBP_QUALITY = { large: 85, small: 80 }

async function withFrameSource<T>(
    input: {
        source_s3_uri: string
        footer_crop_px?: number
        width?: number
        small_width?: number
        image_format?: 'png' | 'webp'
    },
    run: (source: FrameSource) => Promise<T>
): Promise<T> {
    const location = parseS3Uri(input.source_s3_uri)
    const workDir = await fs.mkdtemp(path.join(process.env.VIDEO_WORK_DIR || os.tmpdir(), 'thumb-'))
    try {
        const sourcePath = path.join(workDir, 'source.mp4')
        await downloadFromS3(location.bucket, location.key, sourcePath)
        const footer = Math.max(0, Math.floor(input.footer_crop_px ?? 0))
        return await run({
            workDir,
            sourcePath,
            // crop before scale: the footer is measured in source pixels.
            footerCrop: `crop=iw:ih-${footer}:0:0`,
            size: await frameSize(sourcePath),
            footer,
            width: Math.max(1, Math.floor(input.width ?? 1280)),
            smallWidth: input.small_width ? Math.max(1, Math.floor(input.small_width)) : null,
            format: input.image_format ?? 'png',
        })
    } finally {
        await fs.rm(workDir, { recursive: true, force: true })
    }
}

function encoderArgs(format: 'png' | 'webp', copy: 'large' | 'small'): string[] {
    return format === 'webp' ? ['-c:v', 'libwebp', '-quality', String(WEBP_QUALITY[copy])] : []
}

async function outputSize(outputPath: string): Promise<number> {
    const stat = await fs.stat(outputPath).catch(() => null)
    return stat?.size ?? 0
}

/** Cut one frame into `outputPath`, and into `smallPath` too when the source asks for a small copy, from one decode. */
async function cutFrame(
    source: FrameSource,
    videoTimeS: number,
    outputPath: string,
    smallPath: string | null
): Promise<{ size: number; smallSize: number }> {
    const { size, footer, footerCrop, width, smallWidth, format } = source
    const rawPath = path.join(source.workDir, 'frame.rgb')
    const letterbox = size
        ? await detectBorder(source.sourcePath, rawPath, videoTimeS, footerCrop, size.width, size.height - footer)
        : null
    if (letterbox) {
        log.info({ ...letterbox, frame: size }, 'cropping the letterbox around the page')
    }

    const crops = letterbox
        ? [footerCrop, `crop=${letterbox.w}:${letterbox.h}:${letterbox.x}:${letterbox.y}`]
        : [footerCrop]
    // Never upscale: a phone-sized page stretched to 1280 is a blurrier, heavier poster.
    const sourceWidth = letterbox?.w ?? size?.width ?? width
    const scale = (target: number): string => `scale=${Math.min(target, sourceWidth)}:-2`

    // -ss before -i seeks by keyframe index rather than decoding to the timestamp, which is what
    // keeps this cheap. -frames:v 1 stops after the first frame it lands on.
    const outputs =
        smallPath !== null && smallWidth !== null
            ? [
                  '-filter_complex',
                  `[0:v]${crops.join(',')},split=2[a][b];[a]${scale(width)}[large];[b]${scale(smallWidth)}[small]`,
                  ...['-map', '[large]', '-frames:v', '1', ...encoderArgs(format, 'large'), '-y', outputPath],
                  ...['-map', '[small]', '-frames:v', '1', ...encoderArgs(format, 'small'), '-y', smallPath],
              ]
            : [
                  '-vf',
                  [...crops, scale(width)].join(','),
                  '-frames:v',
                  '1',
                  ...encoderArgs(format, 'large'),
                  '-y',
                  outputPath,
              ]
    const args = ['-nostdin', '-loglevel', 'error', '-ss', String(videoTimeS), '-i', source.sourcePath, ...outputs]

    try {
        await execFileAsync('ffmpeg', args, { timeout: FFMPEG_TIMEOUT_MS })
    } catch (err) {
        const stderr = (err as { stderr?: string })?.stderr?.slice(0, 500) ?? ''
        log.warn({ err: (err as Error)?.message, stderr }, 'thumbnail extraction failed')
        throw new RasterizationError(
            `ffmpeg could not extract a frame: ${(err as Error)?.message ?? String(err)}`,
            true,
            'THUMBNAIL_EXTRACT_FAILED',
            err
        )
    }

    const largeSize = await outputSize(outputPath)
    if (largeSize === 0) {
        // A seek past the end of the video exits 0 and writes nothing, so size is the real check.
        throw new RasterizationError(`ffmpeg wrote no frame at ${videoTimeS}s`, false, 'THUMBNAIL_EMPTY_OUTPUT')
    }
    return { size: largeSize, smallSize: smallPath ? await outputSize(smallPath) : 0 }
}

/**
 * Cut frames from an already-rendered analysis MP4 and store each as an image, downloading the video once.
 *
 * No browser, so this costs seconds of CPU rather than a recording load. The crop removes the
 * burned-in metadata footer, which is an artifact of the analysis render and not part of the page.
 * A frame past the video's end is left out of the output rather than failing the batch.
 */
export async function extractThumbnails(input: ExtractThumbnailsInput): Promise<ExtractThumbnailsOutput> {
    // Before the download: `-ss NaN` burns every attempt, each pulling the whole MP4 first.
    const frames = input.frames.map((frame) => ({
        id: frame.id,
        videoTimeS: Math.max(0, toFiniteNumber(frame.video_time_s, 'video_time_s')),
        required: frame.required ?? false,
    }))
    return withFrameSource(input, async (source) => {
        const extracted: ExtractThumbnailsOutput['frames'] = []
        for (const frame of frames) {
            const outputPath = path.join(source.workDir, `${frame.id}.${source.format}`)
            const smallId = `${frame.id}-small`
            const smallPath =
                source.smallWidth !== null ? path.join(source.workDir, `${smallId}.${source.format}`) : null
            let cut: { size: number; smallSize: number }
            try {
                cut = await cutFrame(source, frame.videoTimeS, outputPath, smallPath)
            } catch (err) {
                // A local cut fails the same way on a retry, so an optional frame is skipped rather than retried.
                if (!frame.required && err instanceof RasterizationError) {
                    log.warn({ id: frame.id, video_time_s: frame.videoTimeS, code: err.code }, 'skipping a frame')
                    continue
                }
                throw err
            }
            const [s3Uri, smallS3Uri] = await Promise.all([
                uploadToS3(outputPath, input.s3_bucket, input.s3_key_prefix, frame.id, source.format),
                smallPath !== null && cut.smallSize > 0
                    ? uploadToS3(smallPath, input.s3_bucket, input.s3_key_prefix, smallId, source.format)
                    : undefined,
            ])
            extracted.push({ id: frame.id, s3_uri: s3Uri, file_size_bytes: cut.size, small_s3_uri: smallS3Uri })
            await Promise.all(
                [outputPath, ...(smallPath ? [smallPath] : [])].map((file) => fs.rm(file, { force: true }))
            )
        }
        return { frames: extracted }
    })
}