Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.5 KB
Raw Blame
import { METADATA_FOOTER_HEIGHT_PX, NavigationSample } from '@posthog/replay-headless/protocol'

import {
    InactivityPeriod,
    NavigationChange,
    RasterizeRecordingInput,
    RasterizeRecordingOutput,
    RecordingResult,
} from './types'

/**
 * Place each period on the video clock from the timeline measured during capture.
 *
 * `frameSessionMs[i]` is where playback was on captured frame `i`, and one captured frame is one
 * frame of the rendered video, so a period's video position is the frame range whose session times
 * fall inside it. Measuring beats deriving: a skip costs the frame it happens on, and any frame
 * spent elsewhere counts too, neither of which segment durations can predict.
 */
export function videoTimestampsFromFrames(
    periods: InactivityPeriod[],
    frameSessionMs: number[],
    fps: number,
    preRollFrames = 0
): InactivityPeriod[] {
    if (frameSessionMs.length === 0 || fps <= 0) {
        return computeVideoTimestamps(periods)
    }
    // Sample `i` is the frame `preRollFrames + i` of the file: capture runs while the player is still
    // starting, and those frames carry no sample.
    const videoTimeOf = (sample: number): number => (preRollFrames + sample) / fps
    const lastPeriod = periods.length - 1
    // A render can start or stop partway through a stretch, via start_offset_s, a trim, or a timeout.
    // The stretch then spans session time the file never shows, and a consumer interpolating across it
    // reads every moment inside as later than it is. Hold those two edges to what was captured.
    const firstSampleS = frameSessionMs[0] / 1000
    const lastSampleS = frameSessionMs[frameSessionMs.length - 1] / 1000
    // Periods run in order and samples never go backwards, so one cursor walks both. Rescanning the
    // samples per period would be quadratic, and a long recording with many activity changes has
    // hundreds of thousands of each.
    let cursor = 0
    return periods.map((period, index) => {
        const fromMs = period.ts_from_s * 1000
        const toMs = period.ts_to_s != null ? period.ts_to_s * 1000 : Number.POSITIVE_INFINITY
        // Half-open, so a frame sitting exactly on a boundary belongs to the period it starts, not the
        // one it ends. The last period takes its own end, or the final frame would belong to nothing.
        const owns = (t: number): boolean => t >= fromMs && (t < toMs || (index === lastPeriod && t <= toMs))
        while (cursor < frameSessionMs.length && frameSessionMs[cursor] < fromMs) {
            cursor++
        }
        let first = -1
        let last = -1
        while (cursor < frameSessionMs.length && owns(frameSessionMs[cursor])) {
            if (first === -1) {
                first = cursor
            }
            last = cursor
            cursor++
        }
        if (first === -1) {
            // Never on screen: sit at the frame where playback resumed, which is where the cursor now
            // rests. It stops on the first sample at or past this period's end, so the frame landing
            // exactly on that end is taken rather than stepped over.
            const at = videoTimeOf(cursor)
            return { ...period, recording_ts_from_s: at, recording_ts_to_s: at }
        }
        // Only the stretch the capture began in, and the one it ended in, can be cut by it.
        const startsMidPeriod =
            firstSampleS > period.ts_from_s && (period.ts_to_s == null || firstSampleS < period.ts_to_s)
        const endsMidPeriod = period.ts_to_s != null && lastSampleS < period.ts_to_s && lastSampleS >= period.ts_from_s
        return {
            ...period,
            ts_from_s: startsMidPeriod ? firstSampleS : period.ts_from_s,
            ts_to_s: endsMidPeriod ? lastSampleS : period.ts_to_s,
            recording_ts_from_s: videoTimeOf(first),
            recording_ts_to_s: videoTimeOf(last + 1),
        }
    })
}

/**
 * Predicted video-time positions, used when no measured timeline is available.
 *
 * Assumes an inactive period costs no video time, which understates the real video by about a frame
 * per skip. Prefer `videoTimestampsFromFrames`.
 */
export function computeVideoTimestamps(periods: InactivityPeriod[]): InactivityPeriod[] {
    // Pass 1: compute raw video timestamps
    let videoTime = 0
    const results: InactivityPeriod[] = periods.map((period) => {
        if (period.active) {
            const recordingTsFromS = videoTime
            const duration = period.ts_to_s != null ? period.ts_to_s - period.ts_from_s : 0
            videoTime += duration
            return { ...period, recording_ts_from_s: recordingTsFromS, recording_ts_to_s: videoTime }
        } else {
            return { ...period, recording_ts_from_s: videoTime, recording_ts_to_s: videoTime }
        }
    })

    // Pass 2: clamp active periods so they don't overlap the next active period's start
    for (let i = 0; i < results.length; i++) {
        if (!results[i].active) {
            continue
        }
        // Find the next active period
        for (let j = i + 1; j < results.length; j++) {
            if (results[j].active) {
                if (results[i].recording_ts_to_s! > results[j].recording_ts_from_s!) {
                    results[i] = { ...results[i], recording_ts_to_s: results[j].recording_ts_from_s }
                }
                break
            }
        }
    }

    return results
}

export function navigationFromFrames(
    samples: NavigationSample[],
    frameSessionMs: number[],
    videoDurationS: number,
    fps: number,
    preRollFrames: number
): NavigationChange[] {
    if (fps <= 0) {
        return []
    }
    // A change revealed by a skip is recorded on the skip frame, which the citation clock gives to the time before
    // the skip. Place it on the first frame whose sample has reached the time the change shows.
    const firstFrameShowing = (entry: NavigationSample): number => {
        let frame = entry.sample
        while (frame < frameSessionMs.length - 1 && frameSessionMs[frame] < entry.session_ms - 1) {
            frame++
        }
        return frame
    }
    return (
        samples
            .map((entry) => ({
                video_s: (preRollFrames + firstFrameShowing(entry)) / fps,
                session_s: entry.session_ms / 1000,
                window_id: entry.window_id,
                window_number: entry.window_number,
                url: entry.url,
            }))
            // A trimmed capture keeps sampling briefly after the encoder stops writing.
            .filter((change) => change.video_s < videoDurationS)
    )
}

/** The render's output metadata, shared by the Temporal activity and the file CLI so the two cannot drift. */
export function renderOutputFields(
    result: RecordingResult,
    input: RasterizeRecordingInput
): Omit<RasterizeRecordingOutput, 's3_uri' | 'file_size_bytes' | 'timings'> {
    return {
        video_duration_s: result.capture_duration_s,
        playback_speed: result.playback_speed,
        show_metadata_footer: !!input.show_metadata_footer,
        footer_height_px: input.show_metadata_footer ? METADATA_FOOTER_HEIGHT_PX : 0,
        truncated: result.truncated,
        inactivity_periods: videoTimestampsFromFrames(
            result.inactivity_periods,
            result.frame_session_ms,
            result.output_fps,
            result.pre_roll_frames
        ),
        navigation: navigationFromFrames(
            result.navigation_samples,
            result.frame_session_ms,
            result.capture_duration_s,
            result.output_fps,
            result.pre_roll_frames
        ),
    }
}