Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.3 KB
Raw Blame
import { DateTime } from 'luxon'
import { Message, MessageHeader } from 'node-rdkafka'
import { gunzipSync } from 'zlib'

import { parseJSON } from '~/common/utils/json-parse'
import { normalizeSessionId } from '~/common/utils/utils'
import { dlq, drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import {
    EventSchema,
    ParsedMessageData,
    RawEventMessageSchema,
    SnapshotEvent,
    SnapshotEventSchema,
} from '~/ingestion/pipelines/sessionreplay/kafka/types'
import { SessionRecordingIngesterMetrics } from '~/ingestion/pipelines/sessionreplay/metrics'

import { SessionReplayHeaders } from './pipeline-types'

const lz4: { decodeBlock(input: Buffer, output: Buffer): number } = require('lz4')

const MESSAGE_TIMESTAMP_DIFF_THRESHOLD_DAYS = 7
const CLOCK_SKEW_QUANTUM_MS = 5 * 60 * 1000
const GZIP_HEADER = Uint8Array.from([0x1f, 0x8b, 0x08, 0x00])
// Compression-bomb cap (mirrors the Rust addon): ~6x the 10 MB largest payload in a production
// sample; exceeding it DLQs instead of risking an unclassifiable OOM.
const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024

export interface MessageLimits {
    maxWindowIdBytes: number
    maxEventsPerMessage: number
    enforce: boolean
}

const DEFAULT_MESSAGE_LIMITS: MessageLimits = { maxWindowIdBytes: 1024, maxEventsPerMessage: 100_000, enforce: false }

export interface ParseMessageStepInput {
    message: Message
    headers: SessionReplayHeaders
}

export interface ParseMessageStepOutput {
    parsedMessage: ParsedMessageData
}

export function isGzipped(buffer: Buffer): boolean {
    return buffer.subarray(0, GZIP_HEADER.length).equals(GZIP_HEADER)
}

export function getContentEncoding(headers: MessageHeader[] | undefined): string | null {
    if (!headers) {
        return null
    }
    for (const header of headers) {
        const value = header['content-encoding']
        if (value !== undefined) {
            return typeof value === 'string' ? value : value.toString()
        }
    }
    return null
}

/**
 * Decompress a Kafka message's value (lz4 via the content-encoding header, gzip via its magic bytes,
 * else as-is) and record the encoding metric. Throws on corrupt compressed data — callers dlq with
 * `invalid_compressed_data`. Shared by the parse step and the fused native parse+anonymize step.
 */
export function decompressMessageValue(message: Message): Buffer {
    const value = message.value!
    const contentEncoding = getContentEncoding(message.headers)
    let messageUnzipped = value
    if (contentEncoding === 'lz4') {
        const uncompressedSize = value.readUInt32LE(0)
        if (uncompressedSize > MAX_DECOMPRESSED_BYTES) {
            throw new Error(`lz4 uncompressed size ${uncompressedSize} exceeds the decompression cap`)
        }
        const output = Buffer.allocUnsafe(uncompressedSize)
        const decodedLength = lz4.decodeBlock(value.subarray(4), output)
        // Without this check an inflated prefix leaks the uninitialized tail of `output`.
        if (decodedLength !== uncompressedSize) {
            throw new Error(`lz4 decoded ${decodedLength} bytes but the size prefix claimed ${uncompressedSize}`)
        }
        messageUnzipped = output
    } else if (isGzipped(value)) {
        messageUnzipped = gunzipSync(value, { maxOutputLength: MAX_DECOMPRESSED_BYTES })
    }
    SessionRecordingIngesterMetrics.incrementMessagesByEncoding(contentEncoding ?? (isGzipped(value) ? 'gzip' : 'none'))
    return messageUnzipped
}

export function statedDecodedBytes(message: Message): number {
    const value = message.value
    if (!value) {
        return 0
    }
    let stated = 0
    if (value.length >= 4) {
        if (getContentEncoding(message.headers) === 'lz4') {
            stated = value.readUInt32LE(0)
        } else if (isGzipped(value)) {
            stated = value.readUInt32LE(value.length - 4)
        }
    }
    return Math.max(value.length, stated <= MAX_DECOMPRESSED_BYTES ? stated : 0)
}

export function splitByDecodedBytes(messages: Message[], maxBytes: number): Message[][] {
    if (maxBytes <= 0 || messages.length === 0) {
        return [messages]
    }
    const parts: Message[][] = []
    let current: Message[] = []
    let currentBytes = 0
    for (const message of messages) {
        const bytes = statedDecodedBytes(message)
        if (current.length > 0 && currentBytes + bytes > maxBytes) {
            parts.push(current)
            current = []
            currentBytes = 0
        }
        current.push(message)
        currentBytes += bytes
    }
    parts.push(current)
    return parts
}

// null when capture recorded no usable pair, which must not read as zero skew.
function rawClockSkewMs(sentAt: string | undefined, now: string | undefined): number | null {
    if (!sentAt || !now) {
        return null
    }
    const sentAtMs = DateTime.fromISO(sentAt).toMillis()
    const nowMs = DateTime.fromISO(now).toMillis()
    if (!Number.isFinite(sentAtMs) || !Number.isFinite(nowMs)) {
        return null
    }
    return sentAtMs - nowMs
}

// The raw skew includes per-request network latency. Rounding to a coarse quantum gives every
// message of a session the same correction, so latency cannot reorder events the player sorts by
// timestamp, and a well-behaved clock rounds to zero.
function quantizedClockSkewMs(sentAt: string | undefined, now: string | undefined): number {
    const rawSkew = rawClockSkewMs(sentAt, now)
    if (rawSkew === null) {
        return 0
    }
    return Math.round(rawSkew / CLOCK_SKEW_QUANTUM_MS) * CLOCK_SKEW_QUANTUM_MS
}

function getValidEvents(
    events: unknown[],
    clockSkewMs: number
): {
    validEvents: SnapshotEvent[]
    startDateTime: DateTime
    endDateTime: DateTime
} | null {
    const eventsWithDates = events
        .map((event) => {
            const parseResult = SnapshotEventSchema.safeParse(event)
            if (!parseResult.success || parseResult.data.timestamp <= 0) {
                return null
            }
            // Corrected on the event object because the recorder serializes it into the blob.
            parseResult.data.timestamp -= clockSkewMs
            return {
                event: parseResult.data,
                dateTime: DateTime.fromMillis(parseResult.data.timestamp),
            }
        })
        .filter((x): x is { event: SnapshotEvent; dateTime: DateTime } => x !== null)
        .filter(({ dateTime }) => dateTime.isValid)

    if (!eventsWithDates.length) {
        return null
    }

    let startDateTime = eventsWithDates[0].dateTime
    let endDateTime = eventsWithDates[0].dateTime
    for (const { dateTime } of eventsWithDates) {
        if (dateTime < startDateTime) {
            startDateTime = dateTime
        }
        if (dateTime > endDateTime) {
            endDateTime = dateTime
        }
    }

    return {
        validEvents: eventsWithDates.map(({ event }) => event),
        startDateTime,
        endDateTime,
    }
}

/**
 * Record the sender's clock offset: the device clock at upload (`sent_at`) against the server clock
 * at receipt (`now`). Capture stamps both at the same instant, so buffering shifts them equally and
 * cancels; the Kafka message time against rrweb event times does not cancel. Capture omits `sent_at`
 * when the client sends none, so those messages count as unmeasured rather than as zero.
 */
export function observeClockSkew(sentAt: string | undefined, now: string | undefined): void {
    const skewMs = rawClockSkewMs(sentAt, now)
    if (skewMs === null) {
        SessionRecordingIngesterMetrics.incrementMessageClockSkewUnmeasured()
        return
    }
    const direction = skewMs >= 0 ? 'device_ahead' : 'device_behind'
    SessionRecordingIngesterMetrics.observeMessageClockSkew(direction, Math.abs(skewMs) / 1000)
}

/**
 * Creates a step that parses raw Kafka messages into ParsedMessageData.
 * This step is additive - it preserves all input properties and adds parsedMessage.
 *
 * This step processes one message at a time since there are no batch-level optimizations.
 * Gzip decompression is done synchronously since the pipeline already runs steps concurrently.
 */
export function createParseMessageStep<T extends ParseMessageStepInput>(
    limits: MessageLimits = DEFAULT_MESSAGE_LIMITS
): ProcessingStep<T, T & ParseMessageStepOutput> {
    return async function parseMessageStep(input) {
        const { message } = input

        if (!message.value || !message.timestamp) {
            return dlq('message_value_or_timestamp_is_empty')
        }
        let messageUnzipped: Buffer
        try {
            messageUnzipped = decompressMessageValue(message)
        } catch (error) {
            return dlq('invalid_compressed_data', error)
        }

        let rawPayload: unknown
        try {
            rawPayload = parseJSON(messageUnzipped.toString())
        } catch (error) {
            return dlq('invalid_json', error)
        }

        const messageResult = RawEventMessageSchema.safeParse(rawPayload)
        if (!messageResult.success) {
            return dlq('invalid_message_payload', messageResult.error)
        }

        let eventData: unknown
        try {
            eventData = parseJSON(messageResult.data.data)
        } catch (error) {
            return dlq('received_non_snapshot_message', error)
        }
        const eventResult = EventSchema.safeParse(eventData)
        if (!eventResult.success) {
            return dlq('received_non_snapshot_message', eventResult.error)
        }

        const { $snapshot_items, $session_id, $window_id, $snapshot_source, $lib } = eventResult.data.properties

        if (eventResult.data.event !== '$snapshot_items' || !$snapshot_items || !$session_id) {
            return dlq('received_non_snapshot_message')
        }

        const sessionId = normalizeSessionId($session_id)

        const windowIdBytes = $window_id ? Buffer.byteLength($window_id) : 0
        const events = $snapshot_items.length
        SessionRecordingIngesterMetrics.observeMessageShape(windowIdBytes, events)
        const overLimit =
            windowIdBytes > limits.maxWindowIdBytes
                ? 'message_window_id_too_long'
                : events > limits.maxEventsPerMessage
                  ? 'message_too_many_events'
                  : null
        if (overLimit) {
            SessionRecordingIngesterMetrics.incrementMessagesOverLimit(overLimit)
            if (limits.enforce) {
                return drop(overLimit, [], [{ type: overLimit, details: { sessionId, windowIdBytes, events } }])
            }
        }

        const clockSkewMs = quantizedClockSkewMs(messageResult.data.sent_at, messageResult.data.now)
        if (clockSkewMs !== 0) {
            SessionRecordingIngesterMetrics.observeClockSkewCorrection(clockSkewMs)
        }
        const result = getValidEvents($snapshot_items, clockSkewMs)
        if (!result) {
            return drop(
                'message_contained_no_valid_rrweb_events',
                [],
                [
                    {
                        type: 'message_contained_no_valid_rrweb_events',
                        details: {},
                    },
                ]
            )
        }
        const { validEvents, startDateTime, endDateTime } = result

        const startDiff = Math.abs(startDateTime.diffNow('day').days)
        const endDiff = Math.abs(endDateTime.diffNow('day').days)
        if (startDiff >= MESSAGE_TIMESTAMP_DIFF_THRESHOLD_DAYS || endDiff >= MESSAGE_TIMESTAMP_DIFF_THRESHOLD_DAYS) {
            return drop(
                'message_timestamp_diff_too_large',
                [],
                [
                    {
                        type: 'message_timestamp_diff_too_large',
                        details: {
                            startDiffDays: startDiff,
                            endDiffDays: endDiff,
                            thresholdDays: MESSAGE_TIMESTAMP_DIFF_THRESHOLD_DAYS,
                        },
                    },
                ]
            )
        }

        // session_id and distinct_id are carried both in the headers (set by capture) and in the
        // message body; they must agree — a mismatch means the message is corrupt or mis-routed.
        // headers.session_id is already normalized by the validate step, matching the body's.
        const { headers } = input
        if (headers.session_id !== sessionId) {
            return dlq('session_id_header_body_mismatch')
        }
        if (headers.distinct_id !== messageResult.data.distinct_id) {
            return dlq('distinct_id_header_body_mismatch')
        }

        observeClockSkew(messageResult.data.sent_at, messageResult.data.now)

        const parsedMessage: ParsedMessageData = {
            metadata: {
                partition: message.partition,
                topic: message.topic,
                rawSize: message.size,
                offset: message.offset,
                timestamp: message.timestamp,
            },
            distinct_id: messageResult.data.distinct_id,
            session_id: sessionId,
            token: headers.token,
            eventsByWindowId: {
                [$window_id ?? '']: validEvents,
            },
            eventsRange: {
                start: startDateTime,
                end: endDateTime,
            },
            snapshot_source: $snapshot_source ?? null,
            snapshot_library: $lib ?? null,
        }

        return Promise.resolve(ok({ ...input, parsedMessage }))
    }
}