Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

34.3 KB
Raw Blame
import { Message } from 'node-rdkafka'
import { promisify } from 'node:util'
import { gzip, gzipSync } from 'zlib'

import { PipelineResultType } from '~/ingestion/framework/results'

import { SessionRecordingIngesterMetrics } from './metrics'
import {
    ParseMessageStepInput,
    createParseMessageStep,
    decompressMessageValue,
    observeClockSkew,
    splitByDecodedBytes,
    statedDecodedBytes,
} from './parse-message-step'
import { SessionReplayHeaders } from './pipeline-types'

const compressWithGzip = promisify(gzip)

describe('createParseMessageStep', () => {
    // Use a fixed time for tests that need predictable timestamp behavior
    const fixedTime = new Date('2023-01-01T00:00:00.000Z')
    const fixedTimeMs = fixedTime.getTime() // 1672531200000

    afterEach(() => {
        jest.useRealTimers()
    })

    function createSnapshotPayload(options: {
        sessionId: string
        windowId?: string
        snapshotItems: Array<{ type: number; timestamp: number }>
        snapshotSource?: string
        lib?: string
        distinctId?: string
        sentAt?: string
        now?: string
    }): string {
        const event = {
            event: '$snapshot_items',
            properties: {
                $session_id: options.sessionId,
                $window_id: options.windowId ?? 'window-1',
                $snapshot_items: options.snapshotItems,
                ...(options.snapshotSource !== undefined && { $snapshot_source: options.snapshotSource }),
                ...(options.lib !== undefined && { $lib: options.lib }),
            },
        }
        const rawMessage: Record<string, unknown> = {
            data: JSON.stringify(event),
        }
        if (options.distinctId !== undefined) {
            rawMessage.distinct_id = options.distinctId
        }
        if (options.sentAt !== undefined) {
            rawMessage.sent_at = options.sentAt
        }
        if (options.now !== undefined) {
            rawMessage.now = options.now
        }
        return JSON.stringify(rawMessage)
    }

    function createValidSnapshotPayload(sessionId: string, windowId = 'window-1'): string {
        const now = Date.now()
        return createSnapshotPayload({
            sessionId,
            windowId,
            snapshotItems: [
                { type: 2, timestamp: now },
                { type: 3, timestamp: now + 1000 },
            ],
            distinctId: 'user-123',
        })
    }

    function createMessage(
        partition: number,
        offset: number,
        payload?: string,
        headers?: Record<string, string>,
        overrides?: Partial<Message>
    ): Message {
        const kafkaHeaders: Message['headers'] = headers
            ? Object.entries(headers).map(([key, value]) => ({ [key]: Buffer.from(value) }))
            : []

        const message: Message = {
            partition,
            offset,
            topic: 'test-topic',
            value: payload ? Buffer.from(payload) : null,
            key: Buffer.from('test-key'),
            timestamp: Date.now(),
            headers: kafkaHeaders,
            size: payload?.length ?? 0,
        }

        return { ...message, ...overrides }
    }

    // The validate step guarantees these before parse; they must mirror the message body's
    // session_id/distinct_id or parse DLQs the message as inconsistent.
    function createReplayHeaders(overrides: Partial<SessionReplayHeaders> = {}): SessionReplayHeaders {
        return { token: 'test-token', session_id: 'session-1', distinct_id: 'user-123', ...overrides }
    }

    function createInput(
        partition: number,
        offset: number,
        payload?: string,
        headers?: Record<string, string>,
        overrides?: Partial<Message>,
        replayHeaders: SessionReplayHeaders = createReplayHeaders()
    ): ParseMessageStepInput {
        return {
            message: createMessage(partition, offset, payload, headers, overrides),
            headers: replayHeaders,
        }
    }

    it('should parse valid messages and return ok results', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.session_id).toBe('session-1')
            expect(result.value.parsedMessage.distinct_id).toBe('user-123')
        }
    })

    it('should send messages with no value to DLQ', async () => {
        const step = createParseMessageStep()
        const input = createInput(0, 1, undefined)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('message_value_or_timestamp_is_empty')
        }
    })

    it('should send messages with invalid JSON to DLQ', async () => {
        const step = createParseMessageStep()
        const input = createInput(0, 1, 'not valid json')

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('invalid_json')
        }
    })

    it('should send messages with invalid message payload schema to DLQ', async () => {
        const step = createParseMessageStep()
        const input = createInput(0, 1, JSON.stringify({ invalid: 'schema' }))

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('invalid_message_payload')
        }
    })

    it('should send non-snapshot messages to DLQ', async () => {
        const step = createParseMessageStep()
        const payload = JSON.stringify({
            distinct_id: 'user-123',
            data: JSON.stringify({ event: 'some_other_event', properties: {} }),
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('received_non_snapshot_message')
        }
    })

    it('should drop messages with no valid rrweb events with ingestion warning', async () => {
        const step = createParseMessageStep()
        const event = {
            event: '$snapshot_items',
            properties: {
                $session_id: 'session-1',
                $window_id: 'window-1',
                $snapshot_items: [], // Empty events array
            },
        }
        const payload = JSON.stringify({
            distinct_id: 'user-123',
            data: JSON.stringify(event),
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DROP)
        if (result.type === PipelineResultType.DROP) {
            expect(result.reason).toBe('message_contained_no_valid_rrweb_events')
            expect(result.warnings).toHaveLength(1)
            expect(result.warnings[0]).toEqual({
                type: 'message_contained_no_valid_rrweb_events',
                details: {},
            })
        }
    })

    describe('message limits', () => {
        const limits = { maxWindowIdBytes: 8, maxEventsPerMessage: 2 }
        const event = { type: 2, timestamp: fixedTimeMs }

        beforeEach(() => {
            jest.useFakeTimers({ now: fixedTime })
        })

        afterEach(() => {
            jest.restoreAllMocks()
        })

        it.each([
            [
                'the window ID',
                'w'.repeat(9),
                [event],
                'message_window_id_too_long',
                { sessionId: 'session-1', windowIdBytes: 9, events: 1 },
            ],
            [
                'the window ID in bytes',
                'é'.repeat(5),
                [event],
                'message_window_id_too_long',
                { sessionId: 'session-1', windowIdBytes: 10, events: 1 },
            ],
            [
                'the event count',
                'w',
                [event, event, event],
                'message_too_many_events',
                { sessionId: 'session-1', windowIdBytes: 1, events: 3 },
            ],
        ] as const)(
            'drops a message past the limit on %s when enforced',
            async (_, windowId, items, reason, details) => {
                const payload = createSnapshotPayload({
                    sessionId: 'session-1',
                    distinctId: 'user-123',
                    windowId,
                    snapshotItems: [...items],
                })

                const result = await createParseMessageStep({ ...limits, enforce: true })(createInput(0, 1, payload))

                expect(result).toMatchObject({
                    type: PipelineResultType.DROP,
                    reason,
                    warnings: [{ type: reason, details }],
                })
            }
        )

        it.each([
            ['past a limit, not enforced', 'w'.repeat(9), false, ['message_window_id_too_long']],
            ['at the limits, enforced', 'w'.repeat(8), true, []],
        ])('keeps a message %s', async (_, windowId, enforce, counted) => {
            const overLimitSpy = jest
                .spyOn(SessionRecordingIngesterMetrics, 'incrementMessagesOverLimit')
                .mockImplementation()
            const payload = createSnapshotPayload({
                sessionId: 'session-1',
                distinctId: 'user-123',
                windowId,
                snapshotItems: [event, event],
            })

            const result = await createParseMessageStep({ ...limits, enforce })(createInput(0, 1, payload))

            expect(result.type).toBe(PipelineResultType.OK)
            expect(overLimitSpy.mock.calls.map(([limit]) => limit)).toEqual(counted)
        })
    })

    describe('splitByDecodedBytes', () => {
        const raw = (bytes: number): Message => ({ value: Buffer.alloc(bytes) }) as Message
        const lz4Stating = (bytes: number): Message => {
            const value = Buffer.alloc(8)
            value.writeUInt32LE(bytes, 0)
            return { value, headers: [{ 'content-encoding': 'lz4' }] } as unknown as Message
        }

        it.each([
            ['keeps one part when off', [raw(10), raw(10)], 0, [2]],
            ['keeps messages that fit together', [raw(10), raw(10)], 20, [2]],
            ['starts a new part past the limit', [raw(10), raw(10), raw(1)], 15, [1, 2]],
            ['gives a message over the limit its own part', [raw(1), raw(30), raw(1)], 10, [1, 1, 1]],
            ['counts the stated lz4 size', [lz4Stating(1000), lz4Stating(1000)], 1500, [1, 1]],
            ['counts the bytes when a stated size is past the cap', [lz4Stating(2 ** 31), raw(4)], 20, [2]],
        ] as const)('%s', (_, messages, maxBytes, sizes) => {
            const parts = splitByDecodedBytes([...messages], maxBytes)

            expect(parts.map((part) => part.length)).toEqual(sizes)
            expect(parts.flat()).toEqual(messages)
        })

        it('counts the stated gzip size', () => {
            const gzipped = gzipSync(Buffer.alloc(1000))

            expect(statedDecodedBytes({ value: gzipped } as Message)).toBe(1000)
        })
    })

    it('should handle gzipped messages', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-gzip')
        const gzippedPayload = await compressWithGzip(Buffer.from(payload))

        const message: Message = {
            partition: 0,
            offset: 1,
            topic: 'test-topic',
            value: gzippedPayload,
            key: Buffer.from('test-key'),
            timestamp: Date.now(),
            headers: [],
            size: gzippedPayload.length,
        }

        const result = await step({ message, headers: createReplayHeaders({ session_id: 'session-gzip' }) })

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.session_id).toBe('session-gzip')
        }
    })

    it('should set token from the validated session replay headers', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(0, 1, payload, undefined, undefined, createReplayHeaders({ token: 'my-team-token' }))

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.token).toBe('my-team-token')
        }
    })

    it('DLQs when the header session_id does not match the message body', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(0, 1, payload, undefined, undefined, createReplayHeaders({ session_id: 'session-2' }))

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('session_id_header_body_mismatch')
        }
    })

    it('DLQs when the header distinct_id does not match the message body', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1') // body distinct_id is user-123
        const input = createInput(
            0,
            1,
            payload,
            undefined,
            undefined,
            createReplayHeaders({ distinct_id: 'someone-else' })
        )

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('distinct_id_header_body_mismatch')
        }
    })

    it('should populate metadata from message', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(5, 42, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.metadata.partition).toBe(5)
            expect(result.value.parsedMessage.metadata.offset).toBe(42)
            expect(result.value.parsedMessage.metadata.topic).toBe('test-topic')
        }
    })

    it('should calculate events range from snapshot items', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.eventsRange.start.isValid).toBe(true)
            expect(result.value.parsedMessage.eventsRange.end.isValid).toBe(true)
            expect(result.value.parsedMessage.eventsRange.end >= result.value.parsedMessage.eventsRange.start).toBe(
                true
            )
        }
    })

    it('should send messages with missing timestamp to DLQ', async () => {
        const step = createParseMessageStep()
        const payload = createValidSnapshotPayload('session-1')
        const input = createInput(0, 1, payload, undefined, { timestamp: undefined })

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('message_value_or_timestamp_is_empty')
        }
    })

    it('should send messages with invalid gzip data to DLQ', async () => {
        const step = createParseMessageStep()
        // Create a message with gzip magic bytes but invalid content
        const message: Message = {
            partition: 0,
            offset: 1,
            topic: 'test-topic',
            value: Buffer.from([0x1f, 0x8b, 0x08, 0x00, 0x00]),
            key: Buffer.from('test-key'),
            timestamp: Date.now(),
            headers: [],
            size: 5,
        }

        const result = await step({ message, headers: createReplayHeaders() })

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('invalid_compressed_data')
        }
    })

    it('should send messages with missing distinct_id to DLQ', async () => {
        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [{ type: 2, timestamp: Date.now() }],
            // distinctId intentionally omitted
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DLQ)
        if (result.type === PipelineResultType.DLQ) {
            expect(result.reason).toBe('invalid_message_payload')
        }
    })

    it('should include snapshot_source and snapshot_library when present', async () => {
        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 2, timestamp: Date.now() },
                { type: 3, timestamp: Date.now() + 1000 },
            ],
            snapshotSource: 'test-source',
            lib: 'test-lib',
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.snapshot_source).toBe('test-source')
            expect(result.value.parsedMessage.snapshot_library).toBe('test-lib')
        }
    })

    it('should set snapshot_source and snapshot_library to null when not present', async () => {
        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 2, timestamp: Date.now() },
                { type: 3, timestamp: Date.now() + 1000 },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.snapshot_source).toBeNull()
            expect(result.value.parsedMessage.snapshot_library).toBeNull()
        }
    })

    it('should filter out events with zero or negative timestamps', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: fixedTimeMs },
                { type: 2, timestamp: 0 },
                { type: 3, timestamp: -1000 },
                { type: 4, timestamp: fixedTimeMs + 1000 },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            const events = result.value.parsedMessage.eventsByWindowId['window-1']
            expect(events).toHaveLength(2)
            expect(events).toEqual([
                { type: 1, timestamp: fixedTimeMs },
                { type: 4, timestamp: fixedTimeMs + 1000 },
            ])
        }
    })

    it('should use min/max for timestamp range instead of first/last', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: fixedTimeMs + 2000 }, // Not the smallest
                { type: 2, timestamp: fixedTimeMs }, // Smallest
                { type: 3, timestamp: fixedTimeMs + 4000 }, // Not the largest
                { type: 4, timestamp: fixedTimeMs + 5000 }, // Largest
                { type: 5, timestamp: fixedTimeMs + 3000 }, // Middle
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.eventsRange.start.toMillis()).toBe(fixedTimeMs)
            expect(result.value.parsedMessage.eventsRange.end.toMillis()).toBe(fixedTimeMs + 5000)
        }
    })

    it('should drop messages with events too far in the future (>7 days)', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const eightDaysInFuture = fixedTimeMs + 8 * 24 * 60 * 60 * 1000
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: fixedTimeMs },
                { type: 2, timestamp: eightDaysInFuture },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DROP)
        if (result.type === PipelineResultType.DROP) {
            expect(result.reason).toBe('message_timestamp_diff_too_large')
        }
    })

    it('should drop messages with events too far in the past (>7 days)', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const eightDaysInPast = fixedTimeMs - 8 * 24 * 60 * 60 * 1000
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: fixedTimeMs },
                { type: 2, timestamp: eightDaysInPast },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DROP)
        if (result.type === PipelineResultType.DROP) {
            expect(result.reason).toBe('message_timestamp_diff_too_large')
        }
    })

    it('should accept messages with events within 7 days', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const sixDaysInFuture = fixedTimeMs + 6 * 24 * 60 * 60 * 1000
        const sixDaysInPast = fixedTimeMs - 6 * 24 * 60 * 60 * 1000
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: sixDaysInPast },
                { type: 2, timestamp: fixedTimeMs },
                { type: 3, timestamp: sixDaysInFuture },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.eventsByWindowId['window-1']).toHaveLength(3)
            expect(result.value.parsedMessage.eventsRange.start.toMillis()).toBe(sixDaysInPast)
            expect(result.value.parsedMessage.eventsRange.end.toMillis()).toBe(sixDaysInFuture)
        }
    })

    it('should include ingestion warning when dropping messages with timestamp diff too large', async () => {
        jest.useFakeTimers()
        jest.setSystemTime(fixedTime)

        const step = createParseMessageStep()
        const eightDaysInFuture = fixedTimeMs + 8 * 24 * 60 * 60 * 1000
        const payload = createSnapshotPayload({
            sessionId: 'session-1',
            snapshotItems: [
                { type: 1, timestamp: fixedTimeMs },
                { type: 2, timestamp: eightDaysInFuture },
            ],
            distinctId: 'user-123',
        })
        const input = createInput(0, 1, payload)

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.DROP)
        expect(result.warnings).toHaveLength(1)
        expect(result.warnings[0]).toEqual({
            type: 'message_timestamp_diff_too_large',
            details: {
                startDiffDays: expect.any(Number),
                endDiffDays: expect.any(Number),
                thresholdDays: 7,
            },
        })
        // The end timestamp is 8 days in the future
        expect(result.warnings[0].details.endDiffDays).toBeGreaterThanOrEqual(8)
    })

    it.each([
        ['uppercase UUID', '0192E72A-1DD2-7714-8000-8B3E4C123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
        ['lowercase UUID', '0192e72a-1dd2-7714-8000-8b3e4c123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
        ['mixed case UUID', '0192E72a-1Dd2-7714-8000-8b3E4c123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
        ['non-UUID session ID', 'custom-session-id', 'custom-session-id'],
    ])('should handle %s session_id: %s -> %s', async (_desc, inputSessionId, expectedSessionId) => {
        const step = createParseMessageStep()
        const payload = createSnapshotPayload({
            sessionId: inputSessionId,
            snapshotItems: [
                { type: 2, timestamp: Date.now() },
                { type: 3, timestamp: Date.now() + 1000 },
            ],
            distinctId: 'user-123',
        })
        // The header id is already normalized by the validate step; the body carries the raw id.
        const input = createInput(
            0,
            1,
            payload,
            undefined,
            undefined,
            createReplayHeaders({ session_id: expectedSessionId })
        )

        const result = await step(input)

        expect(result.type).toBe(PipelineResultType.OK)
        if (result.type === PipelineResultType.OK) {
            expect(result.value.parsedMessage.session_id).toBe(expectedSessionId)
        }
    })

    describe('clock skew', () => {
        const HOUR_MS = 60 * 60 * 1000
        const serverNow = '2023-01-01T00:00:00.000Z'
        let observeSpy: jest.SpyInstance
        let unmeasuredSpy: jest.SpyInstance

        beforeEach(() => {
            observeSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeMessageClockSkew').mockImplementation()
            unmeasuredSpy = jest
                .spyOn(SessionRecordingIngesterMetrics, 'incrementMessageClockSkewUnmeasured')
                .mockImplementation()
        })

        afterEach(() => {
            jest.restoreAllMocks()
        })

        it.each([
            ['reads 1h ahead of the server', '2023-01-01T01:00:00.000Z', 'device_ahead', 3600],
            ['reads 1h behind the server', '2022-12-31T23:00:00.000Z', 'device_behind', 3600],
            ['agrees with the server', serverNow, 'device_ahead', 0],
        ])('records the offset when the device clock %s', (_desc, sentAt, direction, seconds) => {
            observeClockSkew(sentAt, serverNow)

            expect(observeSpy).toHaveBeenCalledWith(direction, seconds)
            expect(unmeasuredSpy).not.toHaveBeenCalled()
        })

        it.each([
            ['capture recorded no sent_at', undefined, serverNow],
            ['sent_at is unparseable', 'not-a-date', serverNow],
            ['capture recorded no now', '2023-01-01T01:00:00.000Z', undefined],
        ])('counts the message as unmeasured when %s', (_desc, sentAt, now) => {
            observeClockSkew(sentAt, now)

            expect(observeSpy).not.toHaveBeenCalled()
            expect(unmeasuredSpy).toHaveBeenCalledTimes(1)
        })

        it('reports no offset for a buffered upload from a correctly clocked device', async () => {
            const step = createParseMessageStep()
            const now = Date.now()
            const uploadedAt = new Date(now).toISOString()
            const payload = createSnapshotPayload({
                sessionId: 'session-1',
                // Buffering, not a wrong clock: the Kafka message time would read this as an hour of skew.
                snapshotItems: [{ type: 2, timestamp: now - HOUR_MS }],
                distinctId: 'user-123',
                sentAt: uploadedAt,
                now: uploadedAt,
            })
            const input = createInput(0, 1, payload, undefined, { timestamp: now })

            const result = await step(input)

            expect(result.type).toBe(PipelineResultType.OK)
            expect(result.warnings).toHaveLength(0)
            expect(observeSpy).toHaveBeenCalledWith('device_ahead', 0)
        })
    })

    describe('clock skew correction', () => {
        const SEVEN_HOURS_MS = 7 * 60 * 60 * 1000

        function skewedInput(options: {
            baseMs: number
            deviceOffsetMs?: number
            latencyMs?: number
            sentAt?: string | null
            now?: string | null
        }): ParseMessageStepInput {
            const { baseMs, deviceOffsetMs = 0, latencyMs = 0 } = options
            const now = options.now === undefined ? new Date(baseMs + latencyMs).toISOString() : options.now
            const sentAt =
                options.sentAt === undefined ? new Date(baseMs + deviceOffsetMs).toISOString() : options.sentAt
            const payload = createSnapshotPayload({
                sessionId: 'session-1',
                snapshotItems: [
                    { type: 2, timestamp: baseMs + deviceOffsetMs },
                    { type: 3, timestamp: baseMs + deviceOffsetMs + 1000 },
                ],
                distinctId: 'user-123',
                ...(now !== null && { now }),
                ...(sentAt !== null && { sentAt }),
            })
            return createInput(0, 1, payload)
        }

        async function startMsOf(input: ParseMessageStepInput): Promise<number> {
            const result = await createParseMessageStep()(input)
            expect(result.type).toBe(PipelineResultType.OK)
            if (result.type !== PipelineResultType.OK) {
                throw new Error('expected an ok result')
            }
            return result.value.parsedMessage.eventsRange.start.toMillis()
        }

        it.each([
            ['ahead', SEVEN_HOURS_MS],
            ['behind', -SEVEN_HOURS_MS],
        ])('shifts rrweb timestamps to server time when the device clock is %s', async (_name, deviceOffsetMs) => {
            const baseMs = Date.now()
            const step = createParseMessageStep()

            const result = await step(skewedInput({ baseMs, deviceOffsetMs }))

            expect(result.type).toBe(PipelineResultType.OK)
            if (result.type === PipelineResultType.OK) {
                const { parsedMessage } = result.value
                expect(parsedMessage.eventsRange.start.toMillis()).toBe(baseMs)
                expect(parsedMessage.eventsRange.end.toMillis()).toBe(baseMs + 1000)
                expect(parsedMessage.eventsByWindowId['window-1'].map((e) => e.timestamp)).toEqual([
                    baseMs,
                    baseMs + 1000,
                ])
            }
        })

        it('corrects two messages of one session identically despite different latencies', async () => {
            const baseMs = Date.now()

            const fast = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, latencyMs: 50 }))
            const slow = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, latencyMs: 2500 }))

            expect(fast).toBe(slow)
        })

        it('leaves a well-behaved clock untouched, so latency cannot reorder its events', async () => {
            const baseMs = Date.now()

            const startMs = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: 400, latencyMs: 900 }))

            expect(startMs).toBe(baseMs + 400)
        })

        const validIso = '2023-01-01T00:00:00.000Z'
        it.each([
            ['sent_at is missing', { sentAt: null }],
            ['now is missing', { now: null }],
            ['sent_at is unparseable', { sentAt: 'not-a-date' }],
            ['now is unparseable', { now: 'not-a-date' }],
        ])(
            'leaves timestamps unchanged when %s',
            async (_name, overrides: { now?: string | null; sentAt?: string | null }) => {
                const baseMs = Date.now()

                const startMs = await startMsOf(
                    skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, now: validIso, ...overrides })
                )

                expect(startMs).toBe(baseMs + SEVEN_HOURS_MS)
            }
        )
    })

    describe('decompressMessageValue lz4 hardening', () => {
        const lz4 = require('lz4')

        function lz4Message(value: Buffer): Message {
            return {
                value,
                headers: [{ 'content-encoding': Buffer.from('lz4') }],
            } as unknown as Message
        }

        function lz4Frame(payload: Buffer, claimedSize?: number): Buffer {
            const compressed = Buffer.alloc(lz4.encodeBound(payload.length))
            const n = lz4.encodeBlock(payload, compressed)
            const prefix = Buffer.alloc(4)
            prefix.writeUInt32LE(claimedSize ?? payload.length, 0)
            return Buffer.concat([prefix, compressed.subarray(0, n)])
        }

        it('round-trips a valid block', () => {
            const payload = Buffer.from('{"distinct_id":"d","data":"x"}'.repeat(10))
            expect(decompressMessageValue(lz4Message(lz4Frame(payload)))).toEqual(payload)
        })

        it('rejects a size prefix beyond the decompression cap instead of allocating it', () => {
            const payload = Buffer.from('small')
            const bomb = lz4Frame(payload, 0xffffffff)
            expect(() => decompressMessageValue(lz4Message(bomb))).toThrow(/exceeds/)
        })

        it('rejects a size prefix that does not match the decoded length', () => {
            const payload = Buffer.from('0123456789'.repeat(20))
            const lying = lz4Frame(payload, payload.length + 512)
            expect(() => decompressMessageValue(lz4Message(lying))).toThrow(/decoded/)
        })
    })
})