Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.5 KB
Raw Blame
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'

import { ingestionLagGauge, ingestionLagHistogram } from '~/common/metrics'
import { UsageIngestionClient } from '~/common/usage-ingestion/client'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { PostgresRouter } from '~/common/utils/db/postgres'
import {
    SessionReplayBatchProgress,
    SessionReplayPipeline,
    runSessionReplayPipeline,
} from '~/ingestion/pipelines/sessionreplay'
import { ParsedMessageData } from '~/ingestion/pipelines/sessionreplay/kafka/types'
import { SessionMetadataSink } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'
import { createMockSessionKey } from '~/ingestion/pipelines/sessionreplay/shared/test-helpers'
import { KeyStore, RecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/types'
import { RedisPool } from '~/types'

import { getDefaultSessionRecordingApiConfig, getDefaultSessionRecordingConfig } from './config'
import {
    SessionRecordingIngester,
    SessionRecordingIngesterCollaborators,
    SessionRecordingIngesterConfig,
} from './consumer'
import { SessionRecordingIngesterMetrics } from './metrics'
import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'
import { SessionBatchRecorder } from './sessions/session-batch-recorder'
import { BatchCommitter, StagedBatchRunner } from './staged-batch'

const mockKafkaConsumerConfigs: Record<string, unknown>[] = []

jest.mock('~/common/kafka/consumer/consumer-v2', () => {
    class FakeKafkaConsumerV2 {
        constructor(config: Record<string, unknown>) {
            mockKafkaConsumerConfigs.push(config)
        }
        public connect = jest.fn().mockResolvedValue(undefined)
        public stopConsuming = jest.fn().mockResolvedValue(undefined)
        public disconnect = jest.fn().mockResolvedValue(undefined)
        public offsetsStore = jest.fn()
        public assignments = jest.fn().mockReturnValue([])
        public isHealthy = jest.fn()
    }
    return { KafkaConsumerV2: FakeKafkaConsumerV2 }
})

jest.mock('~/ingestion/pipelines/sessionreplay', () => ({
    ...jest.requireActual('~/ingestion/pipelines/sessionreplay'),
    runSessionReplayPipeline: jest.fn(),
}))

function deferred(): { promise: Promise<void>; resolve: () => void } {
    let resolve!: () => void
    const promise = new Promise<void>((r) => (resolve = r))
    return { promise, resolve }
}

/** Flush all pending microtasks (one macrotask hop). */
function flushMicrotasks(): Promise<void> {
    return new Promise((resolve) => setImmediate(resolve))
}

function kafkaMessage(partition: number, offset: number, capturedAtMs?: number): Message {
    return {
        value: Buffer.from('data'),
        size: 4,
        topic: 'test-session-replay-topic',
        partition,
        offset,
        timestamp: Date.now(),
        headers: capturedAtMs === undefined ? undefined : [{ now: Buffer.from(new Date(capturedAtMs).toISOString()) }],
    }
}

function progress(maxOffsets: Map<number, number>, okMessages: Message[] = []): SessionReplayBatchProgress {
    return { maxOffsets, okMessages }
}

async function lagHistogramCount(partition: string): Promise<number | undefined> {
    const metric = await ingestionLagHistogram.get()
    return metric.values.find(
        (v) => v.metricName === 'ingestion_lag_ms_histogram_count' && v.labels.partition === partition
    )?.value
}

describe('SessionRecordingIngester', () => {
    let ingester: SessionRecordingIngester
    let events: string[]
    let metadataStore: jest.Mocked<SessionMetadataSink>

    const runPipelineMock = jest.mocked(runSessionReplayPipeline)

    const consumeTopic = getDefaultSessionRecordingConfig().INGESTION_SESSION_REPLAY_CONSUMER_CONSUME_TOPIC

    const createIngester = (
        configOverrides: Partial<SessionRecordingIngesterConfig> = {},
        collaborators: Partial<SessionRecordingIngesterCollaborators> = {}
    ): void => {
        const config: SessionRecordingIngesterConfig = {
            ...getDefaultSessionRecordingConfig(),
            ...getDefaultSessionRecordingApiConfig(),
            INGESTION_OVERFLOW_MODE: 'disabled',
            INGESTION_PIPELINE: 'sessionreplay',
            INGESTION_LANE: 'main',
            // Age/size flushes would interleave with the flush under test; keep them out of
            // reach unless a test opts in via overrides.
            SESSION_RECORDING_MAX_BATCH_AGE_MS: 60 * 60 * 1000,
            SESSION_RECORDING_MAX_BATCH_SIZE_KB: 1024 * 1024,
            USAGE_INGESTION_ADDR: '',
            USAGE_INGESTION_TLS: false,
            USAGE_INGESTION_TIMEOUT_MS: 5000,
            USAGE_INGESTION_MAX_BATCH_SIZE: 500,
            USAGE_INGESTION_REPORT_TEAMS: '',
            ...configOverrides,
        }

        const fakeKeyStore = {
            start: jest.fn().mockResolvedValue(undefined),
            stop: jest.fn(),
        } as unknown as KeyStore
        const fakeEncryptor = {
            start: jest.fn().mockResolvedValue(undefined),
            encryptBlockWithKey: jest.fn((_sessionId: string, _teamId: number, data: Buffer) => ({
                data,
            })),
        } as unknown as RecordingEncryptor
        metadataStore = {
            storeSessionBlocks: jest.fn().mockResolvedValue(undefined),
        }

        const sessionTrackerRedis = {
            mget: (keys: string[]) => Promise.resolve(keys.map(() => null)),
            pipeline: () => ({ set: jest.fn().mockReturnThis(), exec: () => Promise.resolve([]) }),
        }

        ingester = new SessionRecordingIngester(
            config,
            {} as jest.Mocked<PostgresRouter>,
            {} as unknown as ConstructorParameters<typeof SessionRecordingIngester>[2],
            {
                acquire: () => Promise.resolve(sessionTrackerRedis),
                release: () => Promise.resolve(),
            } as unknown as RedisPool,
            {} as unknown as RedisPool,
            {
                fileStorage: new BlackholeSessionBatchFileStorage(),
                metadataStore,
                keyStore: fakeKeyStore,
                encryptor: fakeEncryptor,
                createPipeline: () => ({}) as unknown as SessionReplayPipeline,
                ...collaborators,
            }
        )

        jest.mocked(ingester.kafkaConsumer).offsetsStore.mockImplementation(() => {
            events.push('offsets_stored')
        })
        jest.mocked(ingester.kafkaConsumer).disconnect.mockImplementation(() => {
            events.push('disconnected')
            return Promise.resolve()
        })
    }

    const recordOneSession = (): void => {
        const parsedMessage: ParsedMessageData = {
            metadata: { partition: 0, topic: consumeTopic, offset: 42, timestamp: 1000, rawSize: 100 },
            distinct_id: 'distinct-1',
            session_id: 'session-1',
            token: 'token',
            eventsByWindowId: { window1: [{ type: 2, timestamp: 1000, data: {} }] },
            eventsRange: { start: DateTime.fromMillis(1000), end: DateTime.fromMillis(1000) },
            snapshot_source: 'web',
            snapshot_library: 'posthog-js',
        }
        runPipelineMock.mockImplementation(async (_pipeline, _messages, recorder) => {
            await recorder.record(
                {
                    team: { teamId: 1, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },
                    message: parsedMessage,
                },
                '30d',
                createMockSessionKey()
            )
            return progress(new Map([[0, 42]]))
        })
    }

    beforeEach(() => {
        events = []
        runPipelineMock.mockReset()
        createIngester()
    })

    afterEach(() => {
        jest.restoreAllMocks()
    })

    it('stop() stores the in-flight batch offsets only after its side effects settle, before disconnect', async () => {
        const pipelineEntered = deferred()
        const pipelineGate = deferred()

        // Models a poll batch mid-record when stop() arrives: the pipeline holds the batch lock
        // until the gate opens, and only then schedules its side effect (a DLQ/overflow produce
        // that takes one macrotask to become durable) and reports its offsets.
        runPipelineMock.mockImplementation(async (_pipeline, _messages, _recorder, scheduler) => {
            pipelineEntered.resolve()
            await pipelineGate.promise
            void scheduler.schedule(
                new Promise<void>((resolve) =>
                    setImmediate(() => {
                        events.push('side_effect_settled')
                        resolve()
                    })
                )
            )
            return progress(new Map([[0, 42]]))
        })

        const batchPromise = ingester.handleEachBatch([kafkaMessage(0, 42)])
        await pipelineEntered.promise

        // Model the real consumer contract: stopConsuming resolves only once the in-flight
        // eachBatch has settled. (The drain itself is covered by the consumer-v2 tests.)
        jest.mocked(ingester.kafkaConsumer).stopConsuming.mockImplementation(async () => {
            await batchPromise
        })

        const stopPromise = ingester.stop()
        await flushMicrotasks()

        // The batch finishes: it schedules its side effect and reports offset 42 for partition 0.
        pipelineGate.resolve()

        await batchPromise
        await stopPromise

        // The shutdown contract: the batch's produce must be durable before its offset is stored,
        // and the offset must be stored before disconnect commits it to the broker.
        expect(events).toEqual(['side_effect_settled', 'offsets_stored', 'disconnected'])
    })

    it('flushes on the age trigger: side effects drain before the batch offsets are stored, at exactly highest+1', async () => {
        createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })

        // The pipeline schedules a side effect (a DLQ/overflow produce that takes one macrotask
        // to become durable) and reports its offsets; the age trigger then flushes immediately.
        runPipelineMock.mockImplementation((_pipeline, _messages, _recorder, scheduler) => {
            void scheduler.schedule(
                new Promise<void>((resolve) =>
                    setImmediate(() => {
                        events.push('side_effect_settled')
                        resolve()
                    })
                )
            )
            return Promise.resolve(progress(new Map([[0, 42]])))
        })

        await ingester.handleEachBatch([kafkaMessage(0, 42)])

        // Never commit a message's offset before its produce is durable — and commit at
        // exactly the next-to-process offset.
        expect(events).toEqual(['side_effect_settled', 'offsets_stored'])
        expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
            { topic: consumeTopic, partition: 0, offset: 43 },
        ])
    })

    it('reports ingestion lag only once the batch is flushed', async () => {
        ingestionLagGauge.reset()
        ingestionLagHistogram.reset()

        // The default ingester keeps age/size out of reach, so recording a batch does not flush it —
        // and lag must not be observed until the data is durably ingested.
        const bufferedMessage = kafkaMessage(0, 42, Date.now() - 5000)
        runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), [bufferedMessage]))
        await ingester.handleEachBatch([bufferedMessage])
        expect(await lagHistogramCount('0')).toBeUndefined()

        // With the age trigger the batch flushes immediately, so its capture lag is observed post-flush.
        createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
        const flushedMessage = kafkaMessage(0, 43, Date.now() - 5000)
        runPipelineMock.mockResolvedValue(progress(new Map([[0, 43]]), [flushedMessage]))
        await ingester.handleEachBatch([flushedMessage])
        expect(await lagHistogramCount('0')).toBe(1)
    })

    it('persists replay metadata when usage ingestion fails', async () => {
        createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
        jest.spyOn(UsageRecordBatch.prototype, 'flush').mockRejectedValueOnce(new Error('usage unavailable'))
        recordOneSession()

        await expect(ingester.handleEachBatch([kafkaMessage(0, 42)])).resolves.toBeUndefined()

        expect(metadataStore.storeSessionBlocks).toHaveBeenCalledWith([
            expect.objectContaining({ teamId: 1, sessionId: 'session-1' }),
        ])
        expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
            { topic: consumeTopic, partition: 0, offset: 43 },
        ])
    })

    it.each<[string, Partial<SessionRecordingIngesterCollaborators>, unknown[], string[]]>([
        [
            'the main lane bills a persisted session before it stores the offsets',
            {},
            [[[expect.objectContaining({ teamId: 1, usageKey: 'session_replay_recordings', recordId: 'session-1' })]]],
            ['usage_sent', 'offsets_stored'],
        ],
        ['the ML mirror lane bills nothing', { reportUsage: false }, [], ['offsets_stored']],
    ])('%s', async (_name, collaborators, expectedIngestCalls, expectedEvents) => {
        createIngester(
            {
                SESSION_RECORDING_MAX_BATCH_AGE_MS: 0,
                USAGE_INGESTION_ADDR: 'localhost:1',
                USAGE_INGESTION_REPORT_TEAMS: '*',
            },
            collaborators
        )
        const ingest = jest.spyOn(UsageIngestionClient.prototype, 'ingest').mockImplementation((records) => {
            events.push('usage_sent')
            return Promise.resolve(records)
        })
        recordOneSession()

        await ingester.handleEachBatch([kafkaMessage(0, 42)])

        expect(metadataStore.storeSessionBlocks).toHaveBeenCalledTimes(1)
        expect(ingest.mock.calls).toEqual(expectedIngestCalls)
        expect(events).toEqual(expectedEvents)
    })

    it.each([
        ['skips an empty poll', [], []],
        ['records a poll with messages', [kafkaMessage(0, 42)], [[4 / 1024]]],
    ])('decoded batch size %s', async (_, messages, observed) => {
        const decodedKbSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchDecodedKb')
        runPipelineMock.mockResolvedValue(progress(new Map(), []))

        await ingester.handleEachBatch(messages)

        expect(decodedKbSpy.mock.calls).toEqual(observed)
        decodedKbSpy.mockRestore()
    })

    it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {
        ingestionLagHistogram.reset()
        createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })

        // The consumed message carries a capture header, but the pipeline dropped it (no OK results), so
        // no lag is sampled even though the batch flushes.
        runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), []))
        await ingester.handleEachBatch([kafkaMessage(0, 42, Date.now() - 5000)])
        expect(await lagHistogramCount('0')).toBeUndefined()
    })

    it('keeps pending lag samples across a failed flush so a later successful flush still reports them', async () => {
        ingestionLagHistogram.reset()
        createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })

        // A flush that fails after the capture timestamp was buffered must not observe or drop the pending
        // sample. Poll-batch flush failures crash the process; the retain-and-report path that matters is a
        // swallowed revoke-hook flush, after which this still-running process flushes successfully and
        // reports the samples it kept. The failure is simulated here at the offset-store step.
        jest.mocked(ingester.kafkaConsumer).offsetsStore.mockImplementationOnce(() => {
            throw new Error('offset store failed')
        })
        const firstMessage = kafkaMessage(0, 42, Date.now() - 5000)
        runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), [firstMessage]))
        await expect(ingester.handleEachBatch([firstMessage])).rejects.toThrow('offset store failed')
        // The flush threw, so nothing is observed and the sample stays pending.
        expect(await lagHistogramCount('0')).toBeUndefined()

        // A later successful flush reports the retained sample alongside the new one.
        const secondMessage = kafkaMessage(0, 43, Date.now() - 5000)
        runPipelineMock.mockResolvedValue(progress(new Map([[0, 43]]), [secondMessage]))
        await ingester.handleEachBatch([secondMessage])
        expect(await lagHistogramCount('0')).toBe(2)
    })

    it('start() wires the revoke hook, and the hook flushes the tracked offsets', async () => {
        await ingester.start()
        const connectMock = jest.mocked(ingester.kafkaConsumer).connect
        expect(connectMock).toHaveBeenCalledTimes(1)
        const [eachBatch, onPartitionsRevoked] = connectMock.mock.calls[0]
        expect(onPartitionsRevoked).toBeDefined()

        runPipelineMock.mockImplementation(() => Promise.resolve(progress(new Map([[0, 7]]))))
        await eachBatch([kafkaMessage(0, 7)])
        // Age/size are out of reach, so nothing has flushed yet.
        expect(events).toEqual([])

        await onPartitionsRevoked!([{ topic: consumeTopic, partition: 0 }])
        expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
            { topic: consumeTopic, partition: 0, offset: 8 },
        ])

        await ingester.stop()
    })

    it('passes the consumer no background task options for a runner with one stage', () => {
        expect(mockKafkaConsumerConfigs.at(-1)).not.toHaveProperty('maxBackgroundTasks')
        expect(mockKafkaConsumerConfigs.at(-1)).not.toHaveProperty('backgroundTaskTimeoutMs')
    })

    describe('with a runner of one stage', () => {
        let runner: StagedBatchRunner

        beforeEach(() => {
            runner = { stages: ['ingest'], start: jest.fn(), run: jest.fn() }
            createIngester({}, { runner })
        })

        it('completes the batch inside handleEachBatch and returns no task', async () => {
            const done = deferred()
            runner.run = (_messages, batch) => batch.stage('ingest', () => done.promise).done()
            await ingester.start()

            let returned: unknown = 'pending'
            const handled = ingester.handleEachBatch([kafkaMessage(0, 42)]).then((result) => (returned = result))
            await flushMicrotasks()
            expect(returned).toBe('pending')

            done.resolve()
            await handled
            expect(returned).toBeUndefined()
            await ingester.stop()
        })
    })

    describe('with a runner of several stages', () => {
        let runner: StagedBatchRunner

        beforeEach(() => {
            runner = { stages: ['prepare', 'ingest'], start: jest.fn(), run: jest.fn() }
            createIngester({}, { runner })
        })

        it('keeps one batch per stage in flight at the consumer', () => {
            expect(mockKafkaConsumerConfigs.at(-1)).toMatchObject({ maxBackgroundTasks: 2 })
        })

        it('returns each poll batch as a background task, and tells the write which partitions are held under the lock', async () => {
            const writtenInto: SessionBatchRecorder[] = []
            const ownership: boolean[][] = []
            runner.run = (messages, batch, committer) =>
                batch
                    .stage('prepare', () => Promise.resolve())
                    .stage('ingest', () =>
                        committer.commit((recorder, isAssigned) => {
                            writtenInto.push(recorder)
                            ownership.push([isAssigned(0), isAssigned(1)])
                            const held = messages.filter((m) => isAssigned(m.partition))
                            return Promise.resolve({
                                maxOffsets: new Map(held.map((m) => [m.partition, m.offset])),
                                okMessages: held,
                            })
                        })
                    )
                    .done()
            jest.mocked(ingester.kafkaConsumer).assignments.mockReturnValue([{ topic: consumeTopic, partition: 0 }])
            await ingester.start()
            expect(runner.start).toHaveBeenCalledTimes(1)

            const result = await ingester.handleEachBatch([kafkaMessage(0, 42), kafkaMessage(1, 7)])

            expect(result).toEqual({ backgroundTask: expect.any(Promise) })
            await result!.backgroundTask
            expect(runPipelineMock).not.toHaveBeenCalled()
            expect(ownership).toEqual([[true, false]])
            expect(writtenInto).toHaveLength(1)

            await ingester.stop()
            expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
                { topic: consumeTopic, partition: 0, offset: 43 },
            ])
        })

        it('answers a known retention from the batch current at the time of the call, not the one current when the batch was admitted', async () => {
            createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 }, { runner })
            let committer!: BatchCommitter
            runner.run = (_messages, batch, handedCommitter) => {
                committer = handedCommitter
                return batch
                    .stage('prepare', () => Promise.resolve())
                    .stage('ingest', () =>
                        committer.commit(() => Promise.resolve({ maxOffsets: new Map(), okMessages: [] }))
                    )
                    .done()
            }
            await ingester.start()
            const admittedInto = ingester['currentBatch']
            jest.spyOn(admittedInto, 'getRetention').mockReturnValue('90d')

            // The age trigger flushes inside this commit, so the batch current afterwards is a new one.
            await (await ingester.handleEachBatch([kafkaMessage(0, 42)]))!.backgroundTask
            const flushedInto = ingester['currentBatch']
            expect(flushedInto).not.toBe(admittedInto)
            jest.spyOn(flushedInto, 'getRetention').mockReturnValue('30d')

            expect(committer.knownRetention(1, 'session')).toBe('30d')
            expect(admittedInto.getRetention).not.toHaveBeenCalled()
            await ingester.stop()
        })

        it('tracks the offsets of a commit under the same lock as its write, so a revoke flush queued behind it stores them', async () => {
            const writeEntered = deferred()
            const writeGate = deferred()
            runner.run = (messages, batch, committer) =>
                batch
                    .stage('prepare', () => Promise.resolve())
                    .stage('ingest', () =>
                        committer.commit(async () => {
                            writeEntered.resolve()
                            await writeGate.promise
                            return { maxOffsets: new Map([[0, 42]]), okMessages: messages }
                        })
                    )
                    .done()
            jest.mocked(ingester.kafkaConsumer).assignments.mockReturnValue([{ topic: consumeTopic, partition: 0 }])
            await ingester.start()
            const onPartitionsRevoked = jest.mocked(ingester.kafkaConsumer).connect.mock.calls[0][1]!
            const manager = ingester['sessionBatchManager']
            const trackProcessedOffsets = manager.trackProcessedOffsets.bind(manager)
            jest.spyOn(manager, 'trackProcessedOffsets').mockImplementation((offsets) => {
                events.push('offsets_tracked')
                trackProcessedOffsets(offsets)
            })
            const batchBeingWritten = ingester['currentBatch']
            const flush = batchBeingWritten.flush.bind(batchBeingWritten)
            batchBeingWritten.flush = () => {
                events.push('flush_started')
                return flush()
            }

            const result = await ingester.handleEachBatch([kafkaMessage(0, 42)])
            await writeEntered.promise
            // Models a revoke whose drain timed out: its flush queues on the batch lock while the commit still holds it.
            const revoked = onPartitionsRevoked([{ topic: consumeTopic, partition: 0 }])
            await flushMicrotasks()
            writeGate.resolve()
            await result!.backgroundTask
            await revoked
            await ingester.stop()

            expect(events).toEqual(['offsets_tracked', 'flush_started', 'offsets_stored', 'disconnected'])
            expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
                { topic: consumeTopic, partition: 0, offset: 43 },
            ])
        })
    })
})