:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import { ingestionLagGauge, ingestionLagHistogram } from '~/common/metrics'
import { UsageIngestionClient } from '~/common/usage-ingestion/client'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { PostgresRouter } from '~/common/utils/db/postgres'
import {
SessionReplayBatchProgress,
SessionReplayPipeline,
runSessionReplayPipeline,
} from '~/ingestion/pipelines/sessionreplay'
import { ParsedMessageData } from '~/ingestion/pipelines/sessionreplay/kafka/types'
import { SessionMetadataSink } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'
import { createMockSessionKey } from '~/ingestion/pipelines/sessionreplay/shared/test-helpers'
import { KeyStore, RecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/types'
import { RedisPool } from '~/types'
import { getDefaultSessionRecordingApiConfig, getDefaultSessionRecordingConfig } from './config'
import {
SessionRecordingIngester,
SessionRecordingIngesterCollaborators,
SessionRecordingIngesterConfig,
} from './consumer'
import { SessionRecordingIngesterMetrics } from './metrics'
import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'
import { SessionBatchRecorder } from './sessions/session-batch-recorder'
import { BatchCommitter, StagedBatchRunner } from './staged-batch'
const mockKafkaConsumerConfigs: Record<string, unknown>[] = []
jest.mock('~/common/kafka/consumer/consumer-v2', () => {
class FakeKafkaConsumerV2 {
constructor(config: Record<string, unknown>) {
mockKafkaConsumerConfigs.push(config)
}
public connect = jest.fn().mockResolvedValue(undefined)
public stopConsuming = jest.fn().mockResolvedValue(undefined)
public disconnect = jest.fn().mockResolvedValue(undefined)
public offsetsStore = jest.fn()
public assignments = jest.fn().mockReturnValue([])
public isHealthy = jest.fn()
}
return { KafkaConsumerV2: FakeKafkaConsumerV2 }
})
jest.mock('~/ingestion/pipelines/sessionreplay', () => ({
...jest.requireActual('~/ingestion/pipelines/sessionreplay'),
runSessionReplayPipeline: jest.fn(),
}))
function deferred(): { promise: Promise<void>; resolve: () => void } {
let resolve!: () => void
const promise = new Promise<void>((r) => (resolve = r))
return { promise, resolve }
}
/** Flush all pending microtasks (one macrotask hop). */
function flushMicrotasks(): Promise<void> {
return new Promise((resolve) => setImmediate(resolve))
}
function kafkaMessage(partition: number, offset: number, capturedAtMs?: number): Message {
return {
value: Buffer.from('data'),
size: 4,
topic: 'test-session-replay-topic',
partition,
offset,
timestamp: Date.now(),
headers: capturedAtMs === undefined ? undefined : [{ now: Buffer.from(new Date(capturedAtMs).toISOString()) }],
}
}
function progress(maxOffsets: Map<number, number>, okMessages: Message[] = []): SessionReplayBatchProgress {
return { maxOffsets, okMessages }
}
async function lagHistogramCount(partition: string): Promise<number | undefined> {
const metric = await ingestionLagHistogram.get()
return metric.values.find(
(v) => v.metricName === 'ingestion_lag_ms_histogram_count' && v.labels.partition === partition
)?.value
}
describe('SessionRecordingIngester', () => {
let ingester: SessionRecordingIngester
let events: string[]
let metadataStore: jest.Mocked<SessionMetadataSink>
const runPipelineMock = jest.mocked(runSessionReplayPipeline)
const consumeTopic = getDefaultSessionRecordingConfig().INGESTION_SESSION_REPLAY_CONSUMER_CONSUME_TOPIC
const createIngester = (
configOverrides: Partial<SessionRecordingIngesterConfig> = {},
collaborators: Partial<SessionRecordingIngesterCollaborators> = {}
): void => {
const config: SessionRecordingIngesterConfig = {
...getDefaultSessionRecordingConfig(),
...getDefaultSessionRecordingApiConfig(),
INGESTION_OVERFLOW_MODE: 'disabled',
INGESTION_PIPELINE: 'sessionreplay',
INGESTION_LANE: 'main',
// Age/size flushes would interleave with the flush under test; keep them out of
// reach unless a test opts in via overrides.
SESSION_RECORDING_MAX_BATCH_AGE_MS: 60 * 60 * 1000,
SESSION_RECORDING_MAX_BATCH_SIZE_KB: 1024 * 1024,
USAGE_INGESTION_ADDR: '',
USAGE_INGESTION_TLS: false,
USAGE_INGESTION_TIMEOUT_MS: 5000,
USAGE_INGESTION_MAX_BATCH_SIZE: 500,
USAGE_INGESTION_REPORT_TEAMS: '',
...configOverrides,
}
const fakeKeyStore = {
start: jest.fn().mockResolvedValue(undefined),
stop: jest.fn(),
} as unknown as KeyStore
const fakeEncryptor = {
start: jest.fn().mockResolvedValue(undefined),
encryptBlockWithKey: jest.fn((_sessionId: string, _teamId: number, data: Buffer) => ({
data,
})),
} as unknown as RecordingEncryptor
metadataStore = {
storeSessionBlocks: jest.fn().mockResolvedValue(undefined),
}
const sessionTrackerRedis = {
mget: (keys: string[]) => Promise.resolve(keys.map(() => null)),
pipeline: () => ({ set: jest.fn().mockReturnThis(), exec: () => Promise.resolve([]) }),
}
ingester = new SessionRecordingIngester(
config,
{} as jest.Mocked<PostgresRouter>,
{} as unknown as ConstructorParameters<typeof SessionRecordingIngester>[2],
{
acquire: () => Promise.resolve(sessionTrackerRedis),
release: () => Promise.resolve(),
} as unknown as RedisPool,
{} as unknown as RedisPool,
{
fileStorage: new BlackholeSessionBatchFileStorage(),
metadataStore,
keyStore: fakeKeyStore,
encryptor: fakeEncryptor,
createPipeline: () => ({}) as unknown as SessionReplayPipeline,
...collaborators,
}
)
jest.mocked(ingester.kafkaConsumer).offsetsStore.mockImplementation(() => {
events.push('offsets_stored')
})
jest.mocked(ingester.kafkaConsumer).disconnect.mockImplementation(() => {
events.push('disconnected')
return Promise.resolve()
})
}
const recordOneSession = (): void => {
const parsedMessage: ParsedMessageData = {
metadata: { partition: 0, topic: consumeTopic, offset: 42, timestamp: 1000, rawSize: 100 },
distinct_id: 'distinct-1',
session_id: 'session-1',
token: 'token',
eventsByWindowId: { window1: [{ type: 2, timestamp: 1000, data: {} }] },
eventsRange: { start: DateTime.fromMillis(1000), end: DateTime.fromMillis(1000) },
snapshot_source: 'web',
snapshot_library: 'posthog-js',
}
runPipelineMock.mockImplementation(async (_pipeline, _messages, recorder) => {
await recorder.record(
{
team: { teamId: 1, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },
message: parsedMessage,
},
'30d',
createMockSessionKey()
)
return progress(new Map([[0, 42]]))
})
}
beforeEach(() => {
events = []
runPipelineMock.mockReset()
createIngester()
})
afterEach(() => {
jest.restoreAllMocks()
})
it('stop() stores the in-flight batch offsets only after its side effects settle, before disconnect', async () => {
const pipelineEntered = deferred()
const pipelineGate = deferred()
// Models a poll batch mid-record when stop() arrives: the pipeline holds the batch lock
// until the gate opens, and only then schedules its side effect (a DLQ/overflow produce
// that takes one macrotask to become durable) and reports its offsets.
runPipelineMock.mockImplementation(async (_pipeline, _messages, _recorder, scheduler) => {
pipelineEntered.resolve()
await pipelineGate.promise
void scheduler.schedule(
new Promise<void>((resolve) =>
setImmediate(() => {
events.push('side_effect_settled')
resolve()
})
)
)
return progress(new Map([[0, 42]]))
})
const batchPromise = ingester.handleEachBatch([kafkaMessage(0, 42)])
await pipelineEntered.promise
// Model the real consumer contract: stopConsuming resolves only once the in-flight
// eachBatch has settled. (The drain itself is covered by the consumer-v2 tests.)
jest.mocked(ingester.kafkaConsumer).stopConsuming.mockImplementation(async () => {
await batchPromise
})
const stopPromise = ingester.stop()
await flushMicrotasks()
// The batch finishes: it schedules its side effect and reports offset 42 for partition 0.
pipelineGate.resolve()
await batchPromise
await stopPromise
// The shutdown contract: the batch's produce must be durable before its offset is stored,
// and the offset must be stored before disconnect commits it to the broker.
expect(events).toEqual(['side_effect_settled', 'offsets_stored', 'disconnected'])
})
it('flushes on the age trigger: side effects drain before the batch offsets are stored, at exactly highest+1', async () => {
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
// The pipeline schedules a side effect (a DLQ/overflow produce that takes one macrotask
// to become durable) and reports its offsets; the age trigger then flushes immediately.
runPipelineMock.mockImplementation((_pipeline, _messages, _recorder, scheduler) => {
void scheduler.schedule(
new Promise<void>((resolve) =>
setImmediate(() => {
events.push('side_effect_settled')
resolve()
})
)
)
return Promise.resolve(progress(new Map([[0, 42]])))
})
await ingester.handleEachBatch([kafkaMessage(0, 42)])
// Never commit a message's offset before its produce is durable — and commit at
// exactly the next-to-process offset.
expect(events).toEqual(['side_effect_settled', 'offsets_stored'])
expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
{ topic: consumeTopic, partition: 0, offset: 43 },
])
})
it('reports ingestion lag only once the batch is flushed', async () => {
ingestionLagGauge.reset()
ingestionLagHistogram.reset()
// The default ingester keeps age/size out of reach, so recording a batch does not flush it —
// and lag must not be observed until the data is durably ingested.
const bufferedMessage = kafkaMessage(0, 42, Date.now() - 5000)
runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), [bufferedMessage]))
await ingester.handleEachBatch([bufferedMessage])
expect(await lagHistogramCount('0')).toBeUndefined()
// With the age trigger the batch flushes immediately, so its capture lag is observed post-flush.
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
const flushedMessage = kafkaMessage(0, 43, Date.now() - 5000)
runPipelineMock.mockResolvedValue(progress(new Map([[0, 43]]), [flushedMessage]))
await ingester.handleEachBatch([flushedMessage])
expect(await lagHistogramCount('0')).toBe(1)
})
it('persists replay metadata when usage ingestion fails', async () => {
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
jest.spyOn(UsageRecordBatch.prototype, 'flush').mockRejectedValueOnce(new Error('usage unavailable'))
recordOneSession()
await expect(ingester.handleEachBatch([kafkaMessage(0, 42)])).resolves.toBeUndefined()
expect(metadataStore.storeSessionBlocks).toHaveBeenCalledWith([
expect.objectContaining({ teamId: 1, sessionId: 'session-1' }),
])
expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
{ topic: consumeTopic, partition: 0, offset: 43 },
])
})
it.each<[string, Partial<SessionRecordingIngesterCollaborators>, unknown[], string[]]>([
[
'the main lane bills a persisted session before it stores the offsets',
{},
[[[expect.objectContaining({ teamId: 1, usageKey: 'session_replay_recordings', recordId: 'session-1' })]]],
['usage_sent', 'offsets_stored'],
],
['the ML mirror lane bills nothing', { reportUsage: false }, [], ['offsets_stored']],
])('%s', async (_name, collaborators, expectedIngestCalls, expectedEvents) => {
createIngester(
{
SESSION_RECORDING_MAX_BATCH_AGE_MS: 0,
USAGE_INGESTION_ADDR: 'localhost:1',
USAGE_INGESTION_REPORT_TEAMS: '*',
},
collaborators
)
const ingest = jest.spyOn(UsageIngestionClient.prototype, 'ingest').mockImplementation((records) => {
events.push('usage_sent')
return Promise.resolve(records)
})
recordOneSession()
await ingester.handleEachBatch([kafkaMessage(0, 42)])
expect(metadataStore.storeSessionBlocks).toHaveBeenCalledTimes(1)
expect(ingest.mock.calls).toEqual(expectedIngestCalls)
expect(events).toEqual(expectedEvents)
})
it.each([
['skips an empty poll', [], []],
['records a poll with messages', [kafkaMessage(0, 42)], [[4 / 1024]]],
])('decoded batch size %s', async (_, messages, observed) => {
const decodedKbSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchDecodedKb')
runPipelineMock.mockResolvedValue(progress(new Map(), []))
await ingester.handleEachBatch(messages)
expect(decodedKbSpy.mock.calls).toEqual(observed)
decodedKbSpy.mockRestore()
})
it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {
ingestionLagHistogram.reset()
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
// The consumed message carries a capture header, but the pipeline dropped it (no OK results), so
// no lag is sampled even though the batch flushes.
runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), []))
await ingester.handleEachBatch([kafkaMessage(0, 42, Date.now() - 5000)])
expect(await lagHistogramCount('0')).toBeUndefined()
})
it('keeps pending lag samples across a failed flush so a later successful flush still reports them', async () => {
ingestionLagHistogram.reset()
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
// A flush that fails after the capture timestamp was buffered must not observe or drop the pending
// sample. Poll-batch flush failures crash the process; the retain-and-report path that matters is a
// swallowed revoke-hook flush, after which this still-running process flushes successfully and
// reports the samples it kept. The failure is simulated here at the offset-store step.
jest.mocked(ingester.kafkaConsumer).offsetsStore.mockImplementationOnce(() => {
throw new Error('offset store failed')
})
const firstMessage = kafkaMessage(0, 42, Date.now() - 5000)
runPipelineMock.mockResolvedValue(progress(new Map([[0, 42]]), [firstMessage]))
await expect(ingester.handleEachBatch([firstMessage])).rejects.toThrow('offset store failed')
// The flush threw, so nothing is observed and the sample stays pending.
expect(await lagHistogramCount('0')).toBeUndefined()
// A later successful flush reports the retained sample alongside the new one.
const secondMessage = kafkaMessage(0, 43, Date.now() - 5000)
runPipelineMock.mockResolvedValue(progress(new Map([[0, 43]]), [secondMessage]))
await ingester.handleEachBatch([secondMessage])
expect(await lagHistogramCount('0')).toBe(2)
})
it('start() wires the revoke hook, and the hook flushes the tracked offsets', async () => {
await ingester.start()
const connectMock = jest.mocked(ingester.kafkaConsumer).connect
expect(connectMock).toHaveBeenCalledTimes(1)
const [eachBatch, onPartitionsRevoked] = connectMock.mock.calls[0]
expect(onPartitionsRevoked).toBeDefined()
runPipelineMock.mockImplementation(() => Promise.resolve(progress(new Map([[0, 7]]))))
await eachBatch([kafkaMessage(0, 7)])
// Age/size are out of reach, so nothing has flushed yet.
expect(events).toEqual([])
await onPartitionsRevoked!([{ topic: consumeTopic, partition: 0 }])
expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
{ topic: consumeTopic, partition: 0, offset: 8 },
])
await ingester.stop()
})
it('passes the consumer no background task options for a runner with one stage', () => {
expect(mockKafkaConsumerConfigs.at(-1)).not.toHaveProperty('maxBackgroundTasks')
expect(mockKafkaConsumerConfigs.at(-1)).not.toHaveProperty('backgroundTaskTimeoutMs')
})
describe('with a runner of one stage', () => {
let runner: StagedBatchRunner
beforeEach(() => {
runner = { stages: ['ingest'], start: jest.fn(), run: jest.fn() }
createIngester({}, { runner })
})
it('completes the batch inside handleEachBatch and returns no task', async () => {
const done = deferred()
runner.run = (_messages, batch) => batch.stage('ingest', () => done.promise).done()
await ingester.start()
let returned: unknown = 'pending'
const handled = ingester.handleEachBatch([kafkaMessage(0, 42)]).then((result) => (returned = result))
await flushMicrotasks()
expect(returned).toBe('pending')
done.resolve()
await handled
expect(returned).toBeUndefined()
await ingester.stop()
})
})
describe('with a runner of several stages', () => {
let runner: StagedBatchRunner
beforeEach(() => {
runner = { stages: ['prepare', 'ingest'], start: jest.fn(), run: jest.fn() }
createIngester({}, { runner })
})
it('keeps one batch per stage in flight at the consumer', () => {
expect(mockKafkaConsumerConfigs.at(-1)).toMatchObject({ maxBackgroundTasks: 2 })
})
it('returns each poll batch as a background task, and tells the write which partitions are held under the lock', async () => {
const writtenInto: SessionBatchRecorder[] = []
const ownership: boolean[][] = []
runner.run = (messages, batch, committer) =>
batch
.stage('prepare', () => Promise.resolve())
.stage('ingest', () =>
committer.commit((recorder, isAssigned) => {
writtenInto.push(recorder)
ownership.push([isAssigned(0), isAssigned(1)])
const held = messages.filter((m) => isAssigned(m.partition))
return Promise.resolve({
maxOffsets: new Map(held.map((m) => [m.partition, m.offset])),
okMessages: held,
})
})
)
.done()
jest.mocked(ingester.kafkaConsumer).assignments.mockReturnValue([{ topic: consumeTopic, partition: 0 }])
await ingester.start()
expect(runner.start).toHaveBeenCalledTimes(1)
const result = await ingester.handleEachBatch([kafkaMessage(0, 42), kafkaMessage(1, 7)])
expect(result).toEqual({ backgroundTask: expect.any(Promise) })
await result!.backgroundTask
expect(runPipelineMock).not.toHaveBeenCalled()
expect(ownership).toEqual([[true, false]])
expect(writtenInto).toHaveLength(1)
await ingester.stop()
expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
{ topic: consumeTopic, partition: 0, offset: 43 },
])
})
it('answers a known retention from the batch current at the time of the call, not the one current when the batch was admitted', async () => {
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 }, { runner })
let committer!: BatchCommitter
runner.run = (_messages, batch, handedCommitter) => {
committer = handedCommitter
return batch
.stage('prepare', () => Promise.resolve())
.stage('ingest', () =>
committer.commit(() => Promise.resolve({ maxOffsets: new Map(), okMessages: [] }))
)
.done()
}
await ingester.start()
const admittedInto = ingester['currentBatch']
jest.spyOn(admittedInto, 'getRetention').mockReturnValue('90d')
// The age trigger flushes inside this commit, so the batch current afterwards is a new one.
await (await ingester.handleEachBatch([kafkaMessage(0, 42)]))!.backgroundTask
const flushedInto = ingester['currentBatch']
expect(flushedInto).not.toBe(admittedInto)
jest.spyOn(flushedInto, 'getRetention').mockReturnValue('30d')
expect(committer.knownRetention(1, 'session')).toBe('30d')
expect(admittedInto.getRetention).not.toHaveBeenCalled()
await ingester.stop()
})
it('tracks the offsets of a commit under the same lock as its write, so a revoke flush queued behind it stores them', async () => {
const writeEntered = deferred()
const writeGate = deferred()
runner.run = (messages, batch, committer) =>
batch
.stage('prepare', () => Promise.resolve())
.stage('ingest', () =>
committer.commit(async () => {
writeEntered.resolve()
await writeGate.promise
return { maxOffsets: new Map([[0, 42]]), okMessages: messages }
})
)
.done()
jest.mocked(ingester.kafkaConsumer).assignments.mockReturnValue([{ topic: consumeTopic, partition: 0 }])
await ingester.start()
const onPartitionsRevoked = jest.mocked(ingester.kafkaConsumer).connect.mock.calls[0][1]!
const manager = ingester['sessionBatchManager']
const trackProcessedOffsets = manager.trackProcessedOffsets.bind(manager)
jest.spyOn(manager, 'trackProcessedOffsets').mockImplementation((offsets) => {
events.push('offsets_tracked')
trackProcessedOffsets(offsets)
})
const batchBeingWritten = ingester['currentBatch']
const flush = batchBeingWritten.flush.bind(batchBeingWritten)
batchBeingWritten.flush = () => {
events.push('flush_started')
return flush()
}
const result = await ingester.handleEachBatch([kafkaMessage(0, 42)])
await writeEntered.promise
// Models a revoke whose drain timed out: its flush queues on the batch lock while the commit still holds it.
const revoked = onPartitionsRevoked([{ topic: consumeTopic, partition: 0 }])
await flushMicrotasks()
writeGate.resolve()
await result!.backgroundTask
await revoked
await ingester.stop()
expect(events).toEqual(['offsets_tracked', 'flush_started', 'offsets_stored', 'disconnected'])
expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
{ topic: consumeTopic, partition: 0, offset: 43 },
])
})
})
})