Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay): write a poll in parts by decoded size (#127383)

Nicholas Waltzd95032ae0c1e
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/config.ts
index 515d5cf501df..0e6a261113f1 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/config.ts
@@ -88,6 +88,7 @@ export type SessionRecordingConfig = {
SESSION_RECORDING_MAX_WINDOW_ID_BYTES: number
SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: number
SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: boolean
+ SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE: number
SESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: number
SESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: number
SESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: boolean
@@ -154,6 +155,7 @@ export function getDefaultSessionRecordingConfig(): SessionRecordingConfig {
SESSION_RECORDING_MAX_WINDOW_ID_BYTES: 1024,
SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: 100_000,
SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: false,
+ SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE: 0,
SESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: 3000,
SESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: 300,
SESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: false,
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
index 80cef67ddd63..7cdbb71d4d9a 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
@@ -22,6 +22,7 @@ import {
SessionRecordingIngesterCollaborators,
SessionRecordingIngesterConfig,
} from './consumer'
+import { SessionRecordingIngesterMetrics } from './metrics'
import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'
import { SessionBatchRecorder } from './sessions/session-batch-recorder'
import { BatchCommitter, StagedBatchRunner } from './staged-batch'
@@ -329,6 +330,19 @@ describe('SessionRecordingIngester', () => {
expect(events).toEqual(expectedEvents)
})
+ it.each([
+ ['skips an empty poll', [], []],
+ ['records a poll with messages', [kafkaMessage(0, 42)], [[4 / 1024]]],
+ ])('decoded batch size %s', async (_, messages, observed) => {
+ const decodedKbSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchDecodedKb')
+ runPipelineMock.mockResolvedValue(progress(new Map(), []))
+
+ await ingester.handleEachBatch(messages)
+
+ expect(decodedKbSpy.mock.calls).toEqual(observed)
+ decodedKbSpy.mockRestore()
+ })
+
it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {
ingestionLagHistogram.reset()
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
index d4735d999123..34d2fafed848 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
@@ -38,6 +38,7 @@ import { BatchStages } from './batch-stages'
import { SessionRecordingApiConfig, SessionRecordingConfig } from './config'
import { KafkaOffsetManager } from './kafka/offset-manager'
import { SessionRecordingIngesterMetrics } from './metrics'
+import { statedDecodedBytes } from './parse-message-step'
import { SessionReplayLagReporter } from './session-replay-lag-reporter'
import { SessionReplayPipelineFactory, SessionReplayPipelineRunner } from './session-replay-pipeline-runner'
import { billPersistedSessions } from './session-usage-step'
@@ -324,6 +325,11 @@ export class SessionRecordingIngester {
const batchSizeKb = messages.reduce((acc, m) => (m.value?.length ?? 0) + acc, 0) / 1024
SessionRecordingIngesterMetrics.observeKafkaBatchSize(batchSize)
SessionRecordingIngesterMetrics.observeKafkaBatchSizeKb(batchSizeKb)
+ if (messages.length > 0) {
+ SessionRecordingIngesterMetrics.observeKafkaBatchDecodedKb(
+ messages.reduce((acc, m) => statedDecodedBytes(m) + acc, 0) / 1024
+ )
+ }
const batch = this.runner.run(messages, this.batchStages.admit(), this.committer)
// A runner with one stage cannot overlap batches. The batch completes inside this call, which is what the consumer expects from a lane that returns no task.
@@ -401,6 +407,7 @@ export class SessionRecordingIngester {
this.stopEventIngestionRestrictionManager = started.stop
const pipelineConfig: SessionReplayPipelineConfig = {
+ maxDecodedBytesPerWrite: this.config.SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE * 1024,
messageLimits: {
maxWindowIdBytes: this.config.SESSION_RECORDING_MAX_WINDOW_ID_BYTES,
maxEventsPerMessage: this.config.SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE,
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
index 37e4c92ef768..08c2bbb8a912 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
@@ -30,6 +30,18 @@ export class SessionRecordingIngesterMetrics {
buckets: BUCKETS_KB_WRITTEN,
})
+ private static readonly kafkaBatchDecodedKb = new Histogram({
+ name: 'recording_blob_ingestion_v2_kafka_batch_decoded_kb',
+ help: 'The stated decompressed size in kb of the batches we are receiving from Kafka',
+ buckets: BUCKETS_KB_WRITTEN,
+ })
+
+ private static readonly kafkaBatchWriteParts = new Histogram({
+ name: 'recording_blob_ingestion_v2_kafka_batch_write_parts',
+ help: 'The number of parts that a batch from Kafka is written in when the decoded size limit is on',
+ buckets: [1, 2, 3, 4, 5, 10, 20, 50],
+ })
+
private static readonly batchStageDuration = new Histogram({
name: 'recording_blob_ingestion_v2_batch_stage_duration_ms',
help: 'Wall time one poll batch spent running one stage. Batches overlap across stages, so a stage that runs for close to 100% of wall time is the one the lane is bound by',
@@ -125,6 +137,14 @@ export class SessionRecordingIngesterMetrics {
this.messagesByEncoding.labels(encoding).inc()
}
+ public static observeKafkaBatchDecodedKb(decodedKb: number): void {
+ this.kafkaBatchDecodedKb.observe(decodedKb)
+ }
+
+ public static observeKafkaBatchWriteParts(parts: number): void {
+ this.kafkaBatchWriteParts.observe(parts)
+ }
+
public static observeMessageShape(windowIdBytes: number, events: number): void {
this.messageWindowIdBytes.observe(windowIdBytes)
this.messageEvents.observe(events)
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts
index 7a87dbc80b59..fb183ea85549 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts
@@ -1,6 +1,6 @@
import { Message } from 'node-rdkafka'
import { promisify } from 'node:util'
-import { gzip } from 'zlib'
+import { gzip, gzipSync } from 'zlib'
import { PipelineResultType } from '~/ingestion/framework/results'
@@ -10,6 +10,8 @@ import {
createParseMessageStep,
decompressMessageValue,
observeClockSkew,
+ splitByDecodedBytes,
+ statedDecodedBytes,
} from './parse-message-step'
import { SessionReplayHeaders } from './pipeline-types'
@@ -287,6 +289,35 @@ describe('createParseMessageStep', () => {
})
})
+ describe('splitByDecodedBytes', () => {
+ const raw = (bytes: number): Message => ({ value: Buffer.alloc(bytes) }) as Message
+ const lz4Stating = (bytes: number): Message => {
+ const value = Buffer.alloc(8)
+ value.writeUInt32LE(bytes, 0)
+ return { value, headers: [{ 'content-encoding': 'lz4' }] } as unknown as Message
+ }
+
+ it.each([
+ ['keeps one part when off', [raw(10), raw(10)], 0, [2]],
+ ['keeps messages that fit together', [raw(10), raw(10)], 20, [2]],
+ ['starts a new part past the limit', [raw(10), raw(10), raw(1)], 15, [1, 2]],
+ ['gives a message over the limit its own part', [raw(1), raw(30), raw(1)], 10, [1, 1, 1]],
+ ['counts the stated lz4 size', [lz4Stating(1000), lz4Stating(1000)], 1500, [1, 1]],
+ ['counts the bytes when a stated size is past the cap', [lz4Stating(2 ** 31), raw(4)], 20, [2]],
+ ] as const)('%s', (_, messages, maxBytes, sizes) => {
+ const parts = splitByDecodedBytes([...messages], maxBytes)
+
+ expect(parts.map((part) => part.length)).toEqual(sizes)
+ expect(parts.flat()).toEqual(messages)
+ })
+
+ it('counts the stated gzip size', () => {
+ const gzipped = gzipSync(Buffer.alloc(1000))
+
+ expect(statedDecodedBytes({ value: gzipped } as Message)).toBe(1000)
+ })
+ })
+
it('should handle gzipped messages', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-gzip')
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts
index 27cd5a5adf32..3aa94c3f1af6 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts
@@ -88,6 +88,43 @@ export function decompressMessageValue(message: Message): Buffer {
return messageUnzipped
}
+export function statedDecodedBytes(message: Message): number {
+ const value = message.value
+ if (!value) {
+ return 0
+ }
+ let stated = 0
+ if (value.length >= 4) {
+ if (getContentEncoding(message.headers) === 'lz4') {
+ stated = value.readUInt32LE(0)
+ } else if (isGzipped(value)) {
+ stated = value.readUInt32LE(value.length - 4)
+ }
+ }
+ return Math.max(value.length, stated <= MAX_DECOMPRESSED_BYTES ? stated : 0)
+}
+
+export function splitByDecodedBytes(messages: Message[], maxBytes: number): Message[][] {
+ if (maxBytes <= 0 || messages.length === 0) {
+ return [messages]
+ }
+ const parts: Message[][] = []
+ let current: Message[] = []
+ let currentBytes = 0
+ for (const message of messages) {
+ const bytes = statedDecodedBytes(message)
+ if (current.length > 0 && currentBytes + bytes > maxBytes) {
+ parts.push(current)
+ current = []
+ currentBytes = 0
+ }
+ current.push(message)
+ currentBytes += bytes
+ }
+ parts.push(current)
+ return parts
+}
+
// null when capture recorded no usable pair, which must not read as zero skew.
function rawClockSkewMs(sentAt: string | undefined, now: string | undefined): number | null {
if (!sentAt || !now) {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.ts
new file mode 100644
index 000000000000..5d216bb404fa
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.ts
@@ -0,0 +1,32 @@
+import { Message } from 'node-rdkafka'
+
+import { PromiseScheduler } from '~/common/utils/promise-scheduler'
+
+import { BatchStages } from './batch-stages'
+import { SessionRecordingIngesterMetrics } from './metrics'
+import { SessionReplayPipeline, SessionReplayPipelineConfig } from './session-replay-pipeline'
+import { SessionReplayPipelineRunner } from './session-replay-pipeline-runner'
+import { BatchCommitter } from './staged-batch'
+
+describe('SessionReplayPipelineRunner', () => {
+ it.each([
+ ['one commit when the limit is off', 0, 2, 1, []],
+ ['one commit for a poll under the limit', 100, 2, 1, [[1]]],
+ ['one commit for each part of a poll past the limit', 15, 2, 2, [[2]]],
+ ['one commit and no part count for an empty poll', 100, 0, 1, []],
+ ])('makes %s', async (_, maxDecodedBytesPerWrite, messageCount, commits, observedParts) => {
+ const partsSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchWriteParts').mockImplementation()
+ const runner = new SessionReplayPipelineRunner(() => ({}) as SessionReplayPipeline, new PromiseScheduler())
+ runner.start({ maxDecodedBytesPerWrite } as SessionReplayPipelineConfig)
+ const committer: BatchCommitter = {
+ knownRetention: () => undefined,
+ commit: jest.fn().mockResolvedValue(undefined),
+ }
+ const messages = Array.from({ length: messageCount }, () => ({ value: Buffer.alloc(10) })) as Message[]
+
+ await runner.run(messages, new BatchStages(runner.stages).admit(), committer)
+
+ expect(committer.commit).toHaveBeenCalledTimes(commits)
+ expect(partsSpy.mock.calls).toEqual(observedParts)
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts
index 128516c44534..72e375c672f2 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts
@@ -8,6 +8,8 @@ import {
} from '~/ingestion/pipelines/sessionreplay'
import { AdmittedBatch } from './batch-stages'
+import { SessionRecordingIngesterMetrics } from './metrics'
+import { splitByDecodedBytes } from './parse-message-step'
import { BatchCommitter, StagedBatchRunner } from './staged-batch'
/** Builds the session replay pipeline for a deployment. */
@@ -17,6 +19,7 @@ export type SessionReplayPipelineFactory = (config: SessionReplayPipelineConfig)
export class SessionReplayPipelineRunner implements StagedBatchRunner {
public readonly stages = ['ingest'] as const
private pipeline!: SessionReplayPipeline
+ private maxDecodedBytesPerWrite = 0
constructor(
private readonly createPipeline: SessionReplayPipelineFactory,
@@ -25,15 +28,22 @@ export class SessionReplayPipelineRunner implements StagedBatchRunner {
public start(config: SessionReplayPipelineConfig): void {
this.pipeline = this.createPipeline(config)
+ this.maxDecodedBytesPerWrite = config.maxDecodedBytesPerWrite ?? 0
}
public run(messages: Message[], batch: AdmittedBatch<void>, committer: BatchCommitter): Promise<void> {
return batch
- .stage('ingest', () =>
- committer.commit((recorder) =>
- runSessionReplayPipeline(this.pipeline, messages, recorder, this.scheduler)
- )
- )
+ .stage('ingest', async () => {
+ const parts = splitByDecodedBytes(messages, this.maxDecodedBytesPerWrite)
+ if (this.maxDecodedBytesPerWrite > 0 && messages.length > 0) {
+ SessionRecordingIngesterMetrics.observeKafkaBatchWriteParts(parts.length)
+ }
+ for (const part of parts) {
+ await committer.commit((recorder) =>
+ runSessionReplayPipeline(this.pipeline, part, recorder, this.scheduler)
+ )
+ }
+ })
.done()
}
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
index 53cc57de2acd..f4e8b1eefec8 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
@@ -58,6 +58,7 @@ export type SessionReplayPipeline = BatchingPipeline<
export interface SessionReplayPipelineConfig {
messageLimits?: MessageLimits
+ maxDecodedBytesPerWrite?: number
outputs: IngestionOutputs<IngestionWarningsOutput | DlqOutput | OverflowOutput>
eventIngestionRestrictionManager: EventIngestionRestrictionManager
overflowMode: IngestionOverflowMode