:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay): write a poll in parts by decoded size (#127383)
Nicholas Waltzd95032ae0c1e
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/config.tsindex 515d5cf501df..0e6a261113f1 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/config.ts@@ -88,6 +88,7 @@ export type SessionRecordingConfig = {SESSION_RECORDING_MAX_WINDOW_ID_BYTES: numberSESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: numberSESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: boolean+ SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE: numberSESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: numberSESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: numberSESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: boolean@@ -154,6 +155,7 @@ export function getDefaultSessionRecordingConfig(): SessionRecordingConfig {SESSION_RECORDING_MAX_WINDOW_ID_BYTES: 1024,SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: 100_000,SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: false,+ SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE: 0,SESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: 3000,SESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: 300,SESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: false,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.tsindex 80cef67ddd63..7cdbb71d4d9a 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts@@ -22,6 +22,7 @@ import {SessionRecordingIngesterCollaborators,SessionRecordingIngesterConfig,} from './consumer'+import { SessionRecordingIngesterMetrics } from './metrics'import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'import { SessionBatchRecorder } from './sessions/session-batch-recorder'import { BatchCommitter, StagedBatchRunner } from './staged-batch'@@ -329,6 +330,19 @@ describe('SessionRecordingIngester', () => {expect(events).toEqual(expectedEvents)})+ it.each([+ ['skips an empty poll', [], []],+ ['records a poll with messages', [kafkaMessage(0, 42)], [[4 / 1024]]],+ ])('decoded batch size %s', async (_, messages, observed) => {+ const decodedKbSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchDecodedKb')+ runPipelineMock.mockResolvedValue(progress(new Map(), []))++ await ingester.handleEachBatch(messages)++ expect(decodedKbSpy.mock.calls).toEqual(observed)+ decodedKbSpy.mockRestore()+ })+it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {ingestionLagHistogram.reset()createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.tsindex d4735d999123..34d2fafed848 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts@@ -38,6 +38,7 @@ import { BatchStages } from './batch-stages'import { SessionRecordingApiConfig, SessionRecordingConfig } from './config'import { KafkaOffsetManager } from './kafka/offset-manager'import { SessionRecordingIngesterMetrics } from './metrics'+import { statedDecodedBytes } from './parse-message-step'import { SessionReplayLagReporter } from './session-replay-lag-reporter'import { SessionReplayPipelineFactory, SessionReplayPipelineRunner } from './session-replay-pipeline-runner'import { billPersistedSessions } from './session-usage-step'@@ -324,6 +325,11 @@ export class SessionRecordingIngester {const batchSizeKb = messages.reduce((acc, m) => (m.value?.length ?? 0) + acc, 0) / 1024SessionRecordingIngesterMetrics.observeKafkaBatchSize(batchSize)SessionRecordingIngesterMetrics.observeKafkaBatchSizeKb(batchSizeKb)+ if (messages.length > 0) {+ SessionRecordingIngesterMetrics.observeKafkaBatchDecodedKb(+ messages.reduce((acc, m) => statedDecodedBytes(m) + acc, 0) / 1024+ )+ }const batch = this.runner.run(messages, this.batchStages.admit(), this.committer)// A runner with one stage cannot overlap batches. The batch completes inside this call, which is what the consumer expects from a lane that returns no task.@@ -401,6 +407,7 @@ export class SessionRecordingIngester {this.stopEventIngestionRestrictionManager = started.stopconst pipelineConfig: SessionReplayPipelineConfig = {+ maxDecodedBytesPerWrite: this.config.SESSION_RECORDING_MAX_DECODED_KB_PER_WRITE * 1024,messageLimits: {maxWindowIdBytes: this.config.SESSION_RECORDING_MAX_WINDOW_ID_BYTES,maxEventsPerMessage: this.config.SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.tsindex 37e4c92ef768..08c2bbb8a912 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts@@ -30,6 +30,18 @@ export class SessionRecordingIngesterMetrics {buckets: BUCKETS_KB_WRITTEN,})+ private static readonly kafkaBatchDecodedKb = new Histogram({+ name: 'recording_blob_ingestion_v2_kafka_batch_decoded_kb',+ help: 'The stated decompressed size in kb of the batches we are receiving from Kafka',+ buckets: BUCKETS_KB_WRITTEN,+ })++ private static readonly kafkaBatchWriteParts = new Histogram({+ name: 'recording_blob_ingestion_v2_kafka_batch_write_parts',+ help: 'The number of parts that a batch from Kafka is written in when the decoded size limit is on',+ buckets: [1, 2, 3, 4, 5, 10, 20, 50],+ })+private static readonly batchStageDuration = new Histogram({name: 'recording_blob_ingestion_v2_batch_stage_duration_ms',help: 'Wall time one poll batch spent running one stage. Batches overlap across stages, so a stage that runs for close to 100% of wall time is the one the lane is bound by',@@ -125,6 +137,14 @@ export class SessionRecordingIngesterMetrics {this.messagesByEncoding.labels(encoding).inc()}+ public static observeKafkaBatchDecodedKb(decodedKb: number): void {+ this.kafkaBatchDecodedKb.observe(decodedKb)+ }++ public static observeKafkaBatchWriteParts(parts: number): void {+ this.kafkaBatchWriteParts.observe(parts)+ }+public static observeMessageShape(windowIdBytes: number, events: number): void {this.messageWindowIdBytes.observe(windowIdBytes)this.messageEvents.observe(events)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.tsindex 7a87dbc80b59..fb183ea85549 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts@@ -1,6 +1,6 @@import { Message } from 'node-rdkafka'import { promisify } from 'node:util'-import { gzip } from 'zlib'+import { gzip, gzipSync } from 'zlib'import { PipelineResultType } from '~/ingestion/framework/results'@@ -10,6 +10,8 @@ import {createParseMessageStep,decompressMessageValue,observeClockSkew,+ splitByDecodedBytes,+ statedDecodedBytes,} from './parse-message-step'import { SessionReplayHeaders } from './pipeline-types'@@ -287,6 +289,35 @@ describe('createParseMessageStep', () => {})})+ describe('splitByDecodedBytes', () => {+ const raw = (bytes: number): Message => ({ value: Buffer.alloc(bytes) }) as Message+ const lz4Stating = (bytes: number): Message => {+ const value = Buffer.alloc(8)+ value.writeUInt32LE(bytes, 0)+ return { value, headers: [{ 'content-encoding': 'lz4' }] } as unknown as Message+ }++ it.each([+ ['keeps one part when off', [raw(10), raw(10)], 0, [2]],+ ['keeps messages that fit together', [raw(10), raw(10)], 20, [2]],+ ['starts a new part past the limit', [raw(10), raw(10), raw(1)], 15, [1, 2]],+ ['gives a message over the limit its own part', [raw(1), raw(30), raw(1)], 10, [1, 1, 1]],+ ['counts the stated lz4 size', [lz4Stating(1000), lz4Stating(1000)], 1500, [1, 1]],+ ['counts the bytes when a stated size is past the cap', [lz4Stating(2 ** 31), raw(4)], 20, [2]],+ ] as const)('%s', (_, messages, maxBytes, sizes) => {+ const parts = splitByDecodedBytes([...messages], maxBytes)++ expect(parts.map((part) => part.length)).toEqual(sizes)+ expect(parts.flat()).toEqual(messages)+ })++ it('counts the stated gzip size', () => {+ const gzipped = gzipSync(Buffer.alloc(1000))++ expect(statedDecodedBytes({ value: gzipped } as Message)).toBe(1000)+ })+ })+it('should handle gzipped messages', async () => {const step = createParseMessageStep()const payload = createValidSnapshotPayload('session-gzip')diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.tsindex 27cd5a5adf32..3aa94c3f1af6 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts@@ -88,6 +88,43 @@ export function decompressMessageValue(message: Message): Buffer {return messageUnzipped}+export function statedDecodedBytes(message: Message): number {+ const value = message.value+ if (!value) {+ return 0+ }+ let stated = 0+ if (value.length >= 4) {+ if (getContentEncoding(message.headers) === 'lz4') {+ stated = value.readUInt32LE(0)+ } else if (isGzipped(value)) {+ stated = value.readUInt32LE(value.length - 4)+ }+ }+ return Math.max(value.length, stated <= MAX_DECOMPRESSED_BYTES ? stated : 0)+}++export function splitByDecodedBytes(messages: Message[], maxBytes: number): Message[][] {+ if (maxBytes <= 0 || messages.length === 0) {+ return [messages]+ }+ const parts: Message[][] = []+ let current: Message[] = []+ let currentBytes = 0+ for (const message of messages) {+ const bytes = statedDecodedBytes(message)+ if (current.length > 0 && currentBytes + bytes > maxBytes) {+ parts.push(current)+ current = []+ currentBytes = 0+ }+ current.push(message)+ currentBytes += bytes+ }+ parts.push(current)+ return parts+}+// null when capture recorded no usable pair, which must not read as zero skew.function rawClockSkewMs(sentAt: string | undefined, now: string | undefined): number | null {if (!sentAt || !now) {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.tsnew file mode 100644index 000000000000..5d216bb404fa--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.test.ts@@ -0,0 +1,32 @@+import { Message } from 'node-rdkafka'++import { PromiseScheduler } from '~/common/utils/promise-scheduler'++import { BatchStages } from './batch-stages'+import { SessionRecordingIngesterMetrics } from './metrics'+import { SessionReplayPipeline, SessionReplayPipelineConfig } from './session-replay-pipeline'+import { SessionReplayPipelineRunner } from './session-replay-pipeline-runner'+import { BatchCommitter } from './staged-batch'++describe('SessionReplayPipelineRunner', () => {+ it.each([+ ['one commit when the limit is off', 0, 2, 1, []],+ ['one commit for a poll under the limit', 100, 2, 1, [[1]]],+ ['one commit for each part of a poll past the limit', 15, 2, 2, [[2]]],+ ['one commit and no part count for an empty poll', 100, 0, 1, []],+ ])('makes %s', async (_, maxDecodedBytesPerWrite, messageCount, commits, observedParts) => {+ const partsSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeKafkaBatchWriteParts').mockImplementation()+ const runner = new SessionReplayPipelineRunner(() => ({}) as SessionReplayPipeline, new PromiseScheduler())+ runner.start({ maxDecodedBytesPerWrite } as SessionReplayPipelineConfig)+ const committer: BatchCommitter = {+ knownRetention: () => undefined,+ commit: jest.fn().mockResolvedValue(undefined),+ }+ const messages = Array.from({ length: messageCount }, () => ({ value: Buffer.alloc(10) })) as Message[]++ await runner.run(messages, new BatchStages(runner.stages).admit(), committer)++ expect(committer.commit).toHaveBeenCalledTimes(commits)+ expect(partsSpy.mock.calls).toEqual(observedParts)+ })+})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.tsindex 128516c44534..72e375c672f2 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline-runner.ts@@ -8,6 +8,8 @@ import {} from '~/ingestion/pipelines/sessionreplay'import { AdmittedBatch } from './batch-stages'+import { SessionRecordingIngesterMetrics } from './metrics'+import { splitByDecodedBytes } from './parse-message-step'import { BatchCommitter, StagedBatchRunner } from './staged-batch'/** Builds the session replay pipeline for a deployment. */@@ -17,6 +19,7 @@ export type SessionReplayPipelineFactory = (config: SessionReplayPipelineConfig)export class SessionReplayPipelineRunner implements StagedBatchRunner {public readonly stages = ['ingest'] as constprivate pipeline!: SessionReplayPipeline+ private maxDecodedBytesPerWrite = 0constructor(private readonly createPipeline: SessionReplayPipelineFactory,@@ -25,15 +28,22 @@ export class SessionReplayPipelineRunner implements StagedBatchRunner {public start(config: SessionReplayPipelineConfig): void {this.pipeline = this.createPipeline(config)+ this.maxDecodedBytesPerWrite = config.maxDecodedBytesPerWrite ?? 0}public run(messages: Message[], batch: AdmittedBatch<void>, committer: BatchCommitter): Promise<void> {return batch- .stage('ingest', () =>- committer.commit((recorder) =>- runSessionReplayPipeline(this.pipeline, messages, recorder, this.scheduler)- )- )+ .stage('ingest', async () => {+ const parts = splitByDecodedBytes(messages, this.maxDecodedBytesPerWrite)+ if (this.maxDecodedBytesPerWrite > 0 && messages.length > 0) {+ SessionRecordingIngesterMetrics.observeKafkaBatchWriteParts(parts.length)+ }+ for (const part of parts) {+ await committer.commit((recorder) =>+ runSessionReplayPipeline(this.pipeline, part, recorder, this.scheduler)+ )+ }+ }).done()}}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.tsindex 53cc57de2acd..f4e8b1eefec8 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts@@ -58,6 +58,7 @@ export type SessionReplayPipeline = BatchingPipeline<export interface SessionReplayPipelineConfig {messageLimits?: MessageLimits+ maxDecodedBytesPerWrite?: numberoutputs: IngestionOutputs<IngestionWarningsOutput | DlqOutput | OverflowOutput>eventIngestionRestrictionManager: EventIngestionRestrictionManageroverflowMode: IngestionOverflowMode