:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay): count messages past per-message limits and optionally drop them (#127376)
Nicholas Waltz92395b028aa8
diff --git a/nodejs/src/ingestion/common/ingestion-warning-types.ts b/nodejs/src/ingestion/common/ingestion-warning-types.tsindex 86140179b644..e8d8a3b8197a 100644--- a/nodejs/src/ingestion/common/ingestion-warning-types.ts+++ b/nodejs/src/ingestion/common/ingestion-warning-types.ts@@ -124,6 +124,8 @@ export const INGESTION_WARNING_TYPES = {replay_lib_version_too_old: { category: 'replay', severity: 'info' },message_contained_no_valid_rrweb_events: { category: 'replay', severity: 'warning' },message_timestamp_diff_too_large: { category: 'replay', severity: 'warning' },+ message_window_id_too_long: { category: 'replay', severity: 'error' },+ message_too_many_events: { category: 'replay', severity: 'error' },// Capture's replay endpoint (/s), which validates the $snapshot envelope before// the batch is collapsed into one $snapshot_items message. Distinguished from the// three above by `source: capture` and `path: replay` — those describe conditionsdiff --git a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts b/nodejs/src/ingestion/pipelines/sessionreplay/config.tsindex ed8c82b96258..515d5cf501df 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/config.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/config.ts@@ -85,6 +85,9 @@ export type SessionRecordingConfig = {SESSION_RECORDING_REDIS_TIMEOUT_MS: numberSESSION_RECORDING_V2_CONSOLE_LOG_STORE_SYNC_BATCH_LIMIT: numberSESSION_RECORDING_V2_MAX_EVENTS_PER_SESSION_PER_BATCH: number+ SESSION_RECORDING_MAX_WINDOW_ID_BYTES: number+ SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: number+ SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: booleanSESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: numberSESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: numberSESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: boolean@@ -148,6 +151,9 @@ export function getDefaultSessionRecordingConfig(): SessionRecordingConfig {SESSION_RECORDING_REDIS_TIMEOUT_MS: 200,SESSION_RECORDING_V2_CONSOLE_LOG_STORE_SYNC_BATCH_LIMIT: 1000,SESSION_RECORDING_V2_MAX_EVENTS_PER_SESSION_PER_BATCH: Number.MAX_SAFE_INTEGER,+ SESSION_RECORDING_MAX_WINDOW_ID_BYTES: 1024,+ SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE: 100_000,+ SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS: false,SESSION_RECORDING_NEW_SESSION_BUCKET_CAPACITY: 3000,SESSION_RECORDING_NEW_SESSION_BUCKET_REPLENISH_RATE: 300,SESSION_RECORDING_NEW_SESSION_BLOCKING_ENABLED: false,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.tsindex eda264630a08..d4735d999123 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts@@ -401,6 +401,11 @@ export class SessionRecordingIngester {this.stopEventIngestionRestrictionManager = started.stopconst pipelineConfig: SessionReplayPipelineConfig = {+ messageLimits: {+ maxWindowIdBytes: this.config.SESSION_RECORDING_MAX_WINDOW_ID_BYTES,+ maxEventsPerMessage: this.config.SESSION_RECORDING_MAX_EVENTS_PER_MESSAGE,+ enforce: this.config.SESSION_RECORDING_ENFORCE_MESSAGE_LIMITS,+ },outputs: this.outputs,eventIngestionRestrictionManager: this.eventIngestionRestrictionManager,overflowMode: this.config.INGESTION_OVERFLOW_MODE,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.tsindex c5e14339bda1..37e4c92ef768 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts@@ -72,6 +72,24 @@ export class SessionRecordingIngesterMetrics {labelNames: ['content_encoding'],})+ private static readonly messageWindowIdBytes = new Histogram({+ name: 'recording_blob_ingestion_v2_message_window_id_bytes',+ help: 'The size in bytes of the window ID of each message',+ buckets: [36, 64, 128, 256, 512, 1024, 4096, 16384, 65536, Infinity],+ })++ private static readonly messageEvents = new Histogram({+ name: 'recording_blob_ingestion_v2_message_events',+ help: 'The number of events in each message',+ buckets: [1, 10, 100, 500, 1000, 2500, 5000, 10000, 25000, 50000, 100000, Infinity],+ })++ private static readonly messagesOverLimit = new Counter({+ name: 'recording_blob_ingestion_v2_messages_over_limit',+ help: 'The number of messages past a per-message limit, whether or not the limit is enforced',+ labelNames: ['limit'],+ })+private static readonly messageClockSkew = new Histogram({name: 'recording_blob_ingestion_v2_message_clock_skew_seconds',help: 'Absolute offset between the sender device clock at upload (sent_at) and the server clock at receipt (now), by direction. Replay skips the skew correction every other event gets, so a large offset means the recording start_time — the default playlist sort — disagrees with the corrected event time.',@@ -107,6 +125,15 @@ export class SessionRecordingIngesterMetrics {this.messagesByEncoding.labels(encoding).inc()}+ public static observeMessageShape(windowIdBytes: number, events: number): void {+ this.messageWindowIdBytes.observe(windowIdBytes)+ this.messageEvents.observe(events)+ }++ public static incrementMessagesOverLimit(limit: string): void {+ this.messagesOverLimit.labels(limit).inc()+ }+public static incrementMessageClockSkewUnmeasured(): void {this.messageClockSkewUnmeasured.inc()}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.tsindex aec77b8e039a..7a87dbc80b59 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.test.ts@@ -212,6 +212,81 @@ describe('createParseMessageStep', () => {}})+ describe('message limits', () => {+ const limits = { maxWindowIdBytes: 8, maxEventsPerMessage: 2 }+ const event = { type: 2, timestamp: fixedTimeMs }++ beforeEach(() => {+ jest.useFakeTimers({ now: fixedTime })+ })++ afterEach(() => {+ jest.restoreAllMocks()+ })++ it.each([+ [+ 'the window ID',+ 'w'.repeat(9),+ [event],+ 'message_window_id_too_long',+ { sessionId: 'session-1', windowIdBytes: 9, events: 1 },+ ],+ [+ 'the window ID in bytes',+ 'é'.repeat(5),+ [event],+ 'message_window_id_too_long',+ { sessionId: 'session-1', windowIdBytes: 10, events: 1 },+ ],+ [+ 'the event count',+ 'w',+ [event, event, event],+ 'message_too_many_events',+ { sessionId: 'session-1', windowIdBytes: 1, events: 3 },+ ],+ ] as const)(+ 'drops a message past the limit on %s when enforced',+ async (_, windowId, items, reason, details) => {+ const payload = createSnapshotPayload({+ sessionId: 'session-1',+ distinctId: 'user-123',+ windowId,+ snapshotItems: [...items],+ })++ const result = await createParseMessageStep({ ...limits, enforce: true })(createInput(0, 1, payload))++ expect(result).toMatchObject({+ type: PipelineResultType.DROP,+ reason,+ warnings: [{ type: reason, details }],+ })+ }+ )++ it.each([+ ['past a limit, not enforced', 'w'.repeat(9), false, ['message_window_id_too_long']],+ ['at the limits, enforced', 'w'.repeat(8), true, []],+ ])('keeps a message %s', async (_, windowId, enforce, counted) => {+ const overLimitSpy = jest+ .spyOn(SessionRecordingIngesterMetrics, 'incrementMessagesOverLimit')+ .mockImplementation()+ const payload = createSnapshotPayload({+ sessionId: 'session-1',+ distinctId: 'user-123',+ windowId,+ snapshotItems: [event, event],+ })++ const result = await createParseMessageStep({ ...limits, enforce })(createInput(0, 1, payload))++ expect(result.type).toBe(PipelineResultType.OK)+ expect(overLimitSpy.mock.calls.map(([limit]) => limit)).toEqual(counted)+ })+ })+it('should handle gzipped messages', async () => {const step = createParseMessageStep()const payload = createValidSnapshotPayload('session-gzip')diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.tsindex 31030dc3d708..27cd5a5adf32 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/parse-message-step.ts@@ -26,6 +26,14 @@ const GZIP_HEADER = Uint8Array.from([0x1f, 0x8b, 0x08, 0x00])// sample; exceeding it DLQs instead of risking an unclassifiable OOM.const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024+export interface MessageLimits {+ maxWindowIdBytes: number+ maxEventsPerMessage: number+ enforce: boolean+}++const DEFAULT_MESSAGE_LIMITS: MessageLimits = { maxWindowIdBytes: 1024, maxEventsPerMessage: 100_000, enforce: false }+export interface ParseMessageStepInput {message: Messageheaders: SessionReplayHeaders@@ -173,10 +181,9 @@ export function observeClockSkew(sentAt: string | undefined, now: string | undef* This step processes one message at a time since there are no batch-level optimizations.* Gzip decompression is done synchronously since the pipeline already runs steps concurrently.*/-export function createParseMessageStep<T extends ParseMessageStepInput>(): ProcessingStep<- T,- T & ParseMessageStepOutput-> {+export function createParseMessageStep<T extends ParseMessageStepInput>(+ limits: MessageLimits = DEFAULT_MESSAGE_LIMITS+): ProcessingStep<T, T & ParseMessageStepOutput> {return async function parseMessageStep(input) {const { message } = input@@ -221,6 +228,22 @@ export function createParseMessageStep<T extends ParseMessageStepInput>(): Proceconst sessionId = normalizeSessionId($session_id)+ const windowIdBytes = $window_id ? Buffer.byteLength($window_id) : 0+ const events = $snapshot_items.length+ SessionRecordingIngesterMetrics.observeMessageShape(windowIdBytes, events)+ const overLimit =+ windowIdBytes > limits.maxWindowIdBytes+ ? 'message_window_id_too_long'+ : events > limits.maxEventsPerMessage+ ? 'message_too_many_events'+ : null+ if (overLimit) {+ SessionRecordingIngesterMetrics.incrementMessagesOverLimit(overLimit)+ if (limits.enforce) {+ return drop(overLimit, [], [{ type: overLimit, details: { sessionId, windowIdBytes, events } }])+ }+ }+const clockSkewMs = quantizedClockSkewMs(messageResult.data.sent_at, messageResult.data.now)if (clockSkewMs !== 0) {SessionRecordingIngesterMetrics.observeClockSkewCorrection(clockSkewMs)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.tsindex 061496ae050b..53cc57de2acd 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts@@ -22,7 +22,7 @@ import { TeamForReplay } from '~/ingestion/pipelines/sessionreplay/teams/types'import { ValueMatcher } from '~/types'import { createLibVersionMonitorStep } from './lib-version-monitor-step'-import { createParseMessageStep } from './parse-message-step'+import { MessageLimits, createParseMessageStep } from './parse-message-step'import { MessageContext } from './pipeline-types'import { createRecordSessionEventStep } from './record-session-event-step'import { SessionBatchContext } from './session-batch-context'@@ -57,6 +57,7 @@ export type SessionReplayPipeline = BatchingPipeline<>export interface SessionReplayPipelineConfig {+ messageLimits?: MessageLimitsoutputs: IngestionOutputs<IngestionWarningsOutput | DlqOutput | OverflowOutput>eventIngestionRestrictionManager: EventIngestionRestrictionManageroverflowMode: IngestionOverflowMode@@ -140,7 +141,7 @@ export function createSessionReplayPipeline(config: SessionReplayPipelineConfig)b// Parse message content.pipe(- topHogWrapper(createParseMessageStep(), [+ topHogWrapper(createParseMessageStep(config.messageLimits), [timer('parse_time_ms_by_session_id', (input) => ({token: input.headers.token ?? 'unknown',session_id: input.headers.session_id ?? 'unknown',diff --git a/products/ingestion/skills/resolving-ingestion-warnings/SKILL.md b/products/ingestion/skills/resolving-ingestion-warnings/SKILL.mdindex d20d28285eb7..dc3cb8d301b3 100644--- a/products/ingestion/skills/resolving-ingestion-warnings/SKILL.md+++ b/products/ingestion/skills/resolving-ingestion-warnings/SKILL.md@@ -126,3 +126,5 @@ Two producers land in this category, and the `source` column tells them apart. `| `replay_lib_version_too_old` | Recording sent by an outdated posthog-js (1.x < 1.75) | Read [references/fixing-session-replay-warnings.md](references/fixing-session-replay-warnings.md) — recording still processed; upgrade posthog-js || `message_contained_no_valid_rrweb_events` | A replay message carried no usable snapshot data | Read [references/fixing-session-replay-warnings.md](references/fixing-session-replay-warnings.md) — that recording chunk was dropped; usually a rewriting proxy/transport or old SDK || `message_timestamp_diff_too_large` | Replay snapshot timestamps far from arrival time | Read [references/fixing-session-replay-warnings.md](references/fixing-session-replay-warnings.md) — chunk dropped at the 7-day threshold; persistent = clock skew, bursts = buffering |+| `message_window_id_too_long` | Replay snapshot window ID over the limit | Read [references/fixing-session-replay-warnings.md](references/fixing-session-replay-warnings.md) — chunk dropped at the 7-day threshold; persistent = clock skew, bursts = buffering |+| `message_too_many_events` | Replay snapshot message over the event limit | Read [references/fixing-session-replay-warnings.md](references/fixing-session-replay-warnings.md) — chunk dropped at the 7-day threshold; persistent = clock skew, bursts = buffering |diff --git a/products/ingestion/skills/resolving-ingestion-warnings/references/fixing-session-replay-warnings.md b/products/ingestion/skills/resolving-ingestion-warnings/references/fixing-session-replay-warnings.mdindex 0347b876a942..9e07eab22a82 100644--- a/products/ingestion/skills/resolving-ingestion-warnings/references/fixing-session-replay-warnings.md+++ b/products/ingestion/skills/resolving-ingestion-warnings/references/fixing-session-replay-warnings.md@@ -8,10 +8,12 @@ The symptom side is recordings that never appear or play back with gaps: each dr| `replay_lib_version_too_old` | info | Recording sent by posthog-js 1.x below 1.75 — **still processed**, but the SDK is too old to support all recording features. Debounced per version || `message_contained_no_valid_rrweb_events` | warning | A snapshot message carried no usable rrweb events — **that chunk was dropped** || `message_timestamp_diff_too_large` | warning | A snapshot message's event timestamps are 7+ days away from now — **that chunk was dropped** |+| `message_window_id_too_long` | error | A snapshot message's window ID is longer than the limit — **that chunk was dropped** |+| `message_too_many_events` | error | A snapshot message holds more events than the limit — **that chunk was dropped** |## Diagnose-1. Query the warnings with `posthog:execute-sql`: `SELECT timestamp, details FROM system.ingestion_warnings WHERE type IN ('replay_lib_version_too_old', 'message_contained_no_valid_rrweb_events', 'message_timestamp_diff_too_large') AND timestamp > now() - INTERVAL 7 DAY ORDER BY timestamp DESC LIMIT 20` (narrow to a single `type` to isolate one variant). For `message_timestamp_diff_too_large` the `details` JSON carries `startDiffDays`/`endDiffDays` vs the 7-day threshold — the magnitude tells the story (8 days = buffering/late flush; hundreds/negative-looking = a broken client clock).+1. Query the warnings with `posthog:execute-sql`: `SELECT timestamp, details FROM system.ingestion_warnings WHERE type IN ('replay_lib_version_too_old', 'message_contained_no_valid_rrweb_events', 'message_timestamp_diff_too_large', 'message_window_id_too_long', 'message_too_many_events') AND timestamp > now() - INTERVAL 7 DAY ORDER BY timestamp DESC LIMIT 20` (narrow to a single `type` to isolate one variant). For `message_timestamp_diff_too_large` the `details` JSON carries `startDiffDays`/`endDiffDays` vs the 7-day threshold — the magnitude tells the story (8 days = buffering/late flush; hundreds/negative-looking = a broken client clock).2. **Check SDK version clustering first** (`$lib_version` on the affected sessions' events, via `posthog:execute-sql`): `replay_lib_version_too_old` names the outdated version outright in its details, and the other two also concentrate on old or unusual SDK versions when the payload shape is the problem.3. For `message_contained_no_valid_rrweb_events`, look between the SDK and PostHog: a rewriting reverse proxy, compression/decompression middleware, or a custom transport that truncates or re-serializes the snapshot payload is the usual cause when the SDK is current.4. Distinguish clock from buffer for timestamp diffs: a device with a wrong clock produces the warning **continuously** for that user; buffered/offline replay produces **bursts** on reconnect.@@ -20,8 +22,9 @@ The symptom side is recordings that never appear or play back with gaps: each dr- **Old posthog-js** (`replay_lib_version_too_old`): upgrade to a current version — 1.75 is the floor for full recording support, but go to latest; recording pipelines evolve quickly. Hunt pinned snippets so every page ships the same version.- **Mangled snapshots** (`no_valid_rrweb_events`): let replay traffic reach PostHog unmodified — remove or fix payload-rewriting proxies and custom transports; upgrade the SDK if it predates the current snapshot format.+- **Oversized messages** (`message_window_id_too_long`, `message_too_many_events`): use a current PostHog SDK, which sends UUID window IDs and small batches. A custom SDK or transport must do the same.- **Timestamp diffs**: fix client clocks where the skew is persistent. Where it's buffering: replay is designed for near-real-time delivery — snapshot chunks older than 7 days are dropped by design, so recordings can't be imported or replayed into PostHog after the fact.## Verify-Record a fresh session on the affected app/platform, re-query `system.ingestion_warnings` with `posthog:execute-sql` (filter `type IN ('replay_lib_version_too_old', 'message_contained_no_valid_rrweb_events', 'message_timestamp_diff_too_large')`, `timestamp` after your fix) — no new occurrences — and confirm the new recording appears (`posthog:query-session-recordings-list`) and plays back complete, without gaps.+Record a fresh session on the affected app/platform, re-query `system.ingestion_warnings` with `posthog:execute-sql` (filter `type IN ('replay_lib_version_too_old', 'message_contained_no_valid_rrweb_events', 'message_timestamp_diff_too_large', 'message_window_id_too_long', 'message_too_many_events')`, `timestamp` after your fix) — no new occurrences — and confirm the new recording appears (`posthog:query-session-recordings-list`) and plays back complete, without gaps.diff --git a/rust/common/ingestion_warnings/warning_types.generated.json b/rust/common/ingestion_warnings/warning_types.generated.jsonindex 45b0fa59c3cc..421a89d920a0 100644--- a/rust/common/ingestion_warnings/warning_types.generated.json+++ b/rust/common/ingestion_warnings/warning_types.generated.json@@ -253,6 +253,18 @@"severity": "warning","captureProduced": false},+ {+ "type": "message_too_many_events",+ "category": "replay",+ "severity": "error",+ "captureProduced": false+ },+ {+ "type": "message_window_id_too_long",+ "category": "replay",+ "severity": "error",+ "captureProduced": false+ },{"type": "misrouted_event","category": "event",