:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay): bill each persisted recording once (#121613)
Alex Maysb24cafeca780
diff --git a/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts b/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.tsindex 4a832b3fc07f..797cf142c9bb 100644--- a/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts+++ b/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts@@ -54,7 +54,7 @@ export class CdpUsageReporterService {await this.send(() => this.batch.drain())}- private async send(sendBatch: () => Promise<void>): Promise<void> {+ private async send(sendBatch: () => Promise<unknown>): Promise<void> {if (this.timer) {clearTimeout(this.timer)this.timer = nulldiff --git a/nodejs/src/common/usage-ingestion/client.test.ts b/nodejs/src/common/usage-ingestion/client.test.tsindex aca45fc8ff52..9791cb664a09 100644--- a/nodejs/src/common/usage-ingestion/client.test.ts+++ b/nodejs/src/common/usage-ingestion/client.test.ts@@ -51,7 +51,7 @@ describe('UsageIngestionClient', () => {])('counts what the service accepted for $name', async ({ teams, accepted, sent, rejected }) => {acceptedRecordIds = accepted.map((teamId) => `record-${teamId}`)- await client.ingest(teams.map(record))+ expect(await client.ingest(teams.map(record))).toEqual(teams.map(record))expect(await counted('usage_ingestion_records_sent_total')).toEqual(sent === 0 ? [] : [expect.objectContaining({ value: sent })]@@ -92,7 +92,7 @@ describe('UsageIngestionClient', () => {},}- await client.ingest([record(7), record(8)])+ expect(await client.ingest([record(7), record(8)])).toEqual([])expect(calls).toBe(expected)expect(await counted('usage_ingestion_records_failed_total')).toEqual([@@ -117,7 +117,7 @@ describe('UsageIngestionClient', () => {},}- await client.ingest([record(7)])+ expect(await client.ingest([record(7)])).toEqual([record(7)])expect(calls).toBe(3)expect(await counted('usage_ingestion_records_sent_total')).toEqual([expect.objectContaining({ value: 1 })])diff --git a/nodejs/src/common/usage-ingestion/client.ts b/nodejs/src/common/usage-ingestion/client.tsindex 243fb856f8d8..18f7e7b92457 100644--- a/nodejs/src/common/usage-ingestion/client.ts+++ b/nodejs/src/common/usage-ingestion/client.ts@@ -98,18 +98,20 @@ export class UsageIngestionClient {this.maxBatchSize = config.maxBatchSize ?? 500}- async ingest(records: UsageRecordInput[]): Promise<void> {+ /** Returns the records that reached the service, rejected ones included: a retry fails them the same way. */+ async ingest(records: UsageRecordInput[]): Promise<UsageRecordInput[]> {if (records.length === 0) {- return+ return []}const chunks: UsageRecordInput[][] = []for (let i = 0; i < records.length; i += this.maxBatchSize) {chunks.push(records.slice(i, i + this.maxBatchSize))}- await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))+ const received = await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))+ return chunks.filter((_, i) => received[i]).flat()}- private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<void> {+ private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<boolean> {const request = create(IngestBillingUsageRequestSchema, {records: chunk.map((record) =>create(BillingUsageRecordSchema, {@@ -155,14 +157,14 @@ export class UsageIngestionClient {teams: rejectedTeams(chunk, rejected),})}+ return true} catch (error) {const code = error instanceof ConnectError ? error.code : Code.Unknownif (attempt + 1 < MAX_ATTEMPTS && RETRYABLE_CODES.has(code)) {usageRetriesCounter.inc({ producer_id: this.producerId, error_code: Code[code] ?? 'unknown' })// Backoff so a burst of connection churn does not eat every attempt at once.await delay(RETRY_BACKOFF_MS * 2 ** attempt)- await this.ingestChunk(chunk, attempt + 1)- return+ return this.ingestChunk(chunk, attempt + 1)}for (const record of chunk) {usageRecordsFailedCounter.inc({@@ -176,6 +178,7 @@ export class UsageIngestionClient {records: chunk.length,error: String(error),})+ return false}}}diff --git a/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts b/nodejs/src/common/usage-ingestion/usage-record-batch.test.tsindex ba66d4be88af..e66802b44db0 100644--- a/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts+++ b/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts@@ -80,7 +80,7 @@ describe('UsageRecordBatch', () => {* Queues one record, starts the send, then queues a second while the first is in flight.* Both acknowledgements are the test's to resolve, so neither case depends on timing.*/- async function sendWithALateAcknowledgement(send: (b: UsageRecordBatch) => Promise<void>): Promise<void> {+ async function sendWithALateAcknowledgement(send: (b: UsageRecordBatch) => Promise<unknown>): Promise<void> {const b = batch()let acknowledgeFirst!: (info: object) => voidlet acknowledgeSecond!: (info: object) => void@@ -131,6 +131,6 @@ describe('UsageRecordBatch', () => {])('flushes without waiting on acknowledgements for %s', async (_name, b) => {b.addAfterAcknowledgements([new Promise(() => {})], 1, 'events', 'uuid-1')- await expect(b.flush()).resolves.toBeUndefined()+ await expect(b.flush()).resolves.toEqual([])})})diff --git a/nodejs/src/common/usage-ingestion/usage-record-batch.ts b/nodejs/src/common/usage-ingestion/usage-record-batch.tsindex dc5d9a8b1889..c10f849d8b5e 100644--- a/nodejs/src/common/usage-ingestion/usage-record-batch.ts+++ b/nodejs/src/common/usage-ingestion/usage-record-batch.ts@@ -89,12 +89,12 @@ export class UsageRecordBatch {* pass, one send. A caller that is finished adding wants {@link drain} instead, because* nothing would flush that record afterwards.*/- async flush(): Promise<void> {+ async flush(): Promise<UsageRecordInput[]> {const pending = this.pendingAcknowledgementsthis.pendingAcknowledgements = []await Promise.all(pending)if (!this.client || this.records.size === 0) {- return+ return []}// Aggregate producers have no per-record capture time, so retain their flush-time clock.const flushedAtMs = Date.now()@@ -107,7 +107,7 @@ export class UsageRecordBatch {timestampMs: record.timestampMs ?? flushedAtMs,}))this.records.clear()- await this.client.ingest(records)+ return this.client.ingest(records)}/**diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts b/nodejs/src/ingestion/pipelines/sessionreplay/constants.tsindex bedb76a5cba8..bb441ebf839a 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts@@ -22,6 +22,9 @@ export const MAX_SESSION_LENGTH_SECONDS = 24 * 60 * 60// sessions that span across the boundary, delayed messages, etc.)export const SESSION_TRACKER_REDIS_TTL_SECONDS = 2 * MAX_SESSION_LENGTH_SECONDS+// Mobile SDKs upload queued recordings days later, so a billed mobile session's mark must outlive that upload.+export const SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS = 7 * 24 * 60 * 60+// Redis TTL for session blocklist keys// Matches session tracker TTL since blocked sessions should persist for same durationexport const SESSION_FILTER_REDIS_TTL_SECONDS = 2 * MAX_SESSION_LENGTH_SECONDSdiff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.tsindex 6cd4dfeb860f..80cef67ddd63 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts@@ -1,12 +1,18 @@+import { DateTime } from 'luxon'import { Message } from 'node-rdkafka'import { ingestionLagGauge, ingestionLagHistogram } from '~/common/metrics'+import { UsageIngestionClient } from '~/common/usage-ingestion/client'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { PostgresRouter } from '~/common/utils/db/postgres'import {SessionReplayBatchProgress,SessionReplayPipeline,runSessionReplayPipeline,} from '~/ingestion/pipelines/sessionreplay'+import { ParsedMessageData } from '~/ingestion/pipelines/sessionreplay/kafka/types'+import { SessionMetadataSink } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'+import { createMockSessionKey } from '~/ingestion/pipelines/sessionreplay/shared/test-helpers'import { KeyStore, RecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/types'import { RedisPool } from '~/types'@@ -79,6 +85,7 @@ async function lagHistogramCount(partition: string): Promise<number | undefined>describe('SessionRecordingIngester', () => {let ingester: SessionRecordingIngesterlet events: string[]+ let metadataStore: jest.Mocked<SessionMetadataSink>const runPipelineMock = jest.mocked(runSessionReplayPipeline)@@ -112,16 +119,31 @@ describe('SessionRecordingIngester', () => {} as unknown as KeyStoreconst fakeEncryptor = {start: jest.fn().mockResolvedValue(undefined),+ encryptBlockWithKey: jest.fn((_sessionId: string, _teamId: number, data: Buffer) => ({+ data,+ })),} as unknown as RecordingEncryptor+ metadataStore = {+ storeSessionBlocks: jest.fn().mockResolvedValue(undefined),+ }++ const sessionTrackerRedis = {+ mget: (keys: string[]) => Promise.resolve(keys.map(() => null)),+ pipeline: () => ({ set: jest.fn().mockReturnThis(), exec: () => Promise.resolve([]) }),+ }ingester = new SessionRecordingIngester(config,{} as jest.Mocked<PostgresRouter>,{} as unknown as ConstructorParameters<typeof SessionRecordingIngester>[2],- {} as unknown as RedisPool,+ {+ acquire: () => Promise.resolve(sessionTrackerRedis),+ release: () => Promise.resolve(),+ } as unknown as RedisPool,{} as unknown as RedisPool,{fileStorage: new BlackholeSessionBatchFileStorage(),+ metadataStore,keyStore: fakeKeyStore,encryptor: fakeEncryptor,createPipeline: () => ({}) as unknown as SessionReplayPipeline,@@ -138,12 +160,40 @@ describe('SessionRecordingIngester', () => {})}+ const recordOneSession = (): void => {+ const parsedMessage: ParsedMessageData = {+ metadata: { partition: 0, topic: consumeTopic, offset: 42, timestamp: 1000, rawSize: 100 },+ distinct_id: 'distinct-1',+ session_id: 'session-1',+ token: 'token',+ eventsByWindowId: { window1: [{ type: 2, timestamp: 1000, data: {} }] },+ eventsRange: { start: DateTime.fromMillis(1000), end: DateTime.fromMillis(1000) },+ snapshot_source: 'web',+ snapshot_library: 'posthog-js',+ }+ runPipelineMock.mockImplementation(async (_pipeline, _messages, recorder) => {+ await recorder.record(+ {+ team: { teamId: 1, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },+ message: parsedMessage,+ },+ '30d',+ createMockSessionKey()+ )+ return progress(new Map([[0, 42]]))+ })+ }+beforeEach(() => {events = []runPipelineMock.mockReset()createIngester()})+ afterEach(() => {+ jest.restoreAllMocks()+ })+it('stop() stores the in-flight batch offsets only after its side effects settle, before disconnect', async () => {const pipelineEntered = deferred()const pipelineGate = deferred()@@ -234,6 +284,51 @@ describe('SessionRecordingIngester', () => {expect(await lagHistogramCount('0')).toBe(1)})+ it('persists replay metadata when usage ingestion fails', async () => {+ createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })+ jest.spyOn(UsageRecordBatch.prototype, 'flush').mockRejectedValueOnce(new Error('usage unavailable'))+ recordOneSession()++ await expect(ingester.handleEachBatch([kafkaMessage(0, 42)])).resolves.toBeUndefined()++ expect(metadataStore.storeSessionBlocks).toHaveBeenCalledWith([+ expect.objectContaining({ teamId: 1, sessionId: 'session-1' }),+ ])+ expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([+ { topic: consumeTopic, partition: 0, offset: 43 },+ ])+ })++ it.each<[string, Partial<SessionRecordingIngesterCollaborators>, unknown[], string[]]>([+ [+ 'the main lane bills a persisted session before it stores the offsets',+ {},+ [[[expect.objectContaining({ teamId: 1, usageKey: 'session_replay_recordings', recordId: 'session-1' })]]],+ ['usage_sent', 'offsets_stored'],+ ],+ ['the ML mirror lane bills nothing', { reportUsage: false }, [], ['offsets_stored']],+ ])('%s', async (_name, collaborators, expectedIngestCalls, expectedEvents) => {+ createIngester(+ {+ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0,+ USAGE_INGESTION_ADDR: 'localhost:1',+ USAGE_INGESTION_REPORT_TEAMS: '*',+ },+ collaborators+ )+ const ingest = jest.spyOn(UsageIngestionClient.prototype, 'ingest').mockImplementation((records) => {+ events.push('usage_sent')+ return Promise.resolve(records)+ })+ recordOneSession()++ await ingester.handleEachBatch([kafkaMessage(0, 42)])++ expect(metadataStore.storeSessionBlocks).toHaveBeenCalledTimes(1)+ expect(ingest.mock.calls).toEqual(expectedIngestCalls)+ expect(events).toEqual(expectedEvents)+ })+it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {ingestionLagHistogram.reset()createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.tsindex 92d42d22b65d..eda264630a08 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts@@ -40,6 +40,7 @@ import { KafkaOffsetManager } from './kafka/offset-manager'import { SessionRecordingIngesterMetrics } from './metrics'import { SessionReplayLagReporter } from './session-replay-lag-reporter'import { SessionReplayPipelineFactory, SessionReplayPipelineRunner } from './session-replay-pipeline-runner'+import { billPersistedSessions } from './session-usage-step'import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'import { BlockCompression } from './sessions/block-compression'import { RetentionAwareStorage } from './sessions/retention-aware-batch-writer'@@ -89,6 +90,8 @@ export interface SessionRecordingIngesterCollaborators {redisKeyNamespace?: string/** The ML mirror reports how far back its events are complete; the main lane leaves this unset. */captureWatermark?: CaptureWatermark+ /** False on the ML mirror, because the main lane bills the same recordings. */+ reportUsage?: boolean}export class SessionRecordingIngester {@@ -161,10 +164,13 @@ export class SessionRecordingIngester {this.isDebugLoggingEnabled = buildIntegerMatcher(config.SESSION_RECORDING_DEBUG_PARTITION, true)this.promiseScheduler = new PromiseScheduler()- this.usageBatch = new UsageRecordBatch(createUsageIngestionClient(config, 'session_replay'), {- unit: 'recordings',- isTeamEnabled: usageReportTeamMatcher(config),- })+ this.usageBatch = new UsageRecordBatch(+ collaborators.reportUsage === false ? null : createUsageIngestionClient(config, 'session_replay'),+ {+ unit: 'recordings',+ isTeamEnabled: usageReportTeamMatcher(config),+ }+ )this.runner =collaborators.runner ??@@ -365,14 +371,15 @@ export class SessionRecordingIngester {})await this.batchLock(async () => {logger.info('🔁', 'blob_ingester_consumer_v2 - flushing batch', { batchSize: this.currentBatch.size })- // Billing has nothing to wait on the batch for, and a usage outage must not stop lag- // reporting or the batch reset — so it goes out alongside and swallows its own failure.- await Promise.all([- instrumentFn(`recordingingesterv2.handleEachBatch.flush`, async () => this.currentBatch.flush()),- this.usageBatch.flush().catch((error) => {- logger.warn('⚠️', 'blob_ingester_consumer_v2 - usage flush failed', { error })- }),- ])+ // Usage goes out before the offsets commit, so a crash cannot lose the bill of a session with no later+ // batch. A replayed batch resends the same record, and the usage table removes the duplicate.+ await instrumentFn(`recordingingesterv2.handleEachBatch.flush`, async () =>+ this.currentBatch.flush((sessions) =>+ billPersistedSessions(this.usageBatch, this.sessionTracker, sessions).catch((error) => {+ logger.warn('⚠️', 'blob_ingester_consumer_v2 - usage flush failed', { error })+ })+ )+ )// The flush committed the batch's offsets, so its data is now durably ingested — report lag.// Skipped if the flush above threw, so a failed flush records no premature lag sample.this.lagReporter.flush()@@ -406,7 +413,6 @@ export class SessionRecordingIngester {sessionKeyResolutionMaxConcurrency: this.config.SESSION_RECORDING_KEY_RESOLUTION_MAX_CONCURRENCY,topHog: this.topHog,isDebugLoggingEnabled: this.isDebugLoggingEnabled,- usageBatch: this.usageBatch,}this.runner.start(pipelineConfig)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.tsindex 3222578c9931..c5e14339bda1 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts@@ -90,20 +90,10 @@ export class SessionRecordingIngesterMetrics {buckets: [300, 900, 3600, 21600, 43200, 604800, Infinity],})- private static readonly unbilledNewSession = new Counter({- name: 'recording_blob_ingestion_v2_unbilled_new_session',- help: 'New sessions whose first message failed before the usage step, so a later message for the same session bills nothing while the report still counts the recording',- labelNames: ['reason'],- })-public static incrementMessageReceived(partition: number): void {this.messageReceived.labels(partition.toString()).inc()}- public static incrementUnbilledNewSession(reason: string): void {- this.unbilledNewSession.labels(reason).inc()- }-public static observeDroppedByRestrictions(count: number): void {this.messagesDroppedByRestrictions.inc(count)recordMessagesDroppedByRestrictions(count)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.tsindex bf63adec7ae7..a4335ba52da5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts@@ -48,6 +48,7 @@ describe('createRecordSessionEventStep', () => {team: TeamForReplay = defaultTeam): RecordSessionEventStepInput => ({team,+ headers: { now: new Date(1_700_000_000_000) },parsedMessage: createParsedMessage(overrides),retentionPeriod: '30d',sessionKey: createMockSessionKey(),@@ -78,7 +79,8 @@ describe('createRecordSessionEventStep', () => {message: input.parsedMessage,},'30d',- input.sessionKey+ input.sessionKey,+ 1_700_000_000_000)})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.tsindex 0c3eeca90a75..9c01d9a98946 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts@@ -11,6 +11,7 @@ import { ValueMatcher } from '~/types'export interface RecordSessionEventStepInput extends SessionBatchContext {team: TeamForReplay+ headers: { now?: Date }parsedMessage: ParsedMessageDataretentionPeriod: RetentionPeriodsessionKey: SessionKey@@ -59,7 +60,13 @@ export function createRecordSessionEventStep<T extends RecordSessionEventStepInp// Record to the batch's recorder, carried on the element by the pipeline's beforeBatch.const messageWithTeam: MessageWithTeam = { team, message: parsedMessage }- await sessionBatchRecorder.record(messageWithTeam, retentionPeriod, sessionKey)+ const captureTimestampMs = input.headers.now?.getTime()+ await sessionBatchRecorder.record(+ messageWithTeam,+ retentionPeriod,+ sessionKey,+ Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined+ )return ok(input)}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.tsindex a9503632559f..6c3775de7ba5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts@@ -1189,7 +1189,8 @@ describe('session-replay-pipeline', () => {}),}),'30d',- expect.objectContaining({ sessionState: 'cleartext' })+ expect.objectContaining({ sessionState: 'cleartext' }),+ undefined)})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.tsindex c59c0b2a2355..061496ae050b 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts@@ -2,7 +2,6 @@ import { Message } from 'node-rdkafka'import { DlqOutput, IngestionWarningsOutput, OverflowOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'-import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { EventIngestionRestrictionManager } from '~/common/utils/event-ingestion-restrictions'import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { IngestionOverflowMode } from '~/ingestion/config'@@ -32,7 +31,6 @@ import {addSessionReplaySessionResolution,withSessionReplayRecordingMetrics,} from './session-replay-pipeline-stages'-import { createRecordSessionUsageStep, trackUnbilledNewSessions } from './session-usage-step'export interface SessionReplayPipelineInput extends SessionBatchContext {message: Message@@ -78,7 +76,6 @@ export interface SessionReplayPipelineConfig {topHog: TopHogRegistry/** Debug logging matcher for partition-based debugging. */isDebugLoggingEnabled: ValueMatcher<number>- usageBatch?: UsageRecordBatch}/**@@ -93,7 +90,7 @@ export interface SessionReplayPipelineConfig {* 5. Record - Record parsed messages to the batch's recorder*/export function createSessionReplayPipeline(config: SessionReplayPipelineConfig): SessionReplayPipeline {- const { outputs, promiseScheduler, topHog, isDebugLoggingEnabled, usageBatch } = config+ const { outputs, promiseScheduler, topHog, isDebugLoggingEnabled } = configconst pipelineConfig: PipelineConfig<OverflowOutput> = {outputs,@@ -143,14 +140,12 @@ export function createSessionReplayPipeline(config: SessionReplayPipelineConfig)b// Parse message content.pipe(- trackUnbilledNewSessions(- topHogWrapper(createParseMessageStep(), [- timer('parse_time_ms_by_session_id', (input) => ({- token: input.headers.token ?? 'unknown',- session_id: input.headers.session_id ?? 'unknown',- })),- ])- )+ topHogWrapper(createParseMessageStep(), [+ timer('parse_time_ms_by_session_id', (input) => ({+ token: input.headers.token ?? 'unknown',+ session_id: input.headers.session_id ?? 'unknown',+ })),+ ]))// Monitor library version and emit warnings for old versions.pipe(createLibVersionMonitorStep())@@ -162,7 +157,6 @@ export function createSessionReplayPipeline(config: SessionReplayPipelineConfig)})))- .pipe(createRecordSessionUsageStep(usageBatch))).gather())diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.tsindex 6a118ad19550..c23b19ddd7ba 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts@@ -1,128 +1,201 @@-import { DateTime } from 'luxon'-import { register } from 'prom-client'-import { UsageIngestionClient, UsageRecordInput } from '~/common/usage-ingestion/client'import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'-import { drop, ok } from '~/ingestion/framework/results'--import { ParsedMessageData } from './kafka/types'-import { Recordable, SessionReplayHeaders } from './pipeline-types'-import { createRecordSessionUsageStep, trackUnbilledNewSessions } from './session-usage-step'-import { TeamForReplay } from './teams/types'+import { SessionTracker } from '~/ingestion/pipelines/sessionreplay/sessions/session-tracker'+import {+ SessionBlockMetadata,+ createNoopBlockMetadata,+} from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'+import { RedisPool } from '~/types'-type SessionUsageValue = Recordable<{- team: TeamForReplay- headers: SessionReplayHeaders- parsedMessage: ParsedMessageData- isNewSession: boolean-}>+import { billPersistedSessions, recordPersistedSessionUsage } from './session-usage-step'-describe('createRecordSessionUsageStep', () => {+describe('session usage', () => {let ingested: UsageRecordInput[]let usageBatch: UsageRecordBatch+ // 'stored' is a send that the service kept while the client saw it fail.+ let sendFailure: 'none' | 'lost' | 'stored'- beforeEach(() => {- ingested = []+ function newUsageBatch(): UsageRecordBatch {const client = {ingest: jest.fn((records: UsageRecordInput[]) => {- ingested.push(...records)- return Promise.resolve()+ if (sendFailure !== 'lost') {+ ingested.push(...records)+ }+ return Promise.resolve(sendFailure === 'none' ? records : [])}),} as unknown as UsageIngestionClient- usageBatch = new UsageRecordBatch(client, { unit: 'recordings', isTeamEnabled: () => true })+ return new UsageRecordBatch(client, { unit: 'recordings', isTeamEnabled: () => true })+ }++ beforeEach(() => {+ ingested = []+ sendFailure = 'none'+ usageBatch = newUsageBatch()})- function buildValue(- snapshotSource: string | null,- snapshotLibrary: string | null,- isNewSession: boolean- ): SessionUsageValue {+ function metadata(snapshotSource: string | null, captureTimestampMs?: number): SessionBlockMetadata {return {- team: { teamId: 42, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },- headers: { token: 'token', session_id: 'session-1', distinct_id: 'distinct-1' },- parsedMessage: {- distinct_id: 'distinct-1',- session_id: 'session-1',- token: 'token',- eventsByWindowId: { window1: [] },- eventsRange: { start: DateTime.fromMillis(0), end: DateTime.fromMillis(0) },- snapshot_source: snapshotSource,- snapshot_library: snapshotLibrary,- metadata: { partition: 0, topic: 'test-topic', rawSize: 0, offset: 0, timestamp: 0 },- },- isNewSession,- status: 'allowed',- sessionKey: {- plaintextKey: Buffer.alloc(0),- encryptedKey: Buffer.alloc(0),- sessionState: 'ciphertext',- },+ ...createNoopBlockMetadata('session-1', 42),+ snapshotSource,+ captureTimestampMs,}}- async function record(- snapshotSource: string | null,- snapshotLibrary: string | null,- isNewSession = true- ): Promise<string[]> {- const step = createRecordSessionUsageStep(usageBatch)- await step(buildValue(snapshotSource, snapshotLibrary, isNewSession))-+ async function record(sessions: SessionBlockMetadata[]): Promise<string[]> {+ recordPersistedSessionUsage(usageBatch, sessions)await usageBatch.flush()return ingested.map((record) => record.usageKey)}it.each([- ['web', 'posthog-js', ['session_replay_recordings']],- ['mobile', 'posthog-ios', ['mobile_replay_recordings']],- ['mobile', 'posthog-flutter', ['mobile_replay_recordings']],- // The report bills mobile replay only from four named libraries. The meter reads the- // source instead, so a mobile recording bills whatever library it names.- ['mobile', 'posthog-python', ['mobile_replay_recordings']],- ['mobile', null, ['mobile_replay_recordings']],- // Nothing validates the source, so anything but 'mobile' bills as web rather than for free.- ['desktop', 'posthog-js', ['session_replay_recordings']],- [null, null, ['session_replay_recordings']],- ])('bills a %s session from %s under %j', async (source, library, expectedUsageKeys) => {- expect(await record(source, library)).toEqual(expectedUsageKeys)+ ['web', ['session_replay_recordings']],+ ['mobile', ['mobile_replay_recordings']],+ ['desktop', ['session_replay_recordings']],+ [null, ['session_replay_recordings']],+ ])('bills persisted %s metadata under %j', async (source, expectedUsageKeys) => {+ expect(await record([metadata(source, 1_700_000_000_000)])).toEqual(expectedUsageKeys)})- it('bills nothing for a session already seen in this batch', async () => {- expect(await record('web', 'posthog-js', false)).toEqual([])- })+ it('uses the persisted session identity and trusted capture timestamp once', async () => {+ const persisted = metadata('web', 1_700_000_000_000)- it('uses trusted capture time from replay headers', async () => {- const capturedAtMs = 1_700_000_000_000- const value = buildValue('web', 'posthog-js', true)- value.headers.now = new Date(capturedAtMs)+ await record([persisted, persisted])- await createRecordSessionUsageStep(usageBatch)(value)- await usageBatch.flush()+ expect(ingested).toEqual([+ expect.objectContaining({+ recordId: 'session-1',+ quantity: 1,+ timestampMs: 1_700_000_000_000,+ }),+ ])+ })- expect(ingested[0]).toEqual(expect.objectContaining({ timestampMs: capturedAtMs }))+ it('bills nothing without persisted replay metadata', async () => {+ expect(await record([])).toEqual([])})- describe('trackUnbilledNewSessions', () => {- async function unbilledCount(): Promise<number> {- const metric = register.getSingleMetric('recording_blob_ingestion_v2_unbilled_new_session')!- const data = (await metric.get()) as { values: { value: number }[] }- return data.values.reduce((total, entry) => total + entry.value, 0)+ describe('billing each saved session once', () => {+ const FIRST_FLUSH = Date.parse('2026-10-15T23:59:58Z')+ const SECOND_FLUSH = Date.parse('2026-10-16T00:00:03Z')+ const seenKey = '@posthog/replay/session-seen:42:session-1'+ const billedKey = '@posthog/replay/session-billed:42:session-1'++ let redisValues: Map<string, string>+ let redisTtls: Map<string, number>+ let redisAvailable: boolean+ let redisPool: RedisPool++ beforeEach(() => {+ redisValues = new Map()+ redisTtls = new Map()+ redisAvailable = true+ const client = {+ pipeline: () => {+ const commands: (() => string | null)[] = []+ const pipeline = {+ set: (key: string, value: string, _mode: string, ttlSeconds: number, condition?: string) => {+ commands.push(() => {+ if (condition === 'NX' && redisValues.has(key)) {+ return null+ }+ redisValues.set(key, value)+ redisTtls.set(key, ttlSeconds)+ return 'OK'+ })+ return pipeline+ },+ get: (key: string) => {+ commands.push(() => redisValues.get(key) ?? null)+ return pipeline+ },+ exec: () =>+ redisAvailable+ ? Promise.resolve(commands.map((command) => [null, command()]))+ : Promise.reject(new Error('Redis unavailable')),+ }+ return pipeline+ },+ }+ redisPool = {+ acquire: () => Promise.resolve(client),+ release: () => Promise.resolve(),+ } as unknown as RedisPool+ })++ // A new tracker for each flush, so the local cache cannot hide what Redis holds.+ async function flush(sessions: SessionBlockMetadata[]): Promise<void> {+ await billPersistedSessions(usageBatch, new SessionTracker(redisPool, 60_000), sessions)}+ it('bills a session saved in two flushes once, across midnight', async () => {+ await flush([metadata('web', FIRST_FLUSH)])+ await flush([metadata('web', SECOND_FLUSH)])++ expect(ingested).toEqual([expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH })])+ })++ it.each([+ ['loses', 'lost', 1],+ ['stores', 'stored', 2],+ ] as const)(+ 'bills a failed send on the next saved flush with the first flush time when the service %s it',+ async (_name, failure, records) => {+ sendFailure = failure+ await flush([metadata('web', FIRST_FLUSH)])+ sendFailure = 'none'+ await flush([metadata('web', SECOND_FLUSH)])++ expect(ingested).toEqual(+ Array(records).fill(expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH }))+ )+ }+ )++ it('sends one billing time when two consumers bill a session at once', async () => {+ await Promise.all([+ flush([metadata('web', FIRST_FLUSH)]),+ billPersistedSessions(newUsageBatch(), new SessionTracker(redisPool, 60_000), [+ metadata('web', SECOND_FLUSH),+ ]),+ ])++ expect(new Set(ingested.map((record) => record.timestampMs))).toEqual(new Set([FIRST_FLUSH]))+ })++ it('bills a session saved in only one flush when Redis is unavailable', async () => {+ redisAvailable = false+ await flush([metadata('web', FIRST_FLUSH)])++ expect(ingested).toEqual([expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH })])+ })+it.each([- ['counts a new session whose message fails', true, false, 1],- ['counts nothing for a session already seen', false, false, 0],- ['counts nothing for a new session that parses', true, true, 0],- ])('%s', async (_name, isNewSession, succeeds, expectedIncrease) => {- const value = buildValue('web', 'posthog-js', isNewSession)- const before = await unbilledCount()- const step = trackUnbilledNewSessions(() =>- Promise.resolve(succeeds ? ok(value) : drop<typeof value>('message_contained_no_valid_rrweb_events'))- )-- await step(value)-- expect(await unbilledCount()).toEqual(before + expectedIncrease)+ ['a session the tracker marked seen but did not bill', 'seen', ['session-1']],+ ['nothing for a session that an earlier release billed when it first saw it', '1', []],+ ])('bills %s, also after the seen key expires', async (_name, trackerValue, billedSessionIds) => {+ redisValues.set(seenKey, trackerValue)++ await flush([metadata('mobile', FIRST_FLUSH)])+ redisValues.delete(seenKey)+ await flush([metadata('mobile', SECOND_FLUSH)])++ expect(ingested.map((record) => record.recordId)).toEqual(billedSessionIds)})++ it.each([+ ['mobile', 7 * 24 * 60 * 60],+ ['web', 48 * 60 * 60],+ ])(+ 'keeps a billed %s session marked for %i seconds without changing its seen key',+ async (source, ttlSeconds) => {+ redisValues.set(seenKey, 'seen')+ redisTtls.set(seenKey, 48 * 60 * 60)++ await flush([metadata(source, FIRST_FLUSH)])++ expect(redisTtls.get(billedKey)).toBe(ttlSeconds)+ expect([redisValues.get(seenKey), redisTtls.get(seenKey)]).toEqual(['seen', 48 * 60 * 60])+ }+ )})})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.tsindex 3b6e763170da..1119cf283375 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts@@ -1,17 +1,13 @@import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'-import { PipelineResult, isOkResult, ok } from '~/ingestion/framework/results'-import { ProcessingStep } from '~/ingestion/framework/steps'+import {+ SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS,+ SESSION_TRACKER_REDIS_TTL_SECONDS,+} from '~/ingestion/pipelines/sessionreplay/constants'+import { BillingClaim, SessionTracker } from '~/ingestion/pipelines/sessionreplay/sessions/session-tracker'+import { SessionBlockMetadata } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'+import { SessionMap } from '~/ingestion/pipelines/sessionreplay/shared/session-map'-import { ParsedMessageData } from './kafka/types'-import { SessionRecordingIngesterMetrics } from './metrics'-import { NewSessionFlag, Recordable, SessionReplayHeaders } from './pipeline-types'-import { TeamForReplay } from './teams/types'--type SessionUsageInput = {- team: TeamForReplay- headers: SessionReplayHeaders- parsedMessage: ParsedMessageData-} & NewSessionFlag+const MOBILE_METER = 'mobile_replay_recordings'/*** The meter this session bills under. Every session bills exactly one, so no `snapshot_source`@@ -23,49 +19,70 @@ type SessionUsageInput = {* libraries, so it bills neither meter for a session outside those values. Those are holes on its* side rather than a contract to copy.*- * The meter reads the first message processed for the session. The report instead reads the- * earliest snapshot's metadata, so the two disagree if one session's messages disagree with each- * other about their source, which no single client does.+ * The meter reads the snapshot source stored with the replay metadata.*/function billableMeter(snapshotSource: string | null): string {- return snapshotSource === 'mobile' ? 'mobile_replay_recordings' : 'session_replay_recordings'+ return snapshotSource === 'mobile' ? MOBILE_METER : 'session_replay_recordings'}-export function createRecordSessionUsageStep<T extends SessionUsageInput>(- usageBatch?: UsageRecordBatch-): ProcessingStep<Recordable<T>, Recordable<T>> {- return function recordSessionUsage(value): Promise<PipelineResult<Recordable<T>>> {- if (value.isNewSession) {- const meter = billableMeter(value.parsedMessage.snapshot_source)- const captureTimestampMs = value.headers.now?.getTime()- usageBatch?.add(- value.team.teamId,- meter,- value.headers.session_id,- 1,- undefined,- Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined- )- }- return Promise.resolve(ok(value))+function billedTtlSeconds(meter: string): number {+ return meter === MOBILE_METER ? SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS : SESSION_TRACKER_REDIS_TTL_SECONDS+}++export function recordPersistedSessionUsage(usageBatch: UsageRecordBatch, sessions: SessionBlockMetadata[]): void {+ for (const session of sessions) {+ usageBatch.add(+ session.teamId,+ billableMeter(session.snapshotSource),+ session.sessionId,+ 1,+ undefined,+ session.captureTimestampMs+ )}}/**- * Counts the new sessions that lose their flag before they can bill. The pipeline marks a session- * seen before this step's message is parsed, so a first message that fails takes the flag with it:- * a later valid message for the same session bills nothing, while the report still counts the- * recording it produced. Marking later is not free — the mark sits after key resolution so a- * recording is never written in cleartext — so measure how often this happens before paying for it.+ * Bills each saved session once. A session counts as billed only after its record reaches the usage service,+ * so a failed send bills on the session's next saved flush. Every send of a session, on any consumer, carries the+ * billing time that the tracker stored first. The usage table then removes each resend.*/-export function trackUnbilledNewSessions<T extends NewSessionFlag, U>(- step: ProcessingStep<T, U>-): ProcessingStep<T, U> {- return async function trackUnbilledNewSession(value): Promise<PipelineResult<U>> {- const result = await step(value)- if (value.isNewSession && !isOkResult(result)) {- SessionRecordingIngesterMetrics.incrementUnbilledNewSession(result.reason)+export async function billPersistedSessions(+ usageBatch: UsageRecordBatch,+ sessionTracker: SessionTracker,+ sessions: SessionBlockMetadata[]+): Promise<void> {+ const claims = new SessionMap<BillingClaim>()+ for (const session of sessions) {+ if (usageBatch.accepts(session.teamId) && !claims.has(session.teamId, session.sessionId)) {+ claims.set(session.teamId, session.sessionId, {+ teamId: session.teamId,+ sessionId: session.sessionId,+ timestampMs: session.captureTimestampMs ?? Date.now(),+ ttlSeconds: billedTtlSeconds(billableMeter(session.snapshotSource)),+ })}- return result}+ if (claims.size === 0) {+ return+ }++ const billingTimes = await sessionTracker.claimBillingTimes([...claims.values()])+ recordPersistedSessionUsage(+ usageBatch,+ sessions.flatMap((session) => {+ const timestampMs = billingTimes.get(session.teamId, session.sessionId)+ return timestampMs === undefined ? [] : [{ ...session, captureTimestampMs: timestampMs }]+ })+ )+ const received = await usageBatch.flush()+ await sessionTracker.markBilled(+ received.flatMap((record) => {+ const timestampMs = billingTimes.get(record.teamId, record.recordId)+ const claim = claims.get(record.teamId, record.recordId)+ return timestampMs === undefined || !claim+ ? []+ : [{ teamId: record.teamId, sessionId: record.recordId, timestampMs, ttlSeconds: claim.ttlSeconds }]+ })+ )}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.tsindex bde0b3ce083c..5daa606f2c62 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts@@ -218,8 +218,9 @@ describe('SessionBatchRecorder', () => {const record = (message: MessageWithTeam,retentionPeriod: RetentionPeriod = '30d',- sessionKey: SessionKey = createMockSessionKey()- ): Promise<number> => recorder.record(message, retentionPeriod, sessionKey)+ sessionKey: SessionKey = createMockSessionKey(),+ captureTimestampMs?: number+ ): Promise<number> => recorder.record(message, retentionPeriod, sessionKey, captureTimestampMs)beforeEach(() => {jest.clearAllMocks()@@ -384,9 +385,10 @@ describe('SessionBatchRecorder', () => {},])- await record(message)+ await record(message, '30d', createMockSessionKey(), 1_700_000_000_000)- await recorder.flush()+ const persistedSessions = await recorder.flush()+ expect(persistedSessions[0].captureTimestampMs).toBe(1_700_000_000_000)const writtenData = captureWrittenData(mockWriter.writeSession as jest.Mock)expect(mockWriter.finish).toHaveBeenCalledTimes(1)@@ -1681,7 +1683,8 @@ describe('SessionBatchRecorder', () => {expect(bytesWritten2).toBeGreaterThan(0)expect(bytesWritten3).toBe(0)- await recorder.flush()+ const persistedSessions = await recorder.flush()+ expect(persistedSessions).toEqual([])const writtenData = captureWrittenData(mockWriter.writeSession as jest.Mock)expect(writtenData).toHaveLength(0)})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.tsindex e2794cf2cce5..13902cd57033 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts@@ -32,6 +32,7 @@ interface SessionBatchEntry {featureRecorder: SessionFeatureRecordersessionKey: SessionKeyretentionPeriod: RetentionPeriod+ captureTimestampMs?: number}// A flush holds the batch lock, so nothing else here wants the pool while it runs.@@ -120,7 +121,8 @@ export class SessionBatchRecorder {public async record(message: MessageWithTeam,retentionPeriod: RetentionPeriod,- sessionKey: SessionKey+ sessionKey: SessionKey,+ captureTimestampMs?: number): Promise<number> {const { partition } = message.message.metadataconst sessionId = message.message.session_id@@ -181,6 +183,7 @@ export class SessionBatchRecorder {}return 0}+ existingBatchState.captureTimestampMs ??= captureTimestampMs} else {this.sessions.set(teamId, sessionId, {sessionBlockRecorder: new SessionBlockRecorder(sessionId, teamId, this.batchId, this.compression),@@ -198,6 +201,7 @@ export class SessionBatchRecorder {),sessionKey,retentionPeriod,+ captureTimestampMs,})}@@ -246,9 +250,13 @@ export class SessionBatchRecorder {/*** Flushes the session recordings to storage and commits Kafka offsets*+ * @param beforeCommit - Runs after the flush stores the batch and before its offsets commit. It must be safe+ * to repeat, because a crash between the two replays the batch, and it must catch its own errors.* @throws If the flush operation fails*/- public async flush(): Promise<SessionBlockMetadata[]> {+ public async flush(+ beforeCommit?: (sessions: SessionBlockMetadata[]) => Promise<void>+ ): Promise<SessionBlockMetadata[]> {logger.info('🔁', 'session_batch_recorder_flushing', {sessions: this.sessions.size,totalSize: this._size,@@ -280,7 +288,14 @@ export class SessionBatchRecorder {for (const [index,- { sessionBlockRecorder, consoleLogRecorder, featureRecorder, sessionKey, retentionPeriod },+ {+ sessionBlockRecorder,+ consoleLogRecorder,+ featureRecorder,+ sessionKey,+ retentionPeriod,+ captureTimestampMs,+ },] of entries.entries()) {const {buffer,@@ -360,6 +375,7 @@ export class SessionBatchRecorder {retentionPeriodDays,isDeleted: false,...(earliestCapturedAtMs === undefined ? {} : { earliestCapturedAtMs }),+ captureTimestampMs,})totalEvents += eventCount@@ -371,6 +387,7 @@ export class SessionBatchRecorder {await this.consoleLogStore.flush()await this.featureStore.storeSessionFeatures(featureBlocks)await this.metadataStore.storeSessionBlocks(blockMetadata)+ await beforeCommit?.(blockMetadata)await this.offsetManager.commit()// Update metricsdiff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.tsindex 124029d92f49..3b6c77b4ae68 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts@@ -108,8 +108,20 @@ describe('SessionTracker', () => {it('sets each session key with a 48h TTL in one pipeline', async () => {await sessionTracker.markSeen(sessionSet([1, 'a'], [2, 'b']))- expect(mockPipeline.set).toHaveBeenCalledWith('@posthog/replay/session-seen:1:a', '1', 'EX', TTL_SECONDS)- expect(mockPipeline.set).toHaveBeenCalledWith('@posthog/replay/session-seen:2:b', '1', 'EX', TTL_SECONDS)+ expect(mockPipeline.set).toHaveBeenCalledWith(+ '@posthog/replay/session-seen:1:a',+ 'seen',+ 'EX',+ TTL_SECONDS,+ 'NX'+ )+ expect(mockPipeline.set).toHaveBeenCalledWith(+ '@posthog/replay/session-seen:2:b',+ 'seen',+ 'EX',+ TTL_SECONDS,+ 'NX'+ )expect(mockPipeline.exec).toHaveBeenCalledTimes(1)expect(SessionBatchMetrics.incrementNewSessionsDetected).toHaveBeenCalledTimes(2)})@@ -128,9 +140,10 @@ describe('SessionTracker', () => {expect(mockPipeline.set).toHaveBeenCalledWith('@posthog/replay/ml-mirror/session-seen:1:a',- '1',+ 'seen','EX',- TTL_SECONDS+ TTL_SECONDS,+ 'NX')})@@ -155,4 +168,16 @@ describe('SessionTracker', () => {expect(mockRedisClient.mget).not.toHaveBeenCalled()})})++ describe('markBilled', () => {+ it('counts a Redis error that the pipeline returns as a command result', async () => {+ mockPipeline.exec = jest.fn().mockResolvedValue([[new Error('OOM command not allowed'), null]])++ await sessionTracker.markBilled([+ { teamId: 1, sessionId: 'a', timestampMs: 1_700_000_000_000, ttlSeconds: 60 },+ ])++ expect(SessionBatchMetrics.incrementSessionTrackerRedisErrors).toHaveBeenCalledTimes(1)+ })+ })})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.tsindex 2ea25d4b0a8e..cf9d9e90a9c5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts@@ -9,6 +9,51 @@ import { SessionBatchMetrics } from './metrics'const DEFAULT_LOCAL_CACHE_MAX_SIZE = 100_000+// Any value in a seen key means seen. Code that billed a session when it first saw it wrote '1' to the seen key,+// so a session with that value counts as billed.+const SEEN = 'seen'+const BILLED_AT_FIRST_SIGHT = '1'+// The billed key holds the time, in milliseconds, that every usage record for the session carries. The usage table+// removes a duplicate record only within one UTC day, so one stored time lets it remove every resend of the session.+// The prefix marks a session whose record reached the usage service. The billed key is separate from the seen key+// because billing must not change how long a session stays seen. The seen state decides whether a session gets a new+// encryption key or reads its existing one.+const BILLED_PREFIX = 'billed:'++export interface BillingClaim {+ teamId: number+ sessionId: string+ /** The time to store when the session has no stored time. */+ timestampMs: number+ ttlSeconds: number+}++export interface BilledSession {+ teamId: number+ sessionId: string+ timestampMs: number+ ttlSeconds: number+}++type PipelineResults = [Error | null, unknown][] | null++function pipelineValues(results: PipelineResults, count: number): unknown[] {+ if (!results || results.length !== count) {+ throw new Error('Redis pipeline returned an incomplete result')+ }+ return results.map(([error, value]) => {+ if (error) {+ throw error+ }+ return value+ })+}++function storedBillingTime(value: unknown): number | undefined {+ const timestampMs = typeof value === 'string' ? Number(value) : NaN+ return Number.isSafeInteger(timestampMs) && timestampMs > 0 ? timestampMs : undefined+}+/*** Tracks which sessions have been seen before, so a session's first sighting can be treated as new* (rate-limited, key-generated) and later sightings as existing.@@ -43,8 +88,12 @@ const DEFAULT_LOCAL_CACHE_MAX_SIZE = 100_000export class SessionTracker {private readonly keyPrefix: string+ private readonly billedKeyPrefix: string+private readonly localCache: LRUCache<string, true>+ private readonly billedCache: LRUCache<string, true>+/*** @param keyNamespace - Optional Redis key namespace. Omit (the default) for the main lane — this* keeps the key prefix exactly `@posthog/replay/session-seen` so existing sessions aren't seen as@@ -58,11 +107,17 @@ export class SessionTracker {localCacheMaxSize: number = DEFAULT_LOCAL_CACHE_MAX_SIZE,keyNamespace?: string) {- this.keyPrefix = keyNamespace ? `@posthog/replay/${keyNamespace}/session-seen` : '@posthog/replay/session-seen'+ const namespacePrefix = keyNamespace ? `@posthog/replay/${keyNamespace}` : '@posthog/replay'+ this.keyPrefix = `${namespacePrefix}/session-seen`+ this.billedKeyPrefix = `${namespacePrefix}/session-billed`this.localCache = new LRUCache({max: localCacheMaxSize,ttl: localCacheTtlMs,})+ this.billedCache = new LRUCache({+ max: localCacheMaxSize,+ ttl: localCacheTtlMs,+ })}/**@@ -150,7 +205,9 @@ export class SessionTracker {const key = this.generateKey(teamId, sessionId)this.localCache.set(key, true)SessionBatchMetrics.incrementNewSessionsDetected()- pipeline.set(key, '1', 'EX', SESSION_TRACKER_REDIS_TTL_SECONDS)+ // NX keeps a '1' that code billing at first sight wrote after this consumer's hasSeen read. The key+ // normally does not exist here, because only sessions that read as new reach this write.+ pipeline.set(key, SEEN, 'EX', SESSION_TRACKER_REDIS_TTL_SECONDS, 'NX')}await pipeline.exec()} catch (error) {@@ -164,7 +221,106 @@ export class SessionTracker {}}+ /**+ * Returns the billing time of each session that has not billed. The first call for a session stores its+ * candidate time, and every later call, on any consumer, returns the stored time.+ * A session without a billed mark counts as unbilled, because a mark that failed to persist must not stop the bill.+ * On a Redis error every session outside the local billed cache counts as unbilled, with its candidate time. A+ * session saved in only one flush has no later flush to bill it. The usage table removes a resend only when both+ * records fall on the same UTC day.+ */+ public async claimBillingTimes(claims: BillingClaim[]): Promise<SessionMap<number>> {+ const unbilled = new SessionMap<number>()+ const misses = claims.filter(+ ({ teamId, sessionId }) => !this.billedCache.has(this.generateBilledKey(teamId, sessionId))+ )++ if (misses.length === 0) {+ return unbilled+ }++ // Sessions that code billing at first sight already billed. They get a billed mark, so a late upload does+ // not bill them again after their seen key expires.+ const billedAtFirstSight: BilledSession[] = []+ const startTime = performance.now()+ let client+ try {+ client = await this.redisPool.acquire()+ const pipeline = client.pipeline()+ for (const { teamId, sessionId, timestampMs, ttlSeconds } of misses) {+ const billedKey = this.generateBilledKey(teamId, sessionId)+ pipeline.set(billedKey, String(timestampMs), 'EX', ttlSeconds, 'NX')+ pipeline.get(billedKey)+ pipeline.get(this.generateKey(teamId, sessionId))+ }+ const values = pipelineValues(await pipeline.exec(), misses.length * 3)+ for (let i = 0; i < misses.length; i++) {+ const { teamId, sessionId, timestampMs, ttlSeconds } = misses[i]+ const [billedValue, seenValue] = [values[3 * i + 1], values[3 * i + 2]]+ if (typeof billedValue === 'string' && billedValue.startsWith(BILLED_PREFIX)) {+ this.billedCache.set(this.generateBilledKey(teamId, sessionId), true)+ } else if (seenValue === BILLED_AT_FIRST_SIGHT) {+ const storedTimestampMs = storedBillingTime(billedValue) ?? timestampMs+ billedAtFirstSight.push({ teamId, sessionId, timestampMs: storedTimestampMs, ttlSeconds })+ } else {+ unbilled.set(teamId, sessionId, storedBillingTime(billedValue) ?? timestampMs)+ }+ }+ } catch (error) {+ logger.error('session_tracker_claim_billing_times_redis_error', { error: String(error) })+ SessionBatchMetrics.incrementSessionTrackerRedisErrors()+ for (const { teamId, sessionId, timestampMs } of misses) {+ unbilled.set(teamId, sessionId, timestampMs)+ }+ return unbilled+ } finally {+ if (client) {+ await this.redisPool.release(client)+ }+ SessionBatchMetrics.observeSessionTrackerRedisLatency((performance.now() - startTime) / 1000)+ }++ await this.markBilled(billedAtFirstSight)+ return unbilled+ }++ /**+ * Call only after the sessions' usage records reached the usage service. Fails open, like {@link markSeen}. The+ * local cache keeps the mark when the Redis write fails, so a Redis failure does not resend the session on every+ * flush. A later resend carries the billing time that Redis stored, and the usage table removes it.+ */+ public async markBilled(sessions: BilledSession[]): Promise<void> {+ if (sessions.length === 0) {+ return+ }++ const startTime = performance.now()+ let client+ try {+ client = await this.redisPool.acquire()+ const pipeline = client.pipeline()+ for (const { teamId, sessionId, timestampMs, ttlSeconds } of sessions) {+ const key = this.generateBilledKey(teamId, sessionId)+ this.billedCache.set(key, true)+ pipeline.set(key, `${BILLED_PREFIX}${timestampMs}`, 'EX', ttlSeconds)+ }+ pipelineValues(await pipeline.exec(), sessions.length)+ } catch (error) {+ logger.error('session_tracker_mark_billed_redis_error', { error: String(error) })+ SessionBatchMetrics.incrementSessionTrackerRedisErrors()+ } finally {+ if (client) {+ await this.redisPool.release(client)+ }+ SessionBatchMetrics.observeSessionTrackerRedisLatency((performance.now() - startTime) / 1000)+ }+ }+private generateKey(teamId: number, sessionId: string): string {return `${this.keyPrefix}:${teamId}:${sessionId}`}++ private generateBilledKey(teamId: number, sessionId: string): string {+ return `${this.billedKeyPrefix}:${teamId}:${sessionId}`+ }}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.tsindex 23adddff8e82..aebc55908be2 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts@@ -45,6 +45,8 @@ export function createDeletionBlockMetadata(sessionId: string, teamId: number):}export interface SessionBlockMetadata {+ /** Trusted capture timestamp used to attribute billing without persisting it as replay metadata. */+ captureTimestampMs?: numberreplayIndexEntries?: ReplayIndexEntry[]replayIndexTruncated?: boolean/** Unique identifier for the session */diff --git a/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts b/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.tsindex f428821d1148..bb442986219e 100644--- a/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts+++ b/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts@@ -183,6 +183,7 @@ export class IngestionSessionReplayMlMirrorServer extends MlMirrorConsumerServer// would then fetch a missing key and record cleartext.redisKeyNamespace: 'ml-mirror',captureWatermark: new CaptureWatermark('ml_mirror'),+ reportUsage: false,}const ingester = new SessionRecordingIngester(