Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay): bill each persisted recording once (#121613)

Alex Maysb24cafeca780
diff --git a/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts b/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts
index 4a832b3fc07f..797cf142c9bb 100644
--- a/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts
+++ b/nodejs/src/cdp/services/usage/cdp-usage-reporter.service.ts
@@ -54,7 +54,7 @@ export class CdpUsageReporterService {
await this.send(() => this.batch.drain())
}
- private async send(sendBatch: () => Promise<void>): Promise<void> {
+ private async send(sendBatch: () => Promise<unknown>): Promise<void> {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
diff --git a/nodejs/src/common/usage-ingestion/client.test.ts b/nodejs/src/common/usage-ingestion/client.test.ts
index aca45fc8ff52..9791cb664a09 100644
--- a/nodejs/src/common/usage-ingestion/client.test.ts
+++ b/nodejs/src/common/usage-ingestion/client.test.ts
@@ -51,7 +51,7 @@ describe('UsageIngestionClient', () => {
])('counts what the service accepted for $name', async ({ teams, accepted, sent, rejected }) => {
acceptedRecordIds = accepted.map((teamId) => `record-${teamId}`)
- await client.ingest(teams.map(record))
+ expect(await client.ingest(teams.map(record))).toEqual(teams.map(record))
expect(await counted('usage_ingestion_records_sent_total')).toEqual(
sent === 0 ? [] : [expect.objectContaining({ value: sent })]
@@ -92,7 +92,7 @@ describe('UsageIngestionClient', () => {
},
}
- await client.ingest([record(7), record(8)])
+ expect(await client.ingest([record(7), record(8)])).toEqual([])
expect(calls).toBe(expected)
expect(await counted('usage_ingestion_records_failed_total')).toEqual([
@@ -117,7 +117,7 @@ describe('UsageIngestionClient', () => {
},
}
- await client.ingest([record(7)])
+ expect(await client.ingest([record(7)])).toEqual([record(7)])
expect(calls).toBe(3)
expect(await counted('usage_ingestion_records_sent_total')).toEqual([expect.objectContaining({ value: 1 })])
diff --git a/nodejs/src/common/usage-ingestion/client.ts b/nodejs/src/common/usage-ingestion/client.ts
index 243fb856f8d8..18f7e7b92457 100644
--- a/nodejs/src/common/usage-ingestion/client.ts
+++ b/nodejs/src/common/usage-ingestion/client.ts
@@ -98,18 +98,20 @@ export class UsageIngestionClient {
this.maxBatchSize = config.maxBatchSize ?? 500
}
- async ingest(records: UsageRecordInput[]): Promise<void> {
+ /** Returns the records that reached the service, rejected ones included: a retry fails them the same way. */
+ async ingest(records: UsageRecordInput[]): Promise<UsageRecordInput[]> {
if (records.length === 0) {
- return
+ return []
}
const chunks: UsageRecordInput[][] = []
for (let i = 0; i < records.length; i += this.maxBatchSize) {
chunks.push(records.slice(i, i + this.maxBatchSize))
}
- await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))
+ const received = await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))
+ return chunks.filter((_, i) => received[i]).flat()
}
- private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<void> {
+ private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<boolean> {
const request = create(IngestBillingUsageRequestSchema, {
records: chunk.map((record) =>
create(BillingUsageRecordSchema, {
@@ -155,14 +157,14 @@ export class UsageIngestionClient {
teams: rejectedTeams(chunk, rejected),
})
}
+ return true
} catch (error) {
const code = error instanceof ConnectError ? error.code : Code.Unknown
if (attempt + 1 < MAX_ATTEMPTS && RETRYABLE_CODES.has(code)) {
usageRetriesCounter.inc({ producer_id: this.producerId, error_code: Code[code] ?? 'unknown' })
// Backoff so a burst of connection churn does not eat every attempt at once.
await delay(RETRY_BACKOFF_MS * 2 ** attempt)
- await this.ingestChunk(chunk, attempt + 1)
- return
+ return this.ingestChunk(chunk, attempt + 1)
}
for (const record of chunk) {
usageRecordsFailedCounter.inc({
@@ -176,6 +178,7 @@ export class UsageIngestionClient {
records: chunk.length,
error: String(error),
})
+ return false
}
}
}
diff --git a/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts b/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts
index ba66d4be88af..e66802b44db0 100644
--- a/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts
+++ b/nodejs/src/common/usage-ingestion/usage-record-batch.test.ts
@@ -80,7 +80,7 @@ describe('UsageRecordBatch', () => {
* Queues one record, starts the send, then queues a second while the first is in flight.
* Both acknowledgements are the test's to resolve, so neither case depends on timing.
*/
- async function sendWithALateAcknowledgement(send: (b: UsageRecordBatch) => Promise<void>): Promise<void> {
+ async function sendWithALateAcknowledgement(send: (b: UsageRecordBatch) => Promise<unknown>): Promise<void> {
const b = batch()
let acknowledgeFirst!: (info: object) => void
let acknowledgeSecond!: (info: object) => void
@@ -131,6 +131,6 @@ describe('UsageRecordBatch', () => {
])('flushes without waiting on acknowledgements for %s', async (_name, b) => {
b.addAfterAcknowledgements([new Promise(() => {})], 1, 'events', 'uuid-1')
- await expect(b.flush()).resolves.toBeUndefined()
+ await expect(b.flush()).resolves.toEqual([])
})
})
diff --git a/nodejs/src/common/usage-ingestion/usage-record-batch.ts b/nodejs/src/common/usage-ingestion/usage-record-batch.ts
index dc5d9a8b1889..c10f849d8b5e 100644
--- a/nodejs/src/common/usage-ingestion/usage-record-batch.ts
+++ b/nodejs/src/common/usage-ingestion/usage-record-batch.ts
@@ -89,12 +89,12 @@ export class UsageRecordBatch {
* pass, one send. A caller that is finished adding wants {@link drain} instead, because
* nothing would flush that record afterwards.
*/
- async flush(): Promise<void> {
+ async flush(): Promise<UsageRecordInput[]> {
const pending = this.pendingAcknowledgements
this.pendingAcknowledgements = []
await Promise.all(pending)
if (!this.client || this.records.size === 0) {
- return
+ return []
}
// Aggregate producers have no per-record capture time, so retain their flush-time clock.
const flushedAtMs = Date.now()
@@ -107,7 +107,7 @@ export class UsageRecordBatch {
timestampMs: record.timestampMs ?? flushedAtMs,
}))
this.records.clear()
- await this.client.ingest(records)
+ return this.client.ingest(records)
}
/**
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts b/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts
index bedb76a5cba8..bb441ebf839a 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/constants.ts
@@ -22,6 +22,9 @@ export const MAX_SESSION_LENGTH_SECONDS = 24 * 60 * 60
// sessions that span across the boundary, delayed messages, etc.)
export const SESSION_TRACKER_REDIS_TTL_SECONDS = 2 * MAX_SESSION_LENGTH_SECONDS
+// Mobile SDKs upload queued recordings days later, so a billed mobile session's mark must outlive that upload.
+export const SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS = 7 * 24 * 60 * 60
+
// Redis TTL for session blocklist keys
// Matches session tracker TTL since blocked sessions should persist for same duration
export const SESSION_FILTER_REDIS_TTL_SECONDS = 2 * MAX_SESSION_LENGTH_SECONDS
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
index 6cd4dfeb860f..80cef67ddd63 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.test.ts
@@ -1,12 +1,18 @@
+import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import { ingestionLagGauge, ingestionLagHistogram } from '~/common/metrics'
+import { UsageIngestionClient } from '~/common/usage-ingestion/client'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { PostgresRouter } from '~/common/utils/db/postgres'
import {
SessionReplayBatchProgress,
SessionReplayPipeline,
runSessionReplayPipeline,
} from '~/ingestion/pipelines/sessionreplay'
+import { ParsedMessageData } from '~/ingestion/pipelines/sessionreplay/kafka/types'
+import { SessionMetadataSink } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'
+import { createMockSessionKey } from '~/ingestion/pipelines/sessionreplay/shared/test-helpers'
import { KeyStore, RecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/types'
import { RedisPool } from '~/types'
@@ -79,6 +85,7 @@ async function lagHistogramCount(partition: string): Promise<number | undefined>
describe('SessionRecordingIngester', () => {
let ingester: SessionRecordingIngester
let events: string[]
+ let metadataStore: jest.Mocked<SessionMetadataSink>
const runPipelineMock = jest.mocked(runSessionReplayPipeline)
@@ -112,16 +119,31 @@ describe('SessionRecordingIngester', () => {
} as unknown as KeyStore
const fakeEncryptor = {
start: jest.fn().mockResolvedValue(undefined),
+ encryptBlockWithKey: jest.fn((_sessionId: string, _teamId: number, data: Buffer) => ({
+ data,
+ })),
} as unknown as RecordingEncryptor
+ metadataStore = {
+ storeSessionBlocks: jest.fn().mockResolvedValue(undefined),
+ }
+
+ const sessionTrackerRedis = {
+ mget: (keys: string[]) => Promise.resolve(keys.map(() => null)),
+ pipeline: () => ({ set: jest.fn().mockReturnThis(), exec: () => Promise.resolve([]) }),
+ }
ingester = new SessionRecordingIngester(
config,
{} as jest.Mocked<PostgresRouter>,
{} as unknown as ConstructorParameters<typeof SessionRecordingIngester>[2],
- {} as unknown as RedisPool,
+ {
+ acquire: () => Promise.resolve(sessionTrackerRedis),
+ release: () => Promise.resolve(),
+ } as unknown as RedisPool,
{} as unknown as RedisPool,
{
fileStorage: new BlackholeSessionBatchFileStorage(),
+ metadataStore,
keyStore: fakeKeyStore,
encryptor: fakeEncryptor,
createPipeline: () => ({}) as unknown as SessionReplayPipeline,
@@ -138,12 +160,40 @@ describe('SessionRecordingIngester', () => {
})
}
+ const recordOneSession = (): void => {
+ const parsedMessage: ParsedMessageData = {
+ metadata: { partition: 0, topic: consumeTopic, offset: 42, timestamp: 1000, rawSize: 100 },
+ distinct_id: 'distinct-1',
+ session_id: 'session-1',
+ token: 'token',
+ eventsByWindowId: { window1: [{ type: 2, timestamp: 1000, data: {} }] },
+ eventsRange: { start: DateTime.fromMillis(1000), end: DateTime.fromMillis(1000) },
+ snapshot_source: 'web',
+ snapshot_library: 'posthog-js',
+ }
+ runPipelineMock.mockImplementation(async (_pipeline, _messages, recorder) => {
+ await recorder.record(
+ {
+ team: { teamId: 1, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },
+ message: parsedMessage,
+ },
+ '30d',
+ createMockSessionKey()
+ )
+ return progress(new Map([[0, 42]]))
+ })
+ }
+
beforeEach(() => {
events = []
runPipelineMock.mockReset()
createIngester()
})
+ afterEach(() => {
+ jest.restoreAllMocks()
+ })
+
it('stop() stores the in-flight batch offsets only after its side effects settle, before disconnect', async () => {
const pipelineEntered = deferred()
const pipelineGate = deferred()
@@ -234,6 +284,51 @@ describe('SessionRecordingIngester', () => {
expect(await lagHistogramCount('0')).toBe(1)
})
+ it('persists replay metadata when usage ingestion fails', async () => {
+ createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
+ jest.spyOn(UsageRecordBatch.prototype, 'flush').mockRejectedValueOnce(new Error('usage unavailable'))
+ recordOneSession()
+
+ await expect(ingester.handleEachBatch([kafkaMessage(0, 42)])).resolves.toBeUndefined()
+
+ expect(metadataStore.storeSessionBlocks).toHaveBeenCalledWith([
+ expect.objectContaining({ teamId: 1, sessionId: 'session-1' }),
+ ])
+ expect(jest.mocked(ingester.kafkaConsumer).offsetsStore).toHaveBeenCalledWith([
+ { topic: consumeTopic, partition: 0, offset: 43 },
+ ])
+ })
+
+ it.each<[string, Partial<SessionRecordingIngesterCollaborators>, unknown[], string[]]>([
+ [
+ 'the main lane bills a persisted session before it stores the offsets',
+ {},
+ [[[expect.objectContaining({ teamId: 1, usageKey: 'session_replay_recordings', recordId: 'session-1' })]]],
+ ['usage_sent', 'offsets_stored'],
+ ],
+ ['the ML mirror lane bills nothing', { reportUsage: false }, [], ['offsets_stored']],
+ ])('%s', async (_name, collaborators, expectedIngestCalls, expectedEvents) => {
+ createIngester(
+ {
+ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0,
+ USAGE_INGESTION_ADDR: 'localhost:1',
+ USAGE_INGESTION_REPORT_TEAMS: '*',
+ },
+ collaborators
+ )
+ const ingest = jest.spyOn(UsageIngestionClient.prototype, 'ingest').mockImplementation((records) => {
+ events.push('usage_sent')
+ return Promise.resolve(records)
+ })
+ recordOneSession()
+
+ await ingester.handleEachBatch([kafkaMessage(0, 42)])
+
+ expect(metadataStore.storeSessionBlocks).toHaveBeenCalledTimes(1)
+ expect(ingest.mock.calls).toEqual(expectedIngestCalls)
+ expect(events).toEqual(expectedEvents)
+ })
+
it('samples the pipeline OK-result messages, not the raw consumed batch', async () => {
ingestionLagHistogram.reset()
createIngester({ SESSION_RECORDING_MAX_BATCH_AGE_MS: 0 })
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
index 92d42d22b65d..eda264630a08 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/consumer.ts
@@ -40,6 +40,7 @@ import { KafkaOffsetManager } from './kafka/offset-manager'
import { SessionRecordingIngesterMetrics } from './metrics'
import { SessionReplayLagReporter } from './session-replay-lag-reporter'
import { SessionReplayPipelineFactory, SessionReplayPipelineRunner } from './session-replay-pipeline-runner'
+import { billPersistedSessions } from './session-usage-step'
import { BlackholeSessionBatchFileStorage } from './sessions/blackhole-session-batch-writer'
import { BlockCompression } from './sessions/block-compression'
import { RetentionAwareStorage } from './sessions/retention-aware-batch-writer'
@@ -89,6 +90,8 @@ export interface SessionRecordingIngesterCollaborators {
redisKeyNamespace?: string
/** The ML mirror reports how far back its events are complete; the main lane leaves this unset. */
captureWatermark?: CaptureWatermark
+ /** False on the ML mirror, because the main lane bills the same recordings. */
+ reportUsage?: boolean
}
export class SessionRecordingIngester {
@@ -161,10 +164,13 @@ export class SessionRecordingIngester {
this.isDebugLoggingEnabled = buildIntegerMatcher(config.SESSION_RECORDING_DEBUG_PARTITION, true)
this.promiseScheduler = new PromiseScheduler()
- this.usageBatch = new UsageRecordBatch(createUsageIngestionClient(config, 'session_replay'), {
- unit: 'recordings',
- isTeamEnabled: usageReportTeamMatcher(config),
- })
+ this.usageBatch = new UsageRecordBatch(
+ collaborators.reportUsage === false ? null : createUsageIngestionClient(config, 'session_replay'),
+ {
+ unit: 'recordings',
+ isTeamEnabled: usageReportTeamMatcher(config),
+ }
+ )
this.runner =
collaborators.runner ??
@@ -365,14 +371,15 @@ export class SessionRecordingIngester {
})
await this.batchLock(async () => {
logger.info('🔁', 'blob_ingester_consumer_v2 - flushing batch', { batchSize: this.currentBatch.size })
- // Billing has nothing to wait on the batch for, and a usage outage must not stop lag
- // reporting or the batch reset — so it goes out alongside and swallows its own failure.
- await Promise.all([
- instrumentFn(`recordingingesterv2.handleEachBatch.flush`, async () => this.currentBatch.flush()),
- this.usageBatch.flush().catch((error) => {
- logger.warn('⚠️', 'blob_ingester_consumer_v2 - usage flush failed', { error })
- }),
- ])
+ // Usage goes out before the offsets commit, so a crash cannot lose the bill of a session with no later
+ // batch. A replayed batch resends the same record, and the usage table removes the duplicate.
+ await instrumentFn(`recordingingesterv2.handleEachBatch.flush`, async () =>
+ this.currentBatch.flush((sessions) =>
+ billPersistedSessions(this.usageBatch, this.sessionTracker, sessions).catch((error) => {
+ logger.warn('⚠️', 'blob_ingester_consumer_v2 - usage flush failed', { error })
+ })
+ )
+ )
// The flush committed the batch's offsets, so its data is now durably ingested — report lag.
// Skipped if the flush above threw, so a failed flush records no premature lag sample.
this.lagReporter.flush()
@@ -406,7 +413,6 @@ export class SessionRecordingIngester {
sessionKeyResolutionMaxConcurrency: this.config.SESSION_RECORDING_KEY_RESOLUTION_MAX_CONCURRENCY,
topHog: this.topHog,
isDebugLoggingEnabled: this.isDebugLoggingEnabled,
- usageBatch: this.usageBatch,
}
this.runner.start(pipelineConfig)
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
index 3222578c9931..c5e14339bda1 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/metrics.ts
@@ -90,20 +90,10 @@ export class SessionRecordingIngesterMetrics {
buckets: [300, 900, 3600, 21600, 43200, 604800, Infinity],
})
- private static readonly unbilledNewSession = new Counter({
- name: 'recording_blob_ingestion_v2_unbilled_new_session',
- help: 'New sessions whose first message failed before the usage step, so a later message for the same session bills nothing while the report still counts the recording',
- labelNames: ['reason'],
- })
-
public static incrementMessageReceived(partition: number): void {
this.messageReceived.labels(partition.toString()).inc()
}
- public static incrementUnbilledNewSession(reason: string): void {
- this.unbilledNewSession.labels(reason).inc()
- }
-
public static observeDroppedByRestrictions(count: number): void {
this.messagesDroppedByRestrictions.inc(count)
recordMessagesDroppedByRestrictions(count)
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts
index bf63adec7ae7..a4335ba52da5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.test.ts
@@ -48,6 +48,7 @@ describe('createRecordSessionEventStep', () => {
team: TeamForReplay = defaultTeam
): RecordSessionEventStepInput => ({
team,
+ headers: { now: new Date(1_700_000_000_000) },
parsedMessage: createParsedMessage(overrides),
retentionPeriod: '30d',
sessionKey: createMockSessionKey(),
@@ -78,7 +79,8 @@ describe('createRecordSessionEventStep', () => {
message: input.parsedMessage,
},
'30d',
- input.sessionKey
+ input.sessionKey,
+ 1_700_000_000_000
)
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts
index 0c3eeca90a75..9c01d9a98946 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/record-session-event-step.ts
@@ -11,6 +11,7 @@ import { ValueMatcher } from '~/types'
export interface RecordSessionEventStepInput extends SessionBatchContext {
team: TeamForReplay
+ headers: { now?: Date }
parsedMessage: ParsedMessageData
retentionPeriod: RetentionPeriod
sessionKey: SessionKey
@@ -59,7 +60,13 @@ export function createRecordSessionEventStep<T extends RecordSessionEventStepInp
// Record to the batch's recorder, carried on the element by the pipeline's beforeBatch.
const messageWithTeam: MessageWithTeam = { team, message: parsedMessage }
- await sessionBatchRecorder.record(messageWithTeam, retentionPeriod, sessionKey)
+ const captureTimestampMs = input.headers.now?.getTime()
+ await sessionBatchRecorder.record(
+ messageWithTeam,
+ retentionPeriod,
+ sessionKey,
+ Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined
+ )
return ok(input)
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts
index a9503632559f..6c3775de7ba5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.test.ts
@@ -1189,7 +1189,8 @@ describe('session-replay-pipeline', () => {
}),
}),
'30d',
- expect.objectContaining({ sessionState: 'cleartext' })
+ expect.objectContaining({ sessionState: 'cleartext' }),
+ undefined
)
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
index c59c0b2a2355..061496ae050b 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-replay-pipeline.ts
@@ -2,7 +2,6 @@ import { Message } from 'node-rdkafka'
import { DlqOutput, IngestionWarningsOutput, OverflowOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
-import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { EventIngestionRestrictionManager } from '~/common/utils/event-ingestion-restrictions'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { IngestionOverflowMode } from '~/ingestion/config'
@@ -32,7 +31,6 @@ import {
addSessionReplaySessionResolution,
withSessionReplayRecordingMetrics,
} from './session-replay-pipeline-stages'
-import { createRecordSessionUsageStep, trackUnbilledNewSessions } from './session-usage-step'
export interface SessionReplayPipelineInput extends SessionBatchContext {
message: Message
@@ -78,7 +76,6 @@ export interface SessionReplayPipelineConfig {
topHog: TopHogRegistry
/** Debug logging matcher for partition-based debugging. */
isDebugLoggingEnabled: ValueMatcher<number>
- usageBatch?: UsageRecordBatch
}
/**
@@ -93,7 +90,7 @@ export interface SessionReplayPipelineConfig {
* 5. Record - Record parsed messages to the batch's recorder
*/
export function createSessionReplayPipeline(config: SessionReplayPipelineConfig): SessionReplayPipeline {
- const { outputs, promiseScheduler, topHog, isDebugLoggingEnabled, usageBatch } = config
+ const { outputs, promiseScheduler, topHog, isDebugLoggingEnabled } = config
const pipelineConfig: PipelineConfig<OverflowOutput> = {
outputs,
@@ -143,14 +140,12 @@ export function createSessionReplayPipeline(config: SessionReplayPipelineConfig)
b
// Parse message content
.pipe(
- trackUnbilledNewSessions(
- topHogWrapper(createParseMessageStep(), [
- timer('parse_time_ms_by_session_id', (input) => ({
- token: input.headers.token ?? 'unknown',
- session_id: input.headers.session_id ?? 'unknown',
- })),
- ])
- )
+ topHogWrapper(createParseMessageStep(), [
+ timer('parse_time_ms_by_session_id', (input) => ({
+ token: input.headers.token ?? 'unknown',
+ session_id: input.headers.session_id ?? 'unknown',
+ })),
+ ])
)
// Monitor library version and emit warnings for old versions
.pipe(createLibVersionMonitorStep())
@@ -162,7 +157,6 @@ export function createSessionReplayPipeline(config: SessionReplayPipelineConfig)
})
)
)
- .pipe(createRecordSessionUsageStep(usageBatch))
)
.gather()
)
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts
index 6a118ad19550..c23b19ddd7ba 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.test.ts
@@ -1,128 +1,201 @@
-import { DateTime } from 'luxon'
-import { register } from 'prom-client'
-
import { UsageIngestionClient, UsageRecordInput } from '~/common/usage-ingestion/client'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
-import { drop, ok } from '~/ingestion/framework/results'
-
-import { ParsedMessageData } from './kafka/types'
-import { Recordable, SessionReplayHeaders } from './pipeline-types'
-import { createRecordSessionUsageStep, trackUnbilledNewSessions } from './session-usage-step'
-import { TeamForReplay } from './teams/types'
+import { SessionTracker } from '~/ingestion/pipelines/sessionreplay/sessions/session-tracker'
+import {
+ SessionBlockMetadata,
+ createNoopBlockMetadata,
+} from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'
+import { RedisPool } from '~/types'
-type SessionUsageValue = Recordable<{
- team: TeamForReplay
- headers: SessionReplayHeaders
- parsedMessage: ParsedMessageData
- isNewSession: boolean
-}>
+import { billPersistedSessions, recordPersistedSessionUsage } from './session-usage-step'
-describe('createRecordSessionUsageStep', () => {
+describe('session usage', () => {
let ingested: UsageRecordInput[]
let usageBatch: UsageRecordBatch
+ // 'stored' is a send that the service kept while the client saw it fail.
+ let sendFailure: 'none' | 'lost' | 'stored'
- beforeEach(() => {
- ingested = []
+ function newUsageBatch(): UsageRecordBatch {
const client = {
ingest: jest.fn((records: UsageRecordInput[]) => {
- ingested.push(...records)
- return Promise.resolve()
+ if (sendFailure !== 'lost') {
+ ingested.push(...records)
+ }
+ return Promise.resolve(sendFailure === 'none' ? records : [])
}),
} as unknown as UsageIngestionClient
- usageBatch = new UsageRecordBatch(client, { unit: 'recordings', isTeamEnabled: () => true })
+ return new UsageRecordBatch(client, { unit: 'recordings', isTeamEnabled: () => true })
+ }
+
+ beforeEach(() => {
+ ingested = []
+ sendFailure = 'none'
+ usageBatch = newUsageBatch()
})
- function buildValue(
- snapshotSource: string | null,
- snapshotLibrary: string | null,
- isNewSession: boolean
- ): SessionUsageValue {
+ function metadata(snapshotSource: string | null, captureTimestampMs?: number): SessionBlockMetadata {
return {
- team: { teamId: 42, consoleLogIngestionEnabled: false, aiTrainingOptedIn: false },
- headers: { token: 'token', session_id: 'session-1', distinct_id: 'distinct-1' },
- parsedMessage: {
- distinct_id: 'distinct-1',
- session_id: 'session-1',
- token: 'token',
- eventsByWindowId: { window1: [] },
- eventsRange: { start: DateTime.fromMillis(0), end: DateTime.fromMillis(0) },
- snapshot_source: snapshotSource,
- snapshot_library: snapshotLibrary,
- metadata: { partition: 0, topic: 'test-topic', rawSize: 0, offset: 0, timestamp: 0 },
- },
- isNewSession,
- status: 'allowed',
- sessionKey: {
- plaintextKey: Buffer.alloc(0),
- encryptedKey: Buffer.alloc(0),
- sessionState: 'ciphertext',
- },
+ ...createNoopBlockMetadata('session-1', 42),
+ snapshotSource,
+ captureTimestampMs,
}
}
- async function record(
- snapshotSource: string | null,
- snapshotLibrary: string | null,
- isNewSession = true
- ): Promise<string[]> {
- const step = createRecordSessionUsageStep(usageBatch)
- await step(buildValue(snapshotSource, snapshotLibrary, isNewSession))
-
+ async function record(sessions: SessionBlockMetadata[]): Promise<string[]> {
+ recordPersistedSessionUsage(usageBatch, sessions)
await usageBatch.flush()
return ingested.map((record) => record.usageKey)
}
it.each([
- ['web', 'posthog-js', ['session_replay_recordings']],
- ['mobile', 'posthog-ios', ['mobile_replay_recordings']],
- ['mobile', 'posthog-flutter', ['mobile_replay_recordings']],
- // The report bills mobile replay only from four named libraries. The meter reads the
- // source instead, so a mobile recording bills whatever library it names.
- ['mobile', 'posthog-python', ['mobile_replay_recordings']],
- ['mobile', null, ['mobile_replay_recordings']],
- // Nothing validates the source, so anything but 'mobile' bills as web rather than for free.
- ['desktop', 'posthog-js', ['session_replay_recordings']],
- [null, null, ['session_replay_recordings']],
- ])('bills a %s session from %s under %j', async (source, library, expectedUsageKeys) => {
- expect(await record(source, library)).toEqual(expectedUsageKeys)
+ ['web', ['session_replay_recordings']],
+ ['mobile', ['mobile_replay_recordings']],
+ ['desktop', ['session_replay_recordings']],
+ [null, ['session_replay_recordings']],
+ ])('bills persisted %s metadata under %j', async (source, expectedUsageKeys) => {
+ expect(await record([metadata(source, 1_700_000_000_000)])).toEqual(expectedUsageKeys)
})
- it('bills nothing for a session already seen in this batch', async () => {
- expect(await record('web', 'posthog-js', false)).toEqual([])
- })
+ it('uses the persisted session identity and trusted capture timestamp once', async () => {
+ const persisted = metadata('web', 1_700_000_000_000)
- it('uses trusted capture time from replay headers', async () => {
- const capturedAtMs = 1_700_000_000_000
- const value = buildValue('web', 'posthog-js', true)
- value.headers.now = new Date(capturedAtMs)
+ await record([persisted, persisted])
- await createRecordSessionUsageStep(usageBatch)(value)
- await usageBatch.flush()
+ expect(ingested).toEqual([
+ expect.objectContaining({
+ recordId: 'session-1',
+ quantity: 1,
+ timestampMs: 1_700_000_000_000,
+ }),
+ ])
+ })
- expect(ingested[0]).toEqual(expect.objectContaining({ timestampMs: capturedAtMs }))
+ it('bills nothing without persisted replay metadata', async () => {
+ expect(await record([])).toEqual([])
})
- describe('trackUnbilledNewSessions', () => {
- async function unbilledCount(): Promise<number> {
- const metric = register.getSingleMetric('recording_blob_ingestion_v2_unbilled_new_session')!
- const data = (await metric.get()) as { values: { value: number }[] }
- return data.values.reduce((total, entry) => total + entry.value, 0)
+ describe('billing each saved session once', () => {
+ const FIRST_FLUSH = Date.parse('2026-10-15T23:59:58Z')
+ const SECOND_FLUSH = Date.parse('2026-10-16T00:00:03Z')
+ const seenKey = '@posthog/replay/session-seen:42:session-1'
+ const billedKey = '@posthog/replay/session-billed:42:session-1'
+
+ let redisValues: Map<string, string>
+ let redisTtls: Map<string, number>
+ let redisAvailable: boolean
+ let redisPool: RedisPool
+
+ beforeEach(() => {
+ redisValues = new Map()
+ redisTtls = new Map()
+ redisAvailable = true
+ const client = {
+ pipeline: () => {
+ const commands: (() => string | null)[] = []
+ const pipeline = {
+ set: (key: string, value: string, _mode: string, ttlSeconds: number, condition?: string) => {
+ commands.push(() => {
+ if (condition === 'NX' && redisValues.has(key)) {
+ return null
+ }
+ redisValues.set(key, value)
+ redisTtls.set(key, ttlSeconds)
+ return 'OK'
+ })
+ return pipeline
+ },
+ get: (key: string) => {
+ commands.push(() => redisValues.get(key) ?? null)
+ return pipeline
+ },
+ exec: () =>
+ redisAvailable
+ ? Promise.resolve(commands.map((command) => [null, command()]))
+ : Promise.reject(new Error('Redis unavailable')),
+ }
+ return pipeline
+ },
+ }
+ redisPool = {
+ acquire: () => Promise.resolve(client),
+ release: () => Promise.resolve(),
+ } as unknown as RedisPool
+ })
+
+ // A new tracker for each flush, so the local cache cannot hide what Redis holds.
+ async function flush(sessions: SessionBlockMetadata[]): Promise<void> {
+ await billPersistedSessions(usageBatch, new SessionTracker(redisPool, 60_000), sessions)
}
+ it('bills a session saved in two flushes once, across midnight', async () => {
+ await flush([metadata('web', FIRST_FLUSH)])
+ await flush([metadata('web', SECOND_FLUSH)])
+
+ expect(ingested).toEqual([expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH })])
+ })
+
+ it.each([
+ ['loses', 'lost', 1],
+ ['stores', 'stored', 2],
+ ] as const)(
+ 'bills a failed send on the next saved flush with the first flush time when the service %s it',
+ async (_name, failure, records) => {
+ sendFailure = failure
+ await flush([metadata('web', FIRST_FLUSH)])
+ sendFailure = 'none'
+ await flush([metadata('web', SECOND_FLUSH)])
+
+ expect(ingested).toEqual(
+ Array(records).fill(expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH }))
+ )
+ }
+ )
+
+ it('sends one billing time when two consumers bill a session at once', async () => {
+ await Promise.all([
+ flush([metadata('web', FIRST_FLUSH)]),
+ billPersistedSessions(newUsageBatch(), new SessionTracker(redisPool, 60_000), [
+ metadata('web', SECOND_FLUSH),
+ ]),
+ ])
+
+ expect(new Set(ingested.map((record) => record.timestampMs))).toEqual(new Set([FIRST_FLUSH]))
+ })
+
+ it('bills a session saved in only one flush when Redis is unavailable', async () => {
+ redisAvailable = false
+ await flush([metadata('web', FIRST_FLUSH)])
+
+ expect(ingested).toEqual([expect.objectContaining({ recordId: 'session-1', timestampMs: FIRST_FLUSH })])
+ })
+
it.each([
- ['counts a new session whose message fails', true, false, 1],
- ['counts nothing for a session already seen', false, false, 0],
- ['counts nothing for a new session that parses', true, true, 0],
- ])('%s', async (_name, isNewSession, succeeds, expectedIncrease) => {
- const value = buildValue('web', 'posthog-js', isNewSession)
- const before = await unbilledCount()
- const step = trackUnbilledNewSessions(() =>
- Promise.resolve(succeeds ? ok(value) : drop<typeof value>('message_contained_no_valid_rrweb_events'))
- )
-
- await step(value)
-
- expect(await unbilledCount()).toEqual(before + expectedIncrease)
+ ['a session the tracker marked seen but did not bill', 'seen', ['session-1']],
+ ['nothing for a session that an earlier release billed when it first saw it', '1', []],
+ ])('bills %s, also after the seen key expires', async (_name, trackerValue, billedSessionIds) => {
+ redisValues.set(seenKey, trackerValue)
+
+ await flush([metadata('mobile', FIRST_FLUSH)])
+ redisValues.delete(seenKey)
+ await flush([metadata('mobile', SECOND_FLUSH)])
+
+ expect(ingested.map((record) => record.recordId)).toEqual(billedSessionIds)
})
+
+ it.each([
+ ['mobile', 7 * 24 * 60 * 60],
+ ['web', 48 * 60 * 60],
+ ])(
+ 'keeps a billed %s session marked for %i seconds without changing its seen key',
+ async (source, ttlSeconds) => {
+ redisValues.set(seenKey, 'seen')
+ redisTtls.set(seenKey, 48 * 60 * 60)
+
+ await flush([metadata(source, FIRST_FLUSH)])
+
+ expect(redisTtls.get(billedKey)).toBe(ttlSeconds)
+ expect([redisValues.get(seenKey), redisTtls.get(seenKey)]).toEqual(['seen', 48 * 60 * 60])
+ }
+ )
})
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts
index 3b6e763170da..1119cf283375 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/session-usage-step.ts
@@ -1,17 +1,13 @@
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
-import { PipelineResult, isOkResult, ok } from '~/ingestion/framework/results'
-import { ProcessingStep } from '~/ingestion/framework/steps'
+import {
+ SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS,
+ SESSION_TRACKER_REDIS_TTL_SECONDS,
+} from '~/ingestion/pipelines/sessionreplay/constants'
+import { BillingClaim, SessionTracker } from '~/ingestion/pipelines/sessionreplay/sessions/session-tracker'
+import { SessionBlockMetadata } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'
+import { SessionMap } from '~/ingestion/pipelines/sessionreplay/shared/session-map'
-import { ParsedMessageData } from './kafka/types'
-import { SessionRecordingIngesterMetrics } from './metrics'
-import { NewSessionFlag, Recordable, SessionReplayHeaders } from './pipeline-types'
-import { TeamForReplay } from './teams/types'
-
-type SessionUsageInput = {
- team: TeamForReplay
- headers: SessionReplayHeaders
- parsedMessage: ParsedMessageData
-} & NewSessionFlag
+const MOBILE_METER = 'mobile_replay_recordings'
/**
* The meter this session bills under. Every session bills exactly one, so no `snapshot_source`
@@ -23,49 +19,70 @@ type SessionUsageInput = {
* libraries, so it bills neither meter for a session outside those values. Those are holes on its
* side rather than a contract to copy.
*
- * The meter reads the first message processed for the session. The report instead reads the
- * earliest snapshot's metadata, so the two disagree if one session's messages disagree with each
- * other about their source, which no single client does.
+ * The meter reads the snapshot source stored with the replay metadata.
*/
function billableMeter(snapshotSource: string | null): string {
- return snapshotSource === 'mobile' ? 'mobile_replay_recordings' : 'session_replay_recordings'
+ return snapshotSource === 'mobile' ? MOBILE_METER : 'session_replay_recordings'
}
-export function createRecordSessionUsageStep<T extends SessionUsageInput>(
- usageBatch?: UsageRecordBatch
-): ProcessingStep<Recordable<T>, Recordable<T>> {
- return function recordSessionUsage(value): Promise<PipelineResult<Recordable<T>>> {
- if (value.isNewSession) {
- const meter = billableMeter(value.parsedMessage.snapshot_source)
- const captureTimestampMs = value.headers.now?.getTime()
- usageBatch?.add(
- value.team.teamId,
- meter,
- value.headers.session_id,
- 1,
- undefined,
- Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined
- )
- }
- return Promise.resolve(ok(value))
+function billedTtlSeconds(meter: string): number {
+ return meter === MOBILE_METER ? SESSION_TRACKER_MOBILE_BILLED_TTL_SECONDS : SESSION_TRACKER_REDIS_TTL_SECONDS
+}
+
+export function recordPersistedSessionUsage(usageBatch: UsageRecordBatch, sessions: SessionBlockMetadata[]): void {
+ for (const session of sessions) {
+ usageBatch.add(
+ session.teamId,
+ billableMeter(session.snapshotSource),
+ session.sessionId,
+ 1,
+ undefined,
+ session.captureTimestampMs
+ )
}
}
/**
- * Counts the new sessions that lose their flag before they can bill. The pipeline marks a session
- * seen before this step's message is parsed, so a first message that fails takes the flag with it:
- * a later valid message for the same session bills nothing, while the report still counts the
- * recording it produced. Marking later is not free — the mark sits after key resolution so a
- * recording is never written in cleartext — so measure how often this happens before paying for it.
+ * Bills each saved session once. A session counts as billed only after its record reaches the usage service,
+ * so a failed send bills on the session's next saved flush. Every send of a session, on any consumer, carries the
+ * billing time that the tracker stored first. The usage table then removes each resend.
*/
-export function trackUnbilledNewSessions<T extends NewSessionFlag, U>(
- step: ProcessingStep<T, U>
-): ProcessingStep<T, U> {
- return async function trackUnbilledNewSession(value): Promise<PipelineResult<U>> {
- const result = await step(value)
- if (value.isNewSession && !isOkResult(result)) {
- SessionRecordingIngesterMetrics.incrementUnbilledNewSession(result.reason)
+export async function billPersistedSessions(
+ usageBatch: UsageRecordBatch,
+ sessionTracker: SessionTracker,
+ sessions: SessionBlockMetadata[]
+): Promise<void> {
+ const claims = new SessionMap<BillingClaim>()
+ for (const session of sessions) {
+ if (usageBatch.accepts(session.teamId) && !claims.has(session.teamId, session.sessionId)) {
+ claims.set(session.teamId, session.sessionId, {
+ teamId: session.teamId,
+ sessionId: session.sessionId,
+ timestampMs: session.captureTimestampMs ?? Date.now(),
+ ttlSeconds: billedTtlSeconds(billableMeter(session.snapshotSource)),
+ })
}
- return result
}
+ if (claims.size === 0) {
+ return
+ }
+
+ const billingTimes = await sessionTracker.claimBillingTimes([...claims.values()])
+ recordPersistedSessionUsage(
+ usageBatch,
+ sessions.flatMap((session) => {
+ const timestampMs = billingTimes.get(session.teamId, session.sessionId)
+ return timestampMs === undefined ? [] : [{ ...session, captureTimestampMs: timestampMs }]
+ })
+ )
+ const received = await usageBatch.flush()
+ await sessionTracker.markBilled(
+ received.flatMap((record) => {
+ const timestampMs = billingTimes.get(record.teamId, record.recordId)
+ const claim = claims.get(record.teamId, record.recordId)
+ return timestampMs === undefined || !claim
+ ? []
+ : [{ teamId: record.teamId, sessionId: record.recordId, timestampMs, ttlSeconds: claim.ttlSeconds }]
+ })
+ )
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts
index bde0b3ce083c..5daa606f2c62 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.test.ts
@@ -218,8 +218,9 @@ describe('SessionBatchRecorder', () => {
const record = (
message: MessageWithTeam,
retentionPeriod: RetentionPeriod = '30d',
- sessionKey: SessionKey = createMockSessionKey()
- ): Promise<number> => recorder.record(message, retentionPeriod, sessionKey)
+ sessionKey: SessionKey = createMockSessionKey(),
+ captureTimestampMs?: number
+ ): Promise<number> => recorder.record(message, retentionPeriod, sessionKey, captureTimestampMs)
beforeEach(() => {
jest.clearAllMocks()
@@ -384,9 +385,10 @@ describe('SessionBatchRecorder', () => {
},
])
- await record(message)
+ await record(message, '30d', createMockSessionKey(), 1_700_000_000_000)
- await recorder.flush()
+ const persistedSessions = await recorder.flush()
+ expect(persistedSessions[0].captureTimestampMs).toBe(1_700_000_000_000)
const writtenData = captureWrittenData(mockWriter.writeSession as jest.Mock)
expect(mockWriter.finish).toHaveBeenCalledTimes(1)
@@ -1681,7 +1683,8 @@ describe('SessionBatchRecorder', () => {
expect(bytesWritten2).toBeGreaterThan(0)
expect(bytesWritten3).toBe(0)
- await recorder.flush()
+ const persistedSessions = await recorder.flush()
+ expect(persistedSessions).toEqual([])
const writtenData = captureWrittenData(mockWriter.writeSession as jest.Mock)
expect(writtenData).toHaveLength(0)
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts
index e2794cf2cce5..13902cd57033 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-batch-recorder.ts
@@ -32,6 +32,7 @@ interface SessionBatchEntry {
featureRecorder: SessionFeatureRecorder
sessionKey: SessionKey
retentionPeriod: RetentionPeriod
+ captureTimestampMs?: number
}
// A flush holds the batch lock, so nothing else here wants the pool while it runs.
@@ -120,7 +121,8 @@ export class SessionBatchRecorder {
public async record(
message: MessageWithTeam,
retentionPeriod: RetentionPeriod,
- sessionKey: SessionKey
+ sessionKey: SessionKey,
+ captureTimestampMs?: number
): Promise<number> {
const { partition } = message.message.metadata
const sessionId = message.message.session_id
@@ -181,6 +183,7 @@ export class SessionBatchRecorder {
}
return 0
}
+ existingBatchState.captureTimestampMs ??= captureTimestampMs
} else {
this.sessions.set(teamId, sessionId, {
sessionBlockRecorder: new SessionBlockRecorder(sessionId, teamId, this.batchId, this.compression),
@@ -198,6 +201,7 @@ export class SessionBatchRecorder {
),
sessionKey,
retentionPeriod,
+ captureTimestampMs,
})
}
@@ -246,9 +250,13 @@ export class SessionBatchRecorder {
/**
* Flushes the session recordings to storage and commits Kafka offsets
*
+ * @param beforeCommit - Runs after the flush stores the batch and before its offsets commit. It must be safe
+ * to repeat, because a crash between the two replays the batch, and it must catch its own errors.
* @throws If the flush operation fails
*/
- public async flush(): Promise<SessionBlockMetadata[]> {
+ public async flush(
+ beforeCommit?: (sessions: SessionBlockMetadata[]) => Promise<void>
+ ): Promise<SessionBlockMetadata[]> {
logger.info('🔁', 'session_batch_recorder_flushing', {
sessions: this.sessions.size,
totalSize: this._size,
@@ -280,7 +288,14 @@ export class SessionBatchRecorder {
for (const [
index,
- { sessionBlockRecorder, consoleLogRecorder, featureRecorder, sessionKey, retentionPeriod },
+ {
+ sessionBlockRecorder,
+ consoleLogRecorder,
+ featureRecorder,
+ sessionKey,
+ retentionPeriod,
+ captureTimestampMs,
+ },
] of entries.entries()) {
const {
buffer,
@@ -360,6 +375,7 @@ export class SessionBatchRecorder {
retentionPeriodDays,
isDeleted: false,
...(earliestCapturedAtMs === undefined ? {} : { earliestCapturedAtMs }),
+ captureTimestampMs,
})
totalEvents += eventCount
@@ -371,6 +387,7 @@ export class SessionBatchRecorder {
await this.consoleLogStore.flush()
await this.featureStore.storeSessionFeatures(featureBlocks)
await this.metadataStore.storeSessionBlocks(blockMetadata)
+ await beforeCommit?.(blockMetadata)
await this.offsetManager.commit()
// Update metrics
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts
index 124029d92f49..3b6c77b4ae68 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.test.ts
@@ -108,8 +108,20 @@ describe('SessionTracker', () => {
it('sets each session key with a 48h TTL in one pipeline', async () => {
await sessionTracker.markSeen(sessionSet([1, 'a'], [2, 'b']))
- expect(mockPipeline.set).toHaveBeenCalledWith('@posthog/replay/session-seen:1:a', '1', 'EX', TTL_SECONDS)
- expect(mockPipeline.set).toHaveBeenCalledWith('@posthog/replay/session-seen:2:b', '1', 'EX', TTL_SECONDS)
+ expect(mockPipeline.set).toHaveBeenCalledWith(
+ '@posthog/replay/session-seen:1:a',
+ 'seen',
+ 'EX',
+ TTL_SECONDS,
+ 'NX'
+ )
+ expect(mockPipeline.set).toHaveBeenCalledWith(
+ '@posthog/replay/session-seen:2:b',
+ 'seen',
+ 'EX',
+ TTL_SECONDS,
+ 'NX'
+ )
expect(mockPipeline.exec).toHaveBeenCalledTimes(1)
expect(SessionBatchMetrics.incrementNewSessionsDetected).toHaveBeenCalledTimes(2)
})
@@ -128,9 +140,10 @@ describe('SessionTracker', () => {
expect(mockPipeline.set).toHaveBeenCalledWith(
'@posthog/replay/ml-mirror/session-seen:1:a',
- '1',
+ 'seen',
'EX',
- TTL_SECONDS
+ TTL_SECONDS,
+ 'NX'
)
})
@@ -155,4 +168,16 @@ describe('SessionTracker', () => {
expect(mockRedisClient.mget).not.toHaveBeenCalled()
})
})
+
+ describe('markBilled', () => {
+ it('counts a Redis error that the pipeline returns as a command result', async () => {
+ mockPipeline.exec = jest.fn().mockResolvedValue([[new Error('OOM command not allowed'), null]])
+
+ await sessionTracker.markBilled([
+ { teamId: 1, sessionId: 'a', timestampMs: 1_700_000_000_000, ttlSeconds: 60 },
+ ])
+
+ expect(SessionBatchMetrics.incrementSessionTrackerRedisErrors).toHaveBeenCalledTimes(1)
+ })
+ })
})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts
index 2ea25d4b0a8e..cf9d9e90a9c5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/session-tracker.ts
@@ -9,6 +9,51 @@ import { SessionBatchMetrics } from './metrics'
const DEFAULT_LOCAL_CACHE_MAX_SIZE = 100_000
+// Any value in a seen key means seen. Code that billed a session when it first saw it wrote '1' to the seen key,
+// so a session with that value counts as billed.
+const SEEN = 'seen'
+const BILLED_AT_FIRST_SIGHT = '1'
+// The billed key holds the time, in milliseconds, that every usage record for the session carries. The usage table
+// removes a duplicate record only within one UTC day, so one stored time lets it remove every resend of the session.
+// The prefix marks a session whose record reached the usage service. The billed key is separate from the seen key
+// because billing must not change how long a session stays seen. The seen state decides whether a session gets a new
+// encryption key or reads its existing one.
+const BILLED_PREFIX = 'billed:'
+
+export interface BillingClaim {
+ teamId: number
+ sessionId: string
+ /** The time to store when the session has no stored time. */
+ timestampMs: number
+ ttlSeconds: number
+}
+
+export interface BilledSession {
+ teamId: number
+ sessionId: string
+ timestampMs: number
+ ttlSeconds: number
+}
+
+type PipelineResults = [Error | null, unknown][] | null
+
+function pipelineValues(results: PipelineResults, count: number): unknown[] {
+ if (!results || results.length !== count) {
+ throw new Error('Redis pipeline returned an incomplete result')
+ }
+ return results.map(([error, value]) => {
+ if (error) {
+ throw error
+ }
+ return value
+ })
+}
+
+function storedBillingTime(value: unknown): number | undefined {
+ const timestampMs = typeof value === 'string' ? Number(value) : NaN
+ return Number.isSafeInteger(timestampMs) && timestampMs > 0 ? timestampMs : undefined
+}
+
/**
* Tracks which sessions have been seen before, so a session's first sighting can be treated as new
* (rate-limited, key-generated) and later sightings as existing.
@@ -43,8 +88,12 @@ const DEFAULT_LOCAL_CACHE_MAX_SIZE = 100_000
export class SessionTracker {
private readonly keyPrefix: string
+ private readonly billedKeyPrefix: string
+
private readonly localCache: LRUCache<string, true>
+ private readonly billedCache: LRUCache<string, true>
+
/**
* @param keyNamespace - Optional Redis key namespace. Omit (the default) for the main lane — this
* keeps the key prefix exactly `@posthog/replay/session-seen` so existing sessions aren't seen as
@@ -58,11 +107,17 @@ export class SessionTracker {
localCacheMaxSize: number = DEFAULT_LOCAL_CACHE_MAX_SIZE,
keyNamespace?: string
) {
- this.keyPrefix = keyNamespace ? `@posthog/replay/${keyNamespace}/session-seen` : '@posthog/replay/session-seen'
+ const namespacePrefix = keyNamespace ? `@posthog/replay/${keyNamespace}` : '@posthog/replay'
+ this.keyPrefix = `${namespacePrefix}/session-seen`
+ this.billedKeyPrefix = `${namespacePrefix}/session-billed`
this.localCache = new LRUCache({
max: localCacheMaxSize,
ttl: localCacheTtlMs,
})
+ this.billedCache = new LRUCache({
+ max: localCacheMaxSize,
+ ttl: localCacheTtlMs,
+ })
}
/**
@@ -150,7 +205,9 @@ export class SessionTracker {
const key = this.generateKey(teamId, sessionId)
this.localCache.set(key, true)
SessionBatchMetrics.incrementNewSessionsDetected()
- pipeline.set(key, '1', 'EX', SESSION_TRACKER_REDIS_TTL_SECONDS)
+ // NX keeps a '1' that code billing at first sight wrote after this consumer's hasSeen read. The key
+ // normally does not exist here, because only sessions that read as new reach this write.
+ pipeline.set(key, SEEN, 'EX', SESSION_TRACKER_REDIS_TTL_SECONDS, 'NX')
}
await pipeline.exec()
} catch (error) {
@@ -164,7 +221,106 @@ export class SessionTracker {
}
}
+ /**
+ * Returns the billing time of each session that has not billed. The first call for a session stores its
+ * candidate time, and every later call, on any consumer, returns the stored time.
+ * A session without a billed mark counts as unbilled, because a mark that failed to persist must not stop the bill.
+ * On a Redis error every session outside the local billed cache counts as unbilled, with its candidate time. A
+ * session saved in only one flush has no later flush to bill it. The usage table removes a resend only when both
+ * records fall on the same UTC day.
+ */
+ public async claimBillingTimes(claims: BillingClaim[]): Promise<SessionMap<number>> {
+ const unbilled = new SessionMap<number>()
+ const misses = claims.filter(
+ ({ teamId, sessionId }) => !this.billedCache.has(this.generateBilledKey(teamId, sessionId))
+ )
+
+ if (misses.length === 0) {
+ return unbilled
+ }
+
+ // Sessions that code billing at first sight already billed. They get a billed mark, so a late upload does
+ // not bill them again after their seen key expires.
+ const billedAtFirstSight: BilledSession[] = []
+ const startTime = performance.now()
+ let client
+ try {
+ client = await this.redisPool.acquire()
+ const pipeline = client.pipeline()
+ for (const { teamId, sessionId, timestampMs, ttlSeconds } of misses) {
+ const billedKey = this.generateBilledKey(teamId, sessionId)
+ pipeline.set(billedKey, String(timestampMs), 'EX', ttlSeconds, 'NX')
+ pipeline.get(billedKey)
+ pipeline.get(this.generateKey(teamId, sessionId))
+ }
+ const values = pipelineValues(await pipeline.exec(), misses.length * 3)
+ for (let i = 0; i < misses.length; i++) {
+ const { teamId, sessionId, timestampMs, ttlSeconds } = misses[i]
+ const [billedValue, seenValue] = [values[3 * i + 1], values[3 * i + 2]]
+ if (typeof billedValue === 'string' && billedValue.startsWith(BILLED_PREFIX)) {
+ this.billedCache.set(this.generateBilledKey(teamId, sessionId), true)
+ } else if (seenValue === BILLED_AT_FIRST_SIGHT) {
+ const storedTimestampMs = storedBillingTime(billedValue) ?? timestampMs
+ billedAtFirstSight.push({ teamId, sessionId, timestampMs: storedTimestampMs, ttlSeconds })
+ } else {
+ unbilled.set(teamId, sessionId, storedBillingTime(billedValue) ?? timestampMs)
+ }
+ }
+ } catch (error) {
+ logger.error('session_tracker_claim_billing_times_redis_error', { error: String(error) })
+ SessionBatchMetrics.incrementSessionTrackerRedisErrors()
+ for (const { teamId, sessionId, timestampMs } of misses) {
+ unbilled.set(teamId, sessionId, timestampMs)
+ }
+ return unbilled
+ } finally {
+ if (client) {
+ await this.redisPool.release(client)
+ }
+ SessionBatchMetrics.observeSessionTrackerRedisLatency((performance.now() - startTime) / 1000)
+ }
+
+ await this.markBilled(billedAtFirstSight)
+ return unbilled
+ }
+
+ /**
+ * Call only after the sessions' usage records reached the usage service. Fails open, like {@link markSeen}. The
+ * local cache keeps the mark when the Redis write fails, so a Redis failure does not resend the session on every
+ * flush. A later resend carries the billing time that Redis stored, and the usage table removes it.
+ */
+ public async markBilled(sessions: BilledSession[]): Promise<void> {
+ if (sessions.length === 0) {
+ return
+ }
+
+ const startTime = performance.now()
+ let client
+ try {
+ client = await this.redisPool.acquire()
+ const pipeline = client.pipeline()
+ for (const { teamId, sessionId, timestampMs, ttlSeconds } of sessions) {
+ const key = this.generateBilledKey(teamId, sessionId)
+ this.billedCache.set(key, true)
+ pipeline.set(key, `${BILLED_PREFIX}${timestampMs}`, 'EX', ttlSeconds)
+ }
+ pipelineValues(await pipeline.exec(), sessions.length)
+ } catch (error) {
+ logger.error('session_tracker_mark_billed_redis_error', { error: String(error) })
+ SessionBatchMetrics.incrementSessionTrackerRedisErrors()
+ } finally {
+ if (client) {
+ await this.redisPool.release(client)
+ }
+ SessionBatchMetrics.observeSessionTrackerRedisLatency((performance.now() - startTime) / 1000)
+ }
+ }
+
private generateKey(teamId: number, sessionId: string): string {
return `${this.keyPrefix}:${teamId}:${sessionId}`
}
+
+ private generateBilledKey(teamId: number, sessionId: string): string {
+ return `${this.billedKeyPrefix}:${teamId}:${sessionId}`
+ }
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts
index 23adddff8e82..aebc55908be2 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata.ts
@@ -45,6 +45,8 @@ export function createDeletionBlockMetadata(sessionId: string, teamId: number):
}
export interface SessionBlockMetadata {
+ /** Trusted capture timestamp used to attribute billing without persisting it as replay metadata. */
+ captureTimestampMs?: number
replayIndexEntries?: ReplayIndexEntry[]
replayIndexTruncated?: boolean
/** Unique identifier for the session */
diff --git a/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts b/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts
index f428821d1148..bb442986219e 100644
--- a/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts
+++ b/nodejs/src/servers/ingestion-session-replay-ml-mirror-server.ts
@@ -183,6 +183,7 @@ export class IngestionSessionReplayMlMirrorServer extends MlMirrorConsumerServer
// would then fetch a missing key and record cleartext.
redisKeyNamespace: 'ml-mirror',
captureWatermark: new CaptureWatermark('ml_mirror'),
+ reportUsage: false,
}
const ingester = new SessionRecordingIngester(