:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createHash } from 'node:crypto'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { resolvePersonMode } from '~/ingestion/common/steps/event-processing/create-event'
import { IngestedEventInfo } from '~/ingestion/common/steps/event-processing/emit-event-step'
import {
EVENTS_USAGE_KEY,
SURVEY_RESPONSES_USAGE_KEY,
UsageKeyResolver,
} from '~/ingestion/common/usage-records/billable-events'
import { BeforeBatchStep } from '~/ingestion/framework/batching-pipeline'
import { PipelineResult, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { Properties } from '~/plugin-scaffold'
import { EventHeaders, Person } from '~/types'
export interface EventUsageBatchContext {
eventUsageBatch: UsageRecordBatch
}
export function createEventUsageBeforeBatchStep<TInput, CInput, CBatch>(
createBatch: () => UsageRecordBatch
): BeforeBatchStep<TInput, CInput, CBatch, CBatch & EventUsageBatchContext> {
return function eventUsageBeforeBatchStep(input) {
return Promise.resolve(
ok({
elements: input.elements,
batchContext: { ...input.batchContext, eventUsageBatch: createBatch() },
})
)
}
}
export interface RecordEventUsageInput {
preparedEvent: {
teamId: number
event: string
eventUuid: string
distinctId: string
timestamp: string
properties: Properties
}
headers: Pick<EventHeaders, 'now'>
eventUsageBatch: UsageRecordBatch
processPerson?: boolean
person?: Person
}
/**
* Hashed rather than joined: event names and distinct IDs are client-supplied and together
* exceed the 512-byte identifier the service accepts, and one oversized record makes the
* service reject the whole request. The timestamp is UTC-normalized upstream, so its first
* ten characters are the source day used in the events table's sorting key.
*/
function hashedRecordId(timestamp: string, identity: string[]): string {
const day = timestamp.slice(0, 10)
// JSON rather than a separator, so `a\nb` + `c` and `a` + `b\nc` hash differently.
return `${day}:${createHash('sha256').update(JSON.stringify(identity)).digest('hex').slice(0, 32)}`
}
/**
* Mirrors the events table's dedup identity, `(team_id, toDate(timestamp), event,
* distinct_id, uuid)`, minus the team the billing sorting key already carries. The UUID alone
* is not that identity: two events sharing one but differing in day, name or distinct_id are
* separate rows there, and the nightly report counts them separately, so billing must too.
*/
function analyticsRecordId(preparedEvent: RecordEventUsageInput['preparedEvent']): string {
return hashedRecordId(preparedEvent.timestamp, [
preparedEvent.event,
preparedEvent.distinctId,
preparedEvent.eventUuid,
])
}
/**
* Each answered step of a multi-question survey emits its own `survey sent` event. The nightly
* report bills one row per `($survey_id, $survey_submission_id)`, so sharing that identity lets
* the billing table's ReplacingMergeTree collapse the steps the same way.
*/
function surveyResponseRecordId(preparedEvent: RecordEventUsageInput['preparedEvent']): string {
const surveyId = preparedEvent.properties.$survey_id
const submissionId = preparedEvent.properties.$survey_submission_id
if (typeof submissionId !== 'string' || submissionId === '') {
return analyticsRecordId(preparedEvent)
}
return hashedRecordId(preparedEvent.timestamp, [typeof surveyId === 'string' ? surveyId : '', submissionId])
}
export interface EventUsageRecord {
teamId: number
usageKey: string
recordId: string
timestampMs?: number
}
export interface EventUsageRecordContext {
eventUsageRecords?: EventUsageRecord[]
}
/**
* One record per event. Its identity travels inside the event, so a replay reproduces it
* whatever the consumer's batching, which an offset-derived identity cannot.
*
* Person processing bills a second record under its own usage key, matching the nightly report,
* which counts the same event under both meters. It shares the identity, which the usage key
* separates, and it waits on the same acknowledgement: an event that never lands should not bill
* for the person work either.
*
* Capture time assigns usage to the period that receives it, including historical imports.
* Source event time cannot move newly received usage into a closed billing period.
* The record ID retains the source date for event identity and comparison with legacy reports.
*/
export function createRecordEventUsageStep<T extends RecordEventUsageInput>(
resolveUsageKey: UsageKeyResolver
): ProcessingStep<T, T & EventUsageRecordContext> {
return function recordEventUsageStep(input: T): Promise<PipelineResult<T & EventUsageRecordContext>> {
if (!input.eventUsageBatch.accepts(input.preparedEvent.teamId)) {
return Promise.resolve(ok({ ...input, eventUsageRecords: undefined }))
}
const usageKey = resolveUsageKey(input.preparedEvent.event)
if (!usageKey) {
return Promise.resolve(ok({ ...input, eventUsageRecords: undefined }))
}
const { teamId } = input.preparedEvent
const recordId =
usageKey === SURVEY_RESPONSES_USAGE_KEY
? surveyResponseRecordId(input.preparedEvent)
: analyticsRecordId(input.preparedEvent)
const captureTimestampMs = input.headers.now?.getTime()
const timestampMs = Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined
const eventUsageRecords: EventUsageRecord[] = [{ teamId, usageKey, recordId, timestampMs }]
// Mirrors the report's enhanced-persons query: the plain billable count plus a `person_mode`
// filter, so an event billed under its own key is outside both. Reading the mode stored on
// the event rather than `processPerson` keeps force upgrades counted.
if (
usageKey === EVENTS_USAGE_KEY &&
resolvePersonMode(input.person, input.processPerson ?? false) !== 'propertyless'
) {
eventUsageRecords.push({ teamId, usageKey: 'enhanced_person_events', recordId, timestampMs })
}
return Promise.resolve(ok({ ...input, eventUsageRecords }))
}
}
export interface RecordEventUsageAfterIngestInput extends EventUsageRecordContext {
eventUsageBatch?: UsageRecordBatch
ingested: Promise<IngestedEventInfo | null>[]
}
/** Queues billing only once every Kafka write for the logical event has been acknowledged. */
export function createRecordEventUsageAfterIngestStep<T extends RecordEventUsageAfterIngestInput>(): ProcessingStep<
T,
T
> {
return function recordEventUsageAfterIngestStep(input: T): Promise<PipelineResult<T>> {
if (input.eventUsageBatch) {
for (const record of input.eventUsageRecords ?? []) {
input.eventUsageBatch.addAfterAcknowledgements(
input.ingested,
record.teamId,
record.usageKey,
record.recordId,
record.timestampMs
)
}
}
return Promise.resolve(ok(input))
}
}
/** Ends the batch's usage reporting: the batch object goes away after this, so it drains. */
export function createFlushEventUsageStep<T extends { batchContext: EventUsageBatchContext }>(): ProcessingStep<T, T> {
return function flushEventUsageStep(input: T): Promise<PipelineResult<T>> {
return Promise.resolve(ok(input, [input.batchContext.eventUsageBatch.drain()]))
}
}