Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.6 KB
Raw Blame
import { createHash } from 'node:crypto'

import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { resolvePersonMode } from '~/ingestion/common/steps/event-processing/create-event'
import { IngestedEventInfo } from '~/ingestion/common/steps/event-processing/emit-event-step'
import {
    EVENTS_USAGE_KEY,
    SURVEY_RESPONSES_USAGE_KEY,
    UsageKeyResolver,
} from '~/ingestion/common/usage-records/billable-events'
import { BeforeBatchStep } from '~/ingestion/framework/batching-pipeline'
import { PipelineResult, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { Properties } from '~/plugin-scaffold'
import { EventHeaders, Person } from '~/types'

export interface EventUsageBatchContext {
    eventUsageBatch: UsageRecordBatch
}

export function createEventUsageBeforeBatchStep<TInput, CInput, CBatch>(
    createBatch: () => UsageRecordBatch
): BeforeBatchStep<TInput, CInput, CBatch, CBatch & EventUsageBatchContext> {
    return function eventUsageBeforeBatchStep(input) {
        return Promise.resolve(
            ok({
                elements: input.elements,
                batchContext: { ...input.batchContext, eventUsageBatch: createBatch() },
            })
        )
    }
}

export interface RecordEventUsageInput {
    preparedEvent: {
        teamId: number
        event: string
        eventUuid: string
        distinctId: string
        timestamp: string
        properties: Properties
    }
    headers: Pick<EventHeaders, 'now'>
    eventUsageBatch: UsageRecordBatch
    processPerson?: boolean
    person?: Person
}

/**
 * Hashed rather than joined: event names and distinct IDs are client-supplied and together
 * exceed the 512-byte identifier the service accepts, and one oversized record makes the
 * service reject the whole request. The timestamp is UTC-normalized upstream, so its first
 * ten characters are the source day used in the events table's sorting key.
 */
function hashedRecordId(timestamp: string, identity: string[]): string {
    const day = timestamp.slice(0, 10)
    // JSON rather than a separator, so `a\nb` + `c` and `a` + `b\nc` hash differently.
    return `${day}:${createHash('sha256').update(JSON.stringify(identity)).digest('hex').slice(0, 32)}`
}

/**
 * Mirrors the events table's dedup identity, `(team_id, toDate(timestamp), event,
 * distinct_id, uuid)`, minus the team the billing sorting key already carries. The UUID alone
 * is not that identity: two events sharing one but differing in day, name or distinct_id are
 * separate rows there, and the nightly report counts them separately, so billing must too.
 */
function analyticsRecordId(preparedEvent: RecordEventUsageInput['preparedEvent']): string {
    return hashedRecordId(preparedEvent.timestamp, [
        preparedEvent.event,
        preparedEvent.distinctId,
        preparedEvent.eventUuid,
    ])
}

/**
 * Each answered step of a multi-question survey emits its own `survey sent` event. The nightly
 * report bills one row per `($survey_id, $survey_submission_id)`, so sharing that identity lets
 * the billing table's ReplacingMergeTree collapse the steps the same way.
 */
function surveyResponseRecordId(preparedEvent: RecordEventUsageInput['preparedEvent']): string {
    const surveyId = preparedEvent.properties.$survey_id
    const submissionId = preparedEvent.properties.$survey_submission_id
    if (typeof submissionId !== 'string' || submissionId === '') {
        return analyticsRecordId(preparedEvent)
    }
    return hashedRecordId(preparedEvent.timestamp, [typeof surveyId === 'string' ? surveyId : '', submissionId])
}

export interface EventUsageRecord {
    teamId: number
    usageKey: string
    recordId: string
    timestampMs?: number
}

export interface EventUsageRecordContext {
    eventUsageRecords?: EventUsageRecord[]
}

/**
 * One record per event. Its identity travels inside the event, so a replay reproduces it
 * whatever the consumer's batching, which an offset-derived identity cannot.
 *
 * Person processing bills a second record under its own usage key, matching the nightly report,
 * which counts the same event under both meters. It shares the identity, which the usage key
 * separates, and it waits on the same acknowledgement: an event that never lands should not bill
 * for the person work either.
 *
 * Capture time assigns usage to the period that receives it, including historical imports.
 * Source event time cannot move newly received usage into a closed billing period.
 * The record ID retains the source date for event identity and comparison with legacy reports.
 */
export function createRecordEventUsageStep<T extends RecordEventUsageInput>(
    resolveUsageKey: UsageKeyResolver
): ProcessingStep<T, T & EventUsageRecordContext> {
    return function recordEventUsageStep(input: T): Promise<PipelineResult<T & EventUsageRecordContext>> {
        if (!input.eventUsageBatch.accepts(input.preparedEvent.teamId)) {
            return Promise.resolve(ok({ ...input, eventUsageRecords: undefined }))
        }
        const usageKey = resolveUsageKey(input.preparedEvent.event)
        if (!usageKey) {
            return Promise.resolve(ok({ ...input, eventUsageRecords: undefined }))
        }
        const { teamId } = input.preparedEvent
        const recordId =
            usageKey === SURVEY_RESPONSES_USAGE_KEY
                ? surveyResponseRecordId(input.preparedEvent)
                : analyticsRecordId(input.preparedEvent)
        const captureTimestampMs = input.headers.now?.getTime()
        const timestampMs = Number.isFinite(captureTimestampMs) ? captureTimestampMs : undefined
        const eventUsageRecords: EventUsageRecord[] = [{ teamId, usageKey, recordId, timestampMs }]
        // Mirrors the report's enhanced-persons query: the plain billable count plus a `person_mode`
        // filter, so an event billed under its own key is outside both. Reading the mode stored on
        // the event rather than `processPerson` keeps force upgrades counted.
        if (
            usageKey === EVENTS_USAGE_KEY &&
            resolvePersonMode(input.person, input.processPerson ?? false) !== 'propertyless'
        ) {
            eventUsageRecords.push({ teamId, usageKey: 'enhanced_person_events', recordId, timestampMs })
        }
        return Promise.resolve(ok({ ...input, eventUsageRecords }))
    }
}

export interface RecordEventUsageAfterIngestInput extends EventUsageRecordContext {
    eventUsageBatch?: UsageRecordBatch
    ingested: Promise<IngestedEventInfo | null>[]
}

/** Queues billing only once every Kafka write for the logical event has been acknowledged. */
export function createRecordEventUsageAfterIngestStep<T extends RecordEventUsageAfterIngestInput>(): ProcessingStep<
    T,
    T
> {
    return function recordEventUsageAfterIngestStep(input: T): Promise<PipelineResult<T>> {
        if (input.eventUsageBatch) {
            for (const record of input.eventUsageRecords ?? []) {
                input.eventUsageBatch.addAfterAcknowledgements(
                    input.ingested,
                    record.teamId,
                    record.usageKey,
                    record.recordId,
                    record.timestampMs
                )
            }
        }
        return Promise.resolve(ok(input))
    }
}

/** Ends the batch's usage reporting: the batch object goes away after this, so it drains. */
export function createFlushEventUsageStep<T extends { batchContext: EventUsageBatchContext }>(): ProcessingStep<T, T> {
    return function flushEventUsageStep(input: T): Promise<PipelineResult<T>> {
        return Promise.resolve(ok(input, [input.batchContext.eventUsageBatch.drain()]))
    }
}