Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.8 KB
Raw Blame
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'

import { EVENT_NAME_HEADER, TEAM_ID_HEADER } from '~/common/kafka/headers'
import { IngestionWarningsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { MessageSizeTooLarge } from '~/common/utils/db/error'
import { safeClickhouseString } from '~/common/utils/db/utils'
import { castTimestampOrNow, castTimestampToClickhouseFormat } from '~/common/utils/utils'
import { isAiEventName } from '~/ingestion/common/ai-event-types'
import { IngestionWarning, emitIngestionWarning } from '~/ingestion/common/ingestion-warnings'
import { eventProcessedAndIngestedCounter } from '~/ingestion/common/metrics'
import { EventUsageRecord } from '~/ingestion/common/steps/usage-records-steps'
import { ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { EventHeaders, ProcessedEvent, RawKafkaEvent, TimestampFormat } from '~/types'

export interface EventToEmit<O extends string> {
    event: ProcessedEvent
    output: O
}

export interface EmitEventStepConfig<O extends string> {
    outputs: IngestionOutputs<O | IngestionWarningsOutput>
}

export interface EmitEventStepInput<O extends string> {
    eventsToEmit: EventToEmit<O>[]
    teamId: number
    headers: EventHeaders
    message: Message
    eventUsageRecords?: EventUsageRecord[]
    eventUsageBatch?: UsageRecordBatch
}

/**
 * Info about an ingested event, resolved by `ingested` promises once the
 * event has been acked by Kafka.
 */
export interface IngestedEventInfo {
    /** Capture time from the `now` header; undefined when the header is missing. */
    capturedAt?: Date
    /** Topic and partition the event was consumed from. */
    topic: string
    partition: number
}

export interface EmitEventStepOutput {
    /**
     * One promise per emitted event, resolving with the event info once the
     * emission has been acked by Kafka, or with null when the event was not
     * ingested (e.g. rejected as too large). The same promises also flow
     * through side effects for scheduling; this field lets downstream steps
     * (e.g. ingestion lag recording) observe when the events have actually
     * been ingested. Empty when nothing was emitted.
     */
    ingested: Promise<IngestedEventInfo | null>[]
    eventUsageRecords?: EventUsageRecord[]
    eventUsageBatch?: UsageRecordBatch
}

export function createEmitEventStep<O extends string, T extends EmitEventStepInput<O>>(
    config: EmitEventStepConfig<O>
): ProcessingStep<T, EmitEventStepOutput> {
    return function emitEventStep(input) {
        const { eventsToEmit, headers, message } = input
        const { outputs } = config

        const ingestedInfo: IngestedEventInfo = {
            capturedAt: headers.now,
            topic: message.topic,
            partition: message.partition,
        }

        const ingested: Promise<IngestedEventInfo | null>[] = []

        for (const { event, output } of eventsToEmit) {
            const serialized = serializeEvent(event)

            // TODO: It's not great that we put the produce outcome in side effects, we should probably await it here
            //       but it might slow the pipeline down. Historically, it has always been like that.
            //       We should investigate this later.
            const emitPromise = outputs
                .produce(output, {
                    key: serialized.uuid,
                    value: Buffer.from(JSON.stringify(serialized)),
                    headers: {
                        productTrack: productTrackHeader(event),
                        [TEAM_ID_HEADER]: String(serialized.team_id),
                        [EVENT_NAME_HEADER]: serialized.event,
                    },
                    teamId: serialized.team_id,
                })
                .then(() => {
                    eventProcessedAndIngestedCounter.inc()
                    return ingestedInfo
                })
                .catch(async (error) => {
                    // TODO: For now we have to live with the ingestion warning happening here
                    //       Once the batch pipelines support warnings, we'll put it in the result
                    // Some messages end up significantly larger than the original
                    // after plugin processing, person & group enrichment, etc.
                    if (error instanceof MessageSizeTooLarge) {
                        await emitIngestionWarning(
                            outputs,
                            serialized.team_id,
                            eventTooLargeWarning(serialized, 'emit-event')
                        )
                        // The event was not ingested, so there is no info to resolve with
                        return null
                    } else {
                        throw error
                    }
                })

            ingested.push(emitPromise)
        }

        return Promise.resolve(
            ok(
                {
                    ingested,
                    eventUsageRecords: input.eventUsageRecords,
                    eventUsageBatch: input.eventUsageBatch,
                },
                ingested
            )
        )
    }
}

export function eventTooLargeWarning(
    event: { uuid: string; distinct_id: string; person_id?: string },
    pipelineStep: string
): IngestionWarning {
    return {
        type: 'message_size_too_large',
        details: { eventUuid: event.uuid, distinctId: event.distinct_id, personId: event.person_id },
        pipelineStep,
    }
}

export function serializeEvent(event: ProcessedEvent): RawKafkaEvent {
    return {
        uuid: event.uuid,
        event: safeClickhouseString(event.event),
        properties: JSON.stringify(event.properties ?? {}),
        timestamp: castTimestampOrNow(event.timestamp, TimestampFormat.ClickHouse),
        team_id: event.team_id,
        project_id: event.project_id,
        distinct_id: safeClickhouseString(event.distinct_id),
        elements_chain: safeClickhouseString(event.elements_chain),
        created_at: castTimestampOrNow(event.created_at, TimestampFormat.ClickHouse),
        captured_at:
            event.captured_at !== null
                ? castTimestampToClickhouseFormat(DateTime.fromJSDate(event.captured_at), TimestampFormat.ClickHouse)
                : null,
        person_id: event.person_id,
        person_properties: JSON.stringify(event.person_properties ?? {}),
        person_created_at: castTimestampOrNow(event.person_created_at, TimestampFormat.ClickHouseSecondPrecision),
        person_mode: event.person_mode,
        ...(event.historical_migration ? { historical_migration: true } : {}),
    }
}

export function productTrackHeader(event: ProcessedEvent): string {
    return isAiEventName(event.event) ? 'llma' : 'general'
}