Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

19.3 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

import { convertToHogFunctionInvocationGlobals } from '../../cdp/utils'
import { HealthCheckResult, PluginsServerConfig, RawClickHouseEvent, Team } from '../../types'
import { CDP_EVENTS_DLQ_OUTPUT } from '../outputs/outputs'
import { CdpDeadLetterService } from '../services/dead-letter/cdp-dead-letter.service'
import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'
import { HogFunctionInvocationPipeline } from '../services/hog-function-invocation-pipeline.service'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { EventNameGate, buildEventNameGate, runsOnEvents, triggersOnEvents } from './event-name-gate'
import { parseBatch, readEventNameHeader, readTeamIdHeader } from './message-parsing'
import {
    counterEventNameSource,
    counterParseError,
    counterTeamIdHeaderMismatch,
    counterTeamIdSource,
    counterUnmatchedEventName,
    counterUnmatchedEventNameInvoked,
} from './metrics'
import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'

const isEventObject = (value: unknown): value is RawClickHouseEvent =>
    typeof value === 'object' && value !== null && !Array.isArray(value)

type TeamWithEventNameGate = { team: Team; gate: EventNameGate }

export class CdpEventsConsumer<
    TConfig extends PluginsServerConfig = PluginsServerConfig,
> extends CdpConsumerBase<TConfig> {
    protected name = 'CdpEventsConsumer'
    protected hogTypes: HogFunctionTypeType[] = ['destination']
    protected hogQueue: JobQueue
    protected hogflowQueue: JobQueue
    protected kafkaConsumer: KafkaConsumerInterface

    private hogFunctionPipeline: HogFunctionInvocationPipeline
    private hogFlowPipeline: HogFlowInvocationPipeline
    private deadLetterService: CdpDeadLetterService
    /** The message each set of globals was built from, so a failure can find its bytes. */
    private messageByGlobals = new WeakMap<HogFunctionInvocationGlobals, Message>()
    // Shadow scaffolding while the skip is off: a rejected event that still gets invoked is one the skip would drop.
    private rejectedByName = new WeakSet<HogFunctionInvocationGlobals>()

    constructor(
        config: TConfig,
        deps: CdpConsumerBaseDeps,
        jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue },
        topic: string = config.CDP_EVENTS_CONSUMER_TOPIC,
        groupId: string = config.CDP_EVENTS_CONSUMER_GROUP_ID
    ) {
        super(config, deps)
        this.hogQueue = jobQueues.hogQueue
        this.hogflowQueue = jobQueues.hogflowQueue
        this.kafkaConsumer = createKafkaConsumer({ groupId, topic })
        this.deadLetterService = new CdpDeadLetterService(config, {
            outputs: this.outputs,
            output: CDP_EVENTS_DLQ_OUTPUT,
            consumerGroup: groupId,
            resolveMessage: (globals) => this.messageByGlobals.get(globals),
        })
        this.hogFunctionPipeline = new HogFunctionInvocationPipeline(config, {
            hogFunctionManager: this.hogFunctionManager,
            hogInputsService: this.hogInputsService,
            hogWatcher: this.hogWatcher,
            hogWatcherMirror: this.hogWatcherMirror,
            hogMasker: this.hogMasker,
            hogFunctionMonitoringService: this.hogFunctionMonitoringService,
            cdpUsageReporter: this.cdpUsageReporter,
            quotaLimiting: deps.quotaLimiting,
            redis: this.redis,
            valkeyShadow: this.valkeyShadow,
            deadLetterService: this.deadLetterService,
        })
        this.hogFlowPipeline = new HogFlowInvocationPipeline(config, {
            hogFlowManager: this.hogFlowManager,
            hogFlowExecutor: this.hogFlowExecutor,
            hogWatcher: this.hogWatcher,
            hogWatcherMirror: this.hogWatcherMirror,
            hogMasker: this.hogMasker,
            hogFunctionMonitoringService: this.hogFunctionMonitoringService,
            workflowsActivationReporter: this.workflowsActivationReporter,
            quotaLimiting: deps.quotaLimiting,
            redis: this.redis,
            valkeyShadow: this.valkeyShadow,
            deadLetterService: this.deadLetterService,
        })
    }

    public async processBatch(
        invocationGlobals: HogFunctionInvocationGlobals[],
        /** Runs after the build and before anything is queued, so a throw here queues nothing. */
        beforeQueue?: () => Promise<void>
    ): Promise<{ backgroundTask: Promise<any>; invocations: CyclotronJobInvocation[] }> {
        if (!invocationGlobals.length) {
            // A batch whose every message failed to parse or was skipped still has records and metrics to write.
            await beforeQueue?.()
            return { backgroundTask: this.flushMonitoring(), invocations: [] }
        }

        // TODO: Add a helper to hog functions to determine if they require groups or not and then only load those
        await this.groupsManager.addGroupsToGlobalsList(invocationGlobals)

        const [hogBuild, hogflowBuild] = await Promise.all([
            this.hogFunctionPipeline.buildInvocationsWithRelease(invocationGlobals, {
                hogTypes: this.hogTypes,
                filterFn: runsOnEvents,
            }),
            // Source-compatibility lives in the consumer. The events consumer matches event-triggered
            // flows only; other trigger types (data-warehouse-table, batch, schedule, webhook, manual)
            // are dispatched from their respective consumers and never reach the executor from here.
            this.hogFlowPipeline.buildInvocationsWithRelease(invocationGlobals, {
                eligibilityFn: triggersOnEvents,
            }),
        ])
        const hogInvocations = hogBuild.invocations
        const hogflowInvocations = hogflowBuild.invocations
        if (!this.config.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES) {
            this.countInvokedAfterNameRejection(invocationGlobals, [
                ...hogInvocations.map((invocation) => invocation.state.globals.event.uuid),
                ...hogflowInvocations.flatMap((invocation) => invocation.state?.event.uuid ?? []),
            ])
        }

        const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]

        try {
            await beforeQueue?.()
        } catch (error) {
            // The build already wrote the masking claims. The batch is read again after this throw. A
            // claim left in place would mask the retried event as a duplicate.
            await Promise.all([hogBuild.release(), hogflowBuild.release()])
            throw error
        }

        // Emit a `running` lifecycle row for each freshly-created invocation.
        // This fires ONCE per invocation_id at creation — not on every dequeue
        // — so the runs UI can show in-flight work without us writing duplicate
        // running rows across fetch retries. The terminal row is queued later
        // by the cyclotron worker; both collapse under the same `invocation_id`
        // via ReplacingMergeTree, with the terminal row's later `version`
        // superseding the running row on FINAL queries.
        for (const invocation of invocationsToBeQueued) {
            this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
        }

        // Turn any $push_notification_opened events in this batch into `push_opened` app-metrics. Queued
        // here (synchronously) so the monitoring flush in the background task below picks them up.
        await this.trackPushNotificationOpens(invocationGlobals)

        return {
            // This is all IO so we can set them off in the background and start processing the next batch
            backgroundTask: Promise.all([
                instrumentFn({ key: 'cdp.background_task.queue_hog_invocations', sendException: false }, () =>
                    this.hogQueue.queueInvocations(hogInvocations)
                ),
                instrumentFn({ key: 'cdp.background_task.queue_hogflow_invocations', sendException: false }, () =>
                    this.hogflowQueue.queueInvocations(hogflowInvocations)
                ),
                this.flushMonitoring(),
                instrumentFn({ key: 'cdp.background_task.lifecycle_running_flush', sendException: false }, () =>
                    this.invocationResultsService.invocationResultsRowsService.flush()
                ),
            ]),
            invocations: [...hogInvocations, ...hogflowInvocations],
        }
    }

    private flushMonitoring(): Promise<void> {
        return instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {
            try {
                await this.hogFunctionMonitoringService.flush()
            } catch (err) {
                captureException(err)
                logger.error('🔴', 'Error producing queued messages for monitoring', { err })
            }
        })
    }

    // Resolve each $push_notification_opened event's workflow and queue its push_opened app-metric.
    // The attribution + spoof guard live in buildPushOpenedMetric (pure + unit-tested).
    private async trackPushNotificationOpens(globals: HogFunctionInvocationGlobals[]): Promise<void> {
        const opens = globals.filter((g) => g.event.event === PUSH_NOTIFICATION_OPENED_EVENT)
        if (!opens.length) {
            return
        }
        // Match each open's workflow id against the flows its own team owns, which the hog flow pipeline
        // already loaded for this batch (warm by-team cache). Resolving against that bounded, team-owned
        // set rather than looking up the client-supplied id directly keeps a spoofed id from hitting
        // Postgres or polluting the shared by-id cache with negative entries. Best-effort: a lookup
        // failure here must not abort the batch (that would stall all event processing and force a replay).
        try {
            const teamIds = Array.from(new Set(opens.map((g) => g.project.id)))
            const flowsByTeam = await this.hogFlowManager.getHogFlowsForTeams(teamIds)
            for (const g of opens) {
                const workflowId = g.event.properties['$notification_workflow_id']
                if (typeof workflowId !== 'string') {
                    continue
                }
                const flow = (flowsByTeam[g.project.id] ?? []).find((f) => f.id === workflowId) ?? null
                const metric = buildPushOpenedMetric(g.event.properties, g.project.id, flow)
                if (metric) {
                    this.hogFunctionMonitoringService.queueAppMetric(metric, 'hog_flow')
                }
            }
        } catch (error) {
            logger.error('[CdpEventsConsumer] Failed to track push notification opens', { error })
        }
    }

    private countInvokedAfterNameRejection(globalsList: HogFunctionInvocationGlobals[], invokedUuids: string[]): void {
        const rejected = new Set(globalsList.filter((g) => this.rejectedByName.has(g)).map((g) => g.event.uuid))
        if (!rejected.size) {
            return
        }
        const invoked = new Set(invokedUuids.filter((uuid) => rejected.has(uuid)))
        counterUnmatchedEventNameInvoked.inc(invoked.size)
    }

    @instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')
    public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        const park =
            (step: 'parse' | 'lookup') =>
            (message: Message, error: string): boolean =>
                this.deadLetterService.recordMessageFailure(message, { step, error })
        const skip = this.config.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES
        const gates = new Map<number, EventNameGate>()
        // Queued once the whole batch parsed, so a batch read again after a failed lookup records them once.
        const skippedMetrics: EventNameGate[] = []
        const invocationGlobals = await parseBatch<TeamWithEventNameGate>(messages, {
            errorLog: 'Error parsing message',
            parse: (message) => {
                const parseEvent = (): RawClickHouseEvent => {
                    const clickHouseEvent = parseJSON(message.value!.toString())
                    if (!isEventObject(clickHouseEvent)) {
                        throw new Error('not_an_event')
                    }
                    return clickHouseEvent
                }
                const buildGlobals = (team: Team, clickHouseEvent: RawClickHouseEvent) => {
                    const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)
                    // The one place holding both. A failure found later in the pipeline carries
                    // these globals, and this is how it gets back to the bytes to park.
                    this.messageByGlobals.set(globals, message)
                    return globals
                }

                const headerTeamId = readTeamIdHeader(message)
                if (headerTeamId !== null) {
                    counterTeamIdSource.labels({ source: 'header' }).inc()
                    const headerEventName = readEventNameHeader(message)
                    counterEventNameSource.labels({ source: headerEventName === null ? 'none' : 'header' }).inc()
                    // The body stays unparsed until `toGlobals` runs, and it only runs for a team that
                    // has a destination or a flow. Most events belong to a team with neither.
                    return {
                        teamId: headerTeamId,
                        toGlobals: ({ team, gate }) => {
                            const rejected = headerEventName !== null && !gate.admits(headerEventName)
                            if (rejected) {
                                counterUnmatchedEventName.labels({ skipped: String(skip) }).inc()
                                if (skip) {
                                    skippedMetrics.push(gate)
                                    return null
                                }
                            }
                            const clickHouseEvent = parseEvent()
                            if (clickHouseEvent.team_id !== headerTeamId) {
                                // The loaded team comes from the header and the event data comes from
                                // the body. Building globals from two teams would evaluate one team's
                                // event against another team's destinations and send it there.
                                counterTeamIdHeaderMismatch.inc()
                                throw new Error('team_id_header_mismatch')
                            }
                            const globals = buildGlobals(team, clickHouseEvent)
                            if (rejected) {
                                this.rejectedByName.add(globals)
                            }
                            return globals
                        },
                    }
                }

                counterTeamIdSource.labels({ source: 'body' }).inc()
                const clickHouseEvent = parseEvent()
                return {
                    teamId: clickHouseEvent.team_id,
                    toGlobals: ({ team }) => buildGlobals(team, clickHouseEvent),
                }
            },
            loadTeam: (teamId) => this.loadTeamWithEventNameGate(teamId, gates),
            onInvalidTeamId: () => counterParseError.labels({ error: 'invalid_team_id' }).inc(),
            // Parked rather than dropped: a schema we cannot read today is often one a later
            // version can.
            onError: park('parse'),
            // A lookup error that repeats stops the partition when it fails the batch, so this parks the
            // message for a replay. The loaders send the team ids of the whole batch in one query, so one
            // failed query parks every message in it. With the dead-letter topic off, nothing keeps the
            // message, so the error fails the batch. A failed query that covers enough of the batch trips
            // the dead-letter circuit breaker, which also fails the batch.
            parkLookupError: park('lookup'),
        })
        for (const gate of skippedMetrics) {
            const { hogFunctions, hogFlows } = gate.filteredMetrics()
            this.hogFunctionMonitoringService.queueAppMetrics(hogFunctions, 'hog_function')
            this.hogFunctionMonitoringService.queueAppMetrics(hogFlows, 'hog_flow')
        }
        return invocationGlobals
    }

    // Looks up per message, so a failed lookup parks only its own message.
    private async loadTeamWithEventNameGate(
        teamId: number,
        gates: Map<number, EventNameGate>
    ): Promise<TeamWithEventNameGate | null> {
        const work = await this.loadTeamWork(teamId, this.hogTypes)
        if (!work) {
            return null
        }
        const gate =
            gates.get(teamId) ??
            buildEventNameGate({
                hogFunctions: work.hogFunctions,
                hogFlows: work.hogFlows,
                // Push open tracking reads these for any team with a flow, whatever the flows' filters.
                alsoAdmitted: work.hogFlows.length ? [PUSH_NOTIFICATION_OPENED_EVENT] : [],
            })
        gates.set(teamId, gate)
        return { team: work.team, gate }
    }

    protected async startQueueProducers(): Promise<void> {
        await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()])
    }

    protected async stopQueueProducers(): Promise<void> {
        await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()])
    }

    public override async start(): Promise<void> {
        await super.start()
        await this.startQueueProducers()
        await this.deadLetterService.checkTopic()
        // Start consuming messages
        await this.kafkaConsumer.connect(async (messages) => {
            logger.info('🔁', `${this.name} - handling batch`, {
                size: messages.length,
            })

            return await instrumentFn('cdpConsumer.handleEachBatch', async () => {
                const invocationGlobals = await this._parseKafkaBatch(messages)
                // Awaited, not backgrounded: the offsets for these messages are stored once this
                // handler resolves, so a record has to exist by then or the event is gone.
                const { backgroundTask } = await this.processBatch(invocationGlobals, () =>
                    this.deadLetterService.produceForBatch(messages)
                )

                return { backgroundTask }
            })
        })
    }

    public override async stop(): Promise<void> {
        logger.info('💤', 'Stopping consumer...')
        await this.kafkaConsumer.disconnect()
        logger.info('💤', 'Stopping job queues...')
        await this.stopQueueProducers()
        // IMPORTANT: super always comes last
        await super.stop()
        logger.info('💤', 'Consumer stopped!')
    }

    public isHealthy(): HealthCheckResult {
        return this.kafkaConsumer.isHealthy()
    }
}