Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.2 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { KAFKA_CDP_INTERNAL_EVENTS } from '~/common/config/kafka-topics'
import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

import { HealthCheckResult, PluginsServerConfig } from '../../types'
import { CdpInternalEventSchema } from '../schema'
import {
    GITHUB_EVENT_RECEIVED_EVENT,
    SLACK_MESSAGE_RECEIVED_EVENT,
    getInternalEventFilterEventIds,
    hasMatchingInternalEventFilter,
} from '../schema/hogflow'
import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'
import { HogFunctionInvocationPipeline } from '../services/hog-function-invocation-pipeline.service'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { convertInternalEventToHogFunctionInvocationGlobals } from '../utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { parseBatch } from './message-parsing'

/**
 * Whether a GitHub delivery is a write PostHog's own GitHub App made, resolved from the `own_app`
 * property the emit stamps on the event.
 *
 * A workflow that comments back on an issue sees its own comment arrive on this topic, so without
 * this it retriggers itself forever. Checked at eligibility rather than a trigger's stored filters,
 * which a workflow created through the API or MCP would not carry. Unlike Slack's equivalent, this
 * needs no integration lookup: one GitHub App per environment posts for every installation, so the
 * emit can resolve "is this us" from its own instance setting before the event ever reaches Kafka.
 */
function isOwnGithubEvent(globals: HogFunctionInvocationGlobals): boolean {
    return globals.event.event === GITHUB_EVENT_RECEIVED_EVENT && globals.event.properties.own_app === true
}

export class CdpInternalEventsConsumer extends CdpConsumerBase {
    protected name = 'CdpInternalEventsConsumer'
    protected hogTypes: HogFunctionTypeType[] = ['internal_destination']

    protected hogQueue: JobQueue
    protected hogflowQueue: JobQueue
    protected kafkaConsumer: KafkaConsumerInterface
    private hogFunctionPipeline: HogFunctionInvocationPipeline
    private hogFlowPipeline: HogFlowInvocationPipeline

    constructor(
        config: PluginsServerConfig,
        deps: CdpConsumerBaseDeps,
        jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue }
    ) {
        super(config, deps)
        this.hogQueue = jobQueues.hogQueue
        this.hogflowQueue = jobQueues.hogflowQueue
        this.kafkaConsumer = createKafkaConsumer({
            groupId: 'cdp-internal-events-consumer',
            topic: KAFKA_CDP_INTERNAL_EVENTS,
        })
        this.hogFunctionPipeline = new HogFunctionInvocationPipeline(config, {
            hogFunctionManager: this.hogFunctionManager,
            hogInputsService: this.hogInputsService,
            hogWatcher: this.hogWatcher,
            hogWatcherMirror: this.hogWatcherMirror,
            hogMasker: this.hogMasker,
            hogFunctionMonitoringService: this.hogFunctionMonitoringService,
            cdpUsageReporter: this.cdpUsageReporter,
            quotaLimiting: deps.quotaLimiting,
            redis: this.redis,
            valkeyShadow: this.valkeyShadow,
        })
        this.hogFlowPipeline = new HogFlowInvocationPipeline(config, {
            hogFlowManager: this.hogFlowManager,
            hogFlowExecutor: this.hogFlowExecutor,
            hogWatcher: this.hogWatcher,
            hogWatcherMirror: this.hogWatcherMirror,
            hogMasker: this.hogMasker,
            hogFunctionMonitoringService: this.hogFunctionMonitoringService,
            workflowsActivationReporter: this.workflowsActivationReporter,
            quotaLimiting: deps.quotaLimiting,
            redis: this.redis,
            valkeyShadow: this.valkeyShadow,
        })
    }

    public async processBatch(
        invocationGlobals: HogFunctionInvocationGlobals[]
    ): Promise<{ backgroundTask: Promise<any>; invocations: CyclotronJobInvocation[] }> {
        if (!invocationGlobals.length) {
            return { backgroundTask: Promise.resolve(), invocations: [] }
        }

        await this.groupsManager.addGroupsToGlobalsList(invocationGlobals)

        const ownSlackMessages = await this.findOwnSlackMessages(invocationGlobals)

        const [hogInvocations, hogflowInvocations] = await Promise.all([
            this.hogFunctionPipeline.buildInvocations(invocationGlobals, {
                hogTypes: this.hogTypes,
                filterFn: (fn) => getInternalEventFilterEventIds(fn.filters) !== null,
                invocationFilterFn: (fn, globals) => hasMatchingInternalEventFilter(fn.filters, globals.event.event),
            }),
            this.hogFlowPipeline.buildInvocations(invocationGlobals, {
                eligibilityFn: (flow, globals) =>
                    flow.trigger.type === 'internal-event' &&
                    hasMatchingInternalEventFilter(flow.trigger.filters, globals.event.event) &&
                    !ownSlackMessages.has(globals) &&
                    !isOwnGithubEvent(globals),
            }),
        ])

        const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]

        // Emit a `running` lifecycle row per freshly-created invocation so the runs UI shows these as
        // in-flight, matching the event and warehouse consumers. The terminal row is queued later by
        // the cyclotron worker; both collapse under the same `invocation_id` via ReplacingMergeTree.
        for (const invocation of hogflowInvocations) {
            this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
        }

        return {
            backgroundTask: Promise.all([
                instrumentFn({ key: 'cdp.background_task.queue_invocations', sendException: false }, () =>
                    this.hogQueue.queueInvocations(hogInvocations)
                ),
                instrumentFn({ key: 'cdp.background_task.queue_hogflow_invocations', sendException: false }, () =>
                    this.hogflowQueue.queueInvocations(hogflowInvocations)
                ),
                instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {
                    try {
                        await this.hogFunctionMonitoringService.flush()
                    } catch (err) {
                        captureException(err)
                        logger.error('🔴', 'Error producing queued messages for monitoring', { err })
                    }
                }),
                instrumentFn({ key: 'cdp.background_task.lifecycle_running_flush', sendException: false }, () =>
                    this.invocationResultsService.invocationResultsRowsService.flush()
                ),
            ]),
            invocations: invocationsToBeQueued,
        }
    }

    /**
     * Slack messages PostHog's own app posted, resolved through the integration the emit stamped
     * on the event.
     *
     * A workflow that replies in Slack sees its own reply arrive back on this topic, so without
     * this it retriggers itself forever. This is part of eligibility rather than a trigger's stored
     * filters, which a workflow created through the API or MCP would not carry.
     */
    private async findOwnSlackMessages(
        invocationGlobals: HogFunctionInvocationGlobals[]
    ): Promise<Set<HogFunctionInvocationGlobals>> {
        const candidates = invocationGlobals.filter(
            (globals) =>
                globals.event.event === SLACK_MESSAGE_RECEIVED_EVENT &&
                typeof globals.event.properties.app_id === 'string' &&
                typeof globals.event.properties.integration_id === 'number'
        )

        if (!candidates.length) {
            return new Set()
        }

        const integrations = await this.deps.integrationManager.getMany([
            ...new Set(candidates.map((globals) => globals.event.properties.integration_id as number)),
        ])

        return new Set(
            candidates.filter(
                (globals) =>
                    integrations[globals.event.properties.integration_id as number]?.config?.app_id ===
                    globals.event.properties.app_id
            )
        )
    }

    @instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')
    public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        return await parseBatch(messages, {
            errorLog: 'Error parsing message',
            parse: (message) => {
                const event = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
                return {
                    teamId: event.team_id,
                    toGlobals: (team) =>
                        convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL),
                }
            },
            loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),
        })
    }

    public override async start(): Promise<void> {
        await super.start()
        await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()])
        await this.kafkaConsumer.connect(async (messages) => {
            logger.info('🔁', `${this.name} - handling batch`, { size: messages.length })
            return await instrumentFn('cdpConsumer.handleEachBatch', async () => {
                const invocationGlobals = await this._parseKafkaBatch(messages)
                const { backgroundTask } = await this.processBatch(invocationGlobals)
                return { backgroundTask }
            })
        })
    }

    public override async stop(): Promise<void> {
        logger.info('💤', 'Stopping consumer...')
        await this.kafkaConsumer.disconnect()
        await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()])
        await super.stop()
    }

    public isHealthy(): HealthCheckResult {
        return this.kafkaConsumer.isHealthy()
    }
}