:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { KAFKA_CDP_INTERNAL_EVENTS } from '~/common/config/kafka-topics'
import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { HealthCheckResult, PluginsServerConfig } from '../../types'
import { CdpInternalEventSchema } from '../schema'
import {
GITHUB_EVENT_RECEIVED_EVENT,
SLACK_MESSAGE_RECEIVED_EVENT,
getInternalEventFilterEventIds,
hasMatchingInternalEventFilter,
} from '../schema/hogflow'
import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'
import { HogFunctionInvocationPipeline } from '../services/hog-function-invocation-pipeline.service'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { convertInternalEventToHogFunctionInvocationGlobals } from '../utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { parseBatch } from './message-parsing'
/**
* Whether a GitHub delivery is a write PostHog's own GitHub App made, resolved from the `own_app`
* property the emit stamps on the event.
*
* A workflow that comments back on an issue sees its own comment arrive on this topic, so without
* this it retriggers itself forever. Checked at eligibility rather than a trigger's stored filters,
* which a workflow created through the API or MCP would not carry. Unlike Slack's equivalent, this
* needs no integration lookup: one GitHub App per environment posts for every installation, so the
* emit can resolve "is this us" from its own instance setting before the event ever reaches Kafka.
*/
function isOwnGithubEvent(globals: HogFunctionInvocationGlobals): boolean {
return globals.event.event === GITHUB_EVENT_RECEIVED_EVENT && globals.event.properties.own_app === true
}
export class CdpInternalEventsConsumer extends CdpConsumerBase {
protected name = 'CdpInternalEventsConsumer'
protected hogTypes: HogFunctionTypeType[] = ['internal_destination']
protected hogQueue: JobQueue
protected hogflowQueue: JobQueue
protected kafkaConsumer: KafkaConsumerInterface
private hogFunctionPipeline: HogFunctionInvocationPipeline
private hogFlowPipeline: HogFlowInvocationPipeline
constructor(
config: PluginsServerConfig,
deps: CdpConsumerBaseDeps,
jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue }
) {
super(config, deps)
this.hogQueue = jobQueues.hogQueue
this.hogflowQueue = jobQueues.hogflowQueue
this.kafkaConsumer = createKafkaConsumer({
groupId: 'cdp-internal-events-consumer',
topic: KAFKA_CDP_INTERNAL_EVENTS,
})
this.hogFunctionPipeline = new HogFunctionInvocationPipeline(config, {
hogFunctionManager: this.hogFunctionManager,
hogInputsService: this.hogInputsService,
hogWatcher: this.hogWatcher,
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
cdpUsageReporter: this.cdpUsageReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
})
this.hogFlowPipeline = new HogFlowInvocationPipeline(config, {
hogFlowManager: this.hogFlowManager,
hogFlowExecutor: this.hogFlowExecutor,
hogWatcher: this.hogWatcher,
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
})
}
public async processBatch(
invocationGlobals: HogFunctionInvocationGlobals[]
): Promise<{ backgroundTask: Promise<any>; invocations: CyclotronJobInvocation[] }> {
if (!invocationGlobals.length) {
return { backgroundTask: Promise.resolve(), invocations: [] }
}
await this.groupsManager.addGroupsToGlobalsList(invocationGlobals)
const ownSlackMessages = await this.findOwnSlackMessages(invocationGlobals)
const [hogInvocations, hogflowInvocations] = await Promise.all([
this.hogFunctionPipeline.buildInvocations(invocationGlobals, {
hogTypes: this.hogTypes,
filterFn: (fn) => getInternalEventFilterEventIds(fn.filters) !== null,
invocationFilterFn: (fn, globals) => hasMatchingInternalEventFilter(fn.filters, globals.event.event),
}),
this.hogFlowPipeline.buildInvocations(invocationGlobals, {
eligibilityFn: (flow, globals) =>
flow.trigger.type === 'internal-event' &&
hasMatchingInternalEventFilter(flow.trigger.filters, globals.event.event) &&
!ownSlackMessages.has(globals) &&
!isOwnGithubEvent(globals),
}),
])
const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]
// Emit a `running` lifecycle row per freshly-created invocation so the runs UI shows these as
// in-flight, matching the event and warehouse consumers. The terminal row is queued later by
// the cyclotron worker; both collapse under the same `invocation_id` via ReplacingMergeTree.
for (const invocation of hogflowInvocations) {
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
}
return {
backgroundTask: Promise.all([
instrumentFn({ key: 'cdp.background_task.queue_invocations', sendException: false }, () =>
this.hogQueue.queueInvocations(hogInvocations)
),
instrumentFn({ key: 'cdp.background_task.queue_hogflow_invocations', sendException: false }, () =>
this.hogflowQueue.queueInvocations(hogflowInvocations)
),
instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {
try {
await this.hogFunctionMonitoringService.flush()
} catch (err) {
captureException(err)
logger.error('🔴', 'Error producing queued messages for monitoring', { err })
}
}),
instrumentFn({ key: 'cdp.background_task.lifecycle_running_flush', sendException: false }, () =>
this.invocationResultsService.invocationResultsRowsService.flush()
),
]),
invocations: invocationsToBeQueued,
}
}
/**
* Slack messages PostHog's own app posted, resolved through the integration the emit stamped
* on the event.
*
* A workflow that replies in Slack sees its own reply arrive back on this topic, so without
* this it retriggers itself forever. This is part of eligibility rather than a trigger's stored
* filters, which a workflow created through the API or MCP would not carry.
*/
private async findOwnSlackMessages(
invocationGlobals: HogFunctionInvocationGlobals[]
): Promise<Set<HogFunctionInvocationGlobals>> {
const candidates = invocationGlobals.filter(
(globals) =>
globals.event.event === SLACK_MESSAGE_RECEIVED_EVENT &&
typeof globals.event.properties.app_id === 'string' &&
typeof globals.event.properties.integration_id === 'number'
)
if (!candidates.length) {
return new Set()
}
const integrations = await this.deps.integrationManager.getMany([
...new Set(candidates.map((globals) => globals.event.properties.integration_id as number)),
])
return new Set(
candidates.filter(
(globals) =>
integrations[globals.event.properties.integration_id as number]?.config?.app_id ===
globals.event.properties.app_id
)
)
}
@instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')
public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
return await parseBatch(messages, {
errorLog: 'Error parsing message',
parse: (message) => {
const event = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
return {
teamId: event.team_id,
toGlobals: (team) =>
convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL),
}
},
loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),
})
}
public override async start(): Promise<void> {
await super.start()
await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()])
await this.kafkaConsumer.connect(async (messages) => {
logger.info('🔁', `${this.name} - handling batch`, { size: messages.length })
return await instrumentFn('cdpConsumer.handleEachBatch', async () => {
const invocationGlobals = await this._parseKafkaBatch(messages)
const { backgroundTask } = await this.processBatch(invocationGlobals)
return { backgroundTask }
})
})
}
public override async stop(): Promise<void> {
logger.info('💤', 'Stopping consumer...')
await this.kafkaConsumer.disconnect()
await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()])
await super.stop()
}
public isHealthy(): HealthCheckResult {
return this.kafkaConsumer.isHealthy()
}
}