:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { HealthCheckResult, PluginsServerConfig, Team } from '../../types'
import { CdpDataWarehouseEvent, CdpDataWarehouseEventSchema } from '../schema'
import { DWH_SOURCE_TABLE_PROPERTY, WAREHOUSE_SOURCE_ROW_EVENT, WAREHOUSE_VIEW_ROW_EVENT } from '../schema/hogflow'
import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'
import { HogFunctionInvocationPipeline } from '../services/hog-function-invocation-pipeline.service'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import {
CyclotronJobInvocation,
HogFunctionFilterDataWarehouse,
HogFunctionInvocationGlobals,
HogFunctionTypeType,
} from '../types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { parseBatch } from './message-parsing'
// Filter source / trigger type each kind of warehouse row matches against.
const TRIGGER_TYPE_BY_TABLE_TYPE = {
source: 'data-warehouse-table',
view: 'data-warehouse-view',
} as const
type WarehouseTableType = keyof typeof TRIGGER_TYPE_BY_TABLE_TYPE
const WAREHOUSE_TRIGGER_TYPES = new Set<string>(Object.values(TRIGGER_TYPE_BY_TABLE_TYPE))
export class CdpDatawarehouseEventsConsumer extends CdpConsumerBase {
protected name = 'CdpDatawarehouseEventsConsumer'
protected hogTypes: HogFunctionTypeType[] = ['destination']
protected hogQueue: JobQueue
protected hogflowQueue: JobQueue
protected kafkaConsumer: KafkaConsumerInterface
private hogFunctionPipeline: HogFunctionInvocationPipeline
private hogFlowPipeline: HogFlowInvocationPipeline
constructor(
config: PluginsServerConfig,
deps: CdpConsumerBaseDeps,
jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue }
) {
super(config, deps)
this.hogQueue = jobQueues.hogQueue
this.hogflowQueue = jobQueues.hogflowQueue
this.kafkaConsumer = createKafkaConsumer({
groupId: 'cdp-data-warehouse-events-consumer',
topic: 'cdp_data_warehouse_source_table',
})
this.hogFunctionPipeline = new HogFunctionInvocationPipeline(config, {
hogFunctionManager: this.hogFunctionManager,
hogInputsService: this.hogInputsService,
hogWatcher: this.hogWatcher,
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
cdpUsageReporter: this.cdpUsageReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
})
this.hogFlowPipeline = new HogFlowInvocationPipeline(config, {
hogFlowManager: this.hogFlowManager,
hogFlowExecutor: this.hogFlowExecutor,
hogWatcher: this.hogWatcher,
hogWatcherMirror: this.hogWatcherMirror,
hogMasker: this.hogMasker,
hogFunctionMonitoringService: this.hogFunctionMonitoringService,
workflowsActivationReporter: this.workflowsActivationReporter,
quotaLimiting: deps.quotaLimiting,
redis: this.redis,
valkeyShadow: this.valkeyShadow,
})
}
public async processBatch(
invocationGlobals: HogFunctionInvocationGlobals[]
): Promise<{ backgroundTask: Promise<any>; invocations: CyclotronJobInvocation[] }> {
if (!invocationGlobals.length) {
return { backgroundTask: Promise.resolve(), invocations: [] }
}
// Warehouse rows carry no `$groups` property — group enrichment is a no-op here, so we skip the
// call entirely to avoid the per-batch group-types lookup.
const [hogInvocations, hogflowInvocations] = await Promise.all([
this.hogFunctionPipeline.buildInvocations(invocationGlobals, {
hogTypes: this.hogTypes,
filterFn: (fn) => WAREHOUSE_TRIGGER_TYPES.has(fn.filters?.source ?? 'events'),
// Destinations pick their tables in `filters.data_warehouse`, which never reaches the
// filter bytecode, so without this every warehouse-triggered destination in the team
// ran for every table's rows.
invocationFilterFn: (fn, globals) =>
fn.filters?.source === triggerTypeOf(globals) &&
matchesSubscribedTable(fn.filters?.data_warehouse, sourceTableOf(globals)),
}),
// Source-compatibility matching lives in the consumer rather than the executor — the
// consumer knows it's serving warehouse rows, so it filters flows to only those whose
// trigger matches the row's kind and $source_table property. The executor then just
// evaluates filter bytecode on the matched flows.
this.hogFlowPipeline.buildInvocations(invocationGlobals, {
eligibilityFn: (flow, globals) =>
(flow.trigger.type === 'data-warehouse-table' || flow.trigger.type === 'data-warehouse-view') &&
flow.trigger.type === triggerTypeOf(globals) &&
flow.trigger.table_name === sourceTableOf(globals),
}),
])
const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]
// Emit a `running` lifecycle row for each freshly-created invocation so the runs UI shows
// warehouse-triggered flows as in-flight (matching the event consumer). The terminal row is
// queued later by the cyclotron worker; both collapse under the same `invocation_id` via
// ReplacingMergeTree, with the terminal row's later `version` superseding the running row.
for (const invocation of invocationsToBeQueued) {
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
}
return {
backgroundTask: Promise.all([
instrumentFn({ key: 'cdp.background_task.queue_hog_invocations', sendException: false }, () =>
this.hogQueue.queueInvocations(hogInvocations)
),
instrumentFn({ key: 'cdp.background_task.queue_hogflow_invocations', sendException: false }, () =>
this.hogflowQueue.queueInvocations(hogflowInvocations)
),
instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {
try {
await this.hogFunctionMonitoringService.flush()
} catch (err) {
captureException(err)
logger.error('🔴', 'Error producing queued messages for monitoring', { err })
}
}),
instrumentFn({ key: 'cdp.background_task.lifecycle_running_flush', sendException: false }, () =>
this.invocationResultsService.invocationResultsRowsService.flush()
),
]),
invocations: invocationsToBeQueued,
}
}
@instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')
public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
return await parseBatch(messages, {
errorLog: 'Error parsing message',
parse: (message) => {
const event = CdpDataWarehouseEventSchema.parse(parseJSON(message.value!.toString()))
return {
teamId: event.team_id,
toGlobals: (team) =>
convertDataWarehouseEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL),
}
},
loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),
})
}
public override async start(): Promise<void> {
await super.start()
await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()])
await this.kafkaConsumer.connect(async (messages) => {
logger.info('🔁', `${this.name} - handling batch`, { size: messages.length })
return await instrumentFn('cdpConsumer.handleEachBatch', async () => {
const invocationGlobals = await this._parseKafkaBatch(messages)
const { backgroundTask } = await this.processBatch(invocationGlobals)
return { backgroundTask }
})
})
}
public override async stop(): Promise<void> {
logger.info('💤', 'Stopping consumer...')
await this.kafkaConsumer.disconnect()
await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()])
await super.stop()
}
public isHealthy(): HealthCheckResult {
return this.kafkaConsumer.isHealthy()
}
}
/**
* Which trigger a row's globals can match. The synthetic event name is the discriminator, so the
* row's own properties stay exactly what the warehouse produced.
*/
function triggerTypeOf(globals: HogFunctionInvocationGlobals): string {
return globals.event?.event === WAREHOUSE_VIEW_ROW_EVENT
? TRIGGER_TYPE_BY_TABLE_TYPE.view
: TRIGGER_TYPE_BY_TABLE_TYPE.source
}
function sourceTableOf(globals: HogFunctionInvocationGlobals): string {
const table = globals.event?.properties?.[DWH_SOURCE_TABLE_PROPERTY]
return typeof table === 'string' ? table : ''
}
/**
* An empty or absent subscription list keeps the pre-existing "match every table" behavior.
* Destinations saved before per-table matching existed have no entries, and tightening them here
* would silently stop them firing.
*/
function matchesSubscribedTable(
subscriptions: HogFunctionFilterDataWarehouse[] | undefined,
sourceTable: string
): boolean {
if (!subscriptions?.length) {
return true
}
return subscriptions.some((subscription) => subscription.table_name === sourceTable)
}
function convertDataWarehouseEventToHogFunctionInvocationGlobals(
event: CdpDataWarehouseEvent,
team: Team,
siteUrl: string
): HogFunctionInvocationGlobals {
const data = event.properties
const projectUrl = `${siteUrl}/project/${team.id}`
// The synthetic event carries:
// - the producer's deterministic per-row id (CDPProducer._build_event_id) as the uuid, so
// billing dedup (keyed on event.uuid) counts each row distinctly and stably across re-runs
// - `$warehouse_source_row` or `$warehouse_view_row` as the event name, which both identifies
// warehouse-row globals and says which trigger kind they can match
// - the source table name on `properties.$source_table` so consumers can match warehouse
// triggers without a new top-level field on globals
const tableType: WarehouseTableType = event.table_type ?? 'source'
return {
project: {
id: team.id,
name: team.name,
url: projectUrl,
},
event: {
uuid: event.event_id,
event: tableType === 'view' ? WAREHOUSE_VIEW_ROW_EVENT : WAREHOUSE_SOURCE_ROW_EVENT,
elements_chain: '', // Not applicable but left here for compatibility
distinct_id: 'data-warehouse-table-distinct-id-do-not-use',
properties: {
...data,
[DWH_SOURCE_TABLE_PROPERTY]: event.table_name ?? '',
},
timestamp: DateTime.now().toISO(),
url: '',
},
}
}