:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import '../../../tests/helpers/mocks/consumer.mock'
import { createMockJobQueue } from '../../../tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '../../../tests/helpers/mocks/producer.mock'
import { HogFlow } from '~/cdp/schema/hogflow'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { createCdpConsumerDeps } from '../../../tests/helpers/cdp'
import {
createOrganization,
createTeam,
createTestTeamFixture,
getTeam,
updateOrganizationAvailableFeatures,
} from '../../../tests/helpers/sql'
import { Hub, Team } from '../../types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import { insertHogFunction as _insertHogFunction, createKafkaMessage } from '../_tests/fixtures'
import { insertHogFlow as _insertHogFlow } from '../_tests/fixtures-hogflows'
import { CdpDataWarehouseEvent } from '../schema'
import { HogWatcherState } from '../services/monitoring/hog-watcher.service'
import { HogFunctionInvocationGlobals, HogFunctionType } from '../types'
import { CdpDatawarehouseEventsConsumer } from './cdp-data-warehouse-events.consumer'
jest.setTimeout(1000)
describe('CdpDatawarehouseEventsConsumer', () => {
let processor: CdpDatawarehouseEventsConsumer
let hub: Hub
let team: Team
let team2: Team
let mockQueueInvocations: jest.MockedFunction<any>
const createDataWarehouseEvent = (
teamId: number,
properties: Record<string, any> = {},
tableName?: string,
tableType?: 'source' | 'view'
): CdpDataWarehouseEvent => {
return {
team_id: teamId,
table_name: tableName,
table_type: tableType,
event_id: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
properties: {
column1: 'value1',
column2: 123,
...properties,
},
}
}
const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
const teamId = hogFunction.team_id ?? team.id
const item = await _insertHogFunction(hub.postgres, teamId, {
...hogFunction,
type: 'destination',
})
// Trigger the reload that django would do
processor['hogFunctionManager']['onHogFunctionsReloaded'](teamId, [item.id])
return item
}
const insertHogFlow = async (hogFlow: HogFlow): Promise<HogFlow> => {
return await _insertHogFlow(hub.postgres, hogFlow)
}
beforeEach(async () => {
hub = await createHub()
const { organizationId, team: fixtureTeam } = await createTestTeamFixture(hub.postgres)
team = fixtureTeam
await updateOrganizationAvailableFeatures(hub.postgres, organizationId, [
{ key: 'data_pipelines', name: 'Data Pipelines' },
])
// Create second organization without data_pipelines for testing quota limiting
const otherOrganizationId = await createOrganization(hub.postgres)
const team2Id = await createTeam(hub.postgres, otherOrganizationId)
team2 = (await getTeam(hub.postgres, team2Id))! // This team does NOT have data_pipelines
// Set up default quota limiting mock - not limited by default
jest.spyOn(hub.quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(false)
const mockJobQueue = createMockJobQueue()
processor = new CdpDatawarehouseEventsConsumer(hub, createCdpConsumerDeps(hub), {
hogQueue: mockJobQueue,
hogflowQueue: mockJobQueue,
})
mockQueueInvocations = mockJobQueue.queueInvocations
await processor.start()
})
afterEach(async () => {
jest.setTimeout(10000)
await processor.stop()
await closeHub(hub)
})
afterAll(() => {
jest.useRealTimers()
})
describe('_parseKafkaBatch', () => {
it('should parse valid data warehouse events', async () => {
await insertHogFunction({
team_id: team.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const event = createDataWarehouseEvent(team.id, { test_prop: 'test_value' })
const messages = [createKafkaMessage(event)]
const invocations = await processor._parseKafkaBatch(messages)
expect(invocations).toHaveLength(1)
expect(invocations[0].project.id).toBe(team.id)
expect(invocations[0].event.properties).toMatchObject({
column1: 'value1',
column2: 123,
test_prop: 'test_value',
})
// Deterministic per-row id from the producer, surfaced as event.uuid for stable billing dedup.
expect(invocations[0].event.uuid).toBe('aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee')
expect(invocations[0].event.event).toBe('$warehouse_source_row')
})
it('should not parse events for teams without hog functions or flows', async () => {
const event = createDataWarehouseEvent(team.id)
const messages = [createKafkaMessage(event)]
const invocations = await processor._parseKafkaBatch(messages)
expect(invocations).toHaveLength(0)
})
it('should not parse events for teams that do not exist', async () => {
await insertHogFunction({
team_id: team.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const event = createDataWarehouseEvent(99999)
const messages = [createKafkaMessage(event)]
const invocations = await processor._parseKafkaBatch(messages)
expect(invocations).toHaveLength(0)
})
describe('with a hog function for the team', () => {
beforeEach(async () => {
await insertHogFunction({
team_id: team.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
})
const lookupError = new Error('canceling statement due to conflict with recovery')
it.each([
{
lookup: 'the team lookup',
spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),
},
{
lookup: 'the hog function lookup',
spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),
},
{
lookup: 'the hog flow lookup',
spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam'),
},
])(
'should fail the batch when $lookup throws, and queue the row when the batch is read again',
async ({ spy }) => {
spy().mockRejectedValueOnce(lookupError)
const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
const messages = [createKafkaMessage(createDataWarehouseEvent(team.id))]
await expect(handleBatch(messages)).rejects.toBe(lookupError)
expect(mockQueueInvocations).not.toHaveBeenCalled()
const { backgroundTask } = await handleBatch(messages)
await backgroundTask
expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
}
)
it('should skip a row with an out-of-range team_id and keep the rest', async () => {
const invocations = await processor._parseKafkaBatch([
createKafkaMessage({ ...createDataWarehouseEvent(team.id), team_id: 3_000_000_000 }),
createKafkaMessage(createDataWarehouseEvent(team.id)),
])
expect(invocations).toHaveLength(1)
})
})
it('should handle schema validation errors gracefully', async () => {
const invalidEvent = { team_id: 'not-a-number' }
const messages = [createKafkaMessage(invalidEvent)]
const invocations = await processor._parseKafkaBatch(messages)
expect(invocations).toHaveLength(0)
})
it('should filter by team correctly', async () => {
await insertHogFunction({
team_id: team.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const events = [
createKafkaMessage(createDataWarehouseEvent(team.id)),
createKafkaMessage(createDataWarehouseEvent(team2.id)),
]
const invocations = await processor._parseKafkaBatch(events)
expect(invocations).toHaveLength(1)
expect(invocations[0].project.id).toBe(team.id)
await insertHogFunction({
team_id: team2.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const invocations2 = await processor._parseKafkaBatch(events)
expect(invocations2).toHaveLength(2)
})
})
describe('filterHogFunction', () => {
it('should filter for data-warehouse-table source', async () => {
const fnWithDataWarehouseFilter = await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters,
filters: { ...HOG_FILTERS_EXAMPLES.no_filters.filters, source: 'events' },
})
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters,
})
const event = createDataWarehouseEvent(team.id)
const messages = [createKafkaMessage(event)]
const globals = await processor._parseKafkaBatch(messages)
const { invocations } = await processor.processBatch(globals)
// Should only include the function with data-warehouse-table filter
expect(invocations).toHaveLength(1)
expect(invocations[0].teamId).toBe(fnWithDataWarehouseFilter.team_id)
})
it("should only invoke destinations subscribed to the row's table", async () => {
const fnForOrders = await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
filters: {
source: 'data-warehouse-table',
bytecode: ['_h', 29],
data_warehouse: [{ table_name: 'postgres.orders' }],
},
})
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
filters: {
source: 'data-warehouse-table',
bytecode: ['_h', 29],
data_warehouse: [{ table_name: 'stripe.charge' }],
},
})
const messages = [createKafkaMessage(createDataWarehouseEvent(team.id, {}, 'postgres.orders'))]
const globals = await processor._parseKafkaBatch(messages)
const { invocations } = await processor.processBatch(globals)
expect(invocations).toHaveLength(1)
expect((invocations[0] as any).hogFunction.id).toBe(fnForOrders.id)
})
})
describe('processBatch', () => {
let fnFetchNoFilters: HogFunctionType
let globals: HogFunctionInvocationGlobals
beforeEach(async () => {
fnFetchNoFilters = await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const event = createDataWarehouseEvent(team.id, { test_prop: 'test_value' })
const messages = [createKafkaMessage(event)]
globals = (await processor._parseKafkaBatch(messages))[0]
})
it('should process data warehouse events and create invocations', async () => {
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(1)
expect(invocations[0].teamId).toBe(fnFetchNoFilters.team_id)
expect(invocations[0].state?.globals.event.properties).toMatchObject({
column1: 'value1',
column2: 123,
test_prop: 'test_value',
})
expect(mockQueueInvocations).toHaveBeenCalledWith(invocations)
})
it('should return empty invocations for empty batch', async () => {
const { invocations } = await processor.processBatch([])
expect(invocations).toHaveLength(0)
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
it('should filter out disabled hog functions', async () => {
await processor.hogWatcher.forceStateChange(fnFetchNoFilters, HogWatcherState.disabled)
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(0)
expect(mockQueueInvocations).toHaveBeenCalledWith([])
expect(mockProducerObserver.getProducedKafkaMessages()).toMatchObject([
{
topic: 'clickhouse_app_metrics2_test',
value: {
app_source: 'hog_function',
app_source_id: fnFetchNoFilters.id,
count: 1,
metric_kind: 'failure',
metric_name: 'disabled_permanently',
team_id: team.id,
},
},
])
})
it('should handle degraded state by setting queue priority', async () => {
await processor.hogWatcher.forceStateChange(fnFetchNoFilters, HogWatcherState.degraded)
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(1)
expect(invocations[0].queuePriority).toBe(2)
})
it('should log correct metrics for triggered invocations', async () => {
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(1)
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')).toMatchObject(
[
{
key: null,
topic: 'clickhouse_app_metrics2_test',
value: {
app_source: 'hog_function',
app_source_id: fnFetchNoFilters.id,
count: 1,
metric_kind: 'other',
metric_name: 'triggered',
team_id: team.id,
timestamp: expect.any(String),
},
},
// Billing is per-event, not per-destination
{
key: null,
topic: 'clickhouse_app_metrics2_test',
value: {
app_source: 'hog_function',
app_source_id: '_event_trigger',
instance_id: globals.event.uuid,
count: 1,
metric_kind: 'billing',
metric_name: 'billable_invocation',
team_id: team.id,
timestamp: expect.any(String),
},
},
]
)
})
it('should queue a running lifecycle row for each invocation so the runs UI shows in-flight work', async () => {
const queueLifecycleRowSpy = jest.spyOn(
processor['invocationResultsService']['invocationResultsRowsService'],
'queueLifecycleRow'
)
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(1)
expect(queueLifecycleRowSpy).toHaveBeenCalledTimes(1)
expect(queueLifecycleRowSpy).toHaveBeenCalledWith(invocations[0], 'running')
})
it('should bill once per event when multiple destinations match', async () => {
// Add a second function that also matches
const fnSecondDestination = await insertHogFunction({
...HOG_EXAMPLES.input_printer,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
const { invocations } = await processor.processBatch([globals])
// 1 event × 2 destinations = 2 invocations
expect(invocations).toHaveLength(2)
expect(invocations.map((i) => i.functionId).sort()).toEqual(
[fnFetchNoFilters.id, fnSecondDestination.id].sort()
)
const billingMetrics = mockProducerObserver
.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
.filter((m: any) => m.value.metric_name === 'billable_invocation')
// 1 event = 1 billable_invocation (not 2)
expect(billingMetrics).toHaveLength(1)
expect(billingMetrics[0].value).toMatchObject({
app_source_id: '_event_trigger',
instance_id: globals.event.uuid,
metric_name: 'billable_invocation',
})
})
})
describe('hog flow invocations', () => {
const buildDataWarehouseHogFlow = (
teamId: number,
tableName: string,
triggerType: 'data-warehouse-table' | 'data-warehouse-view' = 'data-warehouse-table'
): HogFlow =>
new FixtureHogFlowBuilder()
.withTeamId(teamId)
.withSimpleWorkflow({
trigger: {
type: triggerType,
table_name: tableName,
// Always-true bytecode (return true)
filters: { properties: [], bytecode: ['_h', 29] } as any,
},
})
.build()
it('should build a hog flow invocation when the row table matches the trigger', async () => {
const hogFlow = await insertHogFlow(buildDataWarehouseHogFlow(team.id, 'postgres.table_1'))
const event = createDataWarehouseEvent(team.id, { test_prop: 'test_value' }, 'postgres.table_1')
const globals = await processor._parseKafkaBatch([createKafkaMessage(event)])
expect(globals).toHaveLength(1)
const { invocations } = await processor.processBatch(globals)
const hogFlowInvocations = invocations.filter((i: any) => i.hogFlow)
expect(hogFlowInvocations).toHaveLength(1)
expect(hogFlowInvocations[0].functionId).toBe(hogFlow.id)
})
it('should not build a hog flow invocation when the row table does not match', async () => {
await insertHogFlow(buildDataWarehouseHogFlow(team.id, 'postgres.table_1'))
const event = createDataWarehouseEvent(team.id, {}, 'postgres.other_table')
const globals = await processor._parseKafkaBatch([createKafkaMessage(event)])
const { invocations } = await processor.processBatch(globals)
const hogFlowInvocations = invocations.filter((i: any) => i.hogFlow)
expect(hogFlowInvocations).toHaveLength(0)
})
it('should parse rows for workflow-only teams (no hog functions)', async () => {
await insertHogFlow(buildDataWarehouseHogFlow(team.id, 'postgres.table_1'))
const event = createDataWarehouseEvent(team.id, {}, 'postgres.table_1')
const globals = await processor._parseKafkaBatch([createKafkaMessage(event)])
// Team has a workflow but no hog functions - the row must not be dropped.
// The source table is exposed via event.properties.$source_table so the pipeline's
// eligibilityFn can match warehouse-table triggers without a top-level globals field.
expect(globals).toHaveLength(1)
expect(globals[0].event?.event).toBe('$warehouse_source_row')
expect(globals[0].event?.properties?.$source_table).toBe('postgres.table_1')
})
it('should build a hog flow invocation for a materialized view row', async () => {
const hogFlow = await insertHogFlow(
buildDataWarehouseHogFlow(team.id, 'daily_revenue', 'data-warehouse-view')
)
const event = createDataWarehouseEvent(team.id, {}, 'daily_revenue', 'view')
const globals = await processor._parseKafkaBatch([createKafkaMessage(event)])
expect(globals[0].event?.event).toBe('$warehouse_view_row')
const { invocations } = await processor.processBatch(globals)
const hogFlowInvocations = invocations.filter((i: any) => i.hogFlow)
expect(hogFlowInvocations).toHaveLength(1)
expect(hogFlowInvocations[0].functionId).toBe(hogFlow.id)
})
it.each([
['view', 'data-warehouse-table'],
['source', 'data-warehouse-view'],
] as const)(
'should not invoke a %s row against a %s trigger of the same name',
async (tableType, triggerType) => {
await insertHogFlow(buildDataWarehouseHogFlow(team.id, 'same_name', triggerType))
const event = createDataWarehouseEvent(team.id, {}, 'same_name', tableType)
const globals = await processor._parseKafkaBatch([createKafkaMessage(event)])
const { invocations } = await processor.processBatch(globals)
expect(invocations.filter((i: any) => i.hogFlow)).toHaveLength(0)
}
)
})
describe('quota limiting', () => {
let fnFetchNoFilters: HogFunctionType
let fnDataWarehouseFunction: HogFunctionType
let globals: HogFunctionInvocationGlobals
beforeEach(async () => {
// Create functions for team2 (no data_pipelines feature)
fnFetchNoFilters = await insertHogFunction({
team_id: team2.id,
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
fnDataWarehouseFunction = await insertHogFunction({
team_id: team2.id,
...HOG_EXAMPLES.input_printer,
...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
})
// Globals for team2 (without data_pipelines)
const event = createDataWarehouseEvent(team2.id, { test_prop: 'test_value' })
const messages = [createKafkaMessage(event)]
globals = (await processor._parseKafkaBatch(messages))[0]
})
it('should filter out functions when team is quota limited', async () => {
// Mock quota limiting to return true for team2 (which doesn't have data_pipelines)
jest.mocked(hub.quotaLimiting.isTeamQuotaLimited).mockClear()
jest.mocked(hub.quotaLimiting.isTeamQuotaLimited).mockResolvedValue(true)
const { invocations } = await processor.processBatch([globals])
expect(hub.quotaLimiting.isTeamQuotaLimited).toHaveBeenCalledWith(team2.id, 'cdp_trigger_events')
// Now check invocations length - should be 0 because team2 is quota limited and has no legacy addon
expect(invocations).toHaveLength(0)
// Check that quota_limited metrics were produced
const metrics = mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
expect(metrics).toEqual(
expect.arrayContaining([
expect.objectContaining({
topic: 'clickhouse_app_metrics2_test',
value: expect.objectContaining({
app_source: 'hog_function',
app_source_id: fnFetchNoFilters.id,
count: 1,
metric_kind: 'failure',
metric_name: 'quota_limited',
team_id: team2.id,
}),
}),
expect.objectContaining({
topic: 'clickhouse_app_metrics2_test',
value: expect.objectContaining({
app_source: 'hog_function',
app_source_id: fnDataWarehouseFunction.id,
count: 1,
metric_kind: 'failure',
metric_name: 'quota_limited',
team_id: team2.id,
}),
}),
])
)
})
it('should not filter out functions when team is not quota limited', async () => {
// Mock quota limiting to return false for team2
jest.mocked(hub.quotaLimiting.isTeamQuotaLimited).mockClear()
jest.mocked(hub.quotaLimiting.isTeamQuotaLimited).mockResolvedValue(false)
const { invocations } = await processor.processBatch([globals])
expect(invocations).toHaveLength(2)
expect(hub.quotaLimiting.isTeamQuotaLimited).toHaveBeenCalledWith(team2.id, 'cdp_trigger_events')
// Check that triggered metrics were produced instead
const metrics = mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
expect(metrics).toEqual(
expect.arrayContaining([
expect.objectContaining({
topic: 'clickhouse_app_metrics2_test',
value: expect.objectContaining({
app_source: 'hog_function',
app_source_id: fnFetchNoFilters.id,
count: 1,
metric_kind: 'other',
metric_name: 'triggered',
team_id: team2.id,
}),
}),
expect.objectContaining({
topic: 'clickhouse_app_metrics2_test',
value: expect.objectContaining({
app_source: 'hog_function',
app_source_id: fnDataWarehouseFunction.id,
count: 1,
metric_kind: 'other',
metric_name: 'triggered',
team_id: team2.id,
}),
}),
])
)
})
})
})