:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { Server } from 'http'
import supertest from 'supertest'
import express from 'ultimate-express'
import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '~/cdp/_tests/examples'
import { insertHogFunction as _insertHogFunction, insertBatchExport } from '~/cdp/_tests/fixtures'
import { CdpApi } from '~/cdp/cdp-api'
import { HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
import { GroupReadRepository } from '~/common/groups/repositories/group-repository.interface'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { createTestTeamFixture, updateOrganizationAvailableFeatures } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { GroupTypeIndex, TeamId } from '../../types'
import { GroupsManagerService } from './managers/groups-manager.service'
describe('BatchExportHogFunctionService', () => {
let hub: Hub
let team: Team
let api: CdpApi
let app: express.Application
let server: Server
let batchExportId: string
let hogFunction: HogFunctionType
let clickhouseEvent: Record<string, any>
const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
const item = await _insertHogFunction(hub.postgres, team.id, hogFunction)
api['hogFunctionManager']['onHogFunctionsReloaded'](team.id, [item.id])
return item
}
const invocationUrl = () => `/api/projects/${team.id}/hog_functions/${hogFunction.id}/batch_export_invocations`
const postInvocation = (body: any) => supertest(app).post(invocationUrl()).send(body)
beforeAll(async () => {
hub = await createHub({ SITE_URL: 'http://localhost:8000' })
api = new CdpApi(hub, createCdpConsumerDeps(hub), {
hogQueue: createMockJobQueue(),
hogflowQueue: createMockJobQueue(),
})
app = setupExpressApp()
app.use('/', api.router())
server = app.listen(0, () => {})
})
beforeEach(async () => {
team = (await createTestTeamFixture(hub.postgres)).team
mockFetch.mockClear()
clickhouseEvent = {
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
event: '$pageview',
team_id: team.id,
distinct_id: '123',
timestamp: '2021-09-28T14:00:00Z',
created_at: '2021-09-28T14:00:00Z',
properties: JSON.stringify({ $lib_version: '1.0.0' }),
elements_chain: '',
}
batchExportId = new UUIDT().toString()
await insertBatchExport(hub.postgres, team.id, batchExportId)
hogFunction = await insertHogFunction({
name: 'test batch export hog function',
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.no_filters,
batch_export_id: batchExportId,
})
})
afterAll(async () => {
await api.stop()
server.close()
await closeHub(hub)
})
describe('request body validation', () => {
it.each([
['empty body', {}, 'clickhouse_event'],
[
'missing event uuid',
{ clickhouse_event: { event: '$pageview', team_id: 1, distinct_id: 'x', timestamp: 't' } },
'uuid',
],
[
'missing event name',
{
clickhouse_event: {
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
team_id: 1,
distinct_id: 'x',
timestamp: 't',
},
},
'event',
],
])('rejects %s', async (_label, body, expectedField) => {
const res = await postInvocation(body)
expect(res.status).toEqual(400)
expect(res.body.errors[0]).toContain('Invalid request body')
expect(res.body.errors[0]).toContain(expectedField)
})
it('rejects non-string invocation_id', async () => {
const res = await postInvocation({ clickhouse_event: clickhouseEvent, invocation_id: 12345 })
expect(res.status).toEqual(400)
expect(res.body.errors[0]).toContain('Invalid request body')
expect(res.body.errors[0]).toContain('invocation_id')
})
it('rejects invalid uuid invocation_id', async () => {
const res = await postInvocation({ clickhouse_event: clickhouseEvent, invocation_id: 'not-a-uuid' })
expect(res.status).toEqual(400)
expect(res.body.errors[0]).toContain('Invalid request body')
expect(res.body.errors[0]).toContain('invocation_id')
})
})
describe('resource lookup errors', () => {
it('returns 404 for non-existent team', async () => {
const res = await supertest(app)
.post(`/api/projects/99999/hog_functions/${hogFunction.id}/batch_export_invocations`)
.send({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(404)
expect(res.body.errors[0]).toContain('99999')
})
it('returns 404 for non-existent hog function', async () => {
const fakeId = new UUIDT().toString()
const res = await supertest(app)
.post(`/api/projects/${team.id}/hog_functions/${fakeId}/batch_export_invocations`)
.send({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(404)
expect(res.body.errors[0]).toContain(fakeId)
})
it('returns 404 for hog function without batch_export_id', async () => {
const nonBatchFunction = await insertHogFunction({
name: 'non-batch hog function',
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.no_filters,
})
const res = await supertest(app)
.post(`/api/projects/${team.id}/hog_functions/${nonBatchFunction.id}/batch_export_invocations`)
.send({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(404)
expect(res.body.errors[0]).toContain(nonBatchFunction.id)
})
})
describe('successful invocation', () => {
it('executes the hog function and returns success', async () => {
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
expect(res.body.status).toEqual('success')
expect(res.body.errors).toEqual([])
expect(res.body.logs).toMatchObject([
{ level: 'info', message: expect.stringContaining('Fetch response:') },
{ level: 'debug', message: expect.stringContaining('Function completed in') },
])
})
it('uses provided invocation_id', async () => {
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const invocationId = new UUIDT().toString()
const res = await postInvocation({
clickhouse_event: clickhouseEvent,
invocation_id: invocationId,
})
expect(res.status).toEqual(200)
expect(res.body.status).toEqual('success')
})
it('generates globals with source metadata', async () => {
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
expect(mockFetch).toHaveBeenCalledTimes(1)
const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
expect(fetchBody).toMatchObject({
event: expect.objectContaining({
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
event: '$pageview',
distinct_id: '123',
}),
})
})
it('produces monitoring metrics', async () => {
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
await postInvocation({ clickhouse_event: clickhouseEvent })
await api['batchExportHogFunctionService'].stop()
const metrics = mockProducerObserver
.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
.map((x) => x.value) as any[]
expect(metrics).toEqual(
expect.arrayContaining([
expect.objectContaining({
metric_name: 'succeeded',
}),
])
)
})
})
describe('execution errors', () => {
it('returns a failed invocation, not a 500, when an integration was deleted', async () => {
hogFunction = await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.no_filters,
batch_export_id: batchExportId,
inputs_schema: [
...(HOG_INPUTS_EXAMPLES.simple_fetch.inputs_schema ?? []),
{ key: 'oauth', type: 'integration', required: true },
],
inputs: { ...HOG_INPUTS_EXAMPLES.simple_fetch.inputs, oauth: { value: 99999999 } },
})
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
expect(res.body.status).toEqual('error')
expect(res.body.errors).toEqual([
expect.stringContaining("The integration for input 'oauth' no longer exists"),
])
expect(mockFetch).not.toHaveBeenCalled()
})
it('returns error log when fetch returns 500', async () => {
mockFetch.mockImplementation(() =>
Promise.resolve({
status: 500,
headers: { 'Content-Type': 'text/plain' },
json: () => Promise.reject(new Error('not json')),
text: () => Promise.resolve('Internal Server Error'),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
expect(res.body.logs).toEqual(
expect.arrayContaining([
expect.objectContaining({
level: 'error',
message: expect.stringContaining('HTTP fetch failed'),
}),
])
)
})
})
describe('groups enrichment', () => {
let originalGroupsManager: GroupsManagerService
beforeEach(() => {
originalGroupsManager = api['batchExportHogFunctionService']['groupsManager']
})
afterEach(() => {
api['batchExportHogFunctionService']['groupsManager'] = originalGroupsManager
})
const setupGroups = async () => {
await updateOrganizationAvailableFeatures(hub.postgres, team.organization_id, [
{ key: 'data_pipelines', name: 'Data Pipelines' },
{ key: 'group_analytics', name: 'Group Analytics' },
])
hub.teamManager['lazyLoader'].clear()
const mockGroupRepo: GroupReadRepository = {
fetchGroupsByKeys: jest.fn().mockResolvedValue([
{
team_id: team.id as TeamId,
group_type_index: 0 as GroupTypeIndex,
group_key: 'acme-inc',
group_properties: { name: 'Acme Inc', industry: 'Tech' },
},
]),
fetchGroupTypesByTeamIds: jest.fn().mockImplementation((teamIds: number[]) => {
const result: Record<string, { group_type: string; group_type_index: GroupTypeIndex }[]> = {}
for (const id of teamIds) {
result[id.toString()] = [{ group_type: 'company', group_type_index: 0 as GroupTypeIndex }]
}
return Promise.resolve(result)
}),
fetchGroupTypesByProjectIds: jest.fn().mockResolvedValue({}),
}
api['batchExportHogFunctionService']['groupsManager'] = new GroupsManagerService(
hub.teamManager,
mockGroupRepo
)
}
it('enriches globals with group properties when event has $groups', async () => {
await setupGroups()
clickhouseEvent.properties = JSON.stringify({
$lib_version: '1.0.0',
$groups: { company: 'acme-inc' },
})
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
expect(res.body.status).toEqual('success')
const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
expect(fetchBody.groups).toMatchObject({
company: expect.objectContaining({
id: 'acme-inc',
type: 'company',
properties: { name: 'Acme Inc', industry: 'Tech' },
}),
})
})
it('returns empty groups when event has no $groups property', async () => {
await setupGroups()
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
expect(fetchBody.groups).toEqual({})
})
it('returns empty groups when team lacks group_analytics feature', async () => {
clickhouseEvent.properties = JSON.stringify({
$lib_version: '1.0.0',
$groups: { company: 'acme-inc' },
})
mockFetch.mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
})
)
const res = await postInvocation({ clickhouse_event: clickhouseEvent })
expect(res.status).toEqual(200)
const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
expect(fetchBody.groups).toEqual({})
})
})
})