:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'
import { Server } from 'http'
import { DateTime, Settings } from 'luxon'
import supertest from 'supertest'
import express from 'ultimate-express'
import { insertHogFunction, insertHogFunctionTemplate } from '~/cdp/_tests/fixtures'
import { CdpApi } from '~/cdp/cdp-api'
import { HogFlow } from '~/cdp/schema/hogflow'
import { template as pixelTemplate } from '~/cdp/templates/_sources/pixel/pixel.template'
import { template as incomingWebhookTemplate } from '~/cdp/templates/_sources/webhook/incoming_webhook.template'
import { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult, HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
import { defaultConfig } from '~/common/config/config'
import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { insertHogFlow } from '../_tests/fixtures-hogflows'
import { getCustomHttpResponse } from '../consumers/cdp-source-webhooks.consumer'
import { createIdempotencyValkeyPool } from '../services/hogflows/idempotency-keys'
import { HogWatcherState } from '../services/monitoring/hog-watcher.service'
import { compileHog } from '../templates/compiler'
import { compileInputs } from '../templates/test/test-helpers'
describe('getCustomHttpResponse', () => {
const makeResult = (execResult: any): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> =>
({ execResult }) as any
it('should return null when execResult has no httpResponse', () => {
expect(getCustomHttpResponse(makeResult({}))).toBeNull()
expect(getCustomHttpResponse(makeResult(null))).toBeNull()
expect(getCustomHttpResponse(makeResult(undefined))).toBeNull()
})
it('should extract status and body from httpResponse', () => {
const result = getCustomHttpResponse(makeResult({ httpResponse: { status: 200, body: 'hello' } }))
expect(result).toEqual({
status: 200,
body: 'hello',
contentType: undefined,
isBase64Encoded: undefined,
})
})
it('should default status to 500 when not a number', () => {
const result = getCustomHttpResponse(makeResult({ httpResponse: { status: 'bad', body: 'error' } }))
expect(result?.status).toEqual(500)
})
it('should extract contentType when present', () => {
const result = getCustomHttpResponse(
makeResult({ httpResponse: { status: 200, body: '', contentType: 'image/gif' } })
)
expect(result?.contentType).toEqual('image/gif')
})
it('should extract isBase64Encoded when present', () => {
const result = getCustomHttpResponse(
makeResult({
httpResponse: {
status: 200,
body: 'R0lGODlh',
contentType: 'image/gif',
isBase64Encoded: true,
},
})
)
expect(result).toEqual({
status: 200,
body: 'R0lGODlh',
contentType: 'image/gif',
isBase64Encoded: true,
})
})
it('should handle object body', () => {
const result = getCustomHttpResponse(
makeResult({ httpResponse: { status: 400, body: { error: 'bad request' } } })
)
expect(result?.body).toEqual({ error: 'bad request' })
})
})
describe('SourceWebhooksConsumer', () => {
// One pool for the file, because a pool for each test leaves open connections behind.
const idempotencyValkey = createIdempotencyValkeyPool(defaultConfig)
let hub: Hub
let team: Team
beforeEach(async () => {
hub = await createHub({})
team = (await createTestTeamFixture(hub.postgres)).team
mockFetch.mockClear()
})
afterEach(async () => {
await closeHub(hub)
})
describe('handleWebhook', () => {
// NOTE: These tests are done via the CdpApi router so we can get full coverage of the code
let api: CdpApi
let app: express.Application
let hogFunction: HogFunctionType
let hogFunctionPixel: HogFunctionType
let server: Server
let incomingWebhookTemplateId: string
let mockExecuteSpy: jest.SpyInstance
let mockQueueInvocationsSpy: jest.SpyInstance
let mockQueueHogflowInvocationsSpy: jest.SpyInstance
beforeEach(async () => {
hub.CDP_WATCHER_OBSERVE_RESULTS_BUFFER_TIME_MS = 50
api = new CdpApi(
hub,
{ ...createCdpConsumerDeps(hub), idempotencyValkey },
{
hogQueue: createMockJobQueue(),
hogflowQueue: createMockJobQueue(),
}
)
mockExecuteSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogExecutorAsync'], 'execute')
mockQueueInvocationsSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogQueue'], 'queueInvocations')
mockQueueHogflowInvocationsSpy = jest.spyOn(
api['cdpSourceWebhooksConsumer']['hogflowQueue'],
'queueInvocations'
)
app = setupExpressApp()
app.use('/', api.router())
server = app.listen(0, () => {})
hogFunction = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: incomingWebhookTemplate.code,
bytecode: await compileHog(incomingWebhookTemplate.code),
inputs: await compileInputs(incomingWebhookTemplate, {}),
})
hogFunctionPixel = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: pixelTemplate.code,
bytecode: await compileHog(pixelTemplate.code),
inputs: await compileInputs(pixelTemplate, {}),
})
Settings.defaultZone = 'UTC'
const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
await api.start()
incomingWebhookTemplateId = `${incomingWebhookTemplate.id}-${team.id}`
})
afterEach(async () => {
await api.stop()
server.close()
})
const doPostRequest = async (options: {
webhookId?: string
headers?: Record<string, string>
body?: Record<string, any>
}) => {
return supertest(app)
.post(`/public/webhooks/${options.webhookId ?? hogFunction.id}`)
.set('Content-Type', 'application/json')
.set(options.headers ?? {})
.send(options.body)
}
const doGetRequest = async (options: {
webhookId: string
headers?: Record<string, string>
body?: Record<string, any>
}) => {
return supertest(app)
.get(`/public/webhooks/${options.webhookId}`)
.set(options.headers ?? {})
.send()
}
const waitForBackgroundTasks = async () => {
await api['cdpSourceWebhooksConsumer']['promiseScheduler'].waitForAllSettled()
}
const getLogs = (): string[] => {
const res = mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')
return res.map((x) => x.value.message) as string[]
}
const getMetrics = (): any[] => {
const res = mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
return res.map((x) => x.value) as any[]
}
describe('hog function processing', () => {
it('should 404 if the hog function does not exist', async () => {
const res = await doPostRequest({
webhookId: 'non-existent-hog-function-id',
})
expect(res.status).toEqual(404)
expect(res.body).toEqual({
error: 'Not found',
})
})
it('should 404 if the hog function is deleted', async () => {
// Deletion via the django API is a soft delete that leaves `enabled` untouched
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_hogfunction SET deleted=true, updated_at = NOW() WHERE id = $1`,
[hogFunction.id],
'testKey'
)
const res = await doPostRequest({
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(404)
expect(res.body).toEqual({
error: 'Not found',
})
expect(mockExecuteSpy).not.toHaveBeenCalled()
})
it('should capture an event using internal capture', async () => {
// Mock the monitoring service to verify events ARE captured for regular webhooks
const mockQueueInvocationResults = jest.spyOn(
api['cdpSourceWebhooksConsumer']['hogFunctionMonitoringService'],
'queueInvocationResults'
)
const res = await doPostRequest({
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(200)
expect(res.body).toEqual({
status: 'ok',
})
// Verify that queueInvocationResults WAS called for regular webhooks (this captures the event)
expect(mockQueueInvocationResults).toHaveBeenCalledTimes(1)
const result = mockQueueInvocationResults.mock.calls[0][0][0]
expect(result.capturedPostHogEvents[0].properties).toHaveProperty('$hog_function_execution_count', 1)
await waitForBackgroundTasks()
expect(mockInternalFetch).toHaveBeenCalledTimes(1)
const internalEvents = mockInternalFetch.mock.calls[0][1]
expect(parseJSON(internalEvents.body)).toMatchObject({
api_key: team.api_token,
event: 'my-event',
distinct_id: 'test-distinct-id',
properties: {
$lib: 'posthog-webhook',
$source_url: `/project/${team.id}/functions/${hogFunction.id}`,
$hog_function_execution_count: 1,
capture_internal: true,
},
})
expect(internalEvents.headers).toEqual({ 'Content-Type': 'application/json' })
expect(internalEvents.method).toBe('POST')
})
it('should log custom errors', async () => {
const res = await doPostRequest({
body: {
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(400)
expect(res.body).toEqual({
error: '"event" could not be parsed correctly',
})
expect(getLogs()).toEqual([
expect.stringContaining('Function completed'),
'Responded with response status - 400, reason: {"error":"\\"event\\" could not be parsed correctly"}',
])
})
it('should not receive sensitive headers', async () => {
await doPostRequest({
headers: {
'x-forwarded-for': '127.0.0.1',
cookie: 'test=test',
},
})
const call = mockExecuteSpy.mock.calls[0][0]
expect(call.state.globals.request.headers).toEqual({
'accept-encoding': 'gzip, deflate',
connection: 'close',
'content-length': '0',
'content-type': 'application/json',
host: expect.any(String),
})
})
it('should capture an event using GET request with the pixel template', async () => {
const res = await doGetRequest({
webhookId: hogFunctionPixel.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(200)
expect(res.body).toBeInstanceOf(Buffer)
// Should NOT have charset=utf-8 appended (binary data, not text)
expect(res.headers['content-type']).toEqual('image/gif')
// Verify the response body is the exact expected GIF binary
const expectedGif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64')
expect(Buffer.from(res.body)).toEqual(expectedGif)
})
it('should allow capturing an event using GET request with gif extension', async () => {
const res = await doGetRequest({
webhookId: hogFunctionPixel.id + '.gif',
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(200)
expect(res.body).toBeInstanceOf(Buffer)
expect(res.headers['content-type']).toEqual('image/gif')
const expectedGif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64')
expect(Buffer.from(res.body)).toEqual(expectedGif)
})
it('should return base64-encoded binary response with correct content-type when isBase64Encoded is true', async () => {
const base64Png = 'iVBORw0KGgo='
const customBinaryFunction = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': '${base64Png}', 'contentType': 'image/png', 'isBase64Encoded': true } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': '${base64Png}', 'contentType': 'image/png', 'isBase64Encoded': true } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: customBinaryFunction.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toEqual('image/png')
expect(res.headers['x-content-type-options']).toEqual('nosniff')
expect(res.headers['content-security-policy']).toEqual("default-src 'none'")
expect(Buffer.from(res.body)).toEqual(Buffer.from(base64Png, 'base64'))
})
it('should return plain text string response without charset corruption when isBase64Encoded is false', async () => {
const customTextFunction = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': 'Hello, world!', 'contentType': 'text/plain' } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': 'Hello, world!', 'contentType': 'text/plain' } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: customTextFunction.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toContain('text/plain')
expect(res.headers['x-content-type-options']).toEqual('nosniff')
expect(res.headers['content-security-policy']).toEqual("default-src 'none'")
expect(res.text).toEqual('Hello, world!')
})
it.each([
['text/html', '<script>alert(document.cookie)</script>'],
['image/svg+xml', '<svg onload="alert(1)"></svg>'],
['application/javascript', 'alert(1)'],
['text/javascript', 'alert(1)'],
['application/xhtml+xml', '<html></html>'],
])(
'should reject dangerous content type %s and fall back to text/plain',
async (dangerousType, payload) => {
const fn = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': '${payload}', 'contentType': '${dangerousType}' } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': '${payload}', 'contentType': '${dangerousType}' } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: fn.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toContain('text/plain')
expect(res.headers['content-type']).not.toContain(dangerousType)
expect(res.headers['x-content-type-options']).toEqual('nosniff')
expect(res.headers['content-security-policy']).toEqual("default-src 'none'")
expect(res.text).toEqual(payload)
}
)
it('should reject unknown content types not on the allowlist', async () => {
const fn = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': 'data', 'contentType': 'application/x-custom' } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': 'data', 'contentType': 'application/x-custom' } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: fn.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toContain('text/plain')
})
it('should reject dangerous content type with charset parameter', async () => {
const fn = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': '<script>alert(1)</script>', 'contentType': 'text/html; charset=utf-8' } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': '<script>alert(1)</script>', 'contentType': 'text/html; charset=utf-8' } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: fn.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toContain('text/plain')
expect(res.headers['content-type']).not.toContain('text/html')
})
it('should strip parameters from allowlisted content types', async () => {
const fn = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': 'hello', 'contentType': 'text/plain; charset=utf-7' } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': 'hello', 'contentType': 'text/plain; charset=utf-7' } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: fn.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toContain('text/plain')
expect(res.headers['content-type']).not.toContain('utf-7')
})
it('should default to application/octet-stream when isBase64Encoded is true but no contentType', async () => {
const base64Data = 'AQIDBA=='
const customBinaryFunction = await insertHogFunction(hub.postgres, team.id, {
type: 'source_webhook',
hog: `return { 'httpResponse': { 'status': 200, 'body': '${base64Data}', 'isBase64Encoded': true } }`,
bytecode: await compileHog(
`return { 'httpResponse': { 'status': 200, 'body': '${base64Data}', 'isBase64Encoded': true } }`
),
inputs: {},
})
const res = await doGetRequest({ webhookId: customBinaryFunction.id })
expect(res.status).toEqual(200)
expect(res.headers['content-type']).toEqual('application/octet-stream')
expect(Buffer.from(res.body)).toEqual(Buffer.from(base64Data, 'base64'))
})
})
describe('hog flow processing', () => {
let hogFlow: HogFlow
beforeEach(async () => {
const template = await insertHogFunctionTemplate(hub.postgres, {
...incomingWebhookTemplate,
id: incomingWebhookTemplateId,
})
hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withSimpleWorkflow({
trigger: {
type: 'webhook',
template_id: template.template_id,
inputs: {
event: {
value: 'my-event',
bytecode: await compileHog(`return f'my-event'`),
},
distinct_id: {
value: '{request.body.distinct_id}',
bytecode: await compileHog(`return f'{request.body.distinct_id}'`),
},
method: {
value: 'POST',
bytecode: await compileHog(`return f'POST'`),
},
},
},
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
})
it('should 404 if the hog flow does not exist', async () => {
const res = await doPostRequest({
webhookId: 'non-existent-hog-flow-id',
})
expect(res.status).toEqual(404)
})
it('should invoke a workflow with the parsed inputs', async () => {
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(201)
expect(res.body).toEqual({
status: 'queued',
})
expect(mockExecuteSpy).toHaveBeenCalledTimes(1)
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
const call = mockQueueHogflowInvocationsSpy.mock.calls[0][0][0]
expect(call.queue).toEqual('hogflow')
expect(call.hogFlow).toMatchObject(hogFlow)
})
it('should add logs and metrics', async () => {
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(201)
await waitForBackgroundTasks()
expect(getLogs()).toEqual([expect.stringContaining('[Action:trigger] Function completed in')])
expect(getMetrics()).toEqual([
expect.objectContaining({
metric_kind: 'other',
metric_name: 'triggered',
count: 1,
}),
])
})
it('records a running row and stamps the run start into the queued state', async () => {
// Lifecycle rows are gated on this flag, which is off by default outside dev.
hub.HOG_INVOCATION_RESULTS_ENABLED = true
let stateWhenQueued: string | undefined
// Snapshot at call time, so a stamp applied after queueInvocations does not count.
mockQueueHogflowInvocationsSpy.mockImplementation((invocations: any[]) => {
stateWhenQueued = JSON.stringify(invocations[0].state)
return Promise.resolve()
})
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(201)
await waitForBackgroundTasks()
const rows = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
expect(rows).toHaveLength(1)
expect(rows[0].value).toMatchObject({
invocation_id: mockQueueHogflowInvocationsSpy.mock.calls[0][0][0].id,
team_id: team.id,
function_kind: 'hog_flow',
function_id: hogFlow.id,
status: 'running',
})
expect(parseJSON(stateWhenQueued!).firstScheduledAt).toEqual(rows[0].value.first_scheduled_at)
})
it('records no running row when the workflow cannot be queued', async () => {
hub.HOG_INVOCATION_RESULTS_ENABLED = true
mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(500)
await waitForBackgroundTasks()
// A row here would show a run that never entered cyclotron as permanently running.
expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)).toEqual([])
})
describe('idempotency key', () => {
const postWithKey = (webhookId: string, idempotencyKey: string) =>
doPostRequest({
webhookId,
headers: { 'Idempotency-Key': idempotencyKey },
body: { distinct_id: 'test-distinct-id' },
})
const insertSecondHogFlow = async (): Promise<HogFlow> => {
const other = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withSimpleWorkflow({ trigger: hogFlow.trigger })
.build()
await insertHogFlow(hub.postgres, other)
return other
}
it('queues one run and answers duplicate for a repeated key on the same workflow', async () => {
hub.HOG_INVOCATION_RESULTS_ENABLED = true
const first = await postWithKey(hogFlow.id, 'escalation-1')
const second = await postWithKey(hogFlow.id, 'escalation-1')
expect(first.status).toEqual(201)
expect(first.body.status).toEqual('queued')
expect(second.status).toEqual(200)
expect(second.body.status).toEqual('duplicate')
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
await waitForBackgroundTasks()
expect(
mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
).toHaveLength(1)
expect(getLogs()).toContainEqual(
expect.stringContaining('Skipped duplicate request for Idempotency-Key "escalation-1"')
)
})
it('queues a run on each workflow that receives the same key', async () => {
const otherHogFlow = await insertSecondHogFlow()
const first = await postWithKey(hogFlow.id, 'escalation-2')
const second = await postWithKey(otherHogFlow.id, 'escalation-2')
expect(first.status).toEqual(201)
expect(second.status).toEqual(201)
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
})
it('releases the key when queueing fails so a retry queues the run', async () => {
mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
const failed = await postWithKey(hogFlow.id, 'escalation-3')
const retried = await postWithKey(hogFlow.id, 'escalation-3')
expect(failed.status).toEqual(500)
expect(retried.status).toEqual(201)
expect(retried.body.status).toEqual('queued')
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
})
it('tells a retry to wait while the first request is still queueing', async () => {
let finishQueueing: () => void = () => {}
mockQueueHogflowInvocationsSpy.mockImplementationOnce(
() => new Promise<void>((resolve) => (finishQueueing = resolve))
)
const first = postWithKey(hogFlow.id, 'escalation-5').then((res) => res)
await waitForExpect(() => expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1))
const retry = await postWithKey(hogFlow.id, 'escalation-5')
finishQueueing()
expect(retry.status).toEqual(409)
expect(retry.body.status).toEqual('in_flight')
expect((await first).status).toEqual(201)
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
})
it('queues the run when the idempotency store never answers, and releases it if queueing fails', async () => {
const valkey = api['cdpSourceWebhooksConsumer']['idempotencyKeys']['valkey']!
const useClient = valkey.useClient
let reservations = 0
jest.spyOn(valkey, 'useClient').mockImplementation((options, callback) => {
if (options.name === 'idempotency-reserve' && reservations++ === 0) {
void useClient(options, callback)
return new Promise(() => {})
}
return useClient(options, callback)
})
mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
const failed = await postWithKey(hogFlow.id, 'escalation-4')
const retried = await postWithKey(hogFlow.id, 'escalation-4')
expect(failed.status).toEqual(500)
expect(retried.status).toEqual(201)
expect(retried.body.status).toEqual('queued')
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
})
it('rejects a key longer than 255 characters without queueing', async () => {
const res = await postWithKey(hogFlow.id, 'a'.repeat(256))
expect(res.status).toEqual(400)
expect(res.body.error).toEqual('Idempotency-Key must be 255 characters or fewer')
expect(mockQueueHogflowInvocationsSpy).not.toHaveBeenCalled()
})
})
it('does not report a workflow trigger as CDP usage', async () => {
const reportBillableInvocation = jest.spyOn(
api['cdpSourceWebhooksConsumer']['cdpUsageReporter'],
'reportBillableInvocation'
)
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(201)
await waitForBackgroundTasks()
expect(reportBillableInvocation).not.toHaveBeenCalled()
})
it('does not report usage when queueing the workflow fails', async () => {
const reportBillableInvocation = jest.spyOn(
api['cdpSourceWebhooksConsumer']['cdpUsageReporter'],
'reportBillableInvocation'
)
mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(500)
await waitForBackgroundTasks()
expect(reportBillableInvocation).not.toHaveBeenCalled()
expect(getMetrics()).not.toContainEqual(
expect.objectContaining({
metric_kind: 'billing',
metric_name: 'billable_invocation',
})
)
})
it('should not capture webhook event to database and should remove execution count property', async () => {
// Mock the monitoring service to track what events would be captured
const mockQueueInvocationResults = jest.spyOn(
api['cdpSourceWebhooksConsumer']['hogFunctionMonitoringService'],
'queueInvocationResults'
)
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
properties: {
custom_prop: 'custom_value',
},
},
})
expect(res.status).toEqual(201)
// Verify that queueInvocationResults was NOT called for workflows
// (this is what actually captures the event)
expect(mockQueueInvocationResults).not.toHaveBeenCalled()
// Verify the workflow invocation was queued to hogflowQueue
expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
const invocation = mockQueueHogflowInvocationsSpy.mock.calls[0][0][0]
expect(invocation.state.event).toMatchObject({
event: 'my-event',
distinct_id: 'test-distinct-id',
properties: expect.objectContaining({}),
})
// Explicitly check that $hog_function_execution_count is not in the properties
// this key prevents the infinite loop protection from triggering
expect(invocation.state.event.properties).not.toHaveProperty('$hog_function_execution_count')
})
it('should add logs and metrics for a controlled failed hog flow', async () => {
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
missing_distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(400)
expect(res.body).toEqual({
error: '"distinct_id" could not be parsed correctly',
})
await waitForBackgroundTasks()
expect(getLogs()).toEqual([
expect.stringContaining('[Action:trigger] Function completed in'),
'[Action:trigger] Responded with response status - 400, reason: {"error":"\\"distinct_id\\" could not be parsed correctly"}',
])
expect(getMetrics()).toEqual([
expect.objectContaining({ metric_kind: 'failure', metric_name: 'trigger_failed', count: 1 }),
])
})
it('should add logs and metrics for an uncontrolled failed hog flow', async () => {
// Hacky but otherwise its quite hard to trigger an uncontrolled error
hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withSimpleWorkflow({
trigger: {
type: 'webhook',
template_id: incomingWebhookTemplateId,
inputs: {
distinct_id: {
value: '{i.do.not.exist}',
bytecode: await compileHog(`return f'{i.do.not.exist}'`),
},
},
},
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const res = await doPostRequest({
webhookId: hogFlow.id,
body: {
event: 'my-event',
missing_distinct_id: 'test-distinct-id',
},
})
expect(res.status).toEqual(500)
expect(res.body).toEqual({
status: 'Unhandled error',
})
await waitForBackgroundTasks()
expect(getLogs()).toEqual([
'[Action:trigger] Error triggering flow: Could not execute bytecode for input field: distinct_id: Global variable not found: i.do.not.exist',
])
expect(getMetrics()).toEqual([
expect.objectContaining({ metric_kind: 'failure', metric_name: 'trigger_failed', count: 1 }),
])
})
})
describe('hogwatcher', () => {
it('should return a degraded response if the function is degraded', async () => {
await api['cdpSourceWebhooksConsumer']['hogWatcher'].forceStateChange(
hogFunction,
HogWatcherState.degraded
)
const res = await doPostRequest({
body: {
event: 'my-event',
distinct_id: 'test-distinct-id',
},
})
expect(res.body).toMatchInlineSnapshot(`{}`)
expect(mockExecuteSpy).not.toHaveBeenCalled()
expect(mockQueueInvocationsSpy).toHaveBeenCalledTimes(1)
const call = mockQueueInvocationsSpy.mock.calls[0][0][0]
expect(call.queue).toEqual('hogoverflow')
})
it('should return a disabled response if the function is disabled', async () => {
await api['cdpSourceWebhooksConsumer']['hogWatcher'].forceStateChange(
hogFunction,
HogWatcherState.disabled
)
const res = await doPostRequest({})
expect(res.status).toEqual(429)
expect(res.body).toEqual({
error: 'Disabled',
})
expect(mockExecuteSpy).not.toHaveBeenCalled()
expect(mockQueueInvocationsSpy).not.toHaveBeenCalled()
})
})
})
})