:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { DateTime } from 'luxon'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { configureEventLoopYield, getEventLoopYieldThresholdMs } from '~/common/utils/event-loop-yield'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub, Team } from '../../types'
import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import {
createExampleInvocation,
createHogExecutionGlobals,
createHogFunction,
insertHogFunction,
} from '../_tests/fixtures'
import { compileHog } from '../templates/compiler'
import { CyclotronJobInvocationHogFunction, HogFunctionInvocationGlobalsWithInputs, HogFunctionType } from '../types'
import { destinationE2eLagMsSummary } from '../utils'
import { CdpCyclotronWorker } from './cdp-cyclotron-worker.consumer'
jest.setTimeout(1000)
/**
* NOTE: The internal and normal events consumers are very similar so we can test them together
*/
describe('CdpCyclotronWorker', () => {
let processor: CdpCyclotronWorker
let hub: Hub
let team: Team
let fn: HogFunctionType
let globals: HogFunctionInvocationGlobalsWithInputs
let invocation: CyclotronJobInvocationHogFunction
beforeEach(async () => {
hub = await createHub()
team = (await createTestTeamFixture(hub.postgres)).team
processor = new CdpCyclotronWorker(hub, createCdpConsumerDeps(hub), createMockJobQueue())
fn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'template-webhook',
})
)
globals = {
...createHogExecutionGlobals({}),
inputs: {
url: 'https://posthog.com',
},
}
invocation = createExampleInvocation(fn, globals)
invocation.queueSource = 'postgres'
})
afterEach(async () => {
jest.setTimeout(10000)
await closeHub(hub)
})
describe('processInvocation', () => {
beforeEach(() => {
const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
mockFetch.mockResolvedValue({
status: 200,
json: () => Promise.resolve({}),
text: () => Promise.resolve(JSON.stringify({})),
headers: {},
} as any)
})
it('should process a single fetch invocation fully', async () => {
const results = await processor.processInvocations([invocation])
const result = results[0]
expect(result.finished).toBe(true)
expect(result.error).toBe(undefined)
expect(result.metrics).toEqual([
{
app_source_id: fn.id,
count: 1,
metric_kind: 'other',
metric_name: 'fetch',
team_id: team.id,
},
])
expect(result.logs.map((x) => x.message)).toEqual([
'Fetch response:, {"status":200,"body":{}}',
expect.stringContaining('Function completed in'),
])
})
it('should route hog functions to correct executor services based on template_id', async () => {
const segmentFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'segment-actions-amplitude',
})
)
const nativeFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'native-webhook',
})
)
const pluginFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'plugin-posthog-intercom-plugin',
})
)
const nativeExecutorSpy = jest.spyOn(processor['nativeDestinationExecutorService'], 'execute')
const pluginExecutorSpy = jest.spyOn(processor['pluginDestinationExecutorService'], 'execute')
const segmentExecutorSpy = jest.spyOn(processor['segmentDestinationExecutorService'], 'execute')
const hogExecutorSpy = jest.spyOn(processor['hogExecutorAsync'], 'executeWithAsyncFunctions')
const invocations = [
createExampleInvocation(nativeFn, globals),
createExampleInvocation(pluginFn, globals),
createExampleInvocation(segmentFn, globals),
createExampleInvocation(fn, globals),
]
await processor.processInvocations(invocations)
expect(nativeExecutorSpy).toHaveBeenCalledTimes(1)
expect(nativeExecutorSpy).toHaveBeenCalledWith(
expect.objectContaining({
hogFunction: expect.objectContaining({ template_id: 'native-webhook' }),
})
)
expect(pluginExecutorSpy).toHaveBeenCalledTimes(1)
expect(pluginExecutorSpy).toHaveBeenCalledWith(
expect.objectContaining({
hogFunction: expect.objectContaining({ template_id: 'plugin-posthog-intercom-plugin' }),
})
)
expect(segmentExecutorSpy).toHaveBeenCalledTimes(1)
expect(segmentExecutorSpy).toHaveBeenCalledWith(
expect.objectContaining({
hogFunction: expect.objectContaining({ template_id: 'segment-actions-amplitude' }),
})
)
expect(hogExecutorSpy).toHaveBeenCalledTimes(1)
expect(hogExecutorSpy).toHaveBeenCalledWith(
expect.objectContaining({
hogFunction: expect.objectContaining({ template_id: 'template-webhook' }),
})
)
})
it('should partially process an invocation if multiple fetches are required', async () => {
mockFetch.mockResolvedValueOnce({
status: 500,
json: () => Promise.resolve({}),
text: () => Promise.resolve(JSON.stringify({})),
headers: {},
dump: () => Promise.resolve(),
} as any)
const invocationId = invocation.id
// Capture reference time BEFORE execution to avoid timing race in lower-bound assertion
const beforeExecution = DateTime.now()
const results = await processor.processInvocations([invocation])
const result = results[0]
expect(result.finished).toBe(false)
expect(result.error).toBe(undefined)
expect(result.metrics).toEqual([])
expect(result.invocation.id).toEqual(invocationId)
expect(result.invocation.queue).toEqual('hog')
// NOTE: Check the queue scheduled at is within the bounds of the backoff
expect(result.invocation.queueScheduledAt?.toMillis()).toBeGreaterThanOrEqual(
beforeExecution.plus({ milliseconds: hub.CDP_FETCH_BACKOFF_BASE_MS }).toMillis()
)
expect(result.invocation.queueScheduledAt?.toMillis()).toBeLessThan(
DateTime.now().plus({ milliseconds: hub.CDP_FETCH_BACKOFF_MAX_MS }).toMillis()
)
expect(result.invocation.queueSource).toEqual('postgres')
expect(result.invocation.queueParameters).toMatchInlineSnapshot(`
{
"body": null,
"headers": {
"Content-Type": "application/json",
},
"method": "POST",
"type": "fetch",
"url": "https://posthog.com",
}
`)
expect(result.invocation.queueMetadata).toBeUndefined()
// No logs from initial invoke
expect(result.logs.map((x) => x.message)).toEqual([
expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.'),
])
// Now invoke the result again
const results2 = await processor.processInvocations([result.invocation])
const result2 = results2[0]
expect(result2.invocation.id).toEqual(invocationId)
expect(result2.invocation.queueSource).toEqual('postgres')
expect(result2.finished).toBe(true)
expect(result2.error).toBe(undefined)
expect(result2.metrics).toEqual([
{
app_source_id: fn.id,
count: 1,
metric_kind: 'other',
metric_name: 'fetch',
team_id: team.id,
},
])
expect(result2.logs.map((x) => x.message)).toEqual([
'Fetch response:, {"status":200,"body":{}}',
expect.stringContaining('Function completed in'),
])
})
it('should dequeue an invocation if the hog function cannot be found', async () => {
const dequeueInvocationsSpy = jest
.spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
.mockResolvedValue(undefined)
const recordTerminalFailureSpy = jest
.spyOn(
processor['invocationResultsService'].invocationResultsRowsService,
'recordTerminalFailureDurably'
)
.mockResolvedValue(true)
const invocation = createExampleInvocation(fn, globals)
invocation.functionId = new UUIDT().toString()
const results = await processor.processInvocations([invocation])
expect(results).toEqual([])
expect(dequeueInvocationsSpy).toHaveBeenCalledWith([invocation])
// Without a terminal lifecycle row the runs UI shows the invocation as running forever
expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
invocation,
expect.objectContaining({ errorKind: 'function_not_found' })
)
})
it.each([['project'], ['event']] as const)(
'should DLQ a malformed invocation whose globals is missing %s instead of crashing',
async (field) => {
const dequeueInvocationsSpy = jest
.spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
.mockResolvedValue(undefined)
const recordTerminalFailureSpy = jest
.spyOn(
processor['invocationResultsService'].invocationResultsRowsService,
'recordTerminalFailureDurably'
)
.mockResolvedValue(true)
const malformed = createExampleInvocation(fn, globals)
delete (malformed.state.globals as any)[field]
const results = await processor['loadHogFunctions']([malformed])
expect(results).toEqual([])
expect(dequeueInvocationsSpy).toHaveBeenCalledWith([malformed])
expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
malformed,
expect.objectContaining({ errorKind: 'malformed_invocation' })
)
}
)
it('should skip a loaded function if it is disabled and record the terminal row with real globals', async () => {
const dequeueInvocationsSpy = jest
.spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
.mockResolvedValue(undefined)
const recordTerminalFailureSpy = jest
.spyOn(
processor['invocationResultsService'].invocationResultsRowsService,
'recordTerminalFailureDurably'
)
.mockResolvedValue(true)
const fn2 = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
enabled: false,
})
)
// Simulate a job as it arrives off the queue: state + globals present, but the
// hogFunction not yet loaded — loadHogFunctions is what attaches it. The fixture
// pre-attaches it, which would otherwise mask the lost-globals regression.
const { hogFunction: _unloaded, ...queued } = createExampleInvocation(fn2, globals)
const results = await processor['loadHogFunctions']([queued as CyclotronJobInvocationHogFunction])
expect(results).toEqual([])
expect(dequeueInvocationsSpy).toHaveBeenCalledWith([expect.objectContaining({ id: queued.id })])
// The terminal row must be built from a fully-shaped hog function invocation so its
// globals serialize to the real payload, not '{}'. Otherwise the row wins the
// ReplacingMergeTree argMax and the rerun paginator can't rehydrate it — the re-run
// after re-enabling the function would silently skip.
expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
expect.objectContaining({
id: queued.id,
hogFunction: expect.objectContaining({ id: fn2.id }),
state: expect.objectContaining({ globals: queued.state.globals }),
}),
expect.objectContaining({ errorKind: 'function_disabled' })
)
})
it('should keep the job for a later retry if the terminal row cannot be recorded', async () => {
hub.HOG_INVOCATION_RESULTS_ENABLED = true
const dequeueInvocationsSpy = jest
.spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
.mockResolvedValue(undefined)
jest.spyOn(
processor['invocationResultsService'].invocationResultsRowsService,
'recordTerminalFailureDurably'
).mockResolvedValue(false)
const invocation2 = createExampleInvocation(fn, globals)
invocation2.functionId = new UUIDT().toString()
const results = await processor['loadHogFunctions']([invocation2])
expect(results).toEqual([])
// Dequeuing without the terminal row would recreate the stuck-'running' state
expect(dequeueInvocationsSpy).toHaveBeenCalledWith([])
})
describe('e2e lag metrics tracking', () => {
let dateNowSpy: jest.SpyInstance
const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
beforeEach(() => {
dateNowSpy = jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
})
afterEach(() => {
dateNowSpy.mockRestore()
})
it('should track e2e lag for segment- invocation', async () => {
const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')
const segmentFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'segment-actions-mixpanel',
})
)
const segmentInvocation = createExampleInvocation(segmentFn, {
...globals,
inputs: {
...globals.inputs,
projectToken: 'test-token',
apiSecret: 'test-secret',
internal_partner_action: 'trackEvent',
},
event: {
...globals.event,
captured_at: capturedAt,
},
})
await processor.processInvocations([segmentInvocation])
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(1000)
})
it('should track e2e lag for plugin- invocation', async () => {
const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')
const pluginFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'plugin-posthog-intercom-plugin',
})
)
const pluginInvocation = createExampleInvocation(pluginFn, {
...globals,
event: {
...globals.event,
captured_at: capturedAt,
},
})
await processor.processInvocations([pluginInvocation])
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(1000)
})
it('should track e2e lag for native-webhook invocation', async () => {
const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')
const nativeFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
template_id: 'native-webhook',
})
)
const nativeInvocation = createExampleInvocation(nativeFn, {
...globals,
event: {
...globals.event,
captured_at: capturedAt,
},
})
await processor.processInvocations([nativeInvocation])
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(1000)
})
it('should track e2e lag for executeWithAsyncFunctions invocation', async () => {
const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')
const hogFn = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
})
)
const hogInvocation = createExampleInvocation(hogFn, {
...globals,
event: {
...globals.event,
captured_at: capturedAt,
},
})
await processor.processInvocations([hogInvocation])
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(1000)
})
})
describe('thread relief', () => {
jest.setTimeout(10000)
let interval: NodeJS.Timeout
const blockTime = 200
let originalThresholdMs: number
beforeEach(() => {
jest.spyOn(Date, 'now').mockRestore()
jest.useRealTimers()
originalThresholdMs = getEventLoopYieldThresholdMs()
configureEventLoopYield(blockTime)
})
afterEach(() => {
clearInterval(interval)
configureEventLoopYield(originalThresholdMs)
})
it('should process batches in a way that does not block the main thread', async () => {
let lastCheck = Date.now()
let longestDelay = 0
interval = setInterval(() => {
// Sets up an interval loop so we can see how long the longest delay between ticks is
longestDelay = Math.max(longestDelay, Date.now() - lastCheck)
lastCheck = Date.now()
}, 1)
const evilFunctionCode = `
fn fibonacci(number) {
print('I AM FIBONACCI. ')
if (number < 2) {
return number;
} else {
return fibonacci(number - 1) + fibonacci(number - 2);
}
}
print(f'fib {fibonacci(64)}');`
const evilFunction = await insertHogFunction(
hub.postgres,
team.id,
createHogFunction({
...HOG_FILTERS_EXAMPLES.no_filters,
hog: evilFunctionCode,
bytecode: await compileHog(evilFunctionCode),
})
)
processor.hogExecutorAsync.hogExecutor['config'].executionTimeoutMs = blockTime
const numberToTest = 5
const invocations = Array.from({ length: numberToTest }, () =>
createExampleInvocation(evilFunction, globals)
)
const results = await processor.processInvocations(invocations)
const timings = results.flatMap(
(x) => (x.invocation.state as CyclotronJobInvocationHogFunction['state']).timings
)
const total = timings.reduce((acc, timing) => acc + timing.duration_ms, 0)
// Timings is semi random so we can't test for exact values
expect(total).toBeGreaterThan(200 * numberToTest)
expect(total).toBeLessThan(300 * numberToTest) // the hog exec limiter isn't exact
await new Promise((resolve) => setTimeout(resolve, 1))
expect(longestDelay).toBeLessThan(300) // Rough upper bound of the hog exec limiter
})
})
})
})