Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.7 KB
Raw Blame
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'

import { DateTime } from 'luxon'

import { closeHub, createHub } from '~/common/utils/db/hub'
import { configureEventLoopYield, getEventLoopYieldThresholdMs } from '~/common/utils/event-loop-yield'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { createTestTeamFixture } from '~/tests/helpers/sql'

import { Hub, Team } from '../../types'
import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import {
    createExampleInvocation,
    createHogExecutionGlobals,
    createHogFunction,
    insertHogFunction,
} from '../_tests/fixtures'
import { compileHog } from '../templates/compiler'
import { CyclotronJobInvocationHogFunction, HogFunctionInvocationGlobalsWithInputs, HogFunctionType } from '../types'
import { destinationE2eLagMsSummary } from '../utils'
import { CdpCyclotronWorker } from './cdp-cyclotron-worker.consumer'

jest.setTimeout(1000)

/**
 * NOTE: The internal and normal events consumers are very similar so we can test them together
 */
describe('CdpCyclotronWorker', () => {
    let processor: CdpCyclotronWorker
    let hub: Hub
    let team: Team
    let fn: HogFunctionType
    let globals: HogFunctionInvocationGlobalsWithInputs
    let invocation: CyclotronJobInvocationHogFunction

    beforeEach(async () => {
        hub = await createHub()
        team = (await createTestTeamFixture(hub.postgres)).team
        processor = new CdpCyclotronWorker(hub, createCdpConsumerDeps(hub), createMockJobQueue())

        fn = await insertHogFunction(
            hub.postgres,
            team.id,
            createHogFunction({
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                template_id: 'template-webhook',
            })
        )

        globals = {
            ...createHogExecutionGlobals({}),
            inputs: {
                url: 'https://posthog.com',
            },
        }

        invocation = createExampleInvocation(fn, globals)
        invocation.queueSource = 'postgres'
    })

    afterEach(async () => {
        jest.setTimeout(10000)
        await closeHub(hub)
    })

    describe('processInvocation', () => {
        beforeEach(() => {
            const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
            jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())

            mockFetch.mockResolvedValue({
                status: 200,
                json: () => Promise.resolve({}),
                text: () => Promise.resolve(JSON.stringify({})),
                headers: {},
            } as any)
        })

        it('should process a single fetch invocation fully', async () => {
            const results = await processor.processInvocations([invocation])
            const result = results[0]

            expect(result.finished).toBe(true)
            expect(result.error).toBe(undefined)
            expect(result.metrics).toEqual([
                {
                    app_source_id: fn.id,
                    count: 1,
                    metric_kind: 'other',
                    metric_name: 'fetch',
                    team_id: team.id,
                },
            ])
            expect(result.logs.map((x) => x.message)).toEqual([
                'Fetch response:, {"status":200,"body":{}}',
                expect.stringContaining('Function completed in'),
            ])
        })

        it('should route hog functions to correct executor services based on template_id', async () => {
            const segmentFn = await insertHogFunction(
                hub.postgres,
                team.id,
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                    template_id: 'segment-actions-amplitude',
                })
            )

            const nativeFn = await insertHogFunction(
                hub.postgres,
                team.id,
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                    template_id: 'native-webhook',
                })
            )

            const pluginFn = await insertHogFunction(
                hub.postgres,
                team.id,
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                    template_id: 'plugin-posthog-intercom-plugin',
                })
            )

            const nativeExecutorSpy = jest.spyOn(processor['nativeDestinationExecutorService'], 'execute')
            const pluginExecutorSpy = jest.spyOn(processor['pluginDestinationExecutorService'], 'execute')
            const segmentExecutorSpy = jest.spyOn(processor['segmentDestinationExecutorService'], 'execute')
            const hogExecutorSpy = jest.spyOn(processor['hogExecutorAsync'], 'executeWithAsyncFunctions')

            const invocations = [
                createExampleInvocation(nativeFn, globals),
                createExampleInvocation(pluginFn, globals),
                createExampleInvocation(segmentFn, globals),
                createExampleInvocation(fn, globals),
            ]

            await processor.processInvocations(invocations)

            expect(nativeExecutorSpy).toHaveBeenCalledTimes(1)
            expect(nativeExecutorSpy).toHaveBeenCalledWith(
                expect.objectContaining({
                    hogFunction: expect.objectContaining({ template_id: 'native-webhook' }),
                })
            )

            expect(pluginExecutorSpy).toHaveBeenCalledTimes(1)
            expect(pluginExecutorSpy).toHaveBeenCalledWith(
                expect.objectContaining({
                    hogFunction: expect.objectContaining({ template_id: 'plugin-posthog-intercom-plugin' }),
                })
            )

            expect(segmentExecutorSpy).toHaveBeenCalledTimes(1)
            expect(segmentExecutorSpy).toHaveBeenCalledWith(
                expect.objectContaining({
                    hogFunction: expect.objectContaining({ template_id: 'segment-actions-amplitude' }),
                })
            )

            expect(hogExecutorSpy).toHaveBeenCalledTimes(1)
            expect(hogExecutorSpy).toHaveBeenCalledWith(
                expect.objectContaining({
                    hogFunction: expect.objectContaining({ template_id: 'template-webhook' }),
                })
            )
        })

        it('should partially process an invocation if multiple fetches are required', async () => {
            mockFetch.mockResolvedValueOnce({
                status: 500,
                json: () => Promise.resolve({}),
                text: () => Promise.resolve(JSON.stringify({})),
                headers: {},
                dump: () => Promise.resolve(),
            } as any)

            const invocationId = invocation.id
            // Capture reference time BEFORE execution to avoid timing race in lower-bound assertion
            const beforeExecution = DateTime.now()
            const results = await processor.processInvocations([invocation])
            const result = results[0]

            expect(result.finished).toBe(false)
            expect(result.error).toBe(undefined)
            expect(result.metrics).toEqual([])
            expect(result.invocation.id).toEqual(invocationId)
            expect(result.invocation.queue).toEqual('hog')
            // NOTE: Check the queue scheduled at is within the bounds of the backoff
            expect(result.invocation.queueScheduledAt?.toMillis()).toBeGreaterThanOrEqual(
                beforeExecution.plus({ milliseconds: hub.CDP_FETCH_BACKOFF_BASE_MS }).toMillis()
            )
            expect(result.invocation.queueScheduledAt?.toMillis()).toBeLessThan(
                DateTime.now().plus({ milliseconds: hub.CDP_FETCH_BACKOFF_MAX_MS }).toMillis()
            )
            expect(result.invocation.queueSource).toEqual('postgres')
            expect(result.invocation.queueParameters).toMatchInlineSnapshot(`
                {
                  "body": null,
                  "headers": {
                    "Content-Type": "application/json",
                  },
                  "method": "POST",
                  "type": "fetch",
                  "url": "https://posthog.com",
                }
            `)
            expect(result.invocation.queueMetadata).toBeUndefined()
            // No logs from initial invoke
            expect(result.logs.map((x) => x.message)).toEqual([
                expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.'),
            ])

            // Now invoke the result again
            const results2 = await processor.processInvocations([result.invocation])
            const result2 = results2[0]

            expect(result2.invocation.id).toEqual(invocationId)
            expect(result2.invocation.queueSource).toEqual('postgres')
            expect(result2.finished).toBe(true)
            expect(result2.error).toBe(undefined)
            expect(result2.metrics).toEqual([
                {
                    app_source_id: fn.id,
                    count: 1,
                    metric_kind: 'other',
                    metric_name: 'fetch',
                    team_id: team.id,
                },
            ])
            expect(result2.logs.map((x) => x.message)).toEqual([
                'Fetch response:, {"status":200,"body":{}}',
                expect.stringContaining('Function completed in'),
            ])
        })

        it('should dequeue an invocation if the hog function cannot be found', async () => {
            const dequeueInvocationsSpy = jest
                .spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
                .mockResolvedValue(undefined)
            const recordTerminalFailureSpy = jest
                .spyOn(
                    processor['invocationResultsService'].invocationResultsRowsService,
                    'recordTerminalFailureDurably'
                )
                .mockResolvedValue(true)
            const invocation = createExampleInvocation(fn, globals)
            invocation.functionId = new UUIDT().toString()
            const results = await processor.processInvocations([invocation])
            expect(results).toEqual([])
            expect(dequeueInvocationsSpy).toHaveBeenCalledWith([invocation])
            // Without a terminal lifecycle row the runs UI shows the invocation as running forever
            expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
                invocation,
                expect.objectContaining({ errorKind: 'function_not_found' })
            )
        })

        it.each([['project'], ['event']] as const)(
            'should DLQ a malformed invocation whose globals is missing %s instead of crashing',
            async (field) => {
                const dequeueInvocationsSpy = jest
                    .spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
                    .mockResolvedValue(undefined)
                const recordTerminalFailureSpy = jest
                    .spyOn(
                        processor['invocationResultsService'].invocationResultsRowsService,
                        'recordTerminalFailureDurably'
                    )
                    .mockResolvedValue(true)

                const malformed = createExampleInvocation(fn, globals)
                delete (malformed.state.globals as any)[field]

                const results = await processor['loadHogFunctions']([malformed])

                expect(results).toEqual([])
                expect(dequeueInvocationsSpy).toHaveBeenCalledWith([malformed])
                expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
                    malformed,
                    expect.objectContaining({ errorKind: 'malformed_invocation' })
                )
            }
        )

        it('should skip a loaded function if it is disabled and record the terminal row with real globals', async () => {
            const dequeueInvocationsSpy = jest
                .spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
                .mockResolvedValue(undefined)
            const recordTerminalFailureSpy = jest
                .spyOn(
                    processor['invocationResultsService'].invocationResultsRowsService,
                    'recordTerminalFailureDurably'
                )
                .mockResolvedValue(true)
            const fn2 = await insertHogFunction(
                hub.postgres,
                team.id,
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                    enabled: false,
                })
            )
            // Simulate a job as it arrives off the queue: state + globals present, but the
            // hogFunction not yet loaded — loadHogFunctions is what attaches it. The fixture
            // pre-attaches it, which would otherwise mask the lost-globals regression.
            const { hogFunction: _unloaded, ...queued } = createExampleInvocation(fn2, globals)

            const results = await processor['loadHogFunctions']([queued as CyclotronJobInvocationHogFunction])
            expect(results).toEqual([])
            expect(dequeueInvocationsSpy).toHaveBeenCalledWith([expect.objectContaining({ id: queued.id })])
            // The terminal row must be built from a fully-shaped hog function invocation so its
            // globals serialize to the real payload, not '{}'. Otherwise the row wins the
            // ReplacingMergeTree argMax and the rerun paginator can't rehydrate it — the re-run
            // after re-enabling the function would silently skip.
            expect(recordTerminalFailureSpy).toHaveBeenCalledWith(
                expect.objectContaining({
                    id: queued.id,
                    hogFunction: expect.objectContaining({ id: fn2.id }),
                    state: expect.objectContaining({ globals: queued.state.globals }),
                }),
                expect.objectContaining({ errorKind: 'function_disabled' })
            )
        })

        it('should keep the job for a later retry if the terminal row cannot be recorded', async () => {
            hub.HOG_INVOCATION_RESULTS_ENABLED = true
            const dequeueInvocationsSpy = jest
                .spyOn(processor['cyclotronJobQueue'], 'dequeueInvocations')
                .mockResolvedValue(undefined)
            jest.spyOn(
                processor['invocationResultsService'].invocationResultsRowsService,
                'recordTerminalFailureDurably'
            ).mockResolvedValue(false)
            const invocation2 = createExampleInvocation(fn, globals)
            invocation2.functionId = new UUIDT().toString()

            const results = await processor['loadHogFunctions']([invocation2])

            expect(results).toEqual([])
            // Dequeuing without the terminal row would recreate the stuck-'running' state
            expect(dequeueInvocationsSpy).toHaveBeenCalledWith([])
        })

        describe('e2e lag metrics tracking', () => {
            let dateNowSpy: jest.SpyInstance
            const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })

            beforeEach(() => {
                dateNowSpy = jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
            })

            afterEach(() => {
                dateNowSpy.mockRestore()
            })

            it('should track e2e lag for segment- invocation', async () => {
                const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
                const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')

                const segmentFn = await insertHogFunction(
                    hub.postgres,
                    team.id,
                    createHogFunction({
                        ...HOG_EXAMPLES.simple_fetch,
                        ...HOG_INPUTS_EXAMPLES.simple_fetch,
                        ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                        template_id: 'segment-actions-mixpanel',
                    })
                )

                const segmentInvocation = createExampleInvocation(segmentFn, {
                    ...globals,
                    inputs: {
                        ...globals.inputs,
                        projectToken: 'test-token',
                        apiSecret: 'test-secret',
                        internal_partner_action: 'trackEvent',
                    },
                    event: {
                        ...globals.event,
                        captured_at: capturedAt,
                    },
                })

                await processor.processInvocations([segmentInvocation])

                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(1000)
            })

            it('should track e2e lag for plugin- invocation', async () => {
                const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
                const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')

                const pluginFn = await insertHogFunction(
                    hub.postgres,
                    team.id,
                    createHogFunction({
                        ...HOG_EXAMPLES.simple_fetch,
                        ...HOG_INPUTS_EXAMPLES.simple_fetch,
                        ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                        template_id: 'plugin-posthog-intercom-plugin',
                    })
                )

                const pluginInvocation = createExampleInvocation(pluginFn, {
                    ...globals,
                    event: {
                        ...globals.event,
                        captured_at: capturedAt,
                    },
                })

                await processor.processInvocations([pluginInvocation])

                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(1000)
            })

            it('should track e2e lag for native-webhook invocation', async () => {
                const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
                const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')

                const nativeFn = await insertHogFunction(
                    hub.postgres,
                    team.id,
                    createHogFunction({
                        ...HOG_EXAMPLES.simple_fetch,
                        ...HOG_INPUTS_EXAMPLES.simple_fetch,
                        ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                        template_id: 'native-webhook',
                    })
                )

                const nativeInvocation = createExampleInvocation(nativeFn, {
                    ...globals,
                    event: {
                        ...globals.event,
                        captured_at: capturedAt,
                    },
                })

                await processor.processInvocations([nativeInvocation])

                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(1000)
            })

            it('should track e2e lag for executeWithAsyncFunctions invocation', async () => {
                const capturedAt = new Date(fixedTime.toMillis() - 1000).toISOString()
                const observeSpy = jest.spyOn(destinationE2eLagMsSummary, 'observe')

                const hogFn = await insertHogFunction(
                    hub.postgres,
                    team.id,
                    createHogFunction({
                        ...HOG_EXAMPLES.simple_fetch,
                        ...HOG_INPUTS_EXAMPLES.simple_fetch,
                        ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,
                    })
                )

                const hogInvocation = createExampleInvocation(hogFn, {
                    ...globals,
                    event: {
                        ...globals.event,
                        captured_at: capturedAt,
                    },
                })

                await processor.processInvocations([hogInvocation])

                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(1000)
            })
        })

        describe('thread relief', () => {
            jest.setTimeout(10000)
            let interval: NodeJS.Timeout
            const blockTime = 200
            let originalThresholdMs: number

            beforeEach(() => {
                jest.spyOn(Date, 'now').mockRestore()
                jest.useRealTimers()
                originalThresholdMs = getEventLoopYieldThresholdMs()
                configureEventLoopYield(blockTime)
            })

            afterEach(() => {
                clearInterval(interval)
                configureEventLoopYield(originalThresholdMs)
            })

            it('should process batches in a way that does not block the main thread', async () => {
                let lastCheck = Date.now()
                let longestDelay = 0

                interval = setInterval(() => {
                    // Sets up an interval loop so we can see how long the longest delay between ticks is
                    longestDelay = Math.max(longestDelay, Date.now() - lastCheck)
                    lastCheck = Date.now()
                }, 1)

                const evilFunctionCode = `
                        fn fibonacci(number) {
                            print('I AM FIBONACCI. ')
                            if (number < 2) {
                                return number;
                            } else {
                                return fibonacci(number - 1) + fibonacci(number - 2);
                            }
                        }
                        print(f'fib {fibonacci(64)}');`

                const evilFunction = await insertHogFunction(
                    hub.postgres,
                    team.id,
                    createHogFunction({
                        ...HOG_FILTERS_EXAMPLES.no_filters,
                        hog: evilFunctionCode,
                        bytecode: await compileHog(evilFunctionCode),
                    })
                )

                processor.hogExecutorAsync.hogExecutor['config'].executionTimeoutMs = blockTime

                const numberToTest = 5
                const invocations = Array.from({ length: numberToTest }, () =>
                    createExampleInvocation(evilFunction, globals)
                )
                const results = await processor.processInvocations(invocations)

                const timings = results.flatMap(
                    (x) => (x.invocation.state as CyclotronJobInvocationHogFunction['state']).timings
                )

                const total = timings.reduce((acc, timing) => acc + timing.duration_ms, 0)

                // Timings is semi random so we can't test for exact values
                expect(total).toBeGreaterThan(200 * numberToTest)
                expect(total).toBeLessThan(300 * numberToTest) // the hog exec limiter isn't exact

                await new Promise((resolve) => setTimeout(resolve, 1))

                expect(longestDelay).toBeLessThan(300) // Rough upper bound of the hog exec limiter
            })
        })
    })
})