Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

27.1 KB
Raw Blame
import { DateTime } from 'luxon'

import { HogFlow } from '~/cdp/schema/hogflow'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { Team } from '~/types'

import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { HOG_FLOW_MASK_EXAMPLES } from '../_tests/examples'
import { CdpOutput } from '../cdp-services'
import {
    BatchResolverState,
    MAX_RESOLVER_ATTEMPTS,
    deserializeResolverState,
    serializeResolverState,
} from '../services/hogflows/batch-resolver.types'
import { AudienceFetchTimeoutError } from '../services/hogflows/hogflow-batch-person-query.service'
import {
    HogInvocationResultRow,
    HogInvocationResultsService,
} from '../services/monitoring/hog-invocation-results.service'
import { CyclotronJobInvocationHogFlow } from '../types'
import {
    CdpCyclotronWorkerBatchResolve,
    buildAccountHogFlowInvocation,
} from './cdp-cyclotron-worker-batch-resolve.consumer'

describe('CdpCyclotronWorkerBatchResolve', () => {
    const team = { id: 123, name: 'Test team' } as Team
    // A non-default version, so the flowVersion assertion can't pass by accident.
    const hogFlow: HogFlow = { ...new FixtureHogFlowBuilder().withTeamId(team.id).build(), version: 4 }

    describe('buildAccountHogFlowInvocation', () => {
        it('carries the account group key and no person', () => {
            const invocation = buildAccountHogFlowInvocation({
                siteUrl: 'https://us.posthog.com',
                parentRunId: 'batch-job-1',
                team,
                hogFlow,
                externalId: 'acme-1',
                groupType: 'customer',
                defaultVariables: { greeting: 'hi' },
            })

            const state = invocation.state as CyclotronJobInvocationHogFlow['state']

            expect(state.event.event).toEqual('$batch_hog_flow_invocation')
            // distinct_id doubles as the per-account key for invocation_results; it must NOT
            // resolve to a person (the hogflow worker skips the lookup for account audiences).
            expect(state.event.distinct_id).toEqual('acme-1')
            expect(state.event.properties['$groups']).toEqual({ customer: 'acme-1' })
            expect(state.personId).toBeUndefined()
            // The stamp is what the hogflow worker trusts when the live trigger has been
            // edited to a person audience while these children were still queued.
            expect(state.accountAudience).toBe(true)
            // Account broadcasts convert long after the send, so the run has to carry the version
            // that sent or the conversion is credited to whatever is published by then.
            expect(state.flowVersion).toBe(4)
            expect(state.customerTaskIdempotencyVersion).toBe(1)
            expect(state.variables).toEqual({ greeting: 'hi' })
            expect(state.sensitive).toBeUndefined()
            expect(invocation.parentRunId).toEqual('batch-job-1')
            expect(invocation.queue).toEqual('hogflow')
            expect((invocation as any).person).toBeUndefined()
        })

        // Two things key off this object. HogMaskerService.filterByMasking() only recognizes an
        // invocation as a hog flow invocation (and applies trigger_masking) when it carries a
        // `hogFlow`, and the invocation-results service classifies rows by the same shape — so a
        // regression here silently stops masking batch runs and drops their rows out of the
        // workflow invocations list, which filters on hog_flow.
        it('attaches the hogFlow so trigger_masking can be applied by the batch resolver', () => {
            const maskedHogFlow: HogFlow = new FixtureHogFlowBuilder()
                .withTeamId(team.id)
                .withTriggerMasking(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)
                .build()

            const invocation = buildAccountHogFlowInvocation({
                siteUrl: 'https://us.posthog.com',
                parentRunId: 'batch-job-1',
                team,
                hogFlow: maskedHogFlow,
                externalId: 'acme-1',
                groupType: 'customer',
                defaultVariables: {},
            })

            expect(invocation.hogFlow).toBe(maskedHogFlow)
            expect(invocation.hogFlow.trigger_masking).toEqual(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)
        })

        it('stamps a protected message step on the run so a recovery path with only the workflow id keeps it out of the snapshot', () => {
            const protectedFlow: HogFlow = {
                ...hogFlow,
                actions: [
                    ...hogFlow.actions,
                    {
                        id: 'message',
                        type: 'function_push',
                        config: { template_id: 'template-native-push', sensitive: true, inputs: {} },
                    } as any,
                ],
            }

            const invocation = buildAccountHogFlowInvocation({
                siteUrl: 'https://us.posthog.com',
                parentRunId: 'batch-job-1',
                team,
                hogFlow: protectedFlow,
                externalId: 'acme-1',
                groupType: 'customer',
                defaultVariables: {},
            })

            expect(invocation.state.sensitive).toBe(true)
        })
    })

    describe('processOnePage run-level monitoring', () => {
        const BATCH_JOB_ID = 'batch-job-1'

        let consumer: CdpCyclotronWorkerBatchResolve
        let queueAppMetrics: jest.Mock
        let outputs: jest.Mocked<IngestionOutputs<CdpOutput>>
        let rowsService: HogInvocationResultsService
        let bulkCreateAndCheckIn: jest.Mock
        let filterByMasking: jest.Mock
        let release: jest.Mock

        const state: BatchResolverState = {
            batchJobId: BATCH_JOB_ID,
            teamId: team.id,
            hogFlowId: hogFlow.id,
            cursor: null,
            filters: { properties: [] },
            maxAudienceSize: 100,
            totalEnqueued: 0,
            pagesProcessed: 0,
            attempts: 0,
            variables: {},
            startedAt: '2026-08-11T00:00:00.000Z',
        }

        const processPage = async (): Promise<void> =>
            await (consumer as any).processOnePage({ bulkCreateAndCheckIn, reschedule: jest.fn() }, state)

        // The rows the real service produced on flush. Asserting on these rather than on mock
        // call args is what catches a misclassification: function_kind is derived structurally
        // inside the service ('hogFlow' in invocation), which a mocked service never evaluates.
        const producedRows = (): HogInvocationResultRow[] =>
            outputs.produce.mock.calls.map(
                ([, message]) => parseJSON(message.value!.toString('utf8')) as HogInvocationResultRow
            )

        const triggeredMetrics = (): any[] =>
            queueAppMetrics.mock.calls.flatMap(([metrics]) => metrics).filter((m) => m.metric_name === 'triggered')

        beforeEach(() => {
            queueAppMetrics = jest.fn()
            bulkCreateAndCheckIn = jest.fn().mockResolvedValue({ newJobIds: [] })
            release = jest.fn().mockResolvedValue(undefined)
            // Masking itself is covered by hog-masker.service.test.ts; stubbing the partition
            // here is what lets each case fix which runs are masked and assert how the consumer
            // routes them.
            filterByMasking = jest.fn((invocations) => ({ masked: [], notMasked: invocations, release }))
            outputs = {
                produce: jest.fn().mockResolvedValue(undefined),
            } as unknown as jest.Mocked<IngestionOutputs<CdpOutput>>
            rowsService = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: true })

            // The base constructor builds redis/valkey-backed services, so assemble the
            // instance directly — this exercises processOnePage without any live boundary.
            // The rows service is real (only its produce boundary is mocked) because the
            // regressions to catch live inside it: how it classifies and stamps the
            // invocations this consumer builds.
            consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
            Object.assign(consumer, {
                config: { SITE_URL: 'https://us.posthog.com' },
                deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
                hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
                hogFlowBatchPersonQueryService: {
                    getBlastRadiusPersons: jest.fn().mockResolvedValue({
                        users_affected: ['person-1', 'person-2'],
                        cursor: null,
                        has_more: false,
                    }),
                },
                hogMasker: { filterByMasking },
                hogFunctionMonitoringService: { queueAppMetrics, queueLogs: jest.fn() },
                invocationResultsService: { invocationResultsRowsService: rowsService },
            })
        })

        it('pages a realtime cohort audience from realtime membership, not the Django audience query', async () => {
            const getMemberPage = jest.fn().mockResolvedValue({
                ids: ['member-1', 'member-2', 'member-3'],
                cursor: 'member-3',
                has_more: false,
            })
            Object.assign(consumer, {
                config: { SITE_URL: 'https://us.posthog.com', BEHAVIORAL_COHORTS_DATABASE_URL: 'postgres://cohorts' },
                realtimeCohortAudience: { getMemberPage },
            })

            await (consumer as any).processOnePage(
                { bulkCreateAndCheckIn, reschedule: jest.fn() },
                { ...state, realtimeCohort: { id: 42, teamId: 7 } }
            )

            // Membership rows belong to the cohort's own team, which can differ from the workflow's.
            expect(getMemberPage).toHaveBeenCalledWith(7, 42, null)
            expect((consumer as any).hogFlowBatchPersonQueryService.getBlastRadiusPersons).not.toHaveBeenCalled()
            expect(triggeredMetrics()).toHaveLength(3)
        })

        it('counts each enrolled person as a triggered run against the batch job', async () => {
            await processPage()

            const metrics = triggeredMetrics()
            expect(metrics).toHaveLength(2)
            for (const metric of metrics) {
                // Keyed on the batch job, not the workflow — that is where the batch metrics
                // view looks, and the workflow id would silently read as zero runs started.
                expect(metric.app_source_id).toEqual(BATCH_JOB_ID)
                expect(metric.team_id).toEqual(team.id)
                expect(metric.metric_kind).toEqual('other')
                expect(metric.count).toEqual(1)
                // Run-level metrics carry no instance id; a step id here would hide them from the
                // started/in-progress counters, which filter on the empty instance.
                expect(metric.instance_id).toBeUndefined()
            }
        })

        it('records each enrolled person as a running hog_flow invocation so parked runs are listable', async () => {
            await processPage()
            await rowsService.flush()

            const rows = producedRows()
            expect(rows).toHaveLength(2)
            for (const row of rows) {
                // hog_flow, not hog_function: the workflow invocations API only reads rows with
                // this kind, so a misclassified row is invisible even though it was written.
                expect(row.function_kind).toEqual('hog_flow')
                expect(row.function_id).toEqual(hogFlow.id)
                expect(row.parent_run_id).toEqual(BATCH_JOB_ID)
                expect(row.status).toEqual('running')
            }
        })

        it.each([
            ['an ordinary workflow', false, undefined],
            // A recovery path that holds only the workflow id reads this stamp to keep the message out of the snapshot.
            ['a workflow with a protected message step', true, true],
        ])('persists the enqueue time and run stamps for new person runs of %s', async (_name, sensitive, stamp) => {
            // The stamp is written onto the invocation by queueLifecycleRow, so it only lands in
            // cyclotron if that runs before the state is serialized. Out of order, the terminal
            // row written when the run wakes records the wake time and wins the argMax collapse.
            const flow: HogFlow = {
                ...hogFlow,
                actions: [
                    ...hogFlow.actions,
                    {
                        id: 'message',
                        type: 'function_push',
                        config: { template_id: 'template-native-push', sensitive, inputs: {} },
                    } as any,
                ],
            }
            ;(consumer as any).hogFlowManager.getHogFlow.mockResolvedValue(flow)
            await processPage()

            const { newJobs } = bulkCreateAndCheckIn.mock.calls[0][0]
            expect(newJobs).toHaveLength(2)
            for (const job of newJobs) {
                const jobState = parseJSON(job.state.toString()).state
                expect(jobState).toMatchObject({
                    firstScheduledAt: expect.any(String),
                    actionStepCount: 0,
                    customerTaskIdempotencyVersion: 1,
                })
                expect(jobState.sensitive).toBe(stamp)
            }
        })

        it('leaves masked runs out of both the triggered count and the invocations list', async () => {
            // A masked run is never enqueued, so counting it as started would strand it in the
            // in-progress figure forever, and its running row would never get a terminal row.
            filterByMasking.mockImplementation((invocations: CyclotronJobInvocationHogFlow[]) => ({
                masked: [invocations[0]],
                notMasked: [invocations[1]],
                release,
            }))

            await processPage()
            await rowsService.flush()

            expect(triggeredMetrics()).toHaveLength(1)
            expect(producedRows()).toHaveLength(1)
        })

        it('drops the queued rows and releases the mask claims when the page fails to commit', async () => {
            // The commit is atomic with the cursor advance, so a failure replays the whole page.
            // Rows left queued here would be written twice.
            bulkCreateAndCheckIn.mockRejectedValue(new Error('postgres is down'))

            await expect(processPage()).rejects.toThrow('postgres is down')
            await rowsService.flush()

            expect(producedRows()).toHaveLength(0)
            expect(release).toHaveBeenCalledTimes(1)
            expect(triggeredMetrics()).toHaveLength(0)
        })

        it('terminates the resolver instead of scheduling another page when the check-in is refused by a cancel', async () => {
            // The engine refuses the page when a cancel flag landed mid-build: nothing
            // committed, so the queued running rows and mask claims must be undone (these
            // children will never run) and the resolver must terminate, not retry.
            bulkCreateAndCheckIn.mockResolvedValue({ newJobIds: [], cancelRequested: true })
            const cancel = jest.fn().mockResolvedValue(undefined)
            const reschedule = jest.fn()
            Object.assign(consumer, {
                hogFunctionMonitoringService: {
                    queueAppMetrics,
                    queueLogs: jest.fn(),
                    flush: jest.fn().mockResolvedValue(undefined),
                },
            })

            await (consumer as any).processOnePage({ bulkCreateAndCheckIn, reschedule, cancel }, state)
            await rowsService.flush()

            expect(producedRows()).toHaveLength(0)
            expect(release).toHaveBeenCalledTimes(1)
            expect(triggeredMetrics()).toHaveLength(0)
            expect(cancel).toHaveBeenCalledTimes(1)
            expect(reschedule).not.toHaveBeenCalled()
        })
    })

    describe('cancel-flagged resolver jobs', () => {
        it('terminates on dequeue without fetching a page, even when the state is unparseable', async () => {
            const getBlastRadiusPersons = jest.fn()
            const queueLogs = jest.fn()
            const flush = jest.fn().mockResolvedValue(undefined)
            const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
            Object.assign(consumer, {
                hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
                hogFunctionMonitoringService: { queueLogs, flush },
            })
            const cancel = jest.fn().mockResolvedValue(undefined)
            const job = {
                id: 'job-1',
                teamId: team.id,
                functionId: hogFlow.id,
                parentRunId: 'batch-job-1',
                cancelRequestedAt: DateTime.now(),
                // Unparseable on purpose: the cancel must not depend on state surviving
                // schema drift across deploys.
                state: Buffer.from('not json'),
                cancel,
            }

            await (consumer as any).processResolverJob(job)

            expect(cancel).toHaveBeenCalledTimes(1)
            expect(getBlastRadiusPersons).not.toHaveBeenCalled()
            // The stop is visible on the batch run's log stream, keyed by parent run id.
            expect(queueLogs).toHaveBeenCalledTimes(1)
            const [logs] = queueLogs.mock.calls[0]
            expect(logs[0].log_source_id).toEqual('batch-job-1')
            expect(flush).toHaveBeenCalled()
        })
    })

    describe('resolver job lock heartbeats', () => {
        afterEach(() => {
            jest.useRealTimers()
        })

        it.each([false, true])('clears the heartbeat timer (heartbeat fails: %s)', async (heartbeatFails) => {
            jest.useFakeTimers()
            const state: BatchResolverState = {
                batchJobId: 'batch-job-hb',
                teamId: team.id,
                hogFlowId: hogFlow.id,
                cursor: null,
                filters: { properties: [] },
                maxAudienceSize: 100,
                totalEnqueued: 0,
                pagesProcessed: 0,
                attempts: 0,
                variables: {},
                startedAt: '2026-08-11T00:00:00.000Z',
            }
            // Long enough for two heartbeat ticks, inside the 30s default fetch budget.
            const getBlastRadiusPersons = jest.fn().mockImplementation(async () => {
                await new Promise<void>((resolve) => {
                    setTimeout(() => resolve(), 25_000)
                })
                return { users_affected: [], cursor: null, has_more: false }
            })
            const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
            Object.assign(consumer, {
                config: { SITE_URL: 'https://us.posthog.com' },
                deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
                hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
                hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
                hogMasker: {
                    filterByMasking: jest.fn((invocations) => ({
                        masked: [],
                        notMasked: invocations,
                        release: jest.fn().mockResolvedValue(undefined),
                    })),
                },
                hogFunctionMonitoringService: {
                    queueAppMetrics: jest.fn(),
                    queueLogs: jest.fn(),
                    flush: jest.fn().mockResolvedValue(undefined),
                },
                invocationResultsService: {
                    invocationResultsRowsService: { flush: jest.fn().mockResolvedValue(undefined) },
                },
            })
            const heartbeat = heartbeatFails
                ? jest.fn().mockRejectedValue(new Error('Heartbeat unavailable'))
                : jest.fn().mockResolvedValue(undefined)
            const job = {
                id: 'job-heartbeat',
                teamId: team.id,
                functionId: hogFlow.id,
                parentRunId: 'batch-job-hb',
                cancelRequestedAt: null,
                state: serializeResolverState(state),
                heartbeat,
                bulkCreateAndCheckIn: jest.fn().mockResolvedValue({ newJobIds: [] }),
                reschedule: jest.fn().mockResolvedValue(undefined),
                ack: jest.fn().mockResolvedValue(undefined),
                fail: jest.fn().mockResolvedValue(undefined),
            }

            const processPromise = (consumer as any).processResolverJob(job)
            await jest.advanceTimersByTimeAsync(25_000)
            await processPromise

            expect(heartbeat).toHaveBeenCalledTimes(2)
            expect(getBlastRadiusPersons).toHaveBeenCalledTimes(1)
            expect(job.bulkCreateAndCheckIn).toHaveBeenCalledTimes(1)
            expect(jest.getTimerCount()).toBe(0)
            await jest.advanceTimersByTimeAsync(20_000)
            expect(heartbeat).toHaveBeenCalledTimes(2)
        })
    })
    describe('permanent audience fetch failure', () => {
        const buildConsumer = (getBlastRadiusPersons: jest.Mock, queueLogs: jest.Mock): any => {
            const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
            Object.assign(consumer, {
                config: { SITE_URL: 'https://us.posthog.com' },
                deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
                hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
                hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
                hogFunctionMonitoringService: {
                    queueAppMetrics: jest.fn(),
                    queueLogs,
                    flush: jest.fn().mockResolvedValue(undefined),
                },
                invocationResultsService: {
                    invocationResultsRowsService: { flush: jest.fn().mockResolvedValue(undefined) },
                },
            })
            return consumer
        }

        const buildJob = (state: BatchResolverState): any => ({
            id: 'job-last-attempt',
            teamId: team.id,
            functionId: hogFlow.id,
            parentRunId: state.batchJobId,
            cancelRequestedAt: null,
            state: serializeResolverState(state),
            heartbeat: jest.fn().mockResolvedValue(undefined),
            bulkCreateAndCheckIn: jest.fn().mockResolvedValue({ newJobIds: [] }),
            reschedule: jest.fn().mockResolvedValue(undefined),
            ack: jest.fn().mockResolvedValue(undefined),
            fail: jest.fn().mockResolvedValue(undefined),
        })

        // One attempt short of the cap, so the failure in each case is the permanent one.
        const lastAttemptState = (): BatchResolverState => ({
            batchJobId: 'batch-job-fail',
            teamId: team.id,
            hogFlowId: hogFlow.id,
            cursor: null,
            filters: { properties: [] },
            maxAudienceSize: 100,
            totalEnqueued: 0,
            pagesProcessed: 0,
            attempts: MAX_RESOLVER_ATTEMPTS - 1,
            variables: {},
            startedAt: '2026-08-11T00:00:00.000Z',
        })

        it.each([
            ['before its deadline parks it again without reading members', 3600 * 1000, false],
            ['at its deadline fails the run', -1000, true],
        ])('a realtime cohort resolver woken %s', async (_name, untilOffsetMs, fails) => {
            const getBlastRadiusPersons = jest.fn()
            const queueLogs = jest.fn()
            const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)
            const waitUntil = new Date(Date.now() + untilOffsetMs).toISOString()
            const job = buildJob({
                ...lastAttemptState(),
                attempts: 0,
                realtimeCohort: { id: 7, teamId: team.id, waitUntil },
            })

            await consumer.processResolverJob(job)

            expect(getBlastRadiusPersons).not.toHaveBeenCalled()
            const [{ scheduledAt, state }] = job.reschedule.mock.calls[0]
            if (fails) {
                expect(deserializeResolverState(state).pendingTerminal).toEqual('failed')
                expect(queueLogs.mock.calls[0][0][0].message).toContain("didn't finish calculating")
            } else {
                expect(scheduledAt.toISOString()).toEqual(waitUntil)
                expect(state).toBeUndefined()
            }
        })

        it('tells the customer what to change when the audience query timed out', async () => {
            const queueLogs = jest.fn()
            const consumer = buildConsumer(
                jest.fn().mockRejectedValue(new AudienceFetchTimeoutError('user_blast_radius_persons', 30_000)),
                queueLogs
            )
            const job = buildJob(lastAttemptState())

            await consumer.processResolverJob(job)

            expect(queueLogs).toHaveBeenCalledWith(
                [
                    expect.objectContaining({
                        message:
                            'Batch resolver failed: Audience query timed out after 30s on the last of ' +
                            `${MAX_RESOLVER_ATTEMPTS} attempts. Use fewer or simpler audience filters, or a smaller audience.`,
                    }),
                ],
                'hog_flow'
            )
        })

        it('fails a realtime cohort run at once when the resolver has no behavioral cohorts database', async () => {
            // Without its own URL the router reads the main database, so retrying would never succeed.
            const queueLogs = jest.fn()
            const getBlastRadiusPersons = jest.fn()
            const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)
            const job = buildJob({ ...lastAttemptState(), attempts: 0, realtimeCohort: { id: 42, teamId: team.id } })

            await consumer.processResolverJob(job)

            expect(getBlastRadiusPersons).not.toHaveBeenCalled()
            expect(queueLogs).toHaveBeenCalledWith(
                [
                    expect.objectContaining({
                        message:
                            "Batch resolver failed: Sending to a realtime cohort isn't available here yet. Use a different audience.",
                    }),
                ],
                'hog_flow'
            )
        })

        it('keeps the generic reason for a failure that is not a timeout', async () => {
            const queueLogs = jest.fn()
            const consumer = buildConsumer(jest.fn().mockRejectedValue(new Error('network down')), queueLogs)
            const job = buildJob(lastAttemptState())

            await consumer.processResolverJob(job)

            expect(queueLogs).toHaveBeenCalledWith(
                [
                    expect.objectContaining({
                        message: `Batch resolver failed: Audience fetch failed permanently after ${MAX_RESOLVER_ATTEMPTS} attempts`,
                    }),
                ],
                'hog_flow'
            )
        })
    })
})