:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { HogFlow } from '~/cdp/schema/hogflow'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { Team } from '~/types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { HOG_FLOW_MASK_EXAMPLES } from '../_tests/examples'
import { CdpOutput } from '../cdp-services'
import {
BatchResolverState,
MAX_RESOLVER_ATTEMPTS,
deserializeResolverState,
serializeResolverState,
} from '../services/hogflows/batch-resolver.types'
import { AudienceFetchTimeoutError } from '../services/hogflows/hogflow-batch-person-query.service'
import {
HogInvocationResultRow,
HogInvocationResultsService,
} from '../services/monitoring/hog-invocation-results.service'
import { CyclotronJobInvocationHogFlow } from '../types'
import {
CdpCyclotronWorkerBatchResolve,
buildAccountHogFlowInvocation,
} from './cdp-cyclotron-worker-batch-resolve.consumer'
describe('CdpCyclotronWorkerBatchResolve', () => {
const team = { id: 123, name: 'Test team' } as Team
// A non-default version, so the flowVersion assertion can't pass by accident.
const hogFlow: HogFlow = { ...new FixtureHogFlowBuilder().withTeamId(team.id).build(), version: 4 }
describe('buildAccountHogFlowInvocation', () => {
it('carries the account group key and no person', () => {
const invocation = buildAccountHogFlowInvocation({
siteUrl: 'https://us.posthog.com',
parentRunId: 'batch-job-1',
team,
hogFlow,
externalId: 'acme-1',
groupType: 'customer',
defaultVariables: { greeting: 'hi' },
})
const state = invocation.state as CyclotronJobInvocationHogFlow['state']
expect(state.event.event).toEqual('$batch_hog_flow_invocation')
// distinct_id doubles as the per-account key for invocation_results; it must NOT
// resolve to a person (the hogflow worker skips the lookup for account audiences).
expect(state.event.distinct_id).toEqual('acme-1')
expect(state.event.properties['$groups']).toEqual({ customer: 'acme-1' })
expect(state.personId).toBeUndefined()
// The stamp is what the hogflow worker trusts when the live trigger has been
// edited to a person audience while these children were still queued.
expect(state.accountAudience).toBe(true)
// Account broadcasts convert long after the send, so the run has to carry the version
// that sent or the conversion is credited to whatever is published by then.
expect(state.flowVersion).toBe(4)
expect(state.customerTaskIdempotencyVersion).toBe(1)
expect(state.variables).toEqual({ greeting: 'hi' })
expect(state.sensitive).toBeUndefined()
expect(invocation.parentRunId).toEqual('batch-job-1')
expect(invocation.queue).toEqual('hogflow')
expect((invocation as any).person).toBeUndefined()
})
// Two things key off this object. HogMaskerService.filterByMasking() only recognizes an
// invocation as a hog flow invocation (and applies trigger_masking) when it carries a
// `hogFlow`, and the invocation-results service classifies rows by the same shape — so a
// regression here silently stops masking batch runs and drops their rows out of the
// workflow invocations list, which filters on hog_flow.
it('attaches the hogFlow so trigger_masking can be applied by the batch resolver', () => {
const maskedHogFlow: HogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withTriggerMasking(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)
.build()
const invocation = buildAccountHogFlowInvocation({
siteUrl: 'https://us.posthog.com',
parentRunId: 'batch-job-1',
team,
hogFlow: maskedHogFlow,
externalId: 'acme-1',
groupType: 'customer',
defaultVariables: {},
})
expect(invocation.hogFlow).toBe(maskedHogFlow)
expect(invocation.hogFlow.trigger_masking).toEqual(HOG_FLOW_MASK_EXAMPLES.everyTime.trigger_masking)
})
it('stamps a protected message step on the run so a recovery path with only the workflow id keeps it out of the snapshot', () => {
const protectedFlow: HogFlow = {
...hogFlow,
actions: [
...hogFlow.actions,
{
id: 'message',
type: 'function_push',
config: { template_id: 'template-native-push', sensitive: true, inputs: {} },
} as any,
],
}
const invocation = buildAccountHogFlowInvocation({
siteUrl: 'https://us.posthog.com',
parentRunId: 'batch-job-1',
team,
hogFlow: protectedFlow,
externalId: 'acme-1',
groupType: 'customer',
defaultVariables: {},
})
expect(invocation.state.sensitive).toBe(true)
})
})
describe('processOnePage run-level monitoring', () => {
const BATCH_JOB_ID = 'batch-job-1'
let consumer: CdpCyclotronWorkerBatchResolve
let queueAppMetrics: jest.Mock
let outputs: jest.Mocked<IngestionOutputs<CdpOutput>>
let rowsService: HogInvocationResultsService
let bulkCreateAndCheckIn: jest.Mock
let filterByMasking: jest.Mock
let release: jest.Mock
const state: BatchResolverState = {
batchJobId: BATCH_JOB_ID,
teamId: team.id,
hogFlowId: hogFlow.id,
cursor: null,
filters: { properties: [] },
maxAudienceSize: 100,
totalEnqueued: 0,
pagesProcessed: 0,
attempts: 0,
variables: {},
startedAt: '2026-08-11T00:00:00.000Z',
}
const processPage = async (): Promise<void> =>
await (consumer as any).processOnePage({ bulkCreateAndCheckIn, reschedule: jest.fn() }, state)
// The rows the real service produced on flush. Asserting on these rather than on mock
// call args is what catches a misclassification: function_kind is derived structurally
// inside the service ('hogFlow' in invocation), which a mocked service never evaluates.
const producedRows = (): HogInvocationResultRow[] =>
outputs.produce.mock.calls.map(
([, message]) => parseJSON(message.value!.toString('utf8')) as HogInvocationResultRow
)
const triggeredMetrics = (): any[] =>
queueAppMetrics.mock.calls.flatMap(([metrics]) => metrics).filter((m) => m.metric_name === 'triggered')
beforeEach(() => {
queueAppMetrics = jest.fn()
bulkCreateAndCheckIn = jest.fn().mockResolvedValue({ newJobIds: [] })
release = jest.fn().mockResolvedValue(undefined)
// Masking itself is covered by hog-masker.service.test.ts; stubbing the partition
// here is what lets each case fix which runs are masked and assert how the consumer
// routes them.
filterByMasking = jest.fn((invocations) => ({ masked: [], notMasked: invocations, release }))
outputs = {
produce: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<IngestionOutputs<CdpOutput>>
rowsService = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: true })
// The base constructor builds redis/valkey-backed services, so assemble the
// instance directly — this exercises processOnePage without any live boundary.
// The rows service is real (only its produce boundary is mocked) because the
// regressions to catch live inside it: how it classifies and stamps the
// invocations this consumer builds.
consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
Object.assign(consumer, {
config: { SITE_URL: 'https://us.posthog.com' },
deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
hogFlowBatchPersonQueryService: {
getBlastRadiusPersons: jest.fn().mockResolvedValue({
users_affected: ['person-1', 'person-2'],
cursor: null,
has_more: false,
}),
},
hogMasker: { filterByMasking },
hogFunctionMonitoringService: { queueAppMetrics, queueLogs: jest.fn() },
invocationResultsService: { invocationResultsRowsService: rowsService },
})
})
it('pages a realtime cohort audience from realtime membership, not the Django audience query', async () => {
const getMemberPage = jest.fn().mockResolvedValue({
ids: ['member-1', 'member-2', 'member-3'],
cursor: 'member-3',
has_more: false,
})
Object.assign(consumer, {
config: { SITE_URL: 'https://us.posthog.com', BEHAVIORAL_COHORTS_DATABASE_URL: 'postgres://cohorts' },
realtimeCohortAudience: { getMemberPage },
})
await (consumer as any).processOnePage(
{ bulkCreateAndCheckIn, reschedule: jest.fn() },
{ ...state, realtimeCohort: { id: 42, teamId: 7 } }
)
// Membership rows belong to the cohort's own team, which can differ from the workflow's.
expect(getMemberPage).toHaveBeenCalledWith(7, 42, null)
expect((consumer as any).hogFlowBatchPersonQueryService.getBlastRadiusPersons).not.toHaveBeenCalled()
expect(triggeredMetrics()).toHaveLength(3)
})
it('counts each enrolled person as a triggered run against the batch job', async () => {
await processPage()
const metrics = triggeredMetrics()
expect(metrics).toHaveLength(2)
for (const metric of metrics) {
// Keyed on the batch job, not the workflow — that is where the batch metrics
// view looks, and the workflow id would silently read as zero runs started.
expect(metric.app_source_id).toEqual(BATCH_JOB_ID)
expect(metric.team_id).toEqual(team.id)
expect(metric.metric_kind).toEqual('other')
expect(metric.count).toEqual(1)
// Run-level metrics carry no instance id; a step id here would hide them from the
// started/in-progress counters, which filter on the empty instance.
expect(metric.instance_id).toBeUndefined()
}
})
it('records each enrolled person as a running hog_flow invocation so parked runs are listable', async () => {
await processPage()
await rowsService.flush()
const rows = producedRows()
expect(rows).toHaveLength(2)
for (const row of rows) {
// hog_flow, not hog_function: the workflow invocations API only reads rows with
// this kind, so a misclassified row is invisible even though it was written.
expect(row.function_kind).toEqual('hog_flow')
expect(row.function_id).toEqual(hogFlow.id)
expect(row.parent_run_id).toEqual(BATCH_JOB_ID)
expect(row.status).toEqual('running')
}
})
it.each([
['an ordinary workflow', false, undefined],
// A recovery path that holds only the workflow id reads this stamp to keep the message out of the snapshot.
['a workflow with a protected message step', true, true],
])('persists the enqueue time and run stamps for new person runs of %s', async (_name, sensitive, stamp) => {
// The stamp is written onto the invocation by queueLifecycleRow, so it only lands in
// cyclotron if that runs before the state is serialized. Out of order, the terminal
// row written when the run wakes records the wake time and wins the argMax collapse.
const flow: HogFlow = {
...hogFlow,
actions: [
...hogFlow.actions,
{
id: 'message',
type: 'function_push',
config: { template_id: 'template-native-push', sensitive, inputs: {} },
} as any,
],
}
;(consumer as any).hogFlowManager.getHogFlow.mockResolvedValue(flow)
await processPage()
const { newJobs } = bulkCreateAndCheckIn.mock.calls[0][0]
expect(newJobs).toHaveLength(2)
for (const job of newJobs) {
const jobState = parseJSON(job.state.toString()).state
expect(jobState).toMatchObject({
firstScheduledAt: expect.any(String),
actionStepCount: 0,
customerTaskIdempotencyVersion: 1,
})
expect(jobState.sensitive).toBe(stamp)
}
})
it('leaves masked runs out of both the triggered count and the invocations list', async () => {
// A masked run is never enqueued, so counting it as started would strand it in the
// in-progress figure forever, and its running row would never get a terminal row.
filterByMasking.mockImplementation((invocations: CyclotronJobInvocationHogFlow[]) => ({
masked: [invocations[0]],
notMasked: [invocations[1]],
release,
}))
await processPage()
await rowsService.flush()
expect(triggeredMetrics()).toHaveLength(1)
expect(producedRows()).toHaveLength(1)
})
it('drops the queued rows and releases the mask claims when the page fails to commit', async () => {
// The commit is atomic with the cursor advance, so a failure replays the whole page.
// Rows left queued here would be written twice.
bulkCreateAndCheckIn.mockRejectedValue(new Error('postgres is down'))
await expect(processPage()).rejects.toThrow('postgres is down')
await rowsService.flush()
expect(producedRows()).toHaveLength(0)
expect(release).toHaveBeenCalledTimes(1)
expect(triggeredMetrics()).toHaveLength(0)
})
it('terminates the resolver instead of scheduling another page when the check-in is refused by a cancel', async () => {
// The engine refuses the page when a cancel flag landed mid-build: nothing
// committed, so the queued running rows and mask claims must be undone (these
// children will never run) and the resolver must terminate, not retry.
bulkCreateAndCheckIn.mockResolvedValue({ newJobIds: [], cancelRequested: true })
const cancel = jest.fn().mockResolvedValue(undefined)
const reschedule = jest.fn()
Object.assign(consumer, {
hogFunctionMonitoringService: {
queueAppMetrics,
queueLogs: jest.fn(),
flush: jest.fn().mockResolvedValue(undefined),
},
})
await (consumer as any).processOnePage({ bulkCreateAndCheckIn, reschedule, cancel }, state)
await rowsService.flush()
expect(producedRows()).toHaveLength(0)
expect(release).toHaveBeenCalledTimes(1)
expect(triggeredMetrics()).toHaveLength(0)
expect(cancel).toHaveBeenCalledTimes(1)
expect(reschedule).not.toHaveBeenCalled()
})
})
describe('cancel-flagged resolver jobs', () => {
it('terminates on dequeue without fetching a page, even when the state is unparseable', async () => {
const getBlastRadiusPersons = jest.fn()
const queueLogs = jest.fn()
const flush = jest.fn().mockResolvedValue(undefined)
const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
Object.assign(consumer, {
hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
hogFunctionMonitoringService: { queueLogs, flush },
})
const cancel = jest.fn().mockResolvedValue(undefined)
const job = {
id: 'job-1',
teamId: team.id,
functionId: hogFlow.id,
parentRunId: 'batch-job-1',
cancelRequestedAt: DateTime.now(),
// Unparseable on purpose: the cancel must not depend on state surviving
// schema drift across deploys.
state: Buffer.from('not json'),
cancel,
}
await (consumer as any).processResolverJob(job)
expect(cancel).toHaveBeenCalledTimes(1)
expect(getBlastRadiusPersons).not.toHaveBeenCalled()
// The stop is visible on the batch run's log stream, keyed by parent run id.
expect(queueLogs).toHaveBeenCalledTimes(1)
const [logs] = queueLogs.mock.calls[0]
expect(logs[0].log_source_id).toEqual('batch-job-1')
expect(flush).toHaveBeenCalled()
})
})
describe('resolver job lock heartbeats', () => {
afterEach(() => {
jest.useRealTimers()
})
it.each([false, true])('clears the heartbeat timer (heartbeat fails: %s)', async (heartbeatFails) => {
jest.useFakeTimers()
const state: BatchResolverState = {
batchJobId: 'batch-job-hb',
teamId: team.id,
hogFlowId: hogFlow.id,
cursor: null,
filters: { properties: [] },
maxAudienceSize: 100,
totalEnqueued: 0,
pagesProcessed: 0,
attempts: 0,
variables: {},
startedAt: '2026-08-11T00:00:00.000Z',
}
// Long enough for two heartbeat ticks, inside the 30s default fetch budget.
const getBlastRadiusPersons = jest.fn().mockImplementation(async () => {
await new Promise<void>((resolve) => {
setTimeout(() => resolve(), 25_000)
})
return { users_affected: [], cursor: null, has_more: false }
})
const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
Object.assign(consumer, {
config: { SITE_URL: 'https://us.posthog.com' },
deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
hogMasker: {
filterByMasking: jest.fn((invocations) => ({
masked: [],
notMasked: invocations,
release: jest.fn().mockResolvedValue(undefined),
})),
},
hogFunctionMonitoringService: {
queueAppMetrics: jest.fn(),
queueLogs: jest.fn(),
flush: jest.fn().mockResolvedValue(undefined),
},
invocationResultsService: {
invocationResultsRowsService: { flush: jest.fn().mockResolvedValue(undefined) },
},
})
const heartbeat = heartbeatFails
? jest.fn().mockRejectedValue(new Error('Heartbeat unavailable'))
: jest.fn().mockResolvedValue(undefined)
const job = {
id: 'job-heartbeat',
teamId: team.id,
functionId: hogFlow.id,
parentRunId: 'batch-job-hb',
cancelRequestedAt: null,
state: serializeResolverState(state),
heartbeat,
bulkCreateAndCheckIn: jest.fn().mockResolvedValue({ newJobIds: [] }),
reschedule: jest.fn().mockResolvedValue(undefined),
ack: jest.fn().mockResolvedValue(undefined),
fail: jest.fn().mockResolvedValue(undefined),
}
const processPromise = (consumer as any).processResolverJob(job)
await jest.advanceTimersByTimeAsync(25_000)
await processPromise
expect(heartbeat).toHaveBeenCalledTimes(2)
expect(getBlastRadiusPersons).toHaveBeenCalledTimes(1)
expect(job.bulkCreateAndCheckIn).toHaveBeenCalledTimes(1)
expect(jest.getTimerCount()).toBe(0)
await jest.advanceTimersByTimeAsync(20_000)
expect(heartbeat).toHaveBeenCalledTimes(2)
})
})
describe('permanent audience fetch failure', () => {
const buildConsumer = (getBlastRadiusPersons: jest.Mock, queueLogs: jest.Mock): any => {
const consumer = Object.create(CdpCyclotronWorkerBatchResolve.prototype)
Object.assign(consumer, {
config: { SITE_URL: 'https://us.posthog.com' },
deps: { teamManager: { getTeam: jest.fn().mockResolvedValue(team) } },
hogFlowManager: { getHogFlow: jest.fn().mockResolvedValue(hogFlow) },
hogFlowBatchPersonQueryService: { getBlastRadiusPersons },
hogFunctionMonitoringService: {
queueAppMetrics: jest.fn(),
queueLogs,
flush: jest.fn().mockResolvedValue(undefined),
},
invocationResultsService: {
invocationResultsRowsService: { flush: jest.fn().mockResolvedValue(undefined) },
},
})
return consumer
}
const buildJob = (state: BatchResolverState): any => ({
id: 'job-last-attempt',
teamId: team.id,
functionId: hogFlow.id,
parentRunId: state.batchJobId,
cancelRequestedAt: null,
state: serializeResolverState(state),
heartbeat: jest.fn().mockResolvedValue(undefined),
bulkCreateAndCheckIn: jest.fn().mockResolvedValue({ newJobIds: [] }),
reschedule: jest.fn().mockResolvedValue(undefined),
ack: jest.fn().mockResolvedValue(undefined),
fail: jest.fn().mockResolvedValue(undefined),
})
// One attempt short of the cap, so the failure in each case is the permanent one.
const lastAttemptState = (): BatchResolverState => ({
batchJobId: 'batch-job-fail',
teamId: team.id,
hogFlowId: hogFlow.id,
cursor: null,
filters: { properties: [] },
maxAudienceSize: 100,
totalEnqueued: 0,
pagesProcessed: 0,
attempts: MAX_RESOLVER_ATTEMPTS - 1,
variables: {},
startedAt: '2026-08-11T00:00:00.000Z',
})
it.each([
['before its deadline parks it again without reading members', 3600 * 1000, false],
['at its deadline fails the run', -1000, true],
])('a realtime cohort resolver woken %s', async (_name, untilOffsetMs, fails) => {
const getBlastRadiusPersons = jest.fn()
const queueLogs = jest.fn()
const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)
const waitUntil = new Date(Date.now() + untilOffsetMs).toISOString()
const job = buildJob({
...lastAttemptState(),
attempts: 0,
realtimeCohort: { id: 7, teamId: team.id, waitUntil },
})
await consumer.processResolverJob(job)
expect(getBlastRadiusPersons).not.toHaveBeenCalled()
const [{ scheduledAt, state }] = job.reschedule.mock.calls[0]
if (fails) {
expect(deserializeResolverState(state).pendingTerminal).toEqual('failed')
expect(queueLogs.mock.calls[0][0][0].message).toContain("didn't finish calculating")
} else {
expect(scheduledAt.toISOString()).toEqual(waitUntil)
expect(state).toBeUndefined()
}
})
it('tells the customer what to change when the audience query timed out', async () => {
const queueLogs = jest.fn()
const consumer = buildConsumer(
jest.fn().mockRejectedValue(new AudienceFetchTimeoutError('user_blast_radius_persons', 30_000)),
queueLogs
)
const job = buildJob(lastAttemptState())
await consumer.processResolverJob(job)
expect(queueLogs).toHaveBeenCalledWith(
[
expect.objectContaining({
message:
'Batch resolver failed: Audience query timed out after 30s on the last of ' +
`${MAX_RESOLVER_ATTEMPTS} attempts. Use fewer or simpler audience filters, or a smaller audience.`,
}),
],
'hog_flow'
)
})
it('fails a realtime cohort run at once when the resolver has no behavioral cohorts database', async () => {
// Without its own URL the router reads the main database, so retrying would never succeed.
const queueLogs = jest.fn()
const getBlastRadiusPersons = jest.fn()
const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)
const job = buildJob({ ...lastAttemptState(), attempts: 0, realtimeCohort: { id: 42, teamId: team.id } })
await consumer.processResolverJob(job)
expect(getBlastRadiusPersons).not.toHaveBeenCalled()
expect(queueLogs).toHaveBeenCalledWith(
[
expect.objectContaining({
message:
"Batch resolver failed: Sending to a realtime cohort isn't available here yet. Use a different audience.",
}),
],
'hog_flow'
)
})
it('keeps the generic reason for a failure that is not a timeout', async () => {
const queueLogs = jest.fn()
const consumer = buildConsumer(jest.fn().mockRejectedValue(new Error('network down')), queueLogs)
const job = buildJob(lastAttemptState())
await consumer.processResolverJob(job)
expect(queueLogs).toHaveBeenCalledWith(
[
expect.objectContaining({
message: `Batch resolver failed: Audience fetch failed permanently after ${MAX_RESOLVER_ATTEMPTS} attempts`,
}),
],
'hog_flow'
)
})
})
})