Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): park a batch resolver until its realtime cohort is ready (#118444)

Harley Alexander24a2e3f8f0ac
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 71791a375815..8fab4922bf2c 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -1271,6 +1271,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
disconnect: jest.fn().mockResolvedValue(undefined),
}
@@ -1319,6 +1320,51 @@ describe('CDP API', () => {
}
})
+ it.each([
+ ['a deadline', { id: 7, team_id: 1, wait_until: '2026-10-17T10:00:00Z' }, 200, '2026-10-17T10:00:00.000Z'],
+ ['no deadline', { id: 7, team_id: 1 }, 200, undefined],
+ ['an unreadable deadline', { id: 7, team_id: 1, wait_until: 'tomorrow-ish' }, 400, undefined],
+ ['a non-string deadline', { id: 7, team_id: 1, wait_until: 1234 }, 400, undefined],
+ ])(
+ 'parks a realtime cohort resolver only on a valid deadline: %s',
+ async (_desc, cohort, status, parkedUntil) => {
+ const createJobMock = jest.fn().mockResolvedValue('resolver-job-id')
+ api['batchResolverProducer'] = {
+ createJob: createJobMock,
+ countInFlightJobs: jest.fn(),
+ rescheduleParkedJobs: jest.fn(),
+ cancelJobs: jest.fn(),
+ resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
+ disconnect: jest.fn().mockResolvedValue(undefined),
+ }
+
+ try {
+ const res = await supertest(app)
+ .post(
+ `/api/projects/${batchHogFlow.team_id}/hog_flows/${batchHogFlow.id}/batch_invocations/job-793`
+ )
+ .send({ realtime_cohort: cohort })
+
+ expect(res.status).toEqual(status)
+ if (status !== 200) {
+ expect(createJobMock).not.toHaveBeenCalled()
+ return
+ }
+ const arg = createJobMock.mock.calls[0][0]
+ expect(arg.scheduled?.toISOString()).toEqual(parkedUntil)
+ const state = parseJSON((arg.state as Buffer).toString('utf-8')) as Record<string, any>
+ expect(state.realtimeCohort).toEqual({
+ id: 7,
+ teamId: 1,
+ ...(parkedUntil ? { waitUntil: parkedUntil } : {}),
+ })
+ } finally {
+ api['batchResolverProducer'] = null
+ }
+ }
+ )
+
it('resolves the audience from the posted snapshot, not the live trigger filters', async () => {
// The snapshot was validated at confirm time - re-reading the trigger here would let an
// edit racing the dispatch widen the send past what was previewed.
@@ -1331,6 +1377,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
disconnect: jest.fn().mockResolvedValue(undefined),
}
@@ -1383,6 +1430,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
disconnect: jest.fn().mockResolvedValue(undefined),
}
@@ -1443,6 +1491,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
disconnect: jest.fn().mockResolvedValue(undefined),
}
@@ -1507,6 +1556,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
disconnect: jest.fn().mockResolvedValue(undefined),
}
@@ -1681,6 +1731,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
}
countHogFlow = await insertHogFlow({
@@ -1775,6 +1826,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: mockRescheduleParkedJobs,
cancelJobs: jest.fn(),
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
}
rescheduleHogFlow = await insertHogFlow({
@@ -1951,6 +2003,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: jest.fn(),
resumeParkedSteps: mockResumeParkedSteps,
+ resumeParkedBatchResolver: jest.fn(),
}
})
@@ -2161,6 +2214,74 @@ describe('CDP API', () => {
})
})
+ describe('hogflow batch job resume', () => {
+ let mockResume: jest.Mock
+ const batchJobId = new UUIDT().toString().toLowerCase()
+ const flowId = new UUIDT().toString().toLowerCase()
+
+ // Raw audience literal and Python claim names: the wire contract with Django's
+ // WORKFLOWS_RESUME_BATCH_JWT_PURPOSE.
+ const mintBatchResumeToken = (
+ claims: Record<string, unknown>,
+ { secret = 'local-dev-workflows-step-resume-jwt', audience = 'posthog:workflows:resume_batch' } = {}
+ ) => jwt.sign(claims, secret, { audience, expiresIn: '2m' })
+ const resumeAuth = (teamId: number, claims: Record<string, unknown> = {}) => ({
+ Authorization: `Bearer ${mintBatchResumeToken({
+ team_id: teamId,
+ hog_flow_id: flowId,
+ batch_job_id: batchJobId,
+ ...claims,
+ })}`,
+ })
+
+ beforeEach(() => {
+ mockResume = jest.fn()
+ api['batchResolverProducer'] = {
+ createJob: jest.fn(),
+ disconnect: jest.fn(),
+ countInFlightJobs: jest.fn(),
+ rescheduleParkedJobs: jest.fn(),
+ cancelJobs: jest.fn(),
+ resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: mockResume,
+ }
+ })
+
+ afterEach(() => {
+ api['batchResolverProducer'] = null
+ })
+
+ it.each([
+ ['resumed', 200],
+ ['already_resumed', 200],
+ ['job_running', 409],
+ ['job_completed', 200],
+ ['job_failed', 200],
+ ['job_missing', 200],
+ ])('answers %s with %s, so only a running resolver is retried', async (outcome, status) => {
+ mockResume.mockResolvedValue(outcome)
+
+ const res = await supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/batch_jobs/${batchJobId}/resume`)
+ .set(resumeAuth(team.id))
+ .send({})
+
+ expect(res.status).toEqual(status)
+ expect(res.body).toEqual({ outcome })
+ expect(mockResume).toHaveBeenCalledWith({ teamId: team.id, functionId: flowId, batchJobId })
+ })
+
+ it("rejects another run's token", async () => {
+ const res = await supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/batch_jobs/${batchJobId}/resume`)
+ .set(resumeAuth(team.id, { batch_job_id: new UUIDT().toString() }))
+ .send({})
+
+ expect(res.status).toEqual(401)
+ expect(mockResume).not.toHaveBeenCalled()
+ })
+ })
+
describe('hogflow cancel invocations auth', () => {
let mockCancelJobs: jest.Mock
@@ -2188,6 +2309,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: mockCancelJobs,
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
}
})
@@ -2324,6 +2446,7 @@ describe('CDP API', () => {
rescheduleParkedJobs: jest.fn(),
cancelJobs: mockCancelJobs,
resumeParkedSteps: jest.fn(),
+ resumeParkedBatchResolver: jest.fn(),
}
})
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 0bd16cbe3e8a..65a0120794dd 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -39,6 +39,7 @@ import type { CyclotronV2JobProducer } from './services/cyclotron-v2'
import { HogExecutorAsyncService, HogExecutorExecuteAsyncOptions } from './services/hog-executor-async.service'
import { MAX_ASYNC_STEPS } from './services/hog-executor.service'
import { HogInputsService } from './services/hog-inputs.service'
+import { counterBatchResolverParked } from './services/hogflows/batch-resolver-resume'
import {
BatchResolverState,
HOGFLOW_BATCH_RESOLVE_QUEUE,
@@ -136,14 +137,24 @@ function canDedupeByEmail(hogFlow: { actions?: unknown }): boolean {
export type CdpApiConfig = PluginsServerConfig
export type CdpApiDeps = CdpConsumerBaseDeps
-function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort'] {
+// Null means the cohort asks to wait but its deadline is unreadable. Queuing it unparked would send to a
+// cohort that is still calculating, so the caller rejects it.
+function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort'] | null {
if (typeof value !== 'object' || value === null) {
return undefined
}
- const { id, team_id } = value as { id?: unknown; team_id?: unknown }
- return Number.isInteger(id) && Number.isInteger(team_id)
- ? { id: id as number, teamId: team_id as number }
- : undefined
+ const { id, team_id, wait_until } = value as { id?: unknown; team_id?: unknown; wait_until?: unknown }
+ if (!Number.isInteger(id) || !Number.isInteger(team_id)) {
+ return undefined
+ }
+ if (wait_until === undefined || wait_until === null) {
+ return { id: id as number, teamId: team_id as number }
+ }
+ const waitUntil = typeof wait_until === 'string' ? new Date(wait_until) : null
+ if (!waitUntil || isNaN(waitUntil.getTime())) {
+ return null
+ }
+ return { id: id as number, teamId: team_id as number, waitUntil: waitUntil.toISOString() }
}
export class CdpApi {
@@ -178,6 +189,7 @@ export class CdpApi {
private stepResumeJwt: ScopedServiceJwt
private transactionalEmailJwt: ScopedServiceJwt
private hogFunctionMonitoringService: HogFunctionMonitoringService
+ private resumeBatchJwt: ScopedServiceJwt
constructor(
private config: PluginsServerConfig,
@@ -250,6 +262,12 @@ export class CdpApi {
PosthogJwtAudience.WORKFLOWS_SEND_TRANSACTIONAL_EMAIL,
config.WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET || ''
)
+ // The same workers that wake parked steps wake parked batch resolvers, so they share the
+ // key. The audience keeps a step-resume token from waking a batch, and the reverse.
+ this.resumeBatchJwt = new ScopedServiceJwt(
+ PosthogJwtAudience.WORKFLOWS_RESUME_BATCH,
+ config.WORKFLOWS_STEP_RESUME_JWT_SECRET || ''
+ )
}
public get service(): PluginServerService {
@@ -330,6 +348,10 @@ export class CdpApi {
'/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/cancel',
asyncHandler(this.postHogFlowCancelBatchJob)
)
+ router.post(
+ '/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/resume',
+ asyncHandler(this.postHogFlowResumeBatchJob)
+ )
router.post('/api/projects/:team_id/workflow_steps/resume', asyncHandler(this.postWorkflowStepResume))
router.post(
'/api/projects/:team_id/hog_flows/:id/transactional_email',
@@ -1310,6 +1332,50 @@ export class CdpApi {
}
}
+ /**
+ * Wake a batch resolver parked until its realtime cohort finishes calculating. Django calls this
+ * once the cohort is ready. `already_resumed` is a success: a second wake for the same run is a no-op.
+ * Only a running resolver answers 409, so the caller retries. A finished or removed one already ran, so
+ * a wake retried after a lost response gets a 200 and the caller stops.
+ */
+ private postHogFlowResumeBatchJob = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
+ try {
+ if (!this.batchResolverProducer) {
+ return res.status(503).json({
+ error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',
+ })
+ }
+ if (!this.resumeBatchJwt.enabled) {
+ return res.status(503).json({
+ error: 'Workflows scoped auth not configured (WORKFLOWS_STEP_RESUME_JWT_SECRET unset)',
+ })
+ }
+ const { team_id, id, batch_job_id } = req.params
+ // batch_job_id is pinned in the claims, so a captured token wakes exactly one run.
+ if (!this.verifyScopedWorkflowJwt(this.resumeBatchJwt, req, res, 'resume', { batch_job_id })) {
+ return
+ }
+ const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)
+ if (!team) {
+ return res.status(404).json({ error: 'Team not found' })
+ }
+ if (batch_job_id.length > 200) {
+ return res.status(400).json({ error: 'batch_job_id is too long' })
+ }
+ const outcome = await this.batchResolverProducer.resumeParkedBatchResolver({
+ teamId: team.id,
+ functionId: id,
+ batchJobId: batch_job_id,
+ })
+ return res.status(outcome === 'job_running' ? 409 : 200).json({ outcome })
+ } catch (e) {
+ logger.error('Error resuming hog flow batch job', {
+ error: e instanceof Error ? e.message : String(e),
+ })
+ return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })
+ }
+ }
+
// Wake the parked workflow step that dispatched a task run, with the run's outcome. Django
// calls this from a retrying Celery task when the run reaches a terminal status. One resume
// per call; the outcome tells the caller whether to retry (409: the worker still holds the
@@ -1527,6 +1593,11 @@ export class CdpApi {
throw new Error('Batch resolver producer is not configured (missing CYCLOTRON_NODE_DATABASE_URL)')
}
+ const realtimeCohort = parseRealtimeCohort(req.body.realtime_cohort)
+ if (realtimeCohort === null) {
+ return res.status(400).json({ error: 'realtime_cohort.wait_until must be an ISO 8601 date' })
+ }
+
const snapshotFilters: BatchResolverState['filters'] | undefined = req.body.filters
const audienceType = snapshotFilters?.audience_type ?? hogFlow.trigger.filters.audience_type
// A snapshot saved before assignment statuses existed carries assignee ids or the legacy
@@ -1564,7 +1635,7 @@ export class CdpApi {
// dedupe than dedupe wrongly. Also skip when the flow has no email action at all.
// Account audiences carry no person (and external ids are already unique), so never dedupe.
dedupeKey: audienceType !== 'accounts' && canDedupeByEmail(hogFlow) ? ('email' as const) : undefined,
- realtimeCohort: parseRealtimeCohort(req.body.realtime_cohort),
+ realtimeCohort,
maxAudienceSize: maxAudienceSize ?? this.config.CDP_BATCH_WORKFLOW_MAX_AUDIENCE_SIZE,
cursor: null,
totalEnqueued: 0,
@@ -1572,13 +1643,19 @@ export class CdpApi {
attempts: 0,
startedAt: new Date().toISOString(),
}
+ const waitUntil = initialState.realtimeCohort?.waitUntil
await this.batchResolverProducer.createJob({
teamId: team.id,
queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
parentRunId: parent_run_id,
functionId: hogFlow.id,
state: serializeResolverState(initialState),
+ // A cohort that is still calculating parks the job until Django resumes it.
+ ...(waitUntil ? { scheduled: new Date(waitUntil) } : {}),
})
+ if (waitUntil) {
+ counterBatchResolverParked.inc()
+ }
res.json({ status: 'queued' })
} catch (e) {
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts
index a53ee91c03c8..750306170d67 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts
@@ -11,6 +11,7 @@ import { CdpOutput } from '../cdp-services'
import {
BatchResolverState,
MAX_RESOLVER_ATTEMPTS,
+ deserializeResolverState,
serializeResolverState,
} from '../services/hogflows/batch-resolver.types'
import { AudienceFetchTimeoutError } from '../services/hogflows/hogflow-batch-person-query.service'
@@ -453,6 +454,33 @@ describe('CdpCyclotronWorkerBatchResolve', () => {
startedAt: '2026-08-11T00:00:00.000Z',
})
+ it.each([
+ ['before its deadline parks it again without reading members', 3600 * 1000, false],
+ ['at its deadline fails the run', -1000, true],
+ ])('a realtime cohort resolver woken %s', async (_name, untilOffsetMs, fails) => {
+ const getBlastRadiusPersons = jest.fn()
+ const queueLogs = jest.fn()
+ const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)
+ const waitUntil = new Date(Date.now() + untilOffsetMs).toISOString()
+ const job = buildJob({
+ ...lastAttemptState(),
+ attempts: 0,
+ realtimeCohort: { id: 7, teamId: team.id, waitUntil },
+ })
+
+ await consumer.processResolverJob(job)
+
+ expect(getBlastRadiusPersons).not.toHaveBeenCalled()
+ const [{ scheduledAt, state }] = job.reschedule.mock.calls[0]
+ if (fails) {
+ expect(deserializeResolverState(state).pendingTerminal).toEqual('failed')
+ expect(queueLogs.mock.calls[0][0][0].message).toContain("didn't finish calculating")
+ } else {
+ expect(scheduledAt.toISOString()).toEqual(waitUntil)
+ expect(state).toBeUndefined()
+ }
+ })
+
it('tells the customer what to change when the audience query timed out', async () => {
const queueLogs = jest.fn()
const consumer = buildConsumer(
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts
index 69d52c8c06cb..bdab41da474e 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts
@@ -64,6 +64,12 @@ const counterBatchHogFlowResolverPagesProcessed = new Counter({
labelNames: ['outcome'], // success | fetch_failure | terminal_write_failure | invalid_state
})
+const counterBatchHogFlowResolverCohortWait = new Counter({
+ name: 'cdp_batch_hog_flow_resolver_cohort_wait',
+ help: 'Batch resolvers parked on a realtime cohort that is still calculating, by what woke them',
+ labelNames: ['outcome'], // early_wake | timed_out
+})
+
const counterBatchHogFlowResolverJobs = new Counter({
name: 'cdp_batch_hog_flow_resolver_jobs',
help: 'Batch hog flow resolver jobs by lifecycle outcome',
@@ -196,6 +202,12 @@ export class CdpCyclotronWorkerBatchResolve extends CdpConsumerBase<PluginsServe
return
}
+ const waitUntil = state.realtimeCohort?.waitUntil
+ if (waitUntil) {
+ await this.handleCohortWait(job, state, new Date(waitUntil))
+ return
+ }
+
if (state.totalEnqueued >= state.maxAudienceSize) {
await this.transitionToTruncatedTerminal(job, state)
return
@@ -220,6 +232,29 @@ export class CdpCyclotronWorkerBatchResolve extends CdpConsumerBase<PluginsServe
}
}
+ /**
+ * A resolver parked on a realtime cohort runs here only when nothing resumed it: Django clears
+ * the wait from its state when the cohort is ready. Before the deadline that is an early wake,
+ * so it parks again; at the deadline the cohort never finished, and the run fails.
+ */
+ private async handleCohortWait(
+ job: CyclotronV2DequeuedJob,
+ state: BatchResolverState,
+ waitUntil: Date
+ ): Promise<void> {
+ if (Date.now() < waitUntil.getTime()) {
+ counterBatchHogFlowResolverCohortWait.labels({ outcome: 'early_wake' }).inc()
+ await job.reschedule({ scheduledAt: waitUntil })
+ return
+ }
+ counterBatchHogFlowResolverCohortWait.labels({ outcome: 'timed_out' }).inc()
+ await this.transitionToFailedTerminal(
+ job,
+ state,
+ "the cohort didn't finish calculating in time, so nobody was sent this run. Start it again once the cohort is ready."
+ )
+ }
+
/**
* Terminate a cancel-flagged resolver job: no further pages, and no terminal status
* PUT — Django flips the batch job's status itself as part of the cancel request, and
diff --git a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
index d690b7eda845..188fb18c0c2b 100644
--- a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
+++ b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
@@ -4,6 +4,11 @@ import { v7 as uuidv7 } from 'uuid'
import { parseJSON } from '~/common/utils/json-parse'
+import {
+ HOGFLOW_BATCH_RESOLVE_QUEUE,
+ deserializeResolverState,
+ serializeResolverState,
+} from '../hogflows/batch-resolver.types'
import { HogInvocationResultsService } from '../monitoring/hog-invocation-results.service'
import { CyclotronV2Janitor, JANITOR_POISON_PILL_ERROR_KIND } from './janitor'
import { CyclotronV2Manager } from './manager'
@@ -869,6 +874,64 @@ describe('Cyclotron V2', () => {
})
})
+ describe('resumeParkedBatchResolver', () => {
+ const parkResolver = async (teamId: number, functionId: string, batchJobId: string): Promise<string> =>
+ await manager.createJob({
+ teamId,
+ queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
+ functionId,
+ parentRunId: batchJobId,
+ scheduled: new Date(Date.now() + 3600 * 1000),
+ state: serializeResolverState({
+ batchJobId,
+ teamId,
+ hogFlowId: functionId,
+ filters: { properties: [] },
+ variables: {},
+ realtimeCohort: { id: 7, teamId, waitUntil: new Date(Date.now() + 3600 * 1000).toISOString() },
+ maxAudienceSize: 100,
+ cursor: null,
+ totalEnqueued: 0,
+ pagesProcessed: 0,
+ attempts: 0,
+ startedAt: new Date().toISOString(),
+ }),
+ })
+
+ it('wakes the parked resolver of the batch and clears its wait, once', async () => {
+ const functionId = uuidv7()
+ const batchJobId = uuidv7()
+ const jobId = await parkResolver(1, functionId, batchJobId)
+
+ const first = await manager.resumeParkedBatchResolver({ teamId: 1, functionId, batchJobId })
+ const second = await manager.resumeParkedBatchResolver({ teamId: 1, functionId, batchJobId })
+
+ expect([first, second]).toEqual(['resumed', 'already_resumed'])
+ expect(await jobIsDue(jobId)).toBe(true)
+ const state = deserializeResolverState((await queryJob(jobId)).state)
+ // Without the wait cleared, the resolver would park again and time out.
+ expect(state.realtimeCohort).toEqual({ id: 7, teamId: 1 })
+ })
+
+ it.each([
+ ['another team', 2, true],
+ ['another batch', 1, false],
+ ])('leaves the resolver of %s parked', async (_name, teamId, sameBatch) => {
+ const functionId = uuidv7()
+ const parkedBatchJobId = uuidv7()
+ const jobId = await parkResolver(teamId, functionId, parkedBatchJobId)
+
+ const outcome = await manager.resumeParkedBatchResolver({
+ teamId: 1,
+ functionId,
+ batchJobId: sameBatch ? parkedBatchJobId : uuidv7(),
+ })
+
+ expect(outcome).toBe('job_missing')
+ expect(await jobIsDue(jobId)).toBe(false)
+ })
+ })
+
describe('cancelJobs', () => {
const FAR_FUTURE = () => new Date(Date.now() + 7 * 24 * 3600 * 1000)
diff --git a/nodejs/src/cdp/services/cyclotron-v2/manager.ts b/nodejs/src/cdp/services/cyclotron-v2/manager.ts
index f5bf5d68076f..be60f51ffcc7 100644
--- a/nodejs/src/cdp/services/cyclotron-v2/manager.ts
+++ b/nodejs/src/cdp/services/cyclotron-v2/manager.ts
@@ -6,6 +6,7 @@ import { isTransientPgError } from '~/common/utils/db/postgres'
import { logger } from '~/common/utils/logger'
import { sleep } from '~/common/utils/utils'
+import { BatchResolverResumeOutcome, resumeParkedBatchResolver } from '../hogflows/batch-resolver-resume'
import { StepResume, StepResumeOutcome, processStepResumes } from '../hogflows/step-resume.service'
import {
CYCLOTRON_COUNTER_MAX,
@@ -795,4 +796,12 @@ export class CyclotronV2Manager {
async resumeParkedSteps(teamId: number, resumes: StepResume[]): Promise<Map<string, StepResumeOutcome>> {
return await processStepResumes(this.pool, resumes, teamId)
}
+
+ async resumeParkedBatchResolver(options: {
+ teamId: number
+ functionId: string
+ batchJobId: string
+ }): Promise<BatchResolverResumeOutcome> {
+ return await resumeParkedBatchResolver(this.pool, options)
+ }
}
diff --git a/nodejs/src/cdp/services/cyclotron-v2/types.ts b/nodejs/src/cdp/services/cyclotron-v2/types.ts
index 3c2c43efa276..3ca1e830eb8e 100644
--- a/nodejs/src/cdp/services/cyclotron-v2/types.ts
+++ b/nodejs/src/cdp/services/cyclotron-v2/types.ts
@@ -1,6 +1,7 @@
import { DateTime } from 'luxon'
import { z } from 'zod'
+import type { BatchResolverResumeOutcome } from '~/cdp/services/hogflows/batch-resolver-resume'
import { StepResume, StepResumeOutcome } from '~/cdp/services/hogflows/step-resume.service'
export type CyclotronV2JobStatus = 'available' | 'running' | 'completed' | 'failed' | 'canceled'
@@ -195,6 +196,11 @@ export interface CyclotronV2JobProducer {
rescheduleParkedJobs(options: CyclotronV2RescheduleParkedOptions): Promise<CyclotronV2RescheduleParkedResult>
cancelJobs(options: CyclotronV2CancelJobsOptions): Promise<CyclotronV2CancelJobsResult>
resumeParkedSteps(teamId: number, resumes: StepResume[]): Promise<Map<string, StepResumeOutcome>>
+ resumeParkedBatchResolver(options: {
+ teamId: number
+ functionId: string
+ batchJobId: string
+ }): Promise<BatchResolverResumeOutcome>
disconnect(): Promise<void>
}
diff --git a/nodejs/src/cdp/services/hogflows/batch-resolver-resume.ts b/nodejs/src/cdp/services/hogflows/batch-resolver-resume.ts
new file mode 100644
index 000000000000..d2793fa287e4
--- /dev/null
+++ b/nodejs/src/cdp/services/hogflows/batch-resolver-resume.ts
@@ -0,0 +1,80 @@
+import { Pool } from 'pg'
+import { Counter } from 'prom-client'
+
+import { logger } from '~/common/utils/logger'
+
+import { HOGFLOW_BATCH_RESOLVE_QUEUE, deserializeResolverState, serializeResolverState } from './batch-resolver.types'
+
+// `job_<status>` covers a resolver that is not parked; `resumed` and `already_resumed` both mean
+// the run is on its way.
+export type BatchResolverResumeOutcome = 'resumed' | 'already_resumed' | 'job_missing' | `job_${string}`
+
+export const counterBatchResolverParked = new Counter({
+ name: 'cdp_batch_hog_flow_resolver_parked',
+ help: 'Batch resolvers created parked because their realtime cohort is still calculating.',
+})
+
+export const counterBatchResolverResume = new Counter({
+ name: 'cdp_batch_hog_flow_resolver_resume',
+ help: 'Resumes of batch resolvers parked on a realtime cohort, by outcome.',
+ labelNames: ['outcome'],
+})
+
+/**
+ * Wakes a batch resolver parked until its realtime cohort finishes calculating. Clears the wait
+ * from its state and makes it due now, the same way a step resume wakes a parked workflow step.
+ */
+export async function resumeParkedBatchResolver(
+ pool: Pool,
+ { teamId, functionId, batchJobId }: { teamId: number; functionId: string; batchJobId: string }
+): Promise<BatchResolverResumeOutcome> {
+ const outcome = await resume(pool, teamId, functionId, batchJobId)
+ counterBatchResolverResume.labels({ outcome }).inc()
+ return outcome
+}
+
+async function resume(
+ pool: Pool,
+ teamId: number,
+ functionId: string,
+ batchJobId: string
+): Promise<BatchResolverResumeOutcome> {
+ const client = await pool.connect()
+ try {
+ await client.query('BEGIN')
+ const rows = await client.query(
+ `SELECT id, status, state FROM cyclotron_jobs
+ WHERE team_id = $1 AND function_id = $2 AND parent_run_id = $3 AND queue_name = $4
+ FOR UPDATE`,
+ [teamId, functionId, batchJobId, HOGFLOW_BATCH_RESOLVE_QUEUE]
+ )
+ const row = rows.rows[0]
+ if (!row) {
+ await client.query('COMMIT')
+ return 'job_missing'
+ }
+ // A running resolver owns its state, so a write here would be dropped by its flush.
+ if (row.status !== 'available') {
+ await client.query('COMMIT')
+ return `job_${row.status}`
+ }
+ const state = deserializeResolverState(row.state)
+ if (!state.realtimeCohort?.waitUntil) {
+ await client.query('COMMIT')
+ return 'already_resumed'
+ }
+ const { waitUntil: _waitUntil, ...realtimeCohort } = state.realtimeCohort
+ await client.query(`UPDATE cyclotron_jobs SET scheduled = NOW(), state = $2 WHERE id = $1`, [
+ row.id,
+ serializeResolverState({ ...state, realtimeCohort }),
+ ])
+ await client.query('COMMIT')
+ return 'resumed'
+ } catch (err) {
+ await client.query('ROLLBACK').catch(() => {})
+ logger.error('🔴', 'Failed to resume a parked batch resolver', { batchJobId, error: String(err) })
+ throw err
+ } finally {
+ client.release()
+ }
+}
diff --git a/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts b/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts
index ae8b5f235be2..c761067249b4 100644
--- a/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts
+++ b/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts
@@ -43,7 +43,15 @@ export const BatchResolverStateSchema = z.object({
// Set by Django when the audience is exactly one realtime cohort. The resolver then pages that
// cohort's members from the realtime pipeline's Postgres table instead of the Django audience
// query. `teamId` is the cohort's own team, which owns its membership rows.
- realtimeCohort: z.object({ id: z.number().int(), teamId: z.number().int() }).optional(),
+ realtimeCohort: z
+ .object({
+ id: z.number().int(),
+ teamId: z.number().int(),
+ // Set while the cohort is still calculating: the job stays parked until Django resumes it
+ // once the cohort is ready, or fails at this instant if that never happens.
+ waitUntil: z.string().datetime().optional(),
+ })
+ .optional(),
maxAudienceSize: z.number().int().nonnegative(),
cursor: z.string().nullable(),
totalEnqueued: z.number().int().nonnegative(),
diff --git a/nodejs/src/cdp/utils/jwt-utils.ts b/nodejs/src/cdp/utils/jwt-utils.ts
index 3a34f07f194e..2483297c32b0 100644
--- a/nodejs/src/cdp/utils/jwt-utils.ts
+++ b/nodejs/src/cdp/utils/jwt-utils.ts
@@ -8,6 +8,7 @@ export enum PosthogJwtAudience {
WORKFLOWS_CANCEL_BATCH = 'posthog:workflows:cancel_batch',
WORKFLOWS_STEP_RESUME = 'posthog:workflows:step_resume',
WORKFLOWS_SEND_TRANSACTIONAL_EMAIL = 'posthog:workflows:send_transactional_email',
+ WORKFLOWS_RESUME_BATCH = 'posthog:workflows:resume_batch',
CUSTOMER_TASKS_CREATE = 'posthog:customer-tasks:create',
TASKS_CREATE = 'posthog:tasks:create',
WORKFLOW_SCOUT_RUN = 'posthog:workflows:scout_run',
diff --git a/nodejs/src/common/api/middleware/internal-api-auth.ts b/nodejs/src/common/api/middleware/internal-api-auth.ts
index 17b598412185..9d14943f7e4a 100644
--- a/nodejs/src/common/api/middleware/internal-api-auth.ts
+++ b/nodejs/src/common/api/middleware/internal-api-auth.ts
@@ -34,6 +34,8 @@ const SCOPED_AUTH_PATH_PATTERNS = [
/^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/invocations\/cancel$/,
// CdpApi.postHogFlowCancelBatchJob, verified against WORKFLOWS_CANCEL_JWT_SECRET, audience-pinned.
/^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/batch_jobs\/[^/]+\/cancel$/,
+ // CdpApi.postHogFlowResumeBatchJob, verified against WORKFLOWS_STEP_RESUME_JWT_SECRET, audience-pinned.
+ /^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/batch_jobs\/[^/]+\/resume$/,
// CdpApi.postWorkflowStepResume, verified against WORKFLOWS_STEP_RESUME_JWT_SECRET, audience-pinned.
/^\/api\/projects\/[^/]+\/workflow_steps\/resume$/,
// CdpApi.postTransactionalEmail, verified against WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET, audience-pinned.