:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): park a batch resolver until its realtime cohort is ready (#118444)
Harley Alexander24a2e3f8f0ac
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 71791a375815..8fab4922bf2c 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -1271,6 +1271,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),disconnect: jest.fn().mockResolvedValue(undefined),}@@ -1319,6 +1320,51 @@ describe('CDP API', () => {}})+ it.each([+ ['a deadline', { id: 7, team_id: 1, wait_until: '2026-10-17T10:00:00Z' }, 200, '2026-10-17T10:00:00.000Z'],+ ['no deadline', { id: 7, team_id: 1 }, 200, undefined],+ ['an unreadable deadline', { id: 7, team_id: 1, wait_until: 'tomorrow-ish' }, 400, undefined],+ ['a non-string deadline', { id: 7, team_id: 1, wait_until: 1234 }, 400, undefined],+ ])(+ 'parks a realtime cohort resolver only on a valid deadline: %s',+ async (_desc, cohort, status, parkedUntil) => {+ const createJobMock = jest.fn().mockResolvedValue('resolver-job-id')+ api['batchResolverProducer'] = {+ createJob: createJobMock,+ countInFlightJobs: jest.fn(),+ rescheduleParkedJobs: jest.fn(),+ cancelJobs: jest.fn(),+ resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),+ disconnect: jest.fn().mockResolvedValue(undefined),+ }++ try {+ const res = await supertest(app)+ .post(+ `/api/projects/${batchHogFlow.team_id}/hog_flows/${batchHogFlow.id}/batch_invocations/job-793`+ )+ .send({ realtime_cohort: cohort })++ expect(res.status).toEqual(status)+ if (status !== 200) {+ expect(createJobMock).not.toHaveBeenCalled()+ return+ }+ const arg = createJobMock.mock.calls[0][0]+ expect(arg.scheduled?.toISOString()).toEqual(parkedUntil)+ const state = parseJSON((arg.state as Buffer).toString('utf-8')) as Record<string, any>+ expect(state.realtimeCohort).toEqual({+ id: 7,+ teamId: 1,+ ...(parkedUntil ? { waitUntil: parkedUntil } : {}),+ })+ } finally {+ api['batchResolverProducer'] = null+ }+ }+ )+it('resolves the audience from the posted snapshot, not the live trigger filters', async () => {// The snapshot was validated at confirm time - re-reading the trigger here would let an// edit racing the dispatch widen the send past what was previewed.@@ -1331,6 +1377,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),disconnect: jest.fn().mockResolvedValue(undefined),}@@ -1383,6 +1430,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),disconnect: jest.fn().mockResolvedValue(undefined),}@@ -1443,6 +1491,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),disconnect: jest.fn().mockResolvedValue(undefined),}@@ -1507,6 +1556,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),disconnect: jest.fn().mockResolvedValue(undefined),}@@ -1681,6 +1731,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),}countHogFlow = await insertHogFlow({@@ -1775,6 +1826,7 @@ describe('CDP API', () => {rescheduleParkedJobs: mockRescheduleParkedJobs,cancelJobs: jest.fn(),resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),}rescheduleHogFlow = await insertHogFlow({@@ -1951,6 +2003,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: jest.fn(),resumeParkedSteps: mockResumeParkedSteps,+ resumeParkedBatchResolver: jest.fn(),}})@@ -2161,6 +2214,74 @@ describe('CDP API', () => {})})+ describe('hogflow batch job resume', () => {+ let mockResume: jest.Mock+ const batchJobId = new UUIDT().toString().toLowerCase()+ const flowId = new UUIDT().toString().toLowerCase()++ // Raw audience literal and Python claim names: the wire contract with Django's+ // WORKFLOWS_RESUME_BATCH_JWT_PURPOSE.+ const mintBatchResumeToken = (+ claims: Record<string, unknown>,+ { secret = 'local-dev-workflows-step-resume-jwt', audience = 'posthog:workflows:resume_batch' } = {}+ ) => jwt.sign(claims, secret, { audience, expiresIn: '2m' })+ const resumeAuth = (teamId: number, claims: Record<string, unknown> = {}) => ({+ Authorization: `Bearer ${mintBatchResumeToken({+ team_id: teamId,+ hog_flow_id: flowId,+ batch_job_id: batchJobId,+ ...claims,+ })}`,+ })++ beforeEach(() => {+ mockResume = jest.fn()+ api['batchResolverProducer'] = {+ createJob: jest.fn(),+ disconnect: jest.fn(),+ countInFlightJobs: jest.fn(),+ rescheduleParkedJobs: jest.fn(),+ cancelJobs: jest.fn(),+ resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: mockResume,+ }+ })++ afterEach(() => {+ api['batchResolverProducer'] = null+ })++ it.each([+ ['resumed', 200],+ ['already_resumed', 200],+ ['job_running', 409],+ ['job_completed', 200],+ ['job_failed', 200],+ ['job_missing', 200],+ ])('answers %s with %s, so only a running resolver is retried', async (outcome, status) => {+ mockResume.mockResolvedValue(outcome)++ const res = await supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/batch_jobs/${batchJobId}/resume`)+ .set(resumeAuth(team.id))+ .send({})++ expect(res.status).toEqual(status)+ expect(res.body).toEqual({ outcome })+ expect(mockResume).toHaveBeenCalledWith({ teamId: team.id, functionId: flowId, batchJobId })+ })++ it("rejects another run's token", async () => {+ const res = await supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/batch_jobs/${batchJobId}/resume`)+ .set(resumeAuth(team.id, { batch_job_id: new UUIDT().toString() }))+ .send({})++ expect(res.status).toEqual(401)+ expect(mockResume).not.toHaveBeenCalled()+ })+ })+describe('hogflow cancel invocations auth', () => {let mockCancelJobs: jest.Mock@@ -2188,6 +2309,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: mockCancelJobs,resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),}})@@ -2324,6 +2446,7 @@ describe('CDP API', () => {rescheduleParkedJobs: jest.fn(),cancelJobs: mockCancelJobs,resumeParkedSteps: jest.fn(),+ resumeParkedBatchResolver: jest.fn(),}})diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 0bd16cbe3e8a..65a0120794dd 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -39,6 +39,7 @@ import type { CyclotronV2JobProducer } from './services/cyclotron-v2'import { HogExecutorAsyncService, HogExecutorExecuteAsyncOptions } from './services/hog-executor-async.service'import { MAX_ASYNC_STEPS } from './services/hog-executor.service'import { HogInputsService } from './services/hog-inputs.service'+import { counterBatchResolverParked } from './services/hogflows/batch-resolver-resume'import {BatchResolverState,HOGFLOW_BATCH_RESOLVE_QUEUE,@@ -136,14 +137,24 @@ function canDedupeByEmail(hogFlow: { actions?: unknown }): boolean {export type CdpApiConfig = PluginsServerConfigexport type CdpApiDeps = CdpConsumerBaseDeps-function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort'] {+// Null means the cohort asks to wait but its deadline is unreadable. Queuing it unparked would send to a+// cohort that is still calculating, so the caller rejects it.+function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort'] | null {if (typeof value !== 'object' || value === null) {return undefined}- const { id, team_id } = value as { id?: unknown; team_id?: unknown }- return Number.isInteger(id) && Number.isInteger(team_id)- ? { id: id as number, teamId: team_id as number }- : undefined+ const { id, team_id, wait_until } = value as { id?: unknown; team_id?: unknown; wait_until?: unknown }+ if (!Number.isInteger(id) || !Number.isInteger(team_id)) {+ return undefined+ }+ if (wait_until === undefined || wait_until === null) {+ return { id: id as number, teamId: team_id as number }+ }+ const waitUntil = typeof wait_until === 'string' ? new Date(wait_until) : null+ if (!waitUntil || isNaN(waitUntil.getTime())) {+ return null+ }+ return { id: id as number, teamId: team_id as number, waitUntil: waitUntil.toISOString() }}export class CdpApi {@@ -178,6 +189,7 @@ export class CdpApi {private stepResumeJwt: ScopedServiceJwtprivate transactionalEmailJwt: ScopedServiceJwtprivate hogFunctionMonitoringService: HogFunctionMonitoringService+ private resumeBatchJwt: ScopedServiceJwtconstructor(private config: PluginsServerConfig,@@ -250,6 +262,12 @@ export class CdpApi {PosthogJwtAudience.WORKFLOWS_SEND_TRANSACTIONAL_EMAIL,config.WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET || '')+ // The same workers that wake parked steps wake parked batch resolvers, so they share the+ // key. The audience keeps a step-resume token from waking a batch, and the reverse.+ this.resumeBatchJwt = new ScopedServiceJwt(+ PosthogJwtAudience.WORKFLOWS_RESUME_BATCH,+ config.WORKFLOWS_STEP_RESUME_JWT_SECRET || ''+ )}public get service(): PluginServerService {@@ -330,6 +348,10 @@ export class CdpApi {'/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/cancel',asyncHandler(this.postHogFlowCancelBatchJob))+ router.post(+ '/api/projects/:team_id/hog_flows/:id/batch_jobs/:batch_job_id/resume',+ asyncHandler(this.postHogFlowResumeBatchJob)+ )router.post('/api/projects/:team_id/workflow_steps/resume', asyncHandler(this.postWorkflowStepResume))router.post('/api/projects/:team_id/hog_flows/:id/transactional_email',@@ -1310,6 +1332,50 @@ export class CdpApi {}}+ /**+ * Wake a batch resolver parked until its realtime cohort finishes calculating. Django calls this+ * once the cohort is ready. `already_resumed` is a success: a second wake for the same run is a no-op.+ * Only a running resolver answers 409, so the caller retries. A finished or removed one already ran, so+ * a wake retried after a lost response gets a 200 and the caller stops.+ */+ private postHogFlowResumeBatchJob = async (req: ModifiedRequest, res: express.Response): Promise<any> => {+ try {+ if (!this.batchResolverProducer) {+ return res.status(503).json({+ error: 'Cyclotron producer not initialized (CYCLOTRON_NODE_DATABASE_URL unset)',+ })+ }+ if (!this.resumeBatchJwt.enabled) {+ return res.status(503).json({+ error: 'Workflows scoped auth not configured (WORKFLOWS_STEP_RESUME_JWT_SECRET unset)',+ })+ }+ const { team_id, id, batch_job_id } = req.params+ // batch_job_id is pinned in the claims, so a captured token wakes exactly one run.+ if (!this.verifyScopedWorkflowJwt(this.resumeBatchJwt, req, res, 'resume', { batch_job_id })) {+ return+ }+ const team = await this.deps.teamManager.getTeam(parseInt(team_id)).catch(() => null)+ if (!team) {+ return res.status(404).json({ error: 'Team not found' })+ }+ if (batch_job_id.length > 200) {+ return res.status(400).json({ error: 'batch_job_id is too long' })+ }+ const outcome = await this.batchResolverProducer.resumeParkedBatchResolver({+ teamId: team.id,+ functionId: id,+ batchJobId: batch_job_id,+ })+ return res.status(outcome === 'job_running' ? 409 : 200).json({ outcome })+ } catch (e) {+ logger.error('Error resuming hog flow batch job', {+ error: e instanceof Error ? e.message : String(e),+ })+ return res.status(500).json({ error: e instanceof Error ? e.message : String(e) })+ }+ }+// Wake the parked workflow step that dispatched a task run, with the run's outcome. Django// calls this from a retrying Celery task when the run reaches a terminal status. One resume// per call; the outcome tells the caller whether to retry (409: the worker still holds the@@ -1527,6 +1593,11 @@ export class CdpApi {throw new Error('Batch resolver producer is not configured (missing CYCLOTRON_NODE_DATABASE_URL)')}+ const realtimeCohort = parseRealtimeCohort(req.body.realtime_cohort)+ if (realtimeCohort === null) {+ return res.status(400).json({ error: 'realtime_cohort.wait_until must be an ISO 8601 date' })+ }+const snapshotFilters: BatchResolverState['filters'] | undefined = req.body.filtersconst audienceType = snapshotFilters?.audience_type ?? hogFlow.trigger.filters.audience_type// A snapshot saved before assignment statuses existed carries assignee ids or the legacy@@ -1564,7 +1635,7 @@ export class CdpApi {// dedupe than dedupe wrongly. Also skip when the flow has no email action at all.// Account audiences carry no person (and external ids are already unique), so never dedupe.dedupeKey: audienceType !== 'accounts' && canDedupeByEmail(hogFlow) ? ('email' as const) : undefined,- realtimeCohort: parseRealtimeCohort(req.body.realtime_cohort),+ realtimeCohort,maxAudienceSize: maxAudienceSize ?? this.config.CDP_BATCH_WORKFLOW_MAX_AUDIENCE_SIZE,cursor: null,totalEnqueued: 0,@@ -1572,13 +1643,19 @@ export class CdpApi {attempts: 0,startedAt: new Date().toISOString(),}+ const waitUntil = initialState.realtimeCohort?.waitUntilawait this.batchResolverProducer.createJob({teamId: team.id,queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,parentRunId: parent_run_id,functionId: hogFlow.id,state: serializeResolverState(initialState),+ // A cohort that is still calculating parks the job until Django resumes it.+ ...(waitUntil ? { scheduled: new Date(waitUntil) } : {}),})+ if (waitUntil) {+ counterBatchResolverParked.inc()+ }res.json({ status: 'queued' })} catch (e) {diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.tsindex a53ee91c03c8..750306170d67 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts@@ -11,6 +11,7 @@ import { CdpOutput } from '../cdp-services'import {BatchResolverState,MAX_RESOLVER_ATTEMPTS,+ deserializeResolverState,serializeResolverState,} from '../services/hogflows/batch-resolver.types'import { AudienceFetchTimeoutError } from '../services/hogflows/hogflow-batch-person-query.service'@@ -453,6 +454,33 @@ describe('CdpCyclotronWorkerBatchResolve', () => {startedAt: '2026-08-11T00:00:00.000Z',})+ it.each([+ ['before its deadline parks it again without reading members', 3600 * 1000, false],+ ['at its deadline fails the run', -1000, true],+ ])('a realtime cohort resolver woken %s', async (_name, untilOffsetMs, fails) => {+ const getBlastRadiusPersons = jest.fn()+ const queueLogs = jest.fn()+ const consumer = buildConsumer(getBlastRadiusPersons, queueLogs)+ const waitUntil = new Date(Date.now() + untilOffsetMs).toISOString()+ const job = buildJob({+ ...lastAttemptState(),+ attempts: 0,+ realtimeCohort: { id: 7, teamId: team.id, waitUntil },+ })++ await consumer.processResolverJob(job)++ expect(getBlastRadiusPersons).not.toHaveBeenCalled()+ const [{ scheduledAt, state }] = job.reschedule.mock.calls[0]+ if (fails) {+ expect(deserializeResolverState(state).pendingTerminal).toEqual('failed')+ expect(queueLogs.mock.calls[0][0][0].message).toContain("didn't finish calculating")+ } else {+ expect(scheduledAt.toISOString()).toEqual(waitUntil)+ expect(state).toBeUndefined()+ }+ })+it('tells the customer what to change when the audience query timed out', async () => {const queueLogs = jest.fn()const consumer = buildConsumer(diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.tsindex 69d52c8c06cb..bdab41da474e 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-batch-resolve.consumer.ts@@ -64,6 +64,12 @@ const counterBatchHogFlowResolverPagesProcessed = new Counter({labelNames: ['outcome'], // success | fetch_failure | terminal_write_failure | invalid_state})+const counterBatchHogFlowResolverCohortWait = new Counter({+ name: 'cdp_batch_hog_flow_resolver_cohort_wait',+ help: 'Batch resolvers parked on a realtime cohort that is still calculating, by what woke them',+ labelNames: ['outcome'], // early_wake | timed_out+})+const counterBatchHogFlowResolverJobs = new Counter({name: 'cdp_batch_hog_flow_resolver_jobs',help: 'Batch hog flow resolver jobs by lifecycle outcome',@@ -196,6 +202,12 @@ export class CdpCyclotronWorkerBatchResolve extends CdpConsumerBase<PluginsServereturn}+ const waitUntil = state.realtimeCohort?.waitUntil+ if (waitUntil) {+ await this.handleCohortWait(job, state, new Date(waitUntil))+ return+ }+if (state.totalEnqueued >= state.maxAudienceSize) {await this.transitionToTruncatedTerminal(job, state)return@@ -220,6 +232,29 @@ export class CdpCyclotronWorkerBatchResolve extends CdpConsumerBase<PluginsServe}}+ /**+ * A resolver parked on a realtime cohort runs here only when nothing resumed it: Django clears+ * the wait from its state when the cohort is ready. Before the deadline that is an early wake,+ * so it parks again; at the deadline the cohort never finished, and the run fails.+ */+ private async handleCohortWait(+ job: CyclotronV2DequeuedJob,+ state: BatchResolverState,+ waitUntil: Date+ ): Promise<void> {+ if (Date.now() < waitUntil.getTime()) {+ counterBatchHogFlowResolverCohortWait.labels({ outcome: 'early_wake' }).inc()+ await job.reschedule({ scheduledAt: waitUntil })+ return+ }+ counterBatchHogFlowResolverCohortWait.labels({ outcome: 'timed_out' }).inc()+ await this.transitionToFailedTerminal(+ job,+ state,+ "the cohort didn't finish calculating in time, so nobody was sent this run. Start it again once the cohort is ready."+ )+ }+/*** Terminate a cancel-flagged resolver job: no further pages, and no terminal status* PUT — Django flips the batch job's status itself as part of the cancel request, anddiff --git a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.tsindex d690b7eda845..188fb18c0c2b 100644--- a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts+++ b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts@@ -4,6 +4,11 @@ import { v7 as uuidv7 } from 'uuid'import { parseJSON } from '~/common/utils/json-parse'+import {+ HOGFLOW_BATCH_RESOLVE_QUEUE,+ deserializeResolverState,+ serializeResolverState,+} from '../hogflows/batch-resolver.types'import { HogInvocationResultsService } from '../monitoring/hog-invocation-results.service'import { CyclotronV2Janitor, JANITOR_POISON_PILL_ERROR_KIND } from './janitor'import { CyclotronV2Manager } from './manager'@@ -869,6 +874,64 @@ describe('Cyclotron V2', () => {})})+ describe('resumeParkedBatchResolver', () => {+ const parkResolver = async (teamId: number, functionId: string, batchJobId: string): Promise<string> =>+ await manager.createJob({+ teamId,+ queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,+ functionId,+ parentRunId: batchJobId,+ scheduled: new Date(Date.now() + 3600 * 1000),+ state: serializeResolverState({+ batchJobId,+ teamId,+ hogFlowId: functionId,+ filters: { properties: [] },+ variables: {},+ realtimeCohort: { id: 7, teamId, waitUntil: new Date(Date.now() + 3600 * 1000).toISOString() },+ maxAudienceSize: 100,+ cursor: null,+ totalEnqueued: 0,+ pagesProcessed: 0,+ attempts: 0,+ startedAt: new Date().toISOString(),+ }),+ })++ it('wakes the parked resolver of the batch and clears its wait, once', async () => {+ const functionId = uuidv7()+ const batchJobId = uuidv7()+ const jobId = await parkResolver(1, functionId, batchJobId)++ const first = await manager.resumeParkedBatchResolver({ teamId: 1, functionId, batchJobId })+ const second = await manager.resumeParkedBatchResolver({ teamId: 1, functionId, batchJobId })++ expect([first, second]).toEqual(['resumed', 'already_resumed'])+ expect(await jobIsDue(jobId)).toBe(true)+ const state = deserializeResolverState((await queryJob(jobId)).state)+ // Without the wait cleared, the resolver would park again and time out.+ expect(state.realtimeCohort).toEqual({ id: 7, teamId: 1 })+ })++ it.each([+ ['another team', 2, true],+ ['another batch', 1, false],+ ])('leaves the resolver of %s parked', async (_name, teamId, sameBatch) => {+ const functionId = uuidv7()+ const parkedBatchJobId = uuidv7()+ const jobId = await parkResolver(teamId, functionId, parkedBatchJobId)++ const outcome = await manager.resumeParkedBatchResolver({+ teamId: 1,+ functionId,+ batchJobId: sameBatch ? parkedBatchJobId : uuidv7(),+ })++ expect(outcome).toBe('job_missing')+ expect(await jobIsDue(jobId)).toBe(false)+ })+ })+describe('cancelJobs', () => {const FAR_FUTURE = () => new Date(Date.now() + 7 * 24 * 3600 * 1000)diff --git a/nodejs/src/cdp/services/cyclotron-v2/manager.ts b/nodejs/src/cdp/services/cyclotron-v2/manager.tsindex f5bf5d68076f..be60f51ffcc7 100644--- a/nodejs/src/cdp/services/cyclotron-v2/manager.ts+++ b/nodejs/src/cdp/services/cyclotron-v2/manager.ts@@ -6,6 +6,7 @@ import { isTransientPgError } from '~/common/utils/db/postgres'import { logger } from '~/common/utils/logger'import { sleep } from '~/common/utils/utils'+import { BatchResolverResumeOutcome, resumeParkedBatchResolver } from '../hogflows/batch-resolver-resume'import { StepResume, StepResumeOutcome, processStepResumes } from '../hogflows/step-resume.service'import {CYCLOTRON_COUNTER_MAX,@@ -795,4 +796,12 @@ export class CyclotronV2Manager {async resumeParkedSteps(teamId: number, resumes: StepResume[]): Promise<Map<string, StepResumeOutcome>> {return await processStepResumes(this.pool, resumes, teamId)}++ async resumeParkedBatchResolver(options: {+ teamId: number+ functionId: string+ batchJobId: string+ }): Promise<BatchResolverResumeOutcome> {+ return await resumeParkedBatchResolver(this.pool, options)+ }}diff --git a/nodejs/src/cdp/services/cyclotron-v2/types.ts b/nodejs/src/cdp/services/cyclotron-v2/types.tsindex 3c2c43efa276..3ca1e830eb8e 100644--- a/nodejs/src/cdp/services/cyclotron-v2/types.ts+++ b/nodejs/src/cdp/services/cyclotron-v2/types.ts@@ -1,6 +1,7 @@import { DateTime } from 'luxon'import { z } from 'zod'+import type { BatchResolverResumeOutcome } from '~/cdp/services/hogflows/batch-resolver-resume'import { StepResume, StepResumeOutcome } from '~/cdp/services/hogflows/step-resume.service'export type CyclotronV2JobStatus = 'available' | 'running' | 'completed' | 'failed' | 'canceled'@@ -195,6 +196,11 @@ export interface CyclotronV2JobProducer {rescheduleParkedJobs(options: CyclotronV2RescheduleParkedOptions): Promise<CyclotronV2RescheduleParkedResult>cancelJobs(options: CyclotronV2CancelJobsOptions): Promise<CyclotronV2CancelJobsResult>resumeParkedSteps(teamId: number, resumes: StepResume[]): Promise<Map<string, StepResumeOutcome>>+ resumeParkedBatchResolver(options: {+ teamId: number+ functionId: string+ batchJobId: string+ }): Promise<BatchResolverResumeOutcome>disconnect(): Promise<void>}diff --git a/nodejs/src/cdp/services/hogflows/batch-resolver-resume.ts b/nodejs/src/cdp/services/hogflows/batch-resolver-resume.tsnew file mode 100644index 000000000000..d2793fa287e4--- /dev/null+++ b/nodejs/src/cdp/services/hogflows/batch-resolver-resume.ts@@ -0,0 +1,80 @@+import { Pool } from 'pg'+import { Counter } from 'prom-client'++import { logger } from '~/common/utils/logger'++import { HOGFLOW_BATCH_RESOLVE_QUEUE, deserializeResolverState, serializeResolverState } from './batch-resolver.types'++// `job_<status>` covers a resolver that is not parked; `resumed` and `already_resumed` both mean+// the run is on its way.+export type BatchResolverResumeOutcome = 'resumed' | 'already_resumed' | 'job_missing' | `job_${string}`++export const counterBatchResolverParked = new Counter({+ name: 'cdp_batch_hog_flow_resolver_parked',+ help: 'Batch resolvers created parked because their realtime cohort is still calculating.',+})++export const counterBatchResolverResume = new Counter({+ name: 'cdp_batch_hog_flow_resolver_resume',+ help: 'Resumes of batch resolvers parked on a realtime cohort, by outcome.',+ labelNames: ['outcome'],+})++/**+ * Wakes a batch resolver parked until its realtime cohort finishes calculating. Clears the wait+ * from its state and makes it due now, the same way a step resume wakes a parked workflow step.+ */+export async function resumeParkedBatchResolver(+ pool: Pool,+ { teamId, functionId, batchJobId }: { teamId: number; functionId: string; batchJobId: string }+): Promise<BatchResolverResumeOutcome> {+ const outcome = await resume(pool, teamId, functionId, batchJobId)+ counterBatchResolverResume.labels({ outcome }).inc()+ return outcome+}++async function resume(+ pool: Pool,+ teamId: number,+ functionId: string,+ batchJobId: string+): Promise<BatchResolverResumeOutcome> {+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const rows = await client.query(+ `SELECT id, status, state FROM cyclotron_jobs+ WHERE team_id = $1 AND function_id = $2 AND parent_run_id = $3 AND queue_name = $4+ FOR UPDATE`,+ [teamId, functionId, batchJobId, HOGFLOW_BATCH_RESOLVE_QUEUE]+ )+ const row = rows.rows[0]+ if (!row) {+ await client.query('COMMIT')+ return 'job_missing'+ }+ // A running resolver owns its state, so a write here would be dropped by its flush.+ if (row.status !== 'available') {+ await client.query('COMMIT')+ return `job_${row.status}`+ }+ const state = deserializeResolverState(row.state)+ if (!state.realtimeCohort?.waitUntil) {+ await client.query('COMMIT')+ return 'already_resumed'+ }+ const { waitUntil: _waitUntil, ...realtimeCohort } = state.realtimeCohort+ await client.query(`UPDATE cyclotron_jobs SET scheduled = NOW(), state = $2 WHERE id = $1`, [+ row.id,+ serializeResolverState({ ...state, realtimeCohort }),+ ])+ await client.query('COMMIT')+ return 'resumed'+ } catch (err) {+ await client.query('ROLLBACK').catch(() => {})+ logger.error('🔴', 'Failed to resume a parked batch resolver', { batchJobId, error: String(err) })+ throw err+ } finally {+ client.release()+ }+}diff --git a/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts b/nodejs/src/cdp/services/hogflows/batch-resolver.types.tsindex ae8b5f235be2..c761067249b4 100644--- a/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts+++ b/nodejs/src/cdp/services/hogflows/batch-resolver.types.ts@@ -43,7 +43,15 @@ export const BatchResolverStateSchema = z.object({// Set by Django when the audience is exactly one realtime cohort. The resolver then pages that// cohort's members from the realtime pipeline's Postgres table instead of the Django audience// query. `teamId` is the cohort's own team, which owns its membership rows.- realtimeCohort: z.object({ id: z.number().int(), teamId: z.number().int() }).optional(),+ realtimeCohort: z+ .object({+ id: z.number().int(),+ teamId: z.number().int(),+ // Set while the cohort is still calculating: the job stays parked until Django resumes it+ // once the cohort is ready, or fails at this instant if that never happens.+ waitUntil: z.string().datetime().optional(),+ })+ .optional(),maxAudienceSize: z.number().int().nonnegative(),cursor: z.string().nullable(),totalEnqueued: z.number().int().nonnegative(),diff --git a/nodejs/src/cdp/utils/jwt-utils.ts b/nodejs/src/cdp/utils/jwt-utils.tsindex 3a34f07f194e..2483297c32b0 100644--- a/nodejs/src/cdp/utils/jwt-utils.ts+++ b/nodejs/src/cdp/utils/jwt-utils.ts@@ -8,6 +8,7 @@ export enum PosthogJwtAudience {WORKFLOWS_CANCEL_BATCH = 'posthog:workflows:cancel_batch',WORKFLOWS_STEP_RESUME = 'posthog:workflows:step_resume',WORKFLOWS_SEND_TRANSACTIONAL_EMAIL = 'posthog:workflows:send_transactional_email',+ WORKFLOWS_RESUME_BATCH = 'posthog:workflows:resume_batch',CUSTOMER_TASKS_CREATE = 'posthog:customer-tasks:create',TASKS_CREATE = 'posthog:tasks:create',WORKFLOW_SCOUT_RUN = 'posthog:workflows:scout_run',diff --git a/nodejs/src/common/api/middleware/internal-api-auth.ts b/nodejs/src/common/api/middleware/internal-api-auth.tsindex 17b598412185..9d14943f7e4a 100644--- a/nodejs/src/common/api/middleware/internal-api-auth.ts+++ b/nodejs/src/common/api/middleware/internal-api-auth.ts@@ -34,6 +34,8 @@ const SCOPED_AUTH_PATH_PATTERNS = [/^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/invocations\/cancel$/,// CdpApi.postHogFlowCancelBatchJob, verified against WORKFLOWS_CANCEL_JWT_SECRET, audience-pinned./^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/batch_jobs\/[^/]+\/cancel$/,+ // CdpApi.postHogFlowResumeBatchJob, verified against WORKFLOWS_STEP_RESUME_JWT_SECRET, audience-pinned.+ /^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/batch_jobs\/[^/]+\/resume$/,// CdpApi.postWorkflowStepResume, verified against WORKFLOWS_STEP_RESUME_JWT_SECRET, audience-pinned./^\/api\/projects\/[^/]+\/workflow_steps\/resume$/,// CdpApi.postTransactionalEmail, verified against WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET, audience-pinned.