Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(workflows): stop the run when a cohort lookup fails instead of routing (#116200)

Harley Alexander5e2fecf7e9e8
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
index edbf876c8f46..c2207b09e344 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
@@ -89,6 +89,8 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
startedAtTimestamp: 999,
awaitingResume: { key: 'old-key', deadlineAt: '2020-01-01T00:00:00Z', dispatch: { id: 'old-task' } },
resumeResult: { key: 'old-key', status: 'failed' },
+ cohortLookupAttempts: 2,
+ cohortLookupFailed: true,
hogFunctionState: {
globals: { event: { uuid: 'evt-1' }, source: { name: 'Email', url: '' } },
timings: [],
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
index f4a73a0cb61a..18ef155c4700 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
@@ -795,10 +795,15 @@ export class RerunPaginatorService {
// with "No recipient identifier found". Dropping it makes the
// action re-enter fresh, so inputs re-render against the current
// config, which is what a rerun intends anyway.
+ // The cohort lookup counters are dropped too. A run that aborted after its last
+ // lookup attempt would otherwise restart with no retries left, and the leftover
+ // cohortLookupFailed flag would turn an unrelated error on that step into an abort.
const {
hogFunctionState: _stripped,
awaitingResume: _awaitingResume,
resumeResult: _resumeResult,
+ cohortLookupAttempts: _cohortLookupAttempts,
+ cohortLookupFailed: _cohortLookupFailed,
...restoredCurrentAction
} = persistedState.currentAction ?? {}
diff --git a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts
index cd248eab6ffa..88a922bc2cfe 100644
--- a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts
+++ b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts
@@ -295,10 +295,32 @@ describe('action.conditional_branch', () => {
expect(repository.calls).toEqual([])
})
- it('propagates a lookup failure instead of routing on a made-up answer', async () => {
+ it.each([
+ [0, 10],
+ [1, 60],
+ ])('retries a failed lookup after %s earlier failures in %ss instead of routing', async (earlier, delay) => {
+ const repository = new FakeCohortMembershipRepository([], new Error('lookup timed out'))
+ invocation.state.currentAction!.cohortLookupAttempts = earlier || undefined
+ action.config.conditions = [{ filters: cohortConditionFilters('inCohort') }]
+
+ const result = await new ConditionalBranchHandler(repository).execute({
+ invocation,
+ action,
+ result: createInvocationResult(invocation),
+ })
+
+ expect(result.nextAction).toBeUndefined()
+ expect(result.scheduledAt).toEqual(DateTime.now().plus({ seconds: delay }))
+ expect(invocation.state.currentAction!.cohortLookupAttempts).toBe(earlier + 1)
+ expect(invocation.state.currentAction!.cohortLookupFailed).toBeUndefined()
+ })
+
+ it('marks the step failed and throws once every lookup attempt failed', async () => {
const repository = new FakeCohortMembershipRepository([], new Error('lookup timed out'))
+ invocation.state.currentAction!.cohortLookupAttempts = 2
await expect(executeWithRepository(repository, 'inCohort')).rejects.toThrow('lookup timed out')
+ expect(invocation.state.currentAction!.cohortLookupFailed).toBe(true)
})
})
diff --git a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts
index 6691bd658b02..d6119b28ef12 100644
--- a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts
+++ b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts
@@ -3,7 +3,7 @@ import { Counter } from 'prom-client'
import { HogFlowAction } from '~/cdp/schema/hogflow'
import { CohortMembershipRepository } from '~/cdp/services/cohorts/cohort-membership-repository'
-import { CyclotronJobInvocationHogFlow, HogFunctionFilters } from '~/cdp/types'
+import { CyclotronJobInvocationHogFlow, CyclotronJobInvocationResult, HogFunctionFilters } from '~/cdp/types'
import { filterFunctionInstrumented } from '~/cdp/utils/hog-function-filtering'
import { findContinueAction, findNextAction, isEvaluableCondition } from '../hogflow-utils'
@@ -41,6 +41,25 @@ export const counterHogflowRekeyWake = new Counter({
labelNames: ['outcome'],
})
+// Outcome of a step whose cohort membership lookup failed. 'retried' re-runs the step after a backoff;
+// 'aborted' means every attempt failed and the run stopped. Sustained 'aborted' readings mean the
+// behavioral cohorts DB is unreachable from the workers, and every cohort branch is stopping runs.
+export const counterHogflowCohortLookupFailure = new Counter({
+ name: 'cdp_hogflow_cohort_lookup_failure_total',
+ help: 'Cohort membership lookups that failed in a conditional_branch or wait, by outcome.',
+ labelNames: ['outcome'],
+})
+
+// Seconds to wait before each retry. One more attempt than there are delays: the first try runs inline.
+export const COHORT_LOOKUP_RETRY_DELAYS_SECONDS = [10, 60]
+
+export class CohortMembershipLookupFailedError extends Error {
+ constructor(cause: unknown) {
+ super(`Cohort membership lookup failed: ${String(cause)}`, { cause })
+ this.name = 'CohortMembershipLookupFailedError'
+ }
+}
+
export class ConditionalBranchHandler implements ActionHandler {
constructor(private cohortMembershipRepository: CohortMembershipRepository) {}
@@ -118,14 +137,26 @@ export class ConditionalBranchHandler implements ActionHandler {
},
}
- const conditionResult = await checkConditions(
- invocation,
- conditionalAction,
- this.createMemberCohortIdsLoader(invocation),
- action.type === 'wait_until_condition' && action.config.max_wait_duration
- ? { maxWaitDuration: action.config.max_wait_duration }
- : undefined
- )
+ let conditionResult: Awaited<ReturnType<typeof checkConditions>>
+ try {
+ conditionResult = await checkConditions(
+ invocation,
+ conditionalAction,
+ this.createMemberCohortIdsLoader(invocation),
+ action.type === 'wait_until_condition' && action.config.max_wait_duration
+ ? { maxWaitDuration: action.config.max_wait_duration }
+ : undefined
+ )
+ } catch (error) {
+ if (error instanceof CohortMembershipLookupFailedError) {
+ return retryOrFailCohortLookup(invocation, result, error)
+ }
+ throw error
+ }
+ if (invocation.state.currentAction) {
+ delete invocation.state.currentAction.cohortLookupAttempts
+ delete invocation.state.currentAction.cohortLookupFailed
+ }
const isWait = action.type === 'wait_until_condition'
@@ -159,7 +190,11 @@ export class ConditionalBranchHandler implements ActionHandler {
if (!loaded) {
const personUuid = invocation.person?.id ?? invocation.state.personId
loaded = personUuid
- ? this.cohortMembershipRepository.getMemberCohortIds(invocation.hogFlow.team_id, personUuid)
+ ? this.cohortMembershipRepository
+ .getMemberCohortIds(invocation.hogFlow.team_id, personUuid)
+ .catch((error) => {
+ throw new CohortMembershipLookupFailedError(error)
+ })
: Promise.resolve([])
}
return loaded
@@ -167,6 +202,36 @@ export class ConditionalBranchHandler implements ActionHandler {
}
}
+// A failed lookup must not route the person: no answer is not the same as "not a member". The step
+// re-runs after a backoff, and once the attempts run out it marks cohortLookupFailed and rethrows, so
+// the executor aborts the run instead of following on_error's default continue edge.
+function retryOrFailCohortLookup(
+ invocation: CyclotronJobInvocationHogFlow,
+ result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
+ error: CohortMembershipLookupFailedError
+): ActionHandlerResult {
+ const currentAction = invocation.state.currentAction
+ const failedAttempts = (currentAction?.cohortLookupAttempts ?? 0) + 1
+ const retryDelaySeconds = COHORT_LOOKUP_RETRY_DELAYS_SECONDS[failedAttempts - 1]
+
+ if (currentAction && retryDelaySeconds !== undefined) {
+ currentAction.cohortLookupAttempts = failedAttempts
+ counterHogflowCohortLookupFailure.labels('retried').inc()
+ result.logs.push({
+ level: 'warn',
+ timestamp: DateTime.now(),
+ message: `Could not look up cohort membership (${String(error.cause)}). Retrying in ${retryDelaySeconds}s`,
+ })
+ return { scheduledAt: DateTime.now().plus({ seconds: retryDelaySeconds }) }
+ }
+
+ if (currentAction) {
+ currentAction.cohortLookupFailed = true
+ }
+ counterHogflowCohortLookupFailure.labels('aborted').inc()
+ throw error
+}
+
// Operation.CALL_GLOBAL from @posthog/hogvm, which is a const enum and can't be imported
// under isolatedModules
const CALL_GLOBAL = 2
@@ -228,7 +293,7 @@ export async function checkConditions(
for (const [index, condition] of action.config.conditions.entries()) {
// Loaded only when evaluation actually reaches a cohort condition, so a run whose earlier
// condition matches never touches the behavioral cohorts DB. A lookup failure throws here
- // on purpose (following the action's on_error) instead of guessing non-membership; the
+ // on purpose, and the handler retries and then aborts instead of guessing non-membership; the
// inCohort/notInCohort STL functions read the resulting cohort_ids global. The matcher does
// not watch cohort membership, so a wait gated on a cohort advances at its deadline.
const cohortGlobals =
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
index 7ef5817d7ef0..0c97d4225854 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
@@ -55,6 +55,7 @@ const cleanLogs = (logs: string[]): string[] => {
describe('Hogflow Executor', () => {
let executor: HogFlowExecutorService
+ let cohortMembershipRepository: CohortMembershipRepository
let hogFlowFunctionsService: HogFlowFunctionsService
let hub: Hub
let team: Team
@@ -173,14 +174,14 @@ describe('Hogflow Executor', () => {
await insertHogFunctionTemplate(hub.postgres, posthogCaptureTemplate)
- const stubCohortMembershipRepository: CohortMembershipRepository = {
+ cohortMembershipRepository = {
getMemberCohortIds: () => Promise.resolve([]),
}
executor = new HogFlowExecutorService(
hogFlowFunctionsService,
recipientPreferencesService,
emailValidationService,
- stubCohortMembershipRepository,
+ cohortMembershipRepository,
hub.integrationManager
)
})
@@ -1926,6 +1927,84 @@ describe('Hogflow Executor', () => {
])
)
})
+
+ it('does NOT continue past a branch whose cohort lookup failed on every attempt, despite on_error continue', async () => {
+ jest.spyOn(cohortMembershipRepository, 'getMemberCohortIds').mockRejectedValue(
+ new Error('lookup timed out')
+ )
+ const flow = new FixtureHogFlowBuilder()
+ .withWorkflow({
+ actions: {
+ trigger: {
+ type: 'trigger',
+ config: {
+ type: 'event',
+ filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {},
+ },
+ },
+ branch: {
+ type: 'conditional_branch',
+ config: {
+ conditions: [
+ {
+ filters: {
+ bytecode: [
+ '_H',
+ 1,
+ 33,
+ 42,
+ 32,
+ 'cohort_ids',
+ 1,
+ 1,
+ 2,
+ 'inCohort',
+ 2,
+ ],
+ properties: [{ key: 'id', type: 'cohort', value: 42 }],
+ },
+ },
+ ],
+ },
+ },
+ in_cohort: { type: 'exit', config: {} },
+ not_in_cohort: { type: 'exit', config: {} },
+ },
+ edges: [
+ { from: 'trigger', to: 'branch', type: 'continue' },
+ { from: 'branch', to: 'in_cohort', type: 'branch', index: 0 },
+ { from: 'branch', to: 'not_in_cohort', type: 'continue' },
+ ],
+ })
+ .build()
+ expect(flow.actions.find((a) => a.id === 'branch')!.on_error).toBe('continue')
+
+ const invocation = createExampleHogFlowInvocation(flow)
+ invocation.state.currentAction = {
+ id: 'branch',
+ startedAtTimestamp: DateTime.now().toMillis(),
+ cohortLookupAttempts: 2,
+ }
+
+ const result = await executor.execute(invocation)
+
+ expect(result.error).toContain('lookup timed out')
+ expect(result.finished).toBe(true)
+ expect(result.invocation.state.currentAction?.id).toBe('branch')
+ expect(result.logs.map((l) => l.message)).not.toEqual(
+ expect.arrayContaining([expect.stringContaining('Workflow moved to action')])
+ )
+ expect(result.logs.map((l) => l.message)).toEqual(
+ expect.arrayContaining([
+ expect.stringContaining(
+ 'Workflow is aborting because [Action:branch] could not look up cohort membership'
+ ),
+ expect.stringContaining(
+ 'Workflow encountered an error: Cohort membership lookup failed: Error: lookup timed out'
+ ),
+ ])
+ )
+ })
})
})
})
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts
index 12f77108301a..32b5b912b362 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts
@@ -451,6 +451,17 @@ export class HogFlowExecutorService {
lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId
} could not work out the date to wait for`,
})
+ } else if (result.invocation.state.currentAction?.cohortLookupFailed) {
+ // The default continue edge of a conditional branch is the path for people outside the
+ // cohort, so carrying on would route this person on a lookup that never answered.
+ shouldAbortAfterError = true
+ logs.push({
+ level: 'info',
+ timestamp: DateTime.now(),
+ message: `Workflow is aborting because ${
+ lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId
+ } could not look up cohort membership`,
+ })
}
}
@@ -818,10 +829,15 @@ export class HogFlowExecutorService {
try {
const { invocation } = result
// A delay that could not work out when to continue must never fall through, whatever on_error
- // says: the next step would run immediately, which is what the wait exists to prevent. Bailing
- // here also keeps the flag readable by shouldEndHogFlowExecution, since goToNextAction would
- // replace currentAction with a fresh entry and drop it.
- if (invocation.state.currentAction?.delayUntilUnresolved) {
+ // says: the next step would run immediately, which is what the wait exists to prevent. A cohort
+ // lookup that failed on every attempt must not fall through either, because the continue edge
+ // treats the person as outside the cohort. Bailing here also keeps the flag readable by
+ // shouldEndHogFlowExecution, since goToNextAction would replace currentAction with a fresh
+ // entry and drop it.
+ if (
+ invocation.state.currentAction?.delayUntilUnresolved ||
+ invocation.state.currentAction?.cohortLookupFailed
+ ) {
return
}
// Unless the current action's on_error is set to 'abort', we move to the next action instead of
@@ -1057,7 +1073,8 @@ export class HogFlowExecutorService {
private logExecutionErrorInfo(
result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
- error: Error
+ // executeCurrentAction stores the message string, not the Error, on result.error.
+ error: unknown
): string {
const invocation = result.invocation
const currentActionId = invocation.state.currentAction?.id
@@ -1068,6 +1085,7 @@ export class HogFlowExecutorService {
? `. This workflow was triggered by [Event:${invocation.state.event?.uuid}|${invocation.state.event?.event?.replaceAll('|', '')}|${invocation.state.event?.timestamp}]`
: ''
- return `Workflow encountered an error: ${error.message} at ${currentAction ? actionIdForLogging(currentAction) : 'unknown action'}${triggeredByEvent}`
+ const message = error instanceof Error ? error.message : String(error)
+ return `Workflow encountered an error: ${message} at ${currentAction ? actionIdForLogging(currentAction) : 'unknown action'}${triggeredByEvent}`
}
}
diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.ts
index 43619252bbfb..701531552087 100644
--- a/nodejs/src/cdp/types.ts
+++ b/nodejs/src/cdp/types.ts
@@ -481,6 +481,13 @@ export type HogFlowInvocationContext = {
// on_error says, because "no date" is not an ambiguous failure to carry on from: continuing would
// run the next step immediately, which for a "N days before X" message is worse than not sending.
delayUntilUnresolved?: boolean
+ // Failed cohort membership lookups so far on this step. The step retries on a backoff while
+ // this stays under the limit, because a lookup timeout is usually a short stall.
+ cohortLookupAttempts?: number
+ // Set when a cohort membership lookup failed on every attempt. The run aborts on it whatever
+ // on_error says, because the default 'continue' takes the continue edge, and on a branch that
+ // edge sends a member down the path for people outside the cohort.
+ cohortLookupFailed?: boolean
hogFunctionState?: CyclotronJobInvocationHogFunctionContext
// Set by the subscription matcher consumer when it wakes a wait_until_condition
// job because a matching event arrived (as opposed to a scheduled timeout firing).