:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(workflows): stop the run when a cohort lookup fails instead of routing (#116200)
Harley Alexander5e2fecf7e9e8
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.tsindex edbf876c8f46..c2207b09e344 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts@@ -89,6 +89,8 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {startedAtTimestamp: 999,awaitingResume: { key: 'old-key', deadlineAt: '2020-01-01T00:00:00Z', dispatch: { id: 'old-task' } },resumeResult: { key: 'old-key', status: 'failed' },+ cohortLookupAttempts: 2,+ cohortLookupFailed: true,hogFunctionState: {globals: { event: { uuid: 'evt-1' }, source: { name: 'Email', url: '' } },timings: [],diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.tsindex f4a73a0cb61a..18ef155c4700 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts@@ -795,10 +795,15 @@ export class RerunPaginatorService {// with "No recipient identifier found". Dropping it makes the// action re-enter fresh, so inputs re-render against the current// config, which is what a rerun intends anyway.+ // The cohort lookup counters are dropped too. A run that aborted after its last+ // lookup attempt would otherwise restart with no retries left, and the leftover+ // cohortLookupFailed flag would turn an unrelated error on that step into an abort.const {hogFunctionState: _stripped,awaitingResume: _awaitingResume,resumeResult: _resumeResult,+ cohortLookupAttempts: _cohortLookupAttempts,+ cohortLookupFailed: _cohortLookupFailed,...restoredCurrentAction} = persistedState.currentAction ?? {}diff --git a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.tsindex cd248eab6ffa..88a922bc2cfe 100644--- a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts+++ b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.test.ts@@ -295,10 +295,32 @@ describe('action.conditional_branch', () => {expect(repository.calls).toEqual([])})- it('propagates a lookup failure instead of routing on a made-up answer', async () => {+ it.each([+ [0, 10],+ [1, 60],+ ])('retries a failed lookup after %s earlier failures in %ss instead of routing', async (earlier, delay) => {+ const repository = new FakeCohortMembershipRepository([], new Error('lookup timed out'))+ invocation.state.currentAction!.cohortLookupAttempts = earlier || undefined+ action.config.conditions = [{ filters: cohortConditionFilters('inCohort') }]++ const result = await new ConditionalBranchHandler(repository).execute({+ invocation,+ action,+ result: createInvocationResult(invocation),+ })++ expect(result.nextAction).toBeUndefined()+ expect(result.scheduledAt).toEqual(DateTime.now().plus({ seconds: delay }))+ expect(invocation.state.currentAction!.cohortLookupAttempts).toBe(earlier + 1)+ expect(invocation.state.currentAction!.cohortLookupFailed).toBeUndefined()+ })++ it('marks the step failed and throws once every lookup attempt failed', async () => {const repository = new FakeCohortMembershipRepository([], new Error('lookup timed out'))+ invocation.state.currentAction!.cohortLookupAttempts = 2await expect(executeWithRepository(repository, 'inCohort')).rejects.toThrow('lookup timed out')+ expect(invocation.state.currentAction!.cohortLookupFailed).toBe(true)})})diff --git a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.tsindex 6691bd658b02..d6119b28ef12 100644--- a/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts+++ b/nodejs/src/cdp/services/hogflows/actions/conditional_branch.ts@@ -3,7 +3,7 @@ import { Counter } from 'prom-client'import { HogFlowAction } from '~/cdp/schema/hogflow'import { CohortMembershipRepository } from '~/cdp/services/cohorts/cohort-membership-repository'-import { CyclotronJobInvocationHogFlow, HogFunctionFilters } from '~/cdp/types'+import { CyclotronJobInvocationHogFlow, CyclotronJobInvocationResult, HogFunctionFilters } from '~/cdp/types'import { filterFunctionInstrumented } from '~/cdp/utils/hog-function-filtering'import { findContinueAction, findNextAction, isEvaluableCondition } from '../hogflow-utils'@@ -41,6 +41,25 @@ export const counterHogflowRekeyWake = new Counter({labelNames: ['outcome'],})+// Outcome of a step whose cohort membership lookup failed. 'retried' re-runs the step after a backoff;+// 'aborted' means every attempt failed and the run stopped. Sustained 'aborted' readings mean the+// behavioral cohorts DB is unreachable from the workers, and every cohort branch is stopping runs.+export const counterHogflowCohortLookupFailure = new Counter({+ name: 'cdp_hogflow_cohort_lookup_failure_total',+ help: 'Cohort membership lookups that failed in a conditional_branch or wait, by outcome.',+ labelNames: ['outcome'],+})++// Seconds to wait before each retry. One more attempt than there are delays: the first try runs inline.+export const COHORT_LOOKUP_RETRY_DELAYS_SECONDS = [10, 60]++export class CohortMembershipLookupFailedError extends Error {+ constructor(cause: unknown) {+ super(`Cohort membership lookup failed: ${String(cause)}`, { cause })+ this.name = 'CohortMembershipLookupFailedError'+ }+}+export class ConditionalBranchHandler implements ActionHandler {constructor(private cohortMembershipRepository: CohortMembershipRepository) {}@@ -118,14 +137,26 @@ export class ConditionalBranchHandler implements ActionHandler {},}- const conditionResult = await checkConditions(- invocation,- conditionalAction,- this.createMemberCohortIdsLoader(invocation),- action.type === 'wait_until_condition' && action.config.max_wait_duration- ? { maxWaitDuration: action.config.max_wait_duration }- : undefined- )+ let conditionResult: Awaited<ReturnType<typeof checkConditions>>+ try {+ conditionResult = await checkConditions(+ invocation,+ conditionalAction,+ this.createMemberCohortIdsLoader(invocation),+ action.type === 'wait_until_condition' && action.config.max_wait_duration+ ? { maxWaitDuration: action.config.max_wait_duration }+ : undefined+ )+ } catch (error) {+ if (error instanceof CohortMembershipLookupFailedError) {+ return retryOrFailCohortLookup(invocation, result, error)+ }+ throw error+ }+ if (invocation.state.currentAction) {+ delete invocation.state.currentAction.cohortLookupAttempts+ delete invocation.state.currentAction.cohortLookupFailed+ }const isWait = action.type === 'wait_until_condition'@@ -159,7 +190,11 @@ export class ConditionalBranchHandler implements ActionHandler {if (!loaded) {const personUuid = invocation.person?.id ?? invocation.state.personIdloaded = personUuid- ? this.cohortMembershipRepository.getMemberCohortIds(invocation.hogFlow.team_id, personUuid)+ ? this.cohortMembershipRepository+ .getMemberCohortIds(invocation.hogFlow.team_id, personUuid)+ .catch((error) => {+ throw new CohortMembershipLookupFailedError(error)+ }): Promise.resolve([])}return loaded@@ -167,6 +202,36 @@ export class ConditionalBranchHandler implements ActionHandler {}}+// A failed lookup must not route the person: no answer is not the same as "not a member". The step+// re-runs after a backoff, and once the attempts run out it marks cohortLookupFailed and rethrows, so+// the executor aborts the run instead of following on_error's default continue edge.+function retryOrFailCohortLookup(+ invocation: CyclotronJobInvocationHogFlow,+ result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,+ error: CohortMembershipLookupFailedError+): ActionHandlerResult {+ const currentAction = invocation.state.currentAction+ const failedAttempts = (currentAction?.cohortLookupAttempts ?? 0) + 1+ const retryDelaySeconds = COHORT_LOOKUP_RETRY_DELAYS_SECONDS[failedAttempts - 1]++ if (currentAction && retryDelaySeconds !== undefined) {+ currentAction.cohortLookupAttempts = failedAttempts+ counterHogflowCohortLookupFailure.labels('retried').inc()+ result.logs.push({+ level: 'warn',+ timestamp: DateTime.now(),+ message: `Could not look up cohort membership (${String(error.cause)}). Retrying in ${retryDelaySeconds}s`,+ })+ return { scheduledAt: DateTime.now().plus({ seconds: retryDelaySeconds }) }+ }++ if (currentAction) {+ currentAction.cohortLookupFailed = true+ }+ counterHogflowCohortLookupFailure.labels('aborted').inc()+ throw error+}+// Operation.CALL_GLOBAL from @posthog/hogvm, which is a const enum and can't be imported// under isolatedModulesconst CALL_GLOBAL = 2@@ -228,7 +293,7 @@ export async function checkConditions(for (const [index, condition] of action.config.conditions.entries()) {// Loaded only when evaluation actually reaches a cohort condition, so a run whose earlier// condition matches never touches the behavioral cohorts DB. A lookup failure throws here- // on purpose (following the action's on_error) instead of guessing non-membership; the+ // on purpose, and the handler retries and then aborts instead of guessing non-membership; the// inCohort/notInCohort STL functions read the resulting cohort_ids global. The matcher does// not watch cohort membership, so a wait gated on a cohort advances at its deadline.const cohortGlobals =diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.tsindex 7ef5817d7ef0..0c97d4225854 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts@@ -55,6 +55,7 @@ const cleanLogs = (logs: string[]): string[] => {describe('Hogflow Executor', () => {let executor: HogFlowExecutorService+ let cohortMembershipRepository: CohortMembershipRepositorylet hogFlowFunctionsService: HogFlowFunctionsServicelet hub: Hublet team: Team@@ -173,14 +174,14 @@ describe('Hogflow Executor', () => {await insertHogFunctionTemplate(hub.postgres, posthogCaptureTemplate)- const stubCohortMembershipRepository: CohortMembershipRepository = {+ cohortMembershipRepository = {getMemberCohortIds: () => Promise.resolve([]),}executor = new HogFlowExecutorService(hogFlowFunctionsService,recipientPreferencesService,emailValidationService,- stubCohortMembershipRepository,+ cohortMembershipRepository,hub.integrationManager)})@@ -1926,6 +1927,84 @@ describe('Hogflow Executor', () => {]))})++ it('does NOT continue past a branch whose cohort lookup failed on every attempt, despite on_error continue', async () => {+ jest.spyOn(cohortMembershipRepository, 'getMemberCohortIds').mockRejectedValue(+ new Error('lookup timed out')+ )+ const flow = new FixtureHogFlowBuilder()+ .withWorkflow({+ actions: {+ trigger: {+ type: 'trigger',+ config: {+ type: 'event',+ filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {},+ },+ },+ branch: {+ type: 'conditional_branch',+ config: {+ conditions: [+ {+ filters: {+ bytecode: [+ '_H',+ 1,+ 33,+ 42,+ 32,+ 'cohort_ids',+ 1,+ 1,+ 2,+ 'inCohort',+ 2,+ ],+ properties: [{ key: 'id', type: 'cohort', value: 42 }],+ },+ },+ ],+ },+ },+ in_cohort: { type: 'exit', config: {} },+ not_in_cohort: { type: 'exit', config: {} },+ },+ edges: [+ { from: 'trigger', to: 'branch', type: 'continue' },+ { from: 'branch', to: 'in_cohort', type: 'branch', index: 0 },+ { from: 'branch', to: 'not_in_cohort', type: 'continue' },+ ],+ })+ .build()+ expect(flow.actions.find((a) => a.id === 'branch')!.on_error).toBe('continue')++ const invocation = createExampleHogFlowInvocation(flow)+ invocation.state.currentAction = {+ id: 'branch',+ startedAtTimestamp: DateTime.now().toMillis(),+ cohortLookupAttempts: 2,+ }++ const result = await executor.execute(invocation)++ expect(result.error).toContain('lookup timed out')+ expect(result.finished).toBe(true)+ expect(result.invocation.state.currentAction?.id).toBe('branch')+ expect(result.logs.map((l) => l.message)).not.toEqual(+ expect.arrayContaining([expect.stringContaining('Workflow moved to action')])+ )+ expect(result.logs.map((l) => l.message)).toEqual(+ expect.arrayContaining([+ expect.stringContaining(+ 'Workflow is aborting because [Action:branch] could not look up cohort membership'+ ),+ expect.stringContaining(+ 'Workflow encountered an error: Cohort membership lookup failed: Error: lookup timed out'+ ),+ ])+ )+ })})})})diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.tsindex 12f77108301a..32b5b912b362 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts@@ -451,6 +451,17 @@ export class HogFlowExecutorService {lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId} could not work out the date to wait for`,})+ } else if (result.invocation.state.currentAction?.cohortLookupFailed) {+ // The default continue edge of a conditional branch is the path for people outside the+ // cohort, so carrying on would route this person on a lookup that never answered.+ shouldAbortAfterError = true+ logs.push({+ level: 'info',+ timestamp: DateTime.now(),+ message: `Workflow is aborting because ${+ lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId+ } could not look up cohort membership`,+ })}}@@ -818,10 +829,15 @@ export class HogFlowExecutorService {try {const { invocation } = result// A delay that could not work out when to continue must never fall through, whatever on_error- // says: the next step would run immediately, which is what the wait exists to prevent. Bailing- // here also keeps the flag readable by shouldEndHogFlowExecution, since goToNextAction would- // replace currentAction with a fresh entry and drop it.- if (invocation.state.currentAction?.delayUntilUnresolved) {+ // says: the next step would run immediately, which is what the wait exists to prevent. A cohort+ // lookup that failed on every attempt must not fall through either, because the continue edge+ // treats the person as outside the cohort. Bailing here also keeps the flag readable by+ // shouldEndHogFlowExecution, since goToNextAction would replace currentAction with a fresh+ // entry and drop it.+ if (+ invocation.state.currentAction?.delayUntilUnresolved ||+ invocation.state.currentAction?.cohortLookupFailed+ ) {return}// Unless the current action's on_error is set to 'abort', we move to the next action instead of@@ -1057,7 +1073,8 @@ export class HogFlowExecutorService {private logExecutionErrorInfo(result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,- error: Error+ // executeCurrentAction stores the message string, not the Error, on result.error.+ error: unknown): string {const invocation = result.invocationconst currentActionId = invocation.state.currentAction?.id@@ -1068,6 +1085,7 @@ export class HogFlowExecutorService {? `. This workflow was triggered by [Event:${invocation.state.event?.uuid}|${invocation.state.event?.event?.replaceAll('|', '')}|${invocation.state.event?.timestamp}]`: ''- return `Workflow encountered an error: ${error.message} at ${currentAction ? actionIdForLogging(currentAction) : 'unknown action'}${triggeredByEvent}`+ const message = error instanceof Error ? error.message : String(error)+ return `Workflow encountered an error: ${message} at ${currentAction ? actionIdForLogging(currentAction) : 'unknown action'}${triggeredByEvent}`}}diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.tsindex 43619252bbfb..701531552087 100644--- a/nodejs/src/cdp/types.ts+++ b/nodejs/src/cdp/types.ts@@ -481,6 +481,13 @@ export type HogFlowInvocationContext = {// on_error says, because "no date" is not an ambiguous failure to carry on from: continuing would// run the next step immediately, which for a "N days before X" message is worse than not sending.delayUntilUnresolved?: boolean+ // Failed cohort membership lookups so far on this step. The step retries on a backoff while+ // this stays under the limit, because a lookup timeout is usually a short stall.+ cohortLookupAttempts?: number+ // Set when a cohort membership lookup failed on every attempt. The run aborts on it whatever+ // on_error says, because the default 'continue' takes the continue edge, and on a branch that+ // edge sends a member down the path for people outside the cohort.+ cohortLookupFailed?: booleanhogFunctionState?: CyclotronJobInvocationHogFunctionContext// Set by the subscription matcher consumer when it wakes a wait_until_condition// job because a matching event arrived (as opposed to a scheduled timeout firing).