Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

54.3 KB
Raw Blame
import { get } from 'lodash'
import { DateTime } from 'luxon'
import { Counter } from 'prom-client'

import { HogFlow, HogFlowAction, isRowScopedTrigger } from '~/cdp/schema/hogflow'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'

import {
    ConversionWatcherRow,
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationResult,
    HogFunctionCapturedEvent,
    HogFunctionFilterGlobals,
    HogFunctionInvocationGlobals,
    InvocationBuildFailure,
    LogEntry,
    LogEntryLevel,
    MessageAssetRow,
    MinimalAppMetric,
    MinimalLogEntry,
    WarehouseWebhookPayload,
} from '../../types'
import { convertToHogFunctionFilterGlobal, filterFunctionInstrumented } from '../../utils/hog-function-filtering'
import { createInvocationResult } from '../../utils/invocation-utils'
import {
    COHORT_MEMBERSHIP_CHANGED_EVENT,
    COHORT_MEMBERSHIP_VERSION_PROPERTY,
    CohortMembershipRepository,
    StoredCohortMembership,
} from '../cohorts/cohort-membership-repository'
import { HogExecutorExecuteAsyncOptions } from '../hog-executor-async.service'
import { EmailStatusService } from '../messaging/email-status.service'
import { EmailValidationService } from '../messaging/email-validation.service'
import { RecipientPreferencesService } from '../messaging/recipient-preferences.service'
import { CdpUsageReporterService } from '../usage/cdp-usage-reporter.service'
import { ActionHandler } from './actions/action.interface'
import { ConditionalBranchHandler, counterHogflowCohortLookupFailure } from './actions/conditional_branch'
import { DelayHandler } from './actions/delay'
import { ExitHandler } from './actions/exit.handler'
import { HogFunctionHandler } from './actions/hog_function'
import { RandomCohortBranchHandler } from './actions/random_cohort_branch'
import { SlackAppLookup, TriggerHandler } from './actions/trigger.handler'
import { WaitUntilTimeWindowHandler } from './actions/wait_until_time_window'
import { buildConversionWatcher } from './conversion-watcher'
import { HogFlowDuplicateObserverService } from './hogflow-duplicate-observer.service'
import { HogFlowFunctionsService } from './hogflow-functions.service'
import { hasSensitiveMessageStep } from './hogflow-sensitivity'
import {
    WorkflowChangedError,
    actionIdForLogging,
    aiTaskLogToken,
    ensureCurrentAction,
    findContinueAction,
    shouldSkipAction,
    trackE2eLag,
} from './hogflow-utils'

export const MAX_ACTION_STEPS_HARD_LIMIT = 1000

// Deliberately unlabelled: fleet-wide volume is the signal; which flow/step redirected where is in
// the run's log line, where unbounded cardinality belongs.
const counterRedirectApplied = new Counter({
    name: 'cdp_hogflow_redirect_applied',
    help: 'A run parked on a deleted step was redirected to its surviving successor',
})
// The API guarantees redirect targets exist in the same flow row; nonzero means that invariant broke
// (the affected runs still exit gracefully rather than follow the bad entry).
const counterRedirectTargetMissing = new Counter({
    name: 'cdp_hogflow_redirect_target_missing',
    help: 'A redirect map entry pointed at an action missing from the live graph',
})

const TRIGGER_MISMATCH_EXITS: ReadonlySet<string> = new Set([
    'exit_on_trigger_not_matched',
    'exit_on_trigger_not_matched_or_conversion',
])

// The email service sets teamCapped when the team sending cap delays one send, and that send reads it from the
// job's stored metadata. The actions after it in the same dequeue start from this executor's result, so the
// flag stays off them. Otherwise their queue waits would also count as cap delays.
function withoutTeamCapped(
    metadata: CyclotronJobInvocationHogFlow['queueMetadata']
): CyclotronJobInvocationHogFlow['queueMetadata'] {
    if (!metadata || !('teamCapped' in metadata)) {
        return metadata
    }
    const { teamCapped: _teamCapped, ...rest } = metadata
    return rest
}

export function createHogFlowInvocation(
    globals: HogFunctionInvocationGlobals,
    hogFlow: HogFlow,
    filterGlobals: HogFunctionFilterGlobals
): CyclotronJobInvocationHogFlow {
    // Build default variables from hogFlow, then merge in any provided in globals.variables
    const defaultVariables =
        hogFlow.variables?.reduce(
            (acc, variable) => {
                acc[variable.key] = variable.default || null
                return acc
            },
            {} as Record<string, any>
        ) || {}

    const mergedVariables = {
        ...defaultVariables,
        ...(globals.variables || {}),
    }

    return {
        id: new UUIDT().toString(),
        state: {
            event: globals.event,
            actionStepCount: 0,
            customerTaskIdempotencyVersion: 1,
            variables: mergedVariables,
            // Seeded at run start and persisted with the state, because the flow itself isn't: the
            // job is re-loaded by functionId on every resume, so by the time a conversion lands the
            // manager may be serving a newer version. A message step re-pins this to the version
            // that actually sent (see HogFunctionHandler); until then this is the best answer.
            flowVersion: hogFlow.version,
            ...(hasSensitiveMessageStep(hogFlow) ? { sensitive: true } : {}),
        },
        teamId: hogFlow.team_id,
        // The version lives on `state.flowVersion` rather than here — functionId is the cyclotron
        // lookup key and re-loads the current flow by design.
        functionId: hogFlow.id,
        hogFlow,
        person: globals.person, // This is outside of state as we don't persist it
        groups: globals.groups, // Same as person: in-memory only (test path); real execution re-resolves on dequeue
        filterGlobals,
        queue: 'hogflow',
        queuePriority: 1,
    }
}

export class HogFlowExecutorService {
    private readonly actionHandlers: Record<HogFlowAction['type'], ActionHandler>
    private readonly duplicateObserver: HogFlowDuplicateObserverService | null
    private readonly hogFlowFunctionsService: HogFlowFunctionsService
    private readonly cohortMembershipRepository: CohortMembershipRepository

    constructor(
        hogFlowFunctionsService: HogFlowFunctionsService,
        recipientPreferencesService: RecipientPreferencesService,
        emailValidationService: EmailValidationService,
        cohortMembershipRepository: CohortMembershipRepository,
        integrationManager: SlackAppLookup,
        // Required, so that a caller which builds the executor without it fails to compile. Pass null
        // only where no email status is recorded.
        emailStatusService: EmailStatusService | null,
        duplicateObserver?: HogFlowDuplicateObserverService,
        usageReporter?: CdpUsageReporterService,
        options: { awaitedStepsEnabled?: boolean } = {}
    ) {
        this.hogFlowFunctionsService = hogFlowFunctionsService
        this.cohortMembershipRepository = cohortMembershipRepository
        this.duplicateObserver = duplicateObserver ?? null
        const hogFunctionHandler = new HogFunctionHandler(
            hogFlowFunctionsService,
            recipientPreferencesService,
            emailValidationService,
            'fetch',
            null,
            usageReporter,
            options
        )
        const hogFunctionEmailHandler = new HogFunctionHandler(
            hogFlowFunctionsService,
            recipientPreferencesService,
            emailValidationService,
            'email',
            emailStatusService,
            usageReporter
        )
        const hogFunctionSmsHandler = new HogFunctionHandler(
            hogFlowFunctionsService,
            recipientPreferencesService,
            emailValidationService,
            'sms',
            null,
            usageReporter,
            options
        )
        const hogFunctionPushHandler = new HogFunctionHandler(
            hogFlowFunctionsService,
            recipientPreferencesService,
            emailValidationService,
            'push',
            null,
            usageReporter
        )

        this.actionHandlers = {
            trigger: new TriggerHandler(integrationManager),
            conditional_branch: new ConditionalBranchHandler(cohortMembershipRepository),
            wait_until_condition: new ConditionalBranchHandler(cohortMembershipRepository),
            delay: new DelayHandler(),
            wait_until_time_window: new WaitUntilTimeWindowHandler(),
            random_cohort_branch: new RandomCohortBranchHandler(),
            function: hogFunctionHandler,
            function_sms: hogFunctionSmsHandler,
            function_push: hogFunctionPushHandler,
            function_email: hogFunctionEmailHandler,
            exit: new ExitHandler(),
        }
    }

    // Decrypted secret input values across the flow's function actions, for redacting test-run logs.
    async getSensitiveValues(hogFlow: HogFlow): Promise<string[]> {
        return this.hogFlowFunctionsService.getSensitiveValues(hogFlow)
    }

    async buildHogFlowInvocations(
        hogFlows: HogFlow[],
        triggerGlobals: HogFunctionInvocationGlobals
    ): Promise<{
        invocations: CyclotronJobInvocationHogFlow[]
        metrics: MinimalAppMetric[]
        logs: LogEntry[]
        buildFailures: InvocationBuildFailure[]
    }> {
        const metrics: MinimalAppMetric[] = []
        const logs: LogEntry[] = []
        const invocations: CyclotronJobInvocationHogFlow[] = []
        const buildFailures: InvocationBuildFailure[] = []

        // TRICKY: The frontend generates filters matching the Clickhouse event type so we are converting back
        const filterGlobals = convertToHogFunctionFilterGlobal(triggerGlobals)

        // Trigger-source compatibility is decided by the pipeline's eligibilityFn (see
        // HogFlowInvocationPipeline). Flows that reach this loop are assumed to be source-compatible
        // with the given globals — the executor's job is just to evaluate filter bytecode.
        for (const hogFlow of hogFlows) {
            const trigger = hogFlow.trigger

            if (trigger.type === 'cohort') {
                // A cohort trigger has no filters to evaluate: eligibility already matched the membership change.
                const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
                // A membership change carries no distinct_id, so the worker loads the person by id.
                invocation.state.personId = triggerGlobals.person?.id
                invocations.push(invocation)
                continue
            }

            // Defensive: only the trigger types that carry `filters` make it through eligibility.
            // isRowScopedTrigger covers internal-event too, alongside the two warehouse types.
            if (trigger.type !== 'event' && !isRowScopedTrigger(trigger)) {
                continue
            }

            const filterResults = await filterFunctionInstrumented({
                fn: hogFlow,
                filters: trigger.filters,
                filterGlobals,
                caller: 'build_hogflow_invocations',
            })

            // Add any generated metrics and logs to our collections. These are queued straight by the
            // pipeline rather than riding an invocation result, so the workflow version is stamped here
            // — otherwise `filtered` would be missing from the per-version series, and a trigger change
            // that filters everyone out would look identical to no traffic at all.
            metrics.push(
                ...filterResults.metrics.map((metric) => ({
                    ...metric,
                    app_source_version: { id: hogFlow.id, version: hogFlow.version },
                }))
            )
            logs.push(...filterResults.logs)

            // Checked against undefined, not for truthiness: a thrown error whose message is empty is
            // still a failure, and treating it as success drops the event with no record of it.
            if (filterResults.error !== undefined) {
                buildFailures.push({
                    sourceId: hogFlow.id,
                    sourceKind: 'hog_flow',
                    step: 'filter',
                    error: String(filterResults.error),
                })
            }

            if (!filterResults.match) {
                continue
            }

            const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
            invocations.push(invocation)
        }

        return {
            invocations,
            metrics,
            logs,
            buildFailures,
        }
    }

    private async observeDuplicateInvocation(
        invocation: CyclotronJobInvocationHogFlow,
        currentAction: HogFlowAction
    ): Promise<void> {
        await this.duplicateObserver?.observe(invocation, currentAction)
    }

    async execute(
        invocation: CyclotronJobInvocationHogFlow
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>> {
        let result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow> | null = null
        const metrics: MinimalAppMetric[] = []
        const logs: MinimalLogEntry[] = []
        const capturedPostHogEvents: HogFunctionCapturedEvent[] = []
        const warehouseWebhookPayloads: WarehouseWebhookPayload[] = []
        const messageAssets: MessageAssetRow[] = []
        const conversionWatchers: ConversionWatcherRow[] = []

        // A run that started before a step was protected must keep it out of its snapshot too, even when
        // a recovery path later holds only the workflow id.
        if (hasSensitiveMessageStep(invocation.hogFlow)) {
            invocation.state.sensitive = true
        }

        // A run enrolls on its first execution, before any exit check: a run that exits immediately
        // still consumed an enrollment, and `triggered` already counted it, so leaving it out would
        // overstate the rate.
        const watcher = buildConversionWatcher(invocation)
        const enrollmentConversion = watcher ? await this.conversionAtEnrollment(invocation) : null

        const earlyExitResult = await this.shouldExitEarly(invocation)
        if (earlyExitResult) {
            this.recordEnrollment(
                enrollmentConversion,
                watcher,
                earlyExitResult.metrics,
                earlyExitResult.capturedPostHogEvents,
                earlyExitResult.conversionWatchers
            )
            return earlyExitResult
        }
        this.recordEnrollment(enrollmentConversion, watcher, metrics, capturedPostHogEvents, conversionWatchers)

        // Routing-only reschedule: the previous dequeue moved this job onto a dedicated queue
        // (e.g. 'email' for SES rate-limit gating) and is continuing the same action. Suppress
        // the redundant trigger log — the customer-visible story should be one Resuming line
        // per real wake (delay, wait_until_condition, throttle retry), not a second one for
        // an internal queue transition. The flag stays set so executeCurrentAction can also
        // suppress its "Executing action..." debug log on this same continuation; it clears
        // the flag itself after reading so subsequent actions on this dequeue log normally.
        if (!invocation.state.currentAction?.routingOnlyReschedule) {
            logs.push(this.logExecutionTriggerInfo(invocation))
        }

        while (!result || !result.finished) {
            const nextInvocation: CyclotronJobInvocationHogFlow = result?.invocation ?? invocation

            // Here we could be continuing the hog function side of things?
            result = await this.executeCurrentAction(nextInvocation)

            if (result.finished) {
                if (result.error) {
                    this.log(result, 'error', this.logExecutionErrorInfo(result, result.error))
                } else {
                    this.log(result, 'info', `Workflow completed`)
                }

                trackE2eLag(result)
            }

            logs.push(...result.logs)
            metrics.push(...result.metrics)
            capturedPostHogEvents.push(...result.capturedPostHogEvents)
            warehouseWebhookPayloads.push(...result.warehouseWebhookPayloads)
            messageAssets.push(...result.messageAssets)
            conversionWatchers.push(...result.conversionWatchers)

            if (this.shouldEndHogFlowExecution(result, logs)) {
                break
            }
        }

        result.logs = logs
        result.metrics = metrics
        result.capturedPostHogEvents = capturedPostHogEvents
        result.warehouseWebhookPayloads = warehouseWebhookPayloads
        result.messageAssets = messageAssets
        result.conversionWatchers = conversionWatchers

        return result
    }

    /**
     * A property goal already satisfied when the run enrolls can never be counted from its watcher.
     * The matcher reads the enrollment event off Kafka long before the run has written the row, and
     * on `exit_on_conversion` the run is gone before any later event could claim it — so the run
     * would record `early_exit` and no `conversion` at all.
     *
     * Counting it here keeps the two paths disjoint: a run that converts at enrollment writes no
     * watcher, so it is still counted exactly once.
     */
    private async conversionAtEnrollment(
        invocation: CyclotronJobInvocationHogFlow
    ): Promise<{ metric: MinimalAppMetric; event: HogFunctionCapturedEvent | null } | null> {
        const { hogFlow } = invocation
        if (!invocation.person || !hogFlow.conversion?.filters?.length || !hogFlow.conversion.bytecode?.length) {
            return null
        }
        const filterResult = await filterFunctionInstrumented({
            fn: hogFlow,
            filters: { bytecode: hogFlow.conversion.bytecode, properties: hogFlow.conversion.filters },
            filterGlobals: invocation.filterGlobals,
            caller: 'hogflow_conversion',
        })
        if (!filterResult.match) {
            return null
        }
        const distinctId = invocation.state.event?.distinct_id
        return {
            metric: {
                team_id: hogFlow.team_id,
                app_source_id: invocation.parentRunId ?? hogFlow.id,
                instance_id: hogFlow.id,
                metric_kind: 'other',
                metric_name: 'conversion',
                count: 1,
            },
            // Mirrors the matcher's payload so both paths produce the same billable event. A run with
            // no distinct_id has nothing to attribute the capture to; the metric still counts.
            event: distinctId
                ? {
                      team_id: hogFlow.team_id,
                      event: '$workflows_conversion',
                      distinct_id: distinctId,
                      timestamp: new Date().toISOString(),
                      properties: {
                          $workflow_id: hogFlow.id,
                          $workflow_run_id: invocation.id,
                          $workflow_version: invocation.state.flowVersion ?? hogFlow.version,
                          $workflow_conversion_type: 'property',
                      },
                  }
                : null,
        }
    }

    private recordEnrollment(
        enrollmentConversion: { metric: MinimalAppMetric; event: HogFunctionCapturedEvent | null } | null,
        watcher: ConversionWatcherRow | null,
        metrics: MinimalAppMetric[],
        capturedPostHogEvents: HogFunctionCapturedEvent[],
        conversionWatchers: ConversionWatcherRow[]
    ): void {
        if (enrollmentConversion) {
            metrics.push(enrollmentConversion.metric)
            if (enrollmentConversion.event) {
                capturedPostHogEvents.push(enrollmentConversion.event)
            }
            return
        }
        if (watcher) {
            conversionWatchers.push(watcher)
        }
    }

    private shouldEndHogFlowExecution(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        logs: MinimalLogEntry[]
    ): boolean {
        const finishedWithoutError = result.finished && !result.error
        const delayScheduled = Boolean(result.invocation.queueScheduledAt)

        let shouldAbortAfterError = false
        if (result.error) {
            const lastExecutedActionId = result.invocation.state.currentAction?.id
            const lastExecutedAction = result.invocation.hogFlow.actions.find((a) => a.id === lastExecutedActionId)

            if (lastExecutedAction?.on_error === 'abort') {
                shouldAbortAfterError = true
                logs.push({
                    level: 'info',
                    timestamp: DateTime.now(),
                    message: `Workflow is aborting due to ${actionIdForLogging(lastExecutedAction)} error handling setting being set to abort on error`,
                })
            } else if (result.invocation.state.currentAction?.delayUntilUnresolved) {
                // A delay that cannot work out when to continue overrides on_error's default of carrying
                // on: the next step would run immediately, which is the one outcome the wait exists to
                // prevent. See delayUntilUnresolved.
                shouldAbortAfterError = true
                logs.push({
                    level: 'info',
                    timestamp: DateTime.now(),
                    message: `Workflow is aborting because ${
                        lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId
                    } could not work out the date to wait for`,
                })
            } else if (result.invocation.state.currentAction?.cohortLookupFailed) {
                // The default continue edge of a conditional branch is the path for people outside the
                // cohort, so carrying on would route this person on a lookup that never answered.
                shouldAbortAfterError = true
                logs.push({
                    level: 'info',
                    timestamp: DateTime.now(),
                    message: `Workflow is aborting because ${
                        lastExecutedAction ? actionIdForLogging(lastExecutedAction) : lastExecutedActionId
                    } could not look up cohort membership`,
                })
            }
        }

        /**
         * If one of the following happens:
         * - we have finished the flow successfully
         * - something has been scheduled to run later
         * - there was an error during the action and the action's on_error is set to 'abort'
         * - we have reached the max async functions count
         *
         * then we break the loop
         */
        return finishedWithoutError || delayScheduled || shouldAbortAfterError
    }

    /**
     * Whether the person still matches a cohort trigger: in the cohort for an "entered" trigger, out of it
     * for a "left" one. Undefined when there is no trustworthy answer, which leaves the run alone.
     *
     * The membership writer and the trigger consumer read each change independently, so the table can still
     * hold the state from before the change that started this run. A read counts only once the table has
     * caught up with that change.
     */
    private async matchesCohortTrigger(
        invocation: CyclotronJobInvocationHogFlow,
        trigger: { cohort_id: number; transition: 'entered' | 'left' }
    ): Promise<boolean | undefined> {
        const personUuid = invocation.person?.id ?? invocation.state.personId
        if (!personUuid) {
            return undefined
        }
        const enrolledBy = invocation.state.event?.properties
        // Only the change that enrolled the run can still be pending in the table. A run enrolled by another
        // cohort or transition, because the trigger changed while it was parked, has nothing to wait for.
        const enrolledByThisTrigger =
            invocation.state.event?.event === COHORT_MEMBERSHIP_CHANGED_EVENT &&
            enrolledBy?.cohort_id === trigger.cohort_id &&
            enrolledBy?.status === trigger.transition
        const triggerVersion = enrolledByThisTrigger ? enrolledBy?.[COHORT_MEMBERSHIP_VERSION_PROPERTY] : undefined
        let membership: StoredCohortMembership | null
        try {
            membership = await this.cohortMembershipRepository.getCohortMembership(
                invocation.hogFlow.team_id,
                personUuid,
                trigger.cohort_id,
                typeof triggerVersion === 'string' ? triggerVersion : undefined
            )
        } catch (error) {
            // No answer is not "no longer in the cohort", so the run carries on and the next wake checks again.
            counterHogflowCohortLookupFailure.labels({ outcome: 'exit_check_skipped' }).inc()
            logger.warn('HogFlowExecutorService: cohort membership lookup failed during the exit check', {
                hogFlowId: invocation.hogFlow.id,
                error: String(error),
            })
            return undefined
        }

        const inCohort = membership?.inCohort ?? null
        const rowMatches = trigger.transition === 'entered' ? inCohort === true : inCohort === false
        // The row's version stamp shows whether the writer has applied the change that started this run.
        // Without a stamp, only a read that matches the trigger shows it, because the change flips the state.
        const caughtUp =
            !enrolledByThisTrigger ||
            invocation.state.cohortTriggerConfirmed === true ||
            (membership?.caughtUp ?? rowMatches)
        if (!caughtUp) {
            return undefined
        }
        invocation.state.cohortTriggerConfirmed = true
        // Once the table has caught up, a missing row means out of the cohort.
        return trigger.transition === 'entered' ? inCohort === true : inCohort !== true
    }

    /**
     * Determines if the invocation should exit early based on the hogflow's exit condition
     */
    private async shouldExitEarly(
        invocation: CyclotronJobInvocationHogFlow
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow> | null> {
        let earlyExitResult: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow> | null = null

        const { hogFlow, person } = invocation
        let shouldExit = false
        let exitReason = ''

        let triggerMatch: boolean | undefined = undefined
        let conversionMatch: boolean | undefined = undefined

        if (hogFlow.trigger.type === 'event' && hogFlow.trigger.filters && person) {
            const filterResult = await filterFunctionInstrumented({
                fn: hogFlow,
                filters: hogFlow.trigger.filters,
                filterGlobals: invocation.filterGlobals,
                caller: 'hogflow_exit_condition',
            })
            triggerMatch = filterResult.match
        }
        if (hogFlow.trigger.type === 'cohort' && TRIGGER_MISMATCH_EXITS.has(hogFlow.exit_condition)) {
            triggerMatch = await this.matchesCohortTrigger(invocation, hogFlow.trigger)
        }
        if (hogFlow.conversion?.filters?.length && person) {
            if (hogFlow.conversion.bytecode?.length) {
                const filterResult = await filterFunctionInstrumented({
                    caller: 'hogflow_exit_condition',
                    fn: hogFlow,
                    filters: {
                        bytecode: hogFlow.conversion.bytecode || [],
                        properties: hogFlow.conversion.filters || [],
                    },
                    filterGlobals: invocation.filterGlobals,
                })
                conversionMatch = filterResult.match
            } else {
                logger.error(
                    'HogFlowExecutorService: Conversion filters are set but no bytecode is provided. This means we cannot evaluate the conversion filters to determine if we should exit the flow.',
                    { hogFlowId: hogFlow.id }
                )
            }
        }
        // Counting lives entirely on the run's conversion watcher, which the matcher claims. The run
        // only decides whether to exit. Keeping the two apart is what makes the count survive the run
        // finishing, and stops a watcher and a still-live run from both counting the same conversion.
        //
        // Event-based conversion goals are evaluated by the subscription matcher (against the live
        // event stream), which flags the job when the conversion event fires. The property-based
        // check above can't see those, so honor the flag here. It is a one-shot signal ("the
        // conversion event just fired"), so consume it: clear it after reading so a later, unrelated
        // resume (e.g. after a subsequent delay) can't re-trigger an exit from a stale flag.
        if (invocation.state.conversionMatched) {
            conversionMatch = true
            invocation.state.conversionMatched = false
        }

        switch (hogFlow.exit_condition) {
            case 'exit_on_trigger_not_matched':
                if (triggerMatch === false) {
                    shouldExit = true
                    exitReason = `[Person:${invocation.person?.id ?? 'unknown'}|${invocation.person?.name ?? 'unknown'}] no longer matches trigger filters`
                }
                break
            case 'exit_on_conversion':
                if (conversionMatch === true) {
                    shouldExit = true
                    exitReason = `[Person:${invocation.person?.id ?? 'unknown'}|${invocation.person?.name ?? 'unknown'}] matches conversion filters`
                }
                break
            case 'exit_on_trigger_not_matched_or_conversion':
                if (triggerMatch === false || conversionMatch === true) {
                    shouldExit = true
                    exitReason =
                        triggerMatch === false
                            ? `[Person:${invocation.person?.id ?? 'unknown'}|${invocation.person?.name ?? 'unknown'}] no longer matches trigger filters`
                            : `[Person:${invocation.person?.id ?? 'unknown'}|${invocation.person?.name ?? 'unknown'}] matches conversion filters`
                }
                break
        }

        if (shouldExit && hogFlow.trigger.type === 'cohort' && triggerMatch === false) {
            const movement = hogFlow.trigger.transition === 'entered' ? 'left' : 'rejoined'
            exitReason = `[Person:${invocation.person?.id ?? 'unknown'}|${invocation.person?.name ?? 'unknown'}] ${movement} cohort ${hogFlow.trigger.cohort_id}`
        }

        if (shouldExit) {
            earlyExitResult = createInvocationResult<CyclotronJobInvocationHogFlow>(invocation)
            earlyExitResult.finished = true
            earlyExitResult.logs.push({
                level: 'info',
                timestamp: DateTime.now(),
                message: `Workflow exited early due to exit condition: ${hogFlow.exit_condition} (${exitReason})`,
            })
            earlyExitResult.metrics.push({
                team_id: hogFlow.team_id,
                app_source_id: invocation.parentRunId ?? hogFlow.id,
                instance_id: invocation.state?.currentAction?.id || 'exit_condition',
                metric_kind: 'other',
                metric_name: 'early_exit',
                count: 1,
            })
        }

        return earlyExitResult
    }

    public async executeCurrentAction(
        invocation: CyclotronJobInvocationHogFlow,
        options?: {
            hogExecutorOptions?: HogExecutorExecuteAsyncOptions
        }
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>> {
        // queuePriority is carried over explicitly: createInvocationResult resets it to
        // 0, and the email routing downstream stashes the entry priority as the value to
        // restore when a job leaves the email queue — a reset here would stash 0 for any
        // run whose email action isn't the first action executed.
        const result = createInvocationResult<CyclotronJobInvocationHogFlow>(invocation, {
            queuePriority: invocation.queuePriority,
            queueMetadata: invocation.queue === 'email' ? withoutTeamCapped(invocation.queueMetadata) : undefined,
        })
        result.finished = false // Typically we are never finished unless we error or exit

        try {
            const currentAction = ensureCurrentAction(invocation)

            if (await shouldSkipAction(invocation, currentAction)) {
                this.logAction(result, currentAction, 'info', `Skipped due to filter conditions`)
                this.goToNextAction(result, currentAction, findContinueAction(invocation), 'filtered')

                return result
            }

            await this.observeDuplicateInvocation(invocation, currentAction)

            // Routing-only reschedule continuation (see hog_function.ts): the previous dequeue
            // set this flag so the executor knows the current call is just resuming an action
            // that was momentarily parked to switch queues — not the start of a fresh action
            // step. Suppress the redundant "Executing action..." log and consume the flag so
            // subsequent actions (next handler returns nextAction → loop continues) log normally.
            if (invocation.state.currentAction?.routingOnlyReschedule) {
                invocation.state.currentAction.routingOnlyReschedule = false
            } else {
                result.logs.push({
                    level: 'debug',
                    message: `Executing action ${actionIdForLogging(currentAction)}`,
                    timestamp: DateTime.now(),
                })
            }
            // Deliberately an allowlisted context, not the full action/invocation: an action's
            // config.inputs can hold decrypted secrets (API keys, auth headers) that the encrypted_inputs
            // split keeps out of storage, and dumping them here would put them straight into worker logs.
            logger.debug('🦔', `[HogFlowActionRunner] Running action ${currentAction.type}`, {
                hogFlowId: invocation.hogFlow.id,
                invocationId: invocation.id,
                actionId: currentAction.id,
                actionType: currentAction.type,
            })

            const handler = this.actionHandlers[currentAction.type]
            if (!handler) {
                throw new Error(`Action type '${currentAction.type}' not supported`)
            }

            try {
                const handlerResult = await handler.execute({
                    invocation,
                    action: currentAction,
                    result,
                    hogExecutorOptions: options?.hogExecutorOptions,
                })

                // Stored before the error so `on_error: continue` still sees what the step returned.
                if (handlerResult.result) {
                    this.trackActionResult(result, currentAction, handlerResult.result)
                    result.execResult = handlerResult.result
                }

                if (handlerResult.error) {
                    throw handlerResult.error instanceof Error ? handlerResult.error : new Error(handlerResult.error)
                }

                if (handlerResult.finished) {
                    result.finished = true
                    result.skipped = handlerResult.skipped
                    // A non-matching trigger is filtered, while an exit is a successful finish.
                    this.trackActionMetric(result, currentAction, handlerResult.skipped ? 'filtered' : 'succeeded')
                }

                if (handlerResult.scheduledAt) {
                    this.scheduleInvocation(result, handlerResult.scheduledAt)
                }

                if (handlerResult.nextAction) {
                    this.goToNextAction(result, currentAction, handlerResult.nextAction, 'succeeded')
                }
            } catch (err) {
                // A live-edit WorkflowChangedError is not this action failing - the graph moved
                // underneath the run - so it skips the failure log/metric and is classified by the
                // outer catch. The same error from an untouched flow is a malformed definition and
                // keeps the failure treatment.
                if (!this.isLiveEditWorkflowChange(err, invocation)) {
                    // Add logs and metric specifically for this action
                    this.logAction(result, currentAction, 'error', `Errored: ${String(err)}`) // TODO: Is this enough detail?
                    this.trackActionMetric(result, currentAction, 'failed')
                }

                throw err
            }
        } catch (err) {
            // The workflow was edited underneath this run and its current step (or that step's next
            // edge) no longer exists. That's a user action, not a defect: skip the run forward to the
            // deleted step's surviving successor when the edit recorded one, otherwise finish the run
            // as a deliberate exit - no result.error, so it doesn't count towards the workflow's
            // failure rate - with its own metric so exits are attributable per workflow.
            if (this.isLiveEditWorkflowChange(err, invocation)) {
                if (this.maybeRedirectDeletedAction(result, invocation)) {
                    return result
                }
                result.finished = true
                this.log(
                    result,
                    'info',
                    `Workflow exited: the workflow was edited and this run's current step no longer exists (${err.message})`
                )
                result.metrics.push({
                    team_id: invocation.hogFlow.team_id,
                    app_source_id: invocation.parentRunId ?? invocation.hogFlow.id,
                    instance_id: invocation.state.currentAction?.id,
                    metric_kind: 'other',
                    metric_name: 'exited_workflow_changed',
                    count: 1,
                })

                return result
            }

            // The final catch - in this case we are always just logging the final outcome
            result.error = err.message
            result.finished = true // Explicitly set to true to prevent infinite loops
            // (a WorkflowChangedError from an untouched flow lands here too: the graph was malformed
            // all along, so it stays a failure the author can see rather than a quiet exit)

            this.maybeContinueToNextActionOnError(result)

            logger.error(
                '🦔',
                `[HogFlowExecutor] Error executing hog flow ${invocation.hogFlow.id} - ${invocation.hogFlow.name}. Event: '${invocation.state.event?.url}'`,
                err
            )
        }

        return result
    }

    // Skip-forward for deleted steps: when a live edit deleted the run's current action, the API
    // recorded its next surviving successor in the flow's action_redirects map. Move the run there
    // with a fresh step entry (so a delay/wait at the target parks from redirect time) and let the
    // execute() loop enter it; the caller falls back to the graceful exit when there's no entry (a
    // true dead end). Only ever reached behind isLiveEditWorkflowChange: the run's position is dead,
    // so entries keyed by a *surviving* action id can't match, and after the redirect the fresh
    // startedAtTimestamp means a second structural miss classifies as a plain failure, not a loop.
    private maybeRedirectDeletedAction(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        invocation: CyclotronJobInvocationHogFlow
    ): boolean {
        const deletedActionId = invocation.state.currentAction?.id
        const redirectId = deletedActionId ? invocation.hogFlow.action_redirects?.[deletedActionId] : undefined
        if (!deletedActionId || !redirectId) {
            return false
        }

        // The API keeps every map value present in the same row's actions, so a miss here means a
        // compute bug or torn data - don't trust the map, let the run exit gracefully instead.
        const target = invocation.hogFlow.actions.find((action) => action.id === redirectId)
        if (!target) {
            counterRedirectTargetMissing.inc()
            logger.warn('[HogFlowExecutor] Redirect target missing from live graph', {
                hogFlowId: invocation.hogFlow.id,
                deletedActionId,
                redirectId,
            })
            return false
        }

        result.finished = false
        result.invocation.state.actionStepCount++
        result.invocation.state.currentAction = {
            id: target.id,
            startedAtTimestamp: DateTime.now().toMillis(),
        }
        this.log(
            result,
            'info',
            `Workflow was edited and this run's current step was removed - continuing at ${actionIdForLogging(target)}`
        )
        result.metrics.push({
            team_id: invocation.hogFlow.team_id,
            app_source_id: invocation.parentRunId ?? invocation.hogFlow.id,
            instance_id: deletedActionId,
            metric_kind: 'other',
            metric_name: 'redirected_workflow_changed',
            count: 1,
        })
        counterRedirectApplied.inc()

        return true
    }

    // A structural lookup miss only counts as a live edit when the flow was actually updated after
    // the run arrived at its current step. Otherwise the graph was malformed from the start (a bad
    // save, a lenient draft in a test run) and hiding it as a deliberate exit would bury the defect.
    private isLiveEditWorkflowChange(err: unknown, invocation: CyclotronJobInvocationHogFlow): boolean {
        if (!(err instanceof WorkflowChangedError)) {
            return false
        }
        const stepStartedAt = invocation.state.currentAction?.startedAtTimestamp
        // updated_at is a Date from pg in production and epoch millis in fixtures - normalize
        const updatedAt = invocation.hogFlow.updated_at ? new Date(invocation.hogFlow.updated_at).getTime() : null
        return Boolean(stepStartedAt && updatedAt && updatedAt > stepStartedAt)
    }

    private goToNextAction(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        currentAction: HogFlowAction,
        nextAction: HogFlowAction,
        reason: 'filtered' | 'failed' | 'succeeded'
    ): CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow> {
        result.finished = false

        result.invocation.state.actionStepCount++
        // Update the state to be going to the next action
        result.invocation.state.currentAction = {
            id: nextAction.id,
            startedAtTimestamp: DateTime.now().toMillis(),
        }

        result.logs.push({
            level: 'info',
            timestamp: DateTime.now(),
            message: `Workflow moved to action ${actionIdForLogging(nextAction)}`,
        })

        this.trackActionMetric(result, currentAction, reason)

        return result
    }

    /**
     * Unless the action has on_error set to 'abort' we continue to the next action instead of failing the flow.
     * An action without on_error gets the default, which is to continue.
     */
    private maybeContinueToNextActionOnError(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>
    ): void {
        try {
            const { invocation } = result
            // A delay that could not work out when to continue must never fall through, whatever on_error
            // says: the next step would run immediately, which is what the wait exists to prevent. A cohort
            // lookup that failed on every attempt must not fall through either, because the continue edge
            // treats the person as outside the cohort. Bailing here also keeps the flag readable by
            // shouldEndHogFlowExecution, since goToNextAction would replace currentAction with a fresh
            // entry and drop it.
            if (
                invocation.state.currentAction?.delayUntilUnresolved ||
                invocation.state.currentAction?.cohortLookupFailed
            ) {
                return
            }
            // Unless the current action's on_error is set to 'abort', we move to the next action instead of
            // failing the flow. 'continue' is the default, so an action that never had on_error set gets it too.
            const currentAction = ensureCurrentAction(invocation)
            if (currentAction?.on_error !== 'abort') {
                const nextAction = findContinueAction(invocation)
                if (nextAction) {
                    this.logAction(
                        result,
                        currentAction,
                        'info',
                        `Continuing to next action ${actionIdForLogging(nextAction)} despite error due to error handling setting being set to continue on error`
                    )

                    /**
                     * TODO: Determine if we should track this as a 'succeeded' metric here or
                     * a new metric_name e.g. 'continued_after_error'
                     */
                    this.goToNextAction(result, currentAction, nextAction, 'succeeded')
                }
            }
        } catch (err) {
            logger.error('Error trying to continue to next action on error', { error: err })
        }
    }

    /**
     * Updates the scheduledAt field on the result to indicate that the invocation should be scheduled for the future
     */
    private scheduleInvocation(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        scheduledAt: DateTime
    ): CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow> {
        // If the result has scheduled for the future then we return that triggering a push back to the queue
        result.invocation.queueScheduledAt = scheduledAt
        result.finished = false
        // Routing-only reschedules (hog function moving the job onto a dedicated queue) don't
        // represent a workflow-author-visible pause — the next dequeue fires almost
        // immediately and continues the same action. Skip the "Workflow will pause until..."
        // log in that case so it doesn't surface as a pause the workflow never actually took.
        // Real pauses (delays, wait_until_condition, throttle retries) still log normally.
        if (!result.invocation.state.currentAction?.routingOnlyReschedule) {
            result.logs.push({
                level: 'info',
                timestamp: DateTime.now(),
                message: `Workflow will pause until ${scheduledAt.toUTC().toISO()}`,
            })
        }

        return result
    }

    private logAction(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        action: HogFlowAction,
        level: LogEntryLevel,
        message: string
    ): void {
        this.log(result, level, `${actionIdForLogging(action)} ${message}`)
    }

    private log(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        level: LogEntryLevel,
        message: string
    ): void {
        result.logs.push({
            level,
            timestamp: DateTime.now(),
            message,
        })
    }

    private trackActionMetric(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        action: HogFlowAction,
        metricName: 'failed' | 'succeeded' | 'filtered'
    ): void {
        result.metrics.push({
            team_id: result.invocation.hogFlow.team_id,
            app_source_id: result.invocation.parentRunId ?? result.invocation.hogFlow.id,
            instance_id: action.id,
            metric_kind: metricName === 'failed' ? 'failure' : metricName === 'succeeded' ? 'success' : 'other',
            metric_name: metricName,
            count: 1,
        })
    }

    private trackActionResult(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        action: HogFlowAction,
        actionResult: unknown
    ): void {
        // Normalize output_variable to an array for uniform handling
        const outputVars = Array.isArray(action.output_variable)
            ? action.output_variable
            : action.output_variable
              ? [action.output_variable]
              : []

        if (outputVars.length === 0) {
            return
        }

        if (!actionResult) {
            this.log(
                result,
                'warn',
                `An output variable was specified for [Action:${action.id}], but no output was returned.`
            )
            return
        }

        if (!result.invocation.state.variables) {
            result.invocation.state.variables = {}
        }

        const allStoredKeys: string[] = []

        for (const outputVar of outputVars) {
            if (!outputVar.key) {
                continue
            }

            const resolvedResult = outputVar.result_path ? get(actionResult, outputVar.result_path) : actionResult

            // When spread is true, store each property of the result as a separate variable
            if (
                outputVar.spread &&
                typeof resolvedResult === 'object' &&
                resolvedResult !== null &&
                !Array.isArray(resolvedResult)
            ) {
                const prefix = outputVar.key
                for (const [prop, value] of Object.entries(resolvedResult)) {
                    const spreadKey = `${prefix}_${prop}`
                    result.invocation.state.variables[spreadKey] = value
                    allStoredKeys.push(spreadKey)
                }
            } else {
                result.invocation.state.variables[outputVar.key] = resolvedResult
                allStoredKeys.push(outputVar.key)
            }
        }

        // Check that total variables are below 5KB
        const resultSize = Buffer.byteLength(JSON.stringify(result.invocation.state.variables), 'utf8')
        if (resultSize > 5120) {
            const keyNames = allStoredKeys.join(', ')
            this.log(
                result,
                'error',
                `Total variable size after updating '${keyNames}' exceeds 5KB limit. Use result_path to store only the fields you need.`
            )
            // Clean up all variables we just set
            for (const key of allStoredKeys) {
                delete result.invocation.state.variables[key]
            }
            throw new Error(
                `Total variable size after updating '${keyNames}' exceeds 5KB limit. Use result_path to store only the fields you need.`
            )
        }

        // A protected run keeps its variables out of every stored copy, so the log names them without their values.
        const storedSummary = result.invocation.state.sensitive
            ? allStoredKeys.join(', ')
            : allStoredKeys
                  .map(
                      (key) =>
                          `${key} = ${this.describeStoredVariable(action, result.invocation.state.variables![key])}`
                  )
                  .join(', ')
        this.log(result, 'debug', `Stored action result in variable(s): ${storedSummary}`)
    }

    private describeStoredVariable(action: HogFlowAction, value: unknown): string {
        return aiTaskLogToken(action, value) ?? JSON.stringify(value)
    }

    private logExecutionTriggerInfo(invocation: CyclotronJobInvocationHogFlow): MinimalLogEntry {
        const hasCurrentAction = Boolean(invocation.state.currentAction)
        const currentAction = hasCurrentAction ? `[Action:${invocation.state.currentAction!.id}]` : 'trigger'

        const hasAssociatedPerson = Boolean(invocation.person)
        const hasAssociatedEvent = Boolean(invocation.state.event)
        const isWebhookTriggered = ['webhook', 'manual', 'schedule'].includes(invocation.hogFlow.trigger.type)
        const isBatchWorkflow = invocation.hogFlow.trigger.type === 'batch'

        let triggeredForActor = ''
        if (!hasCurrentAction) {
            triggeredForActor = isWebhookTriggered
                ? ` at request of [Actor:${invocation.state.event?.distinct_id ?? 'unknown'}]`
                : ''
            triggeredForActor += hasAssociatedPerson
                ? ` for [Person:${invocation.person?.id}|${invocation.person?.name ?? 'unknown'}]`
                : ''
        }

        let triggeredByEvent = hasAssociatedEvent
            ? ` on [Event:${invocation.state.event?.uuid}|${invocation.state.event?.event?.replaceAll('|', '')}|${invocation.state.event?.timestamp}]`
            : ''

        // Surface the event that woke the job (not the trigger). The logs view builds the link
        // from uuid + timestamp, so emit the linkable token only when both are present.
        const wakeEvent = invocation.state.currentAction?.eventMatchedEvent
        const wakeEventUuid = invocation.state.currentAction?.eventMatchedEventUuid
        const wakeEventTimestamp = invocation.state.currentAction?.eventMatchedEventTimestamp
        if (hasCurrentAction && invocation.state.currentAction?.eventMatched && wakeEvent) {
            triggeredByEvent +=
                wakeEventUuid && wakeEventTimestamp
                    ? ` (woken by [Event:${wakeEventUuid}|${wakeEvent.replaceAll('|', '')}|${wakeEventTimestamp}])`
                    : ` (woken by event: ${wakeEvent.replaceAll('|', '')})`
        }
        if (hasCurrentAction && invocation.state.currentAction?.resumeResult) {
            triggeredByEvent += ' (woken by the run finishing)'
        }

        return {
            level: 'info',
            message: `${hasCurrentAction ? 'Resuming' : 'Starting'} ${isBatchWorkflow ? 'batch ' : ''}workflow execution at ${currentAction}${triggeredForActor}${triggeredByEvent}`,
            timestamp: DateTime.now(),
        }
    }

    private logExecutionErrorInfo(
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        // executeCurrentAction stores the message string, not the Error, on result.error.
        error: unknown
    ): string {
        const invocation = result.invocation
        const currentActionId = invocation.state.currentAction?.id
        const currentAction = currentActionId ? invocation.hogFlow.actions.find((a) => a.id === currentActionId) : null

        const hasAssociatedEvent = Boolean(invocation.state.event)
        const triggeredByEvent = hasAssociatedEvent
            ? `. This workflow was triggered by [Event:${invocation.state.event?.uuid}|${invocation.state.event?.event?.replaceAll('|', '')}|${invocation.state.event?.timestamp}]`
            : ''

        const message = error instanceof Error ? error.message : String(error)
        return `Workflow encountered an error: ${message} at ${currentAction ? actionIdForLogging(currentAction) : 'unknown action'}${triggeredByEvent}`
    }
}