Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

31.0 KB
Raw Blame
import { DateTime, Duration } from 'luxon'
import { Counter, Histogram } from 'prom-client'

import { HogFlowAction } from '~/cdp/schema/hogflow'
import {
    buildWorkflowStepDispatchKey,
    workflowStepDispatchKeyFromInvocation,
} from '~/cdp/utils/workflow-step-dispatch-key'
import { capWorkflowStepResult } from '~/cdp/utils/workflow-step-result'
import { instrumentFn } from '~/common/tracing/tracing-utils'

import {
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationHogFunction,
    CyclotronJobInvocationResult,
    HogFlowInvocationContext,
    MessageOptOutRecord,
    MinimalLogEntry,
} from '../../../types'
import { HogExecutorExecuteAsyncOptions } from '../../hog-executor-async.service'
import { EmailSkipReason, EmailStatusService } from '../../messaging/email-status.service'
import { EmailValidationService } from '../../messaging/email-validation.service'
import { FrequencyCapCheck, RecipientPreferencesService } from '../../messaging/recipient-preferences.service'
import { CdpUsageReporterService } from '../../usage/cdp-usage-reporter.service'
import { trackHogFlowBillableInvocation } from '../billing-utils'
import { HogFlowFunctionsService } from '../hogflow-functions.service'
import { actionIdForLogging, aiTaskLogToken, findContinueAction } from '../hogflow-utils'
import { observeMissingVariableReferences } from '../hogflow-variable-usage'
import { ActionHandler, ActionHandlerOptions, ActionHandlerResult } from './action.interface'

const OPT_OUT_OF_MESSAGES_TEMPLATE_ID = 'template-posthog-opt-out-of-messages'

/** The run's recorded opt-outs after a step that succeeded, adding the one an "Opt out of messages" step made. */
export function messageOptOutsAfterStep(
    optOuts: MessageOptOutRecord[] | undefined,
    templateId: string,
    inputs: Record<string, unknown> | undefined
): MessageOptOutRecord[] | undefined {
    if (
        templateId !== OPT_OUT_OF_MESSAGES_TEMPLATE_ID ||
        typeof inputs?.identifier !== 'string' ||
        typeof inputs.category_id !== 'string'
    ) {
        return optOuts
    }
    return [...(optOuts ?? []), { identifier: inputs.identifier.trim(), categoryId: inputs.category_id }]
}

type FunctionActionType = 'function' | 'function_email' | 'function_sms'
type HogFlowActionBillingType = 'fetch' | 'email' | 'push' | 'sms'

const WORKFLOW_USAGE_KEYS = {
    fetch: ['workflow_billable_invocations'],
    email: ['workflow_emails_sent'],
    push: ['workflow_push_sent', 'workflow_billable_invocations'],
    sms: ['workflow_sms_sent'],
} as const

type Action = Extract<HogFlowAction, { type: FunctionActionType }>

type AwaitingResume = NonNullable<NonNullable<HogFlowInvocationContext['currentAction']>['awaitingResume']>

// A template parks the step by returning `{ ..., 'await': { 'max_wait': '190m', 'label': 'task' } }`. When the
// job already finished, `'result': {...}` in the request hands over its answer as a wake would, without parking.
type AwaitRequest = { maxWait: Duration; label: string; settled: Record<string, unknown> | null }

const AWAIT_DURATION_REGEX = /^(\d*\.?\d+)([dhms])$/
const SECONDS_PER_UNIT: Record<string, number> = { d: 86400, h: 3600, m: 60, s: 1 }
const AWAIT_MAX_WAIT_CEILING = Duration.fromObject({ hours: 24 })

// A malformed request throws rather than returning null, so the step logs why it did not wait.
const parseAwaitRequest = (execResult: unknown): AwaitRequest | null => {
    const request = (execResult as { await?: unknown } | undefined)?.await
    if (request === undefined || request === null) {
        return null
    }
    if (typeof request !== 'object') {
        throw new Error(`await must be an object, got ${typeof request}`)
    }
    const {
        max_wait: maxWait,
        label,
        result: settled,
    } = request as { max_wait?: unknown; label?: unknown; result?: unknown }
    const match = typeof maxWait === 'string' ? AWAIT_DURATION_REGEX.exec(maxWait) : null
    if (!match) {
        throw new Error(`await.max_wait must be a duration like '190m' or '2h', got ${JSON.stringify(maxWait)}`)
    }
    const requested = Duration.fromObject({ seconds: parseFloat(match[1]) * SECONDS_PER_UNIT[match[2]] })
    return {
        maxWait: requested > AWAIT_MAX_WAIT_CEILING ? AWAIT_MAX_WAIT_CEILING : requested,
        label: typeof label === 'string' && label ? label : 'run',
        settled:
            settled && typeof settled === 'object' && !Array.isArray(settled)
                ? (settled as Record<string, unknown>)
                : null,
    }
}

const humanDuration = (duration: Duration): string => duration.rescale().toHuman()

// Read off the raw resume result, before the variable cap, so a long warning is not cut.
const readWarnings = (resumeResult: unknown): string[] => {
    const warnings = (resumeResult as { warnings?: unknown } | undefined)?.warnings
    return Array.isArray(warnings) ? warnings.filter((warning): warning is string => typeof warning === 'string') : []
}

// Unlabelled: how often a step degrades fleet-wide. Which flow and field goes to the warn log.
const counterAwaitedStepResumedWithWarnings = new Counter({
    name: 'cdp_hogflow_awaited_step_resumed_with_warnings',
    help: 'A parked step resumed and continued, but the product that ran the job reported a warning.',
})

// The product that ran the job reports what went wrong short of failing it, such as
// an agent whose output misses a field. The step continues; the author reads the log.
const logStepWarnings = (
    result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
    action: Action,
    stepResult: unknown
): void => {
    const warnings = readWarnings(stepResult)
    if (warnings.length > 0) {
        counterAwaitedStepResumedWithWarnings.inc()
    }
    for (const warning of warnings) {
        result.logs.push({
            level: 'warn',
            timestamp: DateTime.now(),
            message: `${actionIdForLogging(action)} ${warning}`,
        })
    }
}

const counterAwaitedStepStaleResume = new Counter({
    name: 'cdp_hogflow_awaited_step_stale_resume',
    help: 'A parked step received a wake keyed to an earlier visit of the same step and kept waiting.',
})

const counterAwaitedStepFinished = new Counter({
    name: 'cdp_hogflow_awaited_step_finished',
    help: 'A parked step stopped waiting, by how: the job completed, failed or was cancelled, or the wait timed out.',
    labelNames: ['outcome'],
})

const histogramAwaitedStepWaitSeconds = new Histogram({
    name: 'cdp_hogflow_awaited_step_wait_seconds',
    help: 'How long a parked step waited before it stopped, by outcome.',
    labelNames: ['outcome'],
    buckets: [30, 60, 120, 300, 600, 1200, 1800, 3600, 7200, 10800],
})

const observeAwaitedStepFinished = (outcome: string, awaiting: AwaitingResume): void => {
    counterAwaitedStepFinished.labels({ outcome }).inc()
    if (awaiting.parkedAt) {
        const waited = DateTime.now().diff(DateTime.fromISO(awaiting.parkedAt), 'seconds').seconds
        histogramAwaitedStepWaitSeconds.labels({ outcome }).observe(Math.max(0, waited))
    }
}

export class HogFunctionHandler implements ActionHandler {
    constructor(
        private hogFlowFunctionsService: HogFlowFunctionsService,
        private recipientPreferencesService: RecipientPreferencesService,
        private emailValidationService: EmailValidationService,
        private hogFlowActionBillingType: HogFlowActionBillingType,
        // Required, so that a caller which builds an email handler without it fails to compile. The
        // handlers for other channels pass null.
        private emailStatusService: EmailStatusService | null,
        private usageReporter?: Pick<CdpUsageReporterService, 'reportBillableInvocation'>,
        private options: { awaitedStepsEnabled?: boolean } = {}
    ) {}

    async execute({
        invocation,
        action,
        result,
        hogExecutorOptions,
    }: ActionHandlerOptions<Action>): Promise<ActionHandlerResult> {
        const awaitedStepsEnabled = this.options.awaitedStepsEnabled ?? false
        const awaiting = invocation.state.currentAction?.awaitingResume
        // Resume before anything else: the dispatch already ran and was billed.
        if (awaiting) {
            return this.resumeAwaitedStep(invocation, action, result, awaiting)
        }

        // Inputs are rendered once, on fresh entry into the action (continuations reuse the
        // rendered state in hogFunctionState) - so this also fires at most once per step per run
        if (
            !invocation.state.currentAction?.hogFunctionState &&
            !invocation.state.currentAction?.frequencyCapHoldDeadlineAt
        ) {
            observeMissingVariableReferences(invocation, action, result)
        }

        const functionResult = await this.executeHogFunction(invocation, action, hogExecutorOptions)

        if (functionResult.emailSendResult) {
            result.emailSendResult = functionResult.emailSendResult
        }

        // Add all logs
        functionResult.logs.forEach((log: MinimalLogEntry) => {
            result.logs.push({
                level: log.level,
                timestamp: log.timestamp,
                message: `${actionIdForLogging(action)} ${log.message}`,
            })
        })

        // Collect captured PostHog events and metrics from the function execution
        result.capturedPostHogEvents = [...result.capturedPostHogEvents, ...functionResult.capturedPostHogEvents]
        // Collect warehouse webhook payloads
        result.warehouseWebhookPayloads = [
            ...result.warehouseWebhookPayloads,
            ...functionResult.warehouseWebhookPayloads,
        ]
        result.metrics = [...result.metrics, ...functionResult.metrics]
        result.messageAssets = [...result.messageAssets, ...functionResult.messageAssets]

        if (functionResult.frequencyCapHold) {
            result.invocation.state.currentAction!.frequencyCapHoldDeadlineAt =
                functionResult.frequencyCapHold.deadlineAt
            return { scheduledAt: functionResult.frequencyCapHold.until }
        }

        if (!functionResult.finished) {
            // Set the state of the function result on the substate of the flow for the next execution
            result.invocation.state.currentAction!.hogFunctionState = {
                ...functionResult.invocation.state,
                ...(functionResult.invocation.hogFunction?.metadata?.sensitive === true ? { sensitive: true } : {}),
            }
            // Preserve queue routing and parameters from the function result
            result.invocation.queue = functionResult.invocation.queue
            result.invocation.queuePriority = functionResult.invocation.queuePriority
            result.invocation.queueParameters = functionResult.invocation.queueParameters
            result.invocation.queueMetadata = functionResult.invocation.queueMetadata
            // Routing-only reschedule signature: the queue changed AND no explicit
            // `queueScheduledAt` was set. That's the shape produced by `routeEmailToQueue`
            // and `routeToQueue` in hog-executor.service.ts when moving a job between the
            // hogflow and email queues — the next dequeue continues the same action on the
            // new queue. Tag the action state so the executor can suppress the redundant
            // "Resuming..." / "Workflow will pause until..." pair on the next dequeue.
            //
            // The queue-changed check is what keeps async pauses (fetches, SES throttle
            // retries) out of this branch: both keep `queueScheduledAt` set OR leave the
            // queue unchanged, so they don't satisfy both halves of the condition.
            const queueChanged = functionResult.invocation.queue !== invocation.queue
            if (queueChanged && !functionResult.invocation.queueScheduledAt) {
                result.invocation.state.currentAction!.routingOnlyReschedule = true
            }
            return {
                scheduledAt: functionResult.invocation.queueScheduledAt ?? DateTime.now(),
            }
        }

        if (!functionResult.error) {
            result.invocation.state.messageOptOuts = messageOptOutsAfterStep(
                result.invocation.state.messageOptOuts,
                action.config.template_id,
                functionResult.invocation.state.globals?.inputs
            )
        }

        // Add billable_invocation metric only if the function actually executed (not skipped)
        if (!functionResult.skipped) {
            // A step that does not send leaves this undefined and bills as before.
            if (functionResult.deliveredToRecipient !== false) {
                trackHogFlowBillableInvocation(result, {
                    invocation: functionResult.invocation,
                    billingMetricType: this.hogFlowActionBillingType,
                })

                if (!hogExecutorOptions?.isTest || !hogExecutorOptions.functions) {
                    // actionStepCount holds across a retry of this step but changes on a loop revisit.
                    for (const usageKey of WORKFLOW_USAGE_KEYS[this.hogFlowActionBillingType]) {
                        this.usageReporter?.reportBillableInvocation({
                            teamId: invocation.teamId,
                            usageKey,
                            recordId: `flow:${invocation.id}:${invocation.state.actionStepCount}:${this.hogFlowActionBillingType}`,
                        })
                    }
                }
            }

            // Re-pin the attribution version to the one that actually sent. Live edits reach runs
            // already in flight, so a run that entered on v2 can send its email after v3 is
            // published — and the conversion belongs to the version whose message the person
            // received, which is also the version `email_sent` was counted under. Leaving the
            // run-start stamp here would split a rate across two versions.
            if (this.hogFlowActionBillingType === 'email' || this.hogFlowActionBillingType === 'push') {
                result.invocation.state.flowVersion = invocation.hogFlow.version
            }
        }

        let awaitRequest: AwaitRequest | null = null
        if (awaitedStepsEnabled && !functionResult.error) {
            try {
                awaitRequest = parseAwaitRequest(functionResult.execResult)
            } catch (error) {
                // The template asked to wait but the request is unusable. Continue instead of parking
                // on a guess, and say so in the run log.
                result.logs.push({
                    level: 'warn',
                    timestamp: DateTime.now(),
                    message: `${actionIdForLogging(action)} Ignored the template's wait request: ${(error as Error).message}`,
                })
            }
        }
        if (awaitRequest?.settled) {
            const { await: _await, ...dispatch } = functionResult.execResult as Record<string, unknown>
            // Read before sizing, which can drop the warnings themselves.
            logStepWarnings(result, action, awaitRequest.settled)
            // Sized like a woken step's answer, so it fits the variables the run already holds.
            return {
                nextAction: findContinueAction(invocation),
                result: capWorkflowStepResult(
                    { ...dispatch, status: 'completed' },
                    awaitRequest.settled,
                    result.invocation.state.variables ?? {},
                    action.output_variable
                ),
            }
        }
        if (awaitRequest) {
            return this.parkForAwaitedRun(
                invocation,
                action,
                result,
                awaitRequest,
                functionResult.execResult,
                workflowStepDispatchKeyFromInvocation(functionResult.invocation)
            )
        }

        // A failed step keeps its variable untouched: execResult may still hold an earlier fetch response.
        return {
            nextAction: findContinueAction(invocation),
            result: functionResult.error ? undefined : functionResult.execResult,
            error: functionResult.error,
        }
    }

    private parkForAwaitedRun(
        invocation: CyclotronJobInvocationHogFlow,
        action: Action,
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        awaitRequest: AwaitRequest,
        execResult: unknown,
        dispatchKey: string | null
    ): ActionHandlerResult {
        const { await: _await, ...dispatch } = execResult as Record<string, unknown>
        const key =
            dispatchKey ??
            buildWorkflowStepDispatchKey(
                invocation.id,
                action.id,
                invocation.state.actionStepCount,
                invocation.state.rerunAttempts
            )
        const deadline = DateTime.now().plus(awaitRequest.maxWait)
        result.invocation.state.currentAction!.awaitingResume = {
            key,
            deadlineAt: deadline.toISO()!,
            dispatch,
            label: awaitRequest.label,
            parkedAt: DateTime.now().toISO()!,
        }
        // The stored variable only links the task when it holds the whole result, so link it here too.
        const taskToken = aiTaskLogToken(action, dispatch)
        result.logs.push({
            level: 'info',
            timestamp: DateTime.now(),
            message: `${actionIdForLogging(action)} Waiting for the ${awaitRequest.label} to finish (up to ${humanDuration(awaitRequest.maxWait)})${taskToken ? ` ${taskToken}` : ''}`,
        })
        // Stored now so a step that later fails still leaves the ids for `on_error: continue`.
        return { scheduledAt: deadline, result: dispatch }
    }

    private resumeAwaitedStep(
        invocation: CyclotronJobInvocationHogFlow,
        action: Action,
        result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFlow>,
        awaiting: AwaitingResume
    ): ActionHandlerResult {
        const currentAction = result.invocation.state.currentAction!
        const label = awaiting.label ?? 'run'
        const resume = currentAction.resumeResult
        if (resume?.key === awaiting.key) {
            delete currentAction.awaitingResume
            delete currentAction.resumeResult
            observeAwaitedStepFinished(resume.status, awaiting)
            const payload = capWorkflowStepResult(
                { ...awaiting.dispatch, status: resume.status },
                resume.result ?? {},
                result.invocation.state.variables ?? {},
                action.output_variable
            )
            if (resume.status !== 'completed') {
                const detail = typeof payload.error_message === 'string' ? `: ${payload.error_message}` : ''
                const outcome = resume.status === 'cancelled' ? 'was cancelled' : 'failed'
                return { error: new Error(`The ${label} ${outcome}${detail}`), result: payload }
            }
            result.logs.push({
                level: 'info',
                timestamp: DateTime.now(),
                message: `${actionIdForLogging(action)} The ${label} finished`,
            })
            logStepWarnings(result, action, resume.result)
            return {
                nextAction: findContinueAction(invocation),
                result: payload,
            }
        }
        if (resume) {
            delete currentAction.resumeResult
            counterAwaitedStepStaleResume.inc()
        }
        const deadline = DateTime.fromISO(awaiting.deadlineAt)
        if (DateTime.now() >= deadline) {
            observeAwaitedStepFinished('timed_out', awaiting)
            throw new Error(`Timed out waiting for the ${label} to finish`)
        }
        // Woken early with nothing (clock skew): park again.
        return { scheduledAt: deadline }
    }

    // These checks skip the send before the email service runs, so the email's status is recorded here.
    private recordSkippedEmail(
        invocation: CyclotronJobInvocationHogFunction,
        action: Action,
        reason: EmailSkipReason,
        hogExecutorOptions?: HogExecutorExecuteAsyncOptions
    ): void {
        if (action.type === 'function_email') {
            this.emailStatusService?.queueSkippedSend(invocation, reason, hogExecutorOptions?.isTest ?? false)
        }
    }

    private async executeHogFunction(
        invocation: CyclotronJobInvocationHogFlow,
        action: Action,
        hogExecutorOptions?: HogExecutorExecuteAsyncOptions
    ): Promise<
        CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> & {
            skipped?: boolean
            frequencyCapHold?: FrequencyCapHold
        }
    > {
        const hogFunction = await instrumentFn(
            { key: 'hogFlow.action.hogFunction.buildHogFunction', sendException: false },
            () => this.hogFlowFunctionsService.buildHogFunction(invocation.hogFlow, action.config, action.name)
        )
        // A send that started protected stays protected through its retries, even if protection is turned off
        // before the retry runs: the queued message was rendered under protection.
        if (invocation.state.currentAction?.hogFunctionState?.sensitive) {
            hogFunction.metadata = { ...hogFunction.metadata, sensitive: true }
        }
        const hogFunctionInvocation = await instrumentFn(
            { key: 'hogFlow.action.hogFunction.buildInvocation', sendException: false },
            () =>
                this.hogFlowFunctionsService.buildHogFunctionInvocation(invocation, hogFunction, {
                    event: invocation.state.event,
                    person: invocation.person,
                    groups: invocation.groups,
                    variables: invocation.state.variables,
                })
        )

        const skipReason = await instrumentFn(
            { key: 'hogFlow.action.hogFunction.recipientPreferences', sendException: false },
            () =>
                this.recipientPreferencesService.shouldSkipAction(
                    hogFunctionInvocation,
                    action,
                    invocation.state.messageOptOuts
                )
        )
        if (skipReason) {
            // Suppression and opt-out both short-circuit the send, but a customer reading the run
            // log needs to know which one — the operator response is different (fix the recipient
            // list vs. respect the unsubscribe). `email_suppressed` mirrors the metric name the
            // send-time choke point in email.service.ts emits, so both entry points aggregate.
            const message =
                skipReason === 'suppressed'
                    ? `Skipping send: recipient is on the suppression list.`
                    : `Recipient has opted out, skipping message delivery.`
            const metrics =
                skipReason === 'suppressed'
                    ? [
                          {
                              team_id: hogFunctionInvocation.teamId,
                              app_source_id: hogFunctionInvocation.parentRunId ?? hogFunctionInvocation.functionId,
                              instance_id: action.id,
                              metric_kind: 'email' as const,
                              metric_name: 'email_suppressed' as const,
                              count: 1,
                          },
                      ]
                    : []
            this.recordSkippedEmail(
                hogFunctionInvocation,
                action,
                skipReason === 'suppressed' ? 'recipient_suppressed' : 'recipient_opted_out',
                hogExecutorOptions
            )
            return {
                finished: true,
                skipped: true,
                emailSendResult: action.type === 'function_email' ? { status: 'skipped', reason: message } : undefined,
                invocation: hogFunctionInvocation,
                logs: [{ level: 'info', timestamp: DateTime.now(), message }],
                metrics,
                capturedPostHogEvents: [],
                warehouseWebhookPayloads: [],
                messageAssets: [],
                conversionWatchers: [],
            }
        }

        // Predicted hard bounce (bad syntax / dead domain): skip before the send reaches
        // SES so it never counts against our bounce rate. Runs after the opt-out check so
        // an opted-out recipient never triggers a DNS lookup.
        const emailSkipReason = await instrumentFn(
            { key: 'hogFlow.action.hogFunction.emailValidation', sendException: false },
            () => this.emailValidationService.getSkipReason(hogFunctionInvocation, action)
        )
        if (emailSkipReason) {
            this.recordSkippedEmail(hogFunctionInvocation, action, 'recipient_unreachable', hogExecutorOptions)
            return {
                finished: true,
                skipped: true,
                emailSendResult: { status: 'skipped', reason: emailSkipReason },
                invocation: hogFunctionInvocation,
                logs: [{ level: 'info', timestamp: DateTime.now(), message: emailSkipReason }],
                metrics: [
                    {
                        team_id: hogFunctionInvocation.teamId,
                        app_source_id: hogFunctionInvocation.parentRunId ?? hogFunctionInvocation.functionId,
                        instance_id: action.id,
                        metric_kind: 'email',
                        metric_name: 'email_bounce_prevented',
                        count: 1,
                    },
                ],
                capturedPostHogEvents: [],
                warehouseWebhookPayloads: [],
                messageAssets: [],
                conversionWatchers: [],
            }
        }

        // This check runs last because it also records the send against the cap. If it ran before the
        // opt-out, suppression or bounce checks, a send that one of them skips would still use a slot.
        // A held send that wakes after its hold ended is skipped before the check, so it never claims a slot late.
        const holdExpired = frequencyCapHoldExpired(invocation)
        const frequencyCap =
            hogExecutorOptions?.isTest || holdExpired
                ? ({ capped: false } as const)
                : await this.recipientPreferencesService.checkFrequencyCap(hogFunctionInvocation, action)
        if (holdExpired || frequencyCap.capped) {
            const hold = frequencyCap.capped ? frequencyCapHold(invocation, frequencyCap) : null
            if (hold) {
                return {
                    finished: false,
                    invocation: hogFunctionInvocation,
                    frequencyCapHold: hold,
                    logs: [
                        {
                            level: 'info',
                            timestamp: DateTime.now(),
                            // Worded apart from the skip line, which the metrics drill-down finds by 'reached the frequency cap'.
                            message: `Holding send until ${hold.until.toUTC().toISO()}: the recipient is at the frequency cap.`,
                        },
                    ],
                    // Counted once per held send, not on every wake that finds the cap still full.
                    metrics: hold.firstHold
                        ? [
                              {
                                  team_id: hogFunctionInvocation.teamId,
                                  app_source_id: hogFunctionInvocation.parentRunId ?? hogFunctionInvocation.functionId,
                                  instance_id: action.id,
                                  metric_kind: 'other',
                                  metric_name: 'message_frequency_cap_held',
                                  count: 1,
                              },
                          ]
                        : [],
                    capturedPostHogEvents: [],
                    warehouseWebhookPayloads: [],
                    messageAssets: [],
                    conversionWatchers: [],
                }
            }
            this.recordSkippedEmail(hogFunctionInvocation, action, 'frequency_capped', hogExecutorOptions)
            return {
                finished: true,
                skipped: true,
                invocation: hogFunctionInvocation,
                logs: [
                    {
                        level: 'info',
                        timestamp: DateTime.now(),
                        message: invocation.state.currentAction?.frequencyCapHoldDeadlineAt
                            ? 'Skipping send: recipient reached the frequency cap, and no slot opened before the hold ended.'
                            : 'Skipping send: recipient reached the frequency cap.',
                    },
                ],
                metrics: [
                    {
                        team_id: hogFunctionInvocation.teamId,
                        app_source_id: hogFunctionInvocation.parentRunId ?? hogFunctionInvocation.functionId,
                        instance_id: action.id,
                        metric_kind: 'other',
                        metric_name: 'message_frequency_capped',
                        count: 1,
                    },
                ],
                capturedPostHogEvents: [],
                warehouseWebhookPayloads: [],
                messageAssets: [],
                conversionWatchers: [],
            }
        }

        return instrumentFn({ key: 'hogFlow.action.hogFunction.executeWithAsyncFunctions', sendException: false }, () =>
            this.hogFlowFunctionsService.executeWithAsyncFunctions(hogFunctionInvocation, hogExecutorOptions)
        )
    }
}

type FrequencyCapHold = { until: DateTime; deadlineAt: string; firstHold: boolean }

// Up to a minute of spread, so held sends for one person that wait on the same slot don't all wake at once.
const FREQUENCY_CAP_HOLD_JITTER_MS = 60_000
function frequencyCapHoldExpired(invocation: CyclotronJobInvocationHogFlow): boolean {
    const deadlineAt = invocation.state.currentAction?.frequencyCapHoldDeadlineAt
    return !!deadlineAt && DateTime.now() >= DateTime.fromISO(deadlineAt)
}

/** When the team holds capped messages and the hold has not ended, returns when to check the cap again. */
function frequencyCapHold(
    invocation: CyclotronJobInvocationHogFlow,
    cap: Extract<FrequencyCapCheck, { capped: true }>
): FrequencyCapHold | null {
    // A continuation on the email queue carries a pending send that a hold would drop, so it skips as before.
    if (!cap.holdHours || invocation.state.currentAction?.hogFunctionState) {
        return null
    }
    const now = DateTime.now()
    const startedDeadline = invocation.state.currentAction?.frequencyCapHoldDeadlineAt
    const deadline = startedDeadline ? DateTime.fromISO(startedDeadline) : now.plus({ hours: cap.holdHours })
    // No slot frees before the hold ends, so waiting cannot help.
    if (cap.nextSlotAt >= deadline.toMillis()) {
        return null
    }
    // Wake after the slot frees but before the deadline, which the next wake enforces strictly.
    const jitterMs = Math.min(
        Math.floor(Math.random() * FREQUENCY_CAP_HOLD_JITTER_MS),
        Math.floor((deadline.toMillis() - cap.nextSlotAt) / 2)
    )
    const until = DateTime.max(now, DateTime.fromMillis(cap.nextSlotAt + jitterMs))
    return { until, deadlineAt: deadline.toISO()!, firstHold: !startedDeadline }
}