Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

29.1 KB
Raw Blame
import { promisify } from 'node:util'
import { gunzip, gzip } from 'node:zlib'
import { Counter, Gauge } from 'prom-client'

import { HOG_INVOCATION_RESULTS_OUTPUT, HogInvocationResultsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { safeClickhouseString } from '~/common/utils/db/utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

import type { CdpOutput } from '../../cdp-services'
import { RerunFilter, RerunFunctionKind, rerunWrapperKindFor } from '../../rerun/rerun-job.types'
import {
    CyclotronJobInvocation,
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationHogFunction,
    CyclotronJobInvocationResult,
} from '../../types'
import { TRANSACTIONAL_EMAIL_EVENT, hasSensitiveMessageStep } from '../hogflows/hogflow-sensitivity'

const counterHogInvocationResultRowsProduced = new Counter({
    name: 'cdp_hog_invocation_result_rows_produced',
    help: 'Lifecycle rows queued for the hog_invocation_results ClickHouse table.',
    labelNames: ['function_kind', 'status'],
})

const counterHogInvocationResultProduceFailed = new Counter({
    name: 'cdp_hog_invocation_result_produce_failed',
    help: 'Rows that failed to produce to Kafka.',
})

const hogInvocationResultsPendingMessages = new Gauge({
    name: 'cdp_hog_invocation_results_pending_messages',
    help: 'Rows queued waiting to be flushed to Kafka.',
})

export type HogInvocationResultsServiceOutput = HogInvocationResultsOutput | CdpOutput

/**
 * Lifecycle row produced to ClickHouse via Kafka. Mirrors the columns on the
 * hog_invocation_results_data table. Two such rows are produced per
 * invocation: one when execution starts (`status='running'`) and one when it
 * finishes (`status='succeeded' | 'failed' | 'canceled'`). On a rerun, the
 * cycle repeats with the same `invocation_id`, `is_retry=1`, and `attempts`
 * bumped — the ReplacingMergeTree on
 * `(team_id, function_kind, function_id, invocation_id)` keyed by `version`
 * collapses prior versions at merge time.
 */
export type HogInvocationResultStatus = 'running' | 'succeeded' | 'failed' | 'canceled'

export interface HogInvocationResultRow {
    team_id: number
    // `*_rerun` kinds tag the wrapper row that drives a re-run, so the
    // Invocations list can surface in-flight re-runs alongside the function's
    // normal invocations. See `rerun-job.types.ts` for the helper.
    function_kind: 'hog_function' | 'hog_flow' | 'hog_function_rerun' | 'hog_flow_rerun'
    function_id: string
    invocation_id: string
    parent_run_id: string
    status: HogInvocationResultStatus
    attempts: number
    is_retry: 0 | 1
    scheduled_at: string // ISO microsecond DateTime64
    // Original cyclotron-scheduled time, carried unchanged through retries.
    // ReplacingMergeTree collapses lifecycle rows per invocation_id, so
    // `min(scheduled_at)` post-merge is unreliable — every row stamps this
    // verbatim so `argMax(first_scheduled_at, version)` returns it correctly.
    first_scheduled_at: string
    started_at: string | null
    finished_at: string | null
    duration_ms: number | null
    error_kind: string
    error_message: string
    event_uuid: string
    distinct_id: string
    person_id: string
    invocation_globals: string // globals JSON (inputs/groups/person stripped) — gzip+base64'd on produce
    version: string // microsecond-precision UInt64; serialized as string to dodge JS's 53-bit precision
    is_deleted: 0 | 1
}

const isHogFunctionInvocation = (invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFunction =>
    'hogFunction' in invocation

const isHogFlowInvocation = (invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFlow =>
    'hogFlow' in invocation

// Monotonic microsecond timestamp used as the row `version`. ReplacingMergeTree keeps the
// row with the max `version` per key; without a monotonic version the 'running' + terminal
// rows of a fast invocation can share or invert a version, and CH keeps one arbitrarily —
// leaving the runs UI stuck on 'running' (and, via the rerun paginator's
// `argMax(status, version)` in-flight check, that invocation un-rerunnable).
//
// The rows being compared are produced across DIFFERENT processes — the 'running' row by
// the events consumer, the terminal row by a cyclotron worker — so the base must be a
// clock that stays synchronised across processes. `Date.now()` is continuously
// NTP-disciplined, so cross-process skew stays well under the queue latency between the
// two rows. (`performance.timeOrigin + performance.now()` is monotonic within a process
// but freezes each process's wall-clock offset at start and never re-syncs, so an
// NTP step or VM pause would invert versions across pods — worse for the comparison that
// matters.)
//
// A wall clock alone can still tie, or briefly step backward under NTP, for two rows
// produced close together in the SAME process (e.g. the flaky monotonicity test). Clamp to
// strictly exceed the last value this process issued: that adds strict in-process
// monotonicity on top of the NTP-anchored cross-process ordering.
//
// This module-level counter is process-global and never resets, so a test using
// `jest.setSystemTime()` to a time earlier than an already-issued stamp will see clamped
// `last + 1` values, not the fake time. Don't rewind the clock below a prior stamp in tests.
let lastVersionMicros = 0n
const microsecondsSinceEpoch = (): string => {
    // BigInt avoids the 53-bit cap so the number lines up with ClickHouse UInt64.
    const wallMicros = BigInt(Date.now()) * 1000n
    lastVersionMicros = wallMicros > lastVersionMicros ? wallMicros : lastVersionMicros + 1n
    return lastVersionMicros.toString()
}

const isoMicroseconds = (date: Date): string => {
    // ClickHouse DateTime64(6) accepts 'YYYY-MM-DD HH:MM:SS.ffffff'.
    return date.toISOString().replace('T', ' ').replace('Z', '000')
}

const truncate = (value: string, max: number): string => {
    if (value.length <= max) {
        return value
    }
    return value.slice(0, max)
}

// Best-effort error classification — keeps `error_kind` low-cardinality so the
// status_idx skipping index stays small. Only the message lands in
// `error_message`, never the stack trace — app-level executors pass the whole
// `Error` here, and its stack would otherwise leak into the Invocations tab.
export const classifyError = (error: unknown): { kind: string; message: string } => {
    if (!error) {
        return { kind: '', message: '' }
    }
    const message =
        typeof error === 'string'
            ? error
            : error instanceof Error
              ? // fall back to the error name so an empty message doesn't blank the row (without the stack)
                error.message || error.name
              : (() => {
                    try {
                        return JSON.stringify(error)
                    } catch {
                        return String(error)
                    }
                })()

    const lower = message.toLowerCase()
    let kind = 'hog_error'
    // Anchor the status-code patterns to HTTP-context tokens so messages like
    // "processed 500 items" or "400 rows returned" don't get misclassified as
    // http_5xx / http_4xx. Matches: 'status 503', 'http 502', 'http/2 504',
    // 'status code 500', etc.
    const http5xxRegex = /\b(?:status(?:\s*code)?|http(?:[/ ]\d)?)\s*[:= ]?\s*5\d{2}\b/
    const http4xxRegex = /\b(?:status(?:\s*code)?|http(?:[/ ]\d)?)\s*[:= ]?\s*4\d{2}\b/
    if (lower.includes('timeout') || lower.includes('timed out')) {
        kind = 'timeout'
    } else if (http5xxRegex.test(lower) || lower.includes('server error')) {
        kind = 'http_5xx'
    } else if (http4xxRegex.test(lower)) {
        kind = 'http_4xx'
    } else if (lower.includes('out of memory') || lower.includes('oom')) {
        kind = 'oom'
    }

    return { kind, message: truncate(message, 4096) }
}

const extractTriggerFields = (
    invocation: CyclotronJobInvocation
): { event_uuid: string; distinct_id: string; person_id: string } => {
    if (isHogFunctionInvocation(invocation)) {
        const globals = invocation.state.globals
        return {
            event_uuid: globals.event?.uuid ?? '',
            distinct_id: globals.event?.distinct_id ?? '',
            person_id: globals.person?.id ?? '',
        }
    }

    if (isHogFlowInvocation(invocation)) {
        const event = invocation.state?.event
        return {
            event_uuid: event?.uuid ?? '',
            distinct_id: event?.distinct_id ?? '',
            person_id: invocation.state?.personId ?? invocation.person?.id ?? '',
        }
    }

    return { event_uuid: '', distinct_id: '', person_id: '' }
}

// Strip every `inputs` blob we can find on a globals/state tree. `inputs`
// holds resolved input values for the function — these can include user-
// supplied secrets (API keys, OAuth tokens, etc.) that the function templated
// in at execute time. Persisting them to ClickHouse for 30 days would expand
// the blast radius of any leak. On rerun we re-resolve inputs from the
// current hog function config + integration store, so this is also a no-op
// for rerun correctness — we'll always pick up the latest secrets, not a
// snapshot from when the original invocation ran.
const INPUT_KEYS: ReadonlySet<string> = new Set(['inputs'])
// A sensitive message carries reset links, login codes or private text, so its snapshot keeps no
// variables either, and a rerun can't rebuild the send. A step paused mid-send (e.g. for a provider
// retry) holds the rendered message on its VM stack, and the person and group properties it read, so those go too.
const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups', 'send'])
export const SENSITIVE_VARIABLES_OMITTED = 'sensitive_variables_omitted'

const stripKeys = <T>(value: T, keys: ReadonlySet<string>): T => {
    if (value === null || typeof value !== 'object') {
        return value
    }
    if (Array.isArray(value)) {
        // We don't expect inputs to live inside arrays today; this branch is a
        // defensive cheap pass-through so a future schema change doesn't have
        // to come back here.
        return value.map((item) => stripKeys(item, keys)) as unknown as T
    }
    const obj = value as Record<string, unknown>
    const out: Record<string, unknown> = {}
    for (const [k, v] of Object.entries(obj)) {
        if (keys.has(k)) {
            continue
        }
        out[k] = stripKeys(v, keys)
    }
    return out as T
}

// The payload the rerun path needs to rehydrate the invocation, kept minimal.
// `inputs`, `groups` and `person` are all dropped: the cyclotron worker
// rehydrates `groups`/`person` (loadHogFunctions) and the executor rebuilds
// `inputs` when an invocation arrives without them — so the rerun reconstructs
// everything from the event against the latest config. `inputs` must be
// dropped anyway for security (resolved secrets — see `stripKeys`).
// Recovery paths pass a workflow stub with only an id. A run that started before its template became
// sensitive carries no stamp, so a transactional email run without the actions is treated as sensitive.
const sendsSensitiveMessage = (invocation: CyclotronJobInvocationHogFlow): boolean => {
    if (invocation.state?.sensitive === true) {
        return true
    }
    if (!Array.isArray(invocation.hogFlow?.actions)) {
        return invocation.state?.event?.event === TRANSACTIONAL_EMAIL_EVENT
    }
    return hasSensitiveMessageStep(invocation.hogFlow)
}

const serializeInvocationGlobals = (invocation: CyclotronJobInvocation): string => {
    if (isHogFunctionInvocation(invocation)) {
        const { groups: _groups, person: _person, ...globals } = invocation.state.globals
        return JSON.stringify(stripKeys(globals, INPUT_KEYS))
    }
    if (isHogFlowInvocation(invocation)) {
        // Hog flow state can carry a per-action `currentAction.hogFunctionState.globals.inputs`
        // — `stripKeys` walks the tree and removes those too.
        if (sendsSensitiveMessage(invocation)) {
            return JSON.stringify({
                ...stripKeys(invocation.state ?? {}, SENSITIVE_KEYS),
                [SENSITIVE_VARIABLES_OMITTED]: true,
            })
        }
        return JSON.stringify(stripKeys(invocation.state ?? {}, INPUT_KEYS))
    }
    return '{}'
}

const gzipAsync = promisify(gzip)
const gunzipAsync = promisify(gunzip)

// `invocation_globals` is the bulk of every lifecycle row. Warpstream meters
// the uncompressed message bytes, so gzipping this one field before produce
// directly cuts cyclotron throughput. The row envelope stays plain JSON — only
// this field is opaque — so the ClickHouse Kafka engine still parses the
// message as JSONEachRow. `decodeInvocationGlobals` is the inverse.
const compressInvocationGlobals = async (globalsJson: string): Promise<string> => {
    return (await gzipAsync(globalsJson)).toString('base64')
}

/**
 * Inverse of `compressInvocationGlobals`. Used by the rerun paginator to read
 * `invocation_globals` back off ClickHouse. Rows written before field
 * compression landed are still raw JSON — base64 can never start with `{`, so
 * the prefix is an unambiguous discriminator for the legacy fallback.
 */
export const decodeInvocationGlobals = async (stored: string): Promise<unknown> => {
    if (stored.startsWith('{')) {
        return parseJSON(stored)
    }
    return parseJSON((await gunzipAsync(Buffer.from(stored, 'base64'))).toString('utf8'))
}

const sumDurationMs = (invocation: CyclotronJobInvocation): number | null => {
    if (!isHogFunctionInvocation(invocation)) {
        return null
    }
    const timings = invocation.state?.timings
    if (!timings || timings.length === 0) {
        return null
    }
    // Per-step timings are fractional ms (perf.now() deltas). ClickHouse stores
    // duration_ms as UInt32 — Kafka's JSONEachRow parser rejects the row if the
    // field arrives as a float, so round before serialization.
    return Math.round(timings.reduce((sum, t) => sum + t.duration_ms, 0))
}

/**
 * Per-invocation lifecycle row producer. Lives next to
 * `HogFunctionMonitoringService` (which handles aggregated metrics + log
 * lines) — this one writes a single row per lifecycle event so the new
 * runs/invocations UI and the rerun path can read it back via HogQL.
 *
 * Off by default behind `config.HOG_INVOCATION_RESULTS_ENABLED`. Producing
 * rows for filtered-out events is intentionally skipped — the worker only
 * calls into this service for invocations that are actually queued to run.
 */
export class HogInvocationResultsService {
    private queuedRows: HogInvocationResultRow[] = []

    constructor(
        private outputs: IngestionOutputs<HogInvocationResultsServiceOutput>,
        private config: { HOG_INVOCATION_RESULTS_ENABLED: boolean }
    ) {}

    private functionIdFor(invocation: CyclotronJobInvocation): string {
        if (isHogFunctionInvocation(invocation)) {
            return invocation.hogFunction.id
        }
        if (isHogFlowInvocation(invocation)) {
            return invocation.hogFlow.id
        }
        return invocation.functionId
    }

    private functionKindFor(invocation: CyclotronJobInvocation): 'hog_function' | 'hog_flow' {
        return isHogFlowInvocation(invocation) ? 'hog_flow' : 'hog_function'
    }

    /**
     * Queue a lifecycle row. `status='running'` should be called when the
     * worker dequeues an invocation and is about to execute it; `succeeded` /
     * `failed` are derived inside `queueInvocationResults` from the result.
     *
     * `attempts` and `is_retry` are derived from `invocation.state.rerunAttempts`
     * — set by the rerun paginator on rehydration and never touched by the
     * executor's fetch-retry counter (`state.attempts`).
     */
    queueLifecycleRow(
        invocation: CyclotronJobInvocation,
        status: HogInvocationResultStatus,
        opts: {
            error?: unknown
            errorKind?: string
            startedAt?: Date
            finishedAt?: Date
        } = {}
    ): void {
        if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
            return
        }

        const row = this.buildLifecycleRow(invocation, status, opts)
        counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
        this.queuedRows.push(row)
        hogInvocationResultsPendingMessages.set(this.queuedRows.length)
    }

    private buildLifecycleRow(
        invocation: CyclotronJobInvocation,
        status: HogInvocationResultStatus,
        opts: {
            error?: unknown
            errorKind?: string
            startedAt?: Date
            finishedAt?: Date
        }
    ): HogInvocationResultRow {
        const now = new Date()
        const trigger = extractTriggerFields(invocation)
        const classified = classifyError(opts.error)
        // An explicit `errorKind` overrides the derived one so callers (e.g. the
        // janitor giving up on a poison pill) can stamp a stable, filterable
        // kind that the rerun tooling can target directly.
        const errorKind = opts.errorKind ?? classified.kind
        const errorMessage = classified.message
        const startedAt = opts.startedAt ?? (status === 'running' ? now : undefined)
        const finishedAt = opts.finishedAt ?? (status !== 'running' ? now : undefined)
        const durationMs =
            startedAt && finishedAt
                ? Math.max(0, finishedAt.getTime() - startedAt.getTime())
                : sumDurationMs(invocation)
        // Both hog function and hog flow state carry a `rerunAttempts`
        // counter that the rerun paginator increments on rehydration. Read
        // from whichever shape this invocation is so the `max_attempts` guard
        // applies uniformly.
        const rerunAttempts = isHogFunctionInvocation(invocation)
            ? (invocation.state?.rerunAttempts ?? 0)
            : isHogFlowInvocation(invocation)
              ? (invocation.state?.rerunAttempts ?? 0)
              : 0

        // `firstScheduledAt` records the original cyclotron-scheduled time. The
        // rerun paginator sets it on rehydration; here we also stamp it onto the
        // state the first time we emit a 'running' row, so it survives cyclotron
        // fetch retries (which overwrite `queueScheduledAt`). The 'running' row
        // fires once at invocation creation, before the invocation is enqueued,
        // so the value carries forward in the serialized state. Without this the
        // terminal row — written after a retry — would record the retry time and
        // win the ReplacingMergeTree argMax, mislabeling the run's start time.
        let firstScheduledAt = isHogFunctionInvocation(invocation)
            ? invocation.state?.firstScheduledAt
            : isHogFlowInvocation(invocation)
              ? invocation.state?.firstScheduledAt
              : undefined
        const scheduledAtIso = isoMicroseconds(invocation.queueScheduledAt?.toJSDate() ?? now)
        if (status === 'running' && firstScheduledAt === undefined) {
            firstScheduledAt = scheduledAtIso
            if (isHogFunctionInvocation(invocation)) {
                invocation.state.firstScheduledAt = scheduledAtIso
            } else if (isHogFlowInvocation(invocation) && invocation.state) {
                invocation.state.firstScheduledAt = scheduledAtIso
            }
        }

        const row: HogInvocationResultRow = {
            team_id: invocation.teamId,
            function_kind: this.functionKindFor(invocation),
            function_id: this.functionIdFor(invocation),
            invocation_id: invocation.id,
            parent_run_id: invocation.parentRunId ?? '',
            status,
            attempts: rerunAttempts,
            is_retry: rerunAttempts > 0 ? 1 : 0,
            scheduled_at: scheduledAtIso,
            first_scheduled_at: firstScheduledAt ?? scheduledAtIso,
            started_at: startedAt ? isoMicroseconds(startedAt) : null,
            finished_at: finishedAt ? isoMicroseconds(finishedAt) : null,
            duration_ms: durationMs,
            error_kind: errorKind,
            error_message: errorMessage,
            event_uuid: trigger.event_uuid,
            distinct_id: trigger.distinct_id,
            person_id: trigger.person_id,
            invocation_globals: serializeInvocationGlobals(invocation),
            version: microsecondsSinceEpoch(),
            is_deleted: 0,
        }

        return row
    }

    /**
     * Produce a single terminal `failed` lifecycle row and await the broker
     * ack, returning `true` only once the row is durably enqueued. Bypasses the
     * batched queue/flush path so the caller can establish a strict ordering —
     * the janitor records a poison pill's give-up here BEFORE deleting the
     * cyclotron row, closing the window where the row could be gone with no
     * recovery record. Returns `false` (never throws) when recording is
     * disabled or the produce fails, so the caller can keep the job instead of
     * dropping it silently.
     */
    async recordTerminalFailureDurably(
        invocation: CyclotronJobInvocation,
        opts: { error?: unknown; errorKind?: string } = {}
    ): Promise<boolean> {
        if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
            return false
        }

        // Build inside the try too — a malformed invocation (e.g. an unparseable
        // date) must fail this record safely so the caller keeps the job, never
        // crash the whole janitor cycle.
        try {
            const row = this.buildLifecycleRow(invocation, 'failed', opts)
            await this.produceRow(row)
            counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
            return true
        } catch (error) {
            counterHogInvocationResultProduceFailed.inc()
            logger.error('⚠️', `failed to durably record terminal failure: ${error}`, {
                error: String(error),
                invocation_id: invocation.id,
            })
            captureException(error)
            return false
        }
    }

    private async produceRow(row: HogInvocationResultRow): Promise<void> {
        await this.produceEncodedRow(row, await this.encodeRow(row))
    }

    private async encodeRow(row: HogInvocationResultRow): Promise<Buffer> {
        return Buffer.from(
            safeClickhouseString(
                JSON.stringify({
                    ...row,
                    invocation_globals: await compressInvocationGlobals(row.invocation_globals),
                })
            )
        )
    }

    private async produceEncodedRow(row: HogInvocationResultRow, value: Buffer): Promise<void> {
        await this.outputs.produce(HOG_INVOCATION_RESULTS_OUTPUT, {
            // Partition by invocation_id so all rows for a single invocation
            // land on the same Kafka partition (and the same ClickHouse shard).
            key: Buffer.from(row.invocation_id),
            value,
        })
    }

    /**
     * Queue a lifecycle row for a re-run wrapper job.
     *
     * Conceptually a wrapper is a meta-invocation: one row per re-run rather
     * than one per rerun invocation. Stamping it on `hog_invocation_results`
     * (with `function_kind = *_rerun`) means the same Invocations tab is the
     * only debugging surface for both real invocations and the wrappers that
     * spawned them — no separate polling, no separate UI.
     *
     * Fields we deliberately leave empty: `event_uuid` / `distinct_id` /
     * `person_id` — a wrapper isn't triggered by a single event. The filter
     * blob goes in `invocation_globals` (never exposed via HogQL — same
     * security guarantee as for normal invocations).
     */
    queueRerunWrapperRow(args: {
        teamId: number
        parentFunctionKind: RerunFunctionKind
        functionId: string
        rerunJobId: string
        status: 'running' | 'succeeded' | 'failed'
        pagesProcessed: number
        filter: RerunFilter
        scheduledAt: Date
        startedAt?: Date
        finishedAt?: Date
        error?: unknown
    }): void {
        if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
            return
        }

        const { kind: errorKind, message: errorMessage } = classifyError(args.error)
        const durationMs =
            args.startedAt && args.finishedAt ? Math.max(0, args.finishedAt.getTime() - args.startedAt.getTime()) : null

        const scheduledAtIso = isoMicroseconds(args.scheduledAt)
        const row: HogInvocationResultRow = {
            team_id: args.teamId,
            function_kind: rerunWrapperKindFor(args.parentFunctionKind),
            function_id: args.functionId,
            invocation_id: args.rerunJobId,
            parent_run_id: '',
            status: args.status,
            attempts: args.pagesProcessed,
            is_retry: 0,
            scheduled_at: scheduledAtIso,
            // Wrapper rows don't have retries — first == scheduled. Keeping
            // the field populated so the column reads consistently across
            // both row kinds.
            first_scheduled_at: scheduledAtIso,
            started_at: args.startedAt ? isoMicroseconds(args.startedAt) : null,
            finished_at: args.finishedAt ? isoMicroseconds(args.finishedAt) : null,
            duration_ms: durationMs,
            error_kind: errorKind,
            error_message: errorMessage,
            event_uuid: '',
            distinct_id: '',
            person_id: '',
            invocation_globals: JSON.stringify(args.filter),
            version: microsecondsSinceEpoch(),
            is_deleted: 0,
        }

        counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
        this.queuedRows.push(row)
        hogInvocationResultsPendingMessages.set(this.queuedRows.length)
    }

    queueInvocationResults(results: CyclotronJobInvocationResult[]): void {
        if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
            return
        }

        for (const result of results) {
            if (!(result.finished || result.error)) {
                // Mid-flight intermediate update — no terminal row yet. The
                // 'running' row was emitted at execution start; nothing else
                // to write until completion.
                continue
            }

            const status: HogInvocationResultStatus = result.error
                ? 'failed'
                : result.canceled
                  ? 'canceled'
                  : 'succeeded'
            this.queueLifecycleRow(result.invocation, status, { error: result.error })
        }
    }

    /**
     * Discard any queued (un-flushed) lifecycle rows for the given invocation
     * ids. Used by the rerun paginator when a re-enqueue is refused (existing
     * job is still in-flight) — we already queued a 'running' row for that
     * invocation_id; drop it so we don't surface a stale running marker.
     */
    dropQueuedRowsFor(invocationIds: string[]): void {
        if (invocationIds.length === 0) {
            return
        }
        const drop = new Set(invocationIds)
        this.queuedRows = this.queuedRows.filter((r) => !drop.has(r.invocation_id))
        hogInvocationResultsPendingMessages.set(this.queuedRows.length)
    }

    // Required by ResultSink. This sink holds nothing of its own: it writes through the shared
    // Kafka outputs, which the server disconnects separately on shutdown.
    async stop(): Promise<void> {
        return Promise.resolve()
    }

    async flush(): Promise<void> {
        const { produced } = await this.handOffQueuedRows()
        await produced
    }

    // Resolves once every queued row is encoded and passed to the Kafka producer, which flushes the rows it
    // holds on shutdown. `produced` resolves when the broker has acked them, so a caller that must not wait
    // on the broker awaits only the hand-off. Neither promise rejects.
    async handOffQueuedRows(): Promise<{ produced: Promise<void> }> {
        if (this.queuedRows.length === 0) {
            return { produced: Promise.resolve() }
        }

        const rows = this.queuedRows
        this.queuedRows = []
        hogInvocationResultsPendingMessages.set(0)

        const encoded = await Promise.all(
            rows.map((row) =>
                this.encodeRow(row).catch((error) => {
                    this.onRowProduceFailed(row, error)
                    return null
                })
            )
        )
        const produced = Promise.all(
            rows.flatMap((row, i) => {
                const value = encoded[i]
                return value
                    ? [this.produceEncodedRow(row, value).catch((error) => this.onRowProduceFailed(row, error))]
                    : []
            })
        ).then(() => undefined)
        return { produced }
    }

    private onRowProduceFailed(row: HogInvocationResultRow, error: unknown): void {
        counterHogInvocationResultProduceFailed.inc()
        // Best-effort — never disrupt invocation processing for a
        // monitoring write.
        logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {
            error: String(error),
            invocation_id: row.invocation_id,
        })
        captureException(error)
    }
}