:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { promisify } from 'node:util'
import { gunzip, gzip } from 'node:zlib'
import { Counter, Gauge } from 'prom-client'
import { HOG_INVOCATION_RESULTS_OUTPUT, HogInvocationResultsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { safeClickhouseString } from '~/common/utils/db/utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import type { CdpOutput } from '../../cdp-services'
import { RerunFilter, RerunFunctionKind, rerunWrapperKindFor } from '../../rerun/rerun-job.types'
import {
CyclotronJobInvocation,
CyclotronJobInvocationHogFlow,
CyclotronJobInvocationHogFunction,
CyclotronJobInvocationResult,
} from '../../types'
import { TRANSACTIONAL_EMAIL_EVENT, hasSensitiveMessageStep } from '../hogflows/hogflow-sensitivity'
const counterHogInvocationResultRowsProduced = new Counter({
name: 'cdp_hog_invocation_result_rows_produced',
help: 'Lifecycle rows queued for the hog_invocation_results ClickHouse table.',
labelNames: ['function_kind', 'status'],
})
const counterHogInvocationResultProduceFailed = new Counter({
name: 'cdp_hog_invocation_result_produce_failed',
help: 'Rows that failed to produce to Kafka.',
})
const hogInvocationResultsPendingMessages = new Gauge({
name: 'cdp_hog_invocation_results_pending_messages',
help: 'Rows queued waiting to be flushed to Kafka.',
})
export type HogInvocationResultsServiceOutput = HogInvocationResultsOutput | CdpOutput
/**
* Lifecycle row produced to ClickHouse via Kafka. Mirrors the columns on the
* hog_invocation_results_data table. Two such rows are produced per
* invocation: one when execution starts (`status='running'`) and one when it
* finishes (`status='succeeded' | 'failed' | 'canceled'`). On a rerun, the
* cycle repeats with the same `invocation_id`, `is_retry=1`, and `attempts`
* bumped — the ReplacingMergeTree on
* `(team_id, function_kind, function_id, invocation_id)` keyed by `version`
* collapses prior versions at merge time.
*/
export type HogInvocationResultStatus = 'running' | 'succeeded' | 'failed' | 'canceled'
export interface HogInvocationResultRow {
team_id: number
// `*_rerun` kinds tag the wrapper row that drives a re-run, so the
// Invocations list can surface in-flight re-runs alongside the function's
// normal invocations. See `rerun-job.types.ts` for the helper.
function_kind: 'hog_function' | 'hog_flow' | 'hog_function_rerun' | 'hog_flow_rerun'
function_id: string
invocation_id: string
parent_run_id: string
status: HogInvocationResultStatus
attempts: number
is_retry: 0 | 1
scheduled_at: string // ISO microsecond DateTime64
// Original cyclotron-scheduled time, carried unchanged through retries.
// ReplacingMergeTree collapses lifecycle rows per invocation_id, so
// `min(scheduled_at)` post-merge is unreliable — every row stamps this
// verbatim so `argMax(first_scheduled_at, version)` returns it correctly.
first_scheduled_at: string
started_at: string | null
finished_at: string | null
duration_ms: number | null
error_kind: string
error_message: string
event_uuid: string
distinct_id: string
person_id: string
invocation_globals: string // globals JSON (inputs/groups/person stripped) — gzip+base64'd on produce
version: string // microsecond-precision UInt64; serialized as string to dodge JS's 53-bit precision
is_deleted: 0 | 1
}
const isHogFunctionInvocation = (invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFunction =>
'hogFunction' in invocation
const isHogFlowInvocation = (invocation: CyclotronJobInvocation): invocation is CyclotronJobInvocationHogFlow =>
'hogFlow' in invocation
// Monotonic microsecond timestamp used as the row `version`. ReplacingMergeTree keeps the
// row with the max `version` per key; without a monotonic version the 'running' + terminal
// rows of a fast invocation can share or invert a version, and CH keeps one arbitrarily —
// leaving the runs UI stuck on 'running' (and, via the rerun paginator's
// `argMax(status, version)` in-flight check, that invocation un-rerunnable).
//
// The rows being compared are produced across DIFFERENT processes — the 'running' row by
// the events consumer, the terminal row by a cyclotron worker — so the base must be a
// clock that stays synchronised across processes. `Date.now()` is continuously
// NTP-disciplined, so cross-process skew stays well under the queue latency between the
// two rows. (`performance.timeOrigin + performance.now()` is monotonic within a process
// but freezes each process's wall-clock offset at start and never re-syncs, so an
// NTP step or VM pause would invert versions across pods — worse for the comparison that
// matters.)
//
// A wall clock alone can still tie, or briefly step backward under NTP, for two rows
// produced close together in the SAME process (e.g. the flaky monotonicity test). Clamp to
// strictly exceed the last value this process issued: that adds strict in-process
// monotonicity on top of the NTP-anchored cross-process ordering.
//
// This module-level counter is process-global and never resets, so a test using
// `jest.setSystemTime()` to a time earlier than an already-issued stamp will see clamped
// `last + 1` values, not the fake time. Don't rewind the clock below a prior stamp in tests.
let lastVersionMicros = 0n
const microsecondsSinceEpoch = (): string => {
// BigInt avoids the 53-bit cap so the number lines up with ClickHouse UInt64.
const wallMicros = BigInt(Date.now()) * 1000n
lastVersionMicros = wallMicros > lastVersionMicros ? wallMicros : lastVersionMicros + 1n
return lastVersionMicros.toString()
}
const isoMicroseconds = (date: Date): string => {
// ClickHouse DateTime64(6) accepts 'YYYY-MM-DD HH:MM:SS.ffffff'.
return date.toISOString().replace('T', ' ').replace('Z', '000')
}
const truncate = (value: string, max: number): string => {
if (value.length <= max) {
return value
}
return value.slice(0, max)
}
// Best-effort error classification — keeps `error_kind` low-cardinality so the
// status_idx skipping index stays small. Only the message lands in
// `error_message`, never the stack trace — app-level executors pass the whole
// `Error` here, and its stack would otherwise leak into the Invocations tab.
export const classifyError = (error: unknown): { kind: string; message: string } => {
if (!error) {
return { kind: '', message: '' }
}
const message =
typeof error === 'string'
? error
: error instanceof Error
? // fall back to the error name so an empty message doesn't blank the row (without the stack)
error.message || error.name
: (() => {
try {
return JSON.stringify(error)
} catch {
return String(error)
}
})()
const lower = message.toLowerCase()
let kind = 'hog_error'
// Anchor the status-code patterns to HTTP-context tokens so messages like
// "processed 500 items" or "400 rows returned" don't get misclassified as
// http_5xx / http_4xx. Matches: 'status 503', 'http 502', 'http/2 504',
// 'status code 500', etc.
const http5xxRegex = /\b(?:status(?:\s*code)?|http(?:[/ ]\d)?)\s*[:= ]?\s*5\d{2}\b/
const http4xxRegex = /\b(?:status(?:\s*code)?|http(?:[/ ]\d)?)\s*[:= ]?\s*4\d{2}\b/
if (lower.includes('timeout') || lower.includes('timed out')) {
kind = 'timeout'
} else if (http5xxRegex.test(lower) || lower.includes('server error')) {
kind = 'http_5xx'
} else if (http4xxRegex.test(lower)) {
kind = 'http_4xx'
} else if (lower.includes('out of memory') || lower.includes('oom')) {
kind = 'oom'
}
return { kind, message: truncate(message, 4096) }
}
const extractTriggerFields = (
invocation: CyclotronJobInvocation
): { event_uuid: string; distinct_id: string; person_id: string } => {
if (isHogFunctionInvocation(invocation)) {
const globals = invocation.state.globals
return {
event_uuid: globals.event?.uuid ?? '',
distinct_id: globals.event?.distinct_id ?? '',
person_id: globals.person?.id ?? '',
}
}
if (isHogFlowInvocation(invocation)) {
const event = invocation.state?.event
return {
event_uuid: event?.uuid ?? '',
distinct_id: event?.distinct_id ?? '',
person_id: invocation.state?.personId ?? invocation.person?.id ?? '',
}
}
return { event_uuid: '', distinct_id: '', person_id: '' }
}
// Strip every `inputs` blob we can find on a globals/state tree. `inputs`
// holds resolved input values for the function — these can include user-
// supplied secrets (API keys, OAuth tokens, etc.) that the function templated
// in at execute time. Persisting them to ClickHouse for 30 days would expand
// the blast radius of any leak. On rerun we re-resolve inputs from the
// current hog function config + integration store, so this is also a no-op
// for rerun correctness — we'll always pick up the latest secrets, not a
// snapshot from when the original invocation ran.
const INPUT_KEYS: ReadonlySet<string> = new Set(['inputs'])
// A sensitive message carries reset links, login codes or private text, so its snapshot keeps no
// variables either, and a rerun can't rebuild the send. A step paused mid-send (e.g. for a provider
// retry) holds the rendered message on its VM stack, and the person and group properties it read, so those go too.
const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups', 'send'])
export const SENSITIVE_VARIABLES_OMITTED = 'sensitive_variables_omitted'
const stripKeys = <T>(value: T, keys: ReadonlySet<string>): T => {
if (value === null || typeof value !== 'object') {
return value
}
if (Array.isArray(value)) {
// We don't expect inputs to live inside arrays today; this branch is a
// defensive cheap pass-through so a future schema change doesn't have
// to come back here.
return value.map((item) => stripKeys(item, keys)) as unknown as T
}
const obj = value as Record<string, unknown>
const out: Record<string, unknown> = {}
for (const [k, v] of Object.entries(obj)) {
if (keys.has(k)) {
continue
}
out[k] = stripKeys(v, keys)
}
return out as T
}
// The payload the rerun path needs to rehydrate the invocation, kept minimal.
// `inputs`, `groups` and `person` are all dropped: the cyclotron worker
// rehydrates `groups`/`person` (loadHogFunctions) and the executor rebuilds
// `inputs` when an invocation arrives without them — so the rerun reconstructs
// everything from the event against the latest config. `inputs` must be
// dropped anyway for security (resolved secrets — see `stripKeys`).
// Recovery paths pass a workflow stub with only an id. A run that started before its template became
// sensitive carries no stamp, so a transactional email run without the actions is treated as sensitive.
const sendsSensitiveMessage = (invocation: CyclotronJobInvocationHogFlow): boolean => {
if (invocation.state?.sensitive === true) {
return true
}
if (!Array.isArray(invocation.hogFlow?.actions)) {
return invocation.state?.event?.event === TRANSACTIONAL_EMAIL_EVENT
}
return hasSensitiveMessageStep(invocation.hogFlow)
}
const serializeInvocationGlobals = (invocation: CyclotronJobInvocation): string => {
if (isHogFunctionInvocation(invocation)) {
const { groups: _groups, person: _person, ...globals } = invocation.state.globals
return JSON.stringify(stripKeys(globals, INPUT_KEYS))
}
if (isHogFlowInvocation(invocation)) {
// Hog flow state can carry a per-action `currentAction.hogFunctionState.globals.inputs`
// — `stripKeys` walks the tree and removes those too.
if (sendsSensitiveMessage(invocation)) {
return JSON.stringify({
...stripKeys(invocation.state ?? {}, SENSITIVE_KEYS),
[SENSITIVE_VARIABLES_OMITTED]: true,
})
}
return JSON.stringify(stripKeys(invocation.state ?? {}, INPUT_KEYS))
}
return '{}'
}
const gzipAsync = promisify(gzip)
const gunzipAsync = promisify(gunzip)
// `invocation_globals` is the bulk of every lifecycle row. Warpstream meters
// the uncompressed message bytes, so gzipping this one field before produce
// directly cuts cyclotron throughput. The row envelope stays plain JSON — only
// this field is opaque — so the ClickHouse Kafka engine still parses the
// message as JSONEachRow. `decodeInvocationGlobals` is the inverse.
const compressInvocationGlobals = async (globalsJson: string): Promise<string> => {
return (await gzipAsync(globalsJson)).toString('base64')
}
/**
* Inverse of `compressInvocationGlobals`. Used by the rerun paginator to read
* `invocation_globals` back off ClickHouse. Rows written before field
* compression landed are still raw JSON — base64 can never start with `{`, so
* the prefix is an unambiguous discriminator for the legacy fallback.
*/
export const decodeInvocationGlobals = async (stored: string): Promise<unknown> => {
if (stored.startsWith('{')) {
return parseJSON(stored)
}
return parseJSON((await gunzipAsync(Buffer.from(stored, 'base64'))).toString('utf8'))
}
const sumDurationMs = (invocation: CyclotronJobInvocation): number | null => {
if (!isHogFunctionInvocation(invocation)) {
return null
}
const timings = invocation.state?.timings
if (!timings || timings.length === 0) {
return null
}
// Per-step timings are fractional ms (perf.now() deltas). ClickHouse stores
// duration_ms as UInt32 — Kafka's JSONEachRow parser rejects the row if the
// field arrives as a float, so round before serialization.
return Math.round(timings.reduce((sum, t) => sum + t.duration_ms, 0))
}
/**
* Per-invocation lifecycle row producer. Lives next to
* `HogFunctionMonitoringService` (which handles aggregated metrics + log
* lines) — this one writes a single row per lifecycle event so the new
* runs/invocations UI and the rerun path can read it back via HogQL.
*
* Off by default behind `config.HOG_INVOCATION_RESULTS_ENABLED`. Producing
* rows for filtered-out events is intentionally skipped — the worker only
* calls into this service for invocations that are actually queued to run.
*/
export class HogInvocationResultsService {
private queuedRows: HogInvocationResultRow[] = []
constructor(
private outputs: IngestionOutputs<HogInvocationResultsServiceOutput>,
private config: { HOG_INVOCATION_RESULTS_ENABLED: boolean }
) {}
private functionIdFor(invocation: CyclotronJobInvocation): string {
if (isHogFunctionInvocation(invocation)) {
return invocation.hogFunction.id
}
if (isHogFlowInvocation(invocation)) {
return invocation.hogFlow.id
}
return invocation.functionId
}
private functionKindFor(invocation: CyclotronJobInvocation): 'hog_function' | 'hog_flow' {
return isHogFlowInvocation(invocation) ? 'hog_flow' : 'hog_function'
}
/**
* Queue a lifecycle row. `status='running'` should be called when the
* worker dequeues an invocation and is about to execute it; `succeeded` /
* `failed` are derived inside `queueInvocationResults` from the result.
*
* `attempts` and `is_retry` are derived from `invocation.state.rerunAttempts`
* — set by the rerun paginator on rehydration and never touched by the
* executor's fetch-retry counter (`state.attempts`).
*/
queueLifecycleRow(
invocation: CyclotronJobInvocation,
status: HogInvocationResultStatus,
opts: {
error?: unknown
errorKind?: string
startedAt?: Date
finishedAt?: Date
} = {}
): void {
if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
return
}
const row = this.buildLifecycleRow(invocation, status, opts)
counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
this.queuedRows.push(row)
hogInvocationResultsPendingMessages.set(this.queuedRows.length)
}
private buildLifecycleRow(
invocation: CyclotronJobInvocation,
status: HogInvocationResultStatus,
opts: {
error?: unknown
errorKind?: string
startedAt?: Date
finishedAt?: Date
}
): HogInvocationResultRow {
const now = new Date()
const trigger = extractTriggerFields(invocation)
const classified = classifyError(opts.error)
// An explicit `errorKind` overrides the derived one so callers (e.g. the
// janitor giving up on a poison pill) can stamp a stable, filterable
// kind that the rerun tooling can target directly.
const errorKind = opts.errorKind ?? classified.kind
const errorMessage = classified.message
const startedAt = opts.startedAt ?? (status === 'running' ? now : undefined)
const finishedAt = opts.finishedAt ?? (status !== 'running' ? now : undefined)
const durationMs =
startedAt && finishedAt
? Math.max(0, finishedAt.getTime() - startedAt.getTime())
: sumDurationMs(invocation)
// Both hog function and hog flow state carry a `rerunAttempts`
// counter that the rerun paginator increments on rehydration. Read
// from whichever shape this invocation is so the `max_attempts` guard
// applies uniformly.
const rerunAttempts = isHogFunctionInvocation(invocation)
? (invocation.state?.rerunAttempts ?? 0)
: isHogFlowInvocation(invocation)
? (invocation.state?.rerunAttempts ?? 0)
: 0
// `firstScheduledAt` records the original cyclotron-scheduled time. The
// rerun paginator sets it on rehydration; here we also stamp it onto the
// state the first time we emit a 'running' row, so it survives cyclotron
// fetch retries (which overwrite `queueScheduledAt`). The 'running' row
// fires once at invocation creation, before the invocation is enqueued,
// so the value carries forward in the serialized state. Without this the
// terminal row — written after a retry — would record the retry time and
// win the ReplacingMergeTree argMax, mislabeling the run's start time.
let firstScheduledAt = isHogFunctionInvocation(invocation)
? invocation.state?.firstScheduledAt
: isHogFlowInvocation(invocation)
? invocation.state?.firstScheduledAt
: undefined
const scheduledAtIso = isoMicroseconds(invocation.queueScheduledAt?.toJSDate() ?? now)
if (status === 'running' && firstScheduledAt === undefined) {
firstScheduledAt = scheduledAtIso
if (isHogFunctionInvocation(invocation)) {
invocation.state.firstScheduledAt = scheduledAtIso
} else if (isHogFlowInvocation(invocation) && invocation.state) {
invocation.state.firstScheduledAt = scheduledAtIso
}
}
const row: HogInvocationResultRow = {
team_id: invocation.teamId,
function_kind: this.functionKindFor(invocation),
function_id: this.functionIdFor(invocation),
invocation_id: invocation.id,
parent_run_id: invocation.parentRunId ?? '',
status,
attempts: rerunAttempts,
is_retry: rerunAttempts > 0 ? 1 : 0,
scheduled_at: scheduledAtIso,
first_scheduled_at: firstScheduledAt ?? scheduledAtIso,
started_at: startedAt ? isoMicroseconds(startedAt) : null,
finished_at: finishedAt ? isoMicroseconds(finishedAt) : null,
duration_ms: durationMs,
error_kind: errorKind,
error_message: errorMessage,
event_uuid: trigger.event_uuid,
distinct_id: trigger.distinct_id,
person_id: trigger.person_id,
invocation_globals: serializeInvocationGlobals(invocation),
version: microsecondsSinceEpoch(),
is_deleted: 0,
}
return row
}
/**
* Produce a single terminal `failed` lifecycle row and await the broker
* ack, returning `true` only once the row is durably enqueued. Bypasses the
* batched queue/flush path so the caller can establish a strict ordering —
* the janitor records a poison pill's give-up here BEFORE deleting the
* cyclotron row, closing the window where the row could be gone with no
* recovery record. Returns `false` (never throws) when recording is
* disabled or the produce fails, so the caller can keep the job instead of
* dropping it silently.
*/
async recordTerminalFailureDurably(
invocation: CyclotronJobInvocation,
opts: { error?: unknown; errorKind?: string } = {}
): Promise<boolean> {
if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
return false
}
// Build inside the try too — a malformed invocation (e.g. an unparseable
// date) must fail this record safely so the caller keeps the job, never
// crash the whole janitor cycle.
try {
const row = this.buildLifecycleRow(invocation, 'failed', opts)
await this.produceRow(row)
counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
return true
} catch (error) {
counterHogInvocationResultProduceFailed.inc()
logger.error('⚠️', `failed to durably record terminal failure: ${error}`, {
error: String(error),
invocation_id: invocation.id,
})
captureException(error)
return false
}
}
private async produceRow(row: HogInvocationResultRow): Promise<void> {
await this.produceEncodedRow(row, await this.encodeRow(row))
}
private async encodeRow(row: HogInvocationResultRow): Promise<Buffer> {
return Buffer.from(
safeClickhouseString(
JSON.stringify({
...row,
invocation_globals: await compressInvocationGlobals(row.invocation_globals),
})
)
)
}
private async produceEncodedRow(row: HogInvocationResultRow, value: Buffer): Promise<void> {
await this.outputs.produce(HOG_INVOCATION_RESULTS_OUTPUT, {
// Partition by invocation_id so all rows for a single invocation
// land on the same Kafka partition (and the same ClickHouse shard).
key: Buffer.from(row.invocation_id),
value,
})
}
/**
* Queue a lifecycle row for a re-run wrapper job.
*
* Conceptually a wrapper is a meta-invocation: one row per re-run rather
* than one per rerun invocation. Stamping it on `hog_invocation_results`
* (with `function_kind = *_rerun`) means the same Invocations tab is the
* only debugging surface for both real invocations and the wrappers that
* spawned them — no separate polling, no separate UI.
*
* Fields we deliberately leave empty: `event_uuid` / `distinct_id` /
* `person_id` — a wrapper isn't triggered by a single event. The filter
* blob goes in `invocation_globals` (never exposed via HogQL — same
* security guarantee as for normal invocations).
*/
queueRerunWrapperRow(args: {
teamId: number
parentFunctionKind: RerunFunctionKind
functionId: string
rerunJobId: string
status: 'running' | 'succeeded' | 'failed'
pagesProcessed: number
filter: RerunFilter
scheduledAt: Date
startedAt?: Date
finishedAt?: Date
error?: unknown
}): void {
if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
return
}
const { kind: errorKind, message: errorMessage } = classifyError(args.error)
const durationMs =
args.startedAt && args.finishedAt ? Math.max(0, args.finishedAt.getTime() - args.startedAt.getTime()) : null
const scheduledAtIso = isoMicroseconds(args.scheduledAt)
const row: HogInvocationResultRow = {
team_id: args.teamId,
function_kind: rerunWrapperKindFor(args.parentFunctionKind),
function_id: args.functionId,
invocation_id: args.rerunJobId,
parent_run_id: '',
status: args.status,
attempts: args.pagesProcessed,
is_retry: 0,
scheduled_at: scheduledAtIso,
// Wrapper rows don't have retries — first == scheduled. Keeping
// the field populated so the column reads consistently across
// both row kinds.
first_scheduled_at: scheduledAtIso,
started_at: args.startedAt ? isoMicroseconds(args.startedAt) : null,
finished_at: args.finishedAt ? isoMicroseconds(args.finishedAt) : null,
duration_ms: durationMs,
error_kind: errorKind,
error_message: errorMessage,
event_uuid: '',
distinct_id: '',
person_id: '',
invocation_globals: JSON.stringify(args.filter),
version: microsecondsSinceEpoch(),
is_deleted: 0,
}
counterHogInvocationResultRowsProduced.labels(row.function_kind, row.status).inc()
this.queuedRows.push(row)
hogInvocationResultsPendingMessages.set(this.queuedRows.length)
}
queueInvocationResults(results: CyclotronJobInvocationResult[]): void {
if (!this.config.HOG_INVOCATION_RESULTS_ENABLED) {
return
}
for (const result of results) {
if (!(result.finished || result.error)) {
// Mid-flight intermediate update — no terminal row yet. The
// 'running' row was emitted at execution start; nothing else
// to write until completion.
continue
}
const status: HogInvocationResultStatus = result.error
? 'failed'
: result.canceled
? 'canceled'
: 'succeeded'
this.queueLifecycleRow(result.invocation, status, { error: result.error })
}
}
/**
* Discard any queued (un-flushed) lifecycle rows for the given invocation
* ids. Used by the rerun paginator when a re-enqueue is refused (existing
* job is still in-flight) — we already queued a 'running' row for that
* invocation_id; drop it so we don't surface a stale running marker.
*/
dropQueuedRowsFor(invocationIds: string[]): void {
if (invocationIds.length === 0) {
return
}
const drop = new Set(invocationIds)
this.queuedRows = this.queuedRows.filter((r) => !drop.has(r.invocation_id))
hogInvocationResultsPendingMessages.set(this.queuedRows.length)
}
// Required by ResultSink. This sink holds nothing of its own: it writes through the shared
// Kafka outputs, which the server disconnects separately on shutdown.
async stop(): Promise<void> {
return Promise.resolve()
}
async flush(): Promise<void> {
const { produced } = await this.handOffQueuedRows()
await produced
}
// Resolves once every queued row is encoded and passed to the Kafka producer, which flushes the rows it
// holds on shutdown. `produced` resolves when the broker has acked them, so a caller that must not wait
// on the broker awaits only the hand-off. Neither promise rejects.
async handOffQueuedRows(): Promise<{ produced: Promise<void> }> {
if (this.queuedRows.length === 0) {
return { produced: Promise.resolve() }
}
const rows = this.queuedRows
this.queuedRows = []
hogInvocationResultsPendingMessages.set(0)
const encoded = await Promise.all(
rows.map((row) =>
this.encodeRow(row).catch((error) => {
this.onRowProduceFailed(row, error)
return null
})
)
)
const produced = Promise.all(
rows.flatMap((row, i) => {
const value = encoded[i]
return value
? [this.produceEncodedRow(row, value).catch((error) => this.onRowProduceFailed(row, error))]
: []
})
).then(() => undefined)
return { produced }
}
private onRowProduceFailed(row: HogInvocationResultRow, error: unknown): void {
counterHogInvocationResultProduceFailed.inc()
// Best-effort — never disrupt invocation processing for a
// monitoring write.
logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {
error: String(error),
invocation_id: row.invocation_id,
})
captureException(error)
}
}