:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { ClickHouseClient } from '@clickhouse/client'
import { DateTime } from 'luxon'
import { Counter } from 'prom-client'
import { logger } from '~/common/utils/logger'
import { HogFunctionInvocationGlobalsSchema } from '../schema/cyclotron'
import { CyclotronJobConflictError } from '../services/cyclotron-v2'
import { createHogFlowInvocation } from '../services/hogflows/hogflow-executor.service'
import { HogFlowManagerService } from '../services/hogflows/hogflow-manager.service'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import {
HogInvocationResultsService,
SENSITIVE_VARIABLES_OMITTED,
decodeInvocationGlobals,
} from '../services/monitoring/hog-invocation-results.service'
import {
CyclotronJobInvocation,
CyclotronJobInvocationHogFlow,
CyclotronJobInvocationHogFunction,
HogFunctionFilterGlobals,
HogFunctionInvocationGlobalsWithInputs,
RERUNNABLE_HOG_FUNCTION_TYPES,
} from '../types'
import { convertToHogFunctionFilterGlobal } from '../utils/hog-function-filtering'
import {
RERUN_MAX_CONSECUTIVE_PAGE_ERRORS,
RERUN_PAGE_SIZE,
RerunFunctionKind,
RerunJobProgress,
RerunJobState,
RerunPagingMode,
} from './rerun-job.types'
const counterRerunPageProcessed = new Counter({
name: 'cdp_hog_invocation_rerun_pages_processed_total',
help: 'Rerun paginator pages processed, by function kind and outcome.',
labelNames: ['function_kind', 'outcome'],
})
const counterRerunInvocationsQueued = new Counter({
name: 'cdp_hog_invocation_rerun_queued_total',
help: 'Reruned invocations queued onto cyclotron, by function kind.',
labelNames: ['function_kind'],
})
const counterRerunInvocationsSkipped = new Counter({
name: 'cdp_hog_invocation_rerun_skipped_total',
help: 'Invocations matched by rerun filters but skipped (over max_attempts, missing function, malformed payload).',
labelNames: ['function_kind', 'reason'],
})
// ClickHouse's `DateTime64` parser only accepts 'YYYY-MM-DD HH:MM:SS[.fff]'.
// The Django serializer (and our internal `scheduled_at` column representation)
// uses ISO 8601 with `T` and `Z`. Convert before binding to query params.
const toClickhouseDateTime = (value: string): string => {
if (!value) {
return value
}
// Already in CH form? Leave it alone.
if (!value.includes('T')) {
return value
}
return value
.replace('T', ' ')
.replace(/Z$/, '')
.replace(/([+-]\d{2}):?(\d{2})$/, '')
}
interface Paging {
keyColumn: 'first_scheduled_at' | 'scheduled_at'
orderColumn: 'run_first_scheduled_at' | 'last_scheduled_at'
cursorTime: (row: InvocationRow) => string
}
const PAGING: Record<RerunPagingMode, Paging> = {
// Every row of a run shares `first_scheduled_at`, so the window and cursor keep or drop whole runs.
// There is no `scheduled_at` bound, because a rescheduled row can land before the run's start.
run_start: {
keyColumn: 'first_scheduled_at',
orderColumn: 'run_first_scheduled_at',
cursorTime: (row) => row.run_first_scheduled_at,
},
last_row: {
keyColumn: 'scheduled_at',
orderColumn: 'last_scheduled_at',
cursorTime: (row) => row.last_scheduled_at,
},
}
const pagingFor = (state: RerunJobState): Paging => PAGING[state.paging ?? 'last_row']
interface InvocationRow {
invocation_id: string
parent_run_id: string
// Prior rerun count for this invocation (argMax over version). Named to avoid
// shadowing the raw `attempts` column in the fetch query — see fetchPage.
latest_attempts: number
last_scheduled_at: string
run_first_scheduled_at: string
invocation_globals: string
}
export interface PageOutcome {
/** New job state after this page. The worker writes it back via reschedule(state) — or ack()s if done. */
state: RerunJobState
}
/**
* Re-enqueue targets keyed by rerun function kind. Mirrors the split in
* cdp-events-consumer — hog functions go to kafka, hog flows to postgres-v2.
* Keying by kind (rather than two positional queue args of similar shape)
* stops the two backends from being swapped by mistake.
*/
export interface RerunJobQueues {
hog_function: JobQueue
hog_flow: CyclotronJobQueuePostgresV2
}
/**
* Context the worker passes alongside the parsed state on each page. Lets the
* paginator stamp wrapper lifecycle rows with the right `invocation_id`
* (= cyclotron job id) and a stable `scheduled_at` for the wrapper across pages.
*/
export interface RerunJobContext {
jobId: string
/** When the wrapper job was first created in cyclotron — anchor for the wrapper's `scheduled_at`. */
createdAt: DateTime
}
/**
* Processes a single page of work for a rerun wrapper job. Pure-ish — the
* caller (the worker) handles the cyclotron-v2 ack/reschedule flow with the
* returned state. Splitting the page-of-work logic out of the worker keeps
* the cyclotron plumbing testable in isolation from the ClickHouse paging.
*/
export class RerunPaginatorService {
constructor(
private clickhouse: ClickHouseClient,
private hogFunctionManager: HogFunctionManagerService,
private hogFlowManager: HogFlowManagerService,
private invocationResultsRowsService: HogInvocationResultsService,
// Re-enqueue targets keyed by function kind — see RerunJobQueues.
private jobQueues: RerunJobQueues,
private monitoringService: HogFunctionMonitoringService,
// Mirror of the Django serializer cap (HOG_INVOCATION_RERUN_MAX_COUNT env var).
private maxCount: number
) {}
/**
* Run one page of work for the given rerun job state. Returns the new
* state. `state.progress.done = true` means the worker should `ack()` the
* wrapper job; otherwise it should `reschedule({ state })` to continue.
*/
async processPage(teamId: number, state: RerunJobState, context: RerunJobContext): Promise<PageOutcome> {
const { function_kind, function_id, progress } = state
try {
const rows = await this.fetchPage(teamId, state)
// Stop early if the user's max_count or our hard server cap is reached.
const remainingBudget = this.remainingBudget(state)
const toProcess = rows.slice(0, remainingBudget)
// A `last_row` page, or a row written without its run's first_scheduled_at, can miss the
// newer lifecycle rows a prior replay wrote, so the collapse can report 'failed' for an invocation that has since
// succeeded. Re-check the page against each invocation's latest status across
// all partitions; without this, replaying the same window twice re-fires side
// effects (emails, webhooks) for already-recovered runs.
const staleStatusIds = await this.fetchStaleStatusInvocationIds(
teamId,
function_kind,
function_id,
toProcess.map((row) => row.invocation_id),
this.requestedStatus(state)
)
const { queued, skipped, queuedInvocations } = await this.rehydrateBatch(
teamId,
state,
toProcess,
staleStatusIds
)
let conflictSkipped = 0
if (queuedInvocations.length > 0) {
// Rerun re-uses the original invocation_id. A rerun job is
// scoped to a single function kind, so the whole page routes to
// one backend — hog → kafka, hog_flow → postgres-v2, the same
// split cdp-events-consumer uses.
let invocationsToEnqueue = queuedInvocations
if (function_kind === 'hog_flow') {
// postgres-v2. `overwriteExisting` upserts ONLY when the
// existing cyclotron row is in a terminal state. If a row
// is still active, the v2 manager raises
// CyclotronJobConflictError listing the conflicting ids —
// skip those, still queue the rest.
try {
await this.jobQueues.hog_flow.queueInvocations(invocationsToEnqueue, {
overwriteExisting: true,
})
} catch (e) {
if (!(e instanceof CyclotronJobConflictError)) {
throw e
}
const raw = e.conflictingIds
const conflictingIds = new Set(Array.isArray(raw) ? raw : [raw])
logger.warn('Rerun skipping invocations that are still in-flight', {
rerun_function_kind: function_kind,
rerun_function_id: function_id,
conflicting_invocation_ids: Array.from(conflictingIds),
})
conflictSkipped = conflictingIds.size
for (let i = 0; i < conflictSkipped; i++) {
counterRerunInvocationsSkipped.labels(function_kind, 'still_in_flight').inc()
}
// The conflicting invocations also queued a 'running'
// lifecycle row above — drop them so we don't show a
// stale running row for an invocation that didn't
// actually re-enqueue.
invocationsToEnqueue = queuedInvocations.filter((i) => !conflictingIds.has(i.id))
this.invocationResultsRowsService.dropQueuedRowsFor(Array.from(conflictingIds))
}
} else {
// kafka. There's no PK to reject a duplicate the way the
// postgres-v2 path does, so re-enqueuing an invocation that
// is still running would fire its side effects (webhook,
// email) a second time. Skip any whose latest CH lifecycle
// row is still 'running'. The 'running' rows we queued for
// this page aren't flushed yet, so this reflects prior state.
const runningIds = await this.fetchRunningInvocationIds(
teamId,
function_kind,
function_id,
invocationsToEnqueue.map((i) => i.id)
)
if (runningIds.size > 0) {
logger.warn('Rerun skipping hog invocations that are still in-flight', {
rerun_function_kind: function_kind,
rerun_function_id: function_id,
conflicting_invocation_ids: Array.from(runningIds),
})
conflictSkipped = runningIds.size
for (let i = 0; i < conflictSkipped; i++) {
counterRerunInvocationsSkipped.labels(function_kind, 'still_in_flight_hog').inc()
}
invocationsToEnqueue = queuedInvocations.filter((i) => !runningIds.has(i.id))
this.invocationResultsRowsService.dropQueuedRowsFor(Array.from(runningIds))
}
if (invocationsToEnqueue.length > 0) {
await this.jobQueues.hog_function.queueInvocations(invocationsToEnqueue)
}
}
await this.invocationResultsRowsService.flush()
counterRerunInvocationsQueued.labels(function_kind).inc(invocationsToEnqueue.length)
// One debug log per re-enqueued invocation, keyed on its own
// `instance_id` so the line shows up on that invocation's
// expanded log panel — not on the wrapper's. Makes it obvious
// why the run shows up twice when scanning a function's logs.
const now = DateTime.now()
this.monitoringService.queueLogs(
invocationsToEnqueue.map((inv) => ({
team_id: teamId,
log_source: function_kind,
log_source_id: function_id,
instance_id: inv.id,
timestamp: now,
level: 'debug',
message: `Re-queued by re-run job ${context.jobId}.`,
})),
function_kind
)
}
const nextProgress: RerunJobProgress = {
queued: progress.queued + queued - conflictSkipped,
skipped: progress.skipped + skipped + conflictSkipped,
cursor: this.advanceCursor(state, toProcess),
done: this.isDone(state, rows.length, toProcess.length),
last_error: undefined,
pages_processed: (progress.pages_processed ?? 0) + 1,
}
counterRerunPageProcessed.labels(function_kind, rows.length === 0 ? 'empty' : 'ok').inc()
// Update the wrapper lifecycle row + emit a progress log so the
// Invocations tab reflects the running total without the user
// hitting Refresh between pages.
//
// Best-effort: the page's invocations are already committed to kafka
// or postgres-v2 by this point, so letting a lifecycle-write failure
// reach the catch below would count a page that did its work as a
// page error, hold the cursor back, and on a persistent write outage
// eventually mark a rerun failed that actually ran.
await this.writeWrapperUpdate(teamId, state, context, nextProgress, undefined).catch((writeErr) => {
logger.warn('Rerun paginator could not write the wrapper progress row', {
rerun_function_kind: function_kind,
rerun_function_id: function_id,
error: writeErr instanceof Error ? writeErr.message : String(writeErr),
})
})
return { state: { ...state, progress: nextProgress } }
} catch (err) {
const errMessage = err instanceof Error ? err.message : String(err)
logger.error('Rerun paginator page failed', {
rerun_function_kind: function_kind,
rerun_function_id: function_id,
error: errMessage,
})
counterRerunPageProcessed.labels(function_kind, 'error').inc()
// Give up once a page has errored too many times in a row. Rethrow
// so the worker's handler fails the wrapper job terminally (and
// records a replayable failure row) rather than rescheduling. An
// unbounded retry loop would bump the SMALLINT `transition_count` on
// every reschedule until it overflows at 32767 — poisoning the row
// so even dequeue fails with `smallint out of range`.
const consecutiveErrors = (progress.consecutive_errors ?? 0) + 1
if (consecutiveErrors >= RERUN_MAX_CONSECUTIVE_PAGE_ERRORS) {
throw err
}
// Below the cap: surface the error in job state but don't mark done,
// so the worker reschedules and retries the same page. Track the
// consecutive-error streak — a page that later succeeds resets it.
const errorProgress = { ...progress, last_error: errMessage, consecutive_errors: consecutiveErrors }
await this.writeWrapperUpdate(teamId, state, context, errorProgress, err)
return { state: { ...state, progress: errorProgress } }
}
}
/**
* Write a wrapper lifecycle row + log line reflecting the result of one
* page. Status is `'running'` for in-flight pages, `'succeeded'` /
* `'failed'` for the final page. Errors flow into the lifecycle row's
* `error_kind` / `error_message` for failed terminal writes only — a
* recoverable per-page error gets logged but the row stays `running` so
* the worker can reschedule and try again.
*
* Public so the worker can write a terminal `failed` row from its catch
* handler when the whole wrapper is being given up on.
*/
async writeWrapperUpdate(
teamId: number,
state: RerunJobState,
context: RerunJobContext,
nextProgress: RerunJobProgress,
pageError: unknown | undefined
): Promise<void> {
const status: 'running' | 'succeeded' | 'failed' = nextProgress.done ? 'succeeded' : 'running'
const now = new Date()
this.invocationResultsRowsService.queueRerunWrapperRow({
teamId,
parentFunctionKind: state.function_kind,
functionId: state.function_id,
rerunJobId: context.jobId,
status,
pagesProcessed: nextProgress.pages_processed ?? 0,
filter: state.request.filter,
scheduledAt: context.createdAt.toJSDate(),
startedAt: context.createdAt.toJSDate(),
finishedAt: status !== 'running' ? now : undefined,
error: pageError,
})
const pageErrorMessage = pageError ? (pageError instanceof Error ? pageError.message : String(pageError)) : null
const message = nextProgress.done
? `Re-run finished. queued=${nextProgress.queued} skipped=${nextProgress.skipped}`
: pageErrorMessage
? `Re-run page failed: ${pageErrorMessage}. Worker will retry.`
: `Re-run page done. queued=${nextProgress.queued} skipped=${nextProgress.skipped} cursor=${
nextProgress.cursor
? `${nextProgress.cursor.scheduled_at}/${nextProgress.cursor.invocation_id}`
: 'end'
}`
this.monitoringService.queueLogs(
[
{
team_id: teamId,
log_source: state.function_kind,
log_source_id: state.function_id,
instance_id: context.jobId,
timestamp: DateTime.fromJSDate(now),
level: pageErrorMessage ? 'warn' : nextProgress.done ? 'info' : 'info',
message,
},
],
state.function_kind
)
await Promise.all([this.invocationResultsRowsService.flush(), this.monitoringService.flush()])
}
/**
* Write the terminal `failed` wrapper lifecycle row from the worker's
* unrecoverable catch path. Logs the cause and flushes so the failure is
* visible immediately on the Invocations tab.
*/
async writeWrapperFailure(
teamId: number,
state: RerunJobState,
context: RerunJobContext,
error: unknown
): Promise<void> {
const errMessage = error instanceof Error ? error.message : String(error)
const now = new Date()
this.invocationResultsRowsService.queueRerunWrapperRow({
teamId,
parentFunctionKind: state.function_kind,
functionId: state.function_id,
rerunJobId: context.jobId,
status: 'failed',
pagesProcessed: state.progress.pages_processed ?? 0,
filter: state.request.filter,
scheduledAt: context.createdAt.toJSDate(),
startedAt: context.createdAt.toJSDate(),
finishedAt: now,
error,
})
this.monitoringService.queueLogs(
[
{
team_id: teamId,
log_source: state.function_kind,
log_source_id: state.function_id,
instance_id: context.jobId,
timestamp: DateTime.fromJSDate(now),
level: 'error',
message: `Re-run aborted: ${errMessage}`,
},
],
state.function_kind
)
await Promise.all([this.invocationResultsRowsService.flush(), this.monitoringService.flush()])
}
private remainingBudget(state: RerunJobState): number {
const cap = Math.min(state.request.filter.max_count ?? this.maxCount, this.maxCount)
return Math.max(0, cap - state.progress.queued - state.progress.skipped)
}
private isDone(state: RerunJobState, fetchedCount: number, processedCount: number): boolean {
const budgetSpent = this.remainingBudget(state) <= processedCount
const pageWasPartial = fetchedCount < RERUN_PAGE_SIZE
return budgetSpent || pageWasPartial
}
private advanceCursor(state: RerunJobState, processed: InvocationRow[]): RerunJobState['progress']['cursor'] {
if (processed.length === 0) {
return null
}
const last = processed[processed.length - 1]
return { scheduled_at: pagingFor(state).cursorTime(last), invocation_id: last.invocation_id }
}
/**
* Of the given hog-function invocation ids, return those whose latest
* lifecycle row is still `running`. The query keys on the table's primary
* prefix (team_id, function_kind, function_id, invocation_id) so it stays
* cheap without a partition bound. Used to avoid re-enqueuing an in-flight
* invocation on the kafka path, which has no conflict guard of its own.
*/
private async fetchRunningInvocationIds(
teamId: number,
functionKind: RerunFunctionKind,
functionId: string,
invocationIds: string[]
): Promise<Set<string>> {
if (invocationIds.length === 0) {
return new Set()
}
const result = await this.clickhouse.query({
query: `/* team_id:${teamId} query_type:hog_invocation_rerun_inflight */
SELECT invocation_id
FROM hog_invocation_results
WHERE team_id = {team_id:Int64}
AND function_kind = {function_kind:String}
AND function_id = {function_id:String}
AND invocation_id IN {invocation_ids:Array(String)}
GROUP BY invocation_id
HAVING argMax(is_deleted, version) = 0
AND argMax(status, version) = 'running'`,
query_params: {
team_id: teamId,
function_kind: functionKind,
function_id: functionId,
invocation_ids: invocationIds,
},
format: 'JSONEachRow',
})
const rows = (await result.json()) as { invocation_id: string }[]
return new Set(rows.map((r) => r.invocation_id))
}
private requestedStatus(state: RerunJobState): string[] {
const status = state.request.filter.status
return status?.length ? status : ['failed']
}
/**
* Of the given invocation ids, return those whose latest lifecycle row —
* across ALL partitions, not just the request window — no longer matches
* the requested statuses (or is deleted). Same primary-prefix keying as
* `fetchRunningInvocationIds`, so it stays cheap without a partition bound.
*/
private async fetchStaleStatusInvocationIds(
teamId: number,
functionKind: RerunFunctionKind,
functionId: string,
invocationIds: string[],
requestedStatus: string[]
): Promise<Set<string>> {
if (invocationIds.length === 0) {
return new Set()
}
const result = await this.clickhouse.query({
query: `/* team_id:${teamId} query_type:hog_invocation_rerun_stale_status */
SELECT invocation_id
FROM hog_invocation_results
WHERE team_id = {team_id:Int64}
AND function_kind = {function_kind:String}
AND function_id = {function_id:String}
AND invocation_id IN {invocation_ids:Array(String)}
GROUP BY invocation_id
HAVING argMax(is_deleted, version) = 1
OR argMax(status, version) NOT IN {requested_status:Array(String)}`,
query_params: {
team_id: teamId,
function_kind: functionKind,
function_id: functionId,
invocation_ids: invocationIds,
requested_status: requestedStatus,
},
format: 'JSONEachRow',
})
const rows = (await result.json()) as { invocation_id: string }[]
return new Set(rows.map((r) => r.invocation_id))
}
private async fetchPage(teamId: number, state: RerunJobState): Promise<InvocationRow[]> {
const filter = state.request.filter
const requestedStatus = this.requestedStatus(state)
// The Django serializer accepts ISO 8601 ('2026-05-01T00:00:00Z'), but
// ClickHouse `DateTime64` only parses 'YYYY-MM-DD HH:MM:SS[.fff]'. Convert
// before passing as a query parameter. The bound params below are typed
// `DateTime64(6, 'UTC')` so the (tz-stripped) UTC wall-clock string is
// always interpreted as UTC — an untyped `DateTime64` would be parsed in
// the CH server timezone, shifting the window on non-UTC servers.
const windowStart = toClickhouseDateTime(filter.window_start)
const windowEnd = toClickhouseDateTime(filter.window_end)
const cursorScheduledAt = state.progress.cursor?.scheduled_at
? toClickhouseDateTime(state.progress.cursor.scheduled_at)
: ''
const { keyColumn, orderColumn } = pagingFor(state)
// Keyset pagination on (key, invocation_id). When the cursor
// is undefined we start from the top of the window. ClickHouse needs
// the tuple compared via a single AND-chained inequality.
const cursor = state.progress.cursor
const cursorClause =
cursor && cursor.scheduled_at
? ` AND (${keyColumn}, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})`
: ''
const errorKindClause = filter.error_kind?.length
? 'AND argMax(error_kind, version) IN {error_kind:Array(String)}'
: ''
// position() instead of LIKE so the needle requires no %/_ escaping. Case-insensitive so a
// needle copied from logs matches regardless of capitalization.
const errorMessageClause = filter.error_message_contains
? 'AND positionCaseInsensitive(argMax(error_message, version), {error_message_contains:String}) > 0'
: ''
const maxAttemptsClause =
filter.max_attempts !== undefined ? 'AND argMax(attempts, version) < {max_attempts:UInt8}' : ''
// `invocation_ids` is an OPTIONAL additional restriction layered on top
// of the time window, which is still required.
const invocationIdsClause = filter.invocation_ids?.length
? 'AND invocation_id IN {invocation_ids:Array(String)}'
: ''
const result = await this.clickhouse.query({
query: `/* team_id:${teamId} query_type:hog_invocation_rerun_page */
SELECT
invocation_id,
argMax(parent_run_id, version) AS parent_run_id,
-- NOT aliased 'attempts': that would shadow the raw column, and the
-- max_attempts HAVING below (argMax(attempts, version) < …) would then
-- resolve 'attempts' to this alias — an aggregate inside an aggregate,
-- which ClickHouse rejects. Any caller setting max_attempts (the poison-
-- pill autodrain always does) would fail every page. Keep the names distinct.
argMax(attempts, version) AS latest_attempts,
argMax(invocation_globals, version) AS invocation_globals,
-- Not aliased 'first_scheduled_at': the alias would replace the raw column in the WHERE.
argMax(first_scheduled_at, version) AS run_first_scheduled_at,
max(scheduled_at) AS last_scheduled_at
FROM hog_invocation_results
WHERE team_id = {team_id:Int64}
AND function_kind = {function_kind:String}
AND function_id = {function_id:String}
AND ${keyColumn} >= {window_start:DateTime64(6, 'UTC')}
AND ${keyColumn} < {window_end:DateTime64(6, 'UTC')}
${invocationIdsClause}
${cursorClause}
GROUP BY invocation_id
HAVING argMax(is_deleted, version) = 0
AND argMax(status, version) IN {status:Array(String)}
${errorKindClause}
${errorMessageClause}
${maxAttemptsClause}
ORDER BY ${orderColumn} DESC, invocation_id DESC
LIMIT {limit:UInt32}`,
query_params: {
team_id: teamId,
function_kind: state.function_kind,
function_id: state.function_id,
window_start: windowStart,
window_end: windowEnd,
status: requestedStatus,
error_kind: filter.error_kind ?? [],
error_message_contains: filter.error_message_contains ?? '',
max_attempts: filter.max_attempts ?? 255,
invocation_ids: filter.invocation_ids ?? [],
cursor_scheduled_at: cursorScheduledAt,
cursor_invocation_id: cursor?.invocation_id ?? '',
limit: RERUN_PAGE_SIZE,
},
format: 'JSONEachRow',
})
return (await result.json()) as InvocationRow[]
}
private async rehydrateBatch(
teamId: number,
state: RerunJobState,
rows: InvocationRow[],
staleStatusIds: Set<string>
): Promise<{ queued: number; skipped: number; queuedInvocations: CyclotronJobInvocation[] }> {
const maxAttempts = state.request.filter.max_attempts
// Rehydrate the whole page concurrently — `addGroupsToGlobals` and the
// hog function manager are LazyLoader-backed and batch their DB lookups
// across concurrent callers, so a sequential loop would defeat that.
const rehydrated = await Promise.all(
rows.map(async (row): Promise<CyclotronJobInvocation | null> => {
// Stale rows stay in the page (the cursor advances over them) and count
// as skipped, so pagination never re-fetches them.
if (staleStatusIds.has(row.invocation_id)) {
counterRerunInvocationsSkipped.labels(state.function_kind, 'stale_status').inc()
return null
}
if (maxAttempts !== undefined && row.latest_attempts >= maxAttempts) {
counterRerunInvocationsSkipped.labels(state.function_kind, 'over_max_attempts').inc()
return null
}
try {
const invocation = await this.rehydrateInvocation(
teamId,
state.function_kind,
state.function_id,
row,
state.request.replay_code_sends === true
)
if (!invocation) {
counterRerunInvocationsSkipped.labels(state.function_kind, 'rehydrate_failed').inc()
}
return invocation
} catch (e) {
logger.error('Rerun failed to rehydrate invocation', {
error: e instanceof Error ? e.message : String(e),
invocation_id: row.invocation_id,
})
counterRerunInvocationsSkipped.labels(state.function_kind, 'exception').inc()
return null
}
})
)
const queuedInvocations: CyclotronJobInvocation[] = []
for (const invocation of rehydrated) {
if (!invocation) {
continue
}
// Rerun-start lifecycle row. is_retry/attempts are derived from
// `state.rerunAttempts` (set by rehydrateInvocation). The matching
// terminal row is written by the worker when the invocation
// finishes — same derivation, same is_retry=1.
this.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
queuedInvocations.push(invocation)
}
return {
queued: queuedInvocations.length,
skipped: rows.length - queuedInvocations.length,
queuedInvocations,
}
}
private async rehydrateInvocation(
teamId: number,
functionKind: RerunFunctionKind,
functionId: string,
row: InvocationRow,
replayCodeSends: boolean
): Promise<CyclotronJobInvocation | null> {
let parsedGlobals: unknown
try {
parsedGlobals = await decodeInvocationGlobals(row.invocation_globals)
} catch {
return null
}
if (functionKind === 'hog_function') {
const hogFunction = await this.hogFunctionManager.getHogFunction(functionId)
if (!hogFunction || hogFunction.team_id !== teamId) {
return null
}
// Only re-enqueue types a cyclotron worker executes. Others (source webhooks,
// transformations, site_*) run elsewhere and would never drain from the hog
// queue, so a re-enqueued invocation wedges the partition. The API rejects these
// up front; this is the backstop for any rerun job that reaches the worker anyway.
if (!RERUNNABLE_HOG_FUNCTION_TYPES.has(hogFunction.type)) {
logger.warn('⚠️', 'Skipping rerun of invocation with non-rerunnable function type', {
functionId,
teamId,
type: hogFunction.type,
invocation_id: row.invocation_id,
})
return null
}
// The persisted globals are minimal — `inputs`, `groups` and
// `person` are all stripped. Re-enqueue as-is: the cyclotron worker
// rehydrates `groups`/`person` and the executor rebuilds `inputs`
// from the current hog function config, so the rerun runs against
// the latest config/secrets rather than a stored snapshot.
//
// Validate the shape before trusting it: a snapshot written by an
// older serializer (or otherwise drifted) can be missing
// project/event, which the worker dereferences unguarded — skip the
// row rather than re-enqueue a poison pill.
const parsed = HogFunctionInvocationGlobalsSchema.safeParse(parsedGlobals)
if (!parsed.success) {
logger.warn('⚠️', 'Skipping rerun of invocation with malformed persisted globals', {
functionId,
teamId,
invocation_id: row.invocation_id,
issues: parsed.error.issues.map((issue) => issue.path.join('.')),
})
return null
}
const persistedGlobals = parsedGlobals as HogFunctionInvocationGlobalsWithInputs
const invocation: CyclotronJobInvocationHogFunction = {
// Preserve invocation_id so lifecycle rows collapse under the
// ReplacingMergeTree on the same key.
id: row.invocation_id,
state: {
globals: persistedGlobals,
timings: [],
// `attempts` is the fetch-retry counter and is reset to 0
// for the rerun run. `rerunAttempts` (read from the
// stored row's `attempts` column, which holds the prior
// rerun count) drives `is_retry` and `attempts` on the
// lifecycle rows.
attempts: 0,
rerunAttempts: (row.latest_attempts || 0) + 1,
// Carry the original first-scheduled time forward — the
// producer writes this verbatim on every retry's lifecycle
// rows so ReplacingMergeTree doesn't collapse it away.
firstScheduledAt: row.run_first_scheduled_at,
},
teamId,
functionId,
parentRunId: row.parent_run_id || null,
hogFunction,
queue: 'hog',
queuePriority: 0,
}
return invocation
}
if (functionKind === 'hog_flow') {
const hogFlow = await this.hogFlowManager.getHogFlow(functionId)
if (!hogFlow || hogFlow.team_id !== teamId) {
return null
}
const persistedState = parsedGlobals as Record<string, any>
if (persistedState[SENSITIVE_VARIABLES_OMITTED]) {
logger.warn('⚠️', 'Skipping rerun of a sensitive message whose data was not stored', {
functionId,
teamId,
invocation_id: row.invocation_id,
})
this.monitoringService.queueLogs(
[
{
team_id: teamId,
log_source: 'hog_flow',
log_source_id: functionId,
instance_id: row.invocation_id,
timestamp: DateTime.now(),
level: 'warn',
message:
"Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",
},
],
'hog_flow'
)
return null
}
if (persistedState.send && !replayCodeSends) {
logger.warn('⚠️', 'Skipping rerun of an email sent from code while code sends are off', {
functionId,
teamId,
invocation_id: row.invocation_id,
})
return null
}
const eventForFilter = persistedState.event ?? {}
const filterGlobals: HogFunctionFilterGlobals = convertToHogFunctionFilterGlobal({
event: eventForFilter,
person: undefined,
groups: {},
variables: persistedState.variables ?? {},
} as any)
const invocation: CyclotronJobInvocationHogFlow = createHogFlowInvocation(
{
project: { id: teamId, name: '', url: '' },
event: eventForFilter,
person: undefined,
groups: {},
variables: persistedState.variables ?? {},
source: { name: '', url: '' },
} as any,
hogFlow,
filterGlobals
)
// The parked per-action hog function state is not replayable:
// `stripInputs` removed its `globals.inputs` (resolved secrets)
// before the row was written, and `buildHogFunctionInvocation`
// reuses a present `hogFunctionState` verbatim instead of
// re-rendering inputs — a message action resumed with it fails
// with "No recipient identifier found". Dropping it makes the
// action re-enter fresh, so inputs re-render against the current
// config, which is what a rerun intends anyway.
// The cohort lookup counters are dropped too. A run that aborted after its last
// lookup attempt would otherwise restart with no retries left, and the leftover
// cohortLookupFailed flag would turn an unrelated error on that step into an abort.
const {
hogFunctionState: _stripped,
awaitingResume: _awaitingResume,
resumeResult: _resumeResult,
cohortLookupAttempts: _cohortLookupAttempts,
cohortLookupFailed: _cohortLookupFailed,
...restoredCurrentAction
} = persistedState.currentAction ?? {}
invocation.id = row.invocation_id
invocation.parentRunId = row.parent_run_id || null
invocation.state = {
...invocation.state!,
event: eventForFilter,
actionStepCount: persistedState.actionStepCount ?? 0,
// Absence must override the new-run default so legacy replays keep their original task keys.
customerTaskIdempotencyVersion: persistedState.customerTaskIdempotencyVersion,
variables: persistedState.variables ?? {},
// Restore where the flow had progressed to. The executor resumes
// from `currentAction` (via ensureCurrentAction); without it the
// flow restarts from the trigger and re-runs already-completed
// actions — re-sending emails that already went out. Carrying it
// forward makes replay of a partially-run flow (e.g. a poisoned
// wait_until_condition the janitor gave up on) resume safely.
// `personId` is carried for the same reason — the worker needs it
// to reload person data for batch/manual triggers on resume.
currentAction: persistedState.currentAction
? (restoredCurrentAction as CyclotronJobInvocationHogFlow['state']['currentAction'])
: undefined,
personId: persistedState.personId,
// Later message steps check these before the per-worker preferences cache, which can predate them.
messageOptOuts: persistedState.messageOptOuts,
send: persistedState.send,
// Sticky rerun counter — mirror the hog function path so the
// lifecycle row producer can derive `attempts` / `is_retry`
// for flows too, and the `max_attempts` guard actually trips.
rerunAttempts: (row.latest_attempts || 0) + 1,
firstScheduledAt: row.run_first_scheduled_at,
}
return invocation
}
return null
}
}