Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

42.4 KB
Raw Blame
import { ClickHouseClient } from '@clickhouse/client'
import { DateTime } from 'luxon'
import { Counter } from 'prom-client'

import { logger } from '~/common/utils/logger'

import { HogFunctionInvocationGlobalsSchema } from '../schema/cyclotron'
import { CyclotronJobConflictError } from '../services/cyclotron-v2'
import { createHogFlowInvocation } from '../services/hogflows/hogflow-executor.service'
import { HogFlowManagerService } from '../services/hogflows/hogflow-manager.service'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import {
    HogInvocationResultsService,
    SENSITIVE_VARIABLES_OMITTED,
    decodeInvocationGlobals,
} from '../services/monitoring/hog-invocation-results.service'
import {
    CyclotronJobInvocation,
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationHogFunction,
    HogFunctionFilterGlobals,
    HogFunctionInvocationGlobalsWithInputs,
    RERUNNABLE_HOG_FUNCTION_TYPES,
} from '../types'
import { convertToHogFunctionFilterGlobal } from '../utils/hog-function-filtering'
import {
    RERUN_MAX_CONSECUTIVE_PAGE_ERRORS,
    RERUN_PAGE_SIZE,
    RerunFunctionKind,
    RerunJobProgress,
    RerunJobState,
    RerunPagingMode,
} from './rerun-job.types'

const counterRerunPageProcessed = new Counter({
    name: 'cdp_hog_invocation_rerun_pages_processed_total',
    help: 'Rerun paginator pages processed, by function kind and outcome.',
    labelNames: ['function_kind', 'outcome'],
})

const counterRerunInvocationsQueued = new Counter({
    name: 'cdp_hog_invocation_rerun_queued_total',
    help: 'Reruned invocations queued onto cyclotron, by function kind.',
    labelNames: ['function_kind'],
})

const counterRerunInvocationsSkipped = new Counter({
    name: 'cdp_hog_invocation_rerun_skipped_total',
    help: 'Invocations matched by rerun filters but skipped (over max_attempts, missing function, malformed payload).',
    labelNames: ['function_kind', 'reason'],
})

// ClickHouse's `DateTime64` parser only accepts 'YYYY-MM-DD HH:MM:SS[.fff]'.
// The Django serializer (and our internal `scheduled_at` column representation)
// uses ISO 8601 with `T` and `Z`. Convert before binding to query params.
const toClickhouseDateTime = (value: string): string => {
    if (!value) {
        return value
    }
    // Already in CH form? Leave it alone.
    if (!value.includes('T')) {
        return value
    }
    return value
        .replace('T', ' ')
        .replace(/Z$/, '')
        .replace(/([+-]\d{2}):?(\d{2})$/, '')
}

interface Paging {
    keyColumn: 'first_scheduled_at' | 'scheduled_at'
    orderColumn: 'run_first_scheduled_at' | 'last_scheduled_at'
    cursorTime: (row: InvocationRow) => string
}

const PAGING: Record<RerunPagingMode, Paging> = {
    // Every row of a run shares `first_scheduled_at`, so the window and cursor keep or drop whole runs.
    // There is no `scheduled_at` bound, because a rescheduled row can land before the run's start.
    run_start: {
        keyColumn: 'first_scheduled_at',
        orderColumn: 'run_first_scheduled_at',
        cursorTime: (row) => row.run_first_scheduled_at,
    },
    last_row: {
        keyColumn: 'scheduled_at',
        orderColumn: 'last_scheduled_at',
        cursorTime: (row) => row.last_scheduled_at,
    },
}

const pagingFor = (state: RerunJobState): Paging => PAGING[state.paging ?? 'last_row']

interface InvocationRow {
    invocation_id: string
    parent_run_id: string
    // Prior rerun count for this invocation (argMax over version). Named to avoid
    // shadowing the raw `attempts` column in the fetch query — see fetchPage.
    latest_attempts: number
    last_scheduled_at: string
    run_first_scheduled_at: string
    invocation_globals: string
}

export interface PageOutcome {
    /** New job state after this page. The worker writes it back via reschedule(state) — or ack()s if done. */
    state: RerunJobState
}

/**
 * Re-enqueue targets keyed by rerun function kind. Mirrors the split in
 * cdp-events-consumer — hog functions go to kafka, hog flows to postgres-v2.
 * Keying by kind (rather than two positional queue args of similar shape)
 * stops the two backends from being swapped by mistake.
 */
export interface RerunJobQueues {
    hog_function: JobQueue
    hog_flow: CyclotronJobQueuePostgresV2
}

/**
 * Context the worker passes alongside the parsed state on each page. Lets the
 * paginator stamp wrapper lifecycle rows with the right `invocation_id`
 * (= cyclotron job id) and a stable `scheduled_at` for the wrapper across pages.
 */
export interface RerunJobContext {
    jobId: string
    /** When the wrapper job was first created in cyclotron — anchor for the wrapper's `scheduled_at`. */
    createdAt: DateTime
}

/**
 * Processes a single page of work for a rerun wrapper job. Pure-ish — the
 * caller (the worker) handles the cyclotron-v2 ack/reschedule flow with the
 * returned state. Splitting the page-of-work logic out of the worker keeps
 * the cyclotron plumbing testable in isolation from the ClickHouse paging.
 */
export class RerunPaginatorService {
    constructor(
        private clickhouse: ClickHouseClient,
        private hogFunctionManager: HogFunctionManagerService,
        private hogFlowManager: HogFlowManagerService,
        private invocationResultsRowsService: HogInvocationResultsService,
        // Re-enqueue targets keyed by function kind — see RerunJobQueues.
        private jobQueues: RerunJobQueues,
        private monitoringService: HogFunctionMonitoringService,
        // Mirror of the Django serializer cap (HOG_INVOCATION_RERUN_MAX_COUNT env var).
        private maxCount: number
    ) {}

    /**
     * Run one page of work for the given rerun job state. Returns the new
     * state. `state.progress.done = true` means the worker should `ack()` the
     * wrapper job; otherwise it should `reschedule({ state })` to continue.
     */
    async processPage(teamId: number, state: RerunJobState, context: RerunJobContext): Promise<PageOutcome> {
        const { function_kind, function_id, progress } = state

        try {
            const rows = await this.fetchPage(teamId, state)

            // Stop early if the user's max_count or our hard server cap is reached.
            const remainingBudget = this.remainingBudget(state)
            const toProcess = rows.slice(0, remainingBudget)

            // A `last_row` page, or a row written without its run's first_scheduled_at, can miss the
            // newer lifecycle rows a prior replay wrote, so the collapse can report 'failed' for an invocation that has since
            // succeeded. Re-check the page against each invocation's latest status across
            // all partitions; without this, replaying the same window twice re-fires side
            // effects (emails, webhooks) for already-recovered runs.
            const staleStatusIds = await this.fetchStaleStatusInvocationIds(
                teamId,
                function_kind,
                function_id,
                toProcess.map((row) => row.invocation_id),
                this.requestedStatus(state)
            )

            const { queued, skipped, queuedInvocations } = await this.rehydrateBatch(
                teamId,
                state,
                toProcess,
                staleStatusIds
            )

            let conflictSkipped = 0
            if (queuedInvocations.length > 0) {
                // Rerun re-uses the original invocation_id. A rerun job is
                // scoped to a single function kind, so the whole page routes to
                // one backend — hog → kafka, hog_flow → postgres-v2, the same
                // split cdp-events-consumer uses.
                let invocationsToEnqueue = queuedInvocations
                if (function_kind === 'hog_flow') {
                    // postgres-v2. `overwriteExisting` upserts ONLY when the
                    // existing cyclotron row is in a terminal state. If a row
                    // is still active, the v2 manager raises
                    // CyclotronJobConflictError listing the conflicting ids —
                    // skip those, still queue the rest.
                    try {
                        await this.jobQueues.hog_flow.queueInvocations(invocationsToEnqueue, {
                            overwriteExisting: true,
                        })
                    } catch (e) {
                        if (!(e instanceof CyclotronJobConflictError)) {
                            throw e
                        }
                        const raw = e.conflictingIds
                        const conflictingIds = new Set(Array.isArray(raw) ? raw : [raw])
                        logger.warn('Rerun skipping invocations that are still in-flight', {
                            rerun_function_kind: function_kind,
                            rerun_function_id: function_id,
                            conflicting_invocation_ids: Array.from(conflictingIds),
                        })
                        conflictSkipped = conflictingIds.size
                        for (let i = 0; i < conflictSkipped; i++) {
                            counterRerunInvocationsSkipped.labels(function_kind, 'still_in_flight').inc()
                        }
                        // The conflicting invocations also queued a 'running'
                        // lifecycle row above — drop them so we don't show a
                        // stale running row for an invocation that didn't
                        // actually re-enqueue.
                        invocationsToEnqueue = queuedInvocations.filter((i) => !conflictingIds.has(i.id))
                        this.invocationResultsRowsService.dropQueuedRowsFor(Array.from(conflictingIds))
                    }
                } else {
                    // kafka. There's no PK to reject a duplicate the way the
                    // postgres-v2 path does, so re-enqueuing an invocation that
                    // is still running would fire its side effects (webhook,
                    // email) a second time. Skip any whose latest CH lifecycle
                    // row is still 'running'. The 'running' rows we queued for
                    // this page aren't flushed yet, so this reflects prior state.
                    const runningIds = await this.fetchRunningInvocationIds(
                        teamId,
                        function_kind,
                        function_id,
                        invocationsToEnqueue.map((i) => i.id)
                    )
                    if (runningIds.size > 0) {
                        logger.warn('Rerun skipping hog invocations that are still in-flight', {
                            rerun_function_kind: function_kind,
                            rerun_function_id: function_id,
                            conflicting_invocation_ids: Array.from(runningIds),
                        })
                        conflictSkipped = runningIds.size
                        for (let i = 0; i < conflictSkipped; i++) {
                            counterRerunInvocationsSkipped.labels(function_kind, 'still_in_flight_hog').inc()
                        }
                        invocationsToEnqueue = queuedInvocations.filter((i) => !runningIds.has(i.id))
                        this.invocationResultsRowsService.dropQueuedRowsFor(Array.from(runningIds))
                    }
                    if (invocationsToEnqueue.length > 0) {
                        await this.jobQueues.hog_function.queueInvocations(invocationsToEnqueue)
                    }
                }
                await this.invocationResultsRowsService.flush()
                counterRerunInvocationsQueued.labels(function_kind).inc(invocationsToEnqueue.length)

                // One debug log per re-enqueued invocation, keyed on its own
                // `instance_id` so the line shows up on that invocation's
                // expanded log panel — not on the wrapper's. Makes it obvious
                // why the run shows up twice when scanning a function's logs.
                const now = DateTime.now()
                this.monitoringService.queueLogs(
                    invocationsToEnqueue.map((inv) => ({
                        team_id: teamId,
                        log_source: function_kind,
                        log_source_id: function_id,
                        instance_id: inv.id,
                        timestamp: now,
                        level: 'debug',
                        message: `Re-queued by re-run job ${context.jobId}.`,
                    })),
                    function_kind
                )
            }

            const nextProgress: RerunJobProgress = {
                queued: progress.queued + queued - conflictSkipped,
                skipped: progress.skipped + skipped + conflictSkipped,
                cursor: this.advanceCursor(state, toProcess),
                done: this.isDone(state, rows.length, toProcess.length),
                last_error: undefined,
                pages_processed: (progress.pages_processed ?? 0) + 1,
            }

            counterRerunPageProcessed.labels(function_kind, rows.length === 0 ? 'empty' : 'ok').inc()

            // Update the wrapper lifecycle row + emit a progress log so the
            // Invocations tab reflects the running total without the user
            // hitting Refresh between pages.
            //
            // Best-effort: the page's invocations are already committed to kafka
            // or postgres-v2 by this point, so letting a lifecycle-write failure
            // reach the catch below would count a page that did its work as a
            // page error, hold the cursor back, and on a persistent write outage
            // eventually mark a rerun failed that actually ran.
            await this.writeWrapperUpdate(teamId, state, context, nextProgress, undefined).catch((writeErr) => {
                logger.warn('Rerun paginator could not write the wrapper progress row', {
                    rerun_function_kind: function_kind,
                    rerun_function_id: function_id,
                    error: writeErr instanceof Error ? writeErr.message : String(writeErr),
                })
            })

            return { state: { ...state, progress: nextProgress } }
        } catch (err) {
            const errMessage = err instanceof Error ? err.message : String(err)
            logger.error('Rerun paginator page failed', {
                rerun_function_kind: function_kind,
                rerun_function_id: function_id,
                error: errMessage,
            })
            counterRerunPageProcessed.labels(function_kind, 'error').inc()

            // Give up once a page has errored too many times in a row. Rethrow
            // so the worker's handler fails the wrapper job terminally (and
            // records a replayable failure row) rather than rescheduling. An
            // unbounded retry loop would bump the SMALLINT `transition_count` on
            // every reschedule until it overflows at 32767 — poisoning the row
            // so even dequeue fails with `smallint out of range`.
            const consecutiveErrors = (progress.consecutive_errors ?? 0) + 1
            if (consecutiveErrors >= RERUN_MAX_CONSECUTIVE_PAGE_ERRORS) {
                throw err
            }

            // Below the cap: surface the error in job state but don't mark done,
            // so the worker reschedules and retries the same page. Track the
            // consecutive-error streak — a page that later succeeds resets it.
            const errorProgress = { ...progress, last_error: errMessage, consecutive_errors: consecutiveErrors }
            await this.writeWrapperUpdate(teamId, state, context, errorProgress, err)
            return { state: { ...state, progress: errorProgress } }
        }
    }

    /**
     * Write a wrapper lifecycle row + log line reflecting the result of one
     * page. Status is `'running'` for in-flight pages, `'succeeded'` /
     * `'failed'` for the final page. Errors flow into the lifecycle row's
     * `error_kind` / `error_message` for failed terminal writes only — a
     * recoverable per-page error gets logged but the row stays `running` so
     * the worker can reschedule and try again.
     *
     * Public so the worker can write a terminal `failed` row from its catch
     * handler when the whole wrapper is being given up on.
     */
    async writeWrapperUpdate(
        teamId: number,
        state: RerunJobState,
        context: RerunJobContext,
        nextProgress: RerunJobProgress,
        pageError: unknown | undefined
    ): Promise<void> {
        const status: 'running' | 'succeeded' | 'failed' = nextProgress.done ? 'succeeded' : 'running'
        const now = new Date()
        this.invocationResultsRowsService.queueRerunWrapperRow({
            teamId,
            parentFunctionKind: state.function_kind,
            functionId: state.function_id,
            rerunJobId: context.jobId,
            status,
            pagesProcessed: nextProgress.pages_processed ?? 0,
            filter: state.request.filter,
            scheduledAt: context.createdAt.toJSDate(),
            startedAt: context.createdAt.toJSDate(),
            finishedAt: status !== 'running' ? now : undefined,
            error: pageError,
        })

        const pageErrorMessage = pageError ? (pageError instanceof Error ? pageError.message : String(pageError)) : null
        const message = nextProgress.done
            ? `Re-run finished. queued=${nextProgress.queued} skipped=${nextProgress.skipped}`
            : pageErrorMessage
              ? `Re-run page failed: ${pageErrorMessage}. Worker will retry.`
              : `Re-run page done. queued=${nextProgress.queued} skipped=${nextProgress.skipped} cursor=${
                    nextProgress.cursor
                        ? `${nextProgress.cursor.scheduled_at}/${nextProgress.cursor.invocation_id}`
                        : 'end'
                }`

        this.monitoringService.queueLogs(
            [
                {
                    team_id: teamId,
                    log_source: state.function_kind,
                    log_source_id: state.function_id,
                    instance_id: context.jobId,
                    timestamp: DateTime.fromJSDate(now),
                    level: pageErrorMessage ? 'warn' : nextProgress.done ? 'info' : 'info',
                    message,
                },
            ],
            state.function_kind
        )

        await Promise.all([this.invocationResultsRowsService.flush(), this.monitoringService.flush()])
    }

    /**
     * Write the terminal `failed` wrapper lifecycle row from the worker's
     * unrecoverable catch path. Logs the cause and flushes so the failure is
     * visible immediately on the Invocations tab.
     */
    async writeWrapperFailure(
        teamId: number,
        state: RerunJobState,
        context: RerunJobContext,
        error: unknown
    ): Promise<void> {
        const errMessage = error instanceof Error ? error.message : String(error)
        const now = new Date()
        this.invocationResultsRowsService.queueRerunWrapperRow({
            teamId,
            parentFunctionKind: state.function_kind,
            functionId: state.function_id,
            rerunJobId: context.jobId,
            status: 'failed',
            pagesProcessed: state.progress.pages_processed ?? 0,
            filter: state.request.filter,
            scheduledAt: context.createdAt.toJSDate(),
            startedAt: context.createdAt.toJSDate(),
            finishedAt: now,
            error,
        })

        this.monitoringService.queueLogs(
            [
                {
                    team_id: teamId,
                    log_source: state.function_kind,
                    log_source_id: state.function_id,
                    instance_id: context.jobId,
                    timestamp: DateTime.fromJSDate(now),
                    level: 'error',
                    message: `Re-run aborted: ${errMessage}`,
                },
            ],
            state.function_kind
        )

        await Promise.all([this.invocationResultsRowsService.flush(), this.monitoringService.flush()])
    }

    private remainingBudget(state: RerunJobState): number {
        const cap = Math.min(state.request.filter.max_count ?? this.maxCount, this.maxCount)
        return Math.max(0, cap - state.progress.queued - state.progress.skipped)
    }

    private isDone(state: RerunJobState, fetchedCount: number, processedCount: number): boolean {
        const budgetSpent = this.remainingBudget(state) <= processedCount
        const pageWasPartial = fetchedCount < RERUN_PAGE_SIZE
        return budgetSpent || pageWasPartial
    }

    private advanceCursor(state: RerunJobState, processed: InvocationRow[]): RerunJobState['progress']['cursor'] {
        if (processed.length === 0) {
            return null
        }
        const last = processed[processed.length - 1]
        return { scheduled_at: pagingFor(state).cursorTime(last), invocation_id: last.invocation_id }
    }

    /**
     * Of the given hog-function invocation ids, return those whose latest
     * lifecycle row is still `running`. The query keys on the table's primary
     * prefix (team_id, function_kind, function_id, invocation_id) so it stays
     * cheap without a partition bound. Used to avoid re-enqueuing an in-flight
     * invocation on the kafka path, which has no conflict guard of its own.
     */
    private async fetchRunningInvocationIds(
        teamId: number,
        functionKind: RerunFunctionKind,
        functionId: string,
        invocationIds: string[]
    ): Promise<Set<string>> {
        if (invocationIds.length === 0) {
            return new Set()
        }
        const result = await this.clickhouse.query({
            query: `/* team_id:${teamId} query_type:hog_invocation_rerun_inflight */
                SELECT invocation_id
                FROM hog_invocation_results
                WHERE team_id = {team_id:Int64}
                  AND function_kind = {function_kind:String}
                  AND function_id = {function_id:String}
                  AND invocation_id IN {invocation_ids:Array(String)}
                GROUP BY invocation_id
                HAVING argMax(is_deleted, version) = 0
                   AND argMax(status, version) = 'running'`,
            query_params: {
                team_id: teamId,
                function_kind: functionKind,
                function_id: functionId,
                invocation_ids: invocationIds,
            },
            format: 'JSONEachRow',
        })
        const rows = (await result.json()) as { invocation_id: string }[]
        return new Set(rows.map((r) => r.invocation_id))
    }

    private requestedStatus(state: RerunJobState): string[] {
        const status = state.request.filter.status
        return status?.length ? status : ['failed']
    }

    /**
     * Of the given invocation ids, return those whose latest lifecycle row —
     * across ALL partitions, not just the request window — no longer matches
     * the requested statuses (or is deleted). Same primary-prefix keying as
     * `fetchRunningInvocationIds`, so it stays cheap without a partition bound.
     */
    private async fetchStaleStatusInvocationIds(
        teamId: number,
        functionKind: RerunFunctionKind,
        functionId: string,
        invocationIds: string[],
        requestedStatus: string[]
    ): Promise<Set<string>> {
        if (invocationIds.length === 0) {
            return new Set()
        }
        const result = await this.clickhouse.query({
            query: `/* team_id:${teamId} query_type:hog_invocation_rerun_stale_status */
                SELECT invocation_id
                FROM hog_invocation_results
                WHERE team_id = {team_id:Int64}
                  AND function_kind = {function_kind:String}
                  AND function_id = {function_id:String}
                  AND invocation_id IN {invocation_ids:Array(String)}
                GROUP BY invocation_id
                HAVING argMax(is_deleted, version) = 1
                    OR argMax(status, version) NOT IN {requested_status:Array(String)}`,
            query_params: {
                team_id: teamId,
                function_kind: functionKind,
                function_id: functionId,
                invocation_ids: invocationIds,
                requested_status: requestedStatus,
            },
            format: 'JSONEachRow',
        })
        const rows = (await result.json()) as { invocation_id: string }[]
        return new Set(rows.map((r) => r.invocation_id))
    }

    private async fetchPage(teamId: number, state: RerunJobState): Promise<InvocationRow[]> {
        const filter = state.request.filter
        const requestedStatus = this.requestedStatus(state)
        // The Django serializer accepts ISO 8601 ('2026-05-01T00:00:00Z'), but
        // ClickHouse `DateTime64` only parses 'YYYY-MM-DD HH:MM:SS[.fff]'. Convert
        // before passing as a query parameter. The bound params below are typed
        // `DateTime64(6, 'UTC')` so the (tz-stripped) UTC wall-clock string is
        // always interpreted as UTC — an untyped `DateTime64` would be parsed in
        // the CH server timezone, shifting the window on non-UTC servers.
        const windowStart = toClickhouseDateTime(filter.window_start)
        const windowEnd = toClickhouseDateTime(filter.window_end)
        const cursorScheduledAt = state.progress.cursor?.scheduled_at
            ? toClickhouseDateTime(state.progress.cursor.scheduled_at)
            : ''

        const { keyColumn, orderColumn } = pagingFor(state)

        // Keyset pagination on (key, invocation_id). When the cursor
        // is undefined we start from the top of the window. ClickHouse needs
        // the tuple compared via a single AND-chained inequality.
        const cursor = state.progress.cursor
        const cursorClause =
            cursor && cursor.scheduled_at
                ? `   AND (${keyColumn}, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})`
                : ''
        const errorKindClause = filter.error_kind?.length
            ? 'AND argMax(error_kind, version) IN {error_kind:Array(String)}'
            : ''
        // position() instead of LIKE so the needle requires no %/_ escaping. Case-insensitive so a
        // needle copied from logs matches regardless of capitalization.
        const errorMessageClause = filter.error_message_contains
            ? 'AND positionCaseInsensitive(argMax(error_message, version), {error_message_contains:String}) > 0'
            : ''
        const maxAttemptsClause =
            filter.max_attempts !== undefined ? 'AND argMax(attempts, version) < {max_attempts:UInt8}' : ''
        // `invocation_ids` is an OPTIONAL additional restriction layered on top
        // of the time window, which is still required.
        const invocationIdsClause = filter.invocation_ids?.length
            ? 'AND invocation_id IN {invocation_ids:Array(String)}'
            : ''

        const result = await this.clickhouse.query({
            query: `/* team_id:${teamId} query_type:hog_invocation_rerun_page */
                SELECT
                    invocation_id,
                    argMax(parent_run_id, version)         AS parent_run_id,
                    -- NOT aliased 'attempts': that would shadow the raw column, and the
                    -- max_attempts HAVING below (argMax(attempts, version) < …) would then
                    -- resolve 'attempts' to this alias — an aggregate inside an aggregate,
                    -- which ClickHouse rejects. Any caller setting max_attempts (the poison-
                    -- pill autodrain always does) would fail every page. Keep the names distinct.
                    argMax(attempts, version)              AS latest_attempts,
                    argMax(invocation_globals, version)    AS invocation_globals,
                    -- Not aliased 'first_scheduled_at': the alias would replace the raw column in the WHERE.
                    argMax(first_scheduled_at, version)    AS run_first_scheduled_at,
                    max(scheduled_at)                      AS last_scheduled_at
                FROM hog_invocation_results
                WHERE team_id = {team_id:Int64}
                  AND function_kind = {function_kind:String}
                  AND function_id = {function_id:String}
                  AND ${keyColumn} >= {window_start:DateTime64(6, 'UTC')}
                  AND ${keyColumn} <  {window_end:DateTime64(6, 'UTC')}
                  ${invocationIdsClause}
                ${cursorClause}
                GROUP BY invocation_id
                HAVING argMax(is_deleted, version) = 0
                   AND argMax(status, version) IN {status:Array(String)}
                   ${errorKindClause}
                   ${errorMessageClause}
                   ${maxAttemptsClause}
                ORDER BY ${orderColumn} DESC, invocation_id DESC
                LIMIT {limit:UInt32}`,
            query_params: {
                team_id: teamId,
                function_kind: state.function_kind,
                function_id: state.function_id,
                window_start: windowStart,
                window_end: windowEnd,
                status: requestedStatus,
                error_kind: filter.error_kind ?? [],
                error_message_contains: filter.error_message_contains ?? '',
                max_attempts: filter.max_attempts ?? 255,
                invocation_ids: filter.invocation_ids ?? [],
                cursor_scheduled_at: cursorScheduledAt,
                cursor_invocation_id: cursor?.invocation_id ?? '',
                limit: RERUN_PAGE_SIZE,
            },
            format: 'JSONEachRow',
        })

        return (await result.json()) as InvocationRow[]
    }

    private async rehydrateBatch(
        teamId: number,
        state: RerunJobState,
        rows: InvocationRow[],
        staleStatusIds: Set<string>
    ): Promise<{ queued: number; skipped: number; queuedInvocations: CyclotronJobInvocation[] }> {
        const maxAttempts = state.request.filter.max_attempts

        // Rehydrate the whole page concurrently — `addGroupsToGlobals` and the
        // hog function manager are LazyLoader-backed and batch their DB lookups
        // across concurrent callers, so a sequential loop would defeat that.
        const rehydrated = await Promise.all(
            rows.map(async (row): Promise<CyclotronJobInvocation | null> => {
                // Stale rows stay in the page (the cursor advances over them) and count
                // as skipped, so pagination never re-fetches them.
                if (staleStatusIds.has(row.invocation_id)) {
                    counterRerunInvocationsSkipped.labels(state.function_kind, 'stale_status').inc()
                    return null
                }
                if (maxAttempts !== undefined && row.latest_attempts >= maxAttempts) {
                    counterRerunInvocationsSkipped.labels(state.function_kind, 'over_max_attempts').inc()
                    return null
                }
                try {
                    const invocation = await this.rehydrateInvocation(
                        teamId,
                        state.function_kind,
                        state.function_id,
                        row,
                        state.request.replay_code_sends === true
                    )
                    if (!invocation) {
                        counterRerunInvocationsSkipped.labels(state.function_kind, 'rehydrate_failed').inc()
                    }
                    return invocation
                } catch (e) {
                    logger.error('Rerun failed to rehydrate invocation', {
                        error: e instanceof Error ? e.message : String(e),
                        invocation_id: row.invocation_id,
                    })
                    counterRerunInvocationsSkipped.labels(state.function_kind, 'exception').inc()
                    return null
                }
            })
        )

        const queuedInvocations: CyclotronJobInvocation[] = []
        for (const invocation of rehydrated) {
            if (!invocation) {
                continue
            }
            // Rerun-start lifecycle row. is_retry/attempts are derived from
            // `state.rerunAttempts` (set by rehydrateInvocation). The matching
            // terminal row is written by the worker when the invocation
            // finishes — same derivation, same is_retry=1.
            this.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
            queuedInvocations.push(invocation)
        }

        return {
            queued: queuedInvocations.length,
            skipped: rows.length - queuedInvocations.length,
            queuedInvocations,
        }
    }

    private async rehydrateInvocation(
        teamId: number,
        functionKind: RerunFunctionKind,
        functionId: string,
        row: InvocationRow,
        replayCodeSends: boolean
    ): Promise<CyclotronJobInvocation | null> {
        let parsedGlobals: unknown
        try {
            parsedGlobals = await decodeInvocationGlobals(row.invocation_globals)
        } catch {
            return null
        }

        if (functionKind === 'hog_function') {
            const hogFunction = await this.hogFunctionManager.getHogFunction(functionId)
            if (!hogFunction || hogFunction.team_id !== teamId) {
                return null
            }

            // Only re-enqueue types a cyclotron worker executes. Others (source webhooks,
            // transformations, site_*) run elsewhere and would never drain from the hog
            // queue, so a re-enqueued invocation wedges the partition. The API rejects these
            // up front; this is the backstop for any rerun job that reaches the worker anyway.
            if (!RERUNNABLE_HOG_FUNCTION_TYPES.has(hogFunction.type)) {
                logger.warn('⚠️', 'Skipping rerun of invocation with non-rerunnable function type', {
                    functionId,
                    teamId,
                    type: hogFunction.type,
                    invocation_id: row.invocation_id,
                })
                return null
            }

            // The persisted globals are minimal — `inputs`, `groups` and
            // `person` are all stripped. Re-enqueue as-is: the cyclotron worker
            // rehydrates `groups`/`person` and the executor rebuilds `inputs`
            // from the current hog function config, so the rerun runs against
            // the latest config/secrets rather than a stored snapshot.
            //
            // Validate the shape before trusting it: a snapshot written by an
            // older serializer (or otherwise drifted) can be missing
            // project/event, which the worker dereferences unguarded — skip the
            // row rather than re-enqueue a poison pill.
            const parsed = HogFunctionInvocationGlobalsSchema.safeParse(parsedGlobals)
            if (!parsed.success) {
                logger.warn('⚠️', 'Skipping rerun of invocation with malformed persisted globals', {
                    functionId,
                    teamId,
                    invocation_id: row.invocation_id,
                    issues: parsed.error.issues.map((issue) => issue.path.join('.')),
                })
                return null
            }
            const persistedGlobals = parsedGlobals as HogFunctionInvocationGlobalsWithInputs

            const invocation: CyclotronJobInvocationHogFunction = {
                // Preserve invocation_id so lifecycle rows collapse under the
                // ReplacingMergeTree on the same key.
                id: row.invocation_id,
                state: {
                    globals: persistedGlobals,
                    timings: [],
                    // `attempts` is the fetch-retry counter and is reset to 0
                    // for the rerun run. `rerunAttempts` (read from the
                    // stored row's `attempts` column, which holds the prior
                    // rerun count) drives `is_retry` and `attempts` on the
                    // lifecycle rows.
                    attempts: 0,
                    rerunAttempts: (row.latest_attempts || 0) + 1,
                    // Carry the original first-scheduled time forward — the
                    // producer writes this verbatim on every retry's lifecycle
                    // rows so ReplacingMergeTree doesn't collapse it away.
                    firstScheduledAt: row.run_first_scheduled_at,
                },
                teamId,
                functionId,
                parentRunId: row.parent_run_id || null,
                hogFunction,
                queue: 'hog',
                queuePriority: 0,
            }
            return invocation
        }

        if (functionKind === 'hog_flow') {
            const hogFlow = await this.hogFlowManager.getHogFlow(functionId)
            if (!hogFlow || hogFlow.team_id !== teamId) {
                return null
            }
            const persistedState = parsedGlobals as Record<string, any>
            if (persistedState[SENSITIVE_VARIABLES_OMITTED]) {
                logger.warn('⚠️', 'Skipping rerun of a sensitive message whose data was not stored', {
                    functionId,
                    teamId,
                    invocation_id: row.invocation_id,
                })
                this.monitoringService.queueLogs(
                    [
                        {
                            team_id: teamId,
                            log_source: 'hog_flow',
                            log_source_id: functionId,
                            instance_id: row.invocation_id,
                            timestamp: DateTime.now(),
                            level: 'warn',
                            message:
                                "Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",
                        },
                    ],
                    'hog_flow'
                )
                return null
            }
            if (persistedState.send && !replayCodeSends) {
                logger.warn('⚠️', 'Skipping rerun of an email sent from code while code sends are off', {
                    functionId,
                    teamId,
                    invocation_id: row.invocation_id,
                })
                return null
            }
            const eventForFilter = persistedState.event ?? {}
            const filterGlobals: HogFunctionFilterGlobals = convertToHogFunctionFilterGlobal({
                event: eventForFilter,
                person: undefined,
                groups: {},
                variables: persistedState.variables ?? {},
            } as any)

            const invocation: CyclotronJobInvocationHogFlow = createHogFlowInvocation(
                {
                    project: { id: teamId, name: '', url: '' },
                    event: eventForFilter,
                    person: undefined,
                    groups: {},
                    variables: persistedState.variables ?? {},
                    source: { name: '', url: '' },
                } as any,
                hogFlow,
                filterGlobals
            )
            // The parked per-action hog function state is not replayable:
            // `stripInputs` removed its `globals.inputs` (resolved secrets)
            // before the row was written, and `buildHogFunctionInvocation`
            // reuses a present `hogFunctionState` verbatim instead of
            // re-rendering inputs — a message action resumed with it fails
            // with "No recipient identifier found". Dropping it makes the
            // action re-enter fresh, so inputs re-render against the current
            // config, which is what a rerun intends anyway.
            // The cohort lookup counters are dropped too. A run that aborted after its last
            // lookup attempt would otherwise restart with no retries left, and the leftover
            // cohortLookupFailed flag would turn an unrelated error on that step into an abort.
            const {
                hogFunctionState: _stripped,
                awaitingResume: _awaitingResume,
                resumeResult: _resumeResult,
                cohortLookupAttempts: _cohortLookupAttempts,
                cohortLookupFailed: _cohortLookupFailed,
                ...restoredCurrentAction
            } = persistedState.currentAction ?? {}

            invocation.id = row.invocation_id
            invocation.parentRunId = row.parent_run_id || null
            invocation.state = {
                ...invocation.state!,
                event: eventForFilter,
                actionStepCount: persistedState.actionStepCount ?? 0,
                // Absence must override the new-run default so legacy replays keep their original task keys.
                customerTaskIdempotencyVersion: persistedState.customerTaskIdempotencyVersion,
                variables: persistedState.variables ?? {},
                // Restore where the flow had progressed to. The executor resumes
                // from `currentAction` (via ensureCurrentAction); without it the
                // flow restarts from the trigger and re-runs already-completed
                // actions — re-sending emails that already went out. Carrying it
                // forward makes replay of a partially-run flow (e.g. a poisoned
                // wait_until_condition the janitor gave up on) resume safely.
                // `personId` is carried for the same reason — the worker needs it
                // to reload person data for batch/manual triggers on resume.
                currentAction: persistedState.currentAction
                    ? (restoredCurrentAction as CyclotronJobInvocationHogFlow['state']['currentAction'])
                    : undefined,
                personId: persistedState.personId,
                // Later message steps check these before the per-worker preferences cache, which can predate them.
                messageOptOuts: persistedState.messageOptOuts,
                send: persistedState.send,
                // Sticky rerun counter — mirror the hog function path so the
                // lifecycle row producer can derive `attempts` / `is_retry`
                // for flows too, and the `max_attempts` guard actually trips.
                rerunAttempts: (row.latest_attempts || 0) + 1,
                firstScheduledAt: row.run_first_scheduled_at,
            }
            return invocation
        }

        return null
    }
}