Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

35.6 KB
Raw Blame
import { Pool } from 'pg'
import { Counter, Histogram } from 'prom-client'
import { v7 as uuidv7 } from 'uuid'

import { isTransientPgError } from '~/common/utils/db/postgres'
import { logger } from '~/common/utils/logger'
import { sleep } from '~/common/utils/utils'

import { BatchResolverResumeOutcome, resumeParkedBatchResolver } from '../hogflows/batch-resolver-resume'
import { StepResume, StepResumeOutcome, processStepResumes } from '../hogflows/step-resume.service'
import {
    CYCLOTRON_COUNTER_MAX,
    CyclotronV2CancelJobsOptions,
    CyclotronV2CancelJobsResult,
    CyclotronV2InFlightCounts,
    CyclotronV2JobInit,
    CyclotronV2JobInitSchema,
    CyclotronV2ManagerConfig,
    CyclotronV2RescheduleParkedOptions,
    CyclotronV2RescheduleParkedResult,
} from './types'

/**
 * Block size used to compute the fair-dequeue sort key for email queue jobs:
 *
 *     dequeue_seq = counter * EMAIL_DEQUEUE_BLOCK_SIZE + team_id
 *
 * 2^24 (16,777,216) — supports team_ids up to ~16M (PostHog is well under
 * that). Keeps headroom for ~5 × 10^11 jobs per team before the BIGINT
 * sort key overflows, which is centuries at any realistic email volume.
 *
 * Only used for queue_name='email'. Other queues leave dequeue_seq NULL.
 */
export const EMAIL_DEQUEUE_BLOCK_SIZE = BigInt(16_777_216)

/**
 * Atomically bump the per-team email counter and return the formatted
 * `dequeue_seq` string for one new email job. Exported because the worker's
 * `reschedule()` path also needs to assign `dequeue_seq` when a hog/hogflow
 * job is re-routed into the email queue — without this, those rows land with
 * `NULL dequeue_seq` and bypass the per-team interleave (`NULLS FIRST` would
 * drain them ahead of any fair-ordered rows).
 *
 * A team's *first ever* email starts at `MAX(counter) + 1` across the table
 * (Hatchet's `p_max_assigned` pattern), so a brand-new tenant can't enqueue a
 * burst that gets free priority over established tenants' in-flight emails.
 * Continuing tenants just increment their own counter as before. `COALESCE`
 * handles the empty-table cold-start (`MAX = NULL` → starting point of 0).
 *
 * For bulk inserts of multiple email jobs at once, `bulkCreateJobs` uses its
 * own batched UPSERT for efficiency rather than calling this in a loop.
 */
export async function assignEmailDequeueSeq(pool: Pool, teamId: number): Promise<string> {
    const result = await pool.query<{ counter: string }>(
        `INSERT INTO cyclotron_email_team_seq (team_id, counter)
         VALUES ($1, COALESCE((SELECT MAX(counter) FROM cyclotron_email_team_seq), 0) + 1)
         ON CONFLICT (team_id) DO UPDATE
            SET counter = cyclotron_email_team_seq.counter + 1
         RETURNING counter`,
        [teamId]
    )
    const counter = BigInt(result.rows[0].counter)
    // pg accepts BIGINT as string to avoid JS number precision loss
    return (counter * EMAIL_DEQUEUE_BLOCK_SIZE + BigInt(teamId)).toString()
}

// Counts Postgres write failures from createJob / bulkCreateJobs *after* input
// validation has passed. Zod parse errors and the overwrite-conflict logical
// error do not increment this counter. The `kind` label splits failures into:
//   - "logical": schema drift, constraint violation, anything that won't fix
//     itself. Any non-zero rate is page-worthy.
//   - "transient": PG / pgbouncer connection issues (matched against
//     POSTGRES_UNAVAILABLE_ERROR_MESSAGES). Brief blips are noise; sustained
//     rate indicates the database is unhealthy.
const dbWriteFailureCounter = new Counter({
    name: 'cdp_cyclotron_v2_db_write_failure',
    help: 'Failed Postgres writes to cyclotron_jobs (input already validated), split by kind=logical|transient.',
    labelNames: ['kind'] as const,
})

const rescheduleSweptCounter = new Counter({
    name: 'cdp_cyclotron_v2_reschedule_parked_swept',
    help: 'Parked cyclotron jobs whose scheduled time was pulled forward by a timing-edit reschedule sweep.',
})

const rescheduleWindowHistogram = new Histogram({
    name: 'cdp_cyclotron_v2_reschedule_parked_window_seconds',
    help: 'Spread window sized for a timing-edit reschedule sweep (observed once per sweep, on the sizing slice).',
    buckets: [300, 600, 1800, 3600, 7200, 14400],
})

const rescheduleFailureCounter = new Counter({
    name: 'cdp_cyclotron_v2_reschedule_parked_failures',
    help: 'Failed reschedule sweep slices, split by kind=logical|transient.',
    labelNames: ['kind'] as const,
})

const cancelMarkedCounter = new Counter({
    name: 'cdp_cyclotron_v2_cancel_marked',
    help: 'In-flight cyclotron jobs flagged for cancellation, split by the job status at marking time.',
    labelNames: ['status'] as const,
})

/**
 * Thrown when an `overwriteExisting` createJob / bulkCreateJobs hits a row
 * that's still in an active state ('available' or 'running'). Callers should
 * treat this as a "skip and warn" rather than a hard failure — the user is
 * trying to rerun an invocation that's still mid-flight, which the safer
 * default is to refuse.
 */
export class CyclotronJobConflictError extends Error {
    constructor(public readonly conflictingIds: string | string[]) {
        super(
            `Cyclotron job overwrite refused: existing row(s) ${
                Array.isArray(conflictingIds) ? conflictingIds.join(', ') : conflictingIds
            } are in an active state`
        )
        this.name = 'CyclotronJobConflictError'
    }
}

export class CyclotronV2Manager {
    private pool: Pool
    private readonly depthLimit: number
    private readonly depthCheckIntervalMs: number
    private depthCheckPromise: Promise<boolean> | null = null
    private depthCheckExpiresAt = 0
    private readonly reschedule: {
        floorSeconds: number
        wakeRatePerSecond: number
        minWindowSeconds: number
        maxWindowSeconds: number
        chunkSize: number
        maxChunksPerCall: number
        chunkSleepMs: number
    }

    constructor(config: CyclotronV2ManagerConfig) {
        this.pool = new Pool({
            connectionString: config.pool.dbUrl,
            max: config.pool.maxConnections ?? 10,
            idleTimeoutMillis: config.pool.idleTimeoutMs ?? 30000,
        })
        this.depthLimit = config.depthLimit ?? 1_000_000
        this.depthCheckIntervalMs = config.depthCheckIntervalMs ?? 10_000
        this.reschedule = {
            floorSeconds: config.rescheduleFloorSeconds ?? 600,
            wakeRatePerSecond: config.rescheduleWakeRatePerSecond ?? 200,
            minWindowSeconds: config.rescheduleMinWindowSeconds ?? 300,
            maxWindowSeconds: config.rescheduleMaxWindowSeconds ?? 14_400,
            chunkSize: config.rescheduleChunkSize ?? 5_000,
            maxChunksPerCall: config.rescheduleMaxChunksPerCall ?? 20,
            chunkSleepMs: config.rescheduleChunkSleepMs ?? 100,
        }
    }

    async connect(): Promise<void> {
        const client = await this.pool.connect()
        client.release()
    }

    async createJob(input: CyclotronV2JobInit): Promise<string> {
        const job = CyclotronV2JobInitSchema.parse(input)
        await this.insertGuard()

        const id = job.id ?? uuidv7()
        const now = new Date()
        // Rerun re-uses the original invocation_id so lifecycle rows collapse
        // under the same ReplacingMergeTree key. The ON CONFLICT clause resets
        // a prior _terminal_ job row back to 'available' with fresh state. If
        // the existing row is still active ('available' or 'running'), the
        // UPDATE's WHERE fails, the row isn't returned, and we surface that as
        // a skip so the caller can warn rather than silently clobber in-flight
        // work. `transition_count` bumps so the janitor's poison-pill guard
        // still applies across reruns.
        const upsertClause = job.overwriteExisting
            ? `ON CONFLICT (id) DO UPDATE SET
                 status = 'available',
                 priority = EXCLUDED.priority,
                 scheduled = EXCLUDED.scheduled,
                 lock_id = NULL,
                 last_heartbeat = NULL,
                 last_transition = EXCLUDED.last_transition,
                 transition_count = LEAST(cyclotron_jobs.transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                 parent_run_id = EXCLUDED.parent_run_id,
                 state = EXCLUDED.state,
                 distinct_id = EXCLUDED.distinct_id,
                 person_id = EXCLUDED.person_id,
                 action_id = EXCLUDED.action_id,
                 cancel_requested_at = NULL
               WHERE cyclotron_jobs.status IN ('completed', 'failed', 'canceled')
               RETURNING id`
            : 'RETURNING id'
        let result: { rows: { id: string }[] }
        try {
            result = await this.pool.query<{ id: string }>(
                `INSERT INTO cyclotron_jobs
                 (id, team_id, function_id, queue_name, status, priority, scheduled, created,
                  lock_id, last_heartbeat, janitor_touch_count, transition_count, last_transition,
                  parent_run_id, state, distinct_id, person_id, action_id)
                 VALUES ($1, $2, $3, $4, 'available', $5, $6, $7,
                         NULL, NULL, 0, 0, $7,
                         $8, $9, $10, $11, $12)
                 ${upsertClause}`,
                [
                    id,
                    job.teamId,
                    job.functionId ?? null,
                    job.queueName,
                    job.priority ?? 0,
                    job.scheduled ?? now,
                    now,
                    job.parentRunId ?? null,
                    job.state ?? null,
                    job.distinctId ?? null,
                    job.personId ?? null,
                    job.actionId ?? null,
                ]
            )
        } catch (err) {
            dbWriteFailureCounter.labels({ kind: isTransientPgError(err) ? 'transient' : 'logical' }).inc()
            throw err
        }
        if (job.overwriteExisting && result.rows.length === 0) {
            // Existing row was in an active state — refuse to clobber.
            throw new CyclotronJobConflictError(id)
        }
        return id
    }

    /**
     * Bulk-insert jobs. If any input is flagged `overwriteExisting`, the entire
     * batch uses `ON CONFLICT (id) DO UPDATE` so existing rows are reset back
     * to 'available' rather than colliding on the primary key. This is how the
     * rerun path re-enqueues an invocation while preserving its
     * `invocation_id` (so lifecycle rows collapse under one ReplacingMergeTree
     * key). Mixing overwrite + non-overwrite in the same batch isn't supported
     * — pre-split into separate calls if you need that.
     */
    async bulkCreateJobs(inputs: CyclotronV2JobInit[]): Promise<string[]> {
        if (inputs.length === 0) {
            return []
        }

        const jobs = inputs.map((input) => CyclotronV2JobInitSchema.parse(input))
        const overwriteExisting = jobs.some((j) => j.overwriteExisting)

        await this.insertGuard()

        const ids: string[] = []
        const teamIds: number[] = []
        const functionIds: (string | null)[] = []
        const queueNames: string[] = []
        const priorities: number[] = []
        const scheduleds: Date[] = []
        const parentRunIds: (string | null)[] = []
        const states: (Buffer | null)[] = []
        const distinctIds: (string | null)[] = []
        const personIds: (string | null)[] = []
        const actionIds: (string | null)[] = []

        const now = new Date()

        for (const job of jobs) {
            const id = job.id ?? uuidv7()
            ids.push(id)
            teamIds.push(job.teamId)
            functionIds.push(job.functionId ?? null)
            queueNames.push(job.queueName)
            priorities.push(job.priority ?? 0)
            scheduleds.push(job.scheduled ?? now)
            parentRunIds.push(job.parentRunId ?? null)
            states.push(job.state ?? null)
            distinctIds.push(job.distinctId ?? null)
            personIds.push(job.personId ?? null)
            actionIds.push(job.actionId ?? null)
        }

        // Fair-dequeue sort key for email-queue jobs. NULL for everything else.
        // Atomically bumps the per-team counter (one UPSERT for the whole batch
        // regardless of size) and then derives a unique dequeue_seq per job.
        const dequeueSeqs = await this.computeEmailDequeueSeqs(jobs)

        const upsertClause = overwriteExisting
            ? `ON CONFLICT (id) DO UPDATE SET
                 status = 'available',
                 priority = EXCLUDED.priority,
                 scheduled = EXCLUDED.scheduled,
                 lock_id = NULL,
                 last_heartbeat = NULL,
                 last_transition = EXCLUDED.last_transition,
                 transition_count = LEAST(cyclotron_jobs.transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                 parent_run_id = EXCLUDED.parent_run_id,
                 state = EXCLUDED.state,
                 distinct_id = EXCLUDED.distinct_id,
                 person_id = EXCLUDED.person_id,
                 action_id = EXCLUDED.action_id,
                 dequeue_seq = EXCLUDED.dequeue_seq,
                 cancel_requested_at = NULL
               WHERE cyclotron_jobs.status IN ('completed', 'failed', 'canceled')
               RETURNING id`
            : 'RETURNING id'
        let result: { rows: { id: string }[] }
        try {
            result = await this.pool.query<{ id: string }>(
                `INSERT INTO cyclotron_jobs
                 (id, team_id, function_id, queue_name, status, priority, scheduled, created,
                  lock_id, last_heartbeat, janitor_touch_count, transition_count, last_transition,
                  parent_run_id, state, distinct_id, person_id, action_id, dequeue_seq)
                 SELECT
                    unnest($1::uuid[]),
                    unnest($2::int[]),
                    unnest($3::uuid[]),
                    unnest($4::text[]),
                    'available'::CyclotronJobStatus,
                    unnest($5::smallint[]),
                    unnest($6::timestamptz[]),
                    $12::timestamptz,
                    NULL::uuid,
                    NULL::timestamptz,
                    0::smallint,
                    0::smallint,
                    $12::timestamptz,
                    unnest($7::text[]),
                    unnest($8::bytea[]),
                    unnest($9::text[]),
                    unnest($10::text[]),
                    unnest($11::text[]),
                    unnest($13::bigint[])
                 ${upsertClause}`,
                [
                    ids,
                    teamIds,
                    functionIds,
                    queueNames,
                    priorities,
                    scheduleds,
                    parentRunIds,
                    states,
                    distinctIds,
                    personIds,
                    actionIds,
                    now,
                    dequeueSeqs,
                ]
            )
        } catch (err) {
            dbWriteFailureCounter.labels({ kind: isTransientPgError(err) ? 'transient' : 'logical' }).inc()
            throw err
        }

        if (overwriteExisting) {
            const returnedIds = new Set(result.rows.map((r) => r.id))
            const skipped = ids.filter((id) => !returnedIds.has(id))
            if (skipped.length > 0) {
                throw new CyclotronJobConflictError(skipped)
            }
        }

        return ids
    }

    /**
     * Compute the per-job `dequeue_seq` array for a batch of jobs.
     *
     * Email-queue jobs get `counter * BLOCK_SIZE + team_id` where `counter`
     * is the team's monotonic per-team sequence number. Sorting ascending by
     * dequeue_seq interleaves teams: each team's first job sorts together,
     * then each team's second job, etc.
     *
     * Non-email jobs get NULL (leaves the existing FIFO ordering untouched).
     *
     * One UPSERT for the whole batch regardless of job count — per-team
     * counters bump by the count of email jobs from that team in this batch,
     * and we derive each job's individual counter value in memory.
     *
     * A team's first-ever email starts at `MAX(counter) + 1` across the table
     * (Hatchet's `p_max_assigned` pattern), so a new tenant joining the
     * system can't enqueue a burst that gets free priority over established
     * tenants — established tenants are already at `MAX`, so a new tenant's
     * batch slots in at the same level and the round-robin interleaves them
     * by team_id from there. Continuing teams just keep incrementing.
     */
    private async computeEmailDequeueSeqs(jobs: CyclotronV2JobInit[]): Promise<(string | null)[]> {
        const indicesByTeam = new Map<number, number[]>()
        for (let i = 0; i < jobs.length; i++) {
            if (jobs[i].queueName !== 'email') {
                continue
            }
            const indices = indicesByTeam.get(jobs[i].teamId)
            if (indices) {
                indices.push(i)
            } else {
                indicesByTeam.set(jobs[i].teamId, [i])
            }
        }

        const dequeueSeqs: (string | null)[] = new Array(jobs.length).fill(null)
        if (indicesByTeam.size === 0) {
            return dequeueSeqs
        }

        const teamIds = [...indicesByTeam.keys()]
        const increments = teamIds.map((id) => indicesByTeam.get(id)!.length)

        // Atomic per-team UPSERT. Concurrent inserts for different teams don't
        // contend; same-team inserts serialize briefly on the row. Returns the
        // new counter value per team — we then derive individual job counters
        // by subtracting back through the batch.
        //
        // Hatchet `p_max_assigned`: new-team rows insert at `MAX + increment`
        // so they slot in next to established teams. Existing-team rows still
        // add just `increment` (we subtract the MAX shift back out in the ON
        // CONFLICT branch). MAX is computed once per batch via the CTE.
        const upsertResult = await this.pool.query<{ team_id: number; counter: string }>(
            `WITH max_counter AS (
                 SELECT COALESCE(MAX(counter), 0) AS m FROM cyclotron_email_team_seq
             )
             INSERT INTO cyclotron_email_team_seq (team_id, counter)
             SELECT team_id, increment + (SELECT m FROM max_counter)
             FROM unnest($1::int[], $2::bigint[]) AS t(team_id, increment)
             ON CONFLICT (team_id) DO UPDATE
                SET counter = cyclotron_email_team_seq.counter
                            + (EXCLUDED.counter - (SELECT m FROM max_counter))
             RETURNING team_id, counter`,
            [teamIds, increments]
        )

        const newCounters = new Map<number, bigint>(upsertResult.rows.map((r) => [r.team_id, BigInt(r.counter)]))

        for (const [teamId, indices] of indicesByTeam) {
            const newCounter = newCounters.get(teamId)
            if (newCounter === undefined) {
                // Should never happen — every team in indicesByTeam was UPSERTed.
                logger.error('Missing counter after UPSERT in computeEmailDequeueSeqs', { teamId })
                continue
            }
            const startCounter = newCounter - BigInt(indices.length) + 1n
            const teamIdBigInt = BigInt(teamId)
            for (let k = 0; k < indices.length; k++) {
                const counterForThisJob = startCounter + BigInt(k)
                // pg accepts BIGINT as string to avoid JS number precision loss
                dequeueSeqs[indices[k]] = (counterForThisJob * EMAIL_DEQUEUE_BLOCK_SIZE + teamIdBigInt).toString()
            }
        }

        return dequeueSeqs
    }

    // "In flight" = jobs still owned by the queue: parked waits/delays ('available' with a future
    // scheduled time) and jobs a worker currently holds ('running'). Terminal rows don't count.
    // Grouped by action_id so publish impact can say "N runs are parked on step X"; rows without
    // one (freshly enqueued and not yet executed, or predating the lookup column) land in
    // positionUnknown, never silently dropped.
    async countInFlightJobs(teamId: number, functionId: string): Promise<CyclotronV2InFlightCounts> {
        const result = await this.pool.query<{ action_id: string | null; count: number }>(
            `SELECT action_id, COUNT(*)::int AS count FROM cyclotron_jobs
             WHERE team_id = $1 AND function_id = $2 AND status IN ('available', 'running')
             GROUP BY action_id`,
            [teamId, functionId]
        )
        let count = 0
        let positionUnknown = 0
        const byAction: Record<string, number> = {}
        for (const row of result.rows) {
            count += row.count
            if (row.action_id) {
                byAction[row.action_id] = row.count
            } else {
                positionUnknown = row.count
            }
        }
        return { count, byAction, positionUnknown }
    }

    /**
     * Pull forward the wake times of a workflow's parked jobs after a timing
     * edit (issue #66380): jobs parked on one of `actionIds` get
     * `scheduled = LEAST(scheduled, sweepFloor + random() * window)`, so each
     * wakes somewhere inside the window, re-reads the live config, and either
     * advances or re-parks at the recomputed target. Waking early is
     * behaviourally idempotent (the executor's live-timing-edit contract
     * tests), so this never needs to read job state or compute wake times.
     *
     * Two invariants make the sweep safe:
     * - `LEAST` never moves a wake later than its natural time.
     * - Swept rows land at or before `sweepUntil`, and the candidate predicate
     *   is `scheduled > sweepUntil` — so the sweep is idempotent and naturally
     *   terminating, and rows already waking within the window are untouched.
     *
     * The floor keeps the earliest sweep-induced wake beyond the hog flow
     * cache's worst-case staleness (LazyLoader refreshAge + jitter, ~6 min),
     * so a woken job always executes against the post-edit config even on a
     * worker that missed the reload pubsub. The window is sized from the
     * parked count and a wake rate: mass-waking parked jobs at once has
     * dropped jobs in past load spikes, so wakes are trickled instead.
     *
     * Callers slice large sweeps across calls (maxChunksPerCall bounds one
     * call's work) and MUST thread the returned bounds into follow-up calls —
     * resizing the window per slice would re-compress the tail of the spread.
     * Stale bounds (a retry landing after the window largely elapsed) are
     * re-sized instead of honored: honoring them would `LEAST` the remainder
     * into the past and mass-wake it.
     */
    async rescheduleParkedJobs(
        options: CyclotronV2RescheduleParkedOptions
    ): Promise<CyclotronV2RescheduleParkedResult> {
        const { teamId, functionId, actionIds } = options
        if (actionIds.length === 0) {
            throw new Error('rescheduleParkedJobs requires at least one action id')
        }

        try {
            let bounds: { sweepFloor: Date; sweepUntil: Date } | null = null
            const staleBoundsCutoff = new Date(Date.now() + this.reschedule.minWindowSeconds * 1000)
            if (options.sweepFloor && options.sweepUntil && options.sweepUntil > staleBoundsCutoff) {
                // The floor is this sweep's safety property (no sweep-induced wake sooner than
                // floorSeconds from now — the config-cache staleness bound), so it is enforced
                // server-side regardless of what bounds the caller passed: a floor in the past
                // would land the random targets in the past and mass-wake the backlog. Clamping
                // per slice only compresses the tail of the spread, never the predicate, so
                // cross-slice idempotency (scheduled > sweepUntil) is unaffected.
                const minFloor = new Date(Date.now() + this.reschedule.floorSeconds * 1000)
                const sweepFloor = options.sweepFloor > minFloor ? options.sweepFloor : minFloor
                if (sweepFloor < options.sweepUntil) {
                    bounds = { sweepFloor, sweepUntil: options.sweepUntil }
                }
            }
            if (!bounds) {
                if (options.sweepUntil) {
                    logger.warn('Reschedule sweep bounds are stale or unsafe, re-sizing window', {
                        teamId,
                        functionId,
                        sweepUntil: options.sweepUntil.toISOString(),
                    })
                }
                const sized = await this.sizeRescheduleWindow(teamId, functionId, actionIds)
                if (!sized) {
                    const now = new Date()
                    return { swept: 0, remaining: 0, done: true, sweepFloor: now, sweepUntil: now }
                }
                bounds = sized
            }
            const { sweepFloor, sweepUntil } = bounds

            let swept = 0
            for (let chunk = 0; chunk < this.reschedule.maxChunksPerCall; chunk++) {
                if (chunk > 0) {
                    await sleep(this.reschedule.chunkSleepMs)
                }
                const result = await this.pool.query(
                    `WITH candidates AS (
                        SELECT id FROM cyclotron_jobs
                        WHERE team_id = $1 AND function_id = $2 AND status = 'available'
                          AND action_id = ANY($3::text[])
                          AND scheduled > $5
                        LIMIT $6
                        FOR UPDATE SKIP LOCKED
                    )
                    UPDATE cyclotron_jobs j
                    SET scheduled = LEAST(j.scheduled, $4::timestamptz + random() * ($5::timestamptz - $4::timestamptz))
                    FROM candidates c
                    WHERE j.id = c.id`,
                    [teamId, functionId, actionIds, sweepFloor, sweepUntil, this.reschedule.chunkSize]
                )
                swept += result.rowCount ?? 0
                if ((result.rowCount ?? 0) < this.reschedule.chunkSize) {
                    break
                }
            }
            rescheduleSweptCounter.inc(swept)

            const remainingResult = await this.pool.query<{ count: number }>(
                `SELECT COUNT(*)::int AS count FROM cyclotron_jobs
                 WHERE team_id = $1 AND function_id = $2 AND status = 'available'
                   AND action_id = ANY($3::text[])
                   AND scheduled > $4`,
                [teamId, functionId, actionIds, sweepUntil]
            )
            const remaining = remainingResult.rows[0].count

            logger.info('Reschedule sweep slice completed', {
                teamId,
                functionId,
                actionIds,
                swept,
                remaining,
                sweepFloor: sweepFloor.toISOString(),
                sweepUntil: sweepUntil.toISOString(),
            })

            return { swept, remaining, done: remaining === 0, sweepFloor, sweepUntil }
        } catch (err) {
            rescheduleFailureCounter.labels({ kind: isTransientPgError(err) ? 'transient' : 'logical' }).inc()
            throw err
        }
    }

    /**
     * Flag in-flight jobs for cancellation. This never writes a terminal status
     * itself: the manager has no path to the lifecycle/metric sinks, so the
     * owning worker performs the actual cancellation (terminal status flip plus
     * lifecycle row, metric, and log) when it next observes the job. Parked rows
     * get their wake pulled forward so that happens promptly; rows a worker
     * currently holds are flagged only, never fighting `lock_id`, because the
     * worker's release pulls the wake forward via its cancel-aware scheduled
     * CASE, so they terminate one step later.
     *
     * Chunked like `rescheduleParkedJobs` so one call never updates an
     * unbounded row count; callers loop while `done` is false. Selection
     * deliberately spans queues: a run's job can sit on another queue (e.g.
     * 'email') mid-step and still needs to be cancellable. Callers whose
     * function also owns non-run orchestration jobs exclude those queues via
     * `excludeQueueNames`.
     */
    async cancelJobs(options: CyclotronV2CancelJobsOptions): Promise<CyclotronV2CancelJobsResult> {
        const { teamId, functionId } = options
        const selectorCount = [
            options.jobIds !== undefined,
            options.all === true,
            options.parentRunId !== undefined,
        ].filter(Boolean).length
        if (selectorCount !== 1) {
            throw new Error('cancelJobs requires exactly one selector: jobIds, parentRunId, or all')
        }
        // Rejected rather than treated as unset: the SQL builder below tests truthiness, so an
        // empty string would leave the selector clause empty and silently widen a parent-run
        // cancel to every in-flight job of the workflow.
        if (options.parentRunId !== undefined && options.parentRunId.length === 0) {
            throw new Error('cancelJobs parentRunId must be a non-empty string')
        }

        const jobIds = options.jobIds ? [...new Set(options.jobIds)] : undefined
        if (jobIds && jobIds.length === 0) {
            return { marked: 0, remaining: 0, done: true }
        }

        const params: (number | string | string[])[] = [teamId, functionId]
        let selectorClause = ''
        if (jobIds) {
            params.push(jobIds)
            selectorClause = `AND id = ANY($${params.length}::uuid[])`
        } else if (options.parentRunId) {
            params.push(options.parentRunId)
            selectorClause = `AND parent_run_id = $${params.length}`
        }
        if (options.excludeQueueNames?.length) {
            params.push(options.excludeQueueNames)
            selectorClause += ` AND queue_name != ALL($${params.length}::text[])`
        }

        let marked = 0

        // Parked rows: flag + wake. SKIP LOCKED skips rows a concurrent dequeue
        // is claiming; those turn 'running' and the second UPDATE catches them.
        const chunkLimitParam = params.length + 1
        for (let chunk = 0; chunk < this.reschedule.maxChunksPerCall; chunk++) {
            if (chunk > 0) {
                await sleep(this.reschedule.chunkSleepMs)
            }
            const result = await this.pool.query(
                `WITH candidates AS (
                    SELECT id FROM cyclotron_jobs
                    WHERE team_id = $1 AND function_id = $2 AND status = 'available'
                      AND cancel_requested_at IS NULL
                      ${selectorClause}
                    LIMIT $${chunkLimitParam}
                    FOR UPDATE SKIP LOCKED
                )
                UPDATE cyclotron_jobs j
                SET cancel_requested_at = NOW(), scheduled = LEAST(j.scheduled, NOW())
                FROM candidates c
                WHERE j.id = c.id`,
                [...params, this.reschedule.chunkSize]
            )
            const n = result.rowCount ?? 0
            marked += n
            cancelMarkedCounter.labels({ status: 'available' }).inc(n)
            if (n < this.reschedule.chunkSize) {
                break
            }
        }

        // Rows held by a worker right now: bounded by worker concurrency, so
        // no chunking or wake write needed.
        const runningResult = await this.pool.query(
            `UPDATE cyclotron_jobs
             SET cancel_requested_at = NOW()
             WHERE team_id = $1 AND function_id = $2 AND status = 'running'
               AND cancel_requested_at IS NULL
               ${selectorClause}`,
            params
        )
        const runningMarked = runningResult.rowCount ?? 0
        marked += runningMarked
        cancelMarkedCounter.labels({ status: 'running' }).inc(runningMarked)

        const remainingResult = await this.pool.query<{ count: number }>(
            `SELECT COUNT(*)::int AS count FROM cyclotron_jobs
             WHERE team_id = $1 AND function_id = $2 AND status IN ('available', 'running')
               AND cancel_requested_at IS NULL
               ${selectorClause}`,
            params
        )
        const remaining = remainingResult.rows[0].count

        logger.info('Cancel sweep completed', {
            teamId,
            functionId,
            mode: jobIds ? 'jobIds' : options.parentRunId ? 'parentRunId' : 'all',
            marked,
            remaining,
        })

        return { marked, remaining, done: remaining === 0 }
    }

    /**
     * Size the sweep window for the current parked backlog: floor at
     * `now + floorSeconds`, width `count / wakeRatePerSecond` clamped to
     * [minWindow, maxWindow]. Counts everything beyond the floor — a slight
     * overcount versus the final `scheduled > sweepUntil` predicate, which
     * only widens the window (conservative). Returns null when nothing is
     * parked beyond the floor.
     */
    private async sizeRescheduleWindow(
        teamId: number,
        functionId: string,
        actionIds: string[]
    ): Promise<{ sweepFloor: Date; sweepUntil: Date } | null> {
        const countResult = await this.pool.query<{ count: number }>(
            `SELECT COUNT(*)::int AS count FROM cyclotron_jobs
             WHERE team_id = $1 AND function_id = $2 AND status = 'available'
               AND action_id = ANY($3::text[])
               AND scheduled > NOW() + make_interval(secs => $4)`,
            [teamId, functionId, actionIds, this.reschedule.floorSeconds]
        )
        const count = countResult.rows[0].count
        if (count === 0) {
            return null
        }

        const windowSeconds = Math.min(
            this.reschedule.maxWindowSeconds,
            Math.max(this.reschedule.minWindowSeconds, Math.ceil(count / this.reschedule.wakeRatePerSecond))
        )
        rescheduleWindowHistogram.observe(windowSeconds)

        const sweepFloor = new Date(Date.now() + this.reschedule.floorSeconds * 1000)
        const sweepUntil = new Date(sweepFloor.getTime() + windowSeconds * 1000)
        return { sweepFloor, sweepUntil }
    }

    async disconnect(): Promise<void> {
        await this.pool.end()
    }

    private async insertGuard(): Promise<void> {
        if (await this.isFull()) {
            throw new Error(`Cyclotron V2 queue is full (depth limit: ${this.depthLimit})`)
        }
    }

    private isFull(): Promise<boolean> {
        if (this.depthCheckPromise && Date.now() < this.depthCheckExpiresAt) {
            return this.depthCheckPromise
        }

        this.depthCheckPromise = this.queryDepth()
        this.depthCheckExpiresAt = Date.now() + this.depthCheckIntervalMs
        return this.depthCheckPromise
    }

    private async queryDepth(): Promise<boolean> {
        try {
            const result = await this.pool.query(
                `SELECT COUNT(*) AS count FROM cyclotron_jobs
                 WHERE status = 'available' AND scheduled <= NOW()`
            )
            const count = parseInt(result.rows[0].count, 10)
            const full = count >= this.depthLimit

            if (full) {
                logger.warn('Cyclotron V2 queue at capacity', { count, depthLimit: this.depthLimit })
            }

            return full
        } catch (e) {
            logger.error('Cyclotron V2 depth check failed', { error: String(e) })
            return false
        }
    }

    // Wakes parked workflow steps with the outcome of the run they dispatched. Scoped to one team
    // because the caller's token is; see processStepResumes for the per-job outcomes.
    async resumeParkedSteps(teamId: number, resumes: StepResume[]): Promise<Map<string, StepResumeOutcome>> {
        return await processStepResumes(this.pool, resumes, teamId)
    }

    async resumeParkedBatchResolver(options: {
        teamId: number
        functionId: string
        batchJobId: string
    }): Promise<BatchResolverResumeOutcome> {
        return await resumeParkedBatchResolver(this.pool, options)
    }
}