Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

27.3 KB
Raw Blame
import { DateTime } from 'luxon'
import { Pool, PoolClient } from 'pg'
import { Counter } from 'prom-client'
import { v7 as uuidv7 } from 'uuid'

import { logger } from '~/common/utils/logger'

import { assignEmailDequeueSeq } from './manager'
import {
    CYCLOTRON_COUNTER_MAX,
    CYCLOTRON_TRANSITION_CHURN_THRESHOLD,
    CyclotronV2BulkCreateAndCheckInInput,
    CyclotronV2DequeuedJob,
    CyclotronV2JobInit,
    CyclotronV2JobInitSchema,
    CyclotronV2RescheduleOptions,
    CyclotronV2RescheduleOptionsSchema,
    CyclotronV2WorkerConfig,
} from './types'

export interface RawJobRow {
    id: string
    team_id: number
    function_id: string | null
    queue_name: string
    priority: number
    scheduled: string
    created: string
    parent_run_id: string | null
    transition_count: number
    state: Buffer | null
    distinct_id: string | null
    person_id: string | null
    action_id: string | null
    cancel_requested_at: string | null
    lock_id: string
}

// Read off the row the dequeue already returns, so tracking churn costs no extra query.
const highTransitionDequeuesCounter = new Counter({
    name: 'cdp_cyclotron_v2_high_transition_dequeues',
    help: `Jobs dequeued with transition_count at or above ${CYCLOTRON_TRANSITION_CHURN_THRESHOLD}, meaning they are cycling without completing.`,
    labelNames: ['queue'] as const,
})

// The loop swallows the error and retries, so without this counter a queue whose every
// dequeue throws looks exactly like an idle queue. Exported for the rate-limited
// subclass, which has its own loop and catch.
export const consumerLoopErrorsCounter = new Counter({
    name: 'cdp_cyclotron_v2_consumer_loop_errors_total',
    help: 'Consumer loop iterations that threw, per queue. A sustained positive rate can indicate the queue is not being consumed.',
    labelNames: ['queue'] as const,
})

export function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

/**
 * Insert N new jobs within an open transaction. Mirrors the bulk-insert shape
 * from `CyclotronV2Manager.bulkCreateJobs` but runs on a caller-provided
 * client so the inserts can be wrapped in the same transaction as a sibling
 * UPDATE (used by `bulkCreateAndCheckIn`).
 *
 * Intentionally narrower than manager.bulkCreateJobs: no `overwriteExisting`
 * path, no email-queue `dequeue_seq` computation (the resolver enqueues
 * children onto 'hogflow', not 'email'). Email re-routing happens later via
 * the worker's reschedule path which already handles dequeue_seq assignment.
 */
async function insertNewJobsInTx(client: PoolClient, newJobs: CyclotronV2JobInit[]): Promise<string[]> {
    if (newJobs.length === 0) {
        return []
    }

    const now = new Date()
    const ids: string[] = []
    const teamIds: number[] = []
    const functionIds: (string | null)[] = []
    const queueNames: string[] = []
    const priorities: number[] = []
    const scheduleds: Date[] = []
    const parentRunIds: (string | null)[] = []
    const states: (Buffer | null)[] = []
    const distinctIds: (string | null)[] = []
    const personIds: (string | null)[] = []
    const actionIds: (string | null)[] = []

    for (const job of newJobs) {
        ids.push(job.id ?? uuidv7())
        teamIds.push(job.teamId)
        functionIds.push(job.functionId ?? null)
        queueNames.push(job.queueName)
        priorities.push(job.priority ?? 0)
        scheduleds.push(job.scheduled ?? now)
        parentRunIds.push(job.parentRunId ?? null)
        states.push(job.state ?? null)
        distinctIds.push(job.distinctId ?? null)
        personIds.push(job.personId ?? null)
        actionIds.push(job.actionId ?? null)
    }

    await client.query(
        `INSERT INTO cyclotron_jobs
         (id, team_id, function_id, queue_name, status, priority, scheduled, created,
          lock_id, last_heartbeat, janitor_touch_count, transition_count, last_transition,
          parent_run_id, state, distinct_id, person_id, action_id)
         SELECT
            unnest($1::uuid[]),
            unnest($2::int[]),
            unnest($3::uuid[]),
            unnest($4::text[]),
            'available'::CyclotronJobStatus,
            unnest($5::smallint[]),
            unnest($6::timestamptz[]),
            $12::timestamptz,
            NULL::uuid,
            NULL::timestamptz,
            0::smallint,
            0::smallint,
            $12::timestamptz,
            unnest($7::text[]),
            unnest($8::bytea[]),
            unnest($9::text[]),
            unnest($10::text[]),
            unnest($11::text[])`,
        [
            ids,
            teamIds,
            functionIds,
            queueNames,
            priorities,
            scheduleds,
            parentRunIds,
            states,
            distinctIds,
            personIds,
            actionIds,
            now,
        ]
    )

    return ids
}

/**
 * Update the current worker's job row to reflect the chosen disposition
 * (reschedule for next page / ack / fail). Mirrors the SQL shapes used by
 * the wrapper's `ack`, `fail`, and `reschedule` methods, but runs in a
 * caller-provided transaction so it can be atomic with sibling inserts.
 *
 * Throws if the UPDATE doesn't match exactly one row. The `WHERE lock_id`
 * filter can match zero rows if the janitor reassigns the lock between
 * dequeue and commit (stall-recovery race) — without this guard the child
 * inserts would commit while the self UPDATE silently no-ops, leaving the
 * cursor un-advanced for the new lock-holder to replay.
 */
async function updateSelfInTx(
    client: PoolClient,
    jobId: string,
    lockId: string,
    disposition: CyclotronV2BulkCreateAndCheckInInput['selfDisposition']
): Promise<void> {
    // Deliberate self-checkins zero janitor_touch_count for the same reason the
    // wrapJob dispositions do — the poison budget counts CONSECUTIVE stalls, so a
    // healthy batch-resolve release must not leave a stale touch count behind.
    if (disposition.kind === 'ack') {
        const result = await client.query(
            `UPDATE cyclotron_jobs
             SET status = 'completed', lock_id = NULL, last_heartbeat = NULL,
                 last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                 janitor_touch_count = 0
             WHERE id = $1 AND lock_id = $2`,
            [jobId, lockId]
        )
        assertSelfRowAffected(result.rowCount, jobId, 'ack')
        return
    }
    if (disposition.kind === 'fail') {
        const result = await client.query(
            `UPDATE cyclotron_jobs
             SET status = 'failed', lock_id = NULL, last_heartbeat = NULL,
                 last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                 janitor_touch_count = 0
             WHERE id = $1 AND lock_id = $2`,
            [jobId, lockId]
        )
        assertSelfRowAffected(result.rowCount, jobId, 'fail')
        return
    }
    // reschedule
    const scheduled = disposition.scheduledAt ?? new Date()
    const setClauses = [
        `status = 'available'`,
        `lock_id = NULL`,
        `last_heartbeat = NULL`,
        `last_transition = NOW()`,
        `transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX})`,
        `janitor_touch_count = 0`,
        // A cancel requested while this worker held the job must not sleep the full
        // next delay before it's observed, so it wakes immediately instead.
        `scheduled = CASE WHEN cancel_requested_at IS NOT NULL THEN LEAST($3::timestamptz, NOW()) ELSE $3::timestamptz END`,
    ]
    const params: any[] = [jobId, lockId, scheduled]
    if (disposition.state !== undefined) {
        params.push(disposition.state ?? null)
        setClauses.push(`state = $${params.length}`)
    }
    const result = await client.query(
        `UPDATE cyclotron_jobs SET ${setClauses.join(', ')}
         WHERE id = $1 AND lock_id = $2`,
        params
    )
    assertSelfRowAffected(result.rowCount, jobId, 'reschedule')
}

function assertSelfRowAffected(rowCount: number | null, jobId: string, kind: string): void {
    if (rowCount !== 1) {
        throw new Error(
            `bulkCreateAndCheckIn(${kind}) self row matched ${rowCount} rows for job ${jobId} — lock_id may have been reassigned`
        )
    }
}

export class CyclotronV2Worker {
    private pool: Pool
    protected isConsuming = false
    protected lastPollTime = new Date()
    private consumerLoopPromise: Promise<void> | null = null

    protected readonly queueName: string
    protected readonly batchMaxSize: number
    protected readonly pollDelayMs: number
    private readonly heartbeatTimeoutMs: number
    protected readonly includeEmptyBatches: boolean
    protected readonly fairDequeue: boolean

    constructor(private config: CyclotronV2WorkerConfig) {
        this.pool = new Pool({
            connectionString: config.pool.dbUrl,
            max: config.pool.maxConnections ?? 10,
            idleTimeoutMillis: config.pool.idleTimeoutMs ?? 30000,
        })
        this.queueName = config.queueName
        this.batchMaxSize = config.batchMaxSize ?? 100
        this.pollDelayMs = config.pollDelayMs ?? 50
        this.heartbeatTimeoutMs = config.heartbeatTimeoutMs ?? 30000
        this.includeEmptyBatches = config.includeEmptyBatches ?? false
        // Fair (per-team round-robin) dequeue is the email queue's ordering.
        // `dequeue_seq` is only ever assigned for email jobs (see
        // CyclotronV2Manager), so deriving this from the queue name keeps the
        // worker's ORDER BY in lockstep with where the sort key actually exists.
        this.fairDequeue = config.queueName === 'email'
    }

    async connect(processBatch: (jobs: CyclotronV2DequeuedJob[]) => Promise<void>): Promise<void> {
        const client = await this.pool.connect()
        client.release()

        this.isConsuming = true
        this.consumerLoopPromise = this.runConsumerLoop(processBatch)
    }

    protected async runConsumerLoop(processBatch: (jobs: CyclotronV2DequeuedJob[]) => Promise<void>): Promise<void> {
        while (this.isConsuming) {
            try {
                this.lastPollTime = new Date()
                const rows = this.fairDequeue ? await this.fairDequeueJobs() : await this.dequeueJobs()

                if (rows.length === 0) {
                    if (this.includeEmptyBatches) {
                        await processBatch([])
                    }
                    await sleep(this.pollDelayMs)
                    continue
                }

                const jobs = rows.map((row) => this.wrapJob(row))
                await processBatch(jobs)
            } catch (err) {
                consumerLoopErrorsCounter.labels({ queue: this.queueName }).inc()
                logger.error('CyclotronV2Worker consumer loop error', { error: String(err) })
                await sleep(this.pollDelayMs)
            }
        }
    }

    /**
     * Cheap pre-check that returns how many jobs are currently dequeueable from
     * this worker's queue, capped at `limit`. Used by rate-limited subclasses
     * to size their token-bucket claim to "exactly what we plan to dequeue"
     * instead of always claiming the bucket's full capacity — keeps unused
     * tokens in the bucket when traffic is sparse and a single job per poll is
     * what we end up dequeuing.
     *
     * Hits the same partial index as `dequeueJobs` (`idx_cyclotron_jobs_dequeue`,
     * filtered on `status = 'available'`), so the query is sub-millisecond and
     * strictly cheaper than the `UPDATE ... SKIP LOCKED` we'd otherwise run.
     * A short race exists between this count and the subsequent dequeue —
     * concurrent pods may grab some rows in the gap via `SKIP LOCKED` — so
     * `dequeueJobs` can return fewer rows than counted. That's bounded by
     * concurrency × poll rate and rate-limiter refill absorbs the slack.
     */
    protected async countWork(limit: number): Promise<number> {
        const result = await this.pool.query<{ n: number }>(
            `SELECT COUNT(*)::int AS n FROM (
                 SELECT 1 FROM cyclotron_jobs
                 WHERE status = 'available'
                   AND queue_name = $1
                   AND scheduled <= NOW()
                 LIMIT $2
             ) sub`,
            [this.config.queueName, limit]
        )
        return result.rows[0].n
    }

    protected async dequeueJobs(limit: number = this.batchMaxSize): Promise<RawJobRow[]> {
        const lockId = uuidv7()
        const result = await this.pool.query<RawJobRow>(
            `WITH available AS (
                SELECT id
                FROM cyclotron_jobs
                WHERE status = 'available'
                  AND queue_name = $1
                  AND scheduled <= NOW()
                ORDER BY priority ASC, scheduled ASC
                LIMIT $2
                FOR UPDATE SKIP LOCKED
            )
            UPDATE cyclotron_jobs
            SET status = 'running',
                lock_id = $3,
                last_heartbeat = NOW(),
                last_transition = NOW(),
                transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX})
            FROM available
            WHERE cyclotron_jobs.id = available.id
            RETURNING
                cyclotron_jobs.id,
                cyclotron_jobs.team_id,
                cyclotron_jobs.function_id,
                cyclotron_jobs.queue_name,
                cyclotron_jobs.priority,
                cyclotron_jobs.scheduled,
                cyclotron_jobs.created,
                cyclotron_jobs.parent_run_id,
                cyclotron_jobs.transition_count,
                cyclotron_jobs.state,
                cyclotron_jobs.distinct_id,
                cyclotron_jobs.person_id,
                cyclotron_jobs.action_id,
                cyclotron_jobs.cancel_requested_at,
                cyclotron_jobs.lock_id`,
            [this.config.queueName, limit, lockId]
        )
        // UPDATE...RETURNING doesn't preserve the CTE's ORDER BY,
        // so re-sort to maintain priority ordering within the batch
        // TODO: Do we care about this in reality?
        return result.rows.sort(
            (a, b) => a.priority - b.priority || new Date(a.scheduled).getTime() - new Date(b.scheduled).getTime()
        )
    }

    /**
     * Fair dequeue: priority class leads the sort so transactional-class sends
     * aren't stuck behind a broadcast backlog, then the precomputed
     * `dequeue_seq` interleaves jobs across tenants within a class instead of
     * being strict FIFO. The sort key is assigned at insert time (see
     * `CyclotronV2Manager.bulkCreateJobs` and the helper
     * `cyclotron_email_team_seq`); this method just reads them back in order.
     *
     * Hits the partial index `idx_cyclotron_jobs_email_priority_fair_dequeue`
     * (only indexes email-queue rows with status='available'). NULLS FIRST
     * drains any pre-migration legacy rows ahead of new fair-ordered ones.
     *
     * Email-specific by intent — but mechanically just "ORDER BY different
     * columns", so the SQL shape mirrors `dequeueJobs` exactly. Kept as a
     * separate method so non-fair callers can read `dequeueJobs` end-to-end
     * without following a conditional or an indirection.
     */
    protected async fairDequeueJobs(limit: number = this.batchMaxSize): Promise<RawJobRow[]> {
        const lockId = uuidv7()
        const result = await this.pool.query<RawJobRow>(
            `WITH available AS (
                SELECT id
                FROM cyclotron_jobs
                WHERE status = 'available'
                  AND queue_name = $1
                  AND scheduled <= NOW()
                ORDER BY priority ASC, dequeue_seq ASC NULLS FIRST
                LIMIT $2
                FOR UPDATE SKIP LOCKED
            )
            UPDATE cyclotron_jobs
            SET status = 'running',
                lock_id = $3,
                last_heartbeat = NOW(),
                last_transition = NOW(),
                transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX})
            FROM available
            WHERE cyclotron_jobs.id = available.id
            RETURNING
                cyclotron_jobs.id,
                cyclotron_jobs.team_id,
                cyclotron_jobs.function_id,
                cyclotron_jobs.queue_name,
                cyclotron_jobs.priority,
                cyclotron_jobs.scheduled,
                cyclotron_jobs.created,
                cyclotron_jobs.parent_run_id,
                cyclotron_jobs.transition_count,
                cyclotron_jobs.state,
                cyclotron_jobs.distinct_id,
                cyclotron_jobs.person_id,
                cyclotron_jobs.action_id,
                cyclotron_jobs.cancel_requested_at,
                cyclotron_jobs.lock_id`,
            [this.config.queueName, limit, lockId]
        )
        // Within-batch order is undefined (UPDATE...RETURNING doesn't preserve
        // the CTE's ORDER BY), but the guarantee is *across* batches: the CTE
        // picks the rows with the lowest priority, then the lowest dequeue_seq,
        // so a transactional send never waits behind a marketing backlog and a
        // small-tenant job never gets stuck behind a large-tenant one.
        return result.rows
    }

    protected wrapJob(row: RawJobRow): CyclotronV2DequeuedJob {
        const pool = this.pool
        const lockId = row.lock_id
        let released = false

        if (row.transition_count >= CYCLOTRON_TRANSITION_CHURN_THRESHOLD) {
            highTransitionDequeuesCounter.labels({ queue: row.queue_name }).inc()
        }

        const releaseGuard = (method: string) => {
            if (released) {
                throw new Error(`Job ${row.id} already released, cannot call ${method}`)
            }
            released = true
        }

        return {
            id: row.id,
            teamId: row.team_id,
            functionId: row.function_id,
            queueName: row.queue_name,
            priority: row.priority,
            scheduled: DateTime.fromISO(row.scheduled, { zone: 'utc' }),
            created: DateTime.fromISO(row.created, { zone: 'utc' }),
            parentRunId: row.parent_run_id,
            transitionCount: row.transition_count,
            state: row.state,
            distinctId: row.distinct_id,
            personId: row.person_id,
            actionId: row.action_id,
            cancelRequestedAt: row.cancel_requested_at
                ? DateTime.fromISO(row.cancel_requested_at, { zone: 'utc' })
                : null,

            async ack(): Promise<void> {
                releaseGuard('ack')
                await pool.query(
                    `UPDATE cyclotron_jobs
                     SET status = 'completed', lock_id = NULL, last_heartbeat = NULL,
                         last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                         janitor_touch_count = 0
                     WHERE id = $1 AND lock_id = $2`,
                    [row.id, lockId]
                )
            },

            async fail(): Promise<void> {
                releaseGuard('fail')
                await pool.query(
                    `UPDATE cyclotron_jobs
                     SET status = 'failed', lock_id = NULL, last_heartbeat = NULL,
                         last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                         janitor_touch_count = 0
                     WHERE id = $1 AND lock_id = $2`,
                    [row.id, lockId]
                )
            },

            async reschedule(input?: CyclotronV2RescheduleOptions): Promise<void> {
                releaseGuard('reschedule')
                const options = input ? CyclotronV2RescheduleOptionsSchema.parse(input) : undefined
                const scheduled = options?.scheduledAt ?? new Date()

                const setClauses = [
                    `status = 'available'`,
                    `lock_id = NULL`,
                    `last_heartbeat = NULL`,
                    `last_transition = NOW()`,
                    `transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX})`,
                    // A deliberate release means the worker is healthy, so the
                    // poison budget counts CONSECUTIVE stalls — long-lived jobs
                    // (e.g. wait_until_condition polls) don't accrue touches for
                    // their whole life and get mistaken for poison pills.
                    `janitor_touch_count = 0`,
                    // A cancel requested while this worker held the job must not
                    // sleep the full next delay before it's observed, so it
                    // wakes immediately instead.
                    `scheduled = CASE WHEN cancel_requested_at IS NOT NULL THEN LEAST($3::timestamptz, NOW()) ELSE $3::timestamptz END`,
                ]
                const params: any[] = [row.id, lockId, scheduled]

                if (options?.state !== undefined) {
                    params.push(options.state ?? null)
                    setClauses.push(`state = $${params.length}`)
                }
                if (options?.distinctId !== undefined) {
                    params.push(options.distinctId ?? null)
                    setClauses.push(`distinct_id = $${params.length}`)
                }
                if (options?.personId !== undefined) {
                    params.push(options.personId ?? null)
                    setClauses.push(`person_id = $${params.length}`)
                }
                if (options?.actionId !== undefined) {
                    params.push(options.actionId ?? null)
                    setClauses.push(`action_id = $${params.length}`)
                }
                if (options?.queueName !== undefined) {
                    params.push(options.queueName)
                    setClauses.push(`queue_name = $${params.length}`)
                }
                if (options?.priority !== undefined) {
                    params.push(options.priority)
                    setClauses.push(`priority = $${params.length}`)
                }

                // Cross-queue routing into the email queue: assign a fresh
                // dequeue_seq so the row participates in fair ordering. Without
                // this, hogflow → email re-routing (via job.reschedule({
                // queueName: 'email' })) lands the row with NULL dequeue_seq,
                // which `NULLS FIRST` drains ahead of every fair-ordered row —
                // defeating the per-team interleave for the most common email
                // path. Skipped when the row was already on the email queue,
                // so existing fair-ordered rows keep their place after retry/
                // reschedule without bumping the counter.
                if (options?.queueName === 'email' && row.queue_name !== 'email') {
                    const dequeueSeq = await assignEmailDequeueSeq(pool, row.team_id)
                    params.push(dequeueSeq)
                    setClauses.push(`dequeue_seq = $${params.length}`)
                }

                await pool.query(
                    `UPDATE cyclotron_jobs SET ${setClauses.join(', ')}
                     WHERE id = $1 AND lock_id = $2`,
                    params
                )
            },

            async cancel(): Promise<void> {
                releaseGuard('cancel')
                await pool.query(
                    `UPDATE cyclotron_jobs
                     SET status = 'canceled', lock_id = NULL, last_heartbeat = NULL,
                         last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX}),
                         janitor_touch_count = 0
                     WHERE id = $1 AND lock_id = $2`,
                    [row.id, lockId]
                )
            },

            async heartbeat(): Promise<void> {
                if (released) {
                    throw new Error(`Job ${row.id} already released, cannot heartbeat`)
                }
                await pool.query(
                    `UPDATE cyclotron_jobs
                     SET last_heartbeat = NOW()
                     WHERE id = $1 AND lock_id = $2`,
                    [row.id, lockId]
                )
            },

            async bulkCreateAndCheckIn(
                input: CyclotronV2BulkCreateAndCheckInInput
            ): Promise<{ newJobIds: string[]; cancelRequested?: boolean }> {
                releaseGuard('bulkCreateAndCheckIn')

                // Validate new jobs up front, outside the TX, so a malformed
                // input doesn't burn a connection slot mid-transaction.
                const newJobs: CyclotronV2JobInit[] = input.newJobs.map((j) => CyclotronV2JobInitSchema.parse(j))

                const client = await pool.connect()
                try {
                    await client.query('BEGIN')

                    // Cancel tombstone, checked inside the transaction. The FOR UPDATE takes the
                    // self row's lock, so this serializes against cancelJobs' flag write: a flag
                    // that committed first refuses the whole page here (nothing inserted), and a
                    // page that locked first commits before the flag can land — the cancel
                    // sweep's remaining-count then still sees its jobs. Without this check, a
                    // page could commit after a cancel sweep counted zero remaining, leaving
                    // jobs that never get flagged.
                    const selfRow = await client.query<{ cancel_requested_at: string | null }>(
                        `SELECT cancel_requested_at FROM cyclotron_jobs
                         WHERE id = $1 AND lock_id = $2
                         FOR UPDATE`,
                        [row.id, lockId]
                    )
                    assertSelfRowAffected(selfRow.rowCount, row.id, 'precheck')
                    if (selfRow.rows[0].cancel_requested_at !== null) {
                        await client.query('ROLLBACK')
                        // The job was never released — hand it back to the caller to dispose.
                        released = false
                        return { newJobIds: [], cancelRequested: true }
                    }

                    const newJobIds = await insertNewJobsInTx(client, newJobs)
                    await updateSelfInTx(client, row.id, lockId, input.selfDisposition)

                    await client.query('COMMIT')
                    return { newJobIds }
                } catch (err) {
                    try {
                        await client.query('ROLLBACK')
                    } catch (rollbackErr) {
                        logger.warn('bulkCreateAndCheckIn rollback failed', {
                            error: String(rollbackErr),
                            originalError: String(err),
                        })
                    }
                    throw err
                } finally {
                    client.release()
                }
            },
        }
    }

    isHealthy(): boolean {
        return this.isConsuming && Date.now() - this.lastPollTime.getTime() < this.heartbeatTimeoutMs
    }

    async stopConsuming(): Promise<void> {
        this.isConsuming = false
        if (this.consumerLoopPromise) {
            await this.consumerLoopPromise
            this.consumerLoopPromise = null
        }
    }

    async disconnect(): Promise<void> {
        await this.stopConsuming()
        await this.pool.end()
    }
}