:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { Pool } from 'pg'
import { Counter, Gauge } from 'prom-client'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { CYCLOTRON_INVOCATION_JOB_QUEUES, CyclotronJobInvocationHogFlow } from '../../types'
import { v2JobToInvocation } from '../job-queue/job-queue-postgres-v2'
import { emailAttachmentOutcomesTotal } from '../messaging/email-attachments.service'
import { HogInvocationResultsService } from '../monitoring/hog-invocation-results.service'
import {
CYCLOTRON_COUNTER_MAX,
CyclotronV2CleanupResult,
CyclotronV2DequeuedJob,
CyclotronV2JanitorConfig,
} from './types'
// Stable, low-cardinality `error_kind` stamped on the failed invocation result
// the janitor writes when it gives up on a poison pill. Lets operators target
// exactly these give-ups from the rerun tooling (rerun filter `error_kind`).
export const JANITOR_POISON_PILL_ERROR_KIND = 'janitor_poison_pill'
// Only jobs on a real invocation queue carry a replayable invocation. Everything else on this
// shared cyclotron_jobs table (the rerun + batch-resolve wrappers, and any future meta queue) has a
// `function_id` pointing at a target function and a `state` that is not an invocation — recording
// one as a give-up would tag it `function_kind='hog_flow'` and let the autodrain replay a real flow
// with fabricated globals (see recordAndDeletePoisonPills). So the janitor records give-ups only for
// this allow-list and drops everything else without a record. Keying off the canonical allow-list
// (it *is* the `CyclotronJobQueueKind` type) is deliberately fail-safe: a new meta queue is dropped
// by default, and a new invocation queue has to be added to this list to compile — no hand-kept
// deny-list to fall out of sync (which is how the batch-resolve queue was missed originally).
const INVOCATION_QUEUE_NAMES: string[] = [...CYCLOTRON_INVOCATION_JOB_QUEUES]
// The first stall gets only this small jittered delay (not the full exponential
// base), so a transient stall — a worker restart/deploy, where a healthy worker
// should pick the job right back up — recovers in seconds rather than waiting a
// full backoff step. The jitter still de-syncs a fleet-wide herd on that first
// retry. Repeat stalls (a persistent signal) then pay the growing exponential.
const STALL_BACKOFF_FIRST_RETRY_SPREAD_MS = 5000
const janitorDeletedCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_deleted',
help: 'Number of terminal jobs cleaned up by the janitor',
labelNames: ['status'],
})
const janitorStalledCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_stalled',
help: 'Number of stalled jobs reset by the janitor',
})
const janitorPoisonedCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_poisoned',
help: 'Number of poison pill jobs given up on (recorded as failed, replayable) by the janitor',
})
const janitorGiveUpSkippedCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_give_up_skipped',
help: 'Poison pills the janitor could not record a recovery row for, so kept (not deleted)',
})
const janitorWrapperDroppedCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_wrapper_dropped',
help: 'Poisoned wrapper/meta jobs dropped without a replay record (a wrapper is not a replayable invocation)',
})
const janitorRunCounter = new Counter({
name: 'cdp_cyclotron_v2_janitor_runs',
help: 'Number of janitor cleanup runs completed',
})
// The name is load-bearing: dashboards and alerts query it, so it must not follow the code around.
const conversionWatchersSweptCounter = new Counter({
name: 'cdp_conversion_watchers_swept',
help: 'Expired conversion watcher rows deleted. These are runs that reached the end of their attribution window without converting.',
})
export const EMAIL_ATTACHMENTS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000
const emailAttachmentsTableBytesGauge = new Gauge({
name: 'cdp_email_attachments_table_bytes',
help: 'Total size of cyclotron_email_attachments on disk, with its TOAST data and indexes (pg_total_relation_size). Sampled by the janitor sweep.',
})
const emailAttachmentsRowsGauge = new Gauge({
name: 'cdp_email_attachments_rows',
help: 'Planner estimate (pg_class.reltuples) of the rows in cyclotron_email_attachments, one per file of a transactional email that waits to send. Sampled by the janitor sweep without a table scan.',
})
const queueDepthGauge = new Gauge({
name: 'cdp_cyclotron_v2_queue_depth',
help: 'Number of available jobs per queue',
labelNames: ['queue'],
})
const queueOldestDueAgeGauge = new Gauge({
name: 'cdp_cyclotron_v2_queue_oldest_due_age_ms',
help: 'Milliseconds since the oldest available job per queue became due (its scheduled time). 0 when the queue has no due jobs.',
labelNames: ['queue'],
})
interface PoisonRow {
id: string
team_id: number
function_id: string | null
queue_name: string
priority: number
// pg returns these as ISO strings here (same as the worker's dequeue rows),
// not Date objects — parse with fromISO, never fromJSDate.
scheduled: string
created: string
parent_run_id: string | null
state: Buffer | null
distinct_id: string | null
person_id: string | null
action_id: string | null
janitor_touch_count: number
transition_count: number
}
export class CyclotronV2Janitor {
private pool: Pool
private intervalHandle: ReturnType<typeof setInterval> | null = null
// Queue names this process has seen hold work, so a drained queue can report 0.
private readonly seenQueues = new Set<string>()
private readonly cleanupBatchSize: number
private readonly cleanupIntervalMs: number
private readonly stallTimeoutMs: number
private readonly maxTouchCount: number
private readonly cleanupGraceMs: number
private readonly poisonRecoveryEnabled: boolean
private readonly stallBackoffBaseMs: number
private readonly stallBackoffMaxMs: number
constructor(
config: CyclotronV2JanitorConfig,
// Optional so the v1 postgres paths and unit tests that don't exercise
// the give-up path can construct a janitor without Kafka. When absent
// the janitor never deletes poison pills (it only resets/retries) — it
// refuses to drop a job it can't record a recovery row for.
private invocationResults?: HogInvocationResultsService
) {
this.pool = new Pool({
connectionString: config.pool.dbUrl,
max: config.pool.maxConnections ?? 5,
idleTimeoutMillis: config.pool.idleTimeoutMs ?? 30000,
})
this.cleanupBatchSize = config.cleanupBatchSize ?? 10000
this.cleanupIntervalMs = config.cleanupIntervalMs ?? 10000
this.stallTimeoutMs = config.stallTimeoutMs ?? 30000
this.maxTouchCount = config.maxTouchCount ?? 3
this.cleanupGraceMs = config.cleanupGraceMs ?? 10000
this.poisonRecoveryEnabled = config.poisonRecoveryEnabled ?? true
// Clamp against a misconfigured value: a non-positive base disables backoff
// (same as 0), and a non-positive cap falls back to the default — otherwise
// a negative cap would win the LEAST() and schedule the retry in the past.
const backoffBaseMs = config.stallBackoffBaseMs ?? 60000
const backoffMaxMs = config.stallBackoffMaxMs ?? 600000
this.stallBackoffBaseMs = backoffBaseMs > 0 ? backoffBaseMs : 0
this.stallBackoffMaxMs = backoffMaxMs > 0 ? backoffMaxMs : 600000
if (!this.poisonRecoveryEnabled) {
logger.warn(
'CyclotronV2Janitor poison-pill recovery DISABLED via CYCLOTRON_NODE_POISON_PILL_RECOVERY_ENABLED=false — reverting to legacy behavior: poison pills are marked failed with no replay record'
)
}
}
async start(): Promise<void> {
const client = await this.pool.connect()
client.release()
this.intervalHandle = setInterval(() => {
this.runOnce().catch((err) => {
logger.error('CyclotronV2Janitor run error', { error: String(err) })
})
}, this.cleanupIntervalMs)
// Run immediately on start
await this.runOnce()
}
async runOnce(): Promise<CyclotronV2CleanupResult> {
// Sweep first, before the cyclotron-cleanup stages below. Each of those issues an unguarded
// pool.query, so a failure specific to cyclotron_jobs (a statement timeout or lock pressure on
// that hot table) rejects out of runOnce and would skip the sweep for as long as it lasts. The
// sweep touches a different table and swallows its own errors, so running it up front keeps
// expired watchers draining even while cyclotron cleanup is failing.
await this.sweepExpiredConversionWatchers()
await this.sweepEmailAttachments()
const deletedCounts = await this.cleanupTerminalJobs()
const deleted = Object.values(deletedCounts).reduce((a, b) => a + b, 0)
// Give up on genuine poison pills. The kill-switch picks between two
// self-contained paths with no inline flag checks beyond this branch:
// the new record-then-delete path, or master's legacy mark-failed path.
// The legacy path (and the flag) can be deleted wholesale once we trust
// recording in production.
const poisonedIds = this.poisonRecoveryEnabled
? await this.recordAndDeletePoisonPills()
: await this.failPoisonPills()
const stalled = await this.resetStalledJobs()
const depths = await this.measureQueueDepths()
janitorRunCounter.inc()
return { deleted, stalled, poisoned: poisonedIds.length, poisonedIds, depths }
}
private async cleanupTerminalJobs(): Promise<Record<string, number>> {
const cutoff = new Date(Date.now() - this.cleanupGraceMs)
const result = await this.pool.query<{ status: string; count: string }>(
`WITH to_delete AS (
SELECT id
FROM cyclotron_jobs
WHERE status IN ('completed', 'failed', 'canceled')
AND last_transition < $1
ORDER BY last_transition ASC
LIMIT $2
FOR UPDATE SKIP LOCKED
)
DELETE FROM cyclotron_jobs
USING to_delete
WHERE cyclotron_jobs.id = to_delete.id
RETURNING cyclotron_jobs.status::text`,
[cutoff, this.cleanupBatchSize]
)
let total = 0
const counts: Record<string, number> = {}
for (const row of result.rows) {
counts[row.status] = (counts[row.status] ?? 0) + 1
total++
}
for (const [status, count] of Object.entries(counts)) {
janitorDeletedCounter.inc({ status }, count)
}
if (total > 0) {
logger.info('CyclotronV2Janitor cleaned up terminal jobs', { counts, total })
}
return counts
}
/**
* Kill-switch OFF path — restores master's pre-recovery behavior verbatim:
* mark poison pills `failed` (then `cleanupTerminalJobs` sweeps them like any
* other failed job) with no recovery record. A give-up here is lost to replay,
* exactly as before this change — that is what OFF means. Kept as a
* self-contained counterpart to `recordAndDeletePoisonPills` so this whole
* path, and the flag, can be deleted once recording is trusted in production.
*/
private async failPoisonPills(): Promise<string[]> {
const heartbeatCutoff = new Date(Date.now() - this.stallTimeoutMs)
const result = await this.pool.query<{ id: string }>(
`UPDATE cyclotron_jobs
SET status = 'failed', lock_id = NULL, last_heartbeat = NULL,
last_transition = NOW(), transition_count = LEAST(transition_count + 1, ${CYCLOTRON_COUNTER_MAX})
WHERE id IN (
SELECT id
FROM cyclotron_jobs
WHERE status = 'running'
AND COALESCE(last_heartbeat, $1) <= $1
AND janitor_touch_count >= $2
FOR UPDATE SKIP LOCKED
)
RETURNING id`,
[heartbeatCutoff, this.maxTouchCount]
)
const ids = result.rows.map((r) => r.id)
if (ids.length > 0) {
janitorPoisonedCounter.inc(ids.length)
logger.warn('CyclotronV2Janitor failed poison pill jobs (recovery disabled, no replay record)', {
count: ids.length,
})
}
return ids
}
/**
* Give up on poison pills: running jobs with a stale heartbeat that have
* been reset too many times. Each one is recorded as a `failed`, replayable
* invocation result (discoverable in the Invocations UI, re-runnable by the
* existing rerun tooling) and only then deleted — so there is never a window
* where the cyclotron row is gone but no recovery record exists. The
* original incident deleted these with no trace; here every give-up is
* logged with its id and recorded for replay.
*/
private async recordAndDeletePoisonPills(): Promise<string[]> {
const heartbeatCutoff = new Date(Date.now() - this.stallTimeoutMs)
// Wrapper/meta jobs (rerun, batch-resolve, any future non-invocation queue) share this table.
// A wrapper is not a replayable invocation — recording one as a `failed` result would give it
// `function_kind='hog_flow'` with the target's `function_id` (poisonRowToInvocation always tags
// hogFlow), making it indistinguishable from a genuine poisoned flow, and the autodrain would
// then rediscover and replay a real flow with fabricated globals. So give up on anything not on
// an invocation queue separately: delete it with NO record. This is self-healing — the work a
// wrapper was driving is re-derived on its next run.
await this.dropWrapperPoisonPills(heartbeatCutoff)
const result = await this.pool.query<PoisonRow>(
`SELECT id, team_id, function_id, queue_name, priority, scheduled, created,
parent_run_id, state, distinct_id, person_id, action_id,
janitor_touch_count, transition_count
FROM cyclotron_jobs
WHERE status = 'running'
AND COALESCE(last_heartbeat, $1) <= $1
AND janitor_touch_count >= $2
AND queue_name = ANY($4::text[])
ORDER BY last_transition ASC
LIMIT $3
FOR UPDATE SKIP LOCKED`,
[heartbeatCutoff, this.maxTouchCount, this.cleanupBatchSize, INVOCATION_QUEUE_NAMES]
)
if (result.rows.length === 0) {
return []
}
if (!this.invocationResults) {
// No way to record a recovery row — keep the jobs rather than drop
// them silently. resetStalledJobs will retry them this cycle.
janitorGiveUpSkippedCounter.inc(result.rows.length)
logger.warn('CyclotronV2Janitor cannot record poison pill recovery (no results service), keeping jobs', {
count: result.rows.length,
})
return []
}
const recordedIds: string[] = []
let skipped = 0
for (const row of result.rows) {
const invocation = this.poisonRowToInvocation(row)
const ok = await this.invocationResults.recordTerminalFailureDurably(invocation, {
error: `poison pill: stalled and reset ${row.janitor_touch_count} times without completing`,
errorKind: JANITOR_POISON_PILL_ERROR_KIND,
})
if (ok) {
recordedIds.push(row.id)
} else {
skipped++
}
}
if (skipped > 0) {
janitorGiveUpSkippedCounter.inc(skipped)
logger.warn('CyclotronV2Janitor kept poison pills it could not durably record', { count: skipped })
}
if (recordedIds.length === 0) {
return []
}
// Re-assert the FULL poison predicate (not just status='running') in the
// DELETE. Between the SELECT and here a row could have been reset to
// 'available' and re-dequeued by a worker — that re-dequeue stamps a
// fresh heartbeat, so the stale-heartbeat / touch-count guard no longer
// matches and we won't delete an actively-running job. RETURNING gives
// the rows actually removed, so the metric/log reflect real give-ups
// even if a concurrent janitor or worker raced us to some of them.
const deleted = await this.pool.query<{ id: string }>(
`DELETE FROM cyclotron_jobs
WHERE id = ANY($1::uuid[])
AND status = 'running'
AND COALESCE(last_heartbeat, $2) <= $2
AND janitor_touch_count >= $3
RETURNING id`,
[recordedIds, heartbeatCutoff, this.maxTouchCount]
)
const deletedIds = deleted.rows.map((r) => r.id)
if (deletedIds.length > 0) {
janitorPoisonedCounter.inc(deletedIds.length)
logger.warn('CyclotronV2Janitor gave up on poison pill jobs (recorded as failed, replayable)', {
count: deletedIds.length,
ids: deletedIds,
})
}
return deletedIds
}
/**
* Give up on poisoned wrapper/meta jobs — anything not on an invocation queue (rerun,
* batch-resolve, any future meta queue) — by deleting them with no replay record. See the
* rationale in recordAndDeletePoisonPills. A wrapper is not a replayable invocation, so there is
* nothing meaningful to record; dropping it is safe and self-healing (its work is re-derived on
* the next run).
*/
private async dropWrapperPoisonPills(heartbeatCutoff: Date): Promise<number> {
// Bound the delete to cleanupBatchSize and pick rows via FOR UPDATE SKIP
// LOCKED, mirroring the genuine-poison-pill path below. This runs first in
// the tick: an unbounded delete could both lock a large row set and block on
// a row a worker/other janitor holds, stalling the whole tick before genuine
// pills get recorded. Skipping locked rows and capping the batch keeps the
// wrapper drain from starving the rest of the sweep — leftovers drain next tick.
const deleted = await this.pool.query<{ id: string }>(
`DELETE FROM cyclotron_jobs
WHERE id IN (
SELECT id FROM cyclotron_jobs
WHERE status = 'running'
AND queue_name <> ALL($1::text[])
AND COALESCE(last_heartbeat, $2) <= $2
AND janitor_touch_count >= $3
ORDER BY last_transition ASC
LIMIT $4
FOR UPDATE SKIP LOCKED
)
RETURNING id`,
[INVOCATION_QUEUE_NAMES, heartbeatCutoff, this.maxTouchCount, this.cleanupBatchSize]
)
const count = deleted.rows.length
if (count > 0) {
janitorWrapperDroppedCounter.inc(count)
logger.warn('CyclotronV2Janitor dropped poisoned wrapper jobs (no replay record — meta-job)', {
count,
ids: deleted.rows.map((r) => r.id),
})
}
return count
}
// Turn a raw poisoned row into a hog flow invocation the results service can
// serialize. postgres-v2 backs hog flows, so we tag it as such — the stub
// `hogFlow` carries only the id the lifecycle row needs (function_id), while
// the rerun path rebuilds the full flow from the function id on replay.
private poisonRowToInvocation(row: PoisonRow): CyclotronJobInvocationHogFlow {
const job: CyclotronV2DequeuedJob = {
id: row.id,
teamId: row.team_id,
functionId: row.function_id,
queueName: row.queue_name,
priority: row.priority,
scheduled: DateTime.fromISO(row.scheduled, { zone: 'utc' }),
created: DateTime.fromISO(row.created, { zone: 'utc' }),
parentRunId: row.parent_run_id,
transitionCount: row.transition_count,
state: row.state,
distinctId: row.distinct_id,
personId: row.person_id,
actionId: row.action_id,
cancelRequestedAt: null,
ack: () => Promise.resolve(),
fail: () => Promise.resolve(),
reschedule: () => Promise.resolve(),
cancel: () => Promise.resolve(),
heartbeat: () => Promise.resolve(),
bulkCreateAndCheckIn: () => Promise.resolve({ newJobIds: [] }),
}
const invocation = v2JobToInvocation(job)
return { ...invocation, hogFlow: { id: invocation.functionId } } as CyclotronJobInvocationHogFlow
}
private async resetStalledJobs(): Promise<number> {
const heartbeatCutoff = new Date(Date.now() - this.stallTimeoutMs)
// Defer the reset job's next scheduled time so repeated stalls back off
// and a fleet-wide stall doesn't immediately re-flood the workers that
// just recovered. Two parts, both jittered to break a synchronized herd:
// 1. a small first-retry spread (0..FIRST_RETRY_SPREAD_MS) applied every
// reset — on the FIRST stall it's the whole delay, so a transient
// stall retries within seconds instead of a full backoff step;
// 2. an exponential term shifted by one (`2^touch_count - 1`), so it's 0
// on the first stall and grows ~base, ~3x base, ... on repeats,
// capped at max — half-jitter ([0.5, 1] x) on this part.
// Disabled when stallBackoffBaseMs <= 0 — scheduled is left untouched
// (immediate retry, the pre-backoff behavior).
const backoffEnabled = this.stallBackoffBaseMs > 0
const backoffClause = backoffEnabled
? `, scheduled = NOW() + (
$4::float8 * random()
+ LEAST($2::float8 * (POWER(2, janitor_touch_count) - 1), $3::float8) * (0.5 + 0.5 * random())
) * INTERVAL '1 millisecond'`
: ''
const params: (Date | number)[] = backoffEnabled
? [heartbeatCutoff, this.stallBackoffBaseMs, this.stallBackoffMaxMs, STALL_BACKOFF_FIRST_RETRY_SPREAD_MS]
: [heartbeatCutoff]
const result = await this.pool.query(
`WITH stalled AS (
SELECT id
FROM cyclotron_jobs
WHERE status = 'running'
AND COALESCE(last_heartbeat, $1) <= $1
FOR UPDATE SKIP LOCKED
)
UPDATE cyclotron_jobs
SET status = 'available', lock_id = NULL, last_heartbeat = NULL,
janitor_touch_count = LEAST(janitor_touch_count + 1, ${CYCLOTRON_COUNTER_MAX})${backoffClause}
FROM stalled
WHERE cyclotron_jobs.id = stalled.id`,
params
)
const count = result.rowCount ?? 0
if (count > 0) {
janitorStalledCounter.inc(count)
logger.info('CyclotronV2Janitor reset stalled jobs', { count })
}
return count
}
/**
* Deletes conversion watchers whose attribution window has closed.
*
* Nothing else removes them: a watcher that converts is deleted by the matcher's claim, and one
* that never converts would otherwise live forever. This runs here rather than in the matcher
* because every matcher pod would run its own copy, all deleting from the same table.
*
* A failure is reported to error tracking and swallowed: an expired watcher has already stopped
* matching via the `expires_at` predicate, so a missed sweep costs space and never a conversion,
* and it must not stop the janitor finishing its cyclotron work. The report matters because the
* catch hides the failure from the run otherwise — a persistent one grows the table unbounded.
*/
async sweepExpiredConversionWatchers(): Promise<number> {
try {
const result = await this.pool.query(
`DELETE FROM conversion_watchers
WHERE id IN (
SELECT id FROM conversion_watchers WHERE expires_at <= NOW() LIMIT $1
)`,
[this.cleanupBatchSize]
)
const deleted = result.rowCount ?? 0
if (deleted > 0) {
conversionWatchersSweptCounter.inc(deleted)
logger.info('CyclotronV2Janitor swept expired conversion watchers', { count: deleted })
}
return deleted
} catch (err) {
logger.error('CyclotronV2Janitor conversion watcher sweep failed', { error: String(err) })
captureException(err)
return 0
}
}
async sweepEmailAttachments(): Promise<number> {
try {
const result = await this.pool.query(
`DELETE FROM cyclotron_email_attachments
WHERE id IN (
SELECT id FROM cyclotron_email_attachments WHERE created_at < $1 LIMIT $2
FOR UPDATE SKIP LOCKED
)`,
[new Date(Date.now() - EMAIL_ATTACHMENTS_MAX_AGE_MS), this.cleanupBatchSize]
)
const swept = result.rowCount ?? 0
if (swept > 0) {
emailAttachmentOutcomesTotal.inc({ outcome: 'swept' }, swept)
logger.info('CyclotronV2Janitor swept email attachment rows', { count: swept })
}
const size = await this.pool.query<{ bytes: string; rows: string }>(
`SELECT pg_total_relation_size('cyclotron_email_attachments') AS bytes,
(SELECT GREATEST(reltuples, 0)::bigint FROM pg_class
WHERE oid = 'cyclotron_email_attachments'::regclass) AS rows`
)
emailAttachmentsTableBytesGauge.set(Number(size.rows[0].bytes))
emailAttachmentsRowsGauge.set(Number(size.rows[0].rows))
return swept
} catch (err) {
logger.error('CyclotronV2Janitor email attachment sweep failed', { error: String(err) })
captureException(err)
return 0
}
}
async measureQueueDepths(): Promise<Map<string, number>> {
const result = await this.pool.query<{ queue_name: string; count: string; oldest_due_age_ms: number }>(
`SELECT queue_name, COUNT(*) as count,
EXTRACT(EPOCH FROM NOW() - MIN(scheduled))::float8 * 1000 as oldest_due_age_ms
FROM cyclotron_jobs
WHERE status = 'available' AND scheduled <= NOW()
GROUP BY queue_name`
)
const depths = new Map<string, number>()
for (const row of result.rows) {
const count = parseInt(row.count, 10)
depths.set(row.queue_name, count)
this.seenQueues.add(row.queue_name)
queueDepthGauge.labels({ queue: row.queue_name }).set(count)
queueOldestDueAgeGauge.labels({ queue: row.queue_name }).set(row.oldest_due_age_ms)
}
// GROUP BY returns no row for a queue that is empty. Without the write below,
// that queue keeps the last depth this process set, and an alert on the value
// fires while the queue is idle. Write 0 for every queue this process saw
// before. The set holds only observed queue names, so it needs no maintenance
// and it adds no series for a queue that does not use this table.
for (const queue of this.seenQueues) {
if (!depths.has(queue)) {
queueDepthGauge.labels({ queue }).set(0)
queueOldestDueAgeGauge.labels({ queue }).set(0)
}
}
return depths
}
isRunning(): boolean {
return this.intervalHandle !== null
}
async stop(): Promise<void> {
if (this.intervalHandle) {
clearInterval(this.intervalHandle)
this.intervalHandle = null
}
await this.pool.end()
}
}