Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.4 KB
Raw Blame
import { chunk } from 'lodash'
import { Gauge, Histogram } from 'prom-client'

import { instrumentFn } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'

import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk } from '../../../types'
import { CdpConfig } from '../../config'
import { CyclotronJobInvocation, CyclotronJobInvocationResult, CyclotronJobQueueKind } from '../../types'
import { CyclotronV2DequeuedJob, CyclotronV2JobInit, CyclotronV2Manager, CyclotronV2Worker } from '../cyclotron-v2'
import { CyclotronV2WorkerConfig } from '../cyclotron-v2/types'
import { JobQueue } from './job-queue.interface'
import { cdpJobSizeCompressedKb, cdpJobSizeKb, createInvocationSanitizer, observeConsumedBatch } from './shared'

const pendingJobsGauge = new Gauge({
    name: 'cdp_cyclotron_v2_pending_jobs',
    help: 'Number of postgres-v2 jobs currently held in memory awaiting ack/fail/reschedule',
})

const jobLatenessHistogram = new Histogram({
    name: 'cdp_cyclotron_job_lateness_seconds',
    help: 'Seconds between a postgres-v2 job becoming due and a worker dequeuing it',
    labelNames: ['queue'],
    buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900],
})

/**
 * State blob stored in the single `state` BYTEA column.
 * Mirrors the Kafka serialization: everything in one JSON object.
 */
type SerializedJobState = {
    state: CyclotronJobInvocation['state']
    queueParameters?: CyclotronJobInvocation['queueParameters']
    queueMetadata?: CyclotronJobInvocation['queueMetadata']
}

export class CyclotronJobQueuePostgresV2 implements JobQueue {
    private manager?: CyclotronV2Manager
    private worker?: CyclotronV2Worker
    private pendingJobs = new Map<string, CyclotronV2DequeuedJob>()
    private sanitizer: ReturnType<typeof createInvocationSanitizer>

    constructor(
        // protected — the rate-limited subclass needs this to cap its token
        // claim at the actual batch size we can process (avoids deducting
        // tokens we'd never use).
        protected consumerBatchSize: number,
        private config: Pick<
            CdpConfig,
            | 'CYCLOTRON_NODE_DATABASE_URL'
            | 'CYCLOTRON_SHARD_DEPTH_LIMIT'
            | 'CDP_CYCLOTRON_BATCH_DELAY_MS'
            | 'CDP_CYCLOTRON_INSERT_MAX_BATCH_SIZE'
            | 'CDP_CYCLOTRON_INSERT_PARALLEL_BATCHES'
            | 'CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS'
        >
    ) {
        this.sanitizer = createInvocationSanitizer(config)
    }

    public async startAsProducer(): Promise<void> {
        if (this.manager) {
            return
        }
        if (!this.config.CYCLOTRON_NODE_DATABASE_URL) {
            throw new Error('Cyclotron V2 database URL not set')
        }

        this.manager = new CyclotronV2Manager({
            pool: { dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL },
            depthLimit: this.config.CYCLOTRON_SHARD_DEPTH_LIMIT,
        })
        await this.manager.connect()
    }

    /**
     * Factory hook for the worker. Subclasses override this to plug in a
     * different worker class (e.g. CyclotronV2RateLimitedWorker) without
     * having to reimplement `startAsConsumer`.
     */
    protected createWorker(workerConfig: CyclotronV2WorkerConfig): CyclotronV2Worker {
        return new CyclotronV2Worker(workerConfig)
    }

    public async startAsConsumer(
        queue: CyclotronJobQueueKind,
        consumeBatch: (invocations: CyclotronJobInvocation[]) => Promise<{ backgroundTask: Promise<any> }>
    ): Promise<void> {
        if (!this.config.CYCLOTRON_NODE_DATABASE_URL) {
            throw new Error('Cyclotron V2 database URL not set')
        }

        await this.startAsProducer()

        this.worker = this.createWorker({
            pool: { dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL },
            queueName: queue,
            batchMaxSize: this.consumerBatchSize,
            pollDelayMs: this.config.CDP_CYCLOTRON_BATCH_DELAY_MS,
            includeEmptyBatches: true,
        })

        await this.worker.connect(async (jobs) => {
            const invocations: CyclotronJobInvocation[] = []
            const now = Date.now()
            const jobLateness = jobLatenessHistogram.labels({ queue })

            for (const job of jobs) {
                this.pendingJobs.set(job.id, job)
                invocations.push(v2JobToInvocation(job))
                jobLateness.observe(Math.max(0, now - job.scheduled.toMillis()) / 1000)
            }

            pendingJobsGauge.set(this.pendingJobs.size)

            observeConsumedBatch({
                queue,
                source: 'postgres-v2',
                batchSize: invocations.length,
                maxBatchSize: this.consumerBatchSize,
            })

            await consumeBatch(invocations)

            pendingJobsGauge.set(this.pendingJobs.size)
        })
    }

    public async stopConsumer(): Promise<void> {
        await this.worker?.disconnect()
        this.worker = undefined
    }

    public async stopProducer(): Promise<void> {
        await this.manager?.disconnect()
        this.manager = undefined
    }

    public isHealthy(): HealthCheckResult {
        if (!this.worker) {
            return new HealthCheckResultError('CyclotronV2Worker not initialized', {})
        }
        if (this.worker.isHealthy()) {
            return new HealthCheckResultOk()
        }
        return new HealthCheckResultError('CyclotronV2Worker is not healthy', {})
    }

    public async queueInvocations(
        invocations: CyclotronJobInvocation[],
        options: { overwriteExisting?: boolean } = {}
    ): Promise<void> {
        if (invocations.length === 0) {
            return
        }

        if (!this.manager) {
            throw new Error('CyclotronV2Manager not initialized')
        }

        invocations = this.sanitizer.sanitizeInvocations(invocations)

        const jobs = invocations.map((inv) => ({
            ...invocationToV2JobInit(inv),
            overwriteExisting: options.overwriteExisting,
        }))

        try {
            const chunked = chunk(jobs, this.config.CDP_CYCLOTRON_INSERT_MAX_BATCH_SIZE)
            if (this.config.CDP_CYCLOTRON_INSERT_PARALLEL_BATCHES) {
                await Promise.all(
                    chunked.map((batch) =>
                        instrumentFn(
                            {
                                key: 'cyclotron_v2.bulk_create_jobs',
                                sendException: false,
                                getLoggingContext: () => ({ batchSize: batch.length, parallel: true }),
                            },
                            () => this.manager!.bulkCreateJobs(batch)
                        )
                    )
                )
            } else {
                for (const batch of chunked) {
                    await instrumentFn(
                        {
                            key: 'cyclotron_v2.bulk_create_jobs',
                            sendException: false,
                            getLoggingContext: () => ({ batchSize: batch.length, parallel: false }),
                        },
                        () => this.manager!.bulkCreateJobs(batch)
                    )
                }
            }
        } catch (e) {
            logger.error('Error creating cyclotron V2 jobs', { error: String(e) })
            throw e
        }
    }

    public async queueInvocationResults(invocationResults: CyclotronJobInvocationResult[]): Promise<void> {
        const sanitizedResults = this.sanitizer.sanitizeResults(invocationResults)

        await Promise.all(
            sanitizedResults.map(async (result) => {
                const job = this.pendingJobs.get(result.invocation.id)
                if (!job) {
                    logger.warn('No pending V2 job found for result, creating new job', {
                        id: result.invocation.id,
                    })
                    if (!result.finished && !result.error) {
                        await this.queueInvocations([result.invocation])
                    }
                    return
                }

                this.pendingJobs.delete(result.invocation.id)

                if (result.error) {
                    await job.fail()
                } else if (result.canceled) {
                    await job.cancel()
                } else if (result.finished) {
                    await job.ack()
                } else {
                    const stateBuffer = serializeState(result.invocation)
                    await job.reschedule({
                        state: stateBuffer,
                        scheduledAt: result.invocation.queueScheduledAt?.toJSDate(),
                        distinctId: extractDistinctId(result.invocation),
                        personId: extractPersonId(result.invocation),
                        actionId: extractActionId(result.invocation),
                        queueName: result.invocation.queue,
                        // Persisted only across email-queue transitions: entering carries the
                        // class routeEmailToQueue assigned, leaving restores the origin
                        // priority. Elsewhere the row keeps its insert-time priority, so the
                        // in-memory retry bumps on other queues stay unpersisted as before.
                        priority:
                            result.invocation.queue === 'email' || job.queueName === 'email'
                                ? result.invocation.queuePriority
                                : undefined,
                    })
                }
            })
        )
    }

    public async dequeueInvocations(invocations: CyclotronJobInvocation[]): Promise<void> {
        await Promise.all(
            invocations.map(async (inv) => {
                const job = this.pendingJobs.get(inv.id)
                if (job) {
                    this.pendingJobs.delete(inv.id)
                    await job.fail()
                }
            })
        )
    }

    public async releaseInvocations(invocations: CyclotronJobInvocation[]): Promise<void> {
        await Promise.all(
            invocations.map(async (inv) => {
                const job = this.pendingJobs.get(inv.id)
                if (job) {
                    this.pendingJobs.delete(inv.id)
                    await job.ack()
                }
            })
        )
    }

    public async heartbeatInvocations(invocations: CyclotronJobInvocation[]): Promise<void> {
        await Promise.all(
            invocations.map(async (inv) => {
                const job = this.pendingJobs.get(inv.id)
                if (!job) {
                    return
                }
                try {
                    await job.heartbeat()
                } catch (err) {
                    const message = err instanceof Error ? err.message : String(err)
                    if (message.includes('already released')) {
                        // Benign race with ack/fail/reschedule flipping `released` on the wrapper.
                        logger.debug('CyclotronV2 heartbeat skipped for released job', { id: inv.id })
                    } else {
                        // Real failure (connection error, pool exhaustion, query timeout) —
                        // surface it so we can act. Don't rethrow: the tick continues for
                        // the other jobs and the interval keeps firing so a transient blip
                        // doesn't kill the batch.
                        logger.warn('CyclotronV2 heartbeat failed', { id: inv.id, error: message })
                    }
                }
            })
        )
    }
}

function serializeState(invocation: CyclotronJobInvocation): Buffer {
    const blob: SerializedJobState = {
        state: invocation.state,
        queueParameters: invocation.queueParameters ?? undefined,
        queueMetadata: invocation.queueMetadata ?? undefined,
    }
    return Buffer.from(JSON.stringify(blob))
}

export function invocationToV2JobInit(invocation: CyclotronJobInvocation): CyclotronV2JobInit {
    const state = serializeState(invocation)
    cdpJobSizeKb.labels('postgres-v2').observe(state.length / 1024)
    cdpJobSizeCompressedKb.labels('postgres-v2').observe(state.length / 1024)

    return {
        id: invocation.id,
        teamId: invocation.teamId,
        functionId: invocation.functionId,
        queueName: invocation.queue,
        priority: invocation.queuePriority,
        scheduled: invocation.queueScheduledAt?.toJSDate() ?? new Date(),
        parentRunId: invocation.parentRunId ?? null,
        state,
        distinctId: extractDistinctId(invocation),
        personId: extractPersonId(invocation),
        actionId: extractActionId(invocation),
    }
}

type LookupColumnSource = {
    person?: { id?: string }
    state?: {
        event?: { distinct_id?: string }
        personId?: string
        currentAction?: { id?: string }
    } | null
}

export function extractDistinctId(invocation: CyclotronJobInvocation): string | null {
    return (invocation as LookupColumnSource).state?.event?.distinct_id || null
}

export function extractPersonId(invocation: CyclotronJobInvocation): string | null {
    const inv = invocation as LookupColumnSource
    return inv.person?.id || inv.state?.personId || null
}

export function extractActionId(invocation: CyclotronJobInvocation): string | null {
    return (invocation as LookupColumnSource).state?.currentAction?.id || null
}

export function v2JobToInvocation(job: CyclotronV2DequeuedJob): CyclotronJobInvocation {
    let parsed: SerializedJobState = { state: null }

    if (job.state) {
        try {
            parsed = parseJSON(job.state.toString('utf-8'))
        } catch (e) {
            logger.error('Error parsing V2 job state', { error: String(e), jobId: job.id })
        }
    }

    const invocation: CyclotronJobInvocation = {
        id: job.id,
        teamId: job.teamId,
        functionId: job.functionId ?? '',
        queue: job.queueName as CyclotronJobQueueKind,
        queuePriority: job.priority,
        queueScheduledAt: job.scheduled ?? undefined,
        queueCreatedAt: job.created,
        queueDueAt: job.scheduled ?? undefined,
        queueMetadata: parsed.queueMetadata ?? undefined,
        queueParameters: parsed.queueParameters ?? undefined,
        state: parsed.state,
        queueSource: 'postgres-v2',
    }

    if (job.parentRunId) {
        invocation.parentRunId = job.parentRunId
    }

    if (job.cancelRequestedAt) {
        invocation.cancelRequestedAt = job.cancelRequestedAt
    }

    return invocation
}