:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { ClickHouseClient, createClient as createClickHouseClient } from '@clickhouse/client'
import https from 'https'
import { Counter } from 'prom-client'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk, PluginsServerConfig } from '../../types'
import { RERUN_PAGE_ERROR_BACKOFF_MAX_MS, RERUN_QUEUE_NAME, RerunJobState } from '../rerun/rerun-job.types'
import { RerunJobQueues, RerunPaginatorService } from '../rerun/rerun-paginator.service'
import { CyclotronV2Worker } from '../services/cyclotron-v2'
import { CyclotronV2DequeuedJob } from '../services/cyclotron-v2/types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
// Heartbeat interval — the cyclotron-v2 lock timeout is 30s by default, send
// roughly twice that often so a slow ClickHouse page doesn't lose the lock.
const HEARTBEAT_INTERVAL_MS = 10_000
// Delay between pages of the same rerun job. Keeps a long rerun from hot-looping
// the worker; cyclotron-v2 reschedule + dequeue overhead absorbs the wait anyway.
const RERUN_PAGE_DELAY_MS = 500
/**
* Delay before retrying a page that errored, doubling per consecutive error up
* to `RERUN_PAGE_ERROR_BACKOFF_MAX_MS`. A healthy page (streak 0) keeps the
* flat `RERUN_PAGE_DELAY_MS`, so only the failing case slows down.
*/
export function pageRetryDelayMs(consecutiveErrors: number): number {
if (consecutiveErrors <= 0) {
return RERUN_PAGE_DELAY_MS
}
return Math.min(RERUN_PAGE_DELAY_MS * 2 ** consecutiveErrors, RERUN_PAGE_ERROR_BACKOFF_MAX_MS)
}
const counterRerunJobsAcked = new Counter({
name: 'cdp_hog_invocation_rerun_jobs_acked_total',
help: 'Rerun wrapper jobs terminated by the worker, by outcome.',
labelNames: ['outcome'],
})
/**
* Consumes the cyclotron-v2 'rerun' queue and drives the rerun paginator.
*
* Each dequeued job carries a `RerunJobState` blob describing one user's
* rerun request (by ids or by filter). The worker:
* 1. Parses the state.
* 2. Runs one page of ClickHouse + rehydrate + enqueue work via `RerunPaginatorService`.
* 3. If `progress.done` → `ack()` the wrapper job (terminal).
* Otherwise → `reschedule({ state: updatedState, scheduledAt: now + RERUN_PAGE_DELAY_MS })`
* so the next worker iteration picks it back up.
*
* Deploy with `PLUGIN_SERVER_MODE=cdp-rerun-worker`. Multiple replicas are
* safe — cyclotron-v2 uses `FOR UPDATE SKIP LOCKED`.
*/
export class CdpRerunWorkerConsumer extends CdpConsumerBase<PluginsServerConfig> {
protected name = 'CdpRerunWorkerConsumer'
private worker: CyclotronV2Worker | null = null
private jobQueues: RerunJobQueues
private paginator: RerunPaginatorService | null = null
private clickhouseClient: ClickHouseClient | null = null
constructor(config: PluginsServerConfig, deps: CdpConsumerBaseDeps, jobQueues: RerunJobQueues) {
super(config, deps)
// Used by the paginator to re-enqueue rehydrated invocations as it
// pages — hog functions to kafka, hog flows to postgres-v2.
this.jobQueues = jobQueues
}
override async start(): Promise<void> {
if (!this.config.CYCLOTRON_NODE_DATABASE_URL) {
throw new Error('CYCLOTRON_NODE_DATABASE_URL is required for the rerun worker')
}
await this.jobQueues.hog_function.startAsProducer()
await this.jobQueues.hog_flow.startAsProducer()
// Dedicated ClickHouse client for the paginator. Internal ClickHouse
// uses self-signed certs with a hostname mismatch, same as the
// session-replay recording-api client.
const chScheme = this.config.CLICKHOUSE_SECURE ? 'https' : 'http'
const chPort = this.config.CLICKHOUSE_SECURE ? 8443 : 8123
this.clickhouseClient = createClickHouseClient({
url: `${chScheme}://${this.config.CLICKHOUSE_HOST}:${chPort}`,
username: this.config.CLICKHOUSE_USER,
password: this.config.CLICKHOUSE_PASSWORD || undefined,
database: this.config.CLICKHOUSE_DATABASE,
request_timeout: 60_000,
max_open_connections: 10,
...(this.config.CLICKHOUSE_SECURE
? {
http_agent: new https.Agent({ rejectUnauthorized: false, keepAlive: true, maxSockets: 10 }), // nosemgrep: problem-based-packs.insecure-transport.js-node.bypass-tls-verification.bypass-tls-verification
}
: {}),
})
this.paginator = new RerunPaginatorService(
this.clickhouseClient,
this.hogFunctionManager,
this.hogFlowManager,
this.invocationResultsService.invocationResultsRowsService,
this.jobQueues,
this.invocationResultsService.monitoringService,
this.config.HOG_INVOCATION_RERUN_MAX_COUNT
)
this.worker = new CyclotronV2Worker({
pool: { dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL },
queueName: RERUN_QUEUE_NAME,
// Rerun jobs are heavy (each runs a full ClickHouse query) — the
// default of 1 pulls them one at a time so a single replica can be
// deployed at high concurrency without overwhelming ClickHouse.
// Tunable via CDP_RERUN_WORKER_BATCH_SIZE.
batchMaxSize: this.config.CDP_RERUN_WORKER_BATCH_SIZE,
pollDelayMs: 1000,
})
await this.worker.connect((jobs) => this.handleBatch(jobs))
logger.info('🎬', 'CdpRerunWorkerConsumer started', { queue: RERUN_QUEUE_NAME })
}
override async stop(): Promise<void> {
this.isStopping = true
await this.worker?.disconnect()
await this.jobQueues.hog_function.stopProducer()
await this.jobQueues.hog_flow.stopProducer()
await this.clickhouseClient?.close()
await this.invocationResultsService.flush()
// This consumer overrides stop() without calling super.stop(), so close the conversion-watcher
// pool here too rather than relying on the base teardown.
await this.invocationResultsService.stop()
}
override isHealthy(): HealthCheckResult {
if (this.isStopping) {
return new HealthCheckResultError('CdpRerunWorkerConsumer is stopping', {})
}
if (!this.worker || !this.worker.isHealthy()) {
return new HealthCheckResultError('CdpRerunWorkerConsumer worker not healthy', {})
}
return new HealthCheckResultOk()
}
override get service() {
return {
id: 'cdp-rerun-worker',
onShutdown: () => this.stop(),
healthcheck: () => this.isHealthy(),
}
}
private async handleBatch(jobs: CyclotronV2DequeuedJob[]): Promise<void> {
// Usually one job at a time (CDP_RERUN_WORKER_BATCH_SIZE defaults to 1),
// but the loop handles a larger batch if the env is tuned up.
for (const job of jobs) {
await this.handleJob(job)
}
}
private async handleJob(job: CyclotronV2DequeuedJob): Promise<void> {
if (!this.paginator) {
await job.fail()
return
}
const state = this.parseState(job)
if (!state) {
logger.error('Rerun job has malformed state — failing', { job_id: job.id })
counterRerunJobsAcked.labels('malformed').inc()
await job.fail()
return
}
// Heartbeat the lock while the page is running. ClickHouse queries
// can take longer than the 30s default lock timeout for big windows.
const heartbeat = setInterval(() => {
job.heartbeat().catch((e) => {
logger.warn('Failed to heartbeat rerun job', { job_id: job.id, error: String(e) })
})
}, HEARTBEAT_INTERVAL_MS)
const context = { jobId: job.id, createdAt: job.created }
try {
const outcome = await this.paginator.processPage(job.teamId, state, context)
const nextState = outcome.state
if (nextState.progress.done) {
logger.info('🎬', 'Rerun job complete', {
job_id: job.id,
team_id: job.teamId,
queued: nextState.progress.queued,
skipped: nextState.progress.skipped,
})
counterRerunJobsAcked.labels('done').inc()
await job.ack()
return
}
// Persist progress + reschedule for the next page.
await job.reschedule({
scheduledAt: new Date(Date.now() + pageRetryDelayMs(nextState.progress.consecutive_errors ?? 0)),
state: Buffer.from(JSON.stringify(nextState)),
})
} catch (e) {
logger.error('Rerun worker error processing job — failing', {
job_id: job.id,
error: e instanceof Error ? e.message : String(e),
})
captureException(e)
counterRerunJobsAcked.labels('error').inc()
// Surface the terminal failure on the Invocations tab before we
// ack the cyclotron job — otherwise the wrapper row would be left
// stuck on `status='running'` forever.
await this.paginator?.writeWrapperFailure(job.teamId, state, context, e).catch((logErr) => {
logger.error('Rerun worker failed to write wrapper failure row', {
job_id: job.id,
error: logErr instanceof Error ? logErr.message : String(logErr),
})
})
await job.fail()
} finally {
clearInterval(heartbeat)
}
}
private parseState(job: CyclotronV2DequeuedJob): RerunJobState | null {
if (!job.state) {
return null
}
try {
return parseJSON(job.state.toString('utf8')) as RerunJobState
} catch {
return null
}
}
}