Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): time cyclotron worker phases and job lateness (#126203)

Jon McCallum9dab9c7eaa78
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts
index fbb7942ad887..d59df5980fb2 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts
@@ -1,3 +1,5 @@
+import { Histogram } from 'prom-client'
+
import { instrumented } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
@@ -15,6 +17,14 @@ import { isLegacyPluginHogFunction, isNativeHogFunction, isSegmentPluginHogFunct
import { dualWrite } from '../utils/dual-store'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
+// instrumented_function_duration_seconds buckets start at 25 ms, which hides a change of a few ms.
+const cyclotronWorkerPhaseDuration = new Histogram({
+ name: 'cdp_cyclotron_worker_phase_duration_seconds',
+ help: 'Wall time of one phase of a cyclotron worker batch. invocations blocks the dequeue loop, while background overlaps the next batch.',
+ labelNames: ['queue', 'phase'],
+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+})
+
/**
* CDP worker that consumes and processes hog function / hogflow jobs.
* Receives its job queue backend via constructor injection.
@@ -198,16 +208,23 @@ export class CdpCyclotronWorker<
// batch would otherwise blow past stallTimeoutMs and get poisoned.
const stopHeartbeat = this.startPeriodicHeartbeat(invocations)
+ const endInvocations = cyclotronWorkerPhaseDuration.startTimer({ queue: this.queue, phase: 'invocations' })
let invocationResults: CyclotronJobInvocationResult[]
try {
invocationResults = await this.processInvocations(invocations)
} catch (e) {
stopHeartbeat()
throw e
+ } finally {
+ endInvocations()
}
// NOTE: We can queue and publish all metrics in the background whilst processing the next batch of invocations
- const backgroundTask = this.runBackgroundTasks(invocationResults).finally(stopHeartbeat)
+ const endBackground = cyclotronWorkerPhaseDuration.startTimer({ queue: this.queue, phase: 'background' })
+ const backgroundTask = this.runBackgroundTasks(invocationResults).finally(() => {
+ endBackground()
+ stopHeartbeat()
+ })
return { backgroundTask, invocationResults }
}
diff --git a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
index 1d7822536967..656f987c855f 100644
--- a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
+++ b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
@@ -1,5 +1,5 @@
import { chunk } from 'lodash'
-import { Gauge } from 'prom-client'
+import { Gauge, Histogram } from 'prom-client'
import { instrumentFn } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
@@ -18,6 +18,13 @@ const pendingJobsGauge = new Gauge({
help: 'Number of postgres-v2 jobs currently held in memory awaiting ack/fail/reschedule',
})
+const jobLatenessHistogram = new Histogram({
+ name: 'cdp_cyclotron_job_lateness_seconds',
+ help: 'Seconds between a postgres-v2 job becoming due and a worker dequeuing it',
+ labelNames: ['queue'],
+ buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900],
+})
+
/**
* State blob stored in the single `state` BYTEA column.
* Mirrors the Kafka serialization: everything in one JSON object.
@@ -96,10 +103,13 @@ export class CyclotronJobQueuePostgresV2 implements JobQueue {
await this.worker.connect(async (jobs) => {
const invocations: CyclotronJobInvocation[] = []
+ const now = Date.now()
+ const jobLateness = jobLatenessHistogram.labels({ queue })
for (const job of jobs) {
this.pendingJobs.set(job.id, job)
invocations.push(v2JobToInvocation(job))
+ jobLateness.observe(Math.max(0, now - job.scheduled.toMillis()) / 1000)
}
pendingJobsGauge.set(this.pendingJobs.size)
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
index 65c58f842fb5..8e58ec6ca1ae 100644
--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
@@ -35,6 +35,12 @@ const hogFunctionMonitoringPendingMessages = new Gauge({
help: 'Number of log entries queued and waiting to be flushed to Kafka. App-metric backlog is tracked separately by app_metrics_aggregator_queued_total / app_metrics_aggregator_flushed_total. High values indicate accumulation and potential memory leak.',
})
+const hogFunctionMonitoringFlushDuration = new Histogram({
+ name: 'cdp_hog_function_monitoring_flush_duration_seconds',
+ help: 'Wall time of one flush of queued app metrics and log entries',
+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+})
+
export type MonitoringOutput = AppMetricsOutput | LogEntriesOutput
// `app_metrics2` has no version column, and it can't gain one: it's an AggregatingMergeTree whose sort
@@ -79,6 +85,7 @@ export class HogFunctionMonitoringService {
}
async flush() {
+ const end = hogFunctionMonitoringFlushDuration.startTimer()
const messages = [...this.queuedLogMessages]
this.queuedLogMessages = []
hogFunctionMonitoringPendingMessages.set(0)
@@ -107,7 +114,7 @@ export class HogFunctionMonitoringService {
captureException(error)
})
}),
- ])
+ ]).finally(end)
}
queueAppMetric(metric: MinimalAppMetric, source: MetricLogSource) {