:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): time cyclotron worker phases and job lateness (#126203)
Jon McCallum9dab9c7eaa78
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.tsindex fbb7942ad887..d59df5980fb2 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts@@ -1,3 +1,5 @@+import { Histogram } from 'prom-client'+import { instrumented } from '~/common/tracing/tracing-utils'import { logger } from '~/common/utils/logger'import { captureException } from '~/common/utils/posthog'@@ -15,6 +17,14 @@ import { isLegacyPluginHogFunction, isNativeHogFunction, isSegmentPluginHogFunctimport { dualWrite } from '../utils/dual-store'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'+// instrumented_function_duration_seconds buckets start at 25 ms, which hides a change of a few ms.+const cyclotronWorkerPhaseDuration = new Histogram({+ name: 'cdp_cyclotron_worker_phase_duration_seconds',+ help: 'Wall time of one phase of a cyclotron worker batch. invocations blocks the dequeue loop, while background overlaps the next batch.',+ labelNames: ['queue', 'phase'],+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],+})+/*** CDP worker that consumes and processes hog function / hogflow jobs.* Receives its job queue backend via constructor injection.@@ -198,16 +208,23 @@ export class CdpCyclotronWorker<// batch would otherwise blow past stallTimeoutMs and get poisoned.const stopHeartbeat = this.startPeriodicHeartbeat(invocations)+ const endInvocations = cyclotronWorkerPhaseDuration.startTimer({ queue: this.queue, phase: 'invocations' })let invocationResults: CyclotronJobInvocationResult[]try {invocationResults = await this.processInvocations(invocations)} catch (e) {stopHeartbeat()throw e+ } finally {+ endInvocations()}// NOTE: We can queue and publish all metrics in the background whilst processing the next batch of invocations- const backgroundTask = this.runBackgroundTasks(invocationResults).finally(stopHeartbeat)+ const endBackground = cyclotronWorkerPhaseDuration.startTimer({ queue: this.queue, phase: 'background' })+ const backgroundTask = this.runBackgroundTasks(invocationResults).finally(() => {+ endBackground()+ stopHeartbeat()+ })return { backgroundTask, invocationResults }}diff --git a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.tsindex 1d7822536967..656f987c855f 100644--- a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts+++ b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts@@ -1,5 +1,5 @@import { chunk } from 'lodash'-import { Gauge } from 'prom-client'+import { Gauge, Histogram } from 'prom-client'import { instrumentFn } from '~/common/tracing/tracing-utils'import { parseJSON } from '~/common/utils/json-parse'@@ -18,6 +18,13 @@ const pendingJobsGauge = new Gauge({help: 'Number of postgres-v2 jobs currently held in memory awaiting ack/fail/reschedule',})+const jobLatenessHistogram = new Histogram({+ name: 'cdp_cyclotron_job_lateness_seconds',+ help: 'Seconds between a postgres-v2 job becoming due and a worker dequeuing it',+ labelNames: ['queue'],+ buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900],+})+/*** State blob stored in the single `state` BYTEA column.* Mirrors the Kafka serialization: everything in one JSON object.@@ -96,10 +103,13 @@ export class CyclotronJobQueuePostgresV2 implements JobQueue {await this.worker.connect(async (jobs) => {const invocations: CyclotronJobInvocation[] = []+ const now = Date.now()+ const jobLateness = jobLatenessHistogram.labels({ queue })for (const job of jobs) {this.pendingJobs.set(job.id, job)invocations.push(v2JobToInvocation(job))+ jobLateness.observe(Math.max(0, now - job.scheduled.toMillis()) / 1000)}pendingJobsGauge.set(this.pendingJobs.size)diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.tsindex 65c58f842fb5..8e58ec6ca1ae 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts@@ -35,6 +35,12 @@ const hogFunctionMonitoringPendingMessages = new Gauge({help: 'Number of log entries queued and waiting to be flushed to Kafka. App-metric backlog is tracked separately by app_metrics_aggregator_queued_total / app_metrics_aggregator_flushed_total. High values indicate accumulation and potential memory leak.',})+const hogFunctionMonitoringFlushDuration = new Histogram({+ name: 'cdp_hog_function_monitoring_flush_duration_seconds',+ help: 'Wall time of one flush of queued app metrics and log entries',+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],+})+export type MonitoringOutput = AppMetricsOutput | LogEntriesOutput// `app_metrics2` has no version column, and it can't gain one: it's an AggregatingMergeTree whose sort@@ -79,6 +85,7 @@ export class HogFunctionMonitoringService {}async flush() {+ const end = hogFunctionMonitoringFlushDuration.startTimer()const messages = [...this.queuedLogMessages]this.queuedLogMessages = []hogFunctionMonitoringPendingMessages.set(0)@@ -107,7 +114,7 @@ export class HogFunctionMonitoringService {captureException(error)})}),- ])+ ]).finally(end)}queueAppMetric(metric: MinimalAppMetric, source: MetricLogSource) {