:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Counter, Gauge, Histogram } from 'prom-client'
import { buildIntegerMatcher } from '~/common/config/config'
import { ValueMatcher } from '../../../types'
import { CdpConfig } from '../../config'
import {
CyclotronJobInvocation,
CyclotronJobInvocationResult,
CyclotronJobQueueKind,
CyclotronJobQueueSource,
} from '../../types'
export const cdpJobSizeKb = new Histogram({
name: 'cdp_cyclotron_job_size_kb',
help: 'The size in kb of the jobs we are processing',
buckets: [0, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, Infinity],
labelNames: ['queue_kind'],
})
export const cdpJobSizeCompressedKb = new Histogram({
name: 'cdp_cyclotron_job_size_compressed_kb',
help: 'The size in kb of the compressed jobs we are processing',
buckets: [0, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, Infinity],
labelNames: ['queue_kind'],
})
const cdpCyclotronBatchUtilization = new Gauge({
name: 'cdp_cyclotron_batch_utilization',
help: 'Indicates how big batches are we are processing compared to the max batch size. Useful as a scaling metric',
labelNames: ['queue', 'source'],
})
const cdpCyclotronJobsProcessed = new Counter({
name: 'cdp_cyclotron_jobs_processed',
help: 'The number of jobs we are managing to process',
labelNames: ['queue', 'source'],
})
/**
* Records throughput and batch utilization for a consumed batch.
* `cdp_cyclotron_batch_utilization` is consumed by KEDA to autoscale the cyclotron workers, so this
* must be called on every batch — including empty ones — so idle workers report zero and can scale down.
*/
export function observeConsumedBatch(params: {
queue: CyclotronJobQueueKind
source: CyclotronJobQueueSource
batchSize: number
maxBatchSize: number
}): void {
const { queue, source, batchSize, maxBatchSize } = params
cdpCyclotronBatchUtilization.labels({ queue, source }).set(maxBatchSize > 0 ? batchSize / maxBatchSize : 0)
cdpCyclotronJobsProcessed.inc({ queue, source }, batchSize)
}
/**
* Strip transient data from invocation state before persisting.
* Groups are always stripped. Person is stripped when stripPerson is true.
* These are large and easily reloaded by the worker, so we avoid storing them.
* Returns a new object if modifications are needed, otherwise the original.
*/
export function sanitizeInvocationForPersistence(
invocation: CyclotronJobInvocation,
{ stripPerson }: { stripPerson?: boolean } = {}
): CyclotronJobInvocation {
const globals = invocation.state?.globals
if (!globals) {
return invocation
}
const hasGroups = globals.groups && Object.keys(globals.groups).length > 0
const hasPerson = stripPerson && globals.person
if (!hasGroups && !hasPerson) {
return invocation
}
const { groups: _g, person: _p, ...restGlobals } = globals
const newGlobals: typeof globals = { ...restGlobals }
if (!hasPerson && globals.person) {
newGlobals.person = globals.person
}
return {
...invocation,
state: {
...invocation.state,
globals: newGlobals,
},
}
}
/**
* Creates a sanitizer that strips transient data before persisting.
* Call once at construction time, use the returned functions on every queue operation.
*/
export function createInvocationSanitizer(config: Pick<CdpConfig, 'CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS'>) {
const stripPersonMatcher: ValueMatcher<number> = buildIntegerMatcher(
config.CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS,
true
)
return {
sanitizeInvocations(invocations: CyclotronJobInvocation[]): CyclotronJobInvocation[] {
return invocations.map((inv) =>
sanitizeInvocationForPersistence(inv, { stripPerson: stripPersonMatcher(inv.teamId) })
)
},
sanitizeResults(results: CyclotronJobInvocationResult[]): CyclotronJobInvocationResult[] {
return results.map((result) => ({
...result,
invocation: sanitizeInvocationForPersistence(result.invocation, {
stripPerson: stripPersonMatcher(result.invocation.teamId),
}),
}))
},
}
}