:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { RedisV2 } from '../../common/redis/redis-v2'
import { KeyedRateLimitRequest, KeyedRateLimiterService } from '../../common/services/keyed-rate-limiter.service'
import { QuotaLimiting } from '../../common/services/quota-limiting.service'
import { CdpValkeyShadowPools } from '../cdp-services'
import { counterHogFunctionStateOnEvent, counterRateLimited } from '../consumers/metrics'
import { shouldBlockInvocationDueToQuota } from '../consumers/quota-limiting-helper'
import {
CyclotronJobInvocation,
CyclotronJobInvocationHogFunction,
HogFunctionInvocationGlobals,
HogFunctionType,
HogFunctionTypeType,
MinimalAppMetric,
} from '../types'
import { dualRead } from '../utils/dual-store'
import { buildHogFunctionInvocations } from '../utils/invocation-utils'
import { InvocationFailureSink } from './dead-letter/cdp-dead-letter.service'
import { HogInputsService } from './hog-inputs.service'
import { HogFunctionManagerService } from './managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'
import { HogMaskerService } from './monitoring/hog-masker.service'
import { HogWatcherService, HogWatcherState, sameWatcherStates } from './monitoring/hog-watcher.service'
import { CdpUsageReporterService } from './usage/cdp-usage-reporter.service'
export interface HogFunctionInvocationPipelineConfig {
CDP_RATE_LIMITER_BUCKET_SIZE: number
CDP_RATE_LIMITER_REFILL_RATE: number
CDP_RATE_LIMITER_TTL: number
CDP_OVERFLOW_QUEUE_ENABLED: boolean
}
export interface HogFunctionInvocationPipelineDeps {
hogFunctionManager: HogFunctionManagerService
hogInputsService: HogInputsService
hogWatcher: HogWatcherService
hogWatcherMirror: HogWatcherService
hogMasker: HogMaskerService
hogFunctionMonitoringService: HogFunctionMonitoringService
cdpUsageReporter?: CdpUsageReporterService
quotaLimiting: QuotaLimiting
redis: RedisV2
valkeyShadow: CdpValkeyShadowPools
/** Absent for consumers that have no dead-letter topic; their build failures stay metrics-only. */
deadLetterService?: InvocationFailureSink
}
// The one rule for billing and the destinations quota, so an unbilled invocation (an alert notification) is never quota-dropped.
function isBilledHogFunction(hogFunction: HogFunctionType): boolean {
return hogFunction.type === 'destination'
}
export interface BuildHogFunctionInvocationsOptions {
hogTypes: HogFunctionTypeType[]
filterFn: (fn: HogFunctionType) => boolean
invocationFilterFn?: (fn: HogFunctionType, globals: HogFunctionInvocationGlobals) => boolean
}
/**
* Encapsulates the pipeline that turns event globals into hog function invocations:
* load functions → execute filters → watcher state → rate limit → quota → masking → metrics.
*
* Consumers compose this service rather than inheriting it.
*/
export class HogFunctionInvocationPipeline {
private hogRateLimiter: KeyedRateLimiterService
private hogRateLimiterMirror: KeyedRateLimiterService
constructor(
private config: HogFunctionInvocationPipelineConfig,
private deps: HogFunctionInvocationPipelineDeps
) {
const rateLimiterConfig = {
name: 'hog-rate-limiter',
bucketSize: config.CDP_RATE_LIMITER_BUCKET_SIZE,
refillRate: config.CDP_RATE_LIMITER_REFILL_RATE,
ttlSeconds: config.CDP_RATE_LIMITER_TTL,
}
this.hogRateLimiter = new KeyedRateLimiterService(rateLimiterConfig, deps.redis)
this.hogRateLimiterMirror = new KeyedRateLimiterService(rateLimiterConfig, deps.valkeyShadow.writer)
}
public async buildInvocations(
invocationGlobals: HogFunctionInvocationGlobals[],
opts: BuildHogFunctionInvocationsOptions
): Promise<CyclotronJobInvocationHogFunction[]> {
return (await this.buildInvocationsWithRelease(invocationGlobals, opts)).invocations
}
/**
* Also returns the masker's `release`. Masking claims are written before the caller queues the invocations,
* so a caller that retries after a failed queue must release them, or the retry masks every invocation.
*/
@instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')
public async buildInvocationsWithRelease(
invocationGlobals: HogFunctionInvocationGlobals[],
opts: BuildHogFunctionInvocationsOptions
): Promise<{
invocations: CyclotronJobInvocationHogFunction[]
release: (persisted?: CyclotronJobInvocation[]) => Promise<void>
}> {
const teamsToLoad = [...new Set(invocationGlobals.map((x) => x.project.id))]
const hogFunctionsByTeam = await this.deps.hogFunctionManager.getHogFunctionsForTeams(
teamsToLoad,
opts.hogTypes,
opts.filterFn
)
const possibleInvocations = (
await Promise.all(
invocationGlobals.map(async (globals) => {
const teamHogFunctions = opts.invocationFilterFn
? hogFunctionsByTeam[globals.project.id].filter((fn) => opts.invocationFilterFn!(fn, globals))
: hogFunctionsByTeam[globals.project.id]
try {
const { invocations, metrics, logs, buildFailures } = await buildHogFunctionInvocations(
this.deps.hogInputsService,
teamHogFunctions,
globals
)
this.deps.hogFunctionMonitoringService.queueAppMetrics(metrics, 'hog_function')
this.deps.hogFunctionMonitoringService.queueLogs(logs, 'hog_function')
this.deps.deadLetterService?.recordBuildFailures(globals, buildFailures)
return invocations
} catch (error) {
// Anything the per-function handling did not expect: a filter global that
// cannot be built, a bug in the builder itself. Without this the whole batch
// fails and the partition stalls on one event, forever, until someone
// deploys a fix. Retriable errors still fail the batch, because a dependency
// being down is not the event's fault.
if (error?.isRetriable === true) {
throw error
}
// Swallowing the event is only safe once it is parked. A consumer with no
// dead-letter service, or one running with the flag off, fails the batch
// instead, which is what kept the event before any of this existed.
if (!this.deps.deadLetterService?.recordProcessFailure(globals, error, 'hog_function')) {
throw error
}
return []
}
})
)
).flat()
const hogFunctionIds = possibleInvocations.map((x) => x.hogFunction.id)
const states = await dualRead(
'hog-watcher.getEffectiveStates',
() =>
instrumentFn('cdpConsumer.handleEachBatch.hogWatcher.getEffectiveStates', async () => {
return await this.deps.hogWatcher.getEffectiveStates(hogFunctionIds)
}),
() => this.deps.hogWatcherMirror.getEffectiveStates(hogFunctionIds),
sameWatcherStates
)
const rateLimitInputs: KeyedRateLimitRequest[] = possibleInvocations.map((x) => ({
id: x.hogFunction.id,
cost: 1,
}))
const rateLimits = await dualRead(
'hog-function-rate-limiter.rateLimitGrouped',
() =>
instrumentFn('cdpConsumer.handleEachBatch.hogRateLimiter.rateLimitGrouped', async () => {
return await this.hogRateLimiter.rateLimitGrouped(rateLimitInputs)
}),
() => this.hogRateLimiterMirror.rateLimitGrouped(rateLimitInputs),
(primary, secondary) =>
primary.every(([, result], index) => result.isRateLimited === secondary[index]?.[1].isRateLimited)
)
const validInvocations: CyclotronJobInvocationHogFunction[] = []
await Promise.all(
possibleInvocations.map(async (item, index) => {
try {
const rateLimit = rateLimits[index][1]
if (rateLimit.isRateLimited) {
counterRateLimited.labels({ kind: 'hog_function', function_id: item.functionId }).inc()
// NOTE: We don't return here as we are just monitoring this feature currently
}
} catch (e) {
captureException(e)
logger.error('🔴', 'Error checking rate limit for hog function', { err: e })
}
const isQuotaLimited =
isBilledHogFunction(item.hogFunction) &&
(await shouldBlockInvocationDueToQuota(item, {
quotaLimiting: this.deps.quotaLimiting,
hogFunctionMonitoringService: this.deps.hogFunctionMonitoringService,
}))
if (isQuotaLimited) {
return
}
const state = states[item.hogFunction.id].state
counterHogFunctionStateOnEvent
.labels({
state: HogWatcherState[state],
kind: item.hogFunction.type,
})
.inc()
if (state === HogWatcherState.disabled) {
this.deps.hogFunctionMonitoringService.queueAppMetric(
{
team_id: item.teamId,
app_source_id: item.functionId,
metric_kind: 'failure',
metric_name: 'disabled_permanently',
count: 1,
},
'hog_function'
)
return
}
if (state === HogWatcherState.degraded) {
item.queuePriority = 2
if (this.config.CDP_OVERFLOW_QUEUE_ENABLED) {
item.queue = 'hogoverflow'
}
}
validInvocations.push(item)
})
)
const {
masked,
notMasked: notMaskedInvocations,
release,
} = await this.deps.hogMasker.filterByMasking(validInvocations)
this.deps.hogFunctionMonitoringService.queueAppMetrics(
masked.map((item) => ({
team_id: item.teamId,
app_source_id: item.functionId,
metric_kind: 'other',
metric_name: 'masked',
count: 1,
})),
'hog_function'
)
const triggeredInvocationsMetrics: MinimalAppMetric[] = []
// Track unique events that have been billed (billing is per-event, not per-destination)
const billedEventUuids = new Set<string>()
notMaskedInvocations.forEach((item) => {
triggeredInvocationsMetrics.push({
team_id: item.teamId,
app_source_id: item.functionId,
metric_kind: 'other',
metric_name: 'triggered',
count: 1,
})
// Bill once per triggering event, not per destination
if (isBilledHogFunction(item.hogFunction)) {
const eventUuid = item.state?.globals?.event?.uuid
if (eventUuid && !billedEventUuids.has(eventUuid)) {
billedEventUuids.add(eventUuid)
triggeredInvocationsMetrics.push({
team_id: item.teamId,
app_source_id: '_event_trigger',
instance_id: eventUuid,
metric_kind: 'billing',
metric_name: 'billable_invocation',
count: 1,
})
this.deps.cdpUsageReporter?.reportBillableInvocation({
teamId: item.teamId,
usageKey: 'cdp_billable_invocations',
recordId: `event:${eventUuid}`,
})
}
}
})
this.deps.hogFunctionMonitoringService.queueAppMetrics(triggeredInvocationsMetrics, 'hog_function')
return { invocations: notMaskedInvocations, release }
}
}