Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.8 KB
Raw Blame
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

import { RedisV2 } from '../../common/redis/redis-v2'
import { KeyedRateLimitRequest, KeyedRateLimiterService } from '../../common/services/keyed-rate-limiter.service'
import { QuotaLimiting } from '../../common/services/quota-limiting.service'
import { CdpValkeyShadowPools } from '../cdp-services'
import { counterHogFunctionStateOnEvent, counterRateLimited } from '../consumers/metrics'
import { shouldBlockInvocationDueToQuota } from '../consumers/quota-limiting-helper'
import {
    CyclotronJobInvocation,
    CyclotronJobInvocationHogFunction,
    HogFunctionInvocationGlobals,
    HogFunctionType,
    HogFunctionTypeType,
    MinimalAppMetric,
} from '../types'
import { dualRead } from '../utils/dual-store'
import { buildHogFunctionInvocations } from '../utils/invocation-utils'
import { InvocationFailureSink } from './dead-letter/cdp-dead-letter.service'
import { HogInputsService } from './hog-inputs.service'
import { HogFunctionManagerService } from './managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'
import { HogMaskerService } from './monitoring/hog-masker.service'
import { HogWatcherService, HogWatcherState, sameWatcherStates } from './monitoring/hog-watcher.service'
import { CdpUsageReporterService } from './usage/cdp-usage-reporter.service'

export interface HogFunctionInvocationPipelineConfig {
    CDP_RATE_LIMITER_BUCKET_SIZE: number
    CDP_RATE_LIMITER_REFILL_RATE: number
    CDP_RATE_LIMITER_TTL: number
    CDP_OVERFLOW_QUEUE_ENABLED: boolean
}

export interface HogFunctionInvocationPipelineDeps {
    hogFunctionManager: HogFunctionManagerService
    hogInputsService: HogInputsService
    hogWatcher: HogWatcherService
    hogWatcherMirror: HogWatcherService
    hogMasker: HogMaskerService
    hogFunctionMonitoringService: HogFunctionMonitoringService
    cdpUsageReporter?: CdpUsageReporterService
    quotaLimiting: QuotaLimiting
    redis: RedisV2
    valkeyShadow: CdpValkeyShadowPools
    /** Absent for consumers that have no dead-letter topic; their build failures stay metrics-only. */
    deadLetterService?: InvocationFailureSink
}

// The one rule for billing and the destinations quota, so an unbilled invocation (an alert notification) is never quota-dropped.
function isBilledHogFunction(hogFunction: HogFunctionType): boolean {
    return hogFunction.type === 'destination'
}

export interface BuildHogFunctionInvocationsOptions {
    hogTypes: HogFunctionTypeType[]
    filterFn: (fn: HogFunctionType) => boolean
    invocationFilterFn?: (fn: HogFunctionType, globals: HogFunctionInvocationGlobals) => boolean
}

/**
 * Encapsulates the pipeline that turns event globals into hog function invocations:
 * load functions → execute filters → watcher state → rate limit → quota → masking → metrics.
 *
 * Consumers compose this service rather than inheriting it.
 */
export class HogFunctionInvocationPipeline {
    private hogRateLimiter: KeyedRateLimiterService
    private hogRateLimiterMirror: KeyedRateLimiterService

    constructor(
        private config: HogFunctionInvocationPipelineConfig,
        private deps: HogFunctionInvocationPipelineDeps
    ) {
        const rateLimiterConfig = {
            name: 'hog-rate-limiter',
            bucketSize: config.CDP_RATE_LIMITER_BUCKET_SIZE,
            refillRate: config.CDP_RATE_LIMITER_REFILL_RATE,
            ttlSeconds: config.CDP_RATE_LIMITER_TTL,
        }
        this.hogRateLimiter = new KeyedRateLimiterService(rateLimiterConfig, deps.redis)
        this.hogRateLimiterMirror = new KeyedRateLimiterService(rateLimiterConfig, deps.valkeyShadow.writer)
    }

    public async buildInvocations(
        invocationGlobals: HogFunctionInvocationGlobals[],
        opts: BuildHogFunctionInvocationsOptions
    ): Promise<CyclotronJobInvocationHogFunction[]> {
        return (await this.buildInvocationsWithRelease(invocationGlobals, opts)).invocations
    }

    /**
     * Also returns the masker's `release`. Masking claims are written before the caller queues the invocations,
     * so a caller that retries after a failed queue must release them, or the retry masks every invocation.
     */
    @instrumented('cdpConsumer.handleEachBatch.queueMatchingFunctions')
    public async buildInvocationsWithRelease(
        invocationGlobals: HogFunctionInvocationGlobals[],
        opts: BuildHogFunctionInvocationsOptions
    ): Promise<{
        invocations: CyclotronJobInvocationHogFunction[]
        release: (persisted?: CyclotronJobInvocation[]) => Promise<void>
    }> {
        const teamsToLoad = [...new Set(invocationGlobals.map((x) => x.project.id))]
        const hogFunctionsByTeam = await this.deps.hogFunctionManager.getHogFunctionsForTeams(
            teamsToLoad,
            opts.hogTypes,
            opts.filterFn
        )

        const possibleInvocations = (
            await Promise.all(
                invocationGlobals.map(async (globals) => {
                    const teamHogFunctions = opts.invocationFilterFn
                        ? hogFunctionsByTeam[globals.project.id].filter((fn) => opts.invocationFilterFn!(fn, globals))
                        : hogFunctionsByTeam[globals.project.id]

                    try {
                        const { invocations, metrics, logs, buildFailures } = await buildHogFunctionInvocations(
                            this.deps.hogInputsService,
                            teamHogFunctions,
                            globals
                        )

                        this.deps.hogFunctionMonitoringService.queueAppMetrics(metrics, 'hog_function')
                        this.deps.hogFunctionMonitoringService.queueLogs(logs, 'hog_function')
                        this.deps.deadLetterService?.recordBuildFailures(globals, buildFailures)

                        return invocations
                    } catch (error) {
                        // Anything the per-function handling did not expect: a filter global that
                        // cannot be built, a bug in the builder itself. Without this the whole batch
                        // fails and the partition stalls on one event, forever, until someone
                        // deploys a fix. Retriable errors still fail the batch, because a dependency
                        // being down is not the event's fault.
                        if (error?.isRetriable === true) {
                            throw error
                        }
                        // Swallowing the event is only safe once it is parked. A consumer with no
                        // dead-letter service, or one running with the flag off, fails the batch
                        // instead, which is what kept the event before any of this existed.
                        if (!this.deps.deadLetterService?.recordProcessFailure(globals, error, 'hog_function')) {
                            throw error
                        }
                        return []
                    }
                })
            )
        ).flat()

        const hogFunctionIds = possibleInvocations.map((x) => x.hogFunction.id)
        const states = await dualRead(
            'hog-watcher.getEffectiveStates',
            () =>
                instrumentFn('cdpConsumer.handleEachBatch.hogWatcher.getEffectiveStates', async () => {
                    return await this.deps.hogWatcher.getEffectiveStates(hogFunctionIds)
                }),
            () => this.deps.hogWatcherMirror.getEffectiveStates(hogFunctionIds),
            sameWatcherStates
        )

        const rateLimitInputs: KeyedRateLimitRequest[] = possibleInvocations.map((x) => ({
            id: x.hogFunction.id,
            cost: 1,
        }))
        const rateLimits = await dualRead(
            'hog-function-rate-limiter.rateLimitGrouped',
            () =>
                instrumentFn('cdpConsumer.handleEachBatch.hogRateLimiter.rateLimitGrouped', async () => {
                    return await this.hogRateLimiter.rateLimitGrouped(rateLimitInputs)
                }),
            () => this.hogRateLimiterMirror.rateLimitGrouped(rateLimitInputs),
            (primary, secondary) =>
                primary.every(([, result], index) => result.isRateLimited === secondary[index]?.[1].isRateLimited)
        )

        const validInvocations: CyclotronJobInvocationHogFunction[] = []

        await Promise.all(
            possibleInvocations.map(async (item, index) => {
                try {
                    const rateLimit = rateLimits[index][1]
                    if (rateLimit.isRateLimited) {
                        counterRateLimited.labels({ kind: 'hog_function', function_id: item.functionId }).inc()
                        // NOTE: We don't return here as we are just monitoring this feature currently
                    }
                } catch (e) {
                    captureException(e)
                    logger.error('🔴', 'Error checking rate limit for hog function', { err: e })
                }

                const isQuotaLimited =
                    isBilledHogFunction(item.hogFunction) &&
                    (await shouldBlockInvocationDueToQuota(item, {
                        quotaLimiting: this.deps.quotaLimiting,
                        hogFunctionMonitoringService: this.deps.hogFunctionMonitoringService,
                    }))

                if (isQuotaLimited) {
                    return
                }

                const state = states[item.hogFunction.id].state

                counterHogFunctionStateOnEvent
                    .labels({
                        state: HogWatcherState[state],
                        kind: item.hogFunction.type,
                    })
                    .inc()

                if (state === HogWatcherState.disabled) {
                    this.deps.hogFunctionMonitoringService.queueAppMetric(
                        {
                            team_id: item.teamId,
                            app_source_id: item.functionId,
                            metric_kind: 'failure',
                            metric_name: 'disabled_permanently',
                            count: 1,
                        },
                        'hog_function'
                    )
                    return
                }

                if (state === HogWatcherState.degraded) {
                    item.queuePriority = 2
                    if (this.config.CDP_OVERFLOW_QUEUE_ENABLED) {
                        item.queue = 'hogoverflow'
                    }
                }

                validInvocations.push(item)
            })
        )

        const {
            masked,
            notMasked: notMaskedInvocations,
            release,
        } = await this.deps.hogMasker.filterByMasking(validInvocations)

        this.deps.hogFunctionMonitoringService.queueAppMetrics(
            masked.map((item) => ({
                team_id: item.teamId,
                app_source_id: item.functionId,
                metric_kind: 'other',
                metric_name: 'masked',
                count: 1,
            })),
            'hog_function'
        )

        const triggeredInvocationsMetrics: MinimalAppMetric[] = []

        // Track unique events that have been billed (billing is per-event, not per-destination)
        const billedEventUuids = new Set<string>()

        notMaskedInvocations.forEach((item) => {
            triggeredInvocationsMetrics.push({
                team_id: item.teamId,
                app_source_id: item.functionId,
                metric_kind: 'other',
                metric_name: 'triggered',
                count: 1,
            })

            // Bill once per triggering event, not per destination
            if (isBilledHogFunction(item.hogFunction)) {
                const eventUuid = item.state?.globals?.event?.uuid
                if (eventUuid && !billedEventUuids.has(eventUuid)) {
                    billedEventUuids.add(eventUuid)
                    triggeredInvocationsMetrics.push({
                        team_id: item.teamId,
                        app_source_id: '_event_trigger',
                        instance_id: eventUuid,
                        metric_kind: 'billing',
                        metric_name: 'billable_invocation',
                        count: 1,
                    })
                    this.deps.cdpUsageReporter?.reportBillableInvocation({
                        teamId: item.teamId,
                        usageKey: 'cdp_billable_invocations',
                        recordId: `event:${eventUuid}`,
                    })
                }
            }
        })

        this.deps.hogFunctionMonitoringService.queueAppMetrics(triggeredInvocationsMetrics, 'hog_function')

        return { invocations: notMaskedInvocations, release }
    }
}