Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.6 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { ReadOnlyGroupTypeManager } from '~/common/groups/readonly-group-type-manager'
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import {
    AppMetricsOutput,
    DlqOutput,
    EVENTS_OUTPUT,
    EventOutput,
    IngestionWarningsOutput,
    OverflowOutput,
    TophogOutput,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { EventIngestionRestrictionManager } from '~/common/utils/event-ingestion-restrictions'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { newCommonIngestionPipeline } from '~/ingestion/common/common-ingestion-pipeline'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { EventFilterManager } from '~/ingestion/common/event-filters'
import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'
import {
    EventFiltersBatchContext,
    createApplyEventFiltersStep,
    createEventFiltersBatchAppMetricsBeforeBatchStep,
    createFlushEventFiltersBatchAppMetricsStep,
} from '~/ingestion/common/steps/event-filters-steps'
import {
    createApplyCookielessProcessingStep,
    createApplyEventRestrictionsStep,
    createOverflowLaneTTLRefreshStep,
    createRateLimitToOverflowStep,
} from '~/ingestion/common/steps/event-preprocessing'
import { createCreateEventStep } from '~/ingestion/common/steps/event-processing/create-event-step'
import { EmitEventStepOutput, createEmitEventStep } from '~/ingestion/common/steps/event-processing/emit-event-step'
import { createFetchPersonChunkStep } from '~/ingestion/common/steps/event-processing/fetch-person-chunk-step'
import { createFlushHogTransformerStep } from '~/ingestion/common/steps/event-processing/flush-hog-transformer-step'
import { createHogTransformEventStep } from '~/ingestion/common/steps/event-processing/hog-transform-event-step'
import { createReadOnlyProcessGroupsStep } from '~/ingestion/common/steps/event-processing/readonly-process-groups-step'
import { prefetchHogFunctionsStep } from '~/ingestion/common/steps/prefetch-hog-functions-step'
import { prefetchTeamsStep } from '~/ingestion/common/steps/prefetch-teams-step'
import { createRecordIngestionLagStep } from '~/ingestion/common/steps/record-ingestion-lag'
import {
    EventUsageBatchContext,
    createEventUsageBeforeBatchStep,
    createFlushEventUsageStep,
    createRecordEventUsageAfterIngestStep,
    createRecordEventUsageStep,
} from '~/ingestion/common/steps/usage-records-steps'
import { resolveExceptionUsageKey } from '~/ingestion/common/usage-records/billable-events'
import { IngestionOverflowMode } from '~/ingestion/config'
import { BatchingContext, BatchingPipeline } from '~/ingestion/framework/batching-pipeline'
import { TopHogRegistry, count } from '~/ingestion/framework/extensions/tophog'

import { createAttachMessageBytesStep } from './attach-message-bytes-step'
import { createCymbalProcessingStep } from './cymbal-processing-step'
import { CymbalClient } from './cymbal/client'
import { createErrorTrackingPrepareEventStep } from './prepare-event-step'

/** The hog transformer methods the pipeline uses; lifecycle stays with the owning scope. */
export type ErrorTrackingHogTransformer = Pick<
    HogTransformer,
    'transformEventAndProduceMessages' | 'processInvocationResults' | 'prefetchHogFunctionsForTeams'
>

export interface ErrorTrackingPipelineInput {
    message: Message
}

/**
 * The final step emits to Kafka and outputs handles to the in-flight
 * emissions. Successful events are produced to the output topic, while
 * failures are handled by the result handling pipeline (DLQ, drop, redirect).
 */
export type ErrorTrackingPipelineOutput = EmitEventStepOutput

export type ErrorTrackingPipeline = BatchingPipeline<
    ErrorTrackingPipelineInput,
    ErrorTrackingPipelineOutput,
    { message: Message },
    EventFiltersBatchContext & EventUsageBatchContext,
    { message: Message } & BatchingContext,
    OverflowOutput
>

export type ErrorTrackingOutputs = IngestionOutputs<
    EventOutput | IngestionWarningsOutput | DlqOutput | OverflowOutput | TophogOutput | AppMetricsOutput
>

export interface ErrorTrackingPipelineConfig {
    outputs: ErrorTrackingOutputs
    promiseScheduler: PromiseScheduler
    teamManager: TeamManager
    personRepository: PersonReadRepository
    hogTransformer: ErrorTrackingHogTransformer
    cymbalClient: CymbalClient
    groupTypeManager: ReadOnlyGroupTypeManager
    cookielessManager: CookielessManager
    eventIngestionRestrictionManager: EventIngestionRestrictionManager
    eventFilterManager: EventFilterManager
    overflowMode: IngestionOverflowMode
    /**
     * When true, overflow redirects (both restriction-driven force-overflow
     * and rate-limit-to-overflow) keep the original partition key. When
     * false, redirects emit with a null key so Kafka spreads load across
     * overflow-topic partitions. Cymbal cache locality is enforced one layer
     * down (team_id consistent hashing inside `CymbalClient`), so the
     * partition key on the overflow lane doesn't affect symbolication cache
     * hits.
     */
    preservePartitionLocality: boolean
    /** Service for rate limiting and redirecting to overflow (main lane only). */
    overflowRedirectService?: OverflowRedirectService
    /** Service for refreshing TTLs on overflow lane events. */
    overflowLaneTTLRefreshService?: OverflowRedirectService
    /** TopHog registry for metrics. */
    topHog: TopHogRegistry
    createEventUsageBatch: () => UsageRecordBatch
    teamsPrefetchEnabled: boolean
    hogFunctionsPrefetchEnabled: boolean
}

/**
 * Creates the error tracking pipeline.
 *
 * The pipeline processes exception events through these phases:
 *  1. Parse headers - Extract token, timestamps from Kafka message headers
 *  2. Apply event restrictions - Billing limits, drop/overflow
 *  3. Skip-cookieless rate limit - Redirect non-cookieless rate-limited events to overflow
 *     pre-parse (cookieless events pass through to step 9)
 *  4. Prefetch teams - Warm the team cache for the chunk's tokens (TEAMS_PREFETCH_ENABLED)
 *  5. Parse Kafka message - Parse message body into event
 *  6. Resolve team - Look up team by token
 *  7. Apply event filters - Drop events that match the team's event filter
 *  8. Apply cookieless processing - Rewrite distinct_id for cookieless events
 *  9. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow
 *     using the hashed distinct_id from step 8
 * 10. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)
 * 11. Cymbal processing - Symbolicate, fingerprint, and link issues
 * 12. Person properties - Fetch person by distinct_id (read-only)
 * 13. Hog transformations - Run team transformations (including GeoIP if enabled)
 * 14. Prepare event - Convert to PreIngestionEvent format, track if person found
 * 15. Group type mapping - Map group types to indexes (read-only)
 * 16. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)
 * 17. Emit event - Produce to output topic
 *
 * Note: Cymbal runs before enrichment because it only needs the raw exception data
 * for symbolication and fingerprinting. This reduces payload size and avoids
 * wasted enrichment work if Cymbal suppresses the event.
 */
export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig): ErrorTrackingPipeline {
    const {
        outputs,
        promiseScheduler,
        teamManager,
        personRepository,
        hogTransformer,
        cymbalClient,
        groupTypeManager,
        cookielessManager,
        eventIngestionRestrictionManager,
        eventFilterManager,
        overflowMode,
        preservePartitionLocality,
        overflowRedirectService,
        overflowLaneTTLRefreshService,
        topHog,
        createEventUsageBatch,
        teamsPrefetchEnabled,
        hogFunctionsPrefetchEnabled,
    } = config

    const preCymbal = newCommonIngestionPipeline<ErrorTrackingPipelineInput, { message: Message }, OverflowOutput>({
        teamManager,
        outputs,
        promiseScheduler,
        concurrentBatches: 1,
        topHog,
    })
        .beforeBatch((b) =>
            b
                .pipe(createEventFiltersBatchAppMetricsBeforeBatchStep(outputs))
                .pipe(createEventUsageBeforeBatchStep(createEventUsageBatch))
        )
        // Header-only steps: parse Kafka headers and apply token-level restrictions.
        // Cheap; runs per-event before we touch the body.
        .parseHeaders()
        .pipe(
            createApplyEventRestrictionsStep(eventIngestionRestrictionManager, {
                overflowMode,
                preservePartitionLocality,
                // createFetchPersonChunkStep below only reads persons.
                pipelineWritesPersons: false,
            })
        )
        // Rate-limit events to overflow before parsing the body, keyed on the
        // Kafka message key — the partition key capture computed. Cookieless
        // events count under token:client_ip.
        .pipeChunk(createRateLimitToOverflowStep(preservePartitionLocality, overflowRedirectService))
        .pipeChunk(prefetchTeamsStep(teamManager, teamsPrefetchEnabled))
        .parseMessage()
        .resolveTeam()
        .pipe(createApplyEventFiltersStep(eventFilterManager))
        // Carry the Kafka message byte size through for Cymbal batch chunking.
        .pipe(createAttachMessageBytesStep())
        // Cookieless processing: rewrites event.distinct_id for cookieless
        // events. Must run as a batch and before any step that depends on
        // the final distinct ID.
        .gather()
        .pipeChunk(createApplyCookielessProcessingStep(cookielessManager))
        // Refresh TTLs for overflow lane events (keeps Redis flags alive)
        .pipeChunk(createOverflowLaneTTLRefreshStep(overflowLaneTTLRefreshService))
        .pipeChunk(prefetchHogFunctionsStep(hogTransformer, hogFunctionsPrefetchEnabled))

    const afterCymbal = preCymbal
        // Process through Cymbal as a batch (before enrichment - Cymbal only
        // needs raw exception data, not person/geoip/group data).
        // softDeadlineMs (10s) + ERROR_TRACKING_CYMBAL_TIMEOUT_MS (15s) stays below
        // CONSUMER_MAX_HEARTBEAT_INTERVAL_MS (30s).
        .pipeChunk(createCymbalProcessingStep(cymbalClient), {
            retry: { tries: 5, sleepMs: 100, backoffFactor: 4, softDeadlineMs: 10000, name: 'cymbal_processing' },
        })

    return (
        afterCymbal
            // Batch fetch person (read-only, no updates). The personhog client
            // retries transient gRPC errors for ~150ms; this outer retry
            // absorbs longer blips that would otherwise crash the worker via
            // an unhandled rejection.
            .pipeChunk(createFetchPersonChunkStep(personRepository), {
                retry: { tries: 5, sleepMs: 100, name: 'fetch_person_chunk' },
            })
            // Run Hog transformations (including GeoIP if team has it enabled)
            .pipe(createHogTransformEventStep(hogTransformer))
            // Prepare event for emission
            .pipe(createErrorTrackingPrepareEventStep())
            // Map group types to indexes (read-only, no new group types created)
            .pipe(createReadOnlyProcessGroupsStep(groupTypeManager))
            // After Cymbal, so an exception it suppresses is never billed.
            .pipe(createRecordEventUsageStep(resolveExceptionUsageKey))
            .pipe(createCreateEventStep(EVENTS_OUTPUT))
            .pipe(createEmitEventStep({ outputs }), {
                topHog: [
                    count('emitted_events', (input) => ({
                        team_id: String(input.teamId),
                    })),
                    count('emitted_events_per_distinct_id', (input) => ({
                        team_id: String(input.teamId),
                        distinct_id: input.eventsToEmit[0]?.event.distinct_id ?? '',
                    })),
                ],
            })
            .pipe(createRecordEventUsageAfterIngestStep())
            .pipe(createRecordIngestionLagStep())
            .afterBatch((b) =>
                b
                    .pipe(createFlushEventFiltersBatchAppMetricsStep())
                    .pipe(createFlushEventUsageStep())
                    // Drain hog transformer invocation results once per batch.
                    .pipe(createFlushHogTransformerStep(hogTransformer))
            )
            .build()
    )
}