:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { ReadOnlyGroupTypeManager } from '~/common/groups/readonly-group-type-manager'
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import {
AppMetricsOutput,
DlqOutput,
EVENTS_OUTPUT,
EventOutput,
IngestionWarningsOutput,
OverflowOutput,
TophogOutput,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { EventIngestionRestrictionManager } from '~/common/utils/event-ingestion-restrictions'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { newCommonIngestionPipeline } from '~/ingestion/common/common-ingestion-pipeline'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { EventFilterManager } from '~/ingestion/common/event-filters'
import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'
import {
EventFiltersBatchContext,
createApplyEventFiltersStep,
createEventFiltersBatchAppMetricsBeforeBatchStep,
createFlushEventFiltersBatchAppMetricsStep,
} from '~/ingestion/common/steps/event-filters-steps'
import {
createApplyCookielessProcessingStep,
createApplyEventRestrictionsStep,
createOverflowLaneTTLRefreshStep,
createRateLimitToOverflowStep,
} from '~/ingestion/common/steps/event-preprocessing'
import { createCreateEventStep } from '~/ingestion/common/steps/event-processing/create-event-step'
import { EmitEventStepOutput, createEmitEventStep } from '~/ingestion/common/steps/event-processing/emit-event-step'
import { createFetchPersonChunkStep } from '~/ingestion/common/steps/event-processing/fetch-person-chunk-step'
import { createFlushHogTransformerStep } from '~/ingestion/common/steps/event-processing/flush-hog-transformer-step'
import { createHogTransformEventStep } from '~/ingestion/common/steps/event-processing/hog-transform-event-step'
import { createReadOnlyProcessGroupsStep } from '~/ingestion/common/steps/event-processing/readonly-process-groups-step'
import { prefetchHogFunctionsStep } from '~/ingestion/common/steps/prefetch-hog-functions-step'
import { prefetchTeamsStep } from '~/ingestion/common/steps/prefetch-teams-step'
import { createRecordIngestionLagStep } from '~/ingestion/common/steps/record-ingestion-lag'
import {
EventUsageBatchContext,
createEventUsageBeforeBatchStep,
createFlushEventUsageStep,
createRecordEventUsageAfterIngestStep,
createRecordEventUsageStep,
} from '~/ingestion/common/steps/usage-records-steps'
import { resolveExceptionUsageKey } from '~/ingestion/common/usage-records/billable-events'
import { IngestionOverflowMode } from '~/ingestion/config'
import { BatchingContext, BatchingPipeline } from '~/ingestion/framework/batching-pipeline'
import { TopHogRegistry, count } from '~/ingestion/framework/extensions/tophog'
import { createAttachMessageBytesStep } from './attach-message-bytes-step'
import { createCymbalProcessingStep } from './cymbal-processing-step'
import { CymbalClient } from './cymbal/client'
import { createErrorTrackingPrepareEventStep } from './prepare-event-step'
/** The hog transformer methods the pipeline uses; lifecycle stays with the owning scope. */
export type ErrorTrackingHogTransformer = Pick<
HogTransformer,
'transformEventAndProduceMessages' | 'processInvocationResults' | 'prefetchHogFunctionsForTeams'
>
export interface ErrorTrackingPipelineInput {
message: Message
}
/**
* The final step emits to Kafka and outputs handles to the in-flight
* emissions. Successful events are produced to the output topic, while
* failures are handled by the result handling pipeline (DLQ, drop, redirect).
*/
export type ErrorTrackingPipelineOutput = EmitEventStepOutput
export type ErrorTrackingPipeline = BatchingPipeline<
ErrorTrackingPipelineInput,
ErrorTrackingPipelineOutput,
{ message: Message },
EventFiltersBatchContext & EventUsageBatchContext,
{ message: Message } & BatchingContext,
OverflowOutput
>
export type ErrorTrackingOutputs = IngestionOutputs<
EventOutput | IngestionWarningsOutput | DlqOutput | OverflowOutput | TophogOutput | AppMetricsOutput
>
export interface ErrorTrackingPipelineConfig {
outputs: ErrorTrackingOutputs
promiseScheduler: PromiseScheduler
teamManager: TeamManager
personRepository: PersonReadRepository
hogTransformer: ErrorTrackingHogTransformer
cymbalClient: CymbalClient
groupTypeManager: ReadOnlyGroupTypeManager
cookielessManager: CookielessManager
eventIngestionRestrictionManager: EventIngestionRestrictionManager
eventFilterManager: EventFilterManager
overflowMode: IngestionOverflowMode
/**
* When true, overflow redirects (both restriction-driven force-overflow
* and rate-limit-to-overflow) keep the original partition key. When
* false, redirects emit with a null key so Kafka spreads load across
* overflow-topic partitions. Cymbal cache locality is enforced one layer
* down (team_id consistent hashing inside `CymbalClient`), so the
* partition key on the overflow lane doesn't affect symbolication cache
* hits.
*/
preservePartitionLocality: boolean
/** Service for rate limiting and redirecting to overflow (main lane only). */
overflowRedirectService?: OverflowRedirectService
/** Service for refreshing TTLs on overflow lane events. */
overflowLaneTTLRefreshService?: OverflowRedirectService
/** TopHog registry for metrics. */
topHog: TopHogRegistry
createEventUsageBatch: () => UsageRecordBatch
teamsPrefetchEnabled: boolean
hogFunctionsPrefetchEnabled: boolean
}
/**
* Creates the error tracking pipeline.
*
* The pipeline processes exception events through these phases:
* 1. Parse headers - Extract token, timestamps from Kafka message headers
* 2. Apply event restrictions - Billing limits, drop/overflow
* 3. Skip-cookieless rate limit - Redirect non-cookieless rate-limited events to overflow
* pre-parse (cookieless events pass through to step 9)
* 4. Prefetch teams - Warm the team cache for the chunk's tokens (TEAMS_PREFETCH_ENABLED)
* 5. Parse Kafka message - Parse message body into event
* 6. Resolve team - Look up team by token
* 7. Apply event filters - Drop events that match the team's event filter
* 8. Apply cookieless processing - Rewrite distinct_id for cookieless events
* 9. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow
* using the hashed distinct_id from step 8
* 10. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)
* 11. Cymbal processing - Symbolicate, fingerprint, and link issues
* 12. Person properties - Fetch person by distinct_id (read-only)
* 13. Hog transformations - Run team transformations (including GeoIP if enabled)
* 14. Prepare event - Convert to PreIngestionEvent format, track if person found
* 15. Group type mapping - Map group types to indexes (read-only)
* 16. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)
* 17. Emit event - Produce to output topic
*
* Note: Cymbal runs before enrichment because it only needs the raw exception data
* for symbolication and fingerprinting. This reduces payload size and avoids
* wasted enrichment work if Cymbal suppresses the event.
*/
export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig): ErrorTrackingPipeline {
const {
outputs,
promiseScheduler,
teamManager,
personRepository,
hogTransformer,
cymbalClient,
groupTypeManager,
cookielessManager,
eventIngestionRestrictionManager,
eventFilterManager,
overflowMode,
preservePartitionLocality,
overflowRedirectService,
overflowLaneTTLRefreshService,
topHog,
createEventUsageBatch,
teamsPrefetchEnabled,
hogFunctionsPrefetchEnabled,
} = config
const preCymbal = newCommonIngestionPipeline<ErrorTrackingPipelineInput, { message: Message }, OverflowOutput>({
teamManager,
outputs,
promiseScheduler,
concurrentBatches: 1,
topHog,
})
.beforeBatch((b) =>
b
.pipe(createEventFiltersBatchAppMetricsBeforeBatchStep(outputs))
.pipe(createEventUsageBeforeBatchStep(createEventUsageBatch))
)
// Header-only steps: parse Kafka headers and apply token-level restrictions.
// Cheap; runs per-event before we touch the body.
.parseHeaders()
.pipe(
createApplyEventRestrictionsStep(eventIngestionRestrictionManager, {
overflowMode,
preservePartitionLocality,
// createFetchPersonChunkStep below only reads persons.
pipelineWritesPersons: false,
})
)
// Rate-limit events to overflow before parsing the body, keyed on the
// Kafka message key — the partition key capture computed. Cookieless
// events count under token:client_ip.
.pipeChunk(createRateLimitToOverflowStep(preservePartitionLocality, overflowRedirectService))
.pipeChunk(prefetchTeamsStep(teamManager, teamsPrefetchEnabled))
.parseMessage()
.resolveTeam()
.pipe(createApplyEventFiltersStep(eventFilterManager))
// Carry the Kafka message byte size through for Cymbal batch chunking.
.pipe(createAttachMessageBytesStep())
// Cookieless processing: rewrites event.distinct_id for cookieless
// events. Must run as a batch and before any step that depends on
// the final distinct ID.
.gather()
.pipeChunk(createApplyCookielessProcessingStep(cookielessManager))
// Refresh TTLs for overflow lane events (keeps Redis flags alive)
.pipeChunk(createOverflowLaneTTLRefreshStep(overflowLaneTTLRefreshService))
.pipeChunk(prefetchHogFunctionsStep(hogTransformer, hogFunctionsPrefetchEnabled))
const afterCymbal = preCymbal
// Process through Cymbal as a batch (before enrichment - Cymbal only
// needs raw exception data, not person/geoip/group data).
// softDeadlineMs (10s) + ERROR_TRACKING_CYMBAL_TIMEOUT_MS (15s) stays below
// CONSUMER_MAX_HEARTBEAT_INTERVAL_MS (30s).
.pipeChunk(createCymbalProcessingStep(cymbalClient), {
retry: { tries: 5, sleepMs: 100, backoffFactor: 4, softDeadlineMs: 10000, name: 'cymbal_processing' },
})
return (
afterCymbal
// Batch fetch person (read-only, no updates). The personhog client
// retries transient gRPC errors for ~150ms; this outer retry
// absorbs longer blips that would otherwise crash the worker via
// an unhandled rejection.
.pipeChunk(createFetchPersonChunkStep(personRepository), {
retry: { tries: 5, sleepMs: 100, name: 'fetch_person_chunk' },
})
// Run Hog transformations (including GeoIP if team has it enabled)
.pipe(createHogTransformEventStep(hogTransformer))
// Prepare event for emission
.pipe(createErrorTrackingPrepareEventStep())
// Map group types to indexes (read-only, no new group types created)
.pipe(createReadOnlyProcessGroupsStep(groupTypeManager))
// After Cymbal, so an exception it suppresses is never billed.
.pipe(createRecordEventUsageStep(resolveExceptionUsageKey))
.pipe(createCreateEventStep(EVENTS_OUTPUT))
.pipe(createEmitEventStep({ outputs }), {
topHog: [
count('emitted_events', (input) => ({
team_id: String(input.teamId),
})),
count('emitted_events_per_distinct_id', (input) => ({
team_id: String(input.teamId),
distinct_id: input.eventsToEmit[0]?.event.distinct_id ?? '',
})),
],
})
.pipe(createRecordEventUsageAfterIngestStep())
.pipe(createRecordIngestionLagStep())
.afterBatch((b) =>
b
.pipe(createFlushEventFiltersBatchAppMetricsStep())
.pipe(createFlushEventUsageStep())
// Drain hog transformer invocation results once per batch.
.pipe(createFlushHogTransformerStep(hogTransformer))
)
.build()
)
}