:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): apply event filters in the error tracking pipeline (#120939)
Paweł Ledwoń79e6dff5cf65
diff --git a/nodejs/src/ingestion/common/event-filters/manager.test.ts b/nodejs/src/ingestion/common/event-filters/manager.test.tsindex 42ddc51fec9c..fc8149addc3d 100644--- a/nodejs/src/ingestion/common/event-filters/manager.test.ts+++ b/nodejs/src/ingestion/common/event-filters/manager.test.ts@@ -143,6 +143,37 @@ describe('EventFilterManager', () => {await stop()})+ it('keeps the loaded filters when a background reload fails', async () => {+ mockPostgres.query.mockResolvedValueOnce({+ rows: [+ {+ id: 'f1',+ team_id: 1,+ mode: 'live',+ filter_tree: { type: 'condition', field: 'event_name', operator: 'exact', value: 'a' },+ },+ ],+ })+ const { manager, stop } = await buildManager()+ mockPostgres.query.mockRejectedValue(new Error('connection refused'))+ const unhandledRejection = jest.fn()+ process.on('unhandledRejection', unhandledRejection)+ jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 61_000)++ try {+ manager.getFilter(1)+ await new Promise((resolve) => setImmediate(resolve))++ expect(mockPostgres.query).toHaveBeenCalledTimes(2)+ expect(unhandledRejection).not.toHaveBeenCalled()+ expect(manager.getFilter(1)!.id).toBe('f1')+ } finally {+ process.off('unhandledRejection', unhandledRejection)+ jest.restoreAllMocks()+ await stop()+ }+ })+it('handles multiple teams', async () => {mockPostgres.query.mockResolvedValue({rows: [diff --git a/nodejs/src/ingestion/common/event-filters/manager.ts b/nodejs/src/ingestion/common/event-filters/manager.tsindex c482a61f3e4f..df2e1473dacd 100644--- a/nodejs/src/ingestion/common/event-filters/manager.ts+++ b/nodejs/src/ingestion/common/event-filters/manager.ts@@ -4,6 +4,7 @@ import { logger } from '~/common/utils/logger'import { Component } from '~/ingestion/common/scopes'import { treeHasConditions } from './evaluate'+import { eventFiltersRefreshErrors } from './metrics'import { EventFilterRowSchema, EventFilterRule } from './schema'/**@@ -17,7 +18,16 @@ export class EventFilterManager {private refresher: BackgroundRefresher<Map<number, EventFilterRule>>constructor(private readonly postgres: PostgresRouter) {- this.refresher = new BackgroundRefresher(async () => this.fetchAllFilters(), 60_000)+ // A failed reload keeps the last loaded filters. The default handler rethrows from a+ // detached promise, and the server stops on an unhandled rejection.+ this.refresher = new BackgroundRefresher(+ async () => this.fetchAllFilters(),+ 60_000,+ (error) => {+ logger.error('Failed to refresh event filter config', { error })+ eventFiltersRefreshErrors.inc()+ }+ )}async prime(): Promise<void> {diff --git a/nodejs/src/ingestion/common/event-filters/metrics.ts b/nodejs/src/ingestion/common/event-filters/metrics.tsindex 1769e2cf22bf..d0c78d14720f 100644--- a/nodejs/src/ingestion/common/event-filters/metrics.ts+++ b/nodejs/src/ingestion/common/event-filters/metrics.ts@@ -5,3 +5,8 @@ export const eventFiltersEventsEvaluated = new Counter({help: 'Total number of events evaluated by event filters',labelNames: ['outcome'],})++export const eventFiltersRefreshErrors = new Counter({+ name: 'ingestion_filters_refresh_errors',+ help: 'Number of failed background reloads of event filter config',+})diff --git a/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts b/nodejs/src/ingestion/pipelines/errortracking/consumer.test.tsindex 475b30c93fce..01d3d6375824 100644--- a/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts@@ -186,6 +186,7 @@ describe('createErrorTrackingConsumer', () => {const sharedScope = newScope('shared-test', (builder) =>builder+ .add('postgres', staticComponent(infra.postgres)).add('redisPool', staticComponent(infra.redisPool)).add('teamManager', staticComponent(infra.teamManager)).add('cookielessManager', staticComponent(infra.cookielessManager))diff --git a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts b/nodejs/src/ingestion/pipelines/errortracking/consumer.tsindex 3ab93811cfd6..42063f479486 100644--- a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/consumer.ts@@ -6,10 +6,12 @@ import { PersonHogClientComponent } from '~/common/personhog/personhog-client-coimport { PersonHogGroupReadRepository } from '~/common/personhog/personhog-group-read-repository'import { PersonHogPersonReadRepository } from '~/common/personhog/personhog-person-read-repository'import { UsageIngestionConfig, createEventUsageBatchFactory } from '~/common/usage-ingestion'+import { PostgresRouter } from '~/common/utils/db/postgres'import { EventIngestionRestrictionManagerComponent } from '~/common/utils/event-ingestion-restrictions'import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'import { TeamManager } from '~/common/utils/team-manager'import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'+import { EventFilterManagerComponent } from '~/ingestion/common/event-filters'import { CommonIngestionConsumerConfig, CommonIngestionConsumerScope } from '~/ingestion/common/ingestion-consumer'import { DisabledOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/disabled-overflow-redirect'import { MainLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/main-lane-overflow-redirect'@@ -50,6 +52,7 @@ export type ErrorTrackingLaneConfig = CommonIngestionConsumerConfig &* owned by the error tracking scope, not shared in here.*/export type ErrorTrackingSharedScope = Scope<{+ postgres: PostgresRouterredisPool: RedisPoolteamManager: TeamManagercookielessManager: CookielessManager@@ -79,6 +82,7 @@ export function createErrorTrackingConsumer(config: ErrorTrackingLaneConfig, sha'eventIngestionRestrictionManager',new EventIngestionRestrictionManagerComponent(container.redisPool, { pipeline: 'errortracking' }))+ .add('eventFilterManager', new EventFilterManagerComponent(container.postgres)).add('overflowRedirectService',overflowMode === 'redirect'@@ -130,6 +134,7 @@ export function createErrorTrackingConsumer(config: ErrorTrackingLaneConfig, sha),cookielessManager: container.cookielessManager,eventIngestionRestrictionManager: container.eventIngestionRestrictionManager,+ eventFilterManager: container.eventFilterManager,overflowMode,preservePartitionLocality,overflowRedirectService: container.overflowRedirectService,diff --git a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.tsindex 930da01ca5b8..29ded8f3370c 100644--- a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts@@ -15,6 +15,8 @@ import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { TeamManager } from '~/common/utils/team-manager'import { UUIDT } from '~/common/utils/utils'import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'+import { EventFilterManager } from '~/ingestion/common/event-filters'+import { cond } from '~/ingestion/common/event-filters/test-helpers'import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'import { TopHogRegistry } from '~/ingestion/framework/extensions/tophog'import { createOkContext } from '~/ingestion/framework/helpers'@@ -68,6 +70,7 @@ describe('ErrorTrackingPipeline', () => {let mockCymbalClient: jest.Mocked<CymbalClient>let mockGroupTypeManager: jest.Mocked<ReadOnlyGroupTypeManager>let mockEventIngestionRestrictionManager: jest.Mocked<EventIngestionRestrictionManager>+ let mockEventFilterManager: jest.Mocked<EventFilterManager>let mockCookielessManager: jest.Mocked<CookielessManager>let promiseScheduler: PromiseSchedulerlet pipelineConfig: ErrorTrackingPipelineConfig@@ -191,27 +194,26 @@ describe('ErrorTrackingPipeline', () => {return getProducedMessagesForTopic('error_tracking_overflow')}- /**- * Helper to get ingestion warnings that were queued.- * Ingestion warnings are sent via queueMessages to the ingestion_warnings topic.- */- const getIngestionWarnings = (): any[] => {+ const getQueuedMessages = (topicFragment: string): any[] => {return mockKafkaProducer.queueMessages.mock.calls.filter((call) => {const arg = call[0]- // queueMessages can receive a single TopicMessage or an arrayconst topicMessages = Array.isArray(arg) ? arg : [arg]- return topicMessages.some((tm: any) => tm.topic?.includes('ingestion_warnings'))+ return topicMessages.some((tm: any) => tm.topic?.includes(topicFragment))}).flatMap((call) => {const arg = call[0]const topicMessages = Array.isArray(arg) ? arg : [arg]return topicMessages- .filter((tm: any) => tm.topic?.includes('ingestion_warnings'))+ .filter((tm: any) => tm.topic?.includes(topicFragment)).flatMap((tm: any) => tm.messages.map((m: { value: string }) => parseJSON(m.value)))})}+ const getIngestionWarnings = (): any[] => getQueuedMessages('ingestion_warnings')++ const getAppMetrics = (): any[] => getQueuedMessages('app_metrics')+/*** Creates a Cymbal response that includes enriched properties.* In real usage, Cymbal receives the enriched event and returns it with@@ -288,6 +290,10 @@ describe('ErrorTrackingPipeline', () => {forceRefresh: jest.fn(),} as unknown as jest.Mocked<EventIngestionRestrictionManager>+ mockEventFilterManager = {+ getFilter: jest.fn().mockReturnValue(null),+ } as unknown as jest.Mocked<EventFilterManager>+// Passthrough: non-cookieless events round-trip unchanged. The apply step// extracts `.value.event` from each result and reuses the original input// wrapper, so returning `ok(input)` is enough to leave events untouched.@@ -335,6 +341,7 @@ describe('ErrorTrackingPipeline', () => {groupTypeManager: mockGroupTypeManager,cookielessManager: mockCookielessManager,eventIngestionRestrictionManager: mockEventIngestionRestrictionManager,+ eventFilterManager: mockEventFilterManager,overflowMode: 'disabled',preservePartitionLocality: false,topHog: mockTopHog,@@ -597,6 +604,30 @@ describe('ErrorTrackingPipeline', () => {expect(getOverflowMessages()).toHaveLength(0)})+ it('drops events matching a live event filter before Cymbal and records the drop', async () => {+ mockEventFilterManager.getFilter.mockReturnValue({+ id: 'filter-1',+ team_id: team.id,+ mode: 'live',+ filter_tree: cond('distinct_id', 'exact', 'user-123'),+ })++ const pipeline = createErrorTrackingPipeline(pipelineConfig)+ await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])++ expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()+ expect(getProducedEvents()).toHaveLength(0)+ expect(getAppMetrics()).toEqual([+ expect.objectContaining({+ team_id: team.id,+ app_source: 'event_filter',+ app_source_id: 'filter-1',+ metric_name: 'dropped',+ count: 1,+ }),+ ])+ })+it('redirects events to overflow when FORCE_OVERFLOW restriction is set', async () => {mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(new Set([RestrictionType.FORCE_OVERFLOW])diff --git a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.tsindex 36a3d49ef87f..193469a39c64 100644--- a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts@@ -19,7 +19,14 @@ import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { TeamManager } from '~/common/utils/team-manager'import { newCommonIngestionPipeline } from '~/ingestion/common/common-ingestion-pipeline'import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'+import { EventFilterManager } from '~/ingestion/common/event-filters'import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'+import {+ EventFiltersBatchContext,+ createApplyEventFiltersStep,+ createEventFiltersBatchAppMetricsBeforeBatchStep,+ createFlushEventFiltersBatchAppMetricsStep,+} from '~/ingestion/common/steps/event-filters-steps'import {createApplyCookielessProcessingStep,createApplyEventRestrictionsStep,@@ -73,7 +80,7 @@ export type ErrorTrackingPipeline = BatchingPipeline<ErrorTrackingPipelineInput,ErrorTrackingPipelineOutput,{ message: Message },- EventUsageBatchContext,+ EventFiltersBatchContext & EventUsageBatchContext,{ message: Message } & BatchingContext,OverflowOutput>@@ -92,6 +99,7 @@ export interface ErrorTrackingPipelineConfig {groupTypeManager: ReadOnlyGroupTypeManagercookielessManager: CookielessManagereventIngestionRestrictionManager: EventIngestionRestrictionManager+ eventFilterManager: EventFilterManageroverflowMode: IngestionOverflowMode/*** When true, overflow redirects (both restriction-driven force-overflow@@ -121,21 +129,22 @@ export interface ErrorTrackingPipelineConfig {* 1. Parse headers - Extract token, timestamps from Kafka message headers* 2. Apply event restrictions - Billing limits, drop/overflow* 3. Skip-cookieless rate limit - Redirect non-cookieless rate-limited events to overflow- * pre-parse (cookieless events pass through to step 7)+ * pre-parse (cookieless events pass through to step 9)* 4. Prefetch teams - Warm the team cache for the chunk's tokens (TEAMS_PREFETCH_ENABLED)* 5. Parse Kafka message - Parse message body into event* 6. Resolve team - Look up team by token- * 7. Apply cookieless processing - Rewrite distinct_id for cookieless events- * 8. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow- * using the hashed distinct_id from step 7- * 9. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)- * 10. Cymbal processing - Symbolicate, fingerprint, and link issues- * 11. Person properties - Fetch person by distinct_id (read-only)- * 12. Hog transformations - Run team transformations (including GeoIP if enabled)- * 13. Prepare event - Convert to PreIngestionEvent format, track if person found- * 14. Group type mapping - Map group types to indexes (read-only)- * 15. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)- * 16. Emit event - Produce to output topic+ * 7. Apply event filters - Drop events that match the team's event filter+ * 8. Apply cookieless processing - Rewrite distinct_id for cookieless events+ * 9. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow+ * using the hashed distinct_id from step 8+ * 10. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)+ * 11. Cymbal processing - Symbolicate, fingerprint, and link issues+ * 12. Person properties - Fetch person by distinct_id (read-only)+ * 13. Hog transformations - Run team transformations (including GeoIP if enabled)+ * 14. Prepare event - Convert to PreIngestionEvent format, track if person found+ * 15. Group type mapping - Map group types to indexes (read-only)+ * 16. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)+ * 17. Emit event - Produce to output topic** Note: Cymbal runs before enrichment because it only needs the raw exception data* for symbolication and fingerprinting. This reduces payload size and avoids@@ -152,6 +161,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)groupTypeManager,cookielessManager,eventIngestionRestrictionManager,+ eventFilterManager,overflowMode,preservePartitionLocality,overflowRedirectService,@@ -169,7 +179,11 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)concurrentBatches: 1,topHog,})- .beforeBatch((b) => b.pipe(createEventUsageBeforeBatchStep(createEventUsageBatch)))+ .beforeBatch((b) =>+ b+ .pipe(createEventFiltersBatchAppMetricsBeforeBatchStep(outputs))+ .pipe(createEventUsageBeforeBatchStep(createEventUsageBatch))+ )// Header-only steps: parse Kafka headers and apply token-level restrictions.// Cheap; runs per-event before we touch the body..parseHeaders()@@ -188,6 +202,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig).pipeChunk(prefetchTeamsStep(teamManager, teamsPrefetchEnabled)).parseMessage().resolveTeam()+ .pipe(createApplyEventFiltersStep(eventFilterManager))// Carry the Kafka message byte size through for Cymbal batch chunking..pipe(createAttachMessageBytesStep())// Cookieless processing: rewrites event.distinct_id for cookieless@@ -241,6 +256,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig).pipe(createRecordIngestionLagStep()).afterBatch((b) =>b+ .pipe(createFlushEventFiltersBatchAppMetricsStep()).pipe(createFlushEventUsageStep())// Drain hog transformer invocation results once per batch..pipe(createFlushHogTransformerStep(hogTransformer))