Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(error-tracking): apply event filters in the error tracking pipeline (#120939)

Paweł Ledwoń79e6dff5cf65
diff --git a/nodejs/src/ingestion/common/event-filters/manager.test.ts b/nodejs/src/ingestion/common/event-filters/manager.test.ts
index 42ddc51fec9c..fc8149addc3d 100644
--- a/nodejs/src/ingestion/common/event-filters/manager.test.ts
+++ b/nodejs/src/ingestion/common/event-filters/manager.test.ts
@@ -143,6 +143,37 @@ describe('EventFilterManager', () => {
await stop()
})
+ it('keeps the loaded filters when a background reload fails', async () => {
+ mockPostgres.query.mockResolvedValueOnce({
+ rows: [
+ {
+ id: 'f1',
+ team_id: 1,
+ mode: 'live',
+ filter_tree: { type: 'condition', field: 'event_name', operator: 'exact', value: 'a' },
+ },
+ ],
+ })
+ const { manager, stop } = await buildManager()
+ mockPostgres.query.mockRejectedValue(new Error('connection refused'))
+ const unhandledRejection = jest.fn()
+ process.on('unhandledRejection', unhandledRejection)
+ jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 61_000)
+
+ try {
+ manager.getFilter(1)
+ await new Promise((resolve) => setImmediate(resolve))
+
+ expect(mockPostgres.query).toHaveBeenCalledTimes(2)
+ expect(unhandledRejection).not.toHaveBeenCalled()
+ expect(manager.getFilter(1)!.id).toBe('f1')
+ } finally {
+ process.off('unhandledRejection', unhandledRejection)
+ jest.restoreAllMocks()
+ await stop()
+ }
+ })
+
it('handles multiple teams', async () => {
mockPostgres.query.mockResolvedValue({
rows: [
diff --git a/nodejs/src/ingestion/common/event-filters/manager.ts b/nodejs/src/ingestion/common/event-filters/manager.ts
index c482a61f3e4f..df2e1473dacd 100644
--- a/nodejs/src/ingestion/common/event-filters/manager.ts
+++ b/nodejs/src/ingestion/common/event-filters/manager.ts
@@ -4,6 +4,7 @@ import { logger } from '~/common/utils/logger'
import { Component } from '~/ingestion/common/scopes'
import { treeHasConditions } from './evaluate'
+import { eventFiltersRefreshErrors } from './metrics'
import { EventFilterRowSchema, EventFilterRule } from './schema'
/**
@@ -17,7 +18,16 @@ export class EventFilterManager {
private refresher: BackgroundRefresher<Map<number, EventFilterRule>>
constructor(private readonly postgres: PostgresRouter) {
- this.refresher = new BackgroundRefresher(async () => this.fetchAllFilters(), 60_000)
+ // A failed reload keeps the last loaded filters. The default handler rethrows from a
+ // detached promise, and the server stops on an unhandled rejection.
+ this.refresher = new BackgroundRefresher(
+ async () => this.fetchAllFilters(),
+ 60_000,
+ (error) => {
+ logger.error('Failed to refresh event filter config', { error })
+ eventFiltersRefreshErrors.inc()
+ }
+ )
}
async prime(): Promise<void> {
diff --git a/nodejs/src/ingestion/common/event-filters/metrics.ts b/nodejs/src/ingestion/common/event-filters/metrics.ts
index 1769e2cf22bf..d0c78d14720f 100644
--- a/nodejs/src/ingestion/common/event-filters/metrics.ts
+++ b/nodejs/src/ingestion/common/event-filters/metrics.ts
@@ -5,3 +5,8 @@ export const eventFiltersEventsEvaluated = new Counter({
help: 'Total number of events evaluated by event filters',
labelNames: ['outcome'],
})
+
+export const eventFiltersRefreshErrors = new Counter({
+ name: 'ingestion_filters_refresh_errors',
+ help: 'Number of failed background reloads of event filter config',
+})
diff --git a/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts b/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts
index 475b30c93fce..01d3d6375824 100644
--- a/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/errortracking/consumer.test.ts
@@ -186,6 +186,7 @@ describe('createErrorTrackingConsumer', () => {
const sharedScope = newScope('shared-test', (builder) =>
builder
+ .add('postgres', staticComponent(infra.postgres))
.add('redisPool', staticComponent(infra.redisPool))
.add('teamManager', staticComponent(infra.teamManager))
.add('cookielessManager', staticComponent(infra.cookielessManager))
diff --git a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts b/nodejs/src/ingestion/pipelines/errortracking/consumer.ts
index 3ab93811cfd6..42063f479486 100644
--- a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts
+++ b/nodejs/src/ingestion/pipelines/errortracking/consumer.ts
@@ -6,10 +6,12 @@ import { PersonHogClientComponent } from '~/common/personhog/personhog-client-co
import { PersonHogGroupReadRepository } from '~/common/personhog/personhog-group-read-repository'
import { PersonHogPersonReadRepository } from '~/common/personhog/personhog-person-read-repository'
import { UsageIngestionConfig, createEventUsageBatchFactory } from '~/common/usage-ingestion'
+import { PostgresRouter } from '~/common/utils/db/postgres'
import { EventIngestionRestrictionManagerComponent } from '~/common/utils/event-ingestion-restrictions'
import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'
import { TeamManager } from '~/common/utils/team-manager'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
+import { EventFilterManagerComponent } from '~/ingestion/common/event-filters'
import { CommonIngestionConsumerConfig, CommonIngestionConsumerScope } from '~/ingestion/common/ingestion-consumer'
import { DisabledOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/disabled-overflow-redirect'
import { MainLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/main-lane-overflow-redirect'
@@ -50,6 +52,7 @@ export type ErrorTrackingLaneConfig = CommonIngestionConsumerConfig &
* owned by the error tracking scope, not shared in here.
*/
export type ErrorTrackingSharedScope = Scope<{
+ postgres: PostgresRouter
redisPool: RedisPool
teamManager: TeamManager
cookielessManager: CookielessManager
@@ -79,6 +82,7 @@ export function createErrorTrackingConsumer(config: ErrorTrackingLaneConfig, sha
'eventIngestionRestrictionManager',
new EventIngestionRestrictionManagerComponent(container.redisPool, { pipeline: 'errortracking' })
)
+ .add('eventFilterManager', new EventFilterManagerComponent(container.postgres))
.add(
'overflowRedirectService',
overflowMode === 'redirect'
@@ -130,6 +134,7 @@ export function createErrorTrackingConsumer(config: ErrorTrackingLaneConfig, sha
),
cookielessManager: container.cookielessManager,
eventIngestionRestrictionManager: container.eventIngestionRestrictionManager,
+ eventFilterManager: container.eventFilterManager,
overflowMode,
preservePartitionLocality,
overflowRedirectService: container.overflowRedirectService,
diff --git a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts
index 930da01ca5b8..29ded8f3370c 100644
--- a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts
+++ b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts
@@ -15,6 +15,8 @@ import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { UUIDT } from '~/common/utils/utils'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
+import { EventFilterManager } from '~/ingestion/common/event-filters'
+import { cond } from '~/ingestion/common/event-filters/test-helpers'
import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'
import { TopHogRegistry } from '~/ingestion/framework/extensions/tophog'
import { createOkContext } from '~/ingestion/framework/helpers'
@@ -68,6 +70,7 @@ describe('ErrorTrackingPipeline', () => {
let mockCymbalClient: jest.Mocked<CymbalClient>
let mockGroupTypeManager: jest.Mocked<ReadOnlyGroupTypeManager>
let mockEventIngestionRestrictionManager: jest.Mocked<EventIngestionRestrictionManager>
+ let mockEventFilterManager: jest.Mocked<EventFilterManager>
let mockCookielessManager: jest.Mocked<CookielessManager>
let promiseScheduler: PromiseScheduler
let pipelineConfig: ErrorTrackingPipelineConfig
@@ -191,27 +194,26 @@ describe('ErrorTrackingPipeline', () => {
return getProducedMessagesForTopic('error_tracking_overflow')
}
- /**
- * Helper to get ingestion warnings that were queued.
- * Ingestion warnings are sent via queueMessages to the ingestion_warnings topic.
- */
- const getIngestionWarnings = (): any[] => {
+ const getQueuedMessages = (topicFragment: string): any[] => {
return mockKafkaProducer.queueMessages.mock.calls
.filter((call) => {
const arg = call[0]
- // queueMessages can receive a single TopicMessage or an array
const topicMessages = Array.isArray(arg) ? arg : [arg]
- return topicMessages.some((tm: any) => tm.topic?.includes('ingestion_warnings'))
+ return topicMessages.some((tm: any) => tm.topic?.includes(topicFragment))
})
.flatMap((call) => {
const arg = call[0]
const topicMessages = Array.isArray(arg) ? arg : [arg]
return topicMessages
- .filter((tm: any) => tm.topic?.includes('ingestion_warnings'))
+ .filter((tm: any) => tm.topic?.includes(topicFragment))
.flatMap((tm: any) => tm.messages.map((m: { value: string }) => parseJSON(m.value)))
})
}
+ const getIngestionWarnings = (): any[] => getQueuedMessages('ingestion_warnings')
+
+ const getAppMetrics = (): any[] => getQueuedMessages('app_metrics')
+
/**
* Creates a Cymbal response that includes enriched properties.
* In real usage, Cymbal receives the enriched event and returns it with
@@ -288,6 +290,10 @@ describe('ErrorTrackingPipeline', () => {
forceRefresh: jest.fn(),
} as unknown as jest.Mocked<EventIngestionRestrictionManager>
+ mockEventFilterManager = {
+ getFilter: jest.fn().mockReturnValue(null),
+ } as unknown as jest.Mocked<EventFilterManager>
+
// Passthrough: non-cookieless events round-trip unchanged. The apply step
// extracts `.value.event` from each result and reuses the original input
// wrapper, so returning `ok(input)` is enough to leave events untouched.
@@ -335,6 +341,7 @@ describe('ErrorTrackingPipeline', () => {
groupTypeManager: mockGroupTypeManager,
cookielessManager: mockCookielessManager,
eventIngestionRestrictionManager: mockEventIngestionRestrictionManager,
+ eventFilterManager: mockEventFilterManager,
overflowMode: 'disabled',
preservePartitionLocality: false,
topHog: mockTopHog,
@@ -597,6 +604,30 @@ describe('ErrorTrackingPipeline', () => {
expect(getOverflowMessages()).toHaveLength(0)
})
+ it('drops events matching a live event filter before Cymbal and records the drop', async () => {
+ mockEventFilterManager.getFilter.mockReturnValue({
+ id: 'filter-1',
+ team_id: team.id,
+ mode: 'live',
+ filter_tree: cond('distinct_id', 'exact', 'user-123'),
+ })
+
+ const pipeline = createErrorTrackingPipeline(pipelineConfig)
+ await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])
+
+ expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
+ expect(getProducedEvents()).toHaveLength(0)
+ expect(getAppMetrics()).toEqual([
+ expect.objectContaining({
+ team_id: team.id,
+ app_source: 'event_filter',
+ app_source_id: 'filter-1',
+ metric_name: 'dropped',
+ count: 1,
+ }),
+ ])
+ })
+
it('redirects events to overflow when FORCE_OVERFLOW restriction is set', async () => {
mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(
new Set([RestrictionType.FORCE_OVERFLOW])
diff --git a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts
index 36a3d49ef87f..193469a39c64 100644
--- a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts
+++ b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.ts
@@ -19,7 +19,14 @@ import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { newCommonIngestionPipeline } from '~/ingestion/common/common-ingestion-pipeline'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
+import { EventFilterManager } from '~/ingestion/common/event-filters'
import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'
+import {
+ EventFiltersBatchContext,
+ createApplyEventFiltersStep,
+ createEventFiltersBatchAppMetricsBeforeBatchStep,
+ createFlushEventFiltersBatchAppMetricsStep,
+} from '~/ingestion/common/steps/event-filters-steps'
import {
createApplyCookielessProcessingStep,
createApplyEventRestrictionsStep,
@@ -73,7 +80,7 @@ export type ErrorTrackingPipeline = BatchingPipeline<
ErrorTrackingPipelineInput,
ErrorTrackingPipelineOutput,
{ message: Message },
- EventUsageBatchContext,
+ EventFiltersBatchContext & EventUsageBatchContext,
{ message: Message } & BatchingContext,
OverflowOutput
>
@@ -92,6 +99,7 @@ export interface ErrorTrackingPipelineConfig {
groupTypeManager: ReadOnlyGroupTypeManager
cookielessManager: CookielessManager
eventIngestionRestrictionManager: EventIngestionRestrictionManager
+ eventFilterManager: EventFilterManager
overflowMode: IngestionOverflowMode
/**
* When true, overflow redirects (both restriction-driven force-overflow
@@ -121,21 +129,22 @@ export interface ErrorTrackingPipelineConfig {
* 1. Parse headers - Extract token, timestamps from Kafka message headers
* 2. Apply event restrictions - Billing limits, drop/overflow
* 3. Skip-cookieless rate limit - Redirect non-cookieless rate-limited events to overflow
- * pre-parse (cookieless events pass through to step 7)
+ * pre-parse (cookieless events pass through to step 9)
* 4. Prefetch teams - Warm the team cache for the chunk's tokens (TEAMS_PREFETCH_ENABLED)
* 5. Parse Kafka message - Parse message body into event
* 6. Resolve team - Look up team by token
- * 7. Apply cookieless processing - Rewrite distinct_id for cookieless events
- * 8. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow
- * using the hashed distinct_id from step 7
- * 9. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)
- * 10. Cymbal processing - Symbolicate, fingerprint, and link issues
- * 11. Person properties - Fetch person by distinct_id (read-only)
- * 12. Hog transformations - Run team transformations (including GeoIP if enabled)
- * 13. Prepare event - Convert to PreIngestionEvent format, track if person found
- * 14. Group type mapping - Map group types to indexes (read-only)
- * 15. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)
- * 16. Emit event - Produce to output topic
+ * 7. Apply event filters - Drop events that match the team's event filter
+ * 8. Apply cookieless processing - Rewrite distinct_id for cookieless events
+ * 9. Only-cookieless rate limit - Redirect cookieless rate-limited events to overflow
+ * using the hashed distinct_id from step 8
+ * 10. Prefetch hog functions - Warm the transformation cache for the chunk's teams (HOG_FUNCTIONS_PREFETCH_ENABLED)
+ * 11. Cymbal processing - Symbolicate, fingerprint, and link issues
+ * 12. Person properties - Fetch person by distinct_id (read-only)
+ * 13. Hog transformations - Run team transformations (including GeoIP if enabled)
+ * 14. Prepare event - Convert to PreIngestionEvent format, track if person found
+ * 15. Group type mapping - Map group types to indexes (read-only)
+ * 16. Create event - Build ErrorTrackingKafkaEvent (matches Cymbal's output format)
+ * 17. Emit event - Produce to output topic
*
* Note: Cymbal runs before enrichment because it only needs the raw exception data
* for symbolication and fingerprinting. This reduces payload size and avoids
@@ -152,6 +161,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)
groupTypeManager,
cookielessManager,
eventIngestionRestrictionManager,
+ eventFilterManager,
overflowMode,
preservePartitionLocality,
overflowRedirectService,
@@ -169,7 +179,11 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)
concurrentBatches: 1,
topHog,
})
- .beforeBatch((b) => b.pipe(createEventUsageBeforeBatchStep(createEventUsageBatch)))
+ .beforeBatch((b) =>
+ b
+ .pipe(createEventFiltersBatchAppMetricsBeforeBatchStep(outputs))
+ .pipe(createEventUsageBeforeBatchStep(createEventUsageBatch))
+ )
// Header-only steps: parse Kafka headers and apply token-level restrictions.
// Cheap; runs per-event before we touch the body.
.parseHeaders()
@@ -188,6 +202,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)
.pipeChunk(prefetchTeamsStep(teamManager, teamsPrefetchEnabled))
.parseMessage()
.resolveTeam()
+ .pipe(createApplyEventFiltersStep(eventFilterManager))
// Carry the Kafka message byte size through for Cymbal batch chunking.
.pipe(createAttachMessageBytesStep())
// Cookieless processing: rewrites event.distinct_id for cookieless
@@ -241,6 +256,7 @@ export function createErrorTrackingPipeline(config: ErrorTrackingPipelineConfig)
.pipe(createRecordIngestionLagStep())
.afterBatch((b) =>
b
+ .pipe(createFlushEventFiltersBatchAppMetricsStep())
.pipe(createFlushEventUsageStep())
// Drain hog transformer invocation results once per batch.
.pipe(createFlushHogTransformerStep(hogTransformer))