:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): drop all events for an event filter pipeline override (#120984)
Paweł Ledwoń68d2c06589b3
diff --git a/nodejs/src/ingestion/common/event-filters/manager.test.ts b/nodejs/src/ingestion/common/event-filters/manager.test.tsindex f5931a8c6912..cc6197616198 100644--- a/nodejs/src/ingestion/common/event-filters/manager.test.ts+++ b/nodejs/src/ingestion/common/event-filters/manager.test.ts@@ -6,7 +6,7 @@ describe('EventFilterManager', () => {}async function buildManager(): Promise<{ manager: EventFilterManager; stop: () => Promise<void> }> {- const scope = new EventFilterManagerComponent(mockPostgres as any)+ const scope = new EventFilterManagerComponent(mockPostgres as any, { pipeline: 'errortracking' })const started = await scope.start()return { manager: started.value, stop: started.stop }}@@ -143,15 +143,26 @@ describe('EventFilterManager', () => {await stop()})- it('fails to start when the first load fails', async () => {- mockPostgres.query.mockRejectedValue(new Error('connection refused'))+ function mockQueryRows(rowsByQueryName: Record<string, unknown[]>): void {+ mockPostgres.query.mockImplementation((_use, _sql, _params, queryName: string) =>+ Promise.resolve({ rows: rowsByQueryName[queryName] ?? [] })+ )+ }++ it.each([+ { name: 'filters', failingQuery: 'fetchAllEventFilters' },+ { name: 'pipeline overrides', failingQuery: 'fetchEventFilterDropAllEventsTeams' },+ ])('fails to start when the first load of $name fails', async ({ failingQuery }) => {+ mockPostgres.query.mockImplementation((_use, _sql, _params, queryName: string) =>+ queryName === failingQuery ? Promise.reject(new Error('connection refused')) : Promise.resolve({ rows: [] })+ )await expect(buildManager()).rejects.toThrow('connection refused')})- it('keeps the loaded filters when a background reload fails', async () => {- mockPostgres.query.mockResolvedValueOnce({- rows: [+ it('keeps the loaded filters and overrides when a background reload fails', async () => {+ mockQueryRows({+ fetchAllEventFilters: [{id: 'f1',team_id: 1,@@ -159,6 +170,7 @@ describe('EventFilterManager', () => {filter_tree: { type: 'condition', field: 'event_name', operator: 'exact', value: 'a' },},],+ fetchEventFilterDropAllEventsTeams: [{ team_id: 2 }],})const { manager, stop } = await buildManager()mockPostgres.query.mockRejectedValue(new Error('connection refused'))@@ -168,11 +180,13 @@ describe('EventFilterManager', () => {try {manager.getFilter(1)+ manager.dropsAllEvents(2)await new Promise((resolve) => setImmediate(resolve))- expect(mockPostgres.query).toHaveBeenCalledTimes(2)+ expect(mockPostgres.query).toHaveBeenCalledTimes(4)expect(unhandledRejection).not.toHaveBeenCalled()expect(manager.getFilter(1)!.id).toBe('f1')+ expect(manager.dropsAllEvents(2)).toBe(true)} finally {process.off('unhandledRejection', unhandledRejection)jest.restoreAllMocks()diff --git a/nodejs/src/ingestion/common/event-filters/manager.ts b/nodejs/src/ingestion/common/event-filters/manager.tsindex c75a98e07f3c..fb3bd0c8963e 100644--- a/nodejs/src/ingestion/common/event-filters/manager.ts+++ b/nodejs/src/ingestion/common/event-filters/manager.ts@@ -1,23 +1,33 @@import { BackgroundRefresher } from '~/common/utils/background-refresher'import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'+import { IngestionPipeline } from '~/common/utils/event-ingestion-restrictions'import { logger } from '~/common/utils/logger'import { Component } from '~/ingestion/common/scopes'import { treeHasConditions } from './evaluate'-import { eventFiltersRefreshErrors } from './metrics'+import { eventFiltersOverrideRefreshErrors, eventFiltersRefreshErrors } from './metrics'import { EventFilterRowSchema, EventFilterRule } from './schema'+export interface EventFilterManagerOptions {+ pipeline: IngestionPipeline+}+/*** Manages per-team event filter config loaded from Postgres. One filter* per team, cached via a `BackgroundRefresher` that reloads all active- * filters every 60s. The constructor does not prime the cache; call+ * filters every 60s. The pipeline overrides that drop all events for+ * `options.pipeline` reload the same way. The constructor does not prime the cache; call* `prime()` (or wrap construction in `EventFilterManagerComponent`) to* await the initial load.*/export class EventFilterManager {private refresher: BackgroundRefresher<Map<number, EventFilterRule>>+ private dropAllEventsTeams: BackgroundRefresher<Set<number>>- constructor(private readonly postgres: PostgresRouter) {+ constructor(+ private readonly postgres: PostgresRouter,+ private readonly options: EventFilterManagerOptions+ ) {// A failed reload keeps the last loaded filters. The default handler rethrows from a// detached promise, and the server stops on an unhandled rejection.this.refresher = new BackgroundRefresher(@@ -28,12 +38,23 @@ export class EventFilterManager {eventFiltersRefreshErrors.inc()})+ this.dropAllEventsTeams = new BackgroundRefresher(+ async () => this.fetchDropAllEventsTeams(),+ 60_000,+ (error) => {+ logger.error('Failed to refresh event filter pipeline overrides', {+ pipeline: this.options.pipeline,+ error,+ })+ eventFiltersOverrideRefreshErrors.inc()+ }+ )}async prime(): Promise<void> {- // Do not catch a first load error: startup must fail. A consumer without its filters- // ingests the events that a filter drops, and those events cannot be removed later.- await this.refresher.get()+ // Do not catch a first load error: startup must fail. A consumer without its filters or+ // overrides ingests the events that they drop, and those events cannot be removed later.+ await Promise.all([this.refresher.get(), this.dropAllEventsTeams.get()])}/** Returns the filter for a team, or null if disabled or has no conditions. Non-blocking. */@@ -45,6 +66,23 @@ export class EventFilterManager {return filter}+ /** True when the team's override drops every event in this pipeline. Non-blocking. */+ dropsAllEvents(teamId: number): boolean {+ return this.dropAllEventsTeams.tryGet()?.has(teamId) ?? false+ }++ private async fetchDropAllEventsTeams(): Promise<Set<number>> {+ const { rows } = await this.postgres.query<{ team_id: number }>(+ PostgresUse.COMMON_READ,+ `SELECT team_id+ FROM posthog_eventfilterpipelineoverride+ WHERE pipeline = $1 AND drop_all_events`,+ [this.options.pipeline],+ 'fetchEventFilterDropAllEventsTeams'+ )+ return new Set(rows.map((row) => Number(row.team_id)))+ }+private async fetchAllFilters(): Promise<Map<number, EventFilterRule>> {const { rows } = await this.postgres.query<{id: string@@ -86,10 +124,13 @@ export class EventFilterManager {* them.*/export class EventFilterManagerComponent implements Component<EventFilterManager> {- constructor(private readonly postgres: PostgresRouter) {}+ constructor(+ private readonly postgres: PostgresRouter,+ private readonly options: EventFilterManagerOptions+ ) {}async start(): Promise<{ value: EventFilterManager; stop: () => Promise<void> }> {- const manager = new EventFilterManager(this.postgres)+ const manager = new EventFilterManager(this.postgres, this.options)await manager.prime()return { value: manager, stop: () => Promise.resolve() }}diff --git a/nodejs/src/ingestion/common/event-filters/metrics.ts b/nodejs/src/ingestion/common/event-filters/metrics.tsindex d0c78d14720f..e4c9d541faed 100644--- a/nodejs/src/ingestion/common/event-filters/metrics.ts+++ b/nodejs/src/ingestion/common/event-filters/metrics.ts@@ -10,3 +10,8 @@ export const eventFiltersRefreshErrors = new Counter({name: 'ingestion_filters_refresh_errors',help: 'Number of failed background reloads of event filter config',})++export const eventFiltersOverrideRefreshErrors = new Counter({+ name: 'ingestion_filters_override_refresh_errors',+ help: 'Number of failed background reloads of event filter pipeline overrides',+})diff --git a/nodejs/src/ingestion/common/steps/event-filters-steps.test.ts b/nodejs/src/ingestion/common/steps/event-filters-steps.test.tsindex 99f7088b915d..ad776adb6dfc 100644--- a/nodejs/src/ingestion/common/steps/event-filters-steps.test.ts+++ b/nodejs/src/ingestion/common/steps/event-filters-steps.test.ts@@ -81,11 +81,28 @@ describe('createEventFiltersBatchAppMetricsBeforeBatchStep', () => {describe('createApplyEventFiltersStep', () => {const mockManager = {getFilter: jest.fn(),- } as unknown as jest.Mocked<Pick<EventFilterManager, 'getFilter'>> & EventFilterManager+ dropsAllEvents: jest.fn(),+ } as unknown as jest.Mocked<Pick<EventFilterManager, 'getFilter' | 'dropsAllEvents'>> & EventFilterManagerbeforeEach(() => {jest.clearAllMocks()resetMetrics()+ mockManager.dropsAllEvents.mockReturnValue(false)+ })++ it('drops events the pipeline override drops, without a filter', async () => {+ mockManager.dropsAllEvents.mockReturnValue(true)+ mockManager.getFilter.mockReturnValue(null)+ const step = createApplyEventFiltersStep(mockManager)+ const input = {+ team: createTestTeam({ id: 1 }),+ headers: createTestEventHeaders({ event: '$exception' }),+ eventFiltersBatchAppMetrics: new EventFiltersBatchAppMetrics(mockOutputs),+ }++ const result = await step(input)++ expect(result).toEqual(expect.objectContaining({ reason: 'event_filter_pipeline_override' }))})it('passes through when no filter exists for the team', async () => {diff --git a/nodejs/src/ingestion/common/steps/event-filters-steps.ts b/nodejs/src/ingestion/common/steps/event-filters-steps.tsindex 25e5b9ed78f0..f5193079e466 100644--- a/nodejs/src/ingestion/common/steps/event-filters-steps.ts+++ b/nodejs/src/ingestion/common/steps/event-filters-steps.ts@@ -38,6 +38,9 @@ export interface ApplyEventFiltersInput {/*** Creates a pipeline step that evaluates customer-configured event filters.*+ * A pipeline override that drops all events for the team drops the event before the filter+ * tree is evaluated.+ ** Uses event headers (event name, distinct_id) so it can run before the Kafka* message is parsed into a full event — avoiding wasted work on dropped events.*@@ -52,6 +55,11 @@ export function createApplyEventFiltersStep<T extends ApplyEventFiltersInput>(manager: EventFilterManager): ProcessingStep<T, T> {return function applyEventFiltersStep(input: T): Promise<PipelineResult<T>> {+ if (manager.dropsAllEvents(input.team.id)) {+ eventFiltersEventsEvaluated.inc({ outcome: 'pipeline_dropped' })+ return Promise.resolve(drop('event_filter_pipeline_override'))+ }+const filter = manager.getFilter(input.team.id)if (!filter) {diff --git a/nodejs/src/ingestion/ingestion-consumer.ts b/nodejs/src/ingestion/ingestion-consumer.tsindex 4cc94a1a1583..6e2f9028369d 100644--- a/nodejs/src/ingestion/ingestion-consumer.ts+++ b/nodejs/src/ingestion/ingestion-consumer.ts@@ -185,7 +185,7 @@ export class IngestionConsumer {staticSkipPersonTokens: this.tokenDistinctIdsToSkipPersons,staticForceOverflowTokens: this.tokenDistinctIdsToForceOverflow,})- this.eventFilterManagerComponent = new EventFilterManagerComponent(deps.postgres)+ this.eventFilterManagerComponent = new EventFilterManagerComponent(deps.postgres, { pipeline: 'analytics' })// Schema loads run detached in the LazyLoader buffer, so an un-retried transient// failure can surface as an unhandled rejection and restart the worker.this.eventSchemaEnforcementManager = new EventSchemaEnforcementManager(deps.postgres, {diff --git a/nodejs/src/ingestion/pipelines/ai/consumer.ts b/nodejs/src/ingestion/pipelines/ai/consumer.tsindex 32b04d73656c..3c86acf9ab12 100644--- a/nodejs/src/ingestion/pipelines/ai/consumer.ts+++ b/nodejs/src/ingestion/pipelines/ai/consumer.ts@@ -128,7 +128,7 @@ export function createAiConsumer(config: AiConsumerConfig, sharedScope: AiSharedstaticForceOverflowTokens: splitTokens(config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID),}))- .add('eventFilterManager', new EventFilterManagerComponent(container.postgres))+ .add('eventFilterManager', new EventFilterManagerComponent(container.postgres, { pipeline: 'ai' }))// Dedicated 'ai' keyspace so AI overflow never affects analytics..add('overflowRedirectService',diff --git a/nodejs/src/ingestion/pipelines/ai/pipeline.test.ts b/nodejs/src/ingestion/pipelines/ai/pipeline.test.tsindex 128d757c0309..cc7a396dfc62 100644--- a/nodejs/src/ingestion/pipelines/ai/pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/ai/pipeline.test.ts@@ -37,7 +37,7 @@ describe('AiIngestionPipeline', () => {let mockKafkaProducer: jest.Mocked<KafkaProducerWrapper>let mockTeamManager: jest.Mocked<TeamManager>let mockEventIngestionRestrictionManager: jest.Mocked<EventIngestionRestrictionManager>- let mockEventFilterManager: { getFilter: jest.Mock }+ let mockEventFilterManager: { getFilter: jest.Mock; dropsAllEvents: jest.Mock }let mockCookielessManager: jest.Mocked<CookielessManager>let mockHogTransformer: jest.Mocked<Pick<@@ -117,7 +117,10 @@ describe('AiIngestionPipeline', () => {forceRefresh: jest.fn(),} as unknown as jest.Mocked<EventIngestionRestrictionManager>- mockEventFilterManager = { getFilter: jest.fn().mockReturnValue(undefined) }+ mockEventFilterManager = {+ getFilter: jest.fn().mockReturnValue(undefined),+ dropsAllEvents: jest.fn().mockReturnValue(false),+ }mockCookielessManager = {doBatch: jest.fn().mockImplementation((events: any[]) => Promise.resolve(events.map((e) => ok(e)))),diff --git a/nodejs/src/ingestion/pipelines/clientwarnings/consumer.ts b/nodejs/src/ingestion/pipelines/clientwarnings/consumer.tsindex db6ae1db0adf..a81d93659992 100644--- a/nodejs/src/ingestion/pipelines/clientwarnings/consumer.ts+++ b/nodejs/src/ingestion/pipelines/clientwarnings/consumer.ts@@ -40,7 +40,10 @@ export function createClientWarningsConsumer(staticDropEventTokens,}))- .add('eventFilterManager', new EventFilterManagerComponent(container.postgres))+ .add(+ 'eventFilterManager',+ new EventFilterManagerComponent(container.postgres, { pipeline: 'clientwarnings' })+ ).add('outputs',new IngestionOutputsComponent(() => createOutputsRegistry().build(container.producerRegistry, config))diff --git a/nodejs/src/ingestion/pipelines/clientwarnings/pipeline.test.ts b/nodejs/src/ingestion/pipelines/clientwarnings/pipeline.test.tsindex 61e1bc7feb4c..40c49890b994 100644--- a/nodejs/src/ingestion/pipelines/clientwarnings/pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/clientwarnings/pipeline.test.ts@@ -102,6 +102,7 @@ describe('ClientWarningsPipeline', () => {mockEventFilterManager = {getFilter: jest.fn().mockReturnValue(undefined),+ dropsAllEvents: jest.fn().mockReturnValue(false),} as unknown as jest.Mocked<EventFilterManager>promiseScheduler = new PromiseScheduler()diff --git a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts b/nodejs/src/ingestion/pipelines/errortracking/consumer.tsindex 42063f479486..43a88a4c9b76 100644--- a/nodejs/src/ingestion/pipelines/errortracking/consumer.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/consumer.ts@@ -82,7 +82,10 @@ export function createErrorTrackingConsumer(config: ErrorTrackingLaneConfig, sha'eventIngestionRestrictionManager',new EventIngestionRestrictionManagerComponent(container.redisPool, { pipeline: 'errortracking' }))- .add('eventFilterManager', new EventFilterManagerComponent(container.postgres))+ .add(+ 'eventFilterManager',+ new EventFilterManagerComponent(container.postgres, { pipeline: 'errortracking' })+ ).add('overflowRedirectService',overflowMode === 'redirect'diff --git a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.tsindex 29ded8f3370c..f8c08fc34a23 100644--- a/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/errortracking/error-tracking-pipeline.test.ts@@ -292,6 +292,7 @@ describe('ErrorTrackingPipeline', () => {mockEventFilterManager = {getFilter: jest.fn().mockReturnValue(null),+ dropsAllEvents: jest.fn().mockReturnValue(false),} as unknown as jest.Mocked<EventFilterManager>// Passthrough: non-cookieless events round-trip unchanged. The apply stepdiff --git a/nodejs/src/ingestion/pipelines/heatmaps/consumer.ts b/nodejs/src/ingestion/pipelines/heatmaps/consumer.tsindex aedade6724bc..0befa767bd7c 100644--- a/nodejs/src/ingestion/pipelines/heatmaps/consumer.ts+++ b/nodejs/src/ingestion/pipelines/heatmaps/consumer.ts@@ -39,7 +39,7 @@ export function createHeatmapsConsumer(config: HeatmapsConsumerConfig, sharedScostaticDropEventTokens,}))- .add('eventFilterManager', new EventFilterManagerComponent(container.postgres))+ .add('eventFilterManager', new EventFilterManagerComponent(container.postgres, { pipeline: 'heatmaps' })).add('outputs',new IngestionOutputsComponent(() => createOutputsRegistry().build(container.producerRegistry, config))diff --git a/nodejs/src/ingestion/pipelines/heatmaps/pipeline.test.ts b/nodejs/src/ingestion/pipelines/heatmaps/pipeline.test.tsindex 5e226a01e7a7..8974a7b4e9b0 100644--- a/nodejs/src/ingestion/pipelines/heatmaps/pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/heatmaps/pipeline.test.ts@@ -116,6 +116,7 @@ describe('HeatmapsPipeline', () => {mockEventFilterManager = {getFilter: jest.fn().mockReturnValue(undefined),+ dropsAllEvents: jest.fn().mockReturnValue(false),} as unknown as jest.Mocked<EventFilterManager>// Passthrough by default: events round-trip unchanged. The apply step reads `.value.event`diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.tsindex 0808abe704dd..98b6429f10e3 100644--- a/nodejs/src/servers/ingestion-api-server.ts+++ b/nodejs/src/servers/ingestion-api-server.ts@@ -446,7 +446,9 @@ export class IngestionApiServer implements NodeServer {concurrentBatches: this.config.INGESTION_WORKER_CONCURRENT_BATCHES,createEventUsageBatch: createEventUsageBatchFactory(this.config, 'events'),}- const eventFilterManagerStarted = await new EventFilterManagerComponent(this.postgres).start()+ const eventFilterManagerStarted = await new EventFilterManagerComponent(this.postgres, {+ pipeline: 'analytics',+ }).start()const featureFlagCalledDedupService = createFeatureFlagCalledDedupService(this.featureFlagCalledDedupRedisPool,this.configdiff --git a/nodejs/tests/ingestion/common/event-filters/manager.integration.test.ts b/nodejs/tests/ingestion/common/event-filters/manager.integration.test.tsindex 089af4a80540..7d8ef95951b4 100644--- a/nodejs/tests/ingestion/common/event-filters/manager.integration.test.ts+++ b/nodejs/tests/ingestion/common/event-filters/manager.integration.test.ts@@ -2,6 +2,7 @@ import { v4 } from 'uuid'import { defaultConfig } from '~/common/config/config'import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'+import { IngestionPipeline } from '~/common/utils/event-ingestion-restrictions'import { evaluateFilterTree } from '~/ingestion/common/event-filters/evaluate'import { EventFilterManager, EventFilterManagerComponent } from '~/ingestion/common/event-filters/manager'import { and, cond, not, or } from '~/ingestion/common/event-filters/test-helpers'@@ -48,6 +49,30 @@ async function insertRawFilter()}+async function insertPipelineOverride(+ postgres: PostgresRouter,+ teamId: number,+ pipeline: IngestionPipeline,+ dropAllEvents: boolean+) {+ await insertRow(postgres, 'posthog_eventfilterpipelineoverride', {+ id: v4(),+ team_id: teamId,+ pipeline,+ drop_all_events: dropAllEvents,+ created_at: new Date().toISOString(),+ updated_at: new Date().toISOString(),+ })+}++interface PipelineOverrideCase {+ name: string+ input: {+ override: { pipeline: IngestionPipeline; dropAllEvents: boolean } | null+ }+ expected: { dropsAllEvents: boolean }+}+describe('EventFilterManager integration', () => {let postgres: PostgresRouterlet teamId: number@@ -65,15 +90,47 @@ describe('EventFilterManager integration', () => {await postgres.end()})- async function createManagerAndWaitForLoad(): Promise<{+ async function createManagerAndWaitForLoad(pipeline: IngestionPipeline = 'analytics'): Promise<{manager: EventFilterManagerstop: () => Promise<void>}> {- const scope = new EventFilterManagerComponent(postgres)+ const scope = new EventFilterManagerComponent(postgres, { pipeline })const started = await scope.start()return { manager: started.value, stop: started.stop }}+ it.each<PipelineOverrideCase>([+ {+ name: 'no override',+ input: { override: null },+ expected: { dropsAllEvents: false },+ },+ {+ name: 'an override for this pipeline',+ input: { override: { pipeline: 'errortracking', dropAllEvents: true } },+ expected: { dropsAllEvents: true },+ },+ {+ name: 'an override for another pipeline',+ input: { override: { pipeline: 'analytics', dropAllEvents: true } },+ expected: { dropsAllEvents: false },+ },+ {+ name: 'an override that keeps events',+ input: { override: { pipeline: 'errortracking', dropAllEvents: false } },+ expected: { dropsAllEvents: false },+ },+ ])('reads pipeline overrides: $name', async ({ input, expected }) => {+ if (input.override) {+ await insertPipelineOverride(postgres, teamId, input.override.pipeline, input.override.dropAllEvents)+ }++ const { manager, stop } = await createManagerAndWaitForLoad('errortracking')+ expect(manager.dropsAllEvents(teamId)).toBe(expected.dropsAllEvents)++ await stop()+ })+it('loads a live filter from postgres and evaluates it', async () => {await insertFilter(postgres, teamId, 'live', cond('event_name', 'exact', '$internal'))