:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): skip events no destination or workflow admits by name (#129767)
Meikel Ratz7f3c0e27962a
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex d6a96fef1d3e..4bba4d832a1b 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -56,6 +56,9 @@ export type CdpConfig = ClickhouseConfig & {CDP_EVENTS_CONSUMER_TOPIC: stringCDP_EVENTS_CONSUMER_GROUP_ID: string+ // Off until every produced event carries the event name header and+ // cdp_events_unmatched_event_name_invoked stays at zero.+ CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES: booleanCDP_HOGFLOW_SUBSCRIPTION_MATCHER_REALTIME_ONLY_EVENTS_TOPIC: stringCDP_CYCLOTRON_BATCH_DELAY_MS: number@@ -319,6 +322,7 @@ export function getDefaultCdpConfig(): CdpConfig {CDP_EVENTS_CONSUMER_TOPIC: KAFKA_EVENTS_JSON,CDP_EVENTS_CONSUMER_GROUP_ID: 'cdp-processed-events-consumer',+ CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES: false,CDP_HOGFLOW_SUBSCRIPTION_MATCHER_REALTIME_ONLY_EVENTS_TOPIC: '',CDP_CYCLOTRON_BATCH_DELAY_MS: 50,diff --git a/nodejs/src/cdp/consumers/cdp-base.consumer.ts b/nodejs/src/cdp/consumers/cdp-base.consumer.tsindex ddc4e8865226..026737b93d26 100644--- a/nodejs/src/cdp/consumers/cdp-base.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-base.consumer.ts@@ -14,6 +14,7 @@ import {createCdpCoreServices,} from '../cdp-services'import type { CdpConfig } from '../config'+import { HogFlow } from '../schema/hogflow'import { HogExecutorAsyncService } from '../services/hog-executor-async.service'import { HogInputsService } from '../services/hog-inputs.service'import { HogFlowExecutorService } from '../services/hogflows/hogflow-executor.service'@@ -36,7 +37,9 @@ import { WorkflowsActivationReporter } from '../services/monitoring/workflows-acimport { NativeDestinationExecutorService } from '../services/native-destination-executor.service'import { SegmentDestinationExecutorService } from '../services/segment-destination-executor.service'import { CdpUsageReporterService } from '../services/usage/cdp-usage-reporter.service'-import { HogFunctionTypeType } from '../types'+import { HogFunctionType, HogFunctionTypeType } from '../types'++export type TeamWork = { team: Team; hogFunctions: HogFunctionType[]; hogFlows: HogFlow[] }export type CdpConsumerBaseConfig = CdpCoreServicesConfig &Pick<CommonConfig, 'KAFKA_CLIENT_RACK'> &@@ -136,12 +139,16 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = CdteamId: TeamId,hogTypes: HogFunctionTypeType[]): Promise<Team | null> {- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([+ return (await this.loadTeamWork(teamId, hogTypes))?.team ?? null+ }++ protected async loadTeamWork(teamId: TeamId, hogTypes: HogFunctionTypeType[]): Promise<TeamWork | null> {+ const [hogFunctions, hogFlows, team] = await Promise.all([this.hogFunctionManager.getHogFunctionsForTeam(teamId, hogTypes),this.hogFlowManager.getHogFlowsForTeam(teamId),this.deps.teamManager.getTeam(teamId),])- return teamHogFunctions.length || teamHogFlows.length ? team : null+ return team && (hogFunctions.length || hogFlows.length) ? { team, hogFunctions, hogFlows } : null}public async start(): Promise<void> {diff --git a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.tsindex 2b7bdcfc5b83..28f62e51bd49 100644--- a/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-dlq-replay.consumer.ts@@ -23,6 +23,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'import { HogFunctionInvocationGlobals, HogFunctionTypeType, InvocationBuildFailure } from '../types'import { convertToHogFunctionInvocationGlobals } from '../utils'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'+import { runsOnEvents, triggersOnEvents } from './event-name-gate'const counterReplayRecords = new Counter({name: 'cdp_dlq_replay_records_total',@@ -267,12 +268,12 @@ export class CdpDlqReplayConsumer extends CdpConsumerBase<PluginsServerConfig> {const [hogInvocations, hogflowInvocations] = await Promise.all([this.hogFunctionPipeline.buildInvocations(globalsList, {hogTypes: this.hogTypes,- filterFn: (fn) => (fn.filters?.source ?? 'events') === 'events',+ filterFn: runsOnEvents,invocationFilterFn: (fn, globals) => allowsKind('hog_function', globals) && allows(fn.id, globals),}),this.hogFlowPipeline.buildInvocations(globalsList, {eligibilityFn: (flow, globals) =>- flow.trigger.type === 'event' && allowsKind('hog_flow', globals) && allows(flow.id, globals),+ triggersOnEvents(flow) && allowsKind('hog_flow', globals) && allows(flow.id, globals),}),])diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.tsindex 9f07712616cb..9986a21666bf 100644--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts@@ -15,7 +15,7 @@ import {getTeam,updateOrganizationAvailableFeatures,} from '../../../tests/helpers/sql'-import { Hub, Team } from '../../types'+import { Hub, RawClickHouseEvent, Team } from '../../types'import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'import {HOG_EXAMPLES,@@ -36,6 +36,7 @@ import { HogWatcherState } from '../services/monitoring/hog-watcher.service'import { HogFunctionInvocationGlobals, HogFunctionType } from '../types'import { currentRuntimeContractHash } from '../utils/filter-runtime'import { CdpEventsConsumer } from './cdp-events.consumer'+import { counterEventNameSource, counterUnmatchedEventName, counterUnmatchedEventNameInvoked } from './metrics'jest.setTimeout(1000)@@ -858,6 +859,198 @@ describe('CdpEventsConsumer', () => {expect(mockQueueInvocations).not.toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])})+ describe('events no destination admits by name', () => {+ let pageviewFunction: HogFunctionType++ const withHeaders = (event: RawClickHouseEvent, offset = 0) =>+ createKafkaMessage(event, {+ offset,+ headers: [{ team_id: String(event.team_id) }, { event: event.event }],+ })++ beforeEach(async () => {+ pageviewFunction = await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter,+ })+ })++ it.each([+ [true, 0, 1],+ [false, 1, 0],+ ])(+ 'with skipping %s, parks %i unreadable bodies of an event no destination admits and records %i filtered',+ async (skip, parked, filtered) => {+ hub.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES = skip+ const unreadable = {+ ...withHeaders(createIncomingEvent(team.id, { event: 'signed_up' })),+ value: Buffer.from('{'),+ }++ await handleBatch([unreadable])++ const dlq = mockProducerObserver+ .getProducedMessages()+ .filter(({ topic }) => topic === 'cdp_events_dlq_test')+ expect(dlq).toHaveLength(parked)+ const filteredMetrics = mockProducerObserver+ .getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')+ .filter(({ value }: any) => value.app_source_id === pageviewFunction.id)+ .map(({ value }: any) => value.metric_name)+ expect(filteredMetrics).toEqual(Array(filtered).fill('filtered'))+ }+ )++ it('queues the same invocations and records the same metrics whether it skips or not', async () => {+ const hogFlow = await _insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder()+ .withTeamId(team.id)+ .withSimpleWorkflow({+ trigger: {+ type: 'event',+ filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters ?? {},+ },+ })+ .build()+ )+ processor['hogFlowManager']['onHogFlowsReloaded'](team.id, [hogFlow.id])+ const events = [+ createIncomingEvent(team.id, {+ properties: JSON.stringify({ $current_url: 'https://posthog.com' }),+ }),+ createIncomingEvent(team.id, {+ properties: JSON.stringify({ $current_url: 'https://other.com' }),+ }),+ createIncomingEvent(team.id, { event: '$autocapture' }),+ createIncomingEvent(team.id, { event: 'signed_up' }),+ createIncomingEvent(team.id, {+ event: '$push_notification_opened',+ properties: JSON.stringify({+ $notification_workflow_id: hogFlow.id,+ $notification_action_id: 'push_step_1',+ }),+ }),+ ]+ const counted = async () => ({+ unmatched: Object.fromEntries(+ (await counterUnmatchedEventName.get()).values.map(({ labels, value }) => [+ String(labels.skipped),+ value,+ ])+ ),+ invoked: (await counterUnmatchedEventNameInvoked.get()).values[0]?.value ?? 0,+ })+ const outcome = async (skip: boolean) => {+ hub.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES = skip+ mockQueueInvocations.mockClear()+ mockProducerObserver.resetKafkaProducer()+ counterUnmatchedEventName.reset()+ counterUnmatchedEventNameInvoked.reset()+ await handleBatch(events.map(withHeaders))+ const queued = mockQueueInvocations.mock.calls.flatMap(([invocations]: any[]) => invocations)+ return {+ invocations: queued+ .map((x: any) => `${x.functionId} ${x.state.globals?.event.uuid ?? x.state.event.uuid}`)+ .sort(),+ metrics: mockProducerObserver+ .getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')+ .map(({ value }: any) => `${value.app_source_id} ${value.metric_name} ${value.count}`)+ .sort(),+ }+ }++ const parsed = await outcome(false)+ expect(parsed.invocations).toHaveLength(4)+ expect(parsed.metrics).toContain(`${pageviewFunction.id} filtered 3`)+ expect(parsed.metrics).toContain(`${hogFlow.id} push_opened 1`)+ expect(await counted()).toEqual({ unmatched: { false: 1 }, invoked: 0 })++ expect(await outcome(true)).toEqual(parsed)+ expect(await counted()).toEqual({ unmatched: { true: 1 }, invoked: 0 })+ })++ it('parses an event without the name header and leaves it to the pre-filter, with skipping on', async () => {+ hub.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES = true+ counterUnmatchedEventName.reset()+ counterEventNameSource.reset()+ const signup = createIncomingEvent(team.id, { event: 'signed_up' })++ await handleBatch([+ createKafkaMessage(signup, { headers: [{ team_id: String(team.id) }] }),+ withHeaders(createIncomingEvent(team.id, { event: '$autocapture' }), 1),+ ])++ expect((await counterUnmatchedEventName.get()).values).toEqual([])+ expect(+ Object.fromEntries(+ (await counterEventNameSource.get()).values.map(({ labels, value }) => [+ labels.source,+ value,+ ])+ )+ ).toEqual({ none: 1, header: 1 })+ const metrics = mockProducerObserver+ .getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')+ .filter(({ value }: any) => value.app_source_id === pageviewFunction.id)+ .map(({ value }: any) => `${value.metric_name} ${value.count}`)+ .sort()+ expect(metrics).toEqual(['filtered 1', 'triggered 1'])+ })++ it('counts an event the gate rejects that a destination still invokes, while skipping is off', async () => {+ hub.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES = false+ counterUnmatchedEventNameInvoked.reset()+ const pageview = createIncomingEvent(team.id, {+ properties: JSON.stringify({ $current_url: 'https://posthog.com' }),+ })++ await handleBatch([+ createKafkaMessage(pageview, {+ headers: [{ team_id: String(team.id) }, { event: 'signed_up' }],+ }),+ ])++ expect((await counterUnmatchedEventNameInvoked.get()).values[0]?.value).toBe(1)+ })++ it('records the filtered metric of a skipped event once when its batch fails and is read again', async () => {+ hub.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES = true+ const manager = processor['hogFunctionManager']+ const original = manager.getHogFunctionsForTeam.bind(manager)+ const lookup = jest+ .spyOn(manager, 'getHogFunctionsForTeam')+ .mockImplementationOnce(original)+ .mockRejectedValueOnce(+ new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down'))+ )+ const batch = [+ withHeaders(createIncomingEvent(team.id, { event: 'signed_up' })),+ withHeaders(+ createIncomingEvent(team.id, {+ properties: JSON.stringify({ $current_url: 'https://posthog.com' }),+ }),+ 1+ ),+ ]++ try {+ await expect(handleBatch(batch)).rejects.toThrow('Postgres is down')+ } finally {+ lookup.mockRestore()+ }+ await handleBatch(batch)++ const filtered = mockProducerObserver+ .getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')+ .filter(({ value }: any) => value.app_source_id === pageviewFunction.id)+ .filter(({ value }: any) => value.metric_name === 'filtered')+ .reduce((sum: number, { value }: any) => sum + value.count, 0)+ expect(filtered).toBe(1)+ })+ })+it('fails the batch when the record cannot be written, so the offsets stay put', async () => {// Also guards the ordering: the handler has to await the write. Without the await// this rejects in the background and the consumer walks past the event anyway.diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex d228fb95a581..c704fd301959 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -15,13 +15,23 @@ import { HogFunctionInvocationPipeline } from '../services/hog-function-invocatiimport { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { parseBatch, readTeamIdHeader } from './message-parsing'-import { counterParseError, counterTeamIdHeaderMismatch, counterTeamIdSource } from './metrics'+import { EventNameGate, buildEventNameGate, runsOnEvents, triggersOnEvents } from './event-name-gate'+import { parseBatch, readEventNameHeader, readTeamIdHeader } from './message-parsing'+import {+ counterEventNameSource,+ counterParseError,+ counterTeamIdHeaderMismatch,+ counterTeamIdSource,+ counterUnmatchedEventName,+ counterUnmatchedEventNameInvoked,+} from './metrics'import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'const isEventObject = (value: unknown): value is RawClickHouseEvent =>typeof value === 'object' && value !== null && !Array.isArray(value)+type TeamWithEventNameGate = { team: Team; gate: EventNameGate }+export class CdpEventsConsumer<TConfig extends PluginsServerConfig = PluginsServerConfig,> extends CdpConsumerBase<TConfig> {@@ -36,6 +46,8 @@ export class CdpEventsConsumer<private deadLetterService: CdpDeadLetterService/** The message each set of globals was built from, so a failure can find its bytes. */private messageByGlobals = new WeakMap<HogFunctionInvocationGlobals, Message>()+ // Shadow scaffolding while the skip is off: a rejected event that still gets invoked is one the skip would drop.+ private rejectedByName = new WeakSet<HogFunctionInvocationGlobals>()constructor(config: TConfig,@@ -88,9 +100,9 @@ export class CdpEventsConsumer<beforeQueue?: () => Promise<void>): Promise<{ backgroundTask: Promise<any>; invocations: CyclotronJobInvocation[] }> {if (!invocationGlobals.length) {- // A batch whose every message failed to parse still has records to write.+ // A batch whose every message failed to parse or was skipped still has records and metrics to write.await beforeQueue?.()- return { backgroundTask: Promise.resolve(), invocations: [] }+ return { backgroundTask: this.flushMonitoring(), invocations: [] }}// TODO: Add a helper to hog functions to determine if they require groups or not and then only load those@@ -99,17 +111,23 @@ export class CdpEventsConsumer<const [hogBuild, hogflowBuild] = await Promise.all([this.hogFunctionPipeline.buildInvocationsWithRelease(invocationGlobals, {hogTypes: this.hogTypes,- filterFn: (fn) => (fn.filters?.source ?? 'events') === 'events',+ filterFn: runsOnEvents,}),// Source-compatibility lives in the consumer. The events consumer matches event-triggered// flows only; other trigger types (data-warehouse-table, batch, schedule, webhook, manual)// are dispatched from their respective consumers and never reach the executor from here.this.hogFlowPipeline.buildInvocationsWithRelease(invocationGlobals, {- eligibilityFn: (flow) => flow.trigger.type === 'event',+ eligibilityFn: triggersOnEvents,}),])const hogInvocations = hogBuild.invocationsconst hogflowInvocations = hogflowBuild.invocations+ if (!this.config.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES) {+ this.countInvokedAfterNameRejection(invocationGlobals, [+ ...hogInvocations.map((invocation) => invocation.state.globals.event.uuid),+ ...hogflowInvocations.flatMap((invocation) => invocation.state?.event.uuid ?? []),+ ])+ }const invocationsToBeQueued = [...hogInvocations, ...hogflowInvocations]@@ -146,14 +164,7 @@ export class CdpEventsConsumer<instrumentFn({ key: 'cdp.background_task.queue_hogflow_invocations', sendException: false }, () =>this.hogflowQueue.queueInvocations(hogflowInvocations)),- instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {- try {- await this.hogFunctionMonitoringService.flush()- } catch (err) {- captureException(err)- logger.error('🔴', 'Error producing queued messages for monitoring', { err })- }- }),+ this.flushMonitoring(),instrumentFn({ key: 'cdp.background_task.lifecycle_running_flush', sendException: false }, () =>this.invocationResultsService.invocationResultsRowsService.flush()),@@ -162,6 +173,17 @@ export class CdpEventsConsumer<}}+ private flushMonitoring(): Promise<void> {+ return instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {+ try {+ await this.hogFunctionMonitoringService.flush()+ } catch (err) {+ captureException(err)+ logger.error('🔴', 'Error producing queued messages for monitoring', { err })+ }+ })+ }+// Resolve each $push_notification_opened event's workflow and queue its push_opened app-metric.// The attribution + spoof guard live in buildPushOpenedMetric (pure + unit-tested).private async trackPushNotificationOpens(globals: HogFunctionInvocationGlobals[]): Promise<void> {@@ -193,13 +215,26 @@ export class CdpEventsConsumer<}}+ private countInvokedAfterNameRejection(globalsList: HogFunctionInvocationGlobals[], invokedUuids: string[]): void {+ const rejected = new Set(globalsList.filter((g) => this.rejectedByName.has(g)).map((g) => g.event.uuid))+ if (!rejected.size) {+ return+ }+ const invoked = new Set(invokedUuids.filter((uuid) => rejected.has(uuid)))+ counterUnmatchedEventNameInvoked.inc(invoked.size)+ }+@instrumented('cdpConsumer.handleEachBatch.parseKafkaMessages')public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {const park =(step: 'parse' | 'lookup') =>(message: Message, error: string): boolean =>this.deadLetterService.recordMessageFailure(message, { step, error })- return await parseBatch(messages, {+ const skip = this.config.CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES+ const gates = new Map<number, EventNameGate>()+ // Queued once the whole batch parsed, so a batch read again after a failed lookup records them once.+ const skippedMetrics: EventNameGate[] = []+ const invocationGlobals = await parseBatch<TeamWithEventNameGate>(messages, {errorLog: 'Error parsing message',parse: (message) => {const parseEvent = (): RawClickHouseEvent => {@@ -220,11 +255,21 @@ export class CdpEventsConsumer<const headerTeamId = readTeamIdHeader(message)if (headerTeamId !== null) {counterTeamIdSource.labels({ source: 'header' }).inc()+ const headerEventName = readEventNameHeader(message)+ counterEventNameSource.labels({ source: headerEventName === null ? 'none' : 'header' }).inc()// The body stays unparsed until `toGlobals` runs, and it only runs for a team that// has a destination or a flow. Most events belong to a team with neither.return {teamId: headerTeamId,- toGlobals: (team) => {+ toGlobals: ({ team, gate }) => {+ const rejected = headerEventName !== null && !gate.admits(headerEventName)+ if (rejected) {+ counterUnmatchedEventName.labels({ skipped: String(skip) }).inc()+ if (skip) {+ skippedMetrics.push(gate)+ return null+ }+ }const clickHouseEvent = parseEvent()if (clickHouseEvent.team_id !== headerTeamId) {// The loaded team comes from the header and the event data comes from@@ -233,7 +278,11 @@ export class CdpEventsConsumer<counterTeamIdHeaderMismatch.inc()throw new Error('team_id_header_mismatch')}- return buildGlobals(team, clickHouseEvent)+ const globals = buildGlobals(team, clickHouseEvent)+ if (rejected) {+ this.rejectedByName.add(globals)+ }+ return globals},}}@@ -242,10 +291,10 @@ export class CdpEventsConsumer<const clickHouseEvent = parseEvent()return {teamId: clickHouseEvent.team_id,- toGlobals: (team) => buildGlobals(team, clickHouseEvent),+ toGlobals: ({ team }) => buildGlobals(team, clickHouseEvent),}},- loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),+ loadTeam: (teamId) => this.loadTeamWithEventNameGate(teamId, gates),onInvalidTeamId: () => counterParseError.labels({ error: 'invalid_team_id' }).inc(),// Parked rather than dropped: a schema we cannot read today is often one a later// version can.@@ -257,6 +306,33 @@ export class CdpEventsConsumer<// the dead-letter circuit breaker, which also fails the batch.parkLookupError: park('lookup'),})+ for (const gate of skippedMetrics) {+ const { hogFunctions, hogFlows } = gate.filteredMetrics()+ this.hogFunctionMonitoringService.queueAppMetrics(hogFunctions, 'hog_function')+ this.hogFunctionMonitoringService.queueAppMetrics(hogFlows, 'hog_flow')+ }+ return invocationGlobals+ }++ // Looks up per message, so a failed lookup parks only its own message.+ private async loadTeamWithEventNameGate(+ teamId: number,+ gates: Map<number, EventNameGate>+ ): Promise<TeamWithEventNameGate | null> {+ const work = await this.loadTeamWork(teamId, this.hogTypes)+ if (!work) {+ return null+ }+ const gate =+ gates.get(teamId) ??+ buildEventNameGate({+ hogFunctions: work.hogFunctions,+ hogFlows: work.hogFlows,+ // Push open tracking reads these for any team with a flow, whatever the flows' filters.+ alsoAdmitted: work.hogFlows.length ? [PUSH_NOTIFICATION_OPENED_EVENT] : [],+ })+ gates.set(teamId, gate)+ return { team: work.team, gate }}protected async startQueueProducers(): Promise<void> {diff --git a/nodejs/src/cdp/consumers/event-name-gate.test.ts b/nodejs/src/cdp/consumers/event-name-gate.test.tsnew file mode 100644index 000000000000..c319b901e020--- /dev/null+++ b/nodejs/src/cdp/consumers/event-name-gate.test.ts@@ -0,0 +1,89 @@+import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'+import { createHogFunction } from '../_tests/fixtures'+import { HogFlow } from '../schema/hogflow'+import { HogFunctionFilters } from '../types'+import { buildEventNameGate } from './event-name-gate'++const named = (...ids: (string | null)[]) => ids.map((id) => ({ id, type: 'events' as const }))++const flowTriggeredBy = (trigger: HogFlow['trigger']): HogFlow =>+ new FixtureHogFlowBuilder().withSimpleWorkflow({ trigger }).build()++describe('buildEventNameGate', () => {+ it.each<[string, HogFunctionFilters | null, string, boolean]>([+ ['an event the filter names', { events: named('$pageview', 'signed_up') }, 'signed_up', true],+ ['an event the filter does not name', { events: named('$pageview') }, 'signed_up', false],+ [+ 'any event when the filter also matches actions',+ { events: named('$pageview'), actions: [{ id: '1', type: 'actions' }] },+ 'signed_up',+ true,+ ],+ ['any event when the filter lists all events', { events: named('$pageview', null) }, 'signed_up', true],+ ['any event when the filter names no event', { events: [], bytecode: ['_h', 29] }, 'signed_up', true],+ ['any event when the function has no filters', null, 'signed_up', true],+ [+ 'no event for a function that runs on another source',+ { source: 'person-updates', events: [] },+ 'signed_up',+ false,+ ],+ ])('a destination admits %s', (_, filters, eventName, admitted) => {+ const gate = buildEventNameGate({+ hogFunctions: [createHogFunction({ filters })],+ hogFlows: [],+ alsoAdmitted: [],+ })++ expect(gate.admits(eventName)).toBe(admitted)+ })++ it.each<[string, HogFlow['trigger'], string, boolean]>([+ ['an event its trigger names', { type: 'event', filters: { events: named('signed_up') } }, 'signed_up', true],+ ['no other event', { type: 'event', filters: { events: named('signed_up') } }, '$pageview', false],+ ['no event when it is not triggered by events', { type: 'schedule' }, '$pageview', false],+ ])('a workflow admits %s', (_, trigger, eventName, admitted) => {+ const gate = buildEventNameGate({ hogFunctions: [], hogFlows: [flowTriggeredBy(trigger)], alsoAdmitted: [] })++ expect(gate.admits(eventName)).toBe(admitted)+ })++ it('admits the events the caller reads for itself', () => {+ const gate = buildEventNameGate({+ hogFunctions: [createHogFunction({ filters: { events: named('$pageview') } })],+ hogFlows: [],+ alsoAdmitted: ['$push_notification_opened'],+ })++ expect(gate.admits('$push_notification_opened')).toBe(true)+ })++ it('records filtered for each destination and workflow the event would have been filtered by', () => {+ const destination = createHogFunction({ team_id: 7, filters: { events: named('$pageview') } })+ const otherSource = createHogFunction({ team_id: 7, filters: { source: 'person-updates' } })+ const workflow = { ...flowTriggeredBy({ type: 'event', filters: { events: named('signed_up') } }), team_id: 7 }+ const scheduled = flowTriggeredBy({ type: 'schedule' })++ const gate = buildEventNameGate({+ hogFunctions: [destination, otherSource],+ hogFlows: [workflow, scheduled],+ alsoAdmitted: [],+ })++ expect(gate.filteredMetrics()).toEqual({+ hogFunctions: [+ { team_id: 7, app_source_id: destination.id, metric_kind: 'other', metric_name: 'filtered', count: 1 },+ ],+ hogFlows: [+ {+ team_id: 7,+ app_source_id: workflow.id,+ metric_kind: 'other',+ metric_name: 'filtered',+ count: 1,+ app_source_version: { id: workflow.id, version: workflow.version },+ },+ ],+ })+ })+})diff --git a/nodejs/src/cdp/consumers/event-name-gate.ts b/nodejs/src/cdp/consumers/event-name-gate.tsnew file mode 100644index 000000000000..ffe2b1329c99--- /dev/null+++ b/nodejs/src/cdp/consumers/event-name-gate.ts@@ -0,0 +1,41 @@+import { HogFlow } from '../schema/hogflow'+import { HogFunctionType, MinimalAppMetric } from '../types'+import { filteredMetric, rejectsEventName } from '../utils/hog-function-filtering'++export const runsOnEvents = (fn: HogFunctionType): boolean => (fn.filters?.source ?? 'events') === 'events'++export const triggersOnEvents = (+ flow: HogFlow+): flow is HogFlow & { trigger: Extract<HogFlow['trigger'], { type: 'event' }> } => flow.trigger.type === 'event'++export type EventNameGate = {+ admits: (eventName: string) => boolean+ /** The metrics the pre-filter records for an event no source admits, so skipping it records the same. */+ filteredMetrics: () => { hogFunctions: MinimalAppMetric[]; hogFlows: MinimalAppMetric[] }+}++export function buildEventNameGate({+ hogFunctions,+ hogFlows,+ alsoAdmitted,+}: {+ hogFunctions: HogFunctionType[]+ hogFlows: HogFlow[]+ alsoAdmitted: string[]+}): EventNameGate {+ const eventFunctions = hogFunctions.filter(runsOnEvents)+ const eventFlows = hogFlows.filter(triggersOnEvents)+ const filters = [...eventFunctions.map((fn) => fn.filters), ...eventFlows.map((flow) => flow.trigger.filters)]++ return {+ admits: (eventName) =>+ alsoAdmitted.includes(eventName) || filters.some((filter) => !rejectsEventName(filter, eventName)),+ filteredMetrics: () => ({+ hogFunctions: eventFunctions.map(filteredMetric),+ hogFlows: eventFlows.map((flow) => ({+ ...filteredMetric(flow),+ app_source_version: { id: flow.id, version: flow.version },+ })),+ }),+ }+}diff --git a/nodejs/src/cdp/consumers/message-parsing.ts b/nodejs/src/cdp/consumers/message-parsing.tsindex 7007a09c0a18..b40c7b809bca 100644--- a/nodejs/src/cdp/consumers/message-parsing.ts+++ b/nodejs/src/cdp/consumers/message-parsing.ts@@ -1,6 +1,6 @@import { Message } from 'node-rdkafka'-import { TEAM_ID_HEADER } from '~/common/kafka/headers'+import { EVENT_NAME_HEADER, TEAM_ID_HEADER } from '~/common/kafka/headers'import { logger } from '~/common/utils/logger'import { isValidTeamId } from '~/common/utils/team-manager'@@ -26,13 +26,25 @@ export function assertTeamId<T extends { team_id: unknown }>(message: T): T {* the body instead, because a value that is not a usable team id must never become one.*/export function readTeamIdHeader(message: Message): number | null {+ const value = readHeader(message, TEAM_ID_HEADER)+ if (value === null) {+ return null+ }+ const teamId = Number(value)+ return isValidTeamId(teamId) ? teamId : null+}++/** Like the team id header: a caller may skip the body on this name, but reads the event from the body. */+export function readEventNameHeader(message: Message): string | null {+ return readHeader(message, EVENT_NAME_HEADER)+}++function readHeader(message: Message, name: string): string | null {for (const header of message.headers ?? []) {- const value = header[TEAM_ID_HEADER]- if (value === undefined) {- continue+ const value = header[name]+ if (value !== undefined) {+ return value.toString()}- const teamId = Number(value.toString())- return isValidTeamId(teamId) ? teamId : null}return null}@@ -62,7 +74,8 @@ export function skipIfMalformed<T>(errorLog: string, step: () => T, onError?: (e// the team has loaded.type ParsedMessage<TLoaded> = {teamId: unknown- toGlobals: (loaded: TLoaded) => HogFunctionInvocationGlobals+ /** Returns null to skip a message the loaded team has no use for. */+ toGlobals: (loaded: TLoaded) => HogFunctionInvocationGlobals | null}export type ParseBatchOptions<TLoaded = Team> = {diff --git a/nodejs/src/cdp/consumers/metrics.ts b/nodejs/src/cdp/consumers/metrics.tsindex 7b70d17ccea2..150c88e5caec 100644--- a/nodejs/src/cdp/consumers/metrics.ts+++ b/nodejs/src/cdp/consumers/metrics.ts@@ -24,6 +24,12 @@ export const counterBatchHogFlowTriggerFailed = new Counter({labelNames: ['hog_flow_id', 'reason'],})+export const counterEventNameSource = new Counter({+ name: 'cdp_events_event_name_source',+ help: 'For events whose team id came from the header, whether the event name header was there too. Only "header" events go through the name gate; "none" events are parsed and pre-filtered as before.',+ labelNames: ['source'],+})+export const counterTeamIdSource = new Counter({name: 'cdp_events_team_id_source',help: 'Where the consumer read an event team id from. "body" means the message carried no usable team id header, so the consumer parsed the event to find one.',@@ -34,3 +40,14 @@ export const counterTeamIdHeaderMismatch = new Counter({name: 'cdp_events_team_id_header_mismatch',help: 'An event body named a different team than its header. The event is parked, because the two teams would send one team data to another team destinations.',})++export const counterUnmatchedEventName = new Counter({+ name: 'cdp_events_unmatched_event_name',+ help: 'Events whose name no destination or workflow of their team admits. "skipped" is false while CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES is off and the consumer still parses them.',+ labelNames: ['skipped'],+})++export const counterUnmatchedEventNameInvoked = new Counter({+ name: 'cdp_events_unmatched_event_name_invoked',+ help: 'Events the name gate rejected that a destination or workflow still invoked once parsed. Counted while CDP_EVENTS_SKIP_UNMATCHED_EVENT_NAMES is off. Each one is an event the skip would drop, so it must stay at zero before the switch goes on. It counts only invocations that survive the watcher, rate limits, quotas and masking, so a wrong rejection for a disabled or limited destination does not show.',+})diff --git a/nodejs/src/cdp/utils/hog-function-filtering.ts b/nodejs/src/cdp/utils/hog-function-filtering.tsindex 98f9cdb34d63..cefbc2260469 100644--- a/nodejs/src/cdp/utils/hog-function-filtering.ts+++ b/nodejs/src/cdp/utils/hog-function-filtering.ts@@ -356,21 +356,22 @@ export function convertToHogFunctionFilterGlobal(const HOG_FILTERING_TIMEOUT_MS = 100-function preFilterResult(filters: HogFunctionType['filters'], filterGlobals: HogFunctionFilterGlobals): boolean {- const eventMatches = filters?.events?.some((eventFilter) => {- // We need to test if the id is null (all events) or if it is in the list of event matchers- return eventFilter.id === null || eventFilter.id === filterGlobals.event- })-- // If none of the event filters match we return false- if (!eventMatches) {+/** An action is a saved event filter, so only an events-only list can reject on the name; a null id admits all. */+export function rejectsEventName(filters: HogFunctionType['filters'], eventName: string): boolean {+ if (!filters?.events?.length || filters.actions?.length) {return false}- // If we get here, there is at least one event filter and it checks this event type- // hence we say its a match and return true- return true+ return !filters.events.some((eventFilter) => eventFilter.id === null || eventFilter.id === eventName)}+export const filteredMetric = (source: { team_id: number; id: string }): MinimalAppMetric => ({+ team_id: source.team_id,+ app_source_id: source.id,+ metric_kind: 'other',+ metric_name: 'filtered',+ count: 1,+})+/*** Shared utility to check if an event matches the filters of a HogFunction.* Used by both the HogExecutorService (for destinations) and HogTransformerService (for transformations).@@ -413,7 +414,6 @@ export async function filterFunctionInstrumented(options: {metrics,}- let preFilterMatch = null// Narrows what the catch block can blame. Each step sets it before the call that can throw.let reason: FilterErrorReason = 'unknown'@@ -428,23 +428,12 @@ export async function filterFunctionInstrumented(options: {return result}- // check whether we have a match with our pre-filter- // Only run if we have event filters and NO action filters (as actions are pre-saved event filters)- if (filters?.events?.length && !filters?.actions?.length) {- reason = 'prefilter'- preFilterMatch = preFilterResult(filters, filterGlobals)- if (preFilterMatch === false) {- hogFunctionPreFilterCounter.inc({ result: 'bytecode_execution_skipped__pre_filtered_out' })- result.match = false- metrics.push({- team_id: fn.team_id,- app_source_id: fn.id,- metric_kind: 'other',- metric_name: 'filtered',- count: 1,- })- return result- }+ reason = 'prefilter'+ if (rejectsEventName(filters, filterGlobals.event)) {+ hogFunctionPreFilterCounter.inc({ result: 'bytecode_execution_skipped__pre_filtered_out' })+ result.match = false+ metrics.push(filteredMetric(fn))+ return result}if (!filters?.bytecode) {@@ -484,13 +473,7 @@ export async function filterFunctionInstrumented(options: {result.match = typeof execHogOutcome.execResult.result === 'boolean' && execHogOutcome.execResult.resultif (!result.match) {- metrics.push({- team_id: fn.team_id,- app_source_id: fn.id,- metric_kind: 'other',- metric_name: 'filtered',- count: 1,- })+ metrics.push(filteredMetric(fn))}} catch (error) {const errorClass = classifyFilterError(error, reason, filters)