:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): stop dropping events when a consumer's team lookup fails (#121819)
Phil Haack54e3087e2182
diff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.tsindex 9c3700e7a958..93259bc1ba79 100644--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts@@ -15,6 +15,7 @@ import { insertHogFlow } from '../_tests/fixtures-hogflows'import { COHORT_MEMBERSHIP_CHANGED_EVENT } from '../services/cohorts/cohort-membership-repository'import { CyclotronJobInvocationHogFlow } from '../types'import { CdpCohortMembershipTriggerConsumer } from './cdp-cohort-membership-trigger.consumer'+import { counterParseError } from './metrics'describe('CdpCohortMembershipTriggerConsumer', () => {let processor: CdpCohortMembershipTriggerConsumer@@ -136,6 +137,23 @@ describe('CdpCohortMembershipTriggerConsumer', () => {await expect(processor._parseKafkaBatch([createKafkaMessage(change)])).rejects.toThrow('conflict with recovery')})+ it('skips a change with an out-of-range team_id as a parse error and keeps the rest of the batch', async () => {+ // The dropped-changes alert counts parse errors. It does not count a skip with any other reason.+ const invalidTeamIdErrors = async (): Promise<number> =>+ (await counterParseError.get()).values.find((v) => v.labels.error === 'invalid_team_id')?.value ?? 0+ await insertHogFlow(hub.postgres, cohortFlow(42, 'entered'))+ const change = createCohortMembershipEvent({ team_id: team.id, cohort_id: 42, status: 'entered' })+ const before = await invalidTeamIdErrors()++ const globals = await processor._parseKafkaBatch([+ createKafkaMessage({ ...change, team_id: 3_000_000_000 }),+ createKafkaMessage(change),+ ])++ expect(globals).toHaveLength(1)+ expect(await invalidTeamIdErrors()).toBe(before + 1)+ })+it("loads a team's workflows once per batch, however many changes the batch holds for it", async () => {await insertHogFlow(hub.postgres, cohortFlow(42, 'entered'))const lookup = jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeams')diff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.tsindex 6da16031af5a..1a93587dd9ed 100644--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts@@ -22,7 +22,7 @@ import { CyclotronJobInvocation, HogFunctionInvocationGlobals } from '../types'import { convertBatchHogFlowRequestToHogFunctionInvocationGlobals } from '../utils'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'import { CohortMembershipChange, CohortMembershipChangeSchema } from './cdp-cohort-membership.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'export const counterCohortMembershipTriggerSkipped = new Counter({name: 'cdp_cohort_membership_trigger_skipped',@@ -159,12 +159,10 @@ export class CdpCohortMembershipTriggerConsumer extends CdpConsumerBase {public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {const changes: CohortMembershipChange[] = []for (const message of messages) {- let change: CohortMembershipChange- try {- change = CohortMembershipChangeSchema.parse(parseJSON(message.value!.toString()))- } catch (e) {- logger.error('Error parsing cohort membership change', e)- counterParseError.labels({ error: e.message }).inc()+ const change = skipIfMalformed('Error parsing cohort membership change', () =>+ assertTeamId(CohortMembershipChangeSchema.parse(parseJSON(message.value!.toString())))+ )+ if (!change) {continue}diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.tsindex 815b764f3a1c..a2ccc873f079 100644--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts@@ -155,6 +155,56 @@ describe('CdpDatawarehouseEventsConsumer', () => {expect(invocations).toHaveLength(0)})+ describe('with a hog function for the team', () => {+ beforeEach(async () => {+ await insertHogFunction({+ team_id: team.id,+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,+ ...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,+ })+ })++ const lookupError = new Error('canceling statement due to conflict with recovery')++ it.each([+ {+ lookup: 'the team lookup',+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),+ },+ {+ lookup: 'the hog function lookup',+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),+ },+ {+ lookup: 'the hog flow lookup',+ spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam'),+ },+ ])(+ 'should fail the batch when $lookup throws, and queue the row when the batch is read again',+ async ({ spy }) => {+ spy().mockRejectedValueOnce(lookupError)+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]+ const messages = [createKafkaMessage(createDataWarehouseEvent(team.id))]++ await expect(handleBatch(messages)).rejects.toBe(lookupError)+ expect(mockQueueInvocations).not.toHaveBeenCalled()++ const { backgroundTask } = await handleBatch(messages)+ await backgroundTask+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ }+ )++ it('should skip a row with an out-of-range team_id and keep the rest', async () => {+ const invocations = await processor._parseKafkaBatch([+ createKafkaMessage({ ...createDataWarehouseEvent(team.id), team_id: 3_000_000_000 }),+ createKafkaMessage(createDataWarehouseEvent(team.id)),+ ])+ expect(invocations).toHaveLength(1)+ })+ })+it('should handle schema validation errors gracefully', async () => {const invalidEvent = { team_id: 'not-a-number' }const messages = [createKafkaMessage(invalidEvent)]diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.tsindex a3c869edf4e7..d638ae6cbe5b 100644--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts@@ -20,7 +20,7 @@ import {HogFunctionTypeType,} from '../types'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'// Filter source / trigger type each kind of warehouse row matches against.const TRIGGER_TYPE_BY_TABLE_TYPE = {@@ -152,26 +152,29 @@ export class CdpDatawarehouseEventsConsumer extends CdpConsumerBase {await Promise.all(messages.map(async (message) => {- try {- const kafkaEvent = parseJSON(message.value!.toString()) as unknown- const event = CdpDataWarehouseEventSchema.parse(kafkaEvent)+ const event = skipIfMalformed('Error parsing message', () =>+ assertTeamId(CdpDataWarehouseEventSchema.parse(parseJSON(message.value!.toString())))+ )+ if (!event) {+ return+ }- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([- this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),- this.hogFlowManager.getHogFlowsForTeam(event.team_id),- this.deps.teamManager.getTeam(event.team_id),- ])+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.+ const [teamHogFunctions, teamHogFlows, team] = await Promise.all([+ this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),+ this.hogFlowManager.getHogFlowsForTeam(event.team_id),+ this.deps.teamManager.getTeam(event.team_id),+ ])- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {- return- }+ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {+ return+ }- events.push(- convertDataWarehouseEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)- )- } catch (e) {- logger.error('Error parsing message', e)- counterParseError.labels({ error: e.message }).inc()+ const globals = skipIfMalformed('Error parsing message', () =>+ convertDataWarehouseEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)+ )+ if (globals) {+ events.push(globals)}}))diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.tsindex ff4c4ef444e6..8ad9283dddf5 100644--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts@@ -51,6 +51,18 @@ describe('CdpEventsConsumer', () => {return item}+ const handleBatch = async (messages: any[]): Promise<void> => {+ const connect = processor['kafkaConsumer'].connect as jest.Mock+ const { backgroundTask } = await connect.mock.calls[0][0](messages)+ await backgroundTask+ }++ const lookupSpies = [+ { lookup: 'team', spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam') },+ { lookup: 'hog function', spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam') },+ { lookup: 'hog flow', spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam') },+ ]+beforeEach(async () => {hub = await createHub()const fixture = await createTestTeamFixture(hub.postgres)@@ -118,7 +130,7 @@ describe('CdpEventsConsumer', () => {})})- describe('parse failures', () => {+ describe('lookup failures', () => {beforeEach(async () => {await insertHogFunction({team_id: team.id,@@ -128,43 +140,37 @@ describe('CdpEventsConsumer', () => {})})- it.each([- [- 'rethrows a retriable team lookup error so the batch is retried',- new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset')),- true,- ],- ['drops the message on a non-retriable team lookup error', new Error('event shape is wrong'), false],- ])('%s', async (_name, error, shouldRethrow) => {- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)- const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]-- if (shouldRethrow) {- await expect(processor._parseKafkaBatch(messages)).rejects.toThrow(error)- } else {- await expect(processor._parseKafkaBatch(messages)).resolves.toEqual([])- }- })-- it('lets a retriable failure escape the batch handler, and processes the batch on a later attempt', async () => {- // The consumer stores offsets only after this handler resolves, so the handler- // rejecting is what leaves the batch unacknowledged. A try/catch added around- // _parseKafkaBatch here would swallow the rejection and silently restore the old- // drop-everything behaviour, which the unit assertion above cannot see.- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))- const getTeam = jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)- const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]- const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]-- await expect(handleBatch(messages)).rejects.toThrow(error)- expect(mockQueueInvocations).not.toHaveBeenCalled()-- getTeam.mockRestore()- const { backgroundTask } = await handleBatch(messages)- await backgroundTask+ it.each(+ lookupSpies.flatMap(({ lookup, spy }) => [+ {+ lookup,+ kind: 'a retriable error',+ dlqEnabled: true,+ spy,+ lookupError: new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset')),+ },+ {+ lookup,+ kind: 'a non-retriable error with the dead-letter topic off',+ dlqEnabled: false,+ spy,+ lookupError: new Error('column "bytecode" does not exist'),+ },+ ])+ )(+ 'fails the batch when the $lookup lookup throws $kind, and queues the event when the batch is read again',+ async ({ spy, dlqEnabled, lookupError }) => {+ hub.CDP_DLQ_ENABLED = dlqEnabled+ spy().mockRejectedValueOnce(lookupError)+ const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]++ await expect(handleBatch(messages)).rejects.toBe(lookupError)+ expect(mockQueueInvocations).not.toHaveBeenCalled()- expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])- })+ await handleBatch(messages)+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ }+ )})describe('general event processing', () => {@@ -596,12 +602,6 @@ describe('CdpEventsConsumer', () => {})describe('dead letter queue', () => {- const handleBatch = async (messages: any[]): Promise<void> => {- const connect = processor['kafkaConsumer'].connect as jest.Mock- const { backgroundTask } = await connect.mock.calls[0][0](messages)- await backgroundTask- }-beforeEach(() => {hub.CDP_DLQ_ENABLED = true})@@ -756,18 +756,41 @@ describe('CdpEventsConsumer', () => {expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)})- it('parks a message it cannot parse', async () => {+ it.each([+ {+ name: 'a message that is not an event object',+ bad: () => createKafkaMessage('not json at all'),+ },+ {+ name: 'an event whose properties are not JSON',+ bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),+ },+ {+ name: 'an event whose team_id is out of int4 range',+ bad: (teamId: number) =>+ createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: 3_000_000_000 }),+ },+ ...lookupSpies.map(({ lookup, spy }) => ({+ name: `an event whose ${lookup} lookup throws a non-retriable error`,+ // The first message in the batch makes the first lookup call, so only it fails.+ bad: (teamId: number) => {+ spy().mockRejectedValueOnce(new Error('column "bytecode" does not exist'))+ return createKafkaMessage(createIncomingEvent(teamId, {}))+ },+ })),+ ])('parks $name and processes the rest of the batch', async ({ bad }) => {await insertHogFunction({...HOG_EXAMPLES.simple_fetch,...HOG_INPUTS_EXAMPLES.simple_fetch,...HOG_FILTERS_EXAMPLES.no_filters,})- await handleBatch([createKafkaMessage('not json at all')])+ await handleBatch([bad(team.id), createKafkaMessage(createIncomingEvent(team.id, {}), { offset: 1 })])const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')expect(parked).toHaveLength(1)- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse' })+ expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_offset: '0' })+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])})it('fails the batch when the record cannot be written, so the offsets stay put', async () => {@@ -797,31 +820,13 @@ describe('CdpEventsConsumer', () => {}})- it('parks a message whose failure was not an Error, with a reason a person can read', async () => {- // A thrown string carries no `message`. An undefined reason fails the produce, so- // the partition stalls on the very message the record exists to keep.- await insertHogFunction({- ...HOG_EXAMPLES.simple_fetch,- ...HOG_INPUTS_EXAMPLES.simple_fetch,- ...HOG_FILTERS_EXAMPLES.no_filters,- })- jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam').mockRejectedValue('kaboom' as any)-- await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])-- const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')- expect(parked).toHaveLength(1)- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_reason: 'kaboom' })- })-it('fails the batch on a dependency outage while parsing, rather than parking every event', async () => {await insertHogFunction({...HOG_EXAMPLES.simple_fetch,...HOG_INPUTS_EXAMPLES.simple_fetch,...HOG_FILTERS_EXAMPLES.no_filters,})- const outage: Error & { isRetriable?: boolean } = new Error('Postgres is down')- outage.isRetriable = true+ const outage = new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down'))const lookup = jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam').mockRejectedValue(outage)diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex afb57209d8d8..da5507b1e174 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -15,7 +15,7 @@ import { HogFunctionInvocationPipeline } from '../services/hog-function-invocatiimport { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'export class CdpEventsConsumer<@@ -185,43 +185,60 @@ export class CdpEventsConsumer<await Promise.all(messages.map(async (message) => {- try {- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent+ // Parked rather than dropped: a schema we cannot read today is often one a later+ // version can.+ const park = (error: string): boolean =>+ this.deadLetterService.recordMessageFailure(message, { step: 'parse', error })- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([- this.hogFunctionManager.getHogFunctionsForTeam(clickHouseEvent.team_id, this.hogTypes),- this.hogFlowManager.getHogFlowsForTeam(clickHouseEvent.team_id),- this.deps.teamManager.getTeam(clickHouseEvent.team_id),- ])-- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {- return- }+ const clickHouseEvent = skipIfMalformed(+ 'Error parsing message',+ () => assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent),+ park+ )+ if (!clickHouseEvent) {+ return+ }- const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)- // The one place holding both. A failure found later in the pipeline carries- // these globals, and this is how it gets back to the bytes to park.- this.messageByGlobals.set(globals, message)- events.push(globals)- } catch (e) {- // A dependency outage is not a poison message. Rethrowing fails the batch, so the- // offsets stay put and the consumer retries, instead of dropping every event of a- // team for the length of a Postgres blip.+ // A retriable error fails the batch, so the consumer reads the batch again. Any other+ // lookup error parks the message for a replay, because failing the batch on an error+ // that repeats stops the partition. The loaders send the team ids of the whole batch+ // in one query, so one failed query parks every message in it. With the dead-letter+ // topic off, nothing keeps the message, so the error fails the batch too.+ const lookups = await Promise.all([+ this.hogFunctionManager.getHogFunctionsForTeam(clickHouseEvent.team_id, this.hogTypes),+ this.hogFlowManager.getHogFlowsForTeam(clickHouseEvent.team_id),+ this.deps.teamManager.getTeam(clickHouseEvent.team_id),+ ]).catch((e) => {if (e?.isRetriable === true) {throw e}- // A thrown non-Error has no `message`, and an undefined reason fails the- // produce, which stalls the partition on the message this exists to keep.- const error = e instanceof Error ? e.message : String(e)- logger.error('Error parsing message', e)- counterParseError.labels({ error }).inc()- // Parked rather than dropped: a schema we cannot read today is often one a- // later version can.- this.deadLetterService.recordMessageFailure(message, {- step: 'parse',- error,- })+ if (!park(e instanceof Error ? e.message : String(e))) {+ throw e+ }+ logger.error('Error looking up team data', e)+ return null+ })+ if (!lookups) {+ return+ }+ const [teamHogFunctions, teamHogFlows, team] = lookups++ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {+ return+ }++ const globals = skipIfMalformed(+ 'Error parsing message',+ () => convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL),+ park+ )+ if (!globals) {+ return}+ // The one place holding both. A failure found later in the pipeline carries+ // these globals, and this is how it gets back to the bytes to park.+ this.messageByGlobals.set(globals, message)+ events.push(globals)}))diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.tsindex 3c0004eb62c0..64ccb664ebe7 100644--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts@@ -46,7 +46,7 @@ import {} from '../utils'import { convertToHogFunctionFilterGlobal } from '../utils/hog-function-filtering'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { skipIfMalformed } from './message-parsing'// A waker only needs signals that arrive after a job parks, never history. auto.offset.reset=latest// makes a fresh consumer group start at the tip instead of replaying the (unconsumable at prod@@ -1279,19 +1279,6 @@ function convertClickhousePersonToWakeGlobals(}}-// If the step throws, this logs the error and counts it as a parse error. It then returns null, and the-// caller skips the message. Wrap only a step that cannot fail for a transient reason. A lookup error must-// fail the batch instead, so that Kafka delivers the batch again.-function skipIfMalformed<T>(errorLog: string, step: () => T): T | null {- try {- return step()- } catch (e) {- logger.error(errorLog, e)- counterParseError.labels({ error: e.message }).inc()- return null- }-}-type IndexedBatch = {teamIds: number[]// Parallel (teamId, id) pair arrays for the correlated lookup (see findParkedJobs).diff --git a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.tsindex f10fde615926..80fa95d11a70 100644--- a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts@@ -21,7 +21,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { convertInternalEventToHogFunctionInvocationGlobals } from '../utils'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'/*** Whether a GitHub delivery is a write PostHog's own GitHub App made, resolved from the `own_app`@@ -183,30 +183,29 @@ export class CdpInternalEventsConsumer extends CdpConsumerBase {const events: HogFunctionInvocationGlobals[] = []await Promise.all(messages.map(async (message) => {- try {- const kafkaEvent = parseJSON(message.value!.toString()) as unknown- const event = CdpInternalEventSchema.parse(kafkaEvent)-- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([- this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),- this.hogFlowManager.getHogFlowsForTeam(event.team_id),- this.deps.teamManager.getTeam(event.team_id),- ])-- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {- return- }+ const event = skipIfMalformed('Error parsing message', () =>+ assertTeamId(CdpInternalEventSchema.parse(parseJSON(message.value!.toString())))+ )+ if (!event) {+ return+ }- events.push(convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL))- } catch (e) {- // A dependency outage is not a poison message. Rethrowing fails the batch, so the- // offsets stay put and the consumer retries, instead of dropping every event of a- // team for the length of a Postgres blip.- if (e?.isRetriable === true) {- throw e- }- logger.error('Error parsing message', e)- counterParseError.labels({ error: e.message }).inc()+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.+ const [teamHogFunctions, teamHogFlows, team] = await Promise.all([+ this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),+ this.hogFlowManager.getHogFlowsForTeam(event.team_id),+ this.deps.teamManager.getTeam(event.team_id),+ ])++ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {+ return+ }++ const globals = skipIfMalformed('Error parsing message', () =>+ convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)+ )+ if (globals) {+ events.push(globals)}}))diff --git a/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.tsindex 7cf1b5f36288..f0db3317d926 100644--- a/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts@@ -3,7 +3,6 @@ import { createMockJobQueue } from '../../../tests/helpers/mocks/job-queue.mock'import '../../../tests/helpers/mocks/producer.mock'import { HogFlow } from '~/cdp/schema/hogflow'-import { DependencyUnavailableError } from '~/common/utils/db/error'import { closeHub, createHub } from '~/common/utils/db/hub'import { createCdpConsumerDeps } from '../../../tests/helpers/cdp'@@ -89,24 +88,41 @@ describe('CDP Internal Events Consumer', () => {expect(events).toHaveLength(0)})- it('should fail the batch when the team lookup hits a retriable error', async () => {- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)+ const lookupError = new Error('canceling statement due to conflict with recovery')- await expect(- processor._parseKafkaBatch([createKafkaMessage(createInternalEvent(team.id, {}))])- ).rejects.toThrow(error)- })+ it.each([+ {+ lookup: 'the team lookup',+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),+ },+ {+ lookup: 'the hog function lookup',+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),+ },+ {+ lookup: 'the hog flow lookup',+ spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam'),+ },+ ])(+ 'should fail the batch when $lookup throws, and queue the event when the batch is read again',+ async ({ spy }) => {+ await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ filters: internalEventFilters(),+ })+ spy().mockRejectedValueOnce(lookupError)+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]+ const messages = [createKafkaMessage(createInternalEvent(team.id, {}))]- it('should let that failure escape the batch handler rather than swallowing it', async () => {- // Offsets are stored only after the handler resolves, so the rejection reaching the- // consumer is what leaves the batch unacknowledged.- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)- const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]+ await expect(handleBatch(messages)).rejects.toBe(lookupError)+ expect(mockQueueInvocations).not.toHaveBeenCalled()- await expect(handleBatch([createKafkaMessage(createInternalEvent(team.id, {}))])).rejects.toThrow(error)- })+ const { backgroundTask } = await handleBatch(messages)+ await backgroundTask+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ }+ )describe('with an existing team and hog function', () => {beforeEach(async () => {@@ -117,6 +133,14 @@ describe('CDP Internal Events Consumer', () => {})})+ it('should skip a message with an out-of-range team_id and keep the rest', async () => {+ const events = await processor._parseKafkaBatch([+ createKafkaMessage({ ...createInternalEvent(team.id, {}), team_id: 3_000_000_000 }),+ createKafkaMessage(createInternalEvent(team.id, {})),+ ])+ expect(events).toHaveLength(1)+ })+it('should ignore invalid payloads', async () => {const events = await processor._parseKafkaBatch([createKafkaMessage(diff --git a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.tsindex 12bcf0ffaff2..ba36bed503de 100644--- a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts@@ -1,6 +1,8 @@+import '~/tests/helpers/mocks/consumer.mock'import { mockFetch } from '~/tests/helpers/mocks/request.mock'import { DateTime } from 'luxon'+import { Message } from 'node-rdkafka'import { register } from 'prom-client'import { closeHub, createHub } from '~/common/utils/db/hub'@@ -11,7 +13,12 @@ import { createTestTeamFixture } from '~/tests/helpers/sql'import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'import { Hub, Team } from '../../types'-import { createHogExecutionGlobals, insertHogFunction } from '../_tests/fixtures'+import {+ createHogExecutionGlobals,+ createIncomingEvent,+ createKafkaMessage,+ insertHogFunction,+} from '../_tests/fixtures'import { DESTINATION_PLUGINS_BY_ID } from '../legacy-plugins'import { LegacyPluginExecutorService } from '../services/legacy-plugin-executor.service'import { HogFunctionInvocationGlobals } from '../types'@@ -645,6 +652,114 @@ describe('CdpLegacyEventsConsumer', () => {})})+ describe('batch handler', () => {+ const lookupError = new Error('canceling statement due to conflict with recovery')++ beforeEach(async () => {+ consumer['legacyWebhookService']['actionMatcher'].hasWebhooks = jest.fn().mockReturnValue(true)+ jest.spyOn(consumer['deps'].teamManager, 'hasAvailableFeature').mockResolvedValue(true)+ await consumer.start()+ })++ afterEach(async () => {+ await consumer.stop()+ })++ it.each([+ { lookup: 'the team lookup', spy: () => jest.spyOn(consumer['deps'].teamManager, 'getTeam') },+ { lookup: 'the plugin config lookup', spy: () => jest.spyOn(consumer['pluginConfigsLoader'], 'get') },+ {+ lookup: 'the webhook feature lookup',+ spy: () => jest.spyOn(consumer['deps'].teamManager, 'hasAvailableFeature'),+ },+ ])(+ 'fails the batch when $lookup throws without sending anything, and sends once when the batch is read again',+ async ({ spy }) => {+ const parses = [+ jest.spyOn(consumer['legacyWebhookService'], 'parseBatch'),+ jest.spyOn(consumer, '_parseKafkaBatch'),+ ]+ const sendWebhooks = jest.spyOn(consumer['legacyWebhookService'], 'processEvents')+ const runPlugins = jest.spyOn(consumer, 'processBatch')+ spy().mockRejectedValueOnce(lookupError)+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]+ const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]++ await expect(handleBatch(messages)).rejects.toBe(lookupError)+ // The other side's parse can still be running when the batch fails. It must not send afterwards.+ await Promise.allSettled(parses.flatMap((parse) => parse.mock.results.map((result) => result.value)))+ expect(sendWebhooks).not.toHaveBeenCalled()+ expect(runPlugins).not.toHaveBeenCalled()++ const { backgroundTask } = await handleBatch(messages)+ await backgroundTask+ expect(sendWebhooks).toHaveBeenCalledTimes(1)+ expect(sendWebhooks).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ expect(runPlugins).toHaveBeenCalledTimes(1)+ expect(runPlugins).toHaveBeenCalledWith([+ expect.objectContaining({ project: expect.objectContaining({ id: team.id }) }),+ ])+ }+ )++ it.each([+ {+ name: 'a message that is not JSON',+ bad: (): Message => ({ ...createKafkaMessage({}), value: Buffer.from('not json') }),+ },+ {+ name: 'an event whose properties are not JSON',+ bad: (teamId: number): Message => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),+ },+ {+ // The team lookup sends a team_id that is not a number to its token query, which Postgres+ // rejects when the text holds a NUL byte.+ name: 'an event whose team_id is a string with a NUL byte',+ bad: (teamId: number): Message =>+ createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: `${teamId}\u0000` }),+ },+ ])('skips $name and sends the rest of the batch on both sides', async ({ bad }) => {+ const sendWebhooks = jest.spyOn(consumer['legacyWebhookService'], 'processEvents')+ const runPlugins = jest.spyOn(consumer, 'processBatch')+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]++ const { backgroundTask } = await handleBatch([+ bad(team.id),+ createKafkaMessage(createIncomingEvent(team.id, {})),+ ])+ await backgroundTask++ expect(sendWebhooks).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ expect(runPlugins).toHaveBeenCalledWith([+ expect.objectContaining({ project: expect.objectContaining({ id: team.id }) }),+ ])+ })+ })++ describe('plugin execution after the batch handler parses', () => {+ beforeEach(async () => {+ await consumer.start()+ })++ afterEach(async () => {+ await consumer.stop()+ })++ it('runs plugins with the configs the parse loaded, so a later config lookup cannot fail the batch', async () => {+ const loader = consumer['pluginConfigsLoader']+ const loadConfigs = loader.get.bind(loader)+ jest.spyOn(loader, 'get')+ .mockImplementationOnce(loadConfigs)+ .mockRejectedValue(new Error('canceling statement due to conflict with recovery'))+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]++ const { backgroundTask } = await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])+ await backgroundTask++ expect(mockFetch).toHaveBeenCalled()+ })+ })+describe('shutdown behavior', () => {it('should flush invocation results when stopping', async () => {const flushSpy = jest.spyOn(consumer['invocationResultsService'], 'flush')diff --git a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.tsindex f1f76905095a..095749449cc8 100644--- a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts@@ -5,7 +5,7 @@ import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consimport { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'import { PostgresUse } from '~/common/utils/db/postgres'import { parseJSON } from '~/common/utils/json-parse'-import { LazyLoader } from '~/common/utils/lazy-loader'+import { DEFAULT_LOADER_RETRY, LazyLoader } from '~/common/utils/lazy-loader'import { logger } from '~/common/utils/logger'import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { validTeamIds } from '~/common/utils/team-manager'@@ -29,7 +29,7 @@ import type {import { convertToHogFunctionInvocationGlobals } from '../utils'import { createInvocation } from '../utils/invocation-utils'import { CdpConsumerBase, CdpConsumerBaseConfig, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'export type LightweightPluginConfig = {id: number@@ -73,6 +73,9 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsConsprotected kafkaConsumer: KafkaConsumerInterfaceprivate pluginConfigsLoader: LazyLoader<PluginConfigHogFunction[]>+ // Execution reads the plugin configs that _parseKafkaBatch loaded. A second lookup could fail after this+ // batch's webhooks start, and reading the batch again would then send those webhooks twice.+ private pluginConfigsByGlobals = new WeakMap<HogFunctionInvocationGlobals, PluginConfigHogFunction[]>()private legacyPluginExecutor: LegacyPluginExecutorServiceprivate legacyWebhookService: LegacyWebhookService@@ -96,6 +99,7 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsConsrefreshAgeMs: 600000, // 10 minutesrefreshBackgroundAgeMs: 300000, // 5 minutesbufferMs: 10, // 10ms buffer for batching+ loaderRetry: DEFAULT_LOADER_RETRY,})// This loader caches the combined plugin-config and migrated-function view, so the manager@@ -381,25 +385,32 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsConsawait Promise.all(messages.map(async (message) => {- try {- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent+ const clickHouseEvent = skipIfMalformed('Error parsing message', () =>+ assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent)+ )+ if (!clickHouseEvent) {+ return+ }- const team = await this.deps.teamManager.getTeam(clickHouseEvent.team_id)+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.+ const team = await this.deps.teamManager.getTeam(clickHouseEvent.team_id)- if (!team) {- return- }+ if (!team) {+ return+ }- const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(team.id.toString())+ const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(team.id.toString())- if (!pluginConfigHogFunctions?.length) {- return- }+ if (!pluginConfigHogFunctions?.length) {+ return+ }- events.push(convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL))- } catch (e) {- logger.error('Error parsing message', e)- counterParseError.labels({ error: e.message }).inc()+ const globals = skipIfMalformed('Error parsing message', () =>+ convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)+ )+ if (globals) {+ this.pluginConfigsByGlobals.set(globals, pluginConfigHogFunctions)+ events.push(globals)}}))@@ -410,7 +421,11 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsConsprivate async getLegacyPluginHogFunctionInvocations(invocation: HogFunctionInvocationGlobals): Promise<{ invocation: CyclotronJobInvocationHogFunction; pluginConfigId: number | null }[]> {- const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(invocation.project.id.toString())+ // The batch handler always finds the configs that _parseKafkaBatch stored, so it never makes a second lookup.+ // Only callers that build their own globals reach the loader call.+ const pluginConfigHogFunctions =+ this.pluginConfigsByGlobals.get(invocation) ??+ (await this.pluginConfigsLoader.get(invocation.project.id.toString()))if (!pluginConfigHogFunctions) {return []@@ -450,10 +465,14 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons})return await instrumentFn('cdpLegacyConsumer.handleEachBatch', async () => {- const [webhookBatch, pluginBatch] = await Promise.all([- this.legacyWebhookService.processBatch(messages),- this._parseKafkaBatch(messages).then((invocations) => this.processBatch(invocations)),+ // Both parses finish before either side sends. A parse lookup error then fails the batch+ // before it sends a webhook or a plugin call, so reading the batch again does not repeat one.+ const [webhookEvents, invocations] = await Promise.all([+ this.legacyWebhookService.parseBatch(messages),+ this._parseKafkaBatch(messages),])+ const webhookBatch = this.legacyWebhookService.processEvents(webhookEvents)+ const pluginBatch = await this.processBatch(invocations)return {backgroundTask: Promise.all([webhookBatch.backgroundTask, pluginBatch.backgroundTask]).then(() =>this.invocationResultsService.flush()diff --git a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.tsindex dfc34c41df46..9a6534795fab 100644--- a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts@@ -23,6 +23,7 @@ describe('CDP Person Updates Consumer', () => {let hub: Hublet team: Teamlet hogFunction: HogFunctionType+ let mockQueueInvocations: jest.MockedFunction<any>const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {const item = await _insertHogFunction(hub.postgres, team.id, hogFunction)@@ -39,6 +40,7 @@ describe('CDP Person Updates Consumer', () => {const mockJobQueue = createMockJobQueue()processor = new CdpPersonUpdatesConsumer(hub, createCdpConsumerDeps(hub), mockJobQueue)+ mockQueueInvocations = mockJobQueue.queueInvocationsawait processor.start()hogFunction = createHogFunction({@@ -71,6 +73,45 @@ describe('CDP Person Updates Consumer', () => {const events = await processor._parseKafkaBatch([createKafkaMessage(createClickhousePerson(999999, {}))])expect(events).toHaveLength(0)})++ const lookupError = new Error('canceling statement due to conflict with recovery')++ it.each([+ {+ lookup: 'the team lookup',+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),+ },+ {+ lookup: 'the hog function lookup',+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),+ },+ ])(+ 'should fail the batch when $lookup throws, and queue the person when the batch is read again',+ async ({ spy }) => {+ spy().mockRejectedValueOnce(lookupError)+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]+ const messages = [createKafkaMessage(createClickhousePerson(team.id, {}))]++ await expect(handleBatch(messages)).rejects.toBe(lookupError)+ expect(mockQueueInvocations).not.toHaveBeenCalled()++ const { backgroundTask } = await handleBatch(messages)+ await backgroundTask+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ }+ )++ it.each([+ { name: 'properties that are not JSON', person: { properties: '{' } },+ { name: 'an out-of-range team_id', person: { team_id: 3_000_000_000 } },+ ])('should skip a person with $name and keep the rest', async ({ person }) => {+ const events = await processor._parseKafkaBatch([+ createKafkaMessage({ ...createClickhousePerson(team.id, {}), ...person }),+ createKafkaMessage(createClickhousePerson(team.id, {})),+ ])+ expect(events).toHaveLength(1)+ })+it('should parse a valid message with an existing team and hog function ', async () => {const event = createClickhousePerson(team.id, {id: 'person-id-1',diff --git a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.tsindex 7aea74893f1e..639b0ec00bd1 100644--- a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts@@ -14,7 +14,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, CyclotronPerson, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { getPersonDisplayName } from '../utils'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { counterParseError } from './metrics'+import { assertTeamId, skipIfMalformed } from './message-parsing'export class CdpPersonUpdatesConsumer extends CdpConsumerBase {protected name = 'CdpPersonUpdatesConsumer'@@ -82,26 +82,30 @@ export class CdpPersonUpdatesConsumer extends CdpConsumerBase {const globals: HogFunctionInvocationGlobals[] = []await Promise.all(messages.map(async (message) => {- try {- const data = parseJSON(message.value!.toString()) as ClickHousePerson+ const data = skipIfMalformed('Error parsing message', () =>+ assertTeamId(parseJSON(message.value!.toString()) as ClickHousePerson)+ )+ if (!data) {+ return+ }- const [teamHogFunctions, team] = await Promise.all([- this.hogFunctionManager.getHogFunctionsForTeam(data.team_id, this.hogTypes),- this.deps.teamManager.getTeam(data.team_id),- ])+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.+ const [teamHogFunctions, team] = await Promise.all([+ this.hogFunctionManager.getHogFunctionsForTeam(data.team_id, this.hogTypes),+ this.deps.teamManager.getTeam(data.team_id),+ ])- const filteredHogFunctions = teamHogFunctions.filter(- (fn) => fn.filters?.source === 'person-updates'- )+ const filteredHogFunctions = teamHogFunctions.filter((fn) => fn.filters?.source === 'person-updates')- if (!filteredHogFunctions.length || !team) {- return- }+ if (!filteredHogFunctions.length || !team) {+ return+ }- globals.push(convertClickhousePersonToInvocationGlobals(data, team, this.config.SITE_URL))- } catch (e) {- logger.error('Error parsing message', e)- counterParseError.labels({ error: e.message }).inc()+ const invocationGlobals = skipIfMalformed('Error parsing message', () =>+ convertClickhousePersonToInvocationGlobals(data, team, this.config.SITE_URL)+ )+ if (invocationGlobals) {+ globals.push(invocationGlobals)}}))diff --git a/nodejs/src/cdp/consumers/message-parsing.test.ts b/nodejs/src/cdp/consumers/message-parsing.test.tsnew file mode 100644index 000000000000..0e60069ef244--- /dev/null+++ b/nodejs/src/cdp/consumers/message-parsing.test.ts@@ -0,0 +1,32 @@+import { assertTeamId, skipIfMalformed } from './message-parsing'++describe('message-parsing', () => {+ describe('assertTeamId', () => {+ it.each([1, 2_147_483_647])('accepts team_id %p', (teamId) => {+ expect(assertTeamId({ team_id: teamId })).toEqual({ team_id: teamId })+ })++ it.each([2_147_483_648, 0, -1, 1.5, NaN, '1', null, undefined])('rejects team_id %p', (teamId) => {+ expect(() => assertTeamId({ team_id: teamId })).toThrow('invalid_team_id')+ })+ })++ describe('skipIfMalformed', () => {+ it('reports a thrown non-Error as a reason a person can read', () => {+ // A thrown string carries no `message`. An undefined reason fails the dead-letter produce, so+ // the partition stalls on the very message the record exists to keep.+ const onError = jest.fn()++ const result = skipIfMalformed(+ 'Error parsing message',+ () => {+ throw 'kaboom'+ },+ onError+ )++ expect(result).toBeNull()+ expect(onError).toHaveBeenCalledWith('kaboom')+ })+ })+})diff --git a/nodejs/src/cdp/consumers/message-parsing.ts b/nodejs/src/cdp/consumers/message-parsing.tsnew file mode 100644index 000000000000..7e13130d4e7e--- /dev/null+++ b/nodejs/src/cdp/consumers/message-parsing.ts@@ -0,0 +1,30 @@+import { logger } from '~/common/utils/logger'+import { isValidTeamId } from '~/common/utils/team-manager'++import { counterParseError } from './metrics'++// The loaders batch team ids from many messages into one query. One invalid id fails that query for+// every message in it, on every read.+export function assertTeamId<T extends { team_id: unknown }>(message: T): T {+ if (!isValidTeamId(message.team_id)) {+ throw new Error('invalid_team_id')+ }+ return message+}++/**+ * If the step throws, this logs the error and counts it as a parse error. It then returns null, and+ * the caller skips the message. Wrap only a step that reads the message. A dependency lookup must not+ * run inside `step`. Its error is not the message's fault, so skipping the message would lose the event.+ */+export function skipIfMalformed<T>(errorLog: string, step: () => T, onError?: (error: string) => void): T | null {+ try {+ return step()+ } catch (e) {+ const error = e instanceof Error ? e.message : String(e)+ logger.error(errorLog, e)+ counterParseError.labels({ error }).inc()+ onError?.(error)+ return null+ }+}diff --git a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.tsindex 63025bc3ffe0..dcf6f40f33b7 100644--- a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts+++ b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts@@ -54,6 +54,10 @@ describe('LegacyWebhookService', () => {}}+ const parseAndSend = async (messages: Message[]): Promise<void> => {+ await service.processEvents(await service.parseBatch(messages)).backgroundTask+ }+beforeEach(async () => {hub = await createHub()team = (await createTestTeamFixture(hub.postgres)).team@@ -79,15 +83,14 @@ describe('LegacyWebhookService', () => {jest.clearAllMocks()})- describe('processBatch', () => {+ describe('parseBatch and processEvents', () => {it('should parse events for teams with both webhooks and zapier', async () => {service['actionMatcher'].hasWebhooks = jest.fn().mockReturnValue(true)hub.teamManager.hasAvailableFeature = jest.fn().mockResolvedValue(true)const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(service['actionMatcher'].hasWebhooks).toHaveBeenCalledWith(team.id)})@@ -102,8 +105,7 @@ describe('LegacyWebhookService', () => {createKafkaMessage(createIncomingEvent(team2.id, { event: '$pageview' })),]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(service.processEvent).not.toHaveBeenCalled()})@@ -115,8 +117,7 @@ describe('LegacyWebhookService', () => {const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(service.processEvent).not.toHaveBeenCalled()})@@ -128,8 +129,7 @@ describe('LegacyWebhookService', () => {const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(service.processEvent).not.toHaveBeenCalled()})@@ -144,8 +144,7 @@ describe('LegacyWebhookService', () => {createKafkaMessage(createIncomingEvent(team2.id, { event: '$pageview' })),]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(service.processEvent).toHaveBeenCalledTimes(1)})@@ -167,8 +166,7 @@ describe('LegacyWebhookService', () => {),]- const result = await service.processBatch(messages)- await result.backgroundTask+ await parseAndSend(messages)expect(fetchGroupsByKeysSpy).not.toHaveBeenCalled()expect(fetchGroupSpy).not.toHaveBeenCalled()@@ -199,8 +197,7 @@ describe('LegacyWebhookService', () => {// group_analytics ON (and zapier ON)hub.teamManager.hasAvailableFeature = jest.fn().mockResolvedValue(true)- let result = await service.processBatch([makeMessage()])- await result.backgroundTask+ await parseAndSend([makeMessage()])const bodyWithGroupAnalytics = parseJSON(mockFetch.mock.calls[0][1].body)mockFetch.mockClear()@@ -209,8 +206,7 @@ describe('LegacyWebhookService', () => {hub.teamManager.hasAvailableFeature = jest.fn().mockImplementation((_teamId, feature) => Promise.resolve(feature === 'zapier'))- result = await service.processBatch([makeMessage()])- await result.backgroundTask+ await parseAndSend([makeMessage()])const bodyWithoutGroupAnalytics = parseJSON(mockFetch.mock.calls[0][1].body)// The payload is byte-for-byte identical whether or not group_analytics is enableddiff --git a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.tsindex 8e5df3474885..d4830529a379 100644--- a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts+++ b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts@@ -9,7 +9,7 @@ import { TeamManager } from '~/common/utils/team-manager'import { clickHouseTimestampSecondPrecisionToISO, clickHouseTimestampToISO } from '~/common/utils/utils'import { Action, Hook, HookPayload, PostIngestionEvent, RawClickHouseEvent, RawKafkaEvent, Team } from '../../types'-import { counterParseError } from '../consumers/metrics'+import { assertTeamId, skipIfMalformed } from '../consumers/message-parsing'import { ActionManager } from '../legacy-webhooks/action-manager'import { ActionMatcher } from '../legacy-webhooks/action-matcher'import { cdpTrackedFetch } from '../utils/cdp-fetch'@@ -94,29 +94,39 @@ export class LegacyWebhookService {)}- public async processBatch(messages: Message[]): Promise<{ backgroundTask: Promise<any> }> {+ public async parseBatch(messages: Message[]): Promise<PostIngestionEvent[]> {const events: PostIngestionEvent[] = []await Promise.all(messages.map(async (message) => {- try {- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent-- if (- !this.actionMatcher.hasWebhooks(clickHouseEvent.team_id) ||- !(await this.teamManager.hasAvailableFeature(clickHouseEvent.team_id, 'zapier'))- ) {- return- }-- events.push(convertToPostIngestionEvent(clickHouseEvent))- } catch (e) {- logger.error('Error parsing message', e)- counterParseError.labels({ error: e.message }).inc()+ const clickHouseEvent = skipIfMalformed('Error parsing message', () =>+ assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent)+ )+ if (!clickHouseEvent) {+ return+ }++ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.+ if (+ !this.actionMatcher.hasWebhooks(clickHouseEvent.team_id) ||+ !(await this.teamManager.hasAvailableFeature(clickHouseEvent.team_id, 'zapier'))+ ) {+ return+ }++ const event = skipIfMalformed('Error parsing message', () =>+ convertToPostIngestionEvent(clickHouseEvent)+ )+ if (event) {+ events.push(event)}}))+ return events+ }++ public processEvents(events: PostIngestionEvent[]): { backgroundTask: Promise<any> } {return { backgroundTask: Promise.all(events.map((event) => this.processEvent(event))) }}diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.tsindex b0c91bbdd05c..7b97ab1d8577 100644--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts@@ -184,12 +184,17 @@ export class CdpDeadLetterService implements InvocationFailureSink {return true}- /** For a message that never became an event, so there are no globals to attach it to. */- public recordMessageFailure(message: Message, failure: Omit<PendingFailure, 'sourceId' | 'sourceKind'>): void {+ /**+ * For a message that never became an event, so there are no globals to attach it to.+ *+ * False means the message was not retained.+ */+ public recordMessageFailure(message: Message, failure: Omit<PendingFailure, 'sourceId' | 'sourceKind'>): boolean {if (!this.config.CDP_DLQ_ENABLED) {- return+ return false}this.messageFailures.push({ message, failure })+ return true}/**