Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): stop dropping events when a consumer's team lookup fails (#121819)

Phil Haack54e3087e2182
diff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts
index 9c3700e7a958..93259bc1ba79 100644
--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.test.ts
@@ -15,6 +15,7 @@ import { insertHogFlow } from '../_tests/fixtures-hogflows'
import { COHORT_MEMBERSHIP_CHANGED_EVENT } from '../services/cohorts/cohort-membership-repository'
import { CyclotronJobInvocationHogFlow } from '../types'
import { CdpCohortMembershipTriggerConsumer } from './cdp-cohort-membership-trigger.consumer'
+import { counterParseError } from './metrics'
describe('CdpCohortMembershipTriggerConsumer', () => {
let processor: CdpCohortMembershipTriggerConsumer
@@ -136,6 +137,23 @@ describe('CdpCohortMembershipTriggerConsumer', () => {
await expect(processor._parseKafkaBatch([createKafkaMessage(change)])).rejects.toThrow('conflict with recovery')
})
+ it('skips a change with an out-of-range team_id as a parse error and keeps the rest of the batch', async () => {
+ // The dropped-changes alert counts parse errors. It does not count a skip with any other reason.
+ const invalidTeamIdErrors = async (): Promise<number> =>
+ (await counterParseError.get()).values.find((v) => v.labels.error === 'invalid_team_id')?.value ?? 0
+ await insertHogFlow(hub.postgres, cohortFlow(42, 'entered'))
+ const change = createCohortMembershipEvent({ team_id: team.id, cohort_id: 42, status: 'entered' })
+ const before = await invalidTeamIdErrors()
+
+ const globals = await processor._parseKafkaBatch([
+ createKafkaMessage({ ...change, team_id: 3_000_000_000 }),
+ createKafkaMessage(change),
+ ])
+
+ expect(globals).toHaveLength(1)
+ expect(await invalidTeamIdErrors()).toBe(before + 1)
+ })
+
it("loads a team's workflows once per batch, however many changes the batch holds for it", async () => {
await insertHogFlow(hub.postgres, cohortFlow(42, 'entered'))
const lookup = jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeams')
diff --git a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
index 6da16031af5a..1a93587dd9ed 100644
--- a/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cohort-membership-trigger.consumer.ts
@@ -22,7 +22,7 @@ import { CyclotronJobInvocation, HogFunctionInvocationGlobals } from '../types'
import { convertBatchHogFlowRequestToHogFunctionInvocationGlobals } from '../utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { CohortMembershipChange, CohortMembershipChangeSchema } from './cdp-cohort-membership.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
export const counterCohortMembershipTriggerSkipped = new Counter({
name: 'cdp_cohort_membership_trigger_skipped',
@@ -159,12 +159,10 @@ export class CdpCohortMembershipTriggerConsumer extends CdpConsumerBase {
public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
const changes: CohortMembershipChange[] = []
for (const message of messages) {
- let change: CohortMembershipChange
- try {
- change = CohortMembershipChangeSchema.parse(parseJSON(message.value!.toString()))
- } catch (e) {
- logger.error('Error parsing cohort membership change', e)
- counterParseError.labels({ error: e.message }).inc()
+ const change = skipIfMalformed('Error parsing cohort membership change', () =>
+ assertTeamId(CohortMembershipChangeSchema.parse(parseJSON(message.value!.toString())))
+ )
+ if (!change) {
continue
}
diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts
index 815b764f3a1c..a2ccc873f079 100644
--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.test.ts
@@ -155,6 +155,56 @@ describe('CdpDatawarehouseEventsConsumer', () => {
expect(invocations).toHaveLength(0)
})
+ describe('with a hog function for the team', () => {
+ beforeEach(async () => {
+ await insertHogFunction({
+ team_id: team.id,
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch_data_warehouse_table,
+ ...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
+ })
+ })
+
+ const lookupError = new Error('canceling statement due to conflict with recovery')
+
+ it.each([
+ {
+ lookup: 'the team lookup',
+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),
+ },
+ {
+ lookup: 'the hog function lookup',
+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),
+ },
+ {
+ lookup: 'the hog flow lookup',
+ spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam'),
+ },
+ ])(
+ 'should fail the batch when $lookup throws, and queue the row when the batch is read again',
+ async ({ spy }) => {
+ spy().mockRejectedValueOnce(lookupError)
+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+ const messages = [createKafkaMessage(createDataWarehouseEvent(team.id))]
+
+ await expect(handleBatch(messages)).rejects.toBe(lookupError)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+
+ const { backgroundTask } = await handleBatch(messages)
+ await backgroundTask
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ }
+ )
+
+ it('should skip a row with an out-of-range team_id and keep the rest', async () => {
+ const invocations = await processor._parseKafkaBatch([
+ createKafkaMessage({ ...createDataWarehouseEvent(team.id), team_id: 3_000_000_000 }),
+ createKafkaMessage(createDataWarehouseEvent(team.id)),
+ ])
+ expect(invocations).toHaveLength(1)
+ })
+ })
+
it('should handle schema validation errors gracefully', async () => {
const invalidEvent = { team_id: 'not-a-number' }
const messages = [createKafkaMessage(invalidEvent)]
diff --git a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
index a3c869edf4e7..d638ae6cbe5b 100644
--- a/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-data-warehouse-events.consumer.ts
@@ -20,7 +20,7 @@ import {
HogFunctionTypeType,
} from '../types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
// Filter source / trigger type each kind of warehouse row matches against.
const TRIGGER_TYPE_BY_TABLE_TYPE = {
@@ -152,26 +152,29 @@ export class CdpDatawarehouseEventsConsumer extends CdpConsumerBase {
await Promise.all(
messages.map(async (message) => {
- try {
- const kafkaEvent = parseJSON(message.value!.toString()) as unknown
- const event = CdpDataWarehouseEventSchema.parse(kafkaEvent)
+ const event = skipIfMalformed('Error parsing message', () =>
+ assertTeamId(CdpDataWarehouseEventSchema.parse(parseJSON(message.value!.toString())))
+ )
+ if (!event) {
+ return
+ }
- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([
- this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),
- this.hogFlowManager.getHogFlowsForTeam(event.team_id),
- this.deps.teamManager.getTeam(event.team_id),
- ])
+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
+ const [teamHogFunctions, teamHogFlows, team] = await Promise.all([
+ this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),
+ this.hogFlowManager.getHogFlowsForTeam(event.team_id),
+ this.deps.teamManager.getTeam(event.team_id),
+ ])
- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
- return
- }
+ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
+ return
+ }
- events.push(
- convertDataWarehouseEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)
- )
- } catch (e) {
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
+ const globals = skipIfMalformed('Error parsing message', () =>
+ convertDataWarehouseEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)
+ )
+ if (globals) {
+ events.push(globals)
}
})
)
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
index ff4c4ef444e6..8ad9283dddf5 100644
--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
@@ -51,6 +51,18 @@ describe('CdpEventsConsumer', () => {
return item
}
+ const handleBatch = async (messages: any[]): Promise<void> => {
+ const connect = processor['kafkaConsumer'].connect as jest.Mock
+ const { backgroundTask } = await connect.mock.calls[0][0](messages)
+ await backgroundTask
+ }
+
+ const lookupSpies = [
+ { lookup: 'team', spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam') },
+ { lookup: 'hog function', spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam') },
+ { lookup: 'hog flow', spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam') },
+ ]
+
beforeEach(async () => {
hub = await createHub()
const fixture = await createTestTeamFixture(hub.postgres)
@@ -118,7 +130,7 @@ describe('CdpEventsConsumer', () => {
})
})
- describe('parse failures', () => {
+ describe('lookup failures', () => {
beforeEach(async () => {
await insertHogFunction({
team_id: team.id,
@@ -128,43 +140,37 @@ describe('CdpEventsConsumer', () => {
})
})
- it.each([
- [
- 'rethrows a retriable team lookup error so the batch is retried',
- new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset')),
- true,
- ],
- ['drops the message on a non-retriable team lookup error', new Error('event shape is wrong'), false],
- ])('%s', async (_name, error, shouldRethrow) => {
- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)
- const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]
-
- if (shouldRethrow) {
- await expect(processor._parseKafkaBatch(messages)).rejects.toThrow(error)
- } else {
- await expect(processor._parseKafkaBatch(messages)).resolves.toEqual([])
- }
- })
-
- it('lets a retriable failure escape the batch handler, and processes the batch on a later attempt', async () => {
- // The consumer stores offsets only after this handler resolves, so the handler
- // rejecting is what leaves the batch unacknowledged. A try/catch added around
- // _parseKafkaBatch here would swallow the rejection and silently restore the old
- // drop-everything behaviour, which the unit assertion above cannot see.
- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))
- const getTeam = jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)
- const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
- const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]
-
- await expect(handleBatch(messages)).rejects.toThrow(error)
- expect(mockQueueInvocations).not.toHaveBeenCalled()
-
- getTeam.mockRestore()
- const { backgroundTask } = await handleBatch(messages)
- await backgroundTask
+ it.each(
+ lookupSpies.flatMap(({ lookup, spy }) => [
+ {
+ lookup,
+ kind: 'a retriable error',
+ dlqEnabled: true,
+ spy,
+ lookupError: new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset')),
+ },
+ {
+ lookup,
+ kind: 'a non-retriable error with the dead-letter topic off',
+ dlqEnabled: false,
+ spy,
+ lookupError: new Error('column "bytecode" does not exist'),
+ },
+ ])
+ )(
+ 'fails the batch when the $lookup lookup throws $kind, and queues the event when the batch is read again',
+ async ({ spy, dlqEnabled, lookupError }) => {
+ hub.CDP_DLQ_ENABLED = dlqEnabled
+ spy().mockRejectedValueOnce(lookupError)
+ const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]
+
+ await expect(handleBatch(messages)).rejects.toBe(lookupError)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
- expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
- })
+ await handleBatch(messages)
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ }
+ )
})
describe('general event processing', () => {
@@ -596,12 +602,6 @@ describe('CdpEventsConsumer', () => {
})
describe('dead letter queue', () => {
- const handleBatch = async (messages: any[]): Promise<void> => {
- const connect = processor['kafkaConsumer'].connect as jest.Mock
- const { backgroundTask } = await connect.mock.calls[0][0](messages)
- await backgroundTask
- }
-
beforeEach(() => {
hub.CDP_DLQ_ENABLED = true
})
@@ -756,18 +756,41 @@ describe('CdpEventsConsumer', () => {
expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)
})
- it('parks a message it cannot parse', async () => {
+ it.each([
+ {
+ name: 'a message that is not an event object',
+ bad: () => createKafkaMessage('not json at all'),
+ },
+ {
+ name: 'an event whose properties are not JSON',
+ bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),
+ },
+ {
+ name: 'an event whose team_id is out of int4 range',
+ bad: (teamId: number) =>
+ createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: 3_000_000_000 }),
+ },
+ ...lookupSpies.map(({ lookup, spy }) => ({
+ name: `an event whose ${lookup} lookup throws a non-retriable error`,
+ // The first message in the batch makes the first lookup call, so only it fails.
+ bad: (teamId: number) => {
+ spy().mockRejectedValueOnce(new Error('column "bytecode" does not exist'))
+ return createKafkaMessage(createIncomingEvent(teamId, {}))
+ },
+ })),
+ ])('parks $name and processes the rest of the batch', async ({ bad }) => {
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.no_filters,
})
- await handleBatch([createKafkaMessage('not json at all')])
+ await handleBatch([bad(team.id), createKafkaMessage(createIncomingEvent(team.id, {}), { offset: 1 })])
const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')
expect(parked).toHaveLength(1)
- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse' })
+ expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_offset: '0' })
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
})
it('fails the batch when the record cannot be written, so the offsets stay put', async () => {
@@ -797,31 +820,13 @@ describe('CdpEventsConsumer', () => {
}
})
- it('parks a message whose failure was not an Error, with a reason a person can read', async () => {
- // A thrown string carries no `message`. An undefined reason fails the produce, so
- // the partition stalls on the very message the record exists to keep.
- await insertHogFunction({
- ...HOG_EXAMPLES.simple_fetch,
- ...HOG_INPUTS_EXAMPLES.simple_fetch,
- ...HOG_FILTERS_EXAMPLES.no_filters,
- })
- jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam').mockRejectedValue('kaboom' as any)
-
- await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])
-
- const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')
- expect(parked).toHaveLength(1)
- expect(parked[0].headers).toMatchObject({ dlq_step: 'parse', dlq_reason: 'kaboom' })
- })
-
it('fails the batch on a dependency outage while parsing, rather than parking every event', async () => {
await insertHogFunction({
...HOG_EXAMPLES.simple_fetch,
...HOG_INPUTS_EXAMPLES.simple_fetch,
...HOG_FILTERS_EXAMPLES.no_filters,
})
- const outage: Error & { isRetriable?: boolean } = new Error('Postgres is down')
- outage.isRetriable = true
+ const outage = new DependencyUnavailableError('Postgres is down', 'Postgres', new Error('down'))
const lookup = jest
.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam')
.mockRejectedValue(outage)
diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
index afb57209d8d8..da5507b1e174 100644
--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
@@ -15,7 +15,7 @@ import { HogFunctionInvocationPipeline } from '../services/hog-function-invocati
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'
export class CdpEventsConsumer<
@@ -185,43 +185,60 @@ export class CdpEventsConsumer<
await Promise.all(
messages.map(async (message) => {
- try {
- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
+ // Parked rather than dropped: a schema we cannot read today is often one a later
+ // version can.
+ const park = (error: string): boolean =>
+ this.deadLetterService.recordMessageFailure(message, { step: 'parse', error })
- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([
- this.hogFunctionManager.getHogFunctionsForTeam(clickHouseEvent.team_id, this.hogTypes),
- this.hogFlowManager.getHogFlowsForTeam(clickHouseEvent.team_id),
- this.deps.teamManager.getTeam(clickHouseEvent.team_id),
- ])
-
- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
- return
- }
+ const clickHouseEvent = skipIfMalformed(
+ 'Error parsing message',
+ () => assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent),
+ park
+ )
+ if (!clickHouseEvent) {
+ return
+ }
- const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)
- // The one place holding both. A failure found later in the pipeline carries
- // these globals, and this is how it gets back to the bytes to park.
- this.messageByGlobals.set(globals, message)
- events.push(globals)
- } catch (e) {
- // A dependency outage is not a poison message. Rethrowing fails the batch, so the
- // offsets stay put and the consumer retries, instead of dropping every event of a
- // team for the length of a Postgres blip.
+ // A retriable error fails the batch, so the consumer reads the batch again. Any other
+ // lookup error parks the message for a replay, because failing the batch on an error
+ // that repeats stops the partition. The loaders send the team ids of the whole batch
+ // in one query, so one failed query parks every message in it. With the dead-letter
+ // topic off, nothing keeps the message, so the error fails the batch too.
+ const lookups = await Promise.all([
+ this.hogFunctionManager.getHogFunctionsForTeam(clickHouseEvent.team_id, this.hogTypes),
+ this.hogFlowManager.getHogFlowsForTeam(clickHouseEvent.team_id),
+ this.deps.teamManager.getTeam(clickHouseEvent.team_id),
+ ]).catch((e) => {
if (e?.isRetriable === true) {
throw e
}
- // A thrown non-Error has no `message`, and an undefined reason fails the
- // produce, which stalls the partition on the message this exists to keep.
- const error = e instanceof Error ? e.message : String(e)
- logger.error('Error parsing message', e)
- counterParseError.labels({ error }).inc()
- // Parked rather than dropped: a schema we cannot read today is often one a
- // later version can.
- this.deadLetterService.recordMessageFailure(message, {
- step: 'parse',
- error,
- })
+ if (!park(e instanceof Error ? e.message : String(e))) {
+ throw e
+ }
+ logger.error('Error looking up team data', e)
+ return null
+ })
+ if (!lookups) {
+ return
+ }
+ const [teamHogFunctions, teamHogFlows, team] = lookups
+
+ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
+ return
+ }
+
+ const globals = skipIfMalformed(
+ 'Error parsing message',
+ () => convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL),
+ park
+ )
+ if (!globals) {
+ return
}
+ // The one place holding both. A failure found later in the pipeline carries
+ // these globals, and this is how it gets back to the bytes to park.
+ this.messageByGlobals.set(globals, message)
+ events.push(globals)
})
)
diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
index 3c0004eb62c0..64ccb664ebe7 100644
--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
@@ -46,7 +46,7 @@ import {
} from '../utils'
import { convertToHogFunctionFilterGlobal } from '../utils/hog-function-filtering'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { skipIfMalformed } from './message-parsing'
// A waker only needs signals that arrive after a job parks, never history. auto.offset.reset=latest
// makes a fresh consumer group start at the tip instead of replaying the (unconsumable at prod
@@ -1279,19 +1279,6 @@ function convertClickhousePersonToWakeGlobals(
}
}
-// If the step throws, this logs the error and counts it as a parse error. It then returns null, and the
-// caller skips the message. Wrap only a step that cannot fail for a transient reason. A lookup error must
-// fail the batch instead, so that Kafka delivers the batch again.
-function skipIfMalformed<T>(errorLog: string, step: () => T): T | null {
- try {
- return step()
- } catch (e) {
- logger.error(errorLog, e)
- counterParseError.labels({ error: e.message }).inc()
- return null
- }
-}
-
type IndexedBatch = {
teamIds: number[]
// Parallel (teamId, id) pair arrays for the correlated lookup (see findParkedJobs).
diff --git a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
index f10fde615926..80fa95d11a70 100644
--- a/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-internal-event.consumer.ts
@@ -21,7 +21,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { convertInternalEventToHogFunctionInvocationGlobals } from '../utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
/**
* Whether a GitHub delivery is a write PostHog's own GitHub App made, resolved from the `own_app`
@@ -183,30 +183,29 @@ export class CdpInternalEventsConsumer extends CdpConsumerBase {
const events: HogFunctionInvocationGlobals[] = []
await Promise.all(
messages.map(async (message) => {
- try {
- const kafkaEvent = parseJSON(message.value!.toString()) as unknown
- const event = CdpInternalEventSchema.parse(kafkaEvent)
-
- const [teamHogFunctions, teamHogFlows, team] = await Promise.all([
- this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),
- this.hogFlowManager.getHogFlowsForTeam(event.team_id),
- this.deps.teamManager.getTeam(event.team_id),
- ])
-
- if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
- return
- }
+ const event = skipIfMalformed('Error parsing message', () =>
+ assertTeamId(CdpInternalEventSchema.parse(parseJSON(message.value!.toString())))
+ )
+ if (!event) {
+ return
+ }
- events.push(convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL))
- } catch (e) {
- // A dependency outage is not a poison message. Rethrowing fails the batch, so the
- // offsets stay put and the consumer retries, instead of dropping every event of a
- // team for the length of a Postgres blip.
- if (e?.isRetriable === true) {
- throw e
- }
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
+ const [teamHogFunctions, teamHogFlows, team] = await Promise.all([
+ this.hogFunctionManager.getHogFunctionsForTeam(event.team_id, this.hogTypes),
+ this.hogFlowManager.getHogFlowsForTeam(event.team_id),
+ this.deps.teamManager.getTeam(event.team_id),
+ ])
+
+ if ((!teamHogFunctions.length && !teamHogFlows.length) || !team) {
+ return
+ }
+
+ const globals = skipIfMalformed('Error parsing message', () =>
+ convertInternalEventToHogFunctionInvocationGlobals(event, team, this.config.SITE_URL)
+ )
+ if (globals) {
+ events.push(globals)
}
})
)
diff --git a/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts
index 7cf1b5f36288..f0db3317d926 100644
--- a/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-internal-events-consumer.test.ts
@@ -3,7 +3,6 @@ import { createMockJobQueue } from '../../../tests/helpers/mocks/job-queue.mock'
import '../../../tests/helpers/mocks/producer.mock'
import { HogFlow } from '~/cdp/schema/hogflow'
-import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { createCdpConsumerDeps } from '../../../tests/helpers/cdp'
@@ -89,24 +88,41 @@ describe('CDP Internal Events Consumer', () => {
expect(events).toHaveLength(0)
})
- it('should fail the batch when the team lookup hits a retriable error', async () => {
- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))
- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)
+ const lookupError = new Error('canceling statement due to conflict with recovery')
- await expect(
- processor._parseKafkaBatch([createKafkaMessage(createInternalEvent(team.id, {}))])
- ).rejects.toThrow(error)
- })
+ it.each([
+ {
+ lookup: 'the team lookup',
+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),
+ },
+ {
+ lookup: 'the hog function lookup',
+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),
+ },
+ {
+ lookup: 'the hog flow lookup',
+ spy: () => jest.spyOn(processor['hogFlowManager'], 'getHogFlowsForTeam'),
+ },
+ ])(
+ 'should fail the batch when $lookup throws, and queue the event when the batch is read again',
+ async ({ spy }) => {
+ await insertHogFunction({
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch,
+ filters: internalEventFilters(),
+ })
+ spy().mockRejectedValueOnce(lookupError)
+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+ const messages = [createKafkaMessage(createInternalEvent(team.id, {}))]
- it('should let that failure escape the batch handler rather than swallowing it', async () => {
- // Offsets are stored only after the handler resolves, so the rejection reaching the
- // consumer is what leaves the batch unacknowledged.
- const error = new DependencyUnavailableError('connection reset', 'Postgres', new Error('reset'))
- jest.spyOn(processor['deps'].teamManager, 'getTeam').mockRejectedValue(error)
- const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+ await expect(handleBatch(messages)).rejects.toBe(lookupError)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
- await expect(handleBatch([createKafkaMessage(createInternalEvent(team.id, {}))])).rejects.toThrow(error)
- })
+ const { backgroundTask } = await handleBatch(messages)
+ await backgroundTask
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ }
+ )
describe('with an existing team and hog function', () => {
beforeEach(async () => {
@@ -117,6 +133,14 @@ describe('CDP Internal Events Consumer', () => {
})
})
+ it('should skip a message with an out-of-range team_id and keep the rest', async () => {
+ const events = await processor._parseKafkaBatch([
+ createKafkaMessage({ ...createInternalEvent(team.id, {}), team_id: 3_000_000_000 }),
+ createKafkaMessage(createInternalEvent(team.id, {})),
+ ])
+ expect(events).toHaveLength(1)
+ })
+
it('should ignore invalid payloads', async () => {
const events = await processor._parseKafkaBatch([
createKafkaMessage(
diff --git a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts
index 12bcf0ffaff2..ba36bed503de 100644
--- a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.test.ts
@@ -1,6 +1,8 @@
+import '~/tests/helpers/mocks/consumer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { DateTime } from 'luxon'
+import { Message } from 'node-rdkafka'
import { register } from 'prom-client'
import { closeHub, createHub } from '~/common/utils/db/hub'
@@ -11,7 +13,12 @@ import { createTestTeamFixture } from '~/tests/helpers/sql'
import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'
import { Hub, Team } from '../../types'
-import { createHogExecutionGlobals, insertHogFunction } from '../_tests/fixtures'
+import {
+ createHogExecutionGlobals,
+ createIncomingEvent,
+ createKafkaMessage,
+ insertHogFunction,
+} from '../_tests/fixtures'
import { DESTINATION_PLUGINS_BY_ID } from '../legacy-plugins'
import { LegacyPluginExecutorService } from '../services/legacy-plugin-executor.service'
import { HogFunctionInvocationGlobals } from '../types'
@@ -645,6 +652,114 @@ describe('CdpLegacyEventsConsumer', () => {
})
})
+ describe('batch handler', () => {
+ const lookupError = new Error('canceling statement due to conflict with recovery')
+
+ beforeEach(async () => {
+ consumer['legacyWebhookService']['actionMatcher'].hasWebhooks = jest.fn().mockReturnValue(true)
+ jest.spyOn(consumer['deps'].teamManager, 'hasAvailableFeature').mockResolvedValue(true)
+ await consumer.start()
+ })
+
+ afterEach(async () => {
+ await consumer.stop()
+ })
+
+ it.each([
+ { lookup: 'the team lookup', spy: () => jest.spyOn(consumer['deps'].teamManager, 'getTeam') },
+ { lookup: 'the plugin config lookup', spy: () => jest.spyOn(consumer['pluginConfigsLoader'], 'get') },
+ {
+ lookup: 'the webhook feature lookup',
+ spy: () => jest.spyOn(consumer['deps'].teamManager, 'hasAvailableFeature'),
+ },
+ ])(
+ 'fails the batch when $lookup throws without sending anything, and sends once when the batch is read again',
+ async ({ spy }) => {
+ const parses = [
+ jest.spyOn(consumer['legacyWebhookService'], 'parseBatch'),
+ jest.spyOn(consumer, '_parseKafkaBatch'),
+ ]
+ const sendWebhooks = jest.spyOn(consumer['legacyWebhookService'], 'processEvents')
+ const runPlugins = jest.spyOn(consumer, 'processBatch')
+ spy().mockRejectedValueOnce(lookupError)
+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+ const messages = [createKafkaMessage(createIncomingEvent(team.id, {}))]
+
+ await expect(handleBatch(messages)).rejects.toBe(lookupError)
+ // The other side's parse can still be running when the batch fails. It must not send afterwards.
+ await Promise.allSettled(parses.flatMap((parse) => parse.mock.results.map((result) => result.value)))
+ expect(sendWebhooks).not.toHaveBeenCalled()
+ expect(runPlugins).not.toHaveBeenCalled()
+
+ const { backgroundTask } = await handleBatch(messages)
+ await backgroundTask
+ expect(sendWebhooks).toHaveBeenCalledTimes(1)
+ expect(sendWebhooks).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ expect(runPlugins).toHaveBeenCalledTimes(1)
+ expect(runPlugins).toHaveBeenCalledWith([
+ expect.objectContaining({ project: expect.objectContaining({ id: team.id }) }),
+ ])
+ }
+ )
+
+ it.each([
+ {
+ name: 'a message that is not JSON',
+ bad: (): Message => ({ ...createKafkaMessage({}), value: Buffer.from('not json') }),
+ },
+ {
+ name: 'an event whose properties are not JSON',
+ bad: (teamId: number): Message => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),
+ },
+ {
+ // The team lookup sends a team_id that is not a number to its token query, which Postgres
+ // rejects when the text holds a NUL byte.
+ name: 'an event whose team_id is a string with a NUL byte',
+ bad: (teamId: number): Message =>
+ createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: `${teamId}\u0000` }),
+ },
+ ])('skips $name and sends the rest of the batch on both sides', async ({ bad }) => {
+ const sendWebhooks = jest.spyOn(consumer['legacyWebhookService'], 'processEvents')
+ const runPlugins = jest.spyOn(consumer, 'processBatch')
+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+
+ const { backgroundTask } = await handleBatch([
+ bad(team.id),
+ createKafkaMessage(createIncomingEvent(team.id, {})),
+ ])
+ await backgroundTask
+
+ expect(sendWebhooks).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ expect(runPlugins).toHaveBeenCalledWith([
+ expect.objectContaining({ project: expect.objectContaining({ id: team.id }) }),
+ ])
+ })
+ })
+
+ describe('plugin execution after the batch handler parses', () => {
+ beforeEach(async () => {
+ await consumer.start()
+ })
+
+ afterEach(async () => {
+ await consumer.stop()
+ })
+
+ it('runs plugins with the configs the parse loaded, so a later config lookup cannot fail the batch', async () => {
+ const loader = consumer['pluginConfigsLoader']
+ const loadConfigs = loader.get.bind(loader)
+ jest.spyOn(loader, 'get')
+ .mockImplementationOnce(loadConfigs)
+ .mockRejectedValue(new Error('canceling statement due to conflict with recovery'))
+ const handleBatch = (consumer['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+
+ const { backgroundTask } = await handleBatch([createKafkaMessage(createIncomingEvent(team.id, {}))])
+ await backgroundTask
+
+ expect(mockFetch).toHaveBeenCalled()
+ })
+ })
+
describe('shutdown behavior', () => {
it('should flush invocation results when stopping', async () => {
const flushSpy = jest.spyOn(consumer['invocationResultsService'], 'flush')
diff --git a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts
index f1f76905095a..095749449cc8 100644
--- a/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-legacy-event.consumer.ts
@@ -5,7 +5,7 @@ import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/cons
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
-import { LazyLoader } from '~/common/utils/lazy-loader'
+import { DEFAULT_LOADER_RETRY, LazyLoader } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { validTeamIds } from '~/common/utils/team-manager'
@@ -29,7 +29,7 @@ import type {
import { convertToHogFunctionInvocationGlobals } from '../utils'
import { createInvocation } from '../utils/invocation-utils'
import { CdpConsumerBase, CdpConsumerBaseConfig, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
export type LightweightPluginConfig = {
id: number
@@ -73,6 +73,9 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons
protected kafkaConsumer: KafkaConsumerInterface
private pluginConfigsLoader: LazyLoader<PluginConfigHogFunction[]>
+ // Execution reads the plugin configs that _parseKafkaBatch loaded. A second lookup could fail after this
+ // batch's webhooks start, and reading the batch again would then send those webhooks twice.
+ private pluginConfigsByGlobals = new WeakMap<HogFunctionInvocationGlobals, PluginConfigHogFunction[]>()
private legacyPluginExecutor: LegacyPluginExecutorService
private legacyWebhookService: LegacyWebhookService
@@ -96,6 +99,7 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons
refreshAgeMs: 600000, // 10 minutes
refreshBackgroundAgeMs: 300000, // 5 minutes
bufferMs: 10, // 10ms buffer for batching
+ loaderRetry: DEFAULT_LOADER_RETRY,
})
// This loader caches the combined plugin-config and migrated-function view, so the manager
@@ -381,25 +385,32 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons
await Promise.all(
messages.map(async (message) => {
- try {
- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
+ const clickHouseEvent = skipIfMalformed('Error parsing message', () =>
+ assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent)
+ )
+ if (!clickHouseEvent) {
+ return
+ }
- const team = await this.deps.teamManager.getTeam(clickHouseEvent.team_id)
+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
+ const team = await this.deps.teamManager.getTeam(clickHouseEvent.team_id)
- if (!team) {
- return
- }
+ if (!team) {
+ return
+ }
- const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(team.id.toString())
+ const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(team.id.toString())
- if (!pluginConfigHogFunctions?.length) {
- return
- }
+ if (!pluginConfigHogFunctions?.length) {
+ return
+ }
- events.push(convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL))
- } catch (e) {
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
+ const globals = skipIfMalformed('Error parsing message', () =>
+ convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)
+ )
+ if (globals) {
+ this.pluginConfigsByGlobals.set(globals, pluginConfigHogFunctions)
+ events.push(globals)
}
})
)
@@ -410,7 +421,11 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons
private async getLegacyPluginHogFunctionInvocations(
invocation: HogFunctionInvocationGlobals
): Promise<{ invocation: CyclotronJobInvocationHogFunction; pluginConfigId: number | null }[]> {
- const pluginConfigHogFunctions = await this.pluginConfigsLoader.get(invocation.project.id.toString())
+ // The batch handler always finds the configs that _parseKafkaBatch stored, so it never makes a second lookup.
+ // Only callers that build their own globals reach the loader call.
+ const pluginConfigHogFunctions =
+ this.pluginConfigsByGlobals.get(invocation) ??
+ (await this.pluginConfigsLoader.get(invocation.project.id.toString()))
if (!pluginConfigHogFunctions) {
return []
@@ -450,10 +465,14 @@ export class CdpLegacyEventsConsumer extends CdpConsumerBase<CdpLegacyEventsCons
})
return await instrumentFn('cdpLegacyConsumer.handleEachBatch', async () => {
- const [webhookBatch, pluginBatch] = await Promise.all([
- this.legacyWebhookService.processBatch(messages),
- this._parseKafkaBatch(messages).then((invocations) => this.processBatch(invocations)),
+ // Both parses finish before either side sends. A parse lookup error then fails the batch
+ // before it sends a webhook or a plugin call, so reading the batch again does not repeat one.
+ const [webhookEvents, invocations] = await Promise.all([
+ this.legacyWebhookService.parseBatch(messages),
+ this._parseKafkaBatch(messages),
])
+ const webhookBatch = this.legacyWebhookService.processEvents(webhookEvents)
+ const pluginBatch = await this.processBatch(invocations)
return {
backgroundTask: Promise.all([webhookBatch.backgroundTask, pluginBatch.backgroundTask]).then(() =>
this.invocationResultsService.flush()
diff --git a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts
index dfc34c41df46..9a6534795fab 100644
--- a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.test.ts
@@ -23,6 +23,7 @@ describe('CDP Person Updates Consumer', () => {
let hub: Hub
let team: Team
let hogFunction: HogFunctionType
+ let mockQueueInvocations: jest.MockedFunction<any>
const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
const item = await _insertHogFunction(hub.postgres, team.id, hogFunction)
@@ -39,6 +40,7 @@ describe('CDP Person Updates Consumer', () => {
const mockJobQueue = createMockJobQueue()
processor = new CdpPersonUpdatesConsumer(hub, createCdpConsumerDeps(hub), mockJobQueue)
+ mockQueueInvocations = mockJobQueue.queueInvocations
await processor.start()
hogFunction = createHogFunction({
@@ -71,6 +73,45 @@ describe('CDP Person Updates Consumer', () => {
const events = await processor._parseKafkaBatch([createKafkaMessage(createClickhousePerson(999999, {}))])
expect(events).toHaveLength(0)
})
+
+ const lookupError = new Error('canceling statement due to conflict with recovery')
+
+ it.each([
+ {
+ lookup: 'the team lookup',
+ spy: () => jest.spyOn(processor['deps'].teamManager, 'getTeam'),
+ },
+ {
+ lookup: 'the hog function lookup',
+ spy: () => jest.spyOn(processor['hogFunctionManager'], 'getHogFunctionsForTeam'),
+ },
+ ])(
+ 'should fail the batch when $lookup throws, and queue the person when the batch is read again',
+ async ({ spy }) => {
+ spy().mockRejectedValueOnce(lookupError)
+ const handleBatch = (processor['kafkaConsumer'].connect as jest.Mock).mock.calls[0][0]
+ const messages = [createKafkaMessage(createClickhousePerson(team.id, {}))]
+
+ await expect(handleBatch(messages)).rejects.toBe(lookupError)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+
+ const { backgroundTask } = await handleBatch(messages)
+ await backgroundTask
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ }
+ )
+
+ it.each([
+ { name: 'properties that are not JSON', person: { properties: '{' } },
+ { name: 'an out-of-range team_id', person: { team_id: 3_000_000_000 } },
+ ])('should skip a person with $name and keep the rest', async ({ person }) => {
+ const events = await processor._parseKafkaBatch([
+ createKafkaMessage({ ...createClickhousePerson(team.id, {}), ...person }),
+ createKafkaMessage(createClickhousePerson(team.id, {})),
+ ])
+ expect(events).toHaveLength(1)
+ })
+
it('should parse a valid message with an existing team and hog function ', async () => {
const event = createClickhousePerson(team.id, {
id: 'person-id-1',
diff --git a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts
index 7aea74893f1e..639b0ec00bd1 100644
--- a/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-person-updates-consumer.ts
@@ -14,7 +14,7 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, CyclotronPerson, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { getPersonDisplayName } from '../utils'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { counterParseError } from './metrics'
+import { assertTeamId, skipIfMalformed } from './message-parsing'
export class CdpPersonUpdatesConsumer extends CdpConsumerBase {
protected name = 'CdpPersonUpdatesConsumer'
@@ -82,26 +82,30 @@ export class CdpPersonUpdatesConsumer extends CdpConsumerBase {
const globals: HogFunctionInvocationGlobals[] = []
await Promise.all(
messages.map(async (message) => {
- try {
- const data = parseJSON(message.value!.toString()) as ClickHousePerson
+ const data = skipIfMalformed('Error parsing message', () =>
+ assertTeamId(parseJSON(message.value!.toString()) as ClickHousePerson)
+ )
+ if (!data) {
+ return
+ }
- const [teamHogFunctions, team] = await Promise.all([
- this.hogFunctionManager.getHogFunctionsForTeam(data.team_id, this.hogTypes),
- this.deps.teamManager.getTeam(data.team_id),
- ])
+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
+ const [teamHogFunctions, team] = await Promise.all([
+ this.hogFunctionManager.getHogFunctionsForTeam(data.team_id, this.hogTypes),
+ this.deps.teamManager.getTeam(data.team_id),
+ ])
- const filteredHogFunctions = teamHogFunctions.filter(
- (fn) => fn.filters?.source === 'person-updates'
- )
+ const filteredHogFunctions = teamHogFunctions.filter((fn) => fn.filters?.source === 'person-updates')
- if (!filteredHogFunctions.length || !team) {
- return
- }
+ if (!filteredHogFunctions.length || !team) {
+ return
+ }
- globals.push(convertClickhousePersonToInvocationGlobals(data, team, this.config.SITE_URL))
- } catch (e) {
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
+ const invocationGlobals = skipIfMalformed('Error parsing message', () =>
+ convertClickhousePersonToInvocationGlobals(data, team, this.config.SITE_URL)
+ )
+ if (invocationGlobals) {
+ globals.push(invocationGlobals)
}
})
)
diff --git a/nodejs/src/cdp/consumers/message-parsing.test.ts b/nodejs/src/cdp/consumers/message-parsing.test.ts
new file mode 100644
index 000000000000..0e60069ef244
--- /dev/null
+++ b/nodejs/src/cdp/consumers/message-parsing.test.ts
@@ -0,0 +1,32 @@
+import { assertTeamId, skipIfMalformed } from './message-parsing'
+
+describe('message-parsing', () => {
+ describe('assertTeamId', () => {
+ it.each([1, 2_147_483_647])('accepts team_id %p', (teamId) => {
+ expect(assertTeamId({ team_id: teamId })).toEqual({ team_id: teamId })
+ })
+
+ it.each([2_147_483_648, 0, -1, 1.5, NaN, '1', null, undefined])('rejects team_id %p', (teamId) => {
+ expect(() => assertTeamId({ team_id: teamId })).toThrow('invalid_team_id')
+ })
+ })
+
+ describe('skipIfMalformed', () => {
+ it('reports a thrown non-Error as a reason a person can read', () => {
+ // A thrown string carries no `message`. An undefined reason fails the dead-letter produce, so
+ // the partition stalls on the very message the record exists to keep.
+ const onError = jest.fn()
+
+ const result = skipIfMalformed(
+ 'Error parsing message',
+ () => {
+ throw 'kaboom'
+ },
+ onError
+ )
+
+ expect(result).toBeNull()
+ expect(onError).toHaveBeenCalledWith('kaboom')
+ })
+ })
+})
diff --git a/nodejs/src/cdp/consumers/message-parsing.ts b/nodejs/src/cdp/consumers/message-parsing.ts
new file mode 100644
index 000000000000..7e13130d4e7e
--- /dev/null
+++ b/nodejs/src/cdp/consumers/message-parsing.ts
@@ -0,0 +1,30 @@
+import { logger } from '~/common/utils/logger'
+import { isValidTeamId } from '~/common/utils/team-manager'
+
+import { counterParseError } from './metrics'
+
+// The loaders batch team ids from many messages into one query. One invalid id fails that query for
+// every message in it, on every read.
+export function assertTeamId<T extends { team_id: unknown }>(message: T): T {
+ if (!isValidTeamId(message.team_id)) {
+ throw new Error('invalid_team_id')
+ }
+ return message
+}
+
+/**
+ * If the step throws, this logs the error and counts it as a parse error. It then returns null, and
+ * the caller skips the message. Wrap only a step that reads the message. A dependency lookup must not
+ * run inside `step`. Its error is not the message's fault, so skipping the message would lose the event.
+ */
+export function skipIfMalformed<T>(errorLog: string, step: () => T, onError?: (error: string) => void): T | null {
+ try {
+ return step()
+ } catch (e) {
+ const error = e instanceof Error ? e.message : String(e)
+ logger.error(errorLog, e)
+ counterParseError.labels({ error }).inc()
+ onError?.(error)
+ return null
+ }
+}
diff --git a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts
index 63025bc3ffe0..dcf6f40f33b7 100644
--- a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts
+++ b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.test.ts
@@ -54,6 +54,10 @@ describe('LegacyWebhookService', () => {
}
}
+ const parseAndSend = async (messages: Message[]): Promise<void> => {
+ await service.processEvents(await service.parseBatch(messages)).backgroundTask
+ }
+
beforeEach(async () => {
hub = await createHub()
team = (await createTestTeamFixture(hub.postgres)).team
@@ -79,15 +83,14 @@ describe('LegacyWebhookService', () => {
jest.clearAllMocks()
})
- describe('processBatch', () => {
+ describe('parseBatch and processEvents', () => {
it('should parse events for teams with both webhooks and zapier', async () => {
service['actionMatcher'].hasWebhooks = jest.fn().mockReturnValue(true)
hub.teamManager.hasAvailableFeature = jest.fn().mockResolvedValue(true)
const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(service['actionMatcher'].hasWebhooks).toHaveBeenCalledWith(team.id)
})
@@ -102,8 +105,7 @@ describe('LegacyWebhookService', () => {
createKafkaMessage(createIncomingEvent(team2.id, { event: '$pageview' })),
]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(service.processEvent).not.toHaveBeenCalled()
})
@@ -115,8 +117,7 @@ describe('LegacyWebhookService', () => {
const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(service.processEvent).not.toHaveBeenCalled()
})
@@ -128,8 +129,7 @@ describe('LegacyWebhookService', () => {
const messages: Message[] = [createKafkaMessage(createIncomingEvent(team.id, { event: '$pageview' }))]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(service.processEvent).not.toHaveBeenCalled()
})
@@ -144,8 +144,7 @@ describe('LegacyWebhookService', () => {
createKafkaMessage(createIncomingEvent(team2.id, { event: '$pageview' })),
]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(service.processEvent).toHaveBeenCalledTimes(1)
})
@@ -167,8 +166,7 @@ describe('LegacyWebhookService', () => {
),
]
- const result = await service.processBatch(messages)
- await result.backgroundTask
+ await parseAndSend(messages)
expect(fetchGroupsByKeysSpy).not.toHaveBeenCalled()
expect(fetchGroupSpy).not.toHaveBeenCalled()
@@ -199,8 +197,7 @@ describe('LegacyWebhookService', () => {
// group_analytics ON (and zapier ON)
hub.teamManager.hasAvailableFeature = jest.fn().mockResolvedValue(true)
- let result = await service.processBatch([makeMessage()])
- await result.backgroundTask
+ await parseAndSend([makeMessage()])
const bodyWithGroupAnalytics = parseJSON(mockFetch.mock.calls[0][1].body)
mockFetch.mockClear()
@@ -209,8 +206,7 @@ describe('LegacyWebhookService', () => {
hub.teamManager.hasAvailableFeature = jest
.fn()
.mockImplementation((_teamId, feature) => Promise.resolve(feature === 'zapier'))
- result = await service.processBatch([makeMessage()])
- await result.backgroundTask
+ await parseAndSend([makeMessage()])
const bodyWithoutGroupAnalytics = parseJSON(mockFetch.mock.calls[0][1].body)
// The payload is byte-for-byte identical whether or not group_analytics is enabled
diff --git a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts
index 8e5df3474885..d4830529a379 100644
--- a/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts
+++ b/nodejs/src/cdp/legacy-webhooks/legacy-webhook-service.ts
@@ -9,7 +9,7 @@ import { TeamManager } from '~/common/utils/team-manager'
import { clickHouseTimestampSecondPrecisionToISO, clickHouseTimestampToISO } from '~/common/utils/utils'
import { Action, Hook, HookPayload, PostIngestionEvent, RawClickHouseEvent, RawKafkaEvent, Team } from '../../types'
-import { counterParseError } from '../consumers/metrics'
+import { assertTeamId, skipIfMalformed } from '../consumers/message-parsing'
import { ActionManager } from '../legacy-webhooks/action-manager'
import { ActionMatcher } from '../legacy-webhooks/action-matcher'
import { cdpTrackedFetch } from '../utils/cdp-fetch'
@@ -94,29 +94,39 @@ export class LegacyWebhookService {
)
}
- public async processBatch(messages: Message[]): Promise<{ backgroundTask: Promise<any> }> {
+ public async parseBatch(messages: Message[]): Promise<PostIngestionEvent[]> {
const events: PostIngestionEvent[] = []
await Promise.all(
messages.map(async (message) => {
- try {
- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
-
- if (
- !this.actionMatcher.hasWebhooks(clickHouseEvent.team_id) ||
- !(await this.teamManager.hasAvailableFeature(clickHouseEvent.team_id, 'zapier'))
- ) {
- return
- }
-
- events.push(convertToPostIngestionEvent(clickHouseEvent))
- } catch (e) {
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
+ const clickHouseEvent = skipIfMalformed('Error parsing message', () =>
+ assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent)
+ )
+ if (!clickHouseEvent) {
+ return
+ }
+
+ // Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
+ if (
+ !this.actionMatcher.hasWebhooks(clickHouseEvent.team_id) ||
+ !(await this.teamManager.hasAvailableFeature(clickHouseEvent.team_id, 'zapier'))
+ ) {
+ return
+ }
+
+ const event = skipIfMalformed('Error parsing message', () =>
+ convertToPostIngestionEvent(clickHouseEvent)
+ )
+ if (event) {
+ events.push(event)
}
})
)
+ return events
+ }
+
+ public processEvents(events: PostIngestionEvent[]): { backgroundTask: Promise<any> } {
return { backgroundTask: Promise.all(events.map((event) => this.processEvent(event))) }
}
diff --git a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
index b0c91bbdd05c..7b97ab1d8577 100644
--- a/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
+++ b/nodejs/src/cdp/services/dead-letter/cdp-dead-letter.service.ts
@@ -184,12 +184,17 @@ export class CdpDeadLetterService implements InvocationFailureSink {
return true
}
- /** For a message that never became an event, so there are no globals to attach it to. */
- public recordMessageFailure(message: Message, failure: Omit<PendingFailure, 'sourceId' | 'sourceKind'>): void {
+ /**
+ * For a message that never became an event, so there are no globals to attach it to.
+ *
+ * False means the message was not retained.
+ */
+ public recordMessageFailure(message: Message, failure: Omit<PendingFailure, 'sourceId' | 'sourceKind'>): boolean {
if (!this.config.CDP_DLQ_ENABLED) {
- return
+ return false
}
this.messageFailures.push({ message, failure })
+ return true
}
/**