:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): skip events with an unusable team id instead of parking them (#123346)
Meikel Ratzed1ddaf1b418
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.tsindex 73148e935857..9f07712616cb 100644--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts@@ -762,6 +762,20 @@ describe('CdpEventsConsumer', () => {expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)})+ it('skips an event whose team id is not a team id, and processes the rest of the batch', async () => {+ await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.no_filters,+ })+ const bad = createKafkaMessage({ ...createIncomingEvent(team.id, {}), team_id: 3_000_000_000 })++ await handleBatch([bad, createKafkaMessage(createIncomingEvent(team.id, {}), { offset: 1 })])++ expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ })+it.each([{name: 'a message that is not an event object',@@ -769,15 +783,14 @@ describe('CdpEventsConsumer', () => {bad: () => createKafkaMessage('not json at all'),},{- name: 'an event whose properties are not JSON',+ name: 'a message that is an array, not an event object',step: 'parse',- bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),+ bad: () => createKafkaMessage([]),},{- name: 'an event whose team_id is out of int4 range',+ name: 'an event whose properties are not JSON',step: 'parse',- bad: (teamId: number) =>- createKafkaMessage({ ...createIncomingEvent(teamId, {}), team_id: 3_000_000_000 }),+ bad: (teamId: number) => createKafkaMessage(createIncomingEvent(teamId, { properties: '{' })),},...lookupSpies.map(({ lookup, spy }) => ({name: `an event whose ${lookup} lookup throws a non-retriable error`,diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex a6cbbfaf04fc..d228fb95a581 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -16,9 +16,12 @@ import { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'import { parseBatch, readTeamIdHeader } from './message-parsing'-import { counterTeamIdHeaderMismatch, counterTeamIdSource } from './metrics'+import { counterParseError, counterTeamIdHeaderMismatch, counterTeamIdSource } from './metrics'import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'+const isEventObject = (value: unknown): value is RawClickHouseEvent =>+ typeof value === 'object' && value !== null && !Array.isArray(value)+export class CdpEventsConsumer<TConfig extends PluginsServerConfig = PluginsServerConfig,> extends CdpConsumerBase<TConfig> {@@ -199,7 +202,13 @@ export class CdpEventsConsumer<return await parseBatch(messages, {errorLog: 'Error parsing message',parse: (message) => {- const parseEvent = (): RawClickHouseEvent => parseJSON(message.value!.toString()) as RawClickHouseEvent+ const parseEvent = (): RawClickHouseEvent => {+ const clickHouseEvent = parseJSON(message.value!.toString())+ if (!isEventObject(clickHouseEvent)) {+ throw new Error('not_an_event')+ }+ return clickHouseEvent+ }const buildGlobals = (team: Team, clickHouseEvent: RawClickHouseEvent) => {const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)// The one place holding both. A failure found later in the pipeline carries@@ -237,6 +246,7 @@ export class CdpEventsConsumer<}},loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),+ onInvalidTeamId: () => counterParseError.labels({ error: 'invalid_team_id' }).inc(),// Parked rather than dropped: a schema we cannot read today is often one a later// version can.onError: park('parse'),