:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(cdp): skip parsing events the consumer has no work for (#127164)
Meikel Ratz8bb522f0c8a8
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.tsindex b852eca5c76b..73148e935857 100644--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts@@ -803,6 +803,48 @@ describe('CdpEventsConsumer', () => {expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])})+ it('leaves the body unread when the team id header names a team with no hog functions', async () => {+ await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.no_filters,+ })+ // team2 has no hog function and this body cannot be parsed, so a parked record could+ // only come from reading a body the consumer does not need. The second message carries+ // no header, so the batch also covers the fallback that reads the team from the body.+ const unreadable = {+ ...createKafkaMessage(createIncomingEvent(team2.id, {}), {+ headers: [{ team_id: String(team2.id) }],+ }),+ value: Buffer.from('{'),+ }++ await handleBatch([unreadable, createKafkaMessage(createIncomingEvent(team.id, {}), { offset: 1 })])++ expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ })++ it('parks an event whose body names a different team than its header', async () => {+ await insertHogFunction({+ ...HOG_EXAMPLES.simple_fetch,+ ...HOG_INPUTS_EXAMPLES.simple_fetch,+ ...HOG_FILTERS_EXAMPLES.no_filters,+ })+ // The header decides which team's destinations load, the body carries the event data.+ // Letting the two disagree would send one team's event to another team's destinations.+ const mismatched = createKafkaMessage(createIncomingEvent(team2.id, {}), {+ headers: [{ team_id: String(team.id) }],+ })++ await handleBatch([mismatched])++ const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')+ expect(parked).toHaveLength(1)+ expect(parked[0].headers).toMatchObject({ dlq_step: 'parse' })+ expect(mockQueueInvocations).not.toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])+ })+it('fails the batch when the record cannot be written, so the offsets stay put', async () => {// Also guards the ordering: the handler has to await the write. Without the await// this rejects in the background and the consumer walks past the event anyway.diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.tsindex 70f2199e4823..922b44dd7b49 100644--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts@@ -7,7 +7,7 @@ import { logger } from '~/common/utils/logger'import { captureException } from '~/common/utils/posthog'import { convertToHogFunctionInvocationGlobals } from '../../cdp/utils'-import { HealthCheckResult, PluginsServerConfig, RawClickHouseEvent } from '../../types'+import { HealthCheckResult, PluginsServerConfig, RawClickHouseEvent, Team } from '../../types'import { CDP_EVENTS_DLQ_OUTPUT } from '../outputs/outputs'import { CdpDeadLetterService } from '../services/dead-letter/cdp-dead-letter.service'import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'@@ -15,7 +15,8 @@ import { HogFunctionInvocationPipeline } from '../services/hog-function-invocatiimport { JobQueue } from '../services/job-queue/job-queue.interface'import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'-import { parseBatch } from './message-parsing'+import { parseBatch, readTeamIdHeader } from './message-parsing'+import { counterTeamIdHeaderMismatch, counterTeamIdSource } from './metrics'import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'export class CdpEventsConsumer<@@ -197,20 +198,41 @@ export class CdpEventsConsumer<return await parseBatch(messages, {errorLog: 'Error parsing message',parse: (message) => {- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent+ const parseEvent = (): RawClickHouseEvent => parseJSON(message.value!.toString()) as RawClickHouseEvent+ const buildGlobals = (team: Team, clickHouseEvent: RawClickHouseEvent) => {+ const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)+ // The one place holding both. A failure found later in the pipeline carries+ // these globals, and this is how it gets back to the bytes to park.+ this.messageByGlobals.set(globals, message)+ return globals+ }++ const headerTeamId = readTeamIdHeader(message)+ if (headerTeamId !== null) {+ counterTeamIdSource.labels({ source: 'header' }).inc()+ // The body stays unparsed until `toGlobals` runs, and it only runs for a team that+ // has a destination or a flow. Most events belong to a team with neither.+ return {+ teamId: headerTeamId,+ toGlobals: (team) => {+ const clickHouseEvent = parseEvent()+ if (clickHouseEvent.team_id !== headerTeamId) {+ // The loaded team comes from the header and the event data comes from+ // the body. Building globals from two teams would evaluate one team's+ // event against another team's destinations and send it there.+ counterTeamIdHeaderMismatch.inc()+ throw new Error('team_id_header_mismatch')+ }+ return buildGlobals(team, clickHouseEvent)+ },+ }+ }++ counterTeamIdSource.labels({ source: 'body' }).inc()+ const clickHouseEvent = parseEvent()return {teamId: clickHouseEvent.team_id,- toGlobals: (team) => {- const globals = convertToHogFunctionInvocationGlobals(- clickHouseEvent,- team,- this.config.SITE_URL- )- // The one place holding both. A failure found later in the pipeline carries- // these globals, and this is how it gets back to the bytes to park.- this.messageByGlobals.set(globals, message)- return globals- },+ toGlobals: (team) => buildGlobals(team, clickHouseEvent),}},loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),diff --git a/nodejs/src/cdp/consumers/message-parsing.test.ts b/nodejs/src/cdp/consumers/message-parsing.test.tsindex 0e60069ef244..378f3734024e 100644--- a/nodejs/src/cdp/consumers/message-parsing.test.ts+++ b/nodejs/src/cdp/consumers/message-parsing.test.ts@@ -1,4 +1,6 @@-import { assertTeamId, skipIfMalformed } from './message-parsing'+import { Message } from 'node-rdkafka'++import { assertTeamId, readTeamIdHeader, skipIfMalformed } from './message-parsing'describe('message-parsing', () => {describe('assertTeamId', () => {@@ -29,4 +31,34 @@ describe('message-parsing', () => {expect(onError).toHaveBeenCalledWith('kaboom')})})+ describe('readTeamIdHeader', () => {+ function withHeaders(headers?: { [key: string]: string | Buffer }[]): Message {+ return { headers } as unknown as Message+ }++ type Headers = { [key: string]: string | Buffer }[]++ it.each<[string, Headers, number]>([+ ['a string value', [{ team_id: '42' }], 42],+ ['a buffer value', [{ team_id: Buffer.from('42') }], 42],+ ['a header beside others', [{ productTrack: 'general' }, { team_id: '42' }], 42],+ ])('reads %s', (_name, headers, expected) => {+ expect(readTeamIdHeader(withHeaders(headers))).toBe(expected)+ })++ // A value that is not a usable team id must not become one. Returning it would make the+ // consumer load another team's destinations for this event, or none at all.+ it.each<[string, Headers | undefined]>([+ ['no headers at all', undefined],+ ['no team_id header', [{ productTrack: 'general' }]],+ ['an empty value', [{ team_id: '' }]],+ ['a value that is not a number', [{ team_id: 'not-a-number' }]],+ ['a value above the int4 range', [{ team_id: '2147483648' }]],+ ['a zero', [{ team_id: '0' }]],+ ['a negative value', [{ team_id: '-1' }]],+ ['a fractional value', [{ team_id: '1.5' }]],+ ])('falls back to the body for %s', (_name, headers) => {+ expect(readTeamIdHeader(withHeaders(headers))).toBeNull()+ })+ })})diff --git a/nodejs/src/cdp/consumers/message-parsing.ts b/nodejs/src/cdp/consumers/message-parsing.tsindex 85fffeed9523..7007a09c0a18 100644--- a/nodejs/src/cdp/consumers/message-parsing.ts+++ b/nodejs/src/cdp/consumers/message-parsing.ts@@ -1,5 +1,6 @@import { Message } from 'node-rdkafka'+import { TEAM_ID_HEADER } from '~/common/kafka/headers'import { logger } from '~/common/utils/logger'import { isValidTeamId } from '~/common/utils/team-manager'@@ -16,6 +17,26 @@ export function assertTeamId<T extends { team_id: unknown }>(message: T): T {return message}+/**+ * Reads the team id the producer put in a Kafka header, so a consumer can decide whether it needs a+ * message before it parses the body.+ *+ * The body still holds the authoritative team id, and a caller that acts on this header has to check+ * the two agree once it parses the body. Anything unexpected here returns null and the caller reads+ * the body instead, because a value that is not a usable team id must never become one.+ */+export function readTeamIdHeader(message: Message): number | null {+ for (const header of message.headers ?? []) {+ const value = header[TEAM_ID_HEADER]+ if (value === undefined) {+ continue+ }+ const teamId = Number(value.toString())+ return isValidTeamId(teamId) ? teamId : null+ }+ return null+}+function errorMessage(e: unknown): string {return e instanceof Error ? e.message : String(e)}diff --git a/nodejs/src/cdp/consumers/metrics.ts b/nodejs/src/cdp/consumers/metrics.tsindex e09cc4f4fe7f..7b70d17ccea2 100644--- a/nodejs/src/cdp/consumers/metrics.ts+++ b/nodejs/src/cdp/consumers/metrics.ts@@ -23,3 +23,14 @@ export const counterBatchHogFlowTriggerFailed = new Counter({help: 'A batch hog flow run failed during audience resolution and was skipped',labelNames: ['hog_flow_id', 'reason'],})++export const counterTeamIdSource = new Counter({+ name: 'cdp_events_team_id_source',+ help: 'Where the consumer read an event team id from. "body" means the message carried no usable team id header, so the consumer parsed the event to find one.',+ labelNames: ['source'],+})++export const counterTeamIdHeaderMismatch = new Counter({+ name: 'cdp_events_team_id_header_mismatch',+ help: 'An event body named a different team than its header. The event is parked, because the two teams would send one team data to another team destinations.',+})