Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(cdp): skip parsing events the consumer has no work for (#127164)

Meikel Ratz8bb522f0c8a8
diff --git a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
index b852eca5c76b..73148e935857 100644
--- a/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-events-consumer.test.ts
@@ -803,6 +803,48 @@ describe('CdpEventsConsumer', () => {
expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
})
+ it('leaves the body unread when the team id header names a team with no hog functions', async () => {
+ await insertHogFunction({
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch,
+ ...HOG_FILTERS_EXAMPLES.no_filters,
+ })
+ // team2 has no hog function and this body cannot be parsed, so a parked record could
+ // only come from reading a body the consumer does not need. The second message carries
+ // no header, so the batch also covers the fallback that reads the team from the body.
+ const unreadable = {
+ ...createKafkaMessage(createIncomingEvent(team2.id, {}), {
+ headers: [{ team_id: String(team2.id) }],
+ }),
+ value: Buffer.from('{'),
+ }
+
+ await handleBatch([unreadable, createKafkaMessage(createIncomingEvent(team.id, {}), { offset: 1 })])
+
+ expect(mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')).toHaveLength(0)
+ expect(mockQueueInvocations).toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ })
+
+ it('parks an event whose body names a different team than its header', async () => {
+ await insertHogFunction({
+ ...HOG_EXAMPLES.simple_fetch,
+ ...HOG_INPUTS_EXAMPLES.simple_fetch,
+ ...HOG_FILTERS_EXAMPLES.no_filters,
+ })
+ // The header decides which team's destinations load, the body carries the event data.
+ // Letting the two disagree would send one team's event to another team's destinations.
+ const mismatched = createKafkaMessage(createIncomingEvent(team2.id, {}), {
+ headers: [{ team_id: String(team.id) }],
+ })
+
+ await handleBatch([mismatched])
+
+ const parked = mockProducerObserver.getProducedKafkaMessagesForTopic('cdp_events_dlq_test')
+ expect(parked).toHaveLength(1)
+ expect(parked[0].headers).toMatchObject({ dlq_step: 'parse' })
+ expect(mockQueueInvocations).not.toHaveBeenCalledWith([expect.objectContaining({ teamId: team.id })])
+ })
+
it('fails the batch when the record cannot be written, so the offsets stay put', async () => {
// Also guards the ordering: the handler has to await the write. Without the await
// this rejects in the background and the consumer walks past the event anyway.
diff --git a/nodejs/src/cdp/consumers/cdp-events.consumer.ts b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
index 70f2199e4823..922b44dd7b49 100644
--- a/nodejs/src/cdp/consumers/cdp-events.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-events.consumer.ts
@@ -7,7 +7,7 @@ import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { convertToHogFunctionInvocationGlobals } from '../../cdp/utils'
-import { HealthCheckResult, PluginsServerConfig, RawClickHouseEvent } from '../../types'
+import { HealthCheckResult, PluginsServerConfig, RawClickHouseEvent, Team } from '../../types'
import { CDP_EVENTS_DLQ_OUTPUT } from '../outputs/outputs'
import { CdpDeadLetterService } from '../services/dead-letter/cdp-dead-letter.service'
import { HogFlowInvocationPipeline } from '../services/hog-flow-invocation-pipeline.service'
@@ -15,7 +15,8 @@ import { HogFunctionInvocationPipeline } from '../services/hog-function-invocati
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { CyclotronJobInvocation, HogFunctionInvocationGlobals, HogFunctionTypeType } from '../types'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
-import { parseBatch } from './message-parsing'
+import { parseBatch, readTeamIdHeader } from './message-parsing'
+import { counterTeamIdHeaderMismatch, counterTeamIdSource } from './metrics'
import { PUSH_NOTIFICATION_OPENED_EVENT, buildPushOpenedMetric } from './push-open-tracking'
export class CdpEventsConsumer<
@@ -197,20 +198,41 @@ export class CdpEventsConsumer<
return await parseBatch(messages, {
errorLog: 'Error parsing message',
parse: (message) => {
- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
+ const parseEvent = (): RawClickHouseEvent => parseJSON(message.value!.toString()) as RawClickHouseEvent
+ const buildGlobals = (team: Team, clickHouseEvent: RawClickHouseEvent) => {
+ const globals = convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL)
+ // The one place holding both. A failure found later in the pipeline carries
+ // these globals, and this is how it gets back to the bytes to park.
+ this.messageByGlobals.set(globals, message)
+ return globals
+ }
+
+ const headerTeamId = readTeamIdHeader(message)
+ if (headerTeamId !== null) {
+ counterTeamIdSource.labels({ source: 'header' }).inc()
+ // The body stays unparsed until `toGlobals` runs, and it only runs for a team that
+ // has a destination or a flow. Most events belong to a team with neither.
+ return {
+ teamId: headerTeamId,
+ toGlobals: (team) => {
+ const clickHouseEvent = parseEvent()
+ if (clickHouseEvent.team_id !== headerTeamId) {
+ // The loaded team comes from the header and the event data comes from
+ // the body. Building globals from two teams would evaluate one team's
+ // event against another team's destinations and send it there.
+ counterTeamIdHeaderMismatch.inc()
+ throw new Error('team_id_header_mismatch')
+ }
+ return buildGlobals(team, clickHouseEvent)
+ },
+ }
+ }
+
+ counterTeamIdSource.labels({ source: 'body' }).inc()
+ const clickHouseEvent = parseEvent()
return {
teamId: clickHouseEvent.team_id,
- toGlobals: (team) => {
- const globals = convertToHogFunctionInvocationGlobals(
- clickHouseEvent,
- team,
- this.config.SITE_URL
- )
- // The one place holding both. A failure found later in the pipeline carries
- // these globals, and this is how it gets back to the bytes to park.
- this.messageByGlobals.set(globals, message)
- return globals
- },
+ toGlobals: (team) => buildGlobals(team, clickHouseEvent),
}
},
loadTeam: (teamId) => this.loadTeamWithHogFunctionsOrFlows(teamId, this.hogTypes),
diff --git a/nodejs/src/cdp/consumers/message-parsing.test.ts b/nodejs/src/cdp/consumers/message-parsing.test.ts
index 0e60069ef244..378f3734024e 100644
--- a/nodejs/src/cdp/consumers/message-parsing.test.ts
+++ b/nodejs/src/cdp/consumers/message-parsing.test.ts
@@ -1,4 +1,6 @@
-import { assertTeamId, skipIfMalformed } from './message-parsing'
+import { Message } from 'node-rdkafka'
+
+import { assertTeamId, readTeamIdHeader, skipIfMalformed } from './message-parsing'
describe('message-parsing', () => {
describe('assertTeamId', () => {
@@ -29,4 +31,34 @@ describe('message-parsing', () => {
expect(onError).toHaveBeenCalledWith('kaboom')
})
})
+ describe('readTeamIdHeader', () => {
+ function withHeaders(headers?: { [key: string]: string | Buffer }[]): Message {
+ return { headers } as unknown as Message
+ }
+
+ type Headers = { [key: string]: string | Buffer }[]
+
+ it.each<[string, Headers, number]>([
+ ['a string value', [{ team_id: '42' }], 42],
+ ['a buffer value', [{ team_id: Buffer.from('42') }], 42],
+ ['a header beside others', [{ productTrack: 'general' }, { team_id: '42' }], 42],
+ ])('reads %s', (_name, headers, expected) => {
+ expect(readTeamIdHeader(withHeaders(headers))).toBe(expected)
+ })
+
+ // A value that is not a usable team id must not become one. Returning it would make the
+ // consumer load another team's destinations for this event, or none at all.
+ it.each<[string, Headers | undefined]>([
+ ['no headers at all', undefined],
+ ['no team_id header', [{ productTrack: 'general' }]],
+ ['an empty value', [{ team_id: '' }]],
+ ['a value that is not a number', [{ team_id: 'not-a-number' }]],
+ ['a value above the int4 range', [{ team_id: '2147483648' }]],
+ ['a zero', [{ team_id: '0' }]],
+ ['a negative value', [{ team_id: '-1' }]],
+ ['a fractional value', [{ team_id: '1.5' }]],
+ ])('falls back to the body for %s', (_name, headers) => {
+ expect(readTeamIdHeader(withHeaders(headers))).toBeNull()
+ })
+ })
})
diff --git a/nodejs/src/cdp/consumers/message-parsing.ts b/nodejs/src/cdp/consumers/message-parsing.ts
index 85fffeed9523..7007a09c0a18 100644
--- a/nodejs/src/cdp/consumers/message-parsing.ts
+++ b/nodejs/src/cdp/consumers/message-parsing.ts
@@ -1,5 +1,6 @@
import { Message } from 'node-rdkafka'
+import { TEAM_ID_HEADER } from '~/common/kafka/headers'
import { logger } from '~/common/utils/logger'
import { isValidTeamId } from '~/common/utils/team-manager'
@@ -16,6 +17,26 @@ export function assertTeamId<T extends { team_id: unknown }>(message: T): T {
return message
}
+/**
+ * Reads the team id the producer put in a Kafka header, so a consumer can decide whether it needs a
+ * message before it parses the body.
+ *
+ * The body still holds the authoritative team id, and a caller that acts on this header has to check
+ * the two agree once it parses the body. Anything unexpected here returns null and the caller reads
+ * the body instead, because a value that is not a usable team id must never become one.
+ */
+export function readTeamIdHeader(message: Message): number | null {
+ for (const header of message.headers ?? []) {
+ const value = header[TEAM_ID_HEADER]
+ if (value === undefined) {
+ continue
+ }
+ const teamId = Number(value.toString())
+ return isValidTeamId(teamId) ? teamId : null
+ }
+ return null
+}
+
function errorMessage(e: unknown): string {
return e instanceof Error ? e.message : String(e)
}
diff --git a/nodejs/src/cdp/consumers/metrics.ts b/nodejs/src/cdp/consumers/metrics.ts
index e09cc4f4fe7f..7b70d17ccea2 100644
--- a/nodejs/src/cdp/consumers/metrics.ts
+++ b/nodejs/src/cdp/consumers/metrics.ts
@@ -23,3 +23,14 @@ export const counterBatchHogFlowTriggerFailed = new Counter({
help: 'A batch hog flow run failed during audience resolution and was skipped',
labelNames: ['hog_flow_id', 'reason'],
})
+
+export const counterTeamIdSource = new Counter({
+ name: 'cdp_events_team_id_source',
+ help: 'Where the consumer read an event team id from. "body" means the message carried no usable team id header, so the consumer parsed the event to find one.',
+ labelNames: ['source'],
+})
+
+export const counterTeamIdHeaderMismatch = new Counter({
+ name: 'cdp_events_team_id_header_mismatch',
+ help: 'An event body named a different team than its header. The event is parked, because the two teams would send one team data to another team destinations.',
+})