:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): put the event name in a kafka header on produced events (#129766)
Meikel Ratzf26dcdd761ca
diff --git a/nodejs/src/common/kafka/headers.ts b/nodejs/src/common/kafka/headers.tsindex 69abb2f112a2..37f150744d96 100644--- a/nodejs/src/common/kafka/headers.ts+++ b/nodejs/src/common/kafka/headers.ts@@ -15,3 +15,12 @@* message on the header alone, and the team the body names would then never see its event.*/export const TEAM_ID_HEADER = 'team_id'++/**+ * The event's name, so a consumer can skip an event no destination or workflow of its team admits+ * without parsing the body.+ *+ * The same rule as the team id applies: a producer takes it from the `event` it writes into the+ * body. A missing header is safe, a wrong one makes a consumer skip an event a destination wanted.+ */+export const EVENT_NAME_HEADER = 'event'diff --git a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snapindex b50ef846316d..fc57d647d331 100644--- a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap+++ b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap@@ -161,6 +161,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -190,6 +191,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -246,6 +248,7 @@ exports[`IngestionConsumer general should process a cookieless event 1`] = `},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -302,6 +305,7 @@ exports[`IngestionConsumer general should process a standard event 1`] = `},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -346,6 +350,7 @@ exports[`IngestionConsumer typical event processing $identify event 1`] = `},{"headers": {+ "event": "$identify","productTrack": "general","team_id": "2",},@@ -402,6 +407,7 @@ exports[`IngestionConsumer typical event processing ai event 1`] = `},{"headers": {+ "event": "$ai_generation","productTrack": "llma","team_id": "2",},@@ -443,6 +449,7 @@ exports[`IngestionConsumer typical event processing event with common distinct_i[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -529,6 +536,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -553,6 +561,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -577,6 +586,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =},{"headers": {+ "event": "$identify","productTrack": "general","team_id": "2",},@@ -601,6 +611,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -670,6 +681,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `},{"headers": {+ "event": "$groupidentify","productTrack": "general","team_id": "2",},@@ -694,6 +706,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `},{"headers": {+ "event": "$groupidentify","productTrack": "general","team_id": "2",},@@ -718,6 +731,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -774,6 +788,7 @@ exports[`IngestionConsumer typical event processing malformed event 1`] = `},{"headers": {+ "event": "","productTrack": "general","team_id": "2",},@@ -815,6 +830,7 @@ exports[`IngestionConsumer typical event processing malformed person information[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -871,6 +887,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -907,6 +924,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `},{"headers": {+ "event": "$identify","productTrack": "general","team_id": "2",},@@ -931,6 +949,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -1017,6 +1036,7 @@ exports[`IngestionConsumer typical event processing normal event 1`] = `},{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -1058,6 +1078,7 @@ exports[`IngestionConsumer typical event processing person processing off 1`] =[{"headers": {+ "event": "$pageview","productTrack": "general","team_id": "2",},@@ -1087,6 +1108,7 @@ exports[`IngestionConsumer typical event processing person property merging via[{"headers": {+ "event": "custom event","productTrack": "general","team_id": "2",},@@ -1135,6 +1157,7 @@ exports[`IngestionConsumer typical event processing person property merging via},{"headers": {+ "event": "custom event","productTrack": "general","team_id": "2",},@@ -1159,6 +1182,7 @@ exports[`IngestionConsumer typical event processing person property merging via},{"headers": {+ "event": "$create_alias","productTrack": "general","team_id": "2",},diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.tsindex b7a3ac943218..c36560215abc 100644--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts@@ -98,7 +98,7 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {key: 'test-uuid',value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),- headers: { productTrack: 'general', team_id: '1' },+ headers: { productTrack: 'general', team_id: '1', event: 'test-event' },teamId: mockProcessedEvent.team_id,})@@ -172,7 +172,7 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {key: 'test-uuid',value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),- headers: { productTrack: 'general', team_id: '1' },+ headers: { productTrack: 'general', team_id: '1', event: 'test-event' },teamId: mockProcessedEvent.team_id,})} finally {@@ -278,20 +278,20 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT,expect.objectContaining({- headers: { productTrack: 'llma', team_id: '1' },+ headers: { productTrack: 'llma', team_id: '1', event: '$ai_generation' },}))})- it('carries the team of the emitted event in the team_id header', async () => {+ it('carries the team and name of the emitted event in headers', async () => {const step = createEmitEventStep(config)- await step(createInput({ ...mockProcessedEvent, team_id: 42 }))+ await step(createInput({ ...mockProcessedEvent, team_id: 42, event: 'order placed' }))expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT,expect.objectContaining({- headers: { productTrack: 'general', team_id: '42' },+ headers: { productTrack: 'general', team_id: '42', event: 'order placed' },}))})diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.tsindex 65fb1c135b3e..42a7bc61b699 100644--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts@@ -1,7 +1,7 @@import { DateTime } from 'luxon'import { Message } from 'node-rdkafka'-import { TEAM_ID_HEADER } from '~/common/kafka/headers'+import { EVENT_NAME_HEADER, TEAM_ID_HEADER } from '~/common/kafka/headers'import { IngestionWarningsOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'@@ -88,6 +88,7 @@ export function createEmitEventStep<O extends string, T extends EmitEventStepInpheaders: {productTrack: productTrackHeader(event),[TEAM_ID_HEADER]: String(serialized.team_id),+ [EVENT_NAME_HEADER]: serialized.event,},teamId: serialized.team_id,})