:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): put the team id in a kafka header on produced events (#127163)
Meikel Ratz1eed11d6e4e2
diff --git a/nodejs/src/common/kafka/headers.ts b/nodejs/src/common/kafka/headers.tsnew file mode 100644index 000000000000..69abb2f112a2--- /dev/null+++ b/nodejs/src/common/kafka/headers.ts@@ -0,0 +1,17 @@+/**+ * Kafka header names that a producer and a consumer in different trees have to agree on.+ *+ * The name is a shared constant rather than a literal on each side because a drifted name fails+ * open: the consumer finds no header and falls back to the message body, so it costs performance+ * and nothing reports it. A drifted name is not the same as a wrong value, which each header+ * documents below.+ */++/**+ * The event's team, so a consumer can route or skip a message without parsing the body.+ *+ * A producer must take this from the same `team_id` it writes into the body. A missing header is+ * safe, because a consumer falls back to the body, but a wrong one is not: a consumer may skip the+ * message on the header alone, and the team the body names would then never see its event.+ */+export const TEAM_ID_HEADER = 'team_id'diff --git a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snapindex 196364592f75..b50ef846316d 100644--- a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap+++ b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap@@ -162,6 +162,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -190,6 +191,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -245,6 +247,7 @@ exports[`IngestionConsumer general should process a cookieless event 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -300,6 +303,7 @@ exports[`IngestionConsumer general should process a standard event 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -343,6 +347,7 @@ exports[`IngestionConsumer typical event processing $identify event 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -398,6 +403,7 @@ exports[`IngestionConsumer typical event processing ai event 1`] = `{"headers": {"productTrack": "llma",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -438,6 +444,7 @@ exports[`IngestionConsumer typical event processing event with common distinct_i{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -523,6 +530,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] ={"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-2>","topic": "clickhouse_events_json_test",@@ -546,6 +554,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] ={"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-3>","topic": "clickhouse_events_json_test",@@ -569,6 +578,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] ={"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -592,6 +602,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] ={"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-4>","topic": "clickhouse_events_json_test",@@ -660,6 +671,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-2>","topic": "clickhouse_events_json_test",@@ -683,6 +695,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-3>","topic": "clickhouse_events_json_test",@@ -706,6 +719,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -761,6 +775,7 @@ exports[`IngestionConsumer typical event processing malformed event 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -801,6 +816,7 @@ exports[`IngestionConsumer typical event processing malformed person information{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -856,6 +872,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -891,6 +908,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-2>","topic": "clickhouse_events_json_test",@@ -914,6 +932,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-4>","topic": "clickhouse_events_json_test",@@ -999,6 +1018,7 @@ exports[`IngestionConsumer typical event processing normal event 1`] = `{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-1>","topic": "clickhouse_events_json_test",@@ -1039,6 +1059,7 @@ exports[`IngestionConsumer typical event processing person processing off 1`] ={"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -1067,6 +1088,7 @@ exports[`IngestionConsumer typical event processing person property merging via{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-0>","topic": "clickhouse_events_json_test",@@ -1114,6 +1136,7 @@ exports[`IngestionConsumer typical event processing person property merging via{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-4>","topic": "clickhouse_events_json_test",@@ -1137,6 +1160,7 @@ exports[`IngestionConsumer typical event processing person property merging via{"headers": {"productTrack": "general",+ "team_id": "2",},"key": "<REPLACED-UUID-6>","topic": "clickhouse_events_json_test",diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.tsindex 416e60175d8d..b7a3ac943218 100644--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts@@ -98,7 +98,7 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {key: 'test-uuid',value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),- headers: { productTrack: 'general' },+ headers: { productTrack: 'general', team_id: '1' },teamId: mockProcessedEvent.team_id,})@@ -172,7 +172,7 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {key: 'test-uuid',value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),- headers: { productTrack: 'general' },+ headers: { productTrack: 'general', team_id: '1' },teamId: mockProcessedEvent.team_id,})} finally {@@ -278,7 +278,20 @@ describe('emit-event-step', () => {expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT,expect.objectContaining({- headers: { productTrack: 'llma' },+ headers: { productTrack: 'llma', team_id: '1' },+ })+ )+ })++ it('carries the team of the emitted event in the team_id header', async () => {+ const step = createEmitEventStep(config)++ await step(createInput({ ...mockProcessedEvent, team_id: 42 }))++ expect(mockOutputs.produce).toHaveBeenCalledWith(+ EVENTS_OUTPUT,+ expect.objectContaining({+ headers: { productTrack: 'general', team_id: '42' },}))})diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.tsindex 49f311be0626..65fb1c135b3e 100644--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts@@ -1,6 +1,7 @@import { DateTime } from 'luxon'import { Message } from 'node-rdkafka'+import { TEAM_ID_HEADER } from '~/common/kafka/headers'import { IngestionWarningsOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'@@ -84,7 +85,10 @@ export function createEmitEventStep<O extends string, T extends EmitEventStepInp.produce(output, {key: serialized.uuid,value: Buffer.from(JSON.stringify(serialized)),- headers: { productTrack: productTrackHeader(event) },+ headers: {+ productTrack: productTrackHeader(event),+ [TEAM_ID_HEADER]: String(serialized.team_id),+ },teamId: serialized.team_id,}).then(() => {