Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): put the team id in a kafka header on produced events (#127163)

Meikel Ratz1eed11d6e4e2
diff --git a/nodejs/src/common/kafka/headers.ts b/nodejs/src/common/kafka/headers.ts
new file mode 100644
index 000000000000..69abb2f112a2
--- /dev/null
+++ b/nodejs/src/common/kafka/headers.ts
@@ -0,0 +1,17 @@
+/**
+ * Kafka header names that a producer and a consumer in different trees have to agree on.
+ *
+ * The name is a shared constant rather than a literal on each side because a drifted name fails
+ * open: the consumer finds no header and falls back to the message body, so it costs performance
+ * and nothing reports it. A drifted name is not the same as a wrong value, which each header
+ * documents below.
+ */
+
+/**
+ * The event's team, so a consumer can route or skip a message without parsing the body.
+ *
+ * A producer must take this from the same `team_id` it writes into the body. A missing header is
+ * safe, because a consumer falls back to the body, but a wrong one is not: a consumer may skip the
+ * message on the header alone, and the team the body names would then never see its event.
+ */
+export const TEAM_ID_HEADER = 'team_id'
diff --git a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
index 196364592f75..b50ef846316d 100644
--- a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
+++ b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
@@ -162,6 +162,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -190,6 +191,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -245,6 +247,7 @@ exports[`IngestionConsumer general should process a cookieless event 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -300,6 +303,7 @@ exports[`IngestionConsumer general should process a standard event 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -343,6 +347,7 @@ exports[`IngestionConsumer typical event processing $identify event 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -398,6 +403,7 @@ exports[`IngestionConsumer typical event processing ai event 1`] = `
{
"headers": {
"productTrack": "llma",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -438,6 +444,7 @@ exports[`IngestionConsumer typical event processing event with common distinct_i
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -523,6 +530,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-2>",
"topic": "clickhouse_events_json_test",
@@ -546,6 +554,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-3>",
"topic": "clickhouse_events_json_test",
@@ -569,6 +578,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -592,6 +602,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-4>",
"topic": "clickhouse_events_json_test",
@@ -660,6 +671,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-2>",
"topic": "clickhouse_events_json_test",
@@ -683,6 +695,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-3>",
"topic": "clickhouse_events_json_test",
@@ -706,6 +719,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -761,6 +775,7 @@ exports[`IngestionConsumer typical event processing malformed event 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -801,6 +816,7 @@ exports[`IngestionConsumer typical event processing malformed person information
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -856,6 +872,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -891,6 +908,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-2>",
"topic": "clickhouse_events_json_test",
@@ -914,6 +932,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-4>",
"topic": "clickhouse_events_json_test",
@@ -999,6 +1018,7 @@ exports[`IngestionConsumer typical event processing normal event 1`] = `
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-1>",
"topic": "clickhouse_events_json_test",
@@ -1039,6 +1059,7 @@ exports[`IngestionConsumer typical event processing person processing off 1`] =
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -1067,6 +1088,7 @@ exports[`IngestionConsumer typical event processing person property merging via
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-0>",
"topic": "clickhouse_events_json_test",
@@ -1114,6 +1136,7 @@ exports[`IngestionConsumer typical event processing person property merging via
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-4>",
"topic": "clickhouse_events_json_test",
@@ -1137,6 +1160,7 @@ exports[`IngestionConsumer typical event processing person property merging via
{
"headers": {
"productTrack": "general",
+ "team_id": "2",
},
"key": "<REPLACED-UUID-6>",
"topic": "clickhouse_events_json_test",
diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
index 416e60175d8d..b7a3ac943218 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
@@ -98,7 +98,7 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
- headers: { productTrack: 'general' },
+ headers: { productTrack: 'general', team_id: '1' },
teamId: mockProcessedEvent.team_id,
})
@@ -172,7 +172,7 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
- headers: { productTrack: 'general' },
+ headers: { productTrack: 'general', team_id: '1' },
teamId: mockProcessedEvent.team_id,
})
} finally {
@@ -278,7 +278,20 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
- headers: { productTrack: 'llma' },
+ headers: { productTrack: 'llma', team_id: '1' },
+ })
+ )
+ })
+
+ it('carries the team of the emitted event in the team_id header', async () => {
+ const step = createEmitEventStep(config)
+
+ await step(createInput({ ...mockProcessedEvent, team_id: 42 }))
+
+ expect(mockOutputs.produce).toHaveBeenCalledWith(
+ EVENTS_OUTPUT,
+ expect.objectContaining({
+ headers: { productTrack: 'general', team_id: '42' },
})
)
})
diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
index 49f311be0626..65fb1c135b3e 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
@@ -1,6 +1,7 @@
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
+import { TEAM_ID_HEADER } from '~/common/kafka/headers'
import { IngestionWarningsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
@@ -84,7 +85,10 @@ export function createEmitEventStep<O extends string, T extends EmitEventStepInp
.produce(output, {
key: serialized.uuid,
value: Buffer.from(JSON.stringify(serialized)),
- headers: { productTrack: productTrackHeader(event) },
+ headers: {
+ productTrack: productTrackHeader(event),
+ [TEAM_ID_HEADER]: String(serialized.team_id),
+ },
teamId: serialized.team_id,
})
.then(() => {