Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): put the event name in a kafka header on produced events (#129766)

Meikel Ratzf26dcdd761ca
diff --git a/nodejs/src/common/kafka/headers.ts b/nodejs/src/common/kafka/headers.ts
index 69abb2f112a2..37f150744d96 100644
--- a/nodejs/src/common/kafka/headers.ts
+++ b/nodejs/src/common/kafka/headers.ts
@@ -15,3 +15,12 @@
* message on the header alone, and the team the body names would then never see its event.
*/
export const TEAM_ID_HEADER = 'team_id'
+
+/**
+ * The event's name, so a consumer can skip an event no destination or workflow of its team admits
+ * without parsing the body.
+ *
+ * The same rule as the team id applies: a producer takes it from the `event` it writes into the
+ * body. A missing header is safe, a wrong one makes a consumer skip an event a destination wanted.
+ */
+export const EVENT_NAME_HEADER = 'event'
diff --git a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
index b50ef846316d..fc57d647d331 100644
--- a/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
+++ b/nodejs/src/ingestion/__snapshots__/ingestion-consumer.serial.test.ts.snap
@@ -161,6 +161,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -190,6 +191,7 @@ exports[`IngestionConsumer general should not blend person properties from 2 dif
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -246,6 +248,7 @@ exports[`IngestionConsumer general should process a cookieless event 1`] = `
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -302,6 +305,7 @@ exports[`IngestionConsumer general should process a standard event 1`] = `
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -346,6 +350,7 @@ exports[`IngestionConsumer typical event processing $identify event 1`] = `
},
{
"headers": {
+ "event": "$identify",
"productTrack": "general",
"team_id": "2",
},
@@ -402,6 +407,7 @@ exports[`IngestionConsumer typical event processing ai event 1`] = `
},
{
"headers": {
+ "event": "$ai_generation",
"productTrack": "llma",
"team_id": "2",
},
@@ -443,6 +449,7 @@ exports[`IngestionConsumer typical event processing event with common distinct_i
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -529,6 +536,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -553,6 +561,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -577,6 +586,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
},
{
"headers": {
+ "event": "$identify",
"productTrack": "general",
"team_id": "2",
},
@@ -601,6 +611,7 @@ exports[`IngestionConsumer typical event processing forced person upgrade 1`] =
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -670,6 +681,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
},
{
"headers": {
+ "event": "$groupidentify",
"productTrack": "general",
"team_id": "2",
},
@@ -694,6 +706,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
},
{
"headers": {
+ "event": "$groupidentify",
"productTrack": "general",
"team_id": "2",
},
@@ -718,6 +731,7 @@ exports[`IngestionConsumer typical event processing groups 1`] = `
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -774,6 +788,7 @@ exports[`IngestionConsumer typical event processing malformed event 1`] = `
},
{
"headers": {
+ "event": "",
"productTrack": "general",
"team_id": "2",
},
@@ -815,6 +830,7 @@ exports[`IngestionConsumer typical event processing malformed person information
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -871,6 +887,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -907,6 +924,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
},
{
"headers": {
+ "event": "$identify",
"productTrack": "general",
"team_id": "2",
},
@@ -931,6 +949,7 @@ exports[`IngestionConsumer typical event processing multiple events 1`] = `
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -1017,6 +1036,7 @@ exports[`IngestionConsumer typical event processing normal event 1`] = `
},
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -1058,6 +1078,7 @@ exports[`IngestionConsumer typical event processing person processing off 1`] =
[
{
"headers": {
+ "event": "$pageview",
"productTrack": "general",
"team_id": "2",
},
@@ -1087,6 +1108,7 @@ exports[`IngestionConsumer typical event processing person property merging via
[
{
"headers": {
+ "event": "custom event",
"productTrack": "general",
"team_id": "2",
},
@@ -1135,6 +1157,7 @@ exports[`IngestionConsumer typical event processing person property merging via
},
{
"headers": {
+ "event": "custom event",
"productTrack": "general",
"team_id": "2",
},
@@ -1159,6 +1182,7 @@ exports[`IngestionConsumer typical event processing person property merging via
},
{
"headers": {
+ "event": "$create_alias",
"productTrack": "general",
"team_id": "2",
},
diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
index b7a3ac943218..c36560215abc 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.test.ts
@@ -98,7 +98,7 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
- headers: { productTrack: 'general', team_id: '1' },
+ headers: { productTrack: 'general', team_id: '1', event: 'test-event' },
teamId: mockProcessedEvent.team_id,
})
@@ -172,7 +172,7 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
- headers: { productTrack: 'general', team_id: '1' },
+ headers: { productTrack: 'general', team_id: '1', event: 'test-event' },
teamId: mockProcessedEvent.team_id,
})
} finally {
@@ -278,20 +278,20 @@ describe('emit-event-step', () => {
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
- headers: { productTrack: 'llma', team_id: '1' },
+ headers: { productTrack: 'llma', team_id: '1', event: '$ai_generation' },
})
)
})
- it('carries the team of the emitted event in the team_id header', async () => {
+ it('carries the team and name of the emitted event in headers', async () => {
const step = createEmitEventStep(config)
- await step(createInput({ ...mockProcessedEvent, team_id: 42 }))
+ await step(createInput({ ...mockProcessedEvent, team_id: 42, event: 'order placed' }))
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
- headers: { productTrack: 'general', team_id: '42' },
+ headers: { productTrack: 'general', team_id: '42', event: 'order placed' },
})
)
})
diff --git a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
index 65fb1c135b3e..42a7bc61b699 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/emit-event-step.ts
@@ -1,7 +1,7 @@
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
-import { TEAM_ID_HEADER } from '~/common/kafka/headers'
+import { EVENT_NAME_HEADER, TEAM_ID_HEADER } from '~/common/kafka/headers'
import { IngestionWarningsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
@@ -88,6 +88,7 @@ export function createEmitEventStep<O extends string, T extends EmitEventStepInp
headers: {
productTrack: productTrackHeader(event),
[TEAM_ID_HEADER]: String(serialized.team_id),
+ [EVENT_NAME_HEADER]: serialized.event,
},
teamId: serialized.team_id,
})