:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import { IngestionWarningsOutput } from '~/common/outputs'
import { EVENTS_OUTPUT, EventOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { MessageSizeTooLarge } from '~/common/utils/db/error'
import { emitIngestionWarning } from '~/ingestion/common/ingestion-warnings'
import { eventProcessedAndIngestedCounter } from '~/ingestion/common/metrics'
import { isOkResult } from '~/ingestion/framework/results'
import { createTestEventHeaders } from '~/tests/helpers/event-headers'
import { createTestMessage } from '~/tests/helpers/kafka-message'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import { EventHeaders, ISOTimestamp, ProcessedEvent, ProjectId } from '~/types'
import {
EmitEventStepConfig,
EmitEventStepInput,
createEmitEventStep,
productTrackHeader,
serializeEvent,
} from './emit-event-step'
jest.mock('~/ingestion/common/ingestion-warnings', () => ({
emitIngestionWarning: jest.fn().mockResolvedValue(undefined),
}))
// Mock the metrics module
jest.mock('~/ingestion/common/metrics', () => ({
eventProcessedAndIngestedCounter: {
inc: jest.fn(),
},
}))
const mockEmitIngestionWarning = jest.mocked(emitIngestionWarning)
const mockEventProcessedAndIngestedCounter = jest.mocked(eventProcessedAndIngestedCounter)
describe('emit-event-step', () => {
let mockOutputs: jest.Mocked<IngestionOutputs<EventOutput | IngestionWarningsOutput>>
let config: EmitEventStepConfig<EventOutput>
let mockProcessedEvent: ProcessedEvent
let mockHeaders: EventHeaders
let mockMessage: Message
beforeEach(() => {
mockHeaders = createTestEventHeaders()
mockMessage = createTestMessage()
jest.clearAllMocks()
mockOutputs = createMockIngestionOutputs<EventOutput | IngestionWarningsOutput>()
config = {
outputs: mockOutputs,
}
mockProcessedEvent = {
uuid: 'test-uuid',
event: 'test-event',
properties: { test: 'property' },
timestamp: '2023-01-01T00:00:00.000Z' as ISOTimestamp,
team_id: 1,
project_id: 1 as ProjectId,
distinct_id: 'test-distinct-id',
elements_chain: '',
created_at: DateTime.fromISO('2023-01-01T00:01:00.000Z'),
captured_at: null,
person_id: 'person-uuid',
person_properties: {},
person_created_at: DateTime.fromISO('2023-01-01T00:00:00.000Z'),
person_mode: 'full',
}
})
function createInput(event: ProcessedEvent = mockProcessedEvent): EmitEventStepInput<EventOutput> {
return {
eventsToEmit: [{ event, output: EVENTS_OUTPUT }],
teamId: event.team_id,
headers: mockHeaders,
message: mockMessage,
}
}
describe('createEmitEventStep', () => {
it('should emit event successfully when eventsToEmit is present', async () => {
jest.useFakeTimers()
try {
const step = createEmitEventStep(config)
const input = createInput()
const result = await step(input)
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.value.ingested).toHaveLength(1)
expect(result.value.ingested).toEqual(result.sideEffects)
}
expect(result.sideEffects).toHaveLength(1)
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
headers: { productTrack: 'general', team_id: '1', event: 'test-event' },
teamId: mockProcessedEvent.team_id,
})
// The ingested promise resolves with the event info once acked
await expect(result.sideEffects[0]).resolves.toEqual({
capturedAt: mockHeaders.now,
topic: mockMessage.topic,
partition: mockMessage.partition,
})
expect(mockEventProcessedAndIngestedCounter.inc).toHaveBeenCalledTimes(1)
} finally {
jest.useRealTimers()
}
})
it('should handle MessageSizeTooLarge error and capture ingestion warning', async () => {
const messageSizeTooLargeError = new MessageSizeTooLarge('Message too large', new Error('Kafka error'))
mockOutputs.produce.mockRejectedValue(messageSizeTooLargeError)
const step = createEmitEventStep(config)
const input = createInput()
const result = await step(input)
expect(isOkResult(result)).toBe(true)
expect(result.sideEffects).toHaveLength(1)
// The event was not ingested, so the promise resolves with null
await expect(result.sideEffects[0]).resolves.toBeNull()
expect(mockEmitIngestionWarning).toHaveBeenCalledWith(mockOutputs, 1, {
type: 'message_size_too_large',
details: {
eventUuid: 'test-uuid',
distinctId: 'test-distinct-id',
personId: 'person-uuid',
},
pipelineStep: 'emit-event',
})
// Metric should not be incremented when there's an error
expect(mockEventProcessedAndIngestedCounter.inc).not.toHaveBeenCalled()
})
it('should re-throw non-MessageSizeTooLarge errors', async () => {
const genericError = new Error('Generic Kafka error')
mockOutputs.produce.mockRejectedValue(genericError)
const step = createEmitEventStep(config)
const input = createInput()
const result = await step(input)
expect(isOkResult(result)).toBe(true)
expect(result.sideEffects).toHaveLength(1)
// Execute the side effect to test error handling
await expect(result.sideEffects[0]).rejects.toThrow('Generic Kafka error')
expect(mockEmitIngestionWarning).not.toHaveBeenCalled()
// Metric should not be incremented when there's an error
expect(mockEventProcessedAndIngestedCounter.inc).not.toHaveBeenCalled()
})
it('should serialize event correctly for Kafka', async () => {
jest.useFakeTimers()
try {
const step = createEmitEventStep(config)
const input = createInput()
await step(input)
expect(mockOutputs.produce).toHaveBeenCalledWith(EVENTS_OUTPUT, {
key: 'test-uuid',
value: Buffer.from(JSON.stringify(serializeEvent(mockProcessedEvent))),
headers: { productTrack: 'general', team_id: '1', event: 'test-event' },
teamId: mockProcessedEvent.team_id,
})
} finally {
jest.useRealTimers()
}
})
it('should handle events with different UUIDs correctly', async () => {
const step = createEmitEventStep(config)
const eventWithDifferentUuid = { ...mockProcessedEvent, uuid: 'different-uuid' }
const input = createInput(eventWithDifferentUuid)
await step(input)
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
key: 'different-uuid',
})
)
})
it('should handle multiple events in eventsToEmit', async () => {
const step = createEmitEventStep(config)
const event2 = { ...mockProcessedEvent, uuid: 'second-uuid' }
const input: EmitEventStepInput<EventOutput> = {
eventsToEmit: [
{ event: mockProcessedEvent, output: EVENTS_OUTPUT },
{ event: event2, output: EVENTS_OUTPUT },
],
teamId: 1,
headers: mockHeaders,
message: mockMessage,
}
const result = await step(input)
expect(isOkResult(result)).toBe(true)
expect(result.sideEffects).toHaveLength(2)
expect(mockOutputs.produce).toHaveBeenCalledTimes(2)
})
describe('metrics tracking', () => {
it('should increment eventProcessedAndIngestedCounter when event is successfully emitted', async () => {
const step = createEmitEventStep(config)
const input = createInput()
const result = await step(input)
expect(isOkResult(result)).toBe(true)
expect(result.sideEffects).toHaveLength(1)
// Execute the side effect to trigger metric increment
await result.sideEffects[0]
expect(mockEventProcessedAndIngestedCounter.inc).toHaveBeenCalledTimes(1)
expect(mockOutputs.produce).toHaveBeenCalledTimes(1)
})
it('should not increment metric when Kafka produce fails', async () => {
const kafkaError = new Error('Kafka connection failed')
mockOutputs.produce.mockRejectedValue(kafkaError)
const step = createEmitEventStep(config)
const input = createInput()
const result = await step(input)
expect(isOkResult(result)).toBe(true)
expect(result.sideEffects).toHaveLength(1)
// Execute the side effect and expect it to throw
await expect(result.sideEffects[0]).rejects.toThrow('Kafka connection failed')
// Metric should not be incremented on failure
expect(mockEventProcessedAndIngestedCounter.inc).not.toHaveBeenCalled()
})
it('should increment metric only once per successful emit', async () => {
const step = createEmitEventStep(config)
// First emit
const result1 = await step(createInput())
await result1.sideEffects[0]
// Second emit
const result2 = await step(createInput({ ...mockProcessedEvent, uuid: 'different-uuid' }))
await result2.sideEffects[0]
// Metric should be incremented twice, once for each successful emit
expect(mockEventProcessedAndIngestedCounter.inc).toHaveBeenCalledTimes(2)
expect(mockOutputs.produce).toHaveBeenCalledTimes(2)
})
})
it('should emit AI events with llma product track header', async () => {
const aiEvent = { ...mockProcessedEvent, event: '$ai_generation' }
const step = createEmitEventStep(config)
const input = createInput(aiEvent)
await step(input)
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
headers: { productTrack: 'llma', team_id: '1', event: '$ai_generation' },
})
)
})
it('carries the team and name of the emitted event in headers', async () => {
const step = createEmitEventStep(config)
await step(createInput({ ...mockProcessedEvent, team_id: 42, event: 'order placed' }))
expect(mockOutputs.produce).toHaveBeenCalledWith(
EVENTS_OUTPUT,
expect.objectContaining({
headers: { productTrack: 'general', team_id: '42', event: 'order placed' },
})
)
})
})
describe('productTrackHeader', () => {
it('should return "llma" for AI generation events', () => {
const aiEvent = { ...mockProcessedEvent, event: '$ai_generation' }
expect(productTrackHeader(aiEvent)).toBe('llma')
})
it('should return "llma" for AI completion events', () => {
const aiEvent = { ...mockProcessedEvent, event: '$ai_completion' }
expect(productTrackHeader(aiEvent)).toBe('llma')
})
it('should return "general" for non-AI events', () => {
const regularEvent = { ...mockProcessedEvent, event: '$pageview' }
expect(productTrackHeader(regularEvent)).toBe('general')
})
it('should return "general" for custom events', () => {
const customEvent = { ...mockProcessedEvent, event: 'user_signed_up' }
expect(productTrackHeader(customEvent)).toBe('general')
})
})
})