:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { DLQ_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import {
logDroppedMessage,
produceMessageToDLQ,
redirectMessageToOutput,
} from '~/ingestion/framework/result-handling-helpers'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
// Mock all dependencies
jest.mock('~/common/utils/logger')
jest.mock('~/common/utils/posthog')
const mockLogger = logger as jest.Mocked<typeof logger>
const mockCaptureException = captureException as jest.MockedFunction<typeof captureException>
describe('produceMessageToDLQ', () => {
let mockOutputs: jest.Mocked<IngestionOutputs<'dlq' | 'ingestion_warnings'>>
let mockMessage: Message
beforeEach(() => {
jest.clearAllMocks()
mockOutputs = createMockIngestionOutputs<'dlq' | 'ingestion_warnings'>()
mockMessage = {
value: Buffer.from('test message'),
topic: 'test-topic',
partition: 0,
offset: 123,
key: 'test-key',
size: 12,
headers: [
{ team_id: Buffer.from('42') },
{ distinct_id: Buffer.from('test-user') },
{ event: 'pageview' },
{ uuid: 'test-uuid-123' },
],
} as Message
})
it('rethrows a failed DLQ produce after logging it when rethrowOnFailure is set', async () => {
const dlqError = new Error('broker down')
mockOutputs.produce.mockRejectedValueOnce(dlqError)
await expect(
produceMessageToDLQ(mockOutputs, mockMessage, new Error('Test error'), 'test-step', {
rethrowOnFailure: true,
})
).rejects.toBe(dlqError)
expect(mockLogger.error).toHaveBeenCalledWith('Failed to send event to DLQ', expect.anything())
expect(mockCaptureException).toHaveBeenCalledWith(dlqError, expect.anything())
})
it('should send message to DLQ with proper headers and logging', async () => {
const error = new Error('Test error')
const stepName = 'test-step'
await produceMessageToDLQ(mockOutputs, mockMessage, error, stepName)
expect(mockLogger.warn).toHaveBeenCalledWith('Event sent to DLQ', {
step: stepName,
team_id: '42',
uuid: 'test-uuid-123',
distinct_id: 'test-user',
event: 'pageview',
error: 'Test error',
})
expect(mockOutputs.produce).toHaveBeenCalledWith(DLQ_OUTPUT, {
value: mockMessage.value,
key: mockMessage.key,
headers: expect.objectContaining({
team_id: '42',
distinct_id: 'test-user',
event: 'pageview',
dlq_reason: 'Test error',
dlq_step: stepName,
dlq_timestamp: expect.any(String),
dlq_topic: 'test-topic',
dlq_partition: '0',
dlq_offset: '123',
}),
})
})
it('should handle message without headers', async () => {
const messageWithoutHeaders = { ...mockMessage, headers: undefined } as Message
const error = new Error('Test error')
const stepName = 'test-step'
await produceMessageToDLQ(mockOutputs, messageWithoutHeaders, error, stepName)
expect(mockLogger.warn).toHaveBeenCalledWith('Event sent to DLQ', {
step: stepName,
team_id: undefined,
distinct_id: undefined,
event: undefined,
error: 'Test error',
})
})
it('should handle different header value types', async () => {
const messageWithMixedHeaders = {
...mockMessage,
headers: [
{ team_id: 42 }, // number
{ distinct_id: 'test-user' }, // string
{ event: Buffer.from('pageview') }, // Buffer
{ custom: null }, // null
{ undefined: undefined }, // undefined
],
} as Message
const error = new Error('Test error')
const stepName = 'test-step'
await produceMessageToDLQ(mockOutputs, messageWithMixedHeaders, error, stepName)
expect(mockOutputs.produce).toHaveBeenCalledWith(DLQ_OUTPUT, {
value: messageWithMixedHeaders.value,
key: messageWithMixedHeaders.key,
headers: expect.objectContaining({
team_id: '42',
distinct_id: 'test-user',
event: 'pageview',
custom: 'null',
// undefined should not be included
}),
})
})
it('should handle non-Error objects', async () => {
const error = 'String error'
const stepName = 'test-step'
await produceMessageToDLQ(mockOutputs, mockMessage, error, stepName)
expect(mockOutputs.produce).toHaveBeenCalledWith(
DLQ_OUTPUT,
expect.objectContaining({
headers: expect.objectContaining({
dlq_reason: 'String error',
}),
})
)
})
it('should handle DLQ failures gracefully', async () => {
const error = new Error('Test error')
const stepName = 'test-step'
const dlqError = new Error('DLQ failed')
mockOutputs.produce.mockRejectedValue(dlqError)
await produceMessageToDLQ(mockOutputs, mockMessage, error, stepName)
expect(mockLogger.error).toHaveBeenCalledWith('Failed to send event to DLQ', {
step: stepName,
uuid: 'test-uuid-123',
event: 'pageview',
team_id: '42',
distinct_id: 'test-user',
error: dlqError,
})
expect(mockCaptureException).toHaveBeenCalledWith(dlqError, {
tags: { team_id: '42', pipeline_step: stepName },
extra: { originalMessage: mockMessage, error: dlqError },
})
})
})
describe('redirectMessageToOutput', () => {
const TEST_REDIRECT = 'test_redirect' as const
let mockOutputs: jest.Mocked<IngestionOutputs<typeof TEST_REDIRECT>>
let mockPromiseScheduler: PromiseScheduler
let mockMessage: Message
beforeEach(() => {
jest.clearAllMocks()
mockOutputs = {
produce: jest.fn().mockResolvedValue(undefined),
queueMessages: jest.fn().mockResolvedValue(undefined),
} as any
mockPromiseScheduler = {
schedule: jest.fn().mockImplementation((promise) => promise),
} as unknown as PromiseScheduler
mockMessage = {
value: Buffer.from('test message'),
topic: 'test-topic',
partition: 0,
offset: 123,
key: 'test-key',
size: 12,
headers: [{ team_id: Buffer.from('42') }, { distinct_id: Buffer.from('test-user') }, { event: 'pageview' }],
} as Message
})
it('should redirect message to output with default parameters', async () => {
const stepName = 'test-step'
await redirectMessageToOutput(mockOutputs, TEST_REDIRECT, mockPromiseScheduler, mockMessage, stepName)
expect(mockOutputs.produce).toHaveBeenCalledWith(TEST_REDIRECT, {
value: mockMessage.value,
key: mockMessage.key,
headers: expect.objectContaining({
team_id: '42',
distinct_id: 'test-user',
event: 'pageview',
'redirect-step': stepName,
'redirect-timestamp': expect.any(String),
}),
})
expect(mockPromiseScheduler.schedule).toHaveBeenCalled()
})
it('should handle preserveKey = false', async () => {
const stepName = 'test-step'
await redirectMessageToOutput(mockOutputs, TEST_REDIRECT, mockPromiseScheduler, mockMessage, stepName, false)
expect(mockOutputs.produce).toHaveBeenCalledWith(TEST_REDIRECT, {
value: mockMessage.value,
key: null,
// The dropped partition key rides along in a header so the overflow
// lane can still refresh the flag that routed the event there.
headers: expect.objectContaining({ 'redirect-original-key': 'test-key' }),
})
})
it('should handle awaitAck = false', async () => {
const stepName = 'test-step'
const neverResolvingPromise = new Promise(() => {})
mockOutputs.produce = jest.fn().mockReturnValue(neverResolvingPromise)
mockPromiseScheduler.schedule = jest.fn().mockImplementation((promise) => promise)
await redirectMessageToOutput(
mockOutputs,
TEST_REDIRECT,
mockPromiseScheduler,
mockMessage,
stepName,
true,
false
)
expect(mockOutputs.produce).toHaveBeenCalled()
expect(mockPromiseScheduler.schedule).toHaveBeenCalled()
})
it('should handle redirect failures', async () => {
const stepName = 'test-step'
const redirectError = new Error('Redirect failed')
mockOutputs.produce = jest.fn().mockRejectedValue(redirectError)
await expect(
redirectMessageToOutput(mockOutputs, TEST_REDIRECT, mockPromiseScheduler, mockMessage, stepName)
).rejects.toThrow('Redirect failed')
expect(mockCaptureException).toHaveBeenCalledWith(redirectError, {
tags: {
team_id: '42',
pipeline_step: stepName,
},
extra: {
output: TEST_REDIRECT,
distinct_id: 'test-user',
event: 'pageview',
error: redirectError,
},
})
})
it('should use default step name when not provided', async () => {
await redirectMessageToOutput(mockOutputs, TEST_REDIRECT, mockPromiseScheduler, mockMessage)
})
})
describe('logDroppedMessage', () => {
let mockMessage: Message
beforeEach(() => {
jest.clearAllMocks()
mockMessage = {
value: Buffer.from('test message'),
topic: 'test-topic',
partition: 0,
offset: 123,
key: 'test-key',
size: 12,
headers: [{ team_id: Buffer.from('42') }, { distinct_id: Buffer.from('test-user') }, { event: 'pageview' }],
} as Message
})
it('should log dropped message with proper metadata', () => {
const reason = 'Invalid format'
const stepName = 'test-step'
logDroppedMessage(mockMessage, reason, stepName)
expect(mockLogger.debug).toHaveBeenCalledWith('Event dropped', {
step: stepName,
team_id: '42',
distinct_id: 'test-user',
event: 'pageview',
reason,
})
})
it('should handle message without headers', () => {
const messageWithoutHeaders = { ...mockMessage, headers: undefined } as Message
const reason = 'Invalid format'
const stepName = 'test-step'
logDroppedMessage(messageWithoutHeaders, reason, stepName)
expect(mockLogger.debug).toHaveBeenCalledWith('Event dropped', {
step: stepName,
team_id: undefined,
distinct_id: undefined,
event: undefined,
reason,
})
})
it('should use default step name when not provided', () => {
const reason = 'Invalid format'
logDroppedMessage(mockMessage, reason)
expect(mockLogger.debug).toHaveBeenCalledWith(
'Event dropped',
expect.objectContaining({
step: 'unknown',
})
)
})
})
describe('Header processing utilities', () => {
let mockMessage: Message
beforeEach(() => {
mockMessage = {
value: Buffer.from('test message'),
topic: 'test-topic',
partition: 0,
offset: 123,
key: 'test-key',
size: 12,
headers: [
{ stringHeader: 'string-value' },
{ bufferHeader: Buffer.from('buffer-value') },
{ numberHeader: 42 },
{ nullHeader: null },
{ undefinedHeader: undefined },
],
} as Message
})
it('should correctly process different header types in produceMessageToDLQ', async () => {
const headerMockOutputs = createMockIngestionOutputs<'dlq' | 'ingestion_warnings'>()
await produceMessageToDLQ(headerMockOutputs, mockMessage, new Error('test'), 'test-step')
expect(headerMockOutputs.produce).toHaveBeenCalledWith(
DLQ_OUTPUT,
expect.objectContaining({
headers: expect.objectContaining({
stringHeader: 'string-value',
bufferHeader: 'buffer-value',
numberHeader: '42',
nullHeader: 'null',
// undefinedHeader should not be present
}),
})
)
})
it('should correctly process different header types in redirectMessageToOutput', async () => {
const TEST_REDIRECT = 'test_redirect' as const
const mockRedirectOutputs = {
produce: jest.fn().mockResolvedValue(undefined),
} as unknown as IngestionOutputs<typeof TEST_REDIRECT>
const mockPromiseScheduler = {
schedule: jest.fn().mockImplementation((promise) => promise),
} as unknown as PromiseScheduler
await redirectMessageToOutput(
mockRedirectOutputs,
TEST_REDIRECT,
mockPromiseScheduler,
mockMessage,
'test-step'
)
expect(mockRedirectOutputs.produce).toHaveBeenCalledWith(
TEST_REDIRECT,
expect.objectContaining({
headers: expect.objectContaining({
stringHeader: 'string-value',
bufferHeader: 'buffer-value',
numberHeader: '42',
nullHeader: 'null',
// undefinedHeader should not be present
}),
})
)
})
})