:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { DLQ_OUTPUT, DlqOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
/**
* Helper function to copy and extend headers from a Kafka message
* Converts various header value types to strings and adds new headers
*/
function copyAndExtendHeaders(
originalMessage: Message,
additionalHeaders: Record<string, string>
): Record<string, string> {
const originalHeaders = originalMessage.headers || []
const stringHeaders: Record<string, string> = {}
// Kafka headers are always in array format (MessageHeader[])
for (const headerObj of originalHeaders) {
for (const [key, value] of Object.entries(headerObj)) {
if (value === undefined) {
// Skip undefined values
continue
} else if (typeof value === 'string') {
stringHeaders[key] = value
} else if (Buffer.isBuffer(value)) {
stringHeaders[key] = value.toString()
} else {
// Convert all other values (including null) to strings
stringHeaders[key] = String(value)
}
}
}
return {
...stringHeaders,
...additionalHeaders,
}
}
/**
* Extract event metadata from a Kafka message for logging purposes
* Only extracts from headers as strings, never parses the payload
*/
function getEventMetadata(message: Message): { teamId?: string; distinctId?: string; event?: string; uuid?: string } {
const originalHeaders = message.headers || []
const headers: Record<string, any> = {}
// Kafka headers are always in array format (MessageHeader[])
for (const headerObj of originalHeaders) {
for (const [key, value] of Object.entries(headerObj)) {
headers[key] = value
}
}
const teamId = headers['team_id'] ? String(headers['team_id']) : undefined
const distinctId = headers['distinct_id'] ? String(headers['distinct_id']) : undefined
const event = headers['event'] ? String(headers['event']) : undefined
const uuid = headers['uuid'] ? String(headers['uuid']) : undefined
return {
teamId,
distinctId,
event,
uuid,
}
}
export interface ProduceToDLQOptions {
/** Rethrow a failed DLQ produce. Set it only when the caller stores no offsets after the error. */
rethrowOnFailure?: boolean
}
/**
* Send a Kafka message to the dead letter queue with proper logging and metrics.
*/
export async function produceMessageToDLQ(
outputs: IngestionOutputs<DlqOutput>,
originalMessage: Message,
error: unknown,
stepName: string,
options: ProduceToDLQOptions = {}
): Promise<void> {
const step = stepName
const messageInfo = getEventMetadata(originalMessage)
logger.warn('Event sent to DLQ', {
step,
team_id: messageInfo.teamId,
distinct_id: messageInfo.distinctId,
event: messageInfo.event,
uuid: messageInfo.uuid,
error: error instanceof Error ? error.message : String(error),
})
try {
await outputs.produce(DLQ_OUTPUT, {
value: originalMessage.value,
key: originalMessage.key ?? null,
headers: copyAndExtendHeaders(originalMessage, {
dlq_reason: error instanceof Error ? error.message : String(error),
dlq_step: step,
dlq_timestamp: new Date().toISOString(),
dlq_topic: originalMessage.topic,
dlq_partition: String(originalMessage.partition),
dlq_offset: String(originalMessage.offset),
}),
})
} catch (dlqError) {
logger.error('Failed to send event to DLQ', {
step,
team_id: messageInfo.teamId,
distinct_id: messageInfo.distinctId,
event: messageInfo.event,
uuid: messageInfo.uuid,
error: dlqError,
})
captureException(dlqError, {
tags: { team_id: messageInfo.teamId, pipeline_step: step },
extra: { originalMessage, error: dlqError },
})
if (options.rethrowOnFailure) {
throw dlqError
}
}
}
/**
* Redirect a Kafka message to a specified Kafka topic
*/
export async function redirectMessageToOutput<O extends string>(
outputs: IngestionOutputs<O>,
output: O,
promiseScheduler: PromiseScheduler,
originalMessage: Message,
stepName?: string,
preserveKey: boolean = true,
awaitAck: boolean = true
): Promise<void> {
const step = stepName || 'unknown'
try {
const extraHeaders: Record<string, string> = {
'redirect-step': step,
'redirect-timestamp': new Date().toISOString(),
}
// Dropping the key spreads the redirected stream across partitions;
// keep the original key recoverable for consumers of the stream.
if (!preserveKey && originalMessage.key) {
extraHeaders['redirect-original-key'] = originalMessage.key.toString()
}
const headers = copyAndExtendHeaders(originalMessage, extraHeaders)
const key = preserveKey ? (originalMessage.key ?? null) : null
const producePromise = outputs.produce(output, {
value: originalMessage.value,
key: key,
headers: headers,
})
const promise = promiseScheduler.schedule(producePromise)
if (awaitAck) {
await promise
}
} catch (redirectError) {
const eventMetadata = getEventMetadata(originalMessage)
captureException(redirectError, {
tags: {
team_id: eventMetadata.teamId,
pipeline_step: step,
},
extra: {
output,
distinct_id: eventMetadata.distinctId,
event: eventMetadata.event,
error: redirectError,
},
})
throw redirectError
}
}
/**
* Log a dropped event from a Kafka message with proper metrics
*/
export function logDroppedMessage(originalMessage: Message, reason: string, stepName?: string): void {
const step = stepName || 'unknown'
const messageInfo = getEventMetadata(originalMessage)
logger.debug('Event dropped', {
step,
team_id: messageInfo.teamId,
distinct_id: messageInfo.distinctId,
event: messageInfo.event,
reason,
})
}