:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { KafkaProducer, MessageKey } from '~/common/kafka/producer'
import { logger } from '~/common/utils/logger'
import { IngestionOutput } from './ingestion-output'
import {
ingestionOutputsBatchSize,
ingestionOutputsErrors,
ingestionOutputsLatency,
ingestionOutputsMessageValueBytes,
} from './metrics'
import { IngestionOutputMessage } from './types'
/** Single-target output. */
export class SingleIngestionOutput implements IngestionOutput {
private readonly labels: { output: string; producer_name: string; topic: string }
constructor(
readonly outputName: string,
readonly topic: string,
readonly producer: KafkaProducer,
readonly producerName: string
) {
this.labels = { output: outputName, producer_name: producerName, topic }
}
async produce(message: IngestionOutputMessage & { key: MessageKey }): Promise<void> {
ingestionOutputsMessageValueBytes.observe(this.labels, message.value?.length ?? 0)
ingestionOutputsBatchSize.observe({ ...this.labels, method: 'produce' }, 1)
await withMetrics(this.labels, 'produce', () => this.producer.produce({ ...message, topic: this.topic }))
}
async queueMessages(messages: IngestionOutputMessage[]): Promise<void> {
for (const m of messages) {
ingestionOutputsMessageValueBytes.observe(this.labels, m.value?.length ?? 0)
}
ingestionOutputsBatchSize.observe({ ...this.labels, method: 'queueMessages' }, messages.length)
await withMetrics(this.labels, 'queueMessages', () =>
this.producer.queueMessages({ topic: this.topic, messages })
)
}
async checkHealth(timeoutMs: number): Promise<void> {
try {
await this.producer.checkConnection(timeoutMs)
} catch (error) {
logger.error('🔴', `Producer health check failed for "${this.producerName}" topic "${this.topic}"`, {
error,
})
throw error
}
}
async checkTopicExists(timeoutMs: number): Promise<void> {
if (!this.topic) {
return
}
try {
await this.producer.checkTopicExists(this.topic, timeoutMs)
} catch (error) {
logger.error('🔴', `Topic check failed for "${this.producerName}" topic "${this.topic}"`, { error })
throw error
}
}
}
async function withMetrics<T>(
labels: { output: string; producer_name: string; topic: string },
method: string,
fn: () => Promise<T>
): Promise<T> {
const metricLabels = { ...labels, method }
const end = ingestionOutputsLatency.startTimer(metricLabels)
try {
return await fn()
} catch (error) {
ingestionOutputsErrors.inc(metricLabels)
throw error
} finally {
end()
}
}