:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { Gauge } from 'prom-client'
import { TophogOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { instrumentFn } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { IngestionConsumerConfig } from '~/ingestion/config'
import { BatchResult, FeedResult } from '~/ingestion/framework/batching-pipeline'
import { createKafkaDebugContext, createOkContext } from '~/ingestion/framework/helpers'
import { OkResultWithContext } from '~/ingestion/framework/pipeline.interface'
import { TopHog, TopHogComponent } from '~/ingestion/framework/tophog'
import { HealthCheckResult, PluginServerService } from '~/types'
import { Scope, extend } from './scopes'
import { KafkaConsumerComponent, KafkaConsumerInterface } from './utils/kafka-consumer'
import { PromiseScheduler } from './utils/promise-scheduler'
type MessageInput = { message: Message }
type MessageContext = { message: Message }
export interface IngestionBatchingPipeline {
feed(
elements: OkResultWithContext<MessageInput, MessageContext>[],
batchContext: Record<never, never>
): Promise<FeedResult>
next(): Promise<BatchResult<unknown> | null>
}
export interface PipelineFactoryContext<S extends Record<string, object>> {
container: S
}
export type PipelineFactory<S extends Record<string, object>> = (
ctx: PipelineFactoryContext<S>
) => IngestionBatchingPipeline
/**
* Constraint on a scope's container: must expose a `promiseScheduler`.
* The pipeline schedules its side effects there; the common consumer
* pulls it from the container to await them after each batch. The
* scheduler is owned by the user-provided scope so its lifetime spans
* the consumer's lifetime.
*/
export type ContainerWithPromiseScheduler = Record<string, object> & { promiseScheduler: PromiseScheduler }
export type CommonIngestionConsumerConfig = Pick<
IngestionConsumerConfig,
| 'INGESTION_CONSUMER_GROUP_ID'
| 'INGESTION_CONSUMER_CONSUME_TOPIC'
| 'INGESTION_PIPELINE'
| 'INGESTION_LANE'
| 'KAFKA_BATCH_START_LOGGING_ENABLED'
>
const latestOffsetTimestampGauge = new Gauge({
name: 'common_ingestion_latest_processed_timestamp_ms',
help: 'Timestamp of the latest offset that has been committed.',
labelNames: ['topic', 'partition', 'groupId'],
aggregator: 'max',
})
/**
* Setup-side of a common ingestion consumer. Constructing it extends the
* user-provided scope with a `kafkaConsumer` entry; `start()` brings up
* the parent scope, builds the pipeline (sync), connects the Kafka
* consumer, and returns a live `CommonIngestionConsumer` that owns the
* started handle.
*/
export class CommonIngestionConsumerScope<
S extends ContainerWithPromiseScheduler & { outputs: IngestionOutputs<TophogOutput> },
> {
private readonly innerScope: Scope<S & { topHog: TopHog } & { kafkaConsumer: KafkaConsumerInterface }>
constructor(
private readonly name: string,
config: CommonIngestionConsumerConfig,
scope: Scope<S>,
pipelineFactory: PipelineFactory<S & { topHog: TopHog }>
) {
const consumerName = this.name
// Every consumer gets a topHog registry scoped to its own outputs and
// pipeline/lane labels. It lives one scope above the consumer so its
// started value is in the container when the pipeline factory runs —
// and so its flush loop starts before, and stops after, the consumer.
const scopeWithTopHog = extend(scope, `${consumerName}-tophog`, (container, builder) =>
builder.add(
'topHog',
new TopHogComponent({
outputs: container.outputs,
pipeline: config.INGESTION_PIPELINE ?? 'unknown',
lane: config.INGESTION_LANE ?? 'unknown',
})
)
)
this.innerScope = extend(scopeWithTopHog, `${consumerName}-consumer`, (container, builder) => {
const pipeline = pipelineFactory({ container })
const handler = new KafkaBatchHandler(config, consumerName, pipeline, container.promiseScheduler)
return builder.add(
'kafkaConsumer',
new KafkaConsumerComponent(
config.INGESTION_CONSUMER_GROUP_ID,
config.INGESTION_CONSUMER_CONSUME_TOPIC,
(messages) => handler.handle(messages)
)
)
})
}
async start(): Promise<{ consumer: CommonIngestionConsumer; stop: () => Promise<void> }> {
const started = await this.innerScope.start()
const consumer = new CommonIngestionConsumer(this.name, started.container.kafkaConsumer)
return { consumer, stop: started.stop }
}
}
/**
* Live, started common ingestion consumer. Exposes only what surrounding
* code actually needs from the started scope: the Kafka consumer (for
* health) and the per-process healthcheck. Lifetime is owned by the
* scope handle returned alongside it from `CommonIngestionConsumerScope.start()`.
*/
export class CommonIngestionConsumer {
constructor(
readonly name: string,
private readonly kafkaConsumer: KafkaConsumerInterface
) {}
isHealthy(): HealthCheckResult {
return this.kafkaConsumer.isHealthy()
}
}
/**
* Builds a `PluginServerService` descriptor for a started common
* ingestion consumer. The scope handle's `stop` is supplied explicitly
* so lifecycle ownership stays with the scope, not the consumer.
*/
export function ingestionConsumerService(
consumer: CommonIngestionConsumer,
stop: () => Promise<void>
): PluginServerService {
return {
id: consumer.name,
onShutdown: stop,
healthcheck: () => consumer.isHealthy(),
}
}
/**
* Per-consumer wrapper that owns the deps a Kafka batch handler needs
* (config, name, pipeline, promise scheduler) and exposes a single
* `handle(messages)` method matching what `KafkaConsumerComponent` expects.
* Internal — only constructed inside `CommonIngestionConsumerScope`.
*/
class KafkaBatchHandler {
constructor(
private readonly config: CommonIngestionConsumerConfig,
private readonly name: string,
private readonly pipeline: IngestionBatchingPipeline,
private readonly promiseScheduler: PromiseScheduler
) {}
async handle(messages: Message[]): Promise<{ backgroundTask?: Promise<unknown> }> {
if (this.config.KAFKA_BATCH_START_LOGGING_ENABLED) {
this.logBatchStart(messages)
}
const batch = messages.map((message) =>
createOkContext({ message }, { message, debugContext: createKafkaDebugContext(message) })
)
const feedResult = await this.pipeline.feed(batch, {})
if (!feedResult.ok) {
throw new Error(`Pipeline rejected batch: ${feedResult.reason}`)
}
// The pipeline handles its own side effects (scheduling them on the
// promise scheduler), so draining results is all that's left to do.
let result = await this.pipeline.next()
while (result !== null) {
result = await this.pipeline.next()
}
for (const message of messages) {
if (message.timestamp) {
latestOffsetTimestampGauge
.labels({
partition: message.partition,
topic: message.topic,
groupId: this.config.INGESTION_CONSUMER_GROUP_ID,
})
.set(message.timestamp)
}
}
return {
backgroundTask: instrumentFn('commonIngestionConsumer.awaitScheduledWork', async () => {
await this.promiseScheduler.waitForAll()
}),
}
}
private logBatchStart(messages: Message[]): void {
const podName = process.env.HOSTNAME || 'unknown'
const partitionEarliestMessages = new Map<number, Message>()
const partitionBatchSizes = new Map<number, number>()
messages.forEach((message) => {
const existing = partitionEarliestMessages.get(message.partition)
if (!existing || message.offset < existing.offset) {
partitionEarliestMessages.set(message.partition, message)
}
partitionBatchSizes.set(message.partition, (partitionBatchSizes.get(message.partition) || 0) + 1)
})
const partitionData = Array.from(partitionEarliestMessages.entries()).map(([partition, message]) => ({
partition,
offset: message.offset,
batchSize: partitionBatchSizes.get(partition) || 0,
}))
logger.info('📖', `KAFKA_BATCH_START: ${this.name}`, {
pod: podName,
totalMessages: messages.length,
partitions: partitionData,
})
}
}