:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Assignment, Message } from 'node-rdkafka'
import { Gauge, Histogram } from 'prom-client'
import { CommonConfig } from '~/common/config'
import { GroupTypeManager } from '~/common/groups/group-type-manager'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { GroupRepository } from '~/common/groups/repositories/group-repository.interface'
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import {
AppMetricsOutput,
DlqOutput,
GroupsOutput,
IngestionWarningsOutput,
OverflowOutput,
TophogOutput,
} from '~/common/outputs'
import {
AsyncOutput,
EventOutput,
FlagEvaluationsOutput,
PersonDistinctIdsOutput,
PersonMergeEventsOutput,
PersonsOutput,
RealtimeOnlyEventsOutput,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PersonRepository } from '~/common/persons/repositories/person-repository'
import { instrumentFn } from '~/common/tracing/tracing-utils'
import { UsageIngestionConfig, createEventUsageBatchFactory } from '~/common/usage-ingestion'
import { PostgresRouter } from '~/common/utils/db/postgres'
import {
EventIngestionRestrictionManager,
EventIngestionRestrictionManagerComponent,
} from '~/common/utils/event-ingestion-restrictions'
import { EventSchemaEnforcementManager } from '~/common/utils/event-schema-enforcement-manager'
import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { BatchWritingGroupStore } from '~/ingestion/common/groups/batch-writing-group-store'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { effectivePersonMergeEventsEnabled } from '~/ingestion/common/persons/person-merge-event'
import { PersonsStore } from '~/ingestion/common/persons/persons-store'
import { createKafkaDebugContext, createOkContext } from '~/ingestion/framework/helpers'
import { TopHog } from '~/ingestion/framework/tophog'
import {
JoinedIngestionPipelineConfig,
JoinedIngestionPipelineContext,
JoinedIngestionPipelineDeps,
JoinedIngestionPipelineInput,
createJoinedIngestionPipeline,
} from '~/ingestion/pipelines/analytics'
import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk, PluginServerService, RedisPool } from '~/types'
import { EventFilterManager, EventFilterManagerComponent } from './common/event-filters'
import {
FeatureFlagCalledDedupService,
createFeatureFlagCalledDedupService,
} from './common/feature-flag-called-dedup/feature-flag-called-dedup-service'
import {
FlagEvaluationsService,
createFlagEvaluationsService,
} from './common/flag-evaluations/flag-evaluations-service'
import { MainLaneOverflowRedirect } from './common/overflow-redirect/main-lane-overflow-redirect'
import { OverflowLaneOverflowRedirect } from './common/overflow-redirect/overflow-lane-overflow-redirect'
import { OverflowRedirectService } from './common/overflow-redirect/overflow-redirect-service'
import { RedisOverflowRepository } from './common/overflow-redirect/overflow-redis-repository'
import { createAnalyticsOverflowStrategies } from './common/overflow-redirect/overflow-strategy'
import { IngestionConsumerConfig, IngestionOutputsConfig } from './config'
export type IngestionConsumerFullConfig = IngestionConsumerConfig &
UsageIngestionConfig &
Pick<CommonConfig, 'KAFKA_CLIENT_RACK'> &
// The general server builds the consumer from a config that includes IngestionOutputsConfig; the
// merge-events gate and the flag-evaluations fork read their topics, so surface them here rather
// than relying on the runtime shape.
Pick<
IngestionOutputsConfig,
| 'INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC'
| 'INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC'
| 'INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC'
>
export interface IngestionConsumerDeps {
postgres: PostgresRouter
redisPool: RedisPool
/** Dedicated pool for $feature_flag_called dedup claims; reuses redisPool when unset */
featureFlagCalledDedupRedisPool?: RedisPool
outputs: IngestionOutputs<
| EventOutput
| FlagEvaluationsOutput
| RealtimeOnlyEventsOutput
| IngestionWarningsOutput
| DlqOutput
| OverflowOutput
| AsyncOutput
| GroupsOutput
| PersonsOutput
| PersonDistinctIdsOutput
| PersonMergeEventsOutput
| AppMetricsOutput
| TophogOutput
>
teamManager: TeamManager
groupTypeManager: GroupTypeManager
groupRepository: GroupRepository
clickhouseGroupRepository: ClickhouseGroupRepository
personRepository: PersonRepository
cookielessManager: CookielessManager
hogTransformer: HogTransformer
}
export const latestOffsetTimestampGauge = new Gauge({
name: 'latest_processed_timestamp_ms',
help: 'Timestamp of the latest offset that has been committed.',
labelNames: ['topic', 'partition', 'groupId'],
aggregator: 'max',
})
const backgroundTaskProducesDuration = new Histogram({
name: 'ingestion_background_task_produces_duration_seconds',
help: 'Time waiting for scheduled Kafka produces in the background task',
labelNames: ['groupId'],
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
})
export class IngestionConsumer {
protected name = 'ingestion-consumer'
protected groupId: string
protected topic: string
protected kafkaConsumer: KafkaConsumerInterface
isStopping = false
public hogTransformer: HogTransformer
private overflowRedirectService?: OverflowRedirectService
private overflowLaneTTLRefreshService?: OverflowRedirectService
private featureFlagCalledDedupService?: FeatureFlagCalledDedupService
private flagEvaluationsService?: FlagEvaluationsService
private tokenDistinctIdsToDrop: string[] = []
private tokenDistinctIdsToSkipPersons: string[] = []
private tokenDistinctIdsToForceOverflow: string[] = []
private personsStore: PersonsStore
public groupStore: BatchWritingGroupStore
private eventFilterManagerComponent: EventFilterManagerComponent
private eventFilterManager!: EventFilterManager
private stopEventFilterManager?: () => Promise<void>
private eventIngestionRestrictionManagerComponent: EventIngestionRestrictionManagerComponent
private eventIngestionRestrictionManager!: EventIngestionRestrictionManager
private stopEventIngestionRestrictionManager?: () => Promise<void>
private eventSchemaEnforcementManager: EventSchemaEnforcementManager
public readonly promiseScheduler = new PromiseScheduler()
private topHog!: TopHog
private joinedPipeline!: ReturnType<
typeof createJoinedIngestionPipeline<JoinedIngestionPipelineInput, JoinedIngestionPipelineContext>
>
constructor(
private config: IngestionConsumerFullConfig,
private deps: IngestionConsumerDeps,
overrides: Partial<
Pick<
IngestionConsumerConfig,
| 'INGESTION_CONSUMER_GROUP_ID'
| 'INGESTION_CONSUMER_CONSUME_TOPIC'
| 'INGESTION_CONSUMER_OVERFLOW_TOPIC'
| 'INGESTION_CONSUMER_DLQ_TOPIC'
>
> = {}
) {
// The group and topic are configurable allowing for multiple ingestion consumers to be run in parallel
this.groupId = overrides.INGESTION_CONSUMER_GROUP_ID ?? config.INGESTION_CONSUMER_GROUP_ID
this.topic = overrides.INGESTION_CONSUMER_CONSUME_TOPIC ?? config.INGESTION_CONSUMER_CONSUME_TOPIC
this.tokenDistinctIdsToDrop = config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID.split(',').filter((x) => !!x)
this.tokenDistinctIdsToSkipPersons = config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID.split(',').filter(
(x) => !!x
)
this.tokenDistinctIdsToForceOverflow = config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID.split(',').filter(
(x) => !!x
)
this.eventIngestionRestrictionManagerComponent = new EventIngestionRestrictionManagerComponent(deps.redisPool, {
pipeline: 'analytics',
staticDropEventTokens: this.tokenDistinctIdsToDrop,
staticSkipPersonTokens: this.tokenDistinctIdsToSkipPersons,
staticForceOverflowTokens: this.tokenDistinctIdsToForceOverflow,
})
this.eventFilterManagerComponent = new EventFilterManagerComponent(deps.postgres, { pipeline: 'analytics' })
// Schema loads run detached in the LazyLoader buffer, so an un-retried transient
// failure can surface as an unhandled rejection and restart the worker.
this.eventSchemaEnforcementManager = new EventSchemaEnforcementManager(deps.postgres, {
loaderRetry: DEFAULT_LOADER_RETRY,
})
this.name = `ingestion-consumer-${this.topic}`
// Create shared Redis repository for overflow redirect services
const overflowRedisRepository = new RedisOverflowRepository({
redisPool: this.deps.redisPool,
redisTTLSeconds: this.config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS,
})
// Overflow role for this consumer (redirect / consume / disabled).
const overflowMode = this.config.INGESTION_OVERFLOW_MODE
// Redirect hot partitions to the overflow topic (main lane).
if (overflowMode === 'redirect') {
this.overflowRedirectService = new MainLaneOverflowRedirect({
redisRepository: overflowRedisRepository,
localCacheTTLSeconds: this.config.INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS,
strategies: createAnalyticsOverflowStrategies({
eventBucketCapacity: this.config.EVENT_OVERFLOW_BUCKET_CAPACITY,
eventReplenishRate: this.config.EVENT_OVERFLOW_BUCKET_REPLENISH_RATE,
mergeEventBucketCapacity: this.config.MERGE_EVENT_OVERFLOW_BUCKET_CAPACITY,
mergeEventReplenishRate: this.config.MERGE_EVENT_OVERFLOW_BUCKET_REPLENISH_RATE,
}),
overflowType: 'events',
})
}
// Drain the overflow topic and refresh stateful TTLs (overflow lane).
if (overflowMode === 'consume') {
this.overflowLaneTTLRefreshService = new OverflowLaneOverflowRedirect({
redisRepository: overflowRedisRepository,
overflowType: 'events',
})
}
this.featureFlagCalledDedupService = createFeatureFlagCalledDedupService(
this.deps.featureFlagCalledDedupRedisPool ?? this.deps.redisPool,
this.config
)
this.flagEvaluationsService = createFlagEvaluationsService(this.config)
this.hogTransformer = deps.hogTransformer
this.personsStore = new BatchWritingPersonsStore(this.deps.personRepository, this.deps.outputs, {
dbWriteMode: this.config.PERSON_BATCH_WRITING_DB_WRITE_MODE,
useBatchUpdates: this.config.PERSON_BATCH_WRITING_USE_BATCH_UPDATES,
maxConcurrentUpdates: this.config.PERSON_BATCH_WRITING_MAX_CONCURRENT_UPDATES,
maxOptimisticUpdateRetries: this.config.PERSON_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES,
optimisticUpdateRetryInterval: this.config.PERSON_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS,
updateAllProperties: this.config.PERSON_PROPERTIES_UPDATE_ALL,
mergeLockedOutcomeTeamAllowlist: this.config.PERSON_MERGE_LOCKED_OUTCOME_TEAM_ALLOWLIST,
mergeEventsEnabled: effectivePersonMergeEventsEnabled(this.config),
mergeEventsPartitionCount: this.config.PERSON_MERGE_EVENTS_PARTITION_COUNT,
mergeEventsTeamAllowlist: this.config.PERSON_MERGE_EVENTS_TEAM_ALLOWLIST,
mergeNoopMappingEmissionEnabled: this.config.PERSON_MERGE_NOOP_MAPPING_EMISSION_ENABLED,
mergeNoopMappingEmissionCacheSize: this.config.PERSON_MERGE_NOOP_MAPPING_EMISSION_CACHE_SIZE,
mergeNoopMappingEmissionTtlMs: this.config.PERSON_MERGE_NOOP_MAPPING_EMISSION_TTL_MS,
})
this.groupStore = new BatchWritingGroupStore(this.deps.groupRepository, this.deps.clickhouseGroupRepository, {
useBatchUpdates: this.config.GROUP_BATCH_WRITING_USE_BATCH_UPDATES,
useBatchCreates: this.config.GROUP_BATCH_WRITING_USE_BATCH_CREATES,
maxConcurrentUpdates: this.config.GROUP_BATCH_WRITING_MAX_CONCURRENT_UPDATES,
maxOptimisticUpdateRetries: this.config.GROUP_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES,
optimisticUpdateRetryInterval: this.config.GROUP_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS,
})
this.kafkaConsumer = createKafkaConsumer({
groupId: this.groupId,
topic: this.topic,
})
this.topHog = new TopHog({
outputs: this.deps.outputs,
pipeline: this.config.INGESTION_PIPELINE ?? 'unknown',
lane: this.config.INGESTION_LANE ?? 'unknown',
})
}
public get service(): PluginServerService {
return {
id: this.name,
onShutdown: async () => await this.stop(),
healthcheck: () => this.isHealthy() ?? false,
}
}
public async start(): Promise<void> {
const startedRestrictions = await this.eventIngestionRestrictionManagerComponent.start()
this.eventIngestionRestrictionManager = startedRestrictions.value
this.stopEventIngestionRestrictionManager = startedRestrictions.stop
const startedFilters = await this.eventFilterManagerComponent.start()
this.eventFilterManager = startedFilters.value
this.stopEventFilterManager = startedFilters.stop
await this.hogTransformer.start()
this.topHog.start()
const outputs = this.deps.outputs
// Verify all output topics exist. When auto_create_topics_enabled=true
// (hobby/dev), this ensures topics are created before first produce.
// When auto-create is off (production), this catches misconfigurations early.
const topicFailures = await outputs.checkTopics()
if (topicFailures.length > 0) {
throw new Error(`Output topic verification failed for: ${topicFailures.join(', ')}`)
}
const joinedPipelineConfig: JoinedIngestionPipelineConfig = {
eventSchemaEnforcementEnabled: this.config.EVENT_SCHEMA_ENFORCEMENT_ENABLED,
overflowMode: this.config.INGESTION_OVERFLOW_MODE,
preservePartitionLocality: this.config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY,
personsPrefetchEnabled: this.config.PERSONS_PREFETCH_ENABLED,
groupsPrefetchEnabled: this.config.GROUPS_PREFETCH_ENABLED,
teamsPrefetchEnabled: this.config.TEAMS_PREFETCH_ENABLED,
eventSchemasPrefetchEnabled: this.config.EVENT_SCHEMAS_PREFETCH_ENABLED,
hogFunctionsPrefetchEnabled: this.config.HOG_FUNCTIONS_PREFETCH_ENABLED,
outputs,
perDistinctIdOptions: {
SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP: this.config.SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP,
PERSON_MERGE_MOVE_DISTINCT_ID_LIMIT: this.config.PERSON_MERGE_MOVE_DISTINCT_ID_LIMIT,
PERSON_MERGE_ASYNC_ENABLED: this.config.PERSON_MERGE_ASYNC_ENABLED,
PERSON_MERGE_SYNC_BATCH_SIZE: this.config.PERSON_MERGE_SYNC_BATCH_SIZE,
PERSON_MERGE_FOLD_ENABLED: this.config.PERSON_MERGE_FOLD_ENABLED,
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: this.config.PERSON_MERGE_FOLD_TEAM_ALLOWLIST,
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: this.config.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,
PERSON_MERGE_NOT_APPLIED_WAIT_MS: this.config.PERSON_MERGE_NOT_APPLIED_WAIT_MS,
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: this.config.PERSON_JSONB_SIZE_ESTIMATE_ENABLE,
PERSON_PROPERTIES_UPDATE_ALL: this.config.PERSON_PROPERTIES_UPDATE_ALL,
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: this.config.FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,
FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS: this.config.FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS,
EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS: this.config.EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS,
},
concurrentBatches: this.config.INGESTION_WORKER_CONCURRENT_BATCHES,
createEventUsageBatch: createEventUsageBatchFactory(this.config, 'events'),
}
const joinedPipelineDeps: JoinedIngestionPipelineDeps = {
personsStore: this.personsStore,
groupStore: this.groupStore,
hogTransformer: this.hogTransformer,
eventFilterManager: this.eventFilterManager,
eventIngestionRestrictionManager: this.eventIngestionRestrictionManager,
eventSchemaEnforcementManager: this.eventSchemaEnforcementManager,
promiseScheduler: this.promiseScheduler,
overflowRedirectService: this.overflowRedirectService,
overflowLaneTTLRefreshService: this.overflowLaneTTLRefreshService,
featureFlagCalledDedupService: this.featureFlagCalledDedupService,
flagEvaluationsService: this.flagEvaluationsService,
teamManager: this.deps.teamManager,
cookielessManager: this.deps.cookielessManager,
groupTypeManager: this.deps.groupTypeManager,
topHog: this.topHog!,
}
this.joinedPipeline = createJoinedIngestionPipeline(joinedPipelineConfig, joinedPipelineDeps)
await this.kafkaConsumer.connect(
async (messages) => {
return await instrumentFn(
{
key: `ingestionConsumer.handleEachBatch`,
sendException: false,
},
async () => await this.handleKafkaBatch(messages)
)
},
(revokedPartitions) => this.onPartitionsRevoked(revokedPartitions)
)
}
public async stop(): Promise<void> {
if (this.isStopping) {
return
}
logger.info('🔁', `${this.name} - stopping`)
this.isStopping = true
// Mark as stopping so that we don't actually process any more incoming messages, but still keep the process alive
logger.info('🔁', `${this.name} - stopping batch consumer`)
await this.kafkaConsumer?.disconnect()
logger.info('🔁', `${this.name} - stopping tophog`)
await this.topHog.stop()
logger.info('🔁', `${this.name} - stopping hog transformer`)
await this.hogTransformer.stop()
await this.stopEventFilterManager?.()
await this.stopEventIngestionRestrictionManager?.()
// Stores must be clean by now — flushBatchStoresStep runs after every
// batch as part of the pipeline. After disconnect, we cannot commit
// offsets, so writing dirty data here would produce duplicates on
// partition rebalance. shutdown() will throw if anything is dirty,
// which surfaces the drain-ordering bug without masking it.
try {
await this.personsStore.shutdown()
} catch (error) {
logger.error('🚨', `${this.name} - personsStore.shutdown() failed`, { error })
}
try {
await this.groupStore.shutdown()
} catch (error) {
logger.error('🚨', `${this.name} - groupStore.shutdown() failed`, { error })
}
logger.info('👍', `${this.name} - stopped!`)
}
public async isHealthy(): Promise<HealthCheckResult> {
if (!this.kafkaConsumer) {
return new HealthCheckResultError('Kafka consumer not initialized', {})
}
const consumerHealth = this.kafkaConsumer.isHealthy()
if (consumerHealth.isError()) {
return consumerHealth
}
if (process.env.INGESTION_OUTPUTS_PRODUCER_HEALTHCHECK === 'true') {
const failures = await this.deps.outputs.checkHealth()
if (failures.length > 0) {
return new HealthCheckResultError('Kafka producer(s) unhealthy', { failedProducers: failures })
}
}
return new HealthCheckResultOk()
}
private runInstrumented<T>(name: string, func: () => Promise<T>): Promise<T> {
return instrumentFn<T>(`ingestionConsumer.${name}`, func)
}
private logBatchStart(messages: Message[]): void {
// Log earliest message from each partition to detect duplicate processing across pods
const podName = process.env.HOSTNAME || 'unknown'
const partitionEarliestMessages = new Map<number, Message>()
const partitionBatchSizes = new Map<number, number>()
messages.forEach((message) => {
const existing = partitionEarliestMessages.get(message.partition)
if (!existing || message.offset < existing.offset) {
partitionEarliestMessages.set(message.partition, message)
}
partitionBatchSizes.set(message.partition, (partitionBatchSizes.get(message.partition) || 0) + 1)
})
// Create partition data array for single log entry
const partitionData = Array.from(partitionEarliestMessages.entries()).map(([partition, message]) => ({
partition,
offset: message.offset,
batchSize: partitionBatchSizes.get(partition) || 0,
}))
logger.info('📖', `KAFKA_BATCH_START: ${this.name}`, {
pod: podName,
totalMessages: messages.length,
partitions: partitionData,
})
}
public async handleKafkaBatch(messages: Message[]): Promise<{ backgroundTask?: Promise<any> }> {
if (this.config.KAFKA_BATCH_START_LOGGING_ENABLED) {
this.logBatchStart(messages)
}
await this.runIngestionPipeline(messages)
for (const message of messages) {
if (message.timestamp) {
latestOffsetTimestampGauge
.labels({ partition: message.partition, topic: message.topic, groupId: this.groupId })
.set(message.timestamp)
}
}
return {
backgroundTask: this.runInstrumented('awaitScheduledWork', async () => {
const labels = { groupId: this.groupId }
// Drains scheduled produces and the hog transformer invocation results, which
// the pipeline's afterBatch flush step schedules as a side effect.
await timedHistogram(backgroundTaskProducesDuration, labels, () => this.promiseScheduler.waitForAll())
}),
}
}
private onPartitionsRevoked(revokedPartitions: Assignment[]): Promise<void> {
for (const { topic, partition } of revokedPartitions) {
latestOffsetTimestampGauge.remove({ topic, partition, groupId: this.groupId })
}
return Promise.resolve()
}
private async runIngestionPipeline(messages: Message[]): Promise<void> {
const batch = messages.map((message) =>
createOkContext({ message }, { message, debugContext: createKafkaDebugContext(message) })
)
const feedResult = await this.joinedPipeline.feed(batch, {})
if (!feedResult.ok) {
throw new Error(`Pipeline rejected batch: ${feedResult.reason}`)
}
// The pipeline handles its own side effects (scheduling them on the
// promise scheduler), so draining results is all that's left to do.
let result = await this.joinedPipeline.next()
while (result !== null) {
result = await this.joinedPipeline.next()
}
}
}
async function timedHistogram<T>(
histogram: Histogram,
labels: Record<string, string>,
fn: () => Promise<T>
): Promise<T> {
const end = histogram.startTimer(labels)
try {
return await fn()
} finally {
end()
}
}