:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { IntegrationManagerService } from '~/cdp/services/managers/integration-manager.service'
import { initializePrometheusLabels } from '~/common/api/router'
import { defaultConfig, overrideConfigWithEnv } from '~/common/config/config'
import {
KAFKA_EVENTS_PLUGIN_INGESTION,
KAFKA_EVENTS_PLUGIN_INGESTION_AI,
KAFKA_EVENTS_PLUGIN_INGESTION_HISTORICAL,
KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW,
} from '~/common/config/kafka-topics'
import {
createCookielessRedisConnectionConfig,
createFeatureFlagCalledDedupRedisConnectionConfig,
createIngestionRedisConnectionConfig,
} from '~/common/config/redis-pools'
import { GroupTypeManager } from '~/common/groups/group-type-manager'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { PostgresGroupRepository } from '~/common/groups/repositories/postgres-group-repository'
import { HogTransformerComponent } from '~/common/hog-transformations/hog-transformer-component'
import { IngestionOutputsComponent } from '~/common/outputs/ingestion-outputs'
import { PersonHogConfig, buildGroupRepository, buildPersonRepository, createPersonHogClient } from '~/common/personhog'
import { PostgresPersonRepository } from '~/common/persons/repositories/postgres-person-repository'
import { UsageIngestionConfig } from '~/common/usage-ingestion'
import { ServerCommands } from '~/common/utils/commands'
import { PostgresRouter, PostgresRouterComponent } from '~/common/utils/db/postgres'
import { RedisPoolComponent } from '~/common/utils/db/redis'
import { GeoIPService } from '~/common/utils/geoip'
import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { PubSub } from '~/common/utils/pubsub'
import { TeamManagerComponent } from '~/common/utils/team-manager'
import { CookielessManagerComponent } from '~/ingestion/common/cookieless/cookieless-manager'
import { KafkaProducerRegistryComponent } from '~/ingestion/common/outputs/producer-registry'
import {
KafkaDownstreamProducerEnvConfig,
KafkaUpstreamProducerEnvConfig,
getDefaultKafkaDownstreamProducerEnvConfig,
getDefaultKafkaUpstreamProducerEnvConfig,
} from '~/ingestion/common/outputs/producers'
import { createAiConsumer } from '~/ingestion/pipelines/ai'
import { createOutputsRegistry as createAiOutputsRegistry } from '~/ingestion/pipelines/ai/outputs/registry'
import { createOutputsRegistry } from '~/ingestion/pipelines/analytics/outputs/registry'
import { createClientWarningsConsumer } from '~/ingestion/pipelines/clientwarnings'
import { createHeatmapsConsumer } from '~/ingestion/pipelines/heatmaps'
import {
HogTransformerServiceConfig,
HogTransformerServiceDeps,
createHogTransformerService,
} from '../cdp/hog-transformations/hog-transformer.service'
import { EncryptedFields } from '../cdp/utils/encryption-utils'
import { CommonConfig, PluginServerMode } from '../common/config'
import { ingestionConsumerService } from '../ingestion/common/ingestion-consumer'
import { extend, newScope } from '../ingestion/common/scopes'
import {
DatabaseConnectionConfig,
IngestionConsumerConfig,
IngestionOutputsConfig,
KafkaBrokerConfig,
KafkaConsumerBaseConfig,
RedisConnectionsConfig,
getDefaultIngestionConsumerConfig,
getDefaultIngestionOutputsConfig,
} from '../ingestion/config'
import { IngestionConsumer, IngestionConsumerDeps } from '../ingestion/ingestion-consumer'
import { PluginServerService, RedisPool } from '../types'
import { BaseServerConfig, CleanupResources, NodeServer, ServerLifecycle } from './base-server'
/**
* Complete config type for an ingestion-v2 deployment.
*
* This is the union of:
* - BaseServerConfig: HTTP server, profiling, pod termination lifecycle
* - IngestionConsumerConfig: ingestion pipeline, person/group processing, overflow, cookieless, etc.
* - HogTransformerServiceConfig: the transformation-only keys the in-process hog transformer reads.
* No CDP delivery config (Redis, watcher, SES, fetch) - transformations run the synchronous
* Hog core alone, so those keys are deliberately absent rather than inherited.
* - Infrastructure configs: Kafka broker, Postgres, Redis, consumer tuning
* - Remaining CommonConfig picks: server mode, services, observability
*
* This type is the source of truth for which env vars ingestion-events-* deployments need.
*/
export type IngestionGeneralServerConfig = BaseServerConfig &
IngestionConsumerConfig &
HogTransformerServiceConfig &
KafkaBrokerConfig &
KafkaUpstreamProducerEnvConfig &
KafkaDownstreamProducerEnvConfig &
IngestionOutputsConfig &
DatabaseConnectionConfig &
RedisConnectionsConfig &
KafkaConsumerBaseConfig &
PersonHogConfig &
UsageIngestionConfig &
Pick<
CommonConfig,
| 'LOG_LEVEL'
| 'PLUGIN_SERVER_MODE'
| 'CLOUD_DEPLOYMENT'
| 'ENCRYPTION_SALT_KEYS'
| 'MMDB_FILE_LOCATION'
| 'CAPTURE_INTERNAL_URL'
| 'LAZY_LOADER_DEFAULT_BUFFER_MS'
| 'LAZY_LOADER_MAX_SIZE'
| 'TASK_TIMEOUT'
| 'POSTHOG_API_KEY'
| 'POSTHOG_HOST_URL'
| 'HEALTHCHECK_MAX_STALE_SECONDS'
| 'KAFKA_HEALTHCHECK_SECONDS'
>
export class IngestionGeneralServer implements NodeServer {
readonly lifecycle: ServerLifecycle
private config: IngestionGeneralServerConfig
private postgres?: PostgresRouter
private redisPool?: RedisPool
private pubsub?: PubSub
private stopSharedServices?: () => Promise<void>
constructor(config: Partial<IngestionGeneralServerConfig> = {}) {
this.config = {
...defaultConfig,
...overrideConfigWithEnv(getDefaultIngestionConsumerConfig()),
...overrideConfigWithEnv(getDefaultKafkaUpstreamProducerEnvConfig()),
...overrideConfigWithEnv(getDefaultKafkaDownstreamProducerEnvConfig()),
...overrideConfigWithEnv(getDefaultIngestionOutputsConfig()),
...config,
}
this.lifecycle = new ServerLifecycle(this.config)
}
async start(): Promise<void> {
return this.lifecycle.start(
() => this.startServices(),
() => this.getCleanupResources()
)
}
async stop(error?: Error): Promise<void> {
return this.lifecycle.stop(() => this.getCleanupResources(), error)
}
private async startServices(): Promise<void> {
initializePrometheusLabels(this.config.INGESTION_PIPELINE, this.config.INGESTION_LANE)
// 1. Shared infrastructure — postgres + redis lifetimes are owned
// by a server-level Scope so consumers can extend off it via
// `Scope.extend` to get them as handles without taking
// ownership.
logger.info('ℹ️', 'Connecting to shared infrastructure...')
const sharedInfraScope = newScope('shared-infra', (builder) =>
builder
.add('postgres', new PostgresRouterComponent(this.config, this.config.PLUGIN_SERVER_MODE!))
.add(
'redisPool',
new RedisPoolComponent({
connection: createIngestionRedisConnectionConfig(this.config),
poolMinSize: this.config.REDIS_POOL_MIN_SIZE,
poolMaxSize: this.config.REDIS_POOL_MAX_SIZE,
})
)
// Cookieless Redis is a separate pool, shared by every consumer that runs cookieless
// processing (analytics, heatmaps, …).
.add(
'cookielessRedisPool',
new RedisPoolComponent({
connection: createCookielessRedisConnectionConfig(this.config),
poolMinSize: this.config.REDIS_POOL_MIN_SIZE,
poolMaxSize: this.config.REDIS_POOL_MAX_SIZE,
})
)
// Dedicated $feature_flag_called dedup Redis, so its claim keys don't compete
// with ingestion's overflow-redirect keys under eviction. Falls back to the
// ingestion connection until the dedup host is configured.
.add(
'featureFlagCalledDedupRedisPool',
new RedisPoolComponent({
connection: createFeatureFlagCalledDedupRedisConnectionConfig(this.config),
poolMinSize: this.config.REDIS_POOL_MIN_SIZE,
poolMaxSize: this.config.REDIS_POOL_MAX_SIZE,
})
)
.add('producerRegistry', new KafkaProducerRegistryComponent(this.config.KAFKA_CLIENT_RACK, this.config))
)
// Services are built off the started infra scope (postgres, redis pools, kafka). They're
// owned by this scope, so consumers can `Scope.extend` off it to read shared services like
// the team manager and cookieless manager without taking ownership of their lifecycle.
const sharedServicesScope = extend(sharedInfraScope, 'shared', (container, builder) =>
builder
.add(
'teamManager',
// Retry transient team-load failures (e.g. a Postgres pooler scale-down returning
// ECONNREFUSED). The team loader runs detached in the LazyLoader buffer, so an un-retried
// transient failure can surface as an unhandled rejection and restart the worker.
new TeamManagerComponent(container.postgres, {
loaderRetry: DEFAULT_LOADER_RETRY,
})
)
.add('cookielessManager', new CookielessManagerComponent(this.config, container.cookielessRedisPool))
)
const sharedServices = await sharedServicesScope.start()
this.postgres = sharedServices.container.postgres
this.redisPool = sharedServices.container.redisPool
const teamManager = sharedServices.container.teamManager
const cookielessManager = sharedServices.container.cookielessManager
this.stopSharedServices = sharedServices.stop
logger.info('👍', 'Postgres Router ready')
logger.info('👍', 'Ingestion Redis ready')
this.pubsub = new PubSub(this.redisPool)
await this.pubsub.start()
// 2. Ingestion + CDP shared services (geoip, repos, encryption)
const geoipService = new GeoIPService(this.config.MMDB_FILE_LOCATION)
await geoipService.get()
const personhogClient = createPersonHogClient(this.config)
const clientLabel = this.config.PLUGIN_SERVER_MODE ?? 'unknown'
const postgresPersonRepository = new PostgresPersonRepository(this.postgres, {
calculatePropertiesSize: this.config.PERSON_UPDATE_CALCULATE_PROPERTIES_SIZE,
personBatchWritePerKeyTeamAllowlist: this.config.PERSON_BATCH_WRITING_PER_KEY_TEAM_ALLOWLIST,
})
const personRepository = buildPersonRepository(
personhogClient,
postgresPersonRepository,
this.config.PERSONHOG_PERSONS_ROLLOUT_PERCENTAGE,
this.config.PERSONHOG_PERSONS_ROLLOUT_TEAM_IDS,
clientLabel
)
const postgresGroupRepository = new PostgresGroupRepository(this.postgres)
const groupRepository = buildGroupRepository(
personhogClient,
postgresGroupRepository,
this.config.PERSONHOG_GROUPS_ROLLOUT_PERCENTAGE,
this.config.PERSONHOG_GROUPS_ROLLOUT_TEAM_IDS,
clientLabel
)
const encryptedFields = new EncryptedFields(this.config.ENCRYPTION_SALT_KEYS)
const integrationManager = new IntegrationManagerService(this.pubsub, this.postgres, encryptedFields)
// 3. Ingestion-specific services
// Same rationale as the team manager's loaderRetry above: group-type loads run
// detached in the LazyLoader buffer, so an un-retried transient failure can
// surface as an unhandled rejection and restart the worker.
const groupTypeManager = new GroupTypeManager(groupRepository, teamManager, {
loaderRetry: DEFAULT_LOADER_RETRY,
})
const serviceLoaders: (() => Promise<PluginServerService>)[] = []
const isCombinedMode = this.config.PLUGIN_SERVER_MODE === PluginServerMode.ingestion_v2_combined
// Producer registry is owned by `sharedInfraScope`; the
// server reads it back from the started services. Outputs is
// a typed view over it — built once here for analytics, and
// separately by each consumer factory as needed.
const ingestionProducerRegistry = sharedServices.container.producerRegistry
const ingestionOutputs = createOutputsRegistry().build(ingestionProducerRegistry, this.config)
const clickhouseGroupRepository = new ClickhouseGroupRepository(ingestionOutputs)
const hogTransformerDeps: HogTransformerServiceDeps = {
geoipService,
postgres: this.postgres,
pubSub: this.pubsub,
encryptedFields,
integrationManager,
monitoringOutputs: ingestionOutputs,
teamManager,
}
const ingestionDeps: IngestionConsumerDeps = {
postgres: this.postgres,
redisPool: this.redisPool,
featureFlagCalledDedupRedisPool: sharedServices.container.featureFlagCalledDedupRedisPool,
outputs: ingestionOutputs,
teamManager,
groupTypeManager,
groupRepository,
clickhouseGroupRepository,
personRepository,
cookielessManager,
hogTransformer: createHogTransformerService(this.config, hogTransformerDeps),
}
const startClientWarnings = (override?: { topic: string; groupId: string }) => {
serviceLoaders.push(async () => {
const consumerConfig = override
? {
...this.config,
INGESTION_CONSUMER_CONSUME_TOPIC: override.topic,
INGESTION_CONSUMER_GROUP_ID: override.groupId,
}
: this.config
const consumerScope = createClientWarningsConsumer(consumerConfig, sharedServicesScope)
const { consumer, stop } = await consumerScope.start()
return ingestionConsumerService(consumer, stop)
})
}
const startHeatmaps = (override?: { topic: string; groupId: string }) => {
serviceLoaders.push(async () => {
const consumerConfig = override
? {
...this.config,
INGESTION_CONSUMER_CONSUME_TOPIC: override.topic,
INGESTION_CONSUMER_GROUP_ID: override.groupId,
}
: this.config
const consumerScope = createHeatmapsConsumer(consumerConfig, sharedServicesScope)
const { consumer, stop } = await consumerScope.start()
return ingestionConsumerService(consumer, stop)
})
}
const startAi = (override?: { topic: string; groupId: string }) => {
serviceLoaders.push(async () => {
const consumerConfig = override
? {
...this.config,
INGESTION_CONSUMER_CONSUME_TOPIC: override.topic,
INGESTION_CONSUMER_GROUP_ID: override.groupId,
}
: this.config
// The AI lane can't construct the cdp-owned hog transformer itself (boundary),
// so the server injects it (and the lane's outputs, which also back the
// transformer's monitoring) through an AI-specific scope. The consumer owns
// everything else (incl. its personhog client), taking only config + parent scope.
const aiOutputs = createAiOutputsRegistry().build(ingestionProducerRegistry, this.config)
const aiSharedScope = extend(sharedServicesScope, 'ai-shared', (_container, builder) =>
builder
.add(
'hogTransformer',
new HogTransformerComponent(() =>
createHogTransformerService(this.config, {
...hogTransformerDeps,
monitoringOutputs: aiOutputs,
})
)
)
.add('outputs', new IngestionOutputsComponent(() => aiOutputs))
)
const consumerScope = createAiConsumer(consumerConfig, aiSharedScope)
const { consumer, stop } = await consumerScope.start()
return ingestionConsumerService(consumer, stop)
})
}
if (isCombinedMode) {
// Local dev / hobby: run multiple consumers for all ingestion topics in one process
const consumersOptions = [
{ topic: KAFKA_EVENTS_PLUGIN_INGESTION, group_id: 'clickhouse-ingestion' },
{ topic: KAFKA_EVENTS_PLUGIN_INGESTION_HISTORICAL, group_id: 'clickhouse-ingestion-historical' },
{ topic: KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW, group_id: 'clickhouse-ingestion-overflow' },
]
for (const consumerOption of consumersOptions) {
serviceLoaders.push(async () => {
const consumer = new IngestionConsumer(this.config, ingestionDeps, {
INGESTION_CONSUMER_CONSUME_TOPIC: consumerOption.topic,
INGESTION_CONSUMER_GROUP_ID: consumerOption.group_id,
})
await consumer.start()
return consumer.service
})
}
startClientWarnings({
topic: 'ingestion-clientwarnings-main-1',
groupId: 'ingestion-clientwarnings-main',
})
startHeatmaps({
topic: 'heatmaps_ingestion',
groupId: 'heatmaps_ingestion',
})
// Capture routes $ai_* events to the dedicated AI topic on every
// deployment shape, so combined mode needs the AI consumer too.
startAi({
topic: KAFKA_EVENTS_PLUGIN_INGESTION_AI,
groupId: 'clickhouse-ingestion-ai',
})
} else if (this.config.INGESTION_PIPELINE === 'clientwarnings') {
startClientWarnings()
} else if (this.config.INGESTION_PIPELINE === 'heatmaps') {
startHeatmaps()
} else if (this.config.INGESTION_PIPELINE === 'ai') {
// Dedicated AI pipeline deployment, consuming the topic capture's
// AI routing produces to (config-provided in production).
startAi()
} else {
// Production ingestion-v2: single consumer using config-provided topic
serviceLoaders.push(async () => {
const consumer = new IngestionConsumer(this.config, ingestionDeps)
await consumer.start()
return consumer.service
})
}
// ServerCommands is always created
serviceLoaders.push(() => {
const serverCommands = new ServerCommands(this.pubsub!)
this.lifecycle.expressApp.use('/', serverCommands.router())
return Promise.resolve(serverCommands.service)
})
const readyServices = await Promise.all(serviceLoaders.map((loader) => loader()))
this.lifecycle.services.push(...readyServices)
}
private getCleanupResources(): CleanupResources {
return {
kafkaProducers: [],
redisPools: [],
pubsub: this.pubsub,
additionalCleanup: async () => {
if (this.stopSharedServices) {
await this.stopSharedServices()
}
},
}
}
}