:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { buildIntegerMatcher } from '~/common/config/config'
import { ReadOnlyGroupTypeManager } from '~/common/groups/readonly-group-type-manager'
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import {
AiEventOutput,
AppMetricsOutput,
DlqOutput,
EventOutput,
IngestionWarningsOutput,
OverflowOutput,
TophogOutput,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'
import { PersonHogConfig } from '~/common/personhog'
import { PersonHogClientComponent } from '~/common/personhog/personhog-client-component'
import { PersonHogGroupReadRepository } from '~/common/personhog/personhog-group-read-repository'
import { PersonHogPersonReadRepository } from '~/common/personhog/personhog-person-read-repository'
import { UsageIngestionConfig, createEventUsageBatchFactory } from '~/common/usage-ingestion'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { EventIngestionRestrictionManagerComponent } from '~/common/utils/event-ingestion-restrictions'
import { EventSchemaEnforcementManager } from '~/common/utils/event-schema-enforcement-manager'
import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'
import { TeamManager } from '~/common/utils/team-manager'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { EventFilterManagerComponent } from '~/ingestion/common/event-filters'
import { CommonIngestionConsumerConfig, CommonIngestionConsumerScope } from '~/ingestion/common/ingestion-consumer'
import { ProducerName } from '~/ingestion/common/outputs/producers'
import { DisabledOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/disabled-overflow-redirect'
import { MainLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/main-lane-overflow-redirect'
import { OverflowLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/overflow-lane-overflow-redirect'
import { RedisOverflowRepositoryComponent } from '~/ingestion/common/overflow-redirect/overflow-redis-repository'
import { eventRateStrategy } from '~/ingestion/common/overflow-redirect/overflow-strategy'
import { Scope, extend } from '~/ingestion/common/scopes'
import { PromiseSchedulerComponent } from '~/ingestion/common/utils/promise-scheduler'
import { IngestionConsumerConfig, IngestionOutputsConfig } from '~/ingestion/config'
import { RedisPool } from '~/types'
import { AiBlobStoreComponent } from './blob-offload/blob-store'
import { createAiIngestionPipeline } from './pipeline'
export type AiConsumerConfig = CommonIngestionConsumerConfig &
IngestionOutputsConfig &
PersonHogConfig &
UsageIngestionConfig &
Pick<
IngestionConsumerConfig,
| 'INGESTION_OVERFLOW_MODE'
| 'INGESTION_CONSUMER_OVERFLOW_TOPIC'
| 'INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS'
| 'INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS'
| 'EVENT_OVERFLOW_BUCKET_CAPACITY'
| 'EVENT_OVERFLOW_BUCKET_REPLENISH_RATE'
| 'INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY'
| 'INGESTION_WORKER_CONCURRENT_BATCHES'
| 'DROP_EVENTS_BY_TOKEN_DISTINCT_ID'
| 'SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID'
| 'INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID'
| 'EVENT_SCHEMA_ENFORCEMENT_ENABLED'
| 'TEAMS_PREFETCH_ENABLED'
| 'HOG_FUNCTIONS_PREFETCH_ENABLED'
| 'AI_BLOB_S3_BUCKET'
| 'AI_BLOB_S3_PREFIX'
| 'AI_BLOB_S3_ENDPOINT'
| 'AI_BLOB_S3_REGION'
| 'AI_BLOB_S3_ACCESS_KEY_ID'
| 'AI_BLOB_S3_SECRET_ACCESS_KEY'
| 'AI_BLOB_S3_TIMEOUT_MS'
| 'AI_BLOB_OFFLOAD_TEAMS'
| 'AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH'
| 'AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT'
| 'AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY'
| 'AI_BLOB_OFFLOAD_TOUCH_AFTER_HOURS'
>
/** Outputs the AI pipeline emits to. The same instance backs the hog transformer's
* monitoring (app_metrics + log_entries), wired up server-side. */
export type AiOutputs = IngestionOutputs<
EventOutput | AiEventOutput | IngestionWarningsOutput | DlqOutput | OverflowOutput | AppMetricsOutput | TophogOutput
>
/**
* Services shared from the server scope. Extends the base shared scope with the
* hog transformer and outputs, which the server must build because the lane can't
* construct the cdp-owned transformer (boundary) and the same outputs instance
* backs the transformer's monitoring. The personhog client is owned by the AI
* scope (created from common, tagged for this pipeline), not shared in here.
*/
export type AiSharedScope = Scope<{
postgres: PostgresRouter
redisPool: RedisPool
teamManager: TeamManager
cookielessManager: CookielessManager
producerRegistry: KafkaProducerRegistry<ProducerName>
hogTransformer: HogTransformer
outputs: AiOutputs
}>
export function createAiConsumer(config: AiConsumerConfig, sharedScope: AiSharedScope) {
const splitTokens = (value: string): string[] => value.split(',').filter((x) => !!x)
const overflowMode = config.INGESTION_OVERFLOW_MODE
const preservePartitionLocality = config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY
// Client name for personhog read metrics: pipeline + lane (e.g. "ai/main").
// The query name is supplied per call (e.g. "person-properties").
const clientLabel = `ai/${config.INGESTION_LANE ?? 'main'}`
// Parent scope: the overflow Redis repository, shared by the main-lane and
// overflow-lane redirect services below (set up here, not inline per service).
const overflowScope = extend(sharedScope, 'ai-overflow', (container, builder) =>
builder.add(
'overflowRedisRepository',
new RedisOverflowRepositoryComponent(
container.redisPool,
config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS
)
)
)
const scope = extend(overflowScope, 'ai', (container, builder) =>
builder
.add('promiseScheduler', new PromiseSchedulerComponent())
.add(
'eventIngestionRestrictionManager',
new EventIngestionRestrictionManagerComponent(container.redisPool, {
pipeline: 'ai',
staticDropEventTokens: splitTokens(config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID),
staticSkipPersonTokens: splitTokens(config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID),
staticForceOverflowTokens: splitTokens(config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID),
})
)
.add('eventFilterManager', new EventFilterManagerComponent(container.postgres, { pipeline: 'ai' }))
// Dedicated 'ai' keyspace so AI overflow never affects analytics.
.add(
'overflowRedirectService',
overflowMode === 'redirect'
? new MainLaneOverflowRedirectComponent({
redisRepository: container.overflowRedisRepository,
localCacheTTLSeconds: config.INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS,
strategies: [
{
...eventRateStrategy(),
bucketCapacity: config.EVENT_OVERFLOW_BUCKET_CAPACITY,
replenishRate: config.EVENT_OVERFLOW_BUCKET_REPLENISH_RATE,
},
],
overflowType: 'ai',
})
: new DisabledOverflowRedirectComponent()
)
.add(
'overflowLaneTTLRefreshService',
overflowMode === 'consume'
? new OverflowLaneOverflowRedirectComponent({
redisRepository: container.overflowRedisRepository,
overflowType: 'ai',
})
: new DisabledOverflowRedirectComponent()
)
// Personhog client owned by the AI scope (created from common, torn down with it).
.add('personhogClient', new PersonHogClientComponent(config))
// Owned by the scope so the store's startup healthcheck runs at
// scope start, before any traffic is consumed.
.add('aiBlobStore', new AiBlobStoreComponent(config))
)
const uploadMaxConcurrency = config.AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY
if (!Number.isInteger(uploadMaxConcurrency) || uploadMaxConcurrency <= 0) {
// The ingestion config layer coerces env overrides by hand (no schema
// validation), so a bad value would otherwise surface as a nameless
// TypeError from p-limit at pipeline construction.
throw new Error(
`AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY must be a positive integer, got ${uploadMaxConcurrency}`
)
}
const createEventUsageBatch = createEventUsageBatchFactory(config, 'ai_events')
const aiBlobOffloadConfig = {
isTeamEnabled: buildIntegerMatcher(config.AI_BLOB_OFFLOAD_TEAMS, true),
minBase64Length: config.AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH,
maxBlobsPerEvent: config.AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT,
uploadMaxConcurrency,
}
return new CommonIngestionConsumerScope('ai', config, scope, ({ container }) =>
createAiIngestionPipeline({
outputs: container.outputs,
teamManager: container.teamManager,
eventIngestionRestrictionManager: container.eventIngestionRestrictionManager,
eventFilterManager: container.eventFilterManager,
cookielessManager: container.cookielessManager,
promiseScheduler: container.promiseScheduler,
hogTransformer: container.hogTransformer,
// Read-only person/group access — read through personhog, never written.
personRepository: new PersonHogPersonReadRepository(container.personhogClient, clientLabel),
groupTypeManager: new ReadOnlyGroupTypeManager(
new PersonHogGroupReadRepository(container.personhogClient, clientLabel)
),
overflowMode,
preservePartitionLocality,
overflowRedirectService: container.overflowRedirectService,
overflowLaneTTLRefreshService: container.overflowLaneTTLRefreshService,
concurrentBatches: config.INGESTION_WORKER_CONCURRENT_BATCHES,
eventSchemaEnforcementEnabled: config.EVENT_SCHEMA_ENFORCEMENT_ENABLED,
teamsPrefetchEnabled: config.TEAMS_PREFETCH_ENABLED,
hogFunctionsPrefetchEnabled: config.HOG_FUNCTIONS_PREFETCH_ENABLED,
// Schema loads run detached in the LazyLoader buffer, so an un-retried transient
// failure can surface as an unhandled rejection and restart the worker.
eventSchemaEnforcementManager: new EventSchemaEnforcementManager(container.postgres, {
loaderRetry: DEFAULT_LOADER_RETRY,
}),
topHog: container.topHog,
aiBlobStore: container.aiBlobStore.store,
aiBlobOffloadConfig,
createEventUsageBatch,
})
)
}