Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
import { buildIntegerMatcher } from '~/common/config/config'
import { ReadOnlyGroupTypeManager } from '~/common/groups/readonly-group-type-manager'
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import {
    AiEventOutput,
    AppMetricsOutput,
    DlqOutput,
    EventOutput,
    IngestionWarningsOutput,
    OverflowOutput,
    TophogOutput,
} from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'
import { PersonHogConfig } from '~/common/personhog'
import { PersonHogClientComponent } from '~/common/personhog/personhog-client-component'
import { PersonHogGroupReadRepository } from '~/common/personhog/personhog-group-read-repository'
import { PersonHogPersonReadRepository } from '~/common/personhog/personhog-person-read-repository'
import { UsageIngestionConfig, createEventUsageBatchFactory } from '~/common/usage-ingestion'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { EventIngestionRestrictionManagerComponent } from '~/common/utils/event-ingestion-restrictions'
import { EventSchemaEnforcementManager } from '~/common/utils/event-schema-enforcement-manager'
import { DEFAULT_LOADER_RETRY } from '~/common/utils/lazy-loader'
import { TeamManager } from '~/common/utils/team-manager'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { EventFilterManagerComponent } from '~/ingestion/common/event-filters'
import { CommonIngestionConsumerConfig, CommonIngestionConsumerScope } from '~/ingestion/common/ingestion-consumer'
import { ProducerName } from '~/ingestion/common/outputs/producers'
import { DisabledOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/disabled-overflow-redirect'
import { MainLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/main-lane-overflow-redirect'
import { OverflowLaneOverflowRedirectComponent } from '~/ingestion/common/overflow-redirect/overflow-lane-overflow-redirect'
import { RedisOverflowRepositoryComponent } from '~/ingestion/common/overflow-redirect/overflow-redis-repository'
import { eventRateStrategy } from '~/ingestion/common/overflow-redirect/overflow-strategy'
import { Scope, extend } from '~/ingestion/common/scopes'
import { PromiseSchedulerComponent } from '~/ingestion/common/utils/promise-scheduler'
import { IngestionConsumerConfig, IngestionOutputsConfig } from '~/ingestion/config'
import { RedisPool } from '~/types'

import { AiBlobStoreComponent } from './blob-offload/blob-store'
import { createAiIngestionPipeline } from './pipeline'

export type AiConsumerConfig = CommonIngestionConsumerConfig &
    IngestionOutputsConfig &
    PersonHogConfig &
    UsageIngestionConfig &
    Pick<
        IngestionConsumerConfig,
        | 'INGESTION_OVERFLOW_MODE'
        | 'INGESTION_CONSUMER_OVERFLOW_TOPIC'
        | 'INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS'
        | 'INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS'
        | 'EVENT_OVERFLOW_BUCKET_CAPACITY'
        | 'EVENT_OVERFLOW_BUCKET_REPLENISH_RATE'
        | 'INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY'
        | 'INGESTION_WORKER_CONCURRENT_BATCHES'
        | 'DROP_EVENTS_BY_TOKEN_DISTINCT_ID'
        | 'SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID'
        | 'INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID'
        | 'EVENT_SCHEMA_ENFORCEMENT_ENABLED'
        | 'TEAMS_PREFETCH_ENABLED'
        | 'HOG_FUNCTIONS_PREFETCH_ENABLED'
        | 'AI_BLOB_S3_BUCKET'
        | 'AI_BLOB_S3_PREFIX'
        | 'AI_BLOB_S3_ENDPOINT'
        | 'AI_BLOB_S3_REGION'
        | 'AI_BLOB_S3_ACCESS_KEY_ID'
        | 'AI_BLOB_S3_SECRET_ACCESS_KEY'
        | 'AI_BLOB_S3_TIMEOUT_MS'
        | 'AI_BLOB_OFFLOAD_TEAMS'
        | 'AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH'
        | 'AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT'
        | 'AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY'
        | 'AI_BLOB_OFFLOAD_TOUCH_AFTER_HOURS'
    >

/** Outputs the AI pipeline emits to. The same instance backs the hog transformer's
 * monitoring (app_metrics + log_entries), wired up server-side. */
export type AiOutputs = IngestionOutputs<
    EventOutput | AiEventOutput | IngestionWarningsOutput | DlqOutput | OverflowOutput | AppMetricsOutput | TophogOutput
>

/**
 * Services shared from the server scope. Extends the base shared scope with the
 * hog transformer and outputs, which the server must build because the lane can't
 * construct the cdp-owned transformer (boundary) and the same outputs instance
 * backs the transformer's monitoring. The personhog client is owned by the AI
 * scope (created from common, tagged for this pipeline), not shared in here.
 */
export type AiSharedScope = Scope<{
    postgres: PostgresRouter
    redisPool: RedisPool
    teamManager: TeamManager
    cookielessManager: CookielessManager
    producerRegistry: KafkaProducerRegistry<ProducerName>
    hogTransformer: HogTransformer
    outputs: AiOutputs
}>

export function createAiConsumer(config: AiConsumerConfig, sharedScope: AiSharedScope) {
    const splitTokens = (value: string): string[] => value.split(',').filter((x) => !!x)
    const overflowMode = config.INGESTION_OVERFLOW_MODE
    const preservePartitionLocality = config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY
    // Client name for personhog read metrics: pipeline + lane (e.g. "ai/main").
    // The query name is supplied per call (e.g. "person-properties").
    const clientLabel = `ai/${config.INGESTION_LANE ?? 'main'}`

    // Parent scope: the overflow Redis repository, shared by the main-lane and
    // overflow-lane redirect services below (set up here, not inline per service).
    const overflowScope = extend(sharedScope, 'ai-overflow', (container, builder) =>
        builder.add(
            'overflowRedisRepository',
            new RedisOverflowRepositoryComponent(
                container.redisPool,
                config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS
            )
        )
    )

    const scope = extend(overflowScope, 'ai', (container, builder) =>
        builder
            .add('promiseScheduler', new PromiseSchedulerComponent())
            .add(
                'eventIngestionRestrictionManager',
                new EventIngestionRestrictionManagerComponent(container.redisPool, {
                    pipeline: 'ai',
                    staticDropEventTokens: splitTokens(config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID),
                    staticSkipPersonTokens: splitTokens(config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID),
                    staticForceOverflowTokens: splitTokens(config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID),
                })
            )
            .add('eventFilterManager', new EventFilterManagerComponent(container.postgres, { pipeline: 'ai' }))
            // Dedicated 'ai' keyspace so AI overflow never affects analytics.
            .add(
                'overflowRedirectService',
                overflowMode === 'redirect'
                    ? new MainLaneOverflowRedirectComponent({
                          redisRepository: container.overflowRedisRepository,
                          localCacheTTLSeconds: config.INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS,
                          strategies: [
                              {
                                  ...eventRateStrategy(),
                                  bucketCapacity: config.EVENT_OVERFLOW_BUCKET_CAPACITY,
                                  replenishRate: config.EVENT_OVERFLOW_BUCKET_REPLENISH_RATE,
                              },
                          ],
                          overflowType: 'ai',
                      })
                    : new DisabledOverflowRedirectComponent()
            )
            .add(
                'overflowLaneTTLRefreshService',
                overflowMode === 'consume'
                    ? new OverflowLaneOverflowRedirectComponent({
                          redisRepository: container.overflowRedisRepository,
                          overflowType: 'ai',
                      })
                    : new DisabledOverflowRedirectComponent()
            )
            // Personhog client owned by the AI scope (created from common, torn down with it).
            .add('personhogClient', new PersonHogClientComponent(config))
            // Owned by the scope so the store's startup healthcheck runs at
            // scope start, before any traffic is consumed.
            .add('aiBlobStore', new AiBlobStoreComponent(config))
    )

    const uploadMaxConcurrency = config.AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY
    if (!Number.isInteger(uploadMaxConcurrency) || uploadMaxConcurrency <= 0) {
        // The ingestion config layer coerces env overrides by hand (no schema
        // validation), so a bad value would otherwise surface as a nameless
        // TypeError from p-limit at pipeline construction.
        throw new Error(
            `AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY must be a positive integer, got ${uploadMaxConcurrency}`
        )
    }
    const createEventUsageBatch = createEventUsageBatchFactory(config, 'ai_events')

    const aiBlobOffloadConfig = {
        isTeamEnabled: buildIntegerMatcher(config.AI_BLOB_OFFLOAD_TEAMS, true),
        minBase64Length: config.AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH,
        maxBlobsPerEvent: config.AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT,
        uploadMaxConcurrency,
    }

    return new CommonIngestionConsumerScope('ai', config, scope, ({ container }) =>
        createAiIngestionPipeline({
            outputs: container.outputs,
            teamManager: container.teamManager,
            eventIngestionRestrictionManager: container.eventIngestionRestrictionManager,
            eventFilterManager: container.eventFilterManager,
            cookielessManager: container.cookielessManager,
            promiseScheduler: container.promiseScheduler,
            hogTransformer: container.hogTransformer,
            // Read-only person/group access — read through personhog, never written.
            personRepository: new PersonHogPersonReadRepository(container.personhogClient, clientLabel),
            groupTypeManager: new ReadOnlyGroupTypeManager(
                new PersonHogGroupReadRepository(container.personhogClient, clientLabel)
            ),
            overflowMode,
            preservePartitionLocality,
            overflowRedirectService: container.overflowRedirectService,
            overflowLaneTTLRefreshService: container.overflowLaneTTLRefreshService,
            concurrentBatches: config.INGESTION_WORKER_CONCURRENT_BATCHES,
            eventSchemaEnforcementEnabled: config.EVENT_SCHEMA_ENFORCEMENT_ENABLED,
            teamsPrefetchEnabled: config.TEAMS_PREFETCH_ENABLED,
            hogFunctionsPrefetchEnabled: config.HOG_FUNCTIONS_PREFETCH_ENABLED,
            // Schema loads run detached in the LazyLoader buffer, so an un-retried transient
            // failure can surface as an unhandled rejection and restart the worker.
            eventSchemaEnforcementManager: new EventSchemaEnforcementManager(container.postgres, {
                loaderRetry: DEFAULT_LOADER_RETRY,
            }),
            topHog: container.topHog,
            aiBlobStore: container.aiBlobStore.store,
            aiBlobOffloadConfig,
            createEventUsageBatch,
        })
    )
}