:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { HogTransformer } from '~/common/hog-transformations/hog-transformer.interface'
import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'
import { PersonHogConfig } from '~/common/personhog'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { TeamManagerComponent } from '~/common/utils/team-manager'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { ProducerName } from '~/ingestion/common/outputs/producers'
import { newScope } from '~/ingestion/common/scopes'
import { IngestionConsumerConfig, IngestionOutputsConfig } from '~/ingestion/config'
import { RedisPool } from '~/types'
import { AiConsumerConfig, AiOutputs, AiSharedScope, createAiConsumer } from './consumer'
import * as pipelineModule from './pipeline'
jest.mock('./pipeline')
describe('createAiConsumer', () => {
function makeConfig(): AiConsumerConfig {
return {
INGESTION_CONSUMER_GROUP_ID: 'g',
INGESTION_CONSUMER_CONSUME_TOPIC: 't',
INGESTION_PIPELINE: 'ai',
INGESTION_LANE: 'main',
INGESTION_OVERFLOW_MODE: 'disabled',
KAFKA_BATCH_START_LOGGING_ENABLED: false,
INGESTION_CONSUMER_OVERFLOW_TOPIC: '',
INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY: false,
INGESTION_WORKER_CONCURRENT_BATCHES: 1,
DROP_EVENTS_BY_TOKEN_DISTINCT_ID: '',
TEAMS_PREFETCH_ENABLED: false,
HOG_FUNCTIONS_PREFETCH_ENABLED: false,
SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID: '',
INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID: '',
EVENT_SCHEMA_ENFORCEMENT_ENABLED: false,
USAGE_INGESTION_ADDR: '',
USAGE_INGESTION_TLS: false,
USAGE_INGESTION_TIMEOUT_MS: 5000,
USAGE_INGESTION_MAX_BATCH_SIZE: 500,
USAGE_INGESTION_REPORT_TEAMS: '',
// Read eagerly by createAiConsumer (not deferred like the scope-builder services
// below), so these need real fail-closed values rather than the cast-away pattern.
AI_BLOB_S3_BUCKET: '',
AI_BLOB_S3_PREFIX: '',
AI_BLOB_S3_ENDPOINT: '',
AI_BLOB_S3_REGION: 'us-east-1',
AI_BLOB_S3_ACCESS_KEY_ID: '',
AI_BLOB_S3_SECRET_ACCESS_KEY: '',
AI_BLOB_S3_TIMEOUT_MS: 30000,
AI_BLOB_OFFLOAD_TEAMS: '',
AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH: 8192,
AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT: 50,
AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY: 8,
AI_BLOB_OFFLOAD_TOUCH_AFTER_HOURS: 20,
...({} as Pick<
IngestionConsumerConfig,
| 'INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS'
| 'INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS'
| 'EVENT_OVERFLOW_BUCKET_CAPACITY'
| 'EVENT_OVERFLOW_BUCKET_REPLENISH_RATE'
>),
...({} as IngestionOutputsConfig),
...({} as PersonHogConfig),
}
}
function makeSharedScope(): AiSharedScope {
// The consumer factory extends this scope but doesn't start it (start happens
// at the caller), so the shape only has to be type-correct.
return newScope('shared-test', (b) =>
b
.add('postgres', {
start: () => Promise.resolve({ value: {} as PostgresRouter, stop: () => Promise.resolve() }),
})
.add('redisPool', {
start: () => Promise.resolve({ value: {} as RedisPool, stop: () => Promise.resolve() }),
})
.add('teamManager', new TeamManagerComponent({} as PostgresRouter))
.add('cookielessManager', {
start: () => Promise.resolve({ value: {} as CookielessManager, stop: () => Promise.resolve() }),
})
.add('producerRegistry', {
start: () =>
Promise.resolve({
value: {} as KafkaProducerRegistry<ProducerName>,
stop: () => Promise.resolve(),
}),
})
.add('hogTransformer', {
start: () => Promise.resolve({ value: {} as HogTransformer, stop: () => Promise.resolve() }),
})
.add('outputs', {
start: () => Promise.resolve({ value: {} as AiOutputs, stop: () => Promise.resolve() }),
})
)
}
beforeEach(() => {
;(pipelineModule.createAiIngestionPipeline as jest.Mock) = jest.fn().mockReturnValue({
feed: jest.fn(),
next: jest.fn(),
})
})
it('defers pipeline construction until scope.start()', () => {
createAiConsumer(makeConfig(), makeSharedScope())
// The pipeline factory runs inside the extend callback at start time, after the
// scope's services come up — not at consumer construction time.
expect(pipelineModule.createAiIngestionPipeline).not.toHaveBeenCalled()
})
})