:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import type { CommonConfig } from '~/common/config'
import {
KAFKA_APP_METRICS_2,
KAFKA_CLICKHOUSE_AI_EVENTS_JSON,
KAFKA_CLICKHOUSE_HEATMAP_EVENTS,
KAFKA_CLICKHOUSE_TOPHOG,
KAFKA_EVENTS_JSON,
KAFKA_EVENTS_PLUGIN_INGESTION,
KAFKA_EVENTS_PLUGIN_INGESTION_ASYNC,
KAFKA_EVENTS_PLUGIN_INGESTION_DLQ,
KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW,
KAFKA_GROUPS,
KAFKA_INGESTION_WARNINGS,
KAFKA_LOG_ENTRIES,
KAFKA_PERSON,
KAFKA_PERSON_DISTINCT_ID,
} from '~/common/config/kafka-topics'
import type { PostgresRouterConfig } from '~/common/utils/db/postgres'
import { isDevEnv, isProdEnv } from '~/common/utils/env-utils'
import { AI_MEDIA_OFFLOAD_MIN_BYTES } from '~/ingestion/common/ai-event-limits'
import {
INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_UPSTREAM_PRODUCER,
type ProducerName,
} from '~/ingestion/common/outputs/producers'
/** Default for FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: '' disables the personless default so it is opt-in per team via config. */
export const DEFAULT_FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS = ''
/** Default for FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS: '' excludes nobody, so the allowlist alone decides. */
export const DEFAULT_FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS = ''
// =============================================================================
// Infrastructure sub-config types
// These group CommonConfig keys by infrastructure concern for use in server
// config types. Defined here (rather than common/config.ts) because they are
// ingestion-specific groupings — other domains may slice CommonConfig differently.
// =============================================================================
/** Kafka broker connection and authentication config */
export type KafkaBrokerConfig = Pick<
CommonConfig,
| 'KAFKA_HOSTS'
| 'KAFKA_CLIENT_RACK'
| 'KAFKA_CLIENT_CERT_B64'
| 'KAFKA_CLIENT_CERT_KEY_B64'
| 'KAFKA_TRUSTED_CERT_B64'
| 'KAFKA_SASL_MECHANISM'
| 'KAFKA_SASL_USER'
| 'KAFKA_SASL_PASSWORD'
>
/** PostgreSQL connection config for the ingestion server — excludes behavioral cohorts */
export type DatabaseConnectionConfig = Omit<PostgresRouterConfig, 'BEHAVIORAL_COHORTS_DATABASE_URL'>
/** Redis connection config — URLs, hosts, and pool sizing */
export type RedisConnectionsConfig = Pick<
CommonConfig,
| 'REDIS_URL'
| 'REDIS_POOL_MIN_SIZE'
| 'REDIS_POOL_MAX_SIZE'
| 'INGESTION_REDIS_HOST'
| 'INGESTION_REDIS_PORT'
| 'POSTHOG_REDIS_HOST'
| 'POSTHOG_REDIS_PORT'
| 'POSTHOG_REDIS_PASSWORD'
>
/** Kafka consumer loop tuning config */
export type KafkaConsumerBaseConfig = Pick<
CommonConfig,
| 'CONSUMER_BATCH_SIZE'
| 'CONSUMER_MAX_HEARTBEAT_INTERVAL_MS'
| 'CONSUMER_LOOP_STALL_THRESHOLD_MS'
| 'CONSUMER_LOG_STATS_LEVEL'
| 'CONSUMER_LOOP_BASED_HEALTH_CHECK'
| 'CONSUMER_MAX_BACKGROUND_TASKS'
| 'CONSUMER_AUTO_CREATE_TOPICS'
>
export type PersonBatchWritingDbWriteMode = 'NO_ASSERT' | 'ASSERT_VERSION'
export type PersonBatchWritingMode = 'BATCH' | 'SHADOW' | 'NONE'
/**
* Real-time lanes process live events and share `main`'s processing-time SLO.
* Delayed lanes process backfilled or async events on their own timeline. The
* distinction is load-bearing for processing-time logic like dedup, so the lane
* type is derived from these two sets to keep the categorization in one place.
*/
export const REALTIME_INGESTION_LANES = ['main', 'overflow', 'turbo', 'team2'] as const
export const DELAYED_INGESTION_LANES = ['historical', 'async'] as const
export type IngestionLane = (typeof REALTIME_INGESTION_LANES)[number] | (typeof DELAYED_INGESTION_LANES)[number]
/**
* How a consumer participates in overflow handling. Explicit and independent of
* the lane name:
* - `redirect` redirect hot partitions to the overflow topic (main lane).
* - `consume` drain the overflow topic and refresh its stateful TTLs (overflow lane).
* - `disabled` no overflow handling.
*/
export const INGESTION_OVERFLOW_MODES = ['redirect', 'consume', 'disabled'] as const
export type IngestionOverflowMode = (typeof INGESTION_OVERFLOW_MODES)[number]
export type IngestionConsumerConfig = {
INGESTION_LANE: IngestionLane | null
INGESTION_OVERFLOW_MODE: IngestionOverflowMode
// Kafka consumer config
INGESTION_CONSUMER_GROUP_ID: string
INGESTION_CONSUMER_CONSUME_TOPIC: string
INGESTION_CONSUMER_DLQ_TOPIC: string
INGESTION_CONSUMER_OVERFLOW_TOPIC: string
// Ingestion pipeline config
INGESTION_BATCH_SIZE: number
INGESTION_OVERFLOW_ENABLED: boolean
INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID: string
INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY: boolean
// Maximum in-flight batches per worker (BatchingPipeline.concurrentBatches).
// Mirrors INGESTION_WORKER_CONCURRENT_BATCHES on the Rust consumer side —
// both values MUST agree, otherwise either the Rust consumer over-limits
// (idle worker capacity) or the worker stalls stream reads at capacity.
INGESTION_WORKER_CONCURRENT_BATCHES: number
// Feed-order sentinel (ingestion API server only): checks that each
// routing key's messages enter the pipeline in Kafka offset order. The
// Rust consumer's sentinels have their own flag
// (CONSUMER_ORDER_SENTINEL_ENABLED).
INGESTION_API_FEED_ORDER_SENTINEL_ENABLED: boolean
// LRU capacity of the sentinel's per-key state; at capacity the
// least-recently-seen key is dropped and rebaselines unchecked.
INGESTION_API_FEED_ORDER_SENTINEL_MAX_KEYS: number
// Streaming ingest (ingestion API server only): the port serving
// ingestion.worker.v1.WorkerIngest. The stream delivers each consumer's
// sub-batches in order.
INGESTION_API_GRPC_PORT: number
// Concurrency caps for the gRPC listener. Legitimate load is roughly one
// stream per connected consumer, so these are generous ceilings that bound
// resource use if a peer misbehaves rather than limits normal traffic.
INGESTION_API_GRPC_MAX_STREAMS: number
INGESTION_API_GRPC_MAX_SESSIONS: number
INGESTION_API_GRPC_MAX_STREAMS_PER_SESSION: number
INGESTION_API_GRPC_SESSION_MEMORY_MB: number
INGESTION_API_GRPC_SESSION_IDLE_TIMEOUT_MS: number
INGESTION_API_GRPC_READ_MAX_BYTES: number
INGESTION_API_GRPC_DRAIN_TIMEOUT_MS: number
// Person batch writing config
PERSON_BATCH_WRITING_DB_WRITE_MODE: PersonBatchWritingDbWriteMode
PERSON_BATCH_WRITING_USE_BATCH_UPDATES: boolean
PERSON_BATCH_WRITING_OPTIMISTIC_UPDATES_ENABLED: boolean
PERSON_BATCH_WRITING_MAX_CONCURRENT_UPDATES: number
PERSON_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES: number
PERSON_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS: number
// Teams whose batch person writes merge per key with the row. Comma-separated team IDs, or '*'; empty means none.
PERSON_BATCH_WRITING_PER_KEY_TEAM_ALLOWLIST: string
/** Concurrent RPC fan-out in the personhog store (batch fetches and flush). */
PERSONHOG_STORE_MAX_CONCURRENT_UPDATES: number
PERSONS_PREFETCH_ENABLED: boolean
// Person properties config
PERSON_UPDATE_CALCULATE_PROPERTIES_SIZE: number
PERSON_PROPERTIES_DB_CONSTRAINT_LIMIT_BYTES: number
PERSON_PROPERTIES_TRIM_TARGET_BYTES: number
PERSON_PROPERTIES_UPDATE_ALL: boolean
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: number
// Person merge config
PERSON_MERGE_MOVE_DISTINCT_ID_LIMIT: number
PERSON_MERGE_ASYNC_TOPIC: string
PERSON_MERGE_ASYNC_ENABLED: boolean
PERSON_MERGE_SYNC_BATCH_SIZE: number
// The saga's per-source move guard in SYNC mode; an over-limit source
// comes back skipped_move_limit for the merge-mode policy.
PERSONHOG_SYNC_MERGE_MOVE_LIMIT: number
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: boolean
// Re-send budget per batch for merges refused at admission, measured from the batch's
// first refusal and shared by every merge and step retry in it. Keep it well under
// CONSUMER_LOOP_STALL_THRESHOLD_MS, which the whole batch has to fit in.
PERSON_MERGE_NOT_APPLIED_WAIT_MS: number
// Kill switch for emitting person_merge_events to the cohort-stream-processor.
// Enable ordering: (1) create the topic, (2) set INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC
// (startup topic verification is then fatal by design), (3) flip this on. Flipping this on before
// the topic env is set is a no-op — see effectivePersonMergeEventsEnabled.
PERSON_MERGE_EVENTS_ENABLED: boolean
// Must equal the person_merge_events topic partition count and the Rust COHORT_PARTITION_COUNT.
PERSON_MERGE_EVENTS_PARTITION_COUNT: number
// Which teams to emit person_merge_events for: comma-separated team IDs, or '*' for all teams.
// Defaults to team 2 only. Unlike the Rust REALTIME_COHORT_TEAM_ALLOWLIST, an empty value here
// means "no teams", not "all teams"; use '*' to open the gate.
PERSON_MERGE_EVENTS_TEAM_ALLOWLIST: string
// Fold consecutive runs of $identify merges for the same distinct_id in a batch into a single
// merge operation (merge-storm mitigation). Master switch; when off, the planning step passes
// every event through unplanned and merges stay sequential.
PERSON_MERGE_FOLD_ENABLED: boolean
// Teams eligible for merge folding: comma-separated team IDs, or '*' for all teams.
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: string
// Teams whose merges lock the person rows and write the survivor in the transaction; other teams queue it for
// the next flush. Enable after PERSON_BATCH_WRITING_PER_KEY_TEAM_ALLOWLIST. Team IDs, or '*'; empty means none.
PERSON_MERGE_LOCKED_OUTCOME_TEAM_ALLOWLIST: string
// Re-emit committed distinct id mappings for merge events that arrive already satisfied,
// debounced per (team, distinct id). Heals ClickHouse mapping rows lost to a crash between
// a merge's commit and its produce; see MergeMappingDebounce for why the cache is in-memory.
PERSON_MERGE_NOOP_MAPPING_EMISSION_ENABLED: boolean
PERSON_MERGE_NOOP_MAPPING_EMISSION_CACHE_SIZE: number
PERSON_MERGE_NOOP_MAPPING_EMISSION_TTL_MS: number
// Group batch writing config
GROUP_BATCH_WRITING_USE_BATCH_UPDATES: boolean
// Defer creation of new groups to flush time and insert them in a single
// batched statement, instead of an inline single-row insert per new group
// during event processing. When off, behavior is unchanged.
GROUP_BATCH_WRITING_USE_BATCH_CREATES: boolean
GROUP_BATCH_WRITING_MAX_CONCURRENT_UPDATES: number
GROUP_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES: number
GROUP_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS: number
GROUPS_PREFETCH_ENABLED: boolean
// Team-keyed cache prefetch config: one batched warm-up per chunk for each cache,
// instead of a per-event lookup in the sequential steps that read it.
TEAMS_PREFETCH_ENABLED: boolean
EVENT_SCHEMAS_PREFETCH_ENABLED: boolean
HOG_FUNCTIONS_PREFETCH_ENABLED: boolean
// Event overflow config
EVENT_OVERFLOW_BUCKET_CAPACITY: number
EVENT_OVERFLOW_BUCKET_REPLENISH_RATE: number
// Merge-event ($identify, $create_alias, $merge_dangerously) overflow rate,
// per token:distinct_id. A capacity of 0 disables the condition.
MERGE_EVENT_OVERFLOW_BUCKET_CAPACITY: number
MERGE_EVENT_OVERFLOW_BUCKET_REPLENISH_RATE: number
// Stateful overflow config
INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS: number
INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS: number
// Per-token/distinct_id restrictions
DROP_EVENTS_BY_TOKEN_DISTINCT_ID: string
SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID: string
MAX_TEAM_ID_TO_BUFFER_ANONYMOUS_EVENTS_FOR: number
// Pipeline step config
SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP: boolean
EVENT_SCHEMA_ENFORCEMENT_ENABLED: boolean
KAFKA_BATCH_START_LOGGING_ENABLED: boolean
/** Teams whose $feature_flag_called events default to personless: '*' for all, '' to disable, or comma-separated team IDs */
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: string
/** Teams held back from the personless default even when the allowlist is '*': '' for none, '*' to disable the default for every team, or comma-separated team IDs */
FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS: string
/** Teams whose multivariate $feature_flag_called events are duplicated as $experiment_exposure: '*' for all, '' to disable, or comma-separated team IDs */
EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS: string
// $feature_flag_called fork into the flag_evaluations ClickHouse table
/** 'disabled' | 'dual_write' (produce to flag_evaluations). With dual_write the event also continues to events unless its organization is on FLAG_EVALUATIONS_ONLY. */
INGESTION_FLAG_EVALUATIONS_MODE: string
/** '*' for all teams, or comma-separated team IDs */
INGESTION_FLAG_EVALUATIONS_TEAMS: string
/** Comma-separated team IDs never forked, even when TEAMS is '*' */
INGESTION_FLAG_EVALUATIONS_EXCLUDED_TEAMS: string
/** Rollback for FLAG_EVALUATIONS_ONLY. True keeps writing $feature_flag_called to events for organizations on that mode, as if they were on READ_FLAG_EVALUATIONS. */
INGESTION_FLAG_EVALUATIONS_ONLY_DISABLED: boolean
// $feature_flag_called keep-first dedup config
/** 'disabled' | 'shadow' (claim + count, never drop) | 'drop' */
INGESTION_FEATURE_FLAG_CALLED_DEDUP_MODE: string
/** '*' for all teams, or comma-separated team IDs */
INGESTION_FEATURE_FLAG_CALLED_DEDUP_TEAMS: string
/** Comma-separated team IDs never deduped, even when TEAMS is '*' */
INGESTION_FEATURE_FLAG_CALLED_DEDUP_EXCLUDED_TEAMS: string
/** Claim TTL: the keep-first dedup window */
INGESTION_FEATURE_FLAG_CALLED_DEDUP_TTL_SECONDS: number
/** Dedicated Redis host for dedup claims; empty reuses the ingestion Redis */
INGESTION_FEATURE_FLAG_CALLED_DEDUP_REDIS_HOST: string
INGESTION_FEATURE_FLAG_CALLED_DEDUP_REDIS_PORT: number
// Clickhouse topics
CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC: string
CLICKHOUSE_AI_EVENTS_KAFKA_TOPIC: string
CLICKHOUSE_HEATMAPS_KAFKA_TOPIC: string
// AI blob offload: content-addressed S3 storage for multimodal payloads.
// Empty bucket or empty teams list disables the offload step entirely.
AI_BLOB_S3_BUCKET: string
AI_BLOB_S3_PREFIX: string
AI_BLOB_S3_ENDPOINT: string
AI_BLOB_S3_REGION: string
AI_BLOB_S3_ACCESS_KEY_ID: string
AI_BLOB_S3_SECRET_ACCESS_KEY: string
AI_BLOB_S3_TIMEOUT_MS: number
AI_BLOB_OFFLOAD_TEAMS: string
AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH: number
AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT: number
AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY: number
AI_BLOB_OFFLOAD_TOUCH_AFTER_HOURS: number
// Cookieless server hash mode config
COOKIELESS_DISABLED: boolean
COOKIELESS_DELETE_EXPIRED_LOCAL_SALTS_INTERVAL_MS: number
COOKIELESS_SESSION_TTL_SECONDS: number
COOKIELESS_SALT_TTL_SECONDS: number
COOKIELESS_SESSION_INACTIVITY_MS: number
COOKIELESS_IDENTIFIES_TTL_SECONDS: number
COOKIELESS_REDIS_HOST: string
COOKIELESS_REDIS_PORT: number
// Property definitions
PROPERTY_DEFS_CONSUMER_GROUP_ID: string
PROPERTY_DEFS_CONSUMER_CONSUME_TOPIC: string
PROPERTY_DEFS_CONSUMER_ENABLED_TEAMS: string
PROPERTY_DEFS_WRITE_DISABLED: boolean
// Ingestion caching
DISTINCT_ID_LRU_SIZE: number
EVENT_PROPERTY_LRU_SIZE: number
PERSON_INFO_CACHE_TTL: number
// Ingestion pipeline
INGESTION_PIPELINE: string | null
PLUGIN_SERVER_EVENTS_INGESTION_PIPELINE: string | null
}
export function getDefaultIngestionConsumerConfig(): IngestionConsumerConfig {
return {
INGESTION_LANE: null,
INGESTION_OVERFLOW_MODE: 'disabled',
// Kafka consumer config
INGESTION_CONSUMER_GROUP_ID: 'events-ingestion-consumer',
INGESTION_CONSUMER_CONSUME_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION,
INGESTION_CONSUMER_DLQ_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION_DLQ,
INGESTION_CONSUMER_OVERFLOW_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW,
// Ingestion pipeline config
INGESTION_BATCH_SIZE: 500,
INGESTION_OVERFLOW_ENABLED: false,
INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID: '',
INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY: false,
INGESTION_WORKER_CONCURRENT_BATCHES: 1,
INGESTION_API_FEED_ORDER_SENTINEL_ENABLED: true,
INGESTION_API_FEED_ORDER_SENTINEL_MAX_KEYS: 200_000,
INGESTION_API_GRPC_PORT: 6739,
INGESTION_API_GRPC_MAX_STREAMS: 256,
INGESTION_API_GRPC_MAX_SESSIONS: 256,
INGESTION_API_GRPC_MAX_STREAMS_PER_SESSION: 8,
INGESTION_API_GRPC_SESSION_MEMORY_MB: 64,
INGESTION_API_GRPC_SESSION_IDLE_TIMEOUT_MS: 300_000,
INGESTION_API_GRPC_READ_MAX_BYTES: 32 * 1024 * 1024,
INGESTION_API_GRPC_DRAIN_TIMEOUT_MS: 15_000,
// Person batch writing config
PERSON_BATCH_WRITING_DB_WRITE_MODE: 'NO_ASSERT',
PERSON_BATCH_WRITING_USE_BATCH_UPDATES: true,
PERSON_BATCH_WRITING_OPTIMISTIC_UPDATES_ENABLED: false,
PERSON_BATCH_WRITING_MAX_CONCURRENT_UPDATES: 10,
PERSON_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES: 5,
PERSON_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS: 50,
PERSON_BATCH_WRITING_PER_KEY_TEAM_ALLOWLIST: '',
PERSONHOG_STORE_MAX_CONCURRENT_UPDATES: 10,
PERSONS_PREFETCH_ENABLED: false,
// Person properties config
PERSON_UPDATE_CALCULATE_PROPERTIES_SIZE: 0,
PERSON_PROPERTIES_DB_CONSTRAINT_LIMIT_BYTES: 655360,
PERSON_PROPERTIES_TRIM_TARGET_BYTES: 512 * 1024,
PERSON_PROPERTIES_UPDATE_ALL: false,
PERSON_JSONB_SIZE_ESTIMATE_ENABLE: 0,
// Person merge config
PERSON_MERGE_MOVE_DISTINCT_ID_LIMIT: 0,
PERSON_MERGE_ASYNC_TOPIC: '',
PERSON_MERGE_ASYNC_ENABLED: false,
PERSON_MERGE_SYNC_BATCH_SIZE: 0,
PERSONHOG_SYNC_MERGE_MOVE_LIMIT: 10_000,
PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: false,
PERSON_MERGE_NOT_APPLIED_WAIT_MS: 20_000,
PERSON_MERGE_EVENTS_ENABLED: false,
PERSON_MERGE_EVENTS_PARTITION_COUNT: 64,
PERSON_MERGE_EVENTS_TEAM_ALLOWLIST: '2',
PERSON_MERGE_FOLD_ENABLED: false,
PERSON_MERGE_FOLD_TEAM_ALLOWLIST: '*',
PERSON_MERGE_LOCKED_OUTCOME_TEAM_ALLOWLIST: '',
PERSON_MERGE_NOOP_MAPPING_EMISSION_ENABLED: false,
PERSON_MERGE_NOOP_MAPPING_EMISSION_CACHE_SIZE: 500_000,
PERSON_MERGE_NOOP_MAPPING_EMISSION_TTL_MS: 60 * 60 * 1000,
// Group batch writing config
GROUP_BATCH_WRITING_USE_BATCH_UPDATES: true,
GROUP_BATCH_WRITING_USE_BATCH_CREATES: false,
GROUP_BATCH_WRITING_MAX_CONCURRENT_UPDATES: 10,
GROUP_BATCH_WRITING_MAX_OPTIMISTIC_UPDATE_RETRIES: 5,
GROUP_BATCH_WRITING_OPTIMISTIC_UPDATE_RETRY_INTERVAL_MS: 50,
GROUPS_PREFETCH_ENABLED: false,
TEAMS_PREFETCH_ENABLED: false,
EVENT_SCHEMAS_PREFETCH_ENABLED: false,
HOG_FUNCTIONS_PREFETCH_ENABLED: false,
// Event overflow config
EVENT_OVERFLOW_BUCKET_CAPACITY: 1000,
EVENT_OVERFLOW_BUCKET_REPLENISH_RATE: 1.0,
MERGE_EVENT_OVERFLOW_BUCKET_CAPACITY: 0,
MERGE_EVENT_OVERFLOW_BUCKET_REPLENISH_RATE: 1.0,
// Stateful overflow config
INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS: 300,
INGESTION_STATEFUL_OVERFLOW_LOCAL_CACHE_TTL_SECONDS: 60,
// Per-token/distinct_id restrictions
DROP_EVENTS_BY_TOKEN_DISTINCT_ID: '',
SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID: '',
MAX_TEAM_ID_TO_BUFFER_ANONYMOUS_EVENTS_FOR: 0,
// Pipeline step config
SKIP_UPDATE_EVENT_AND_PROPERTIES_STEP: false,
EVENT_SCHEMA_ENFORCEMENT_ENABLED: true,
KAFKA_BATCH_START_LOGGING_ENABLED: false,
FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: DEFAULT_FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,
FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS: DEFAULT_FLAG_CALLED_PERSONLESS_EXCLUDED_TEAMS,
EXPERIMENT_EXPOSURE_DUPLICATION_TEAMS: '',
// $feature_flag_called fork into the flag_evaluations ClickHouse table.
// Teams default empty so flipping the mode alone forks nobody; ramp by
// naming teams (or '*') in INGESTION_FLAG_EVALUATIONS_TEAMS.
//
// On a split lane the fork runs in the Node processor pool, not in the
// Rust consumer that dispatches to it, so these belong on the lane's base
// values.yaml, which both roles inherit. Setting them on the consumer
// release alone does nothing.
INGESTION_FLAG_EVALUATIONS_MODE: 'disabled',
INGESTION_FLAG_EVALUATIONS_TEAMS: '',
INGESTION_FLAG_EVALUATIONS_EXCLUDED_TEAMS: '',
INGESTION_FLAG_EVALUATIONS_ONLY_DISABLED: false,
// $feature_flag_called keep-first dedup config
INGESTION_FEATURE_FLAG_CALLED_DEDUP_MODE: 'disabled',
INGESTION_FEATURE_FLAG_CALLED_DEDUP_TEAMS: '',
INGESTION_FEATURE_FLAG_CALLED_DEDUP_EXCLUDED_TEAMS: '',
INGESTION_FEATURE_FLAG_CALLED_DEDUP_TTL_SECONDS: 60 * 60,
INGESTION_FEATURE_FLAG_CALLED_DEDUP_REDIS_HOST: '',
INGESTION_FEATURE_FLAG_CALLED_DEDUP_REDIS_PORT: 6379,
// Clickhouse topics
CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC: KAFKA_EVENTS_JSON,
CLICKHOUSE_AI_EVENTS_KAFKA_TOPIC: KAFKA_CLICKHOUSE_AI_EVENTS_JSON,
CLICKHOUSE_HEATMAPS_KAFKA_TOPIC: KAFKA_CLICKHOUSE_HEATMAP_EVENTS,
// AI blob offload: content-addressed S3 storage for multimodal payloads.
// Empty bucket or empty teams list disables the offload step entirely.
AI_BLOB_S3_BUCKET: '',
// Bucket+prefix are a shared contract with the Django read side (posthog/settings/
// object_storage.py) — defaults must agree or reads 404 while writes succeed.
AI_BLOB_S3_PREFIX: 'aio/',
AI_BLOB_S3_ENDPOINT: '',
AI_BLOB_S3_REGION: 'us-east-1',
AI_BLOB_S3_ACCESS_KEY_ID: '',
AI_BLOB_S3_SECRET_ACCESS_KEY: '',
AI_BLOB_S3_TIMEOUT_MS: 30000,
AI_BLOB_OFFLOAD_TEAMS: '',
// Keep the floor above base64-packed embedding vectors so logged embeddings stay inline as text.
// The shortest base64 that decodes to AI_MEDIA_OFFLOAD_MIN_BYTES, so no media of that size stays inline.
AI_BLOB_OFFLOAD_MIN_BASE64_LENGTH: Math.ceil(AI_MEDIA_OFFLOAD_MIN_BYTES / 3) * 4,
AI_BLOB_OFFLOAD_MAX_BLOBS_PER_EVENT: 50,
// Chunk-wide cap on concurrent blob uploads, so blob-heavy traffic
// can't monopolize the S3 socket pool.
AI_BLOB_OFFLOAD_UPLOAD_MAX_CONCURRENCY: 8,
AI_BLOB_OFFLOAD_TOUCH_AFTER_HOURS: 20,
// Cookieless server hash mode config
COOKIELESS_DISABLED: false,
COOKIELESS_DELETE_EXPIRED_LOCAL_SALTS_INTERVAL_MS: 60 * 60 * 1000,
COOKIELESS_SESSION_TTL_SECONDS: 60 * 60 * (72 + 24),
COOKIELESS_SALT_TTL_SECONDS: 60 * 60 * (72 + 24),
COOKIELESS_SESSION_INACTIVITY_MS: 30 * 60 * 1000,
COOKIELESS_IDENTIFIES_TTL_SECONDS: (72 + 12 + 14 + 24) * 60 * 60,
COOKIELESS_REDIS_HOST: '',
COOKIELESS_REDIS_PORT: 6379,
// Property definitions
PROPERTY_DEFS_CONSUMER_GROUP_ID: 'property-defs-consumer',
PROPERTY_DEFS_CONSUMER_CONSUME_TOPIC: KAFKA_EVENTS_JSON,
PROPERTY_DEFS_CONSUMER_ENABLED_TEAMS: isDevEnv() ? '*' : '',
PROPERTY_DEFS_WRITE_DISABLED: isProdEnv() ? true : false,
// Ingestion caching
DISTINCT_ID_LRU_SIZE: 10000,
EVENT_PROPERTY_LRU_SIZE: 10000,
PERSON_INFO_CACHE_TTL: 5 * 60,
// Ingestion pipeline
INGESTION_PIPELINE: null,
PLUGIN_SERVER_EVENTS_INGESTION_PIPELINE: null,
}
}
/** Config type for all analytics ingestion output keys. */
export type IngestionOutputsConfig = {
INGESTION_OUTPUT_EVENTS_TOPIC: string
INGESTION_OUTPUT_EVENTS_PRODUCER: ProducerName
INGESTION_OUTPUT_AI_EVENTS_TOPIC: string
INGESTION_OUTPUT_AI_EVENTS_PRODUCER: ProducerName
INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: string
INGESTION_OUTPUT_FLAG_EVALUATIONS_PRODUCER: ProducerName
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: string
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_PRODUCER: ProducerName
INGESTION_OUTPUT_HEATMAPS_TOPIC: string
INGESTION_OUTPUT_HEATMAPS_PRODUCER: ProducerName
INGESTION_OUTPUT_INGESTION_WARNINGS_TOPIC: string
INGESTION_OUTPUT_INGESTION_WARNINGS_PRODUCER: ProducerName
INGESTION_OUTPUT_DLQ_TOPIC: string
INGESTION_OUTPUT_DLQ_PRODUCER: ProducerName
INGESTION_OUTPUT_OVERFLOW_TOPIC: string
INGESTION_OUTPUT_OVERFLOW_PRODUCER: ProducerName
INGESTION_OUTPUT_ASYNC_TOPIC: string
INGESTION_OUTPUT_ASYNC_PRODUCER: ProducerName
INGESTION_OUTPUT_GROUPS_TOPIC: string
INGESTION_OUTPUT_GROUPS_PRODUCER: ProducerName
INGESTION_OUTPUT_PERSONS_TOPIC: string
INGESTION_OUTPUT_PERSONS_PRODUCER: ProducerName
INGESTION_OUTPUT_PERSON_DISTINCT_IDS_TOPIC: string
INGESTION_OUTPUT_PERSON_DISTINCT_IDS_PRODUCER: ProducerName
INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC: string
INGESTION_OUTPUT_PERSON_MERGE_EVENTS_PRODUCER: ProducerName
INGESTION_OUTPUT_APP_METRICS_TOPIC: string
INGESTION_OUTPUT_APP_METRICS_PRODUCER: ProducerName
INGESTION_OUTPUT_LOG_ENTRIES_TOPIC: string
INGESTION_OUTPUT_LOG_ENTRIES_PRODUCER: ProducerName
INGESTION_OUTPUT_TOPHOG_TOPIC: string
INGESTION_OUTPUT_TOPHOG_PRODUCER: ProducerName
INGESTION_OUTPUTS_DISABLED: boolean
}
export function getDefaultIngestionOutputsConfig(): IngestionOutputsConfig {
return {
INGESTION_OUTPUT_EVENTS_TOPIC: KAFKA_EVENTS_JSON,
INGESTION_OUTPUT_EVENTS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_AI_EVENTS_TOPIC: KAFKA_CLICKHOUSE_AI_EVENTS_JSON,
INGESTION_OUTPUT_AI_EVENTS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
// Empty topic skips the startup topic-existence check, so deploying this dark cannot
// crash-loop a fleet whose broker lacks the topic. Enable ordering: (1) create the
// clickhouse_flag_evaluations topic, (2) set INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC
// (startup topic verification is then fatal by design), (3) set
// INGESTION_FLAG_EVALUATIONS_MODE. Mode without topic is a no-op; see
// createFlagEvaluationsService.
INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC: '',
INGESTION_OUTPUT_FLAG_EVALUATIONS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
// An empty topic skips the startup topic-existence check. While it is empty, a
// FLAG_EVALUATIONS_ONLY team's flag calls reach no realtime destination, so set it
// on every lane that sets INGESTION_OUTPUT_FLAG_EVALUATIONS_TOPIC. Enable
// ordering: (1) create the realtime_only_events_json topic, (2) start the CDP
// consumers that read it, (3) set INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC. A
// consumer group that joins after step 3 can start past the first routed calls.
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_TOPIC: '',
INGESTION_OUTPUT_REALTIME_ONLY_EVENTS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_HEATMAPS_TOPIC: KAFKA_CLICKHOUSE_HEATMAP_EVENTS,
INGESTION_OUTPUT_HEATMAPS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_INGESTION_WARNINGS_TOPIC: KAFKA_INGESTION_WARNINGS,
INGESTION_OUTPUT_INGESTION_WARNINGS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_DLQ_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION_DLQ,
INGESTION_OUTPUT_DLQ_PRODUCER: INGESTION_UPSTREAM_PRODUCER,
INGESTION_OUTPUT_OVERFLOW_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION_OVERFLOW,
INGESTION_OUTPUT_OVERFLOW_PRODUCER: INGESTION_UPSTREAM_PRODUCER,
INGESTION_OUTPUT_ASYNC_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION_ASYNC,
INGESTION_OUTPUT_ASYNC_PRODUCER: INGESTION_UPSTREAM_PRODUCER,
INGESTION_OUTPUT_GROUPS_TOPIC: KAFKA_GROUPS,
INGESTION_OUTPUT_GROUPS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_PERSONS_TOPIC: KAFKA_PERSON,
INGESTION_OUTPUT_PERSONS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_PERSON_DISTINCT_IDS_TOPIC: KAFKA_PERSON_DISTINCT_ID,
INGESTION_OUTPUT_PERSON_DISTINCT_IDS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
// Empty topic skips the startup topic-existence check.
INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC: '',
INGESTION_OUTPUT_PERSON_MERGE_EVENTS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_APP_METRICS_TOPIC: KAFKA_APP_METRICS_2,
INGESTION_OUTPUT_APP_METRICS_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_LOG_ENTRIES_TOPIC: KAFKA_LOG_ENTRIES,
INGESTION_OUTPUT_LOG_ENTRIES_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUT_TOPHOG_TOPIC: KAFKA_CLICKHOUSE_TOPHOG,
INGESTION_OUTPUT_TOPHOG_PRODUCER: INGESTION_DOWNSTREAM_PRODUCER,
INGESTION_OUTPUTS_DISABLED: false,
}
}