Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.8 KB
Raw Blame
/**
 * CDP producer names + their typed env-var config.
 *
 * Each producer is named explicitly after the cluster it targets, and its env
 * vars share the same prefix so deployment config is unambiguous (e.g.
 * `WARPSTREAM_INGESTION_PRODUCER` ↔ `KAFKA_WARPSTREAM_INGESTION_PRODUCER_*`).
 */
import { AllowedConfigKey } from '~/common/outputs/kafka-producer-config'

/** Targets the shared Warpstream cluster used by ingestion. Current default for CDP. */
export const WARPSTREAM_INGESTION_PRODUCER = 'WARPSTREAM_INGESTION_PRODUCER' as const
export type WarpstreamIngestionProducer = typeof WARPSTREAM_INGESTION_PRODUCER

/** Targets the warehouse webhook Kafka cluster (`KAFKA_WAREHOUSE_PRODUCER_*`). */
export const WAREHOUSE_PRODUCER = 'WAREHOUSE_PRODUCER' as const
export type WarehouseProducer = typeof WAREHOUSE_PRODUCER

/**
 * Targets the Warpstream Cyclotron cluster for outputs (batch hogflow requests).
 * Distinct from the legacy `KAFKA_CDP_PRODUCER_*` env vars used by the cyclotron
 * job queue itself — kept separate so the cluster-named convention holds across
 * all outputs in the CDP registry.
 */
export const WARPSTREAM_CYCLOTRON_PRODUCER = 'WARPSTREAM_CYCLOTRON_PRODUCER' as const
export type WarpstreamCyclotronProducer = typeof WARPSTREAM_CYCLOTRON_PRODUCER

/** Producer names registered by the CDP deployments. */
export type CdpProducerName = WarpstreamIngestionProducer | WarehouseProducer | WarpstreamCyclotronProducer

/** Cluster-named env var prefix mirroring the producer it configures. */
export const WARPSTREAM_INGESTION_PRODUCER_CONFIG_MAP = {
    'client.id': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID',
    'metadata.broker.list': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST',
    'security.protocol': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL',
    'sasl.mechanisms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS',
    'sasl.username': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME',
    'sasl.password': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD',
    'compression.codec': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC',
    'linger.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS',
    'batch.size': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE',
    'queue.buffering.max.messages': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
    'queue.buffering.max.kbytes': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
    'enable.ssl.certificate.verification': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION',
    'enable.idempotence': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE',
    'message.max.bytes': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES',
    'batch.num.messages': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES',
    'sticky.partitioning.linger.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS',
    'topic.metadata.refresh.interval.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS',
    'metadata.max.age.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS',
    'message.send.max.retries': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES',
    'max.in.flight.requests.per.connection':
        'KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION',
} as const satisfies Partial<Record<AllowedConfigKey, string>>

/** Typed env vars referenced by `WARPSTREAM_INGESTION_PRODUCER_CONFIG_MAP`. */
export type KafkaWarpstreamIngestionProducerEnvConfig = {
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: string
}

export function getDefaultKafkaWarpstreamIngestionProducerEnvConfig(): KafkaWarpstreamIngestionProducerEnvConfig {
    return {
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: '',
    }
}

/**
 * Warehouse-webhook producer. Targets the dedicated cluster for warehouse source
 * webhooks — matches the env-var shape the old `KafkaProducerWrapper.create(rack,
 * 'WAREHOUSE_PRODUCER')` path already read via `getKafkaConfigFromEnv`.
 */
export const WAREHOUSE_PRODUCER_CONFIG_MAP = {
    'client.id': 'KAFKA_WAREHOUSE_PRODUCER_CLIENT_ID',
    'metadata.broker.list': 'KAFKA_WAREHOUSE_PRODUCER_METADATA_BROKER_LIST',
    'security.protocol': 'KAFKA_WAREHOUSE_PRODUCER_SECURITY_PROTOCOL',
    'sasl.mechanisms': 'KAFKA_WAREHOUSE_PRODUCER_SASL_MECHANISMS',
    'sasl.username': 'KAFKA_WAREHOUSE_PRODUCER_SASL_USERNAME',
    'sasl.password': 'KAFKA_WAREHOUSE_PRODUCER_SASL_PASSWORD',
    'compression.codec': 'KAFKA_WAREHOUSE_PRODUCER_COMPRESSION_CODEC',
    'linger.ms': 'KAFKA_WAREHOUSE_PRODUCER_LINGER_MS',
    'batch.size': 'KAFKA_WAREHOUSE_PRODUCER_BATCH_SIZE',
    'queue.buffering.max.messages': 'KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
    'queue.buffering.max.kbytes': 'KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
    'enable.ssl.certificate.verification': 'KAFKA_WAREHOUSE_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION',
    'enable.idempotence': 'KAFKA_WAREHOUSE_PRODUCER_ENABLE_IDEMPOTENCE',
    'message.max.bytes': 'KAFKA_WAREHOUSE_PRODUCER_MESSAGE_MAX_BYTES',
    'batch.num.messages': 'KAFKA_WAREHOUSE_PRODUCER_BATCH_NUM_MESSAGES',
    'sticky.partitioning.linger.ms': 'KAFKA_WAREHOUSE_PRODUCER_STICKY_PARTITIONING_LINGER_MS',
    'topic.metadata.refresh.interval.ms': 'KAFKA_WAREHOUSE_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS',
    'metadata.max.age.ms': 'KAFKA_WAREHOUSE_PRODUCER_METADATA_MAX_AGE_MS',
    'message.send.max.retries': 'KAFKA_WAREHOUSE_PRODUCER_RETRIES',
    'max.in.flight.requests.per.connection': 'KAFKA_WAREHOUSE_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION',
} as const satisfies Partial<Record<AllowedConfigKey, string>>

/** Typed env vars referenced by `WAREHOUSE_PRODUCER_CONFIG_MAP`. */
export type KafkaWarehouseProducerEnvConfig = {
    KAFKA_WAREHOUSE_PRODUCER_CLIENT_ID: string
    KAFKA_WAREHOUSE_PRODUCER_METADATA_BROKER_LIST: string
    KAFKA_WAREHOUSE_PRODUCER_SECURITY_PROTOCOL: string
    KAFKA_WAREHOUSE_PRODUCER_SASL_MECHANISMS: string
    KAFKA_WAREHOUSE_PRODUCER_SASL_USERNAME: string
    KAFKA_WAREHOUSE_PRODUCER_SASL_PASSWORD: string
    KAFKA_WAREHOUSE_PRODUCER_COMPRESSION_CODEC: string
    KAFKA_WAREHOUSE_PRODUCER_LINGER_MS: string
    KAFKA_WAREHOUSE_PRODUCER_BATCH_SIZE: string
    KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
    KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
    KAFKA_WAREHOUSE_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: string
    KAFKA_WAREHOUSE_PRODUCER_ENABLE_IDEMPOTENCE: string
    KAFKA_WAREHOUSE_PRODUCER_MESSAGE_MAX_BYTES: string
    KAFKA_WAREHOUSE_PRODUCER_BATCH_NUM_MESSAGES: string
    KAFKA_WAREHOUSE_PRODUCER_STICKY_PARTITIONING_LINGER_MS: string
    KAFKA_WAREHOUSE_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: string
    KAFKA_WAREHOUSE_PRODUCER_METADATA_MAX_AGE_MS: string
    KAFKA_WAREHOUSE_PRODUCER_RETRIES: string
    KAFKA_WAREHOUSE_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: string
}

export function getDefaultKafkaWarehouseProducerEnvConfig(): KafkaWarehouseProducerEnvConfig {
    return {
        KAFKA_WAREHOUSE_PRODUCER_CLIENT_ID: '',
        KAFKA_WAREHOUSE_PRODUCER_METADATA_BROKER_LIST: '',
        KAFKA_WAREHOUSE_PRODUCER_SECURITY_PROTOCOL: '',
        KAFKA_WAREHOUSE_PRODUCER_SASL_MECHANISMS: '',
        KAFKA_WAREHOUSE_PRODUCER_SASL_USERNAME: '',
        KAFKA_WAREHOUSE_PRODUCER_SASL_PASSWORD: '',
        KAFKA_WAREHOUSE_PRODUCER_COMPRESSION_CODEC: '',
        KAFKA_WAREHOUSE_PRODUCER_LINGER_MS: '',
        KAFKA_WAREHOUSE_PRODUCER_BATCH_SIZE: '',
        KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '',
        KAFKA_WAREHOUSE_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '',
        KAFKA_WAREHOUSE_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: '',
        KAFKA_WAREHOUSE_PRODUCER_ENABLE_IDEMPOTENCE: '',
        KAFKA_WAREHOUSE_PRODUCER_MESSAGE_MAX_BYTES: '',
        KAFKA_WAREHOUSE_PRODUCER_BATCH_NUM_MESSAGES: '',
        KAFKA_WAREHOUSE_PRODUCER_STICKY_PARTITIONING_LINGER_MS: '',
        KAFKA_WAREHOUSE_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: '',
        KAFKA_WAREHOUSE_PRODUCER_METADATA_MAX_AGE_MS: '',
        KAFKA_WAREHOUSE_PRODUCER_RETRIES: '',
        KAFKA_WAREHOUSE_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: '',
    }
}

/**
 * Warpstream Cyclotron output producer. Distinct env-var prefix from the legacy
 * `KAFKA_CDP_PRODUCER_*` so output routing is decoupled from the cyclotron job
 * queue's own producer config.
 */
export const WARPSTREAM_CYCLOTRON_PRODUCER_CONFIG_MAP = {
    'client.id': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_CLIENT_ID',
    'metadata.broker.list': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_BROKER_LIST',
    'security.protocol': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_SECURITY_PROTOCOL',
    'compression.codec': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_COMPRESSION_CODEC',
    'linger.ms': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_LINGER_MS',
    'batch.size': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_SIZE',
    'queue.buffering.max.messages': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
    'queue.buffering.max.kbytes': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
    'enable.idempotence': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_ENABLE_IDEMPOTENCE',
    'message.max.bytes': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MESSAGE_MAX_BYTES',
    'batch.num.messages': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_NUM_MESSAGES',
    'sticky.partitioning.linger.ms': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_STICKY_PARTITIONING_LINGER_MS',
    'topic.metadata.refresh.interval.ms': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS',
    'metadata.max.age.ms': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_MAX_AGE_MS',
    'message.send.max.retries': 'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_RETRIES',
    'max.in.flight.requests.per.connection':
        'KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION',
} as const satisfies Partial<Record<AllowedConfigKey, string>>

/** Typed env vars referenced by `WARPSTREAM_CYCLOTRON_PRODUCER_CONFIG_MAP`. */
export type KafkaWarpstreamCyclotronProducerEnvConfig = {
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_CLIENT_ID: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_BROKER_LIST: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_SECURITY_PROTOCOL: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_COMPRESSION_CODEC: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_LINGER_MS: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_SIZE: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_ENABLE_IDEMPOTENCE: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MESSAGE_MAX_BYTES: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_NUM_MESSAGES: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_STICKY_PARTITIONING_LINGER_MS: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_MAX_AGE_MS: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_RETRIES: string
    KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: string
}

export function getDefaultKafkaWarpstreamCyclotronProducerEnvConfig(): KafkaWarpstreamCyclotronProducerEnvConfig {
    return {
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_CLIENT_ID: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_BROKER_LIST: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_SECURITY_PROTOCOL: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_COMPRESSION_CODEC: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_LINGER_MS: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_SIZE: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_ENABLE_IDEMPOTENCE: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MESSAGE_MAX_BYTES: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_BATCH_NUM_MESSAGES: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_STICKY_PARTITIONING_LINGER_MS: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_METADATA_MAX_AGE_MS: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_RETRIES: '',
        KAFKA_WARPSTREAM_CYCLOTRON_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: '',
    }
}