Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.5 KB
Raw Blame
/**
 * Logs/traces ingestion producer names + their typed env-var config.
 *
 * Each producer is named explicitly after the cluster it targets, and its env
 * vars share the same prefix so deployment config is unambiguous (e.g.
 * `WARPSTREAM_LOGS_PRODUCER` ↔ `KAFKA_WARPSTREAM_LOGS_PRODUCER_*`).
 */
import { AllowedConfigKey } from '~/common/outputs/kafka-producer-config'

/**
 * Targets the Warpstream cluster dedicated to logs/traces. The main log/trace
 * data path produces here.
 */
export const WARPSTREAM_LOGS_PRODUCER = 'WARPSTREAM_LOGS_PRODUCER' as const
export type WarpstreamLogsProducer = typeof WARPSTREAM_LOGS_PRODUCER

/** Targets the Warpstream cluster shared with the rest of ingestion. App metrics produce here. */
export const WARPSTREAM_INGESTION_PRODUCER = 'WARPSTREAM_INGESTION_PRODUCER' as const
export type WarpstreamIngestionProducer = typeof WARPSTREAM_INGESTION_PRODUCER

/** Producer names registered by the logs/traces deployments. */
export type LogsProducerName = WarpstreamLogsProducer | WarpstreamIngestionProducer

/** Cluster-named env var prefix mirroring the producer it configures. */
export const WARPSTREAM_LOGS_PRODUCER_CONFIG_MAP = {
    'client.id': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_CLIENT_ID',
    'metadata.broker.list': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_BROKER_LIST',
    'security.protocol': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_SECURITY_PROTOCOL',
    'sasl.mechanisms': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_MECHANISMS',
    'sasl.username': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_USERNAME',
    'sasl.password': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_PASSWORD',
    'compression.codec': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_COMPRESSION_CODEC',
    'linger.ms': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_LINGER_MS',
    'batch.size': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_SIZE',
    'queue.buffering.max.messages': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
    'queue.buffering.max.kbytes': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
    'enable.ssl.certificate.verification': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION',
    'enable.idempotence': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_IDEMPOTENCE',
    'message.max.bytes': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_MESSAGE_MAX_BYTES',
    'batch.num.messages': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_NUM_MESSAGES',
    'sticky.partitioning.linger.ms': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_STICKY_PARTITIONING_LINGER_MS',
    'topic.metadata.refresh.interval.ms': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS',
    'metadata.max.age.ms': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_MAX_AGE_MS',
    'message.send.max.retries': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_RETRIES',
    'max.in.flight.requests.per.connection': 'KAFKA_WARPSTREAM_LOGS_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION',
} as const satisfies Partial<Record<AllowedConfigKey, string>>

/** Typed env vars referenced by `WARPSTREAM_LOGS_PRODUCER_CONFIG_MAP`. */
export type KafkaWarpstreamLogsProducerEnvConfig = {
    KAFKA_WARPSTREAM_LOGS_PRODUCER_CLIENT_ID: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_BROKER_LIST: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_SECURITY_PROTOCOL: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_MECHANISMS: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_USERNAME: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_PASSWORD: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_COMPRESSION_CODEC: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_LINGER_MS: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_SIZE: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_IDEMPOTENCE: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_MESSAGE_MAX_BYTES: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_NUM_MESSAGES: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_STICKY_PARTITIONING_LINGER_MS: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_MAX_AGE_MS: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_RETRIES: string
    KAFKA_WARPSTREAM_LOGS_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: string
}

export function getDefaultKafkaWarpstreamLogsProducerEnvConfig(): KafkaWarpstreamLogsProducerEnvConfig {
    return {
        KAFKA_WARPSTREAM_LOGS_PRODUCER_CLIENT_ID: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_BROKER_LIST: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_SECURITY_PROTOCOL: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_MECHANISMS: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_USERNAME: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_SASL_PASSWORD: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_COMPRESSION_CODEC: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_LINGER_MS: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_SIZE: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_ENABLE_IDEMPOTENCE: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_MESSAGE_MAX_BYTES: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_BATCH_NUM_MESSAGES: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_STICKY_PARTITIONING_LINGER_MS: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_MAX_AGE_MS: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_RETRIES: '',
        KAFKA_WARPSTREAM_LOGS_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: '',
    }
}

/** Cluster-named env var prefix mirroring the producer it configures. */
export const WARPSTREAM_INGESTION_PRODUCER_CONFIG_MAP = {
    'client.id': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID',
    'metadata.broker.list': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST',
    'security.protocol': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL',
    'sasl.mechanisms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS',
    'sasl.username': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME',
    'sasl.password': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD',
    'compression.codec': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC',
    'linger.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS',
    'batch.size': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE',
    'queue.buffering.max.messages': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES',
    'queue.buffering.max.kbytes': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES',
    'enable.ssl.certificate.verification': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION',
    'enable.idempotence': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE',
    'message.max.bytes': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES',
    'batch.num.messages': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES',
    'sticky.partitioning.linger.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS',
    'topic.metadata.refresh.interval.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS',
    'metadata.max.age.ms': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS',
    'message.send.max.retries': 'KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES',
    'max.in.flight.requests.per.connection':
        'KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION',
} as const satisfies Partial<Record<AllowedConfigKey, string>>

/** Typed env vars referenced by `WARPSTREAM_INGESTION_PRODUCER_CONFIG_MAP`. */
export type KafkaWarpstreamIngestionProducerEnvConfig = {
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES: string
    KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: string
}

export function getDefaultKafkaWarpstreamIngestionProducerEnvConfig(): KafkaWarpstreamIngestionProducerEnvConfig {
    return {
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_CLIENT_ID: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SECURITY_PROTOCOL: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_MECHANISMS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_USERNAME: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_SASL_PASSWORD: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_COMPRESSION_CODEC: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_SIZE: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_SSL_CERTIFICATE_VERIFICATION: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_ENABLE_IDEMPOTENCE: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_MESSAGE_MAX_BYTES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_BATCH_NUM_MESSAGES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_STICKY_PARTITIONING_LINGER_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_TOPIC_METADATA_REFRESH_INTERVAL_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_MAX_AGE_MS: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_RETRIES: '',
        KAFKA_WARPSTREAM_INGESTION_PRODUCER_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION: '',
    }
}