:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{net::SocketAddr, num::NonZeroU32};
use common_continuous_profiling::ContinuousProfilingConfig;
use envconfig::Envconfig;
use tracing::Level;
use crate::producers::ProducerName;
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
pub enum CaptureMode {
Events,
Recordings,
Ai,
/// Analytics ingestion for historical backfills (the batch-import-worker).
/// Unlike `Events`, it skips the global rate limiter, drops batches not
/// flagged `historical_migration: true`, serves no AI or OTEL routes (their
/// handlers hardcode `historical_migration: false`), and has no AI overflow
/// lane.
Import,
}
impl CaptureMode {
pub fn as_tag(&self) -> &'static str {
match self {
CaptureMode::Events => "events",
CaptureMode::Recordings => "recordings",
CaptureMode::Ai => "ai",
CaptureMode::Import => "import",
}
}
/// Whether this mode subjects incoming events to the per-(token,
/// distinct_id) global rate limiter. `Import` opts out: historical
/// backfills are internal traffic that must not be throttled.
///
/// Note this is necessary but not sufficient: only the analytics processing
/// paths (legacy `events::analytics` and `v1::analytics`) actually consult
/// the limiter, so `Recordings` never rate-limits despite returning `true`
/// here. The predicate gates the two analytics paths; other paths ignore it.
pub fn applies_global_rate_limit(&self) -> bool {
!matches!(self, CaptureMode::Import)
}
/// Whether this mode drops any batch not marked `historical_migration:
/// true`. Only `Import` does — it exclusively ingests historical data.
pub fn requires_historical_migration(&self) -> bool {
matches!(self, CaptureMode::Import)
}
/// Import has no AI overflow lane: imports must never overflow.
pub fn has_ai_overflow_lane(&self) -> bool {
!matches!(self, CaptureMode::Import)
}
}
impl std::str::FromStr for CaptureMode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.trim().to_lowercase().as_ref() {
"events" => Ok(CaptureMode::Events),
"recordings" => Ok(CaptureMode::Recordings),
"ai" => Ok(CaptureMode::Ai),
"import" => Ok(CaptureMode::Import),
_ => Err(format!("Unknown Capture Type: {s}")),
}
}
}
/// Compression algorithm applied at the Kafka message payload (envelope) level,
/// independent of the broker-level `compression.codec` setting.
/// Enables Warpstream billing reduction by storing compressed bytes.
#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)]
pub enum EnvelopeCompression {
#[default]
None,
Lz4,
}
impl std::str::FromStr for EnvelopeCompression {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.trim().to_lowercase().as_ref() {
"none" => Ok(EnvelopeCompression::None),
"lz4" => Ok(EnvelopeCompression::Lz4),
_ => Err(format!("Unknown EnvelopeCompression: {s}")),
}
}
}
#[derive(Envconfig, Clone)]
pub struct Config {
#[envconfig(default = "false")]
pub print_sink: bool,
#[envconfig(default = "false")]
pub noop_sink: bool,
#[envconfig(default = "127.0.0.1:3000")]
pub address: SocketAddr,
pub redis_url: String,
#[envconfig(default = "100")]
pub redis_response_timeout_ms: u64,
#[envconfig(default = "5000")]
pub redis_connection_timeout_ms: u64,
#[envconfig(default = "false")]
pub global_rate_limit_enabled: bool,
/// When true, the global rate limiter evaluates and emits metrics/logs
/// but does not enforce (events pass through as if not limited).
#[envconfig(default = "false")]
pub global_rate_limit_dry_run: bool,
/// Sliding window interval to apply global rate limiting threshold to
#[envconfig(default = "60")]
pub global_rate_limit_window_interval_secs: u64,
/// Base re-sync cadence (seconds). The pressure tier scales it
#[envconfig(default = "15")]
pub global_rate_limit_sync_interval_secs: u64,
/// Background task cadence for pipeline reads + writes (milliseconds)
#[envconfig(default = "1000")]
pub global_rate_limit_tick_interval_ms: u64,
// --- Token+DistinctId limiter config ---
/// Per-(token, distinct_id) rate limit threshold per window interval
/// Note: default is too high to trigger limiting in production
#[envconfig(default = "300000")]
pub global_rate_limit_token_distinctid_threshold: u64,
/// CSV list of key=value pairs for custom per-(token, distinct_id) thresholds
pub global_rate_limit_token_distinctid_overrides_csv: Option<String>,
/// Max local cache entries for the per-(token, distinct_id) limiter
#[envconfig(default = "5000000")]
pub global_rate_limit_token_distinctid_local_cache_max_entries: u64,
/// Minimum local event count before a key earns a Redis read; `0` reads every key. Keep
/// it well under `threshold / pod_count`, or a key at its limit spread across pods is read
/// more than a window late.
#[envconfig(default = "10")]
pub global_rate_limit_min_sync_floor: u64,
/// Max keys read, and max write entries sent, per tick. The rest wait, so a backlog
/// shows as staleness rather than a tick that overruns its interval.
#[envconfig(default = "20000")]
pub global_rate_limit_max_sync_keys_per_tick: usize,
/// Max Redis keys per command (a read costs two per entity), so one command fits the
/// Redis client's response timeout.
#[envconfig(default = "2000")]
pub global_rate_limit_max_keys_per_command: usize,
/// How many chunked commands may be in flight at once per Redis instance.
#[envconfig(default = "4")]
pub global_rate_limit_max_concurrent_commands: usize,
/// Max distinct (key, epoch) entries held in the deferred write batch per
/// limiter. Merges are always accepted; at the cap, updates for new keys
/// are dropped and counted (fail-open). Bounds limiter memory under
/// unique-key floods that outrun the per-tick write drain.
#[envconfig(default = "200000")]
pub global_rate_limit_max_write_batch_entries: usize,
/// Max keys held in the pending-sync set per limiter. At the cap, new sync
/// requests drop and re-queue on the key's next request (fail-open).
/// Bounds limiter memory alongside the write-batch cap.
#[envconfig(default = "200000")]
pub global_rate_limit_max_pending_sync_entries: usize,
/// Seconds since a cache entry was last written before it is dropped. Every request
/// and read rewrites the entry, so at or above the idle timeout this never fires first.
#[envconfig(default = "600")]
pub global_rate_limit_local_cache_ttl_secs: u64,
/// Evict local cache entries not accessed within this window (seconds).
/// This is the main lever on cache cardinality: with a key space dominated
/// by one-shot identities, most entries are pure churn and hold a slot for
/// the full idle window. Must stay at or above the rate-limit window, or
/// entries expire inside the enforcement window and the limiter loses the
/// counts it is supposed to be accumulating -- values below the window are
/// clamped up, with a warning.
#[envconfig(default = "300")]
pub global_rate_limit_local_cache_idle_timeout_secs: u64,
/// Seconds reads (replica, then primary) must keep failing before a pod stops
/// limiting on its own unconfirmed counts. Unset uses each limiter's window; `0` disables.
pub global_rate_limit_max_read_outage_secs: Option<u64>,
/// The limiter's cap on one Redis read command (milliseconds). The Redis
/// client's response timeout also applies, and the shorter one fires first.
#[envconfig(default = "250")]
pub global_rate_limit_read_timeout_ms: u64,
/// The limiter's cap on one Redis write command (milliseconds). The Redis
/// client's response timeout also applies, and the shorter one fires first.
#[envconfig(default = "250")]
pub global_rate_limit_write_timeout_ms: u64,
// --- Token-only limiter config (not currently used in production, retained for new_token()) ---
/// Per-token rate limit threshold per window interval
/// Note: default is too high to trigger limiting in production
#[envconfig(default = "5000000")]
pub global_rate_limit_token_threshold: u64,
/// CSV list of key=value pairs for custom per-token thresholds
pub global_rate_limit_token_overrides_csv: Option<String>,
/// Max local cache entries for the per-token limiter
#[envconfig(default = "300000")]
pub global_rate_limit_token_local_cache_max_entries: u64,
/// Optional dedicated Redis URL for global rate limiter.
/// If set, creates a separate Redis client for the limiter.
/// Falls back to the shared redis_url if unset.
pub global_rate_limit_redis_url: Option<String>,
/// Optional replica URL: with `global_rate_limit_redis_url` set, reads go here and writes
/// to the primary. A replica read that fails with a recoverable error retries on the primary.
pub global_rate_limit_redis_reader_url: Option<String>,
/// Response timeout (ms) for the dedicated limiter Redis, defaulting to
/// `redis_response_timeout_ms`; ignored unless `global_rate_limit_redis_url` is set.
pub global_rate_limit_redis_response_timeout_ms: Option<u64>,
/// Connection timeout (ms) for the dedicated limiter Redis, defaulting to
/// `redis_connection_timeout_ms`; ignored unless `global_rate_limit_redis_url` is set.
pub global_rate_limit_redis_connection_timeout_ms: Option<u64>,
/// Redis clusters the per-(token, distinct_id) limiter shards its keys across: `1` uses
/// `global_rate_limit_redis_url` only, `2` adds the shard 1 URLs. The AI byte limiter always uses the first.
#[envconfig(default = "1")]
pub global_rate_limit_redis_shards: usize,
/// Writer URL of the second limiter cluster. Capture connects to it only when `global_rate_limit_redis_shards` is 2.
pub global_rate_limit_redis_shard_1_url: Option<String>,
/// Replica URL of the second limiter cluster, used like `global_rate_limit_redis_reader_url`.
pub global_rate_limit_redis_shard_1_reader_url: Option<String>,
/// Redis key holding the dynamic custom per-key rate-limit thresholds
/// (JSON object of `{key: threshold}`), written by Django. When set, the
/// per-(token, distinct_id) limiter refreshes its custom thresholds from
/// this key on a timer, overriding the static CSV overrides. Sourced from
/// the same Redis as event restrictions (`event_restrictions_redis_url`).
/// When unset, the limiter uses only the static CSV overrides.
pub global_rate_limit_custom_threshold_key: Option<String>,
/// How often to refresh the dynamic custom thresholds from Redis (seconds).
#[envconfig(default = "60")]
pub global_rate_limit_custom_threshold_refresh_secs: u64,
// Event restrictions configuration (reads from Redis, synced by Django)
#[envconfig(default = "false")]
pub event_restrictions_enabled: bool,
/// Redis URL for event restrictions (separate from main redis_url)
pub event_restrictions_redis_url: Option<String>,
#[envconfig(default = "30")]
pub event_restrictions_refresh_interval_secs: u64,
#[envconfig(default = "300")]
pub event_restrictions_fail_open_after_secs: u64,
pub otel_url: Option<String>,
#[envconfig(default = "false")]
pub overflow_enabled: bool,
/// Builds the AI lane's overflow limiter when `overflow_enabled` is also
/// set. Restriction overflow does not need it.
#[envconfig(default = "false")]
pub ai_events_overflow_enabled: bool,
#[envconfig(default = "false")]
pub overflow_preserve_partition_locality: bool,
#[envconfig(default = "100")]
pub overflow_per_second_limit: NonZeroU32,
#[envconfig(default = "1000")]
pub overflow_burst_limit: NonZeroU32,
pub ingestion_force_overflow_by_token_distinct_id: Option<String>, // Comma-delimited keys
pub drop_events_by_token_distinct_id: Option<String>, // "<token>:<distinct_id or *>,<distinct_id or *>;<token>..."
#[envconfig(default = "false")]
pub enable_historical_rerouting: bool,
#[envconfig(default = "1")]
pub historical_rerouting_threshold_days: i64,
#[envconfig(nested = true)]
pub outputs: OutputsConfig,
/// Application-level compression of session replay payloads, independent of
/// broker-level compression. Consumers detect and decompress it.
#[envconfig(from = "KAFKA_REPLAY_ENVELOPE_COMPRESSION", default = "none")]
pub replay_envelope_compression: EnvelopeCompression,
/// Refuse to boot when a registered output has an empty topic name (see
/// `OutputTable::check_complete`). Off by default so that a deployment which
/// blanks a topic it never produces to still boots.
#[envconfig(from = "CAPTURE_OUTPUTS_COMPLETENESS_CHECK_ENABLED", default = "false")]
pub outputs_completeness_check_enabled: bool,
#[envconfig(default = "1.0")]
pub otel_sampling_rate: f64,
#[envconfig(default = "capture")]
pub otel_service_name: String,
/// Install the Prometheus recorder and the limiters' metric tasks.
#[envconfig(default = "true")]
pub export_prometheus: bool,
pub redis_key_prefix: Option<String>,
#[envconfig(default = "events")]
pub capture_mode: CaptureMode,
pub concurrency_limit: Option<usize>,
#[envconfig(default = "false")]
pub s3_fallback_enabled: bool,
pub s3_fallback_bucket: Option<String>,
pub s3_fallback_endpoint: Option<String>,
#[envconfig(default = "")]
pub s3_fallback_prefix: String,
#[envconfig(default = "false")]
pub is_mirror_deploy: bool,
#[envconfig(default = "info")]
pub log_level: Level,
// deploy var [0.0..100.0] to sample behavior of interest for verbose logging
#[envconfig(default = "0.0")]
pub verbose_sample_percent: f32,
// AI endpoint size limits
#[envconfig(default = "26214400")] // 25MB in bytes
pub ai_max_sum_of_parts_bytes: usize,
/// Largest single AI-lane event this deployment accepts. Measured on the
/// serialized event body, except on v1, which measures the properties blob
/// — see [`crate::v0_request::exceeds_max_ai_event_bytes`]. `0` disables
/// the ceiling.
///
/// Set it below what the deployment's broker accepts, leaving room for the
/// `CapturedEvent` envelope and the JSON-escaping of `data`:
/// `KAFKA_INGESTION_PRODUCER_MESSAGE_MAX_BYTES` bounds the produced message, not the
/// event inside it. capture-analytics needs a smaller value than capture-ai
/// because its AI topic is on MSK.
///
/// What an over-ceiling event gets back differs by path, because each one
/// keeps its own convention:
///
/// * `/i/v0/ai/batch` and the diverted legacy path — 413, whole request
/// refused, like every other oversize check there.
/// * `/i/v1/analytics/events` — the one event is dropped and reported as
/// `ai_event_too_big` in the 200 body; the rest of the batch publishes.
/// * `/i/v0/ai` (multipart) — 413, the endpoint's pre-existing behavior.
/// * `/i/v0/ai/otel` — the span is shed and the export still succeeds. A
/// collector retries a rejected export, so refusing would stall every
/// span behind one that can never fit. That loss is invisible in the
/// response, so it raises a `MessageSizeTooLarge` ingestion warning.
///
/// The legacy, v1, and OTEL paths count the loss under `ai_event_too_big`,
/// on `capture_events_dropped_total` or `capture_v1_events_dropped`. The
/// legacy path charges the whole batch, because the refusal loses every
/// event in it, not just the offender. The multipart handler counts no
/// drop at all: like every other error it raises, the refusal shows up
/// only on `capture_error_by_stage_and_type`.
/// Keep this under the deployment's `KAFKA_INGESTION_PRODUCER_MESSAGE_MAX_BYTES`.
/// Above it the ceiling stops being a guard: capture reads the body, builds
/// the event, and the producer refuses it anyway. A deployment that has not
/// raised its producer cap wants a lower value than this default; the boot
/// warning says so when the two are out of order.
///
/// Must equal `AI_MAX_EVENT_BYTES_DEFAULT`, or `tests/ai_event_limits_parity.rs` fails.
#[envconfig(default = "8388608")] // 8MiB
pub ai_max_event_bytes: u64,
// HMAC-SHA256 key shared with the AI gateway. When set, $ai_generation events
// carrying a valid PostHog-Ai-Gateway-* signature are stamped verified and
// exempted from the llm_events quota limiter. Unset disables verification
// (all $ai_gateway* props are stripped as untrusted).
pub ai_gateway_signing_secret: Option<String>,
// HTTP/1 header read timeout in milliseconds - closes connections that don't
// send complete headers within this duration (slow loris protection).
// Set env var to enable; unset to disable.
pub http1_header_read_timeout_ms: Option<u64>,
// Body chunk read timeout in milliseconds. If a client stops sending data
// for this duration mid-upload, the request is aborted with 408 to avoid
// pointless gateway retries of stalled mobile requests that can't succeed.
// Set env var to enable; unset to disable (existing behavior).
pub body_chunk_read_timeout_ms: Option<u64>,
// Initial buffer size for body reads in KB. The buffer starts at this size
// (or the request limit, whichever is smaller) and grows as needed.
#[envconfig(default = "256")]
pub body_read_chunk_size_kb: usize,
#[envconfig(nested = true)]
pub continuous_profiling: ContinuousProfilingConfig,
/// Maximum compressed (wire) body size the v1 endpoint will accept (bytes).
#[envconfig(default = "10485760")]
pub capture_v1_max_compressed_body_bytes: usize,
/// Maximum decompressed body size the v1 endpoint will accept (bytes).
#[envconfig(default = "52428800")]
pub capture_v1_max_decompressed_body_bytes: usize,
/// Batch size threshold for parallel scatter-gather serialization; 0 disables fanout.
#[envconfig(default = "8")]
pub capture_v1_scatter_gather_min_batch: usize,
// --- Ingestion warnings emitter (fire-and-forget, best-effort) ---
// Warnings go out as `$$client_ingestion_warning` events through a
// dedicated `common_kafka::config::KafkaConfig` with fire-and-forget
// acks/retries and a small queue, so a slow warnings topic cannot contend
// with the main event producer. Defaults off (fail open).
#[envconfig(default = "false")]
pub capture_ingestion_warnings_enabled: bool,
// The producer's fire-and-forget policy (acks, retries, linger, queue
// depth in messages, message timeout) is fixed in code — see the
// `WARNINGS_KAFKA_*` constants in `setup.rs` — not env-configurable, since
// those define the "a warning is worth less than the cost of retrying it"
// contract rather than operator knobs. Only the two capacity/safety limits
// below stay tunable.
#[envconfig(default = "16")]
pub capture_ingestion_warnings_kafka_queue_mib: u32,
// rdkafka "message.max.bytes": a hard per-message ceiling, independent of
// the main producer's, so an oversized warning envelope (e.g. built from
// attacker-controlled input) cannot inflate past this regardless of what
// the main event producer allows.
#[envconfig(default = "1048576")]
pub capture_ingestion_warnings_kafka_message_max_bytes: u32,
// The warnings emitter's own destination. It serves every pipeline that
// emits (v1 and legacy analytics, both AI endpoints, and replay) but reads
// only these three vars, never the ingestion producer's settings or
// `CAPTURE_OUTPUT_CLIENT_WARNINGS_TOPIC`. charts sets all three per env,
// pointed at the MSK cluster the clientwarnings consumer reads from.
//
// Defaults are inert on purpose: empty hosts or topic makes
// `create_ingestion_warning_emitter` report the emitter disabled and return
// (fail open) rather than produce to a wrong or empty destination. TLS is a
// separate knob from hosts because the warnings cluster's TLS requirement
// need not match the main one — capture-ai is the live example, with a
// PLAINTEXT WarpStream event sink and a TLS MSK warnings destination.
#[envconfig(default = "")]
pub capture_ingestion_warnings_kafka_topic: String,
#[envconfig(default = "")]
pub capture_ingestion_warnings_kafka_hosts: String,
#[envconfig(default = "false")]
pub capture_ingestion_warnings_kafka_tls: bool,
/// Per-token AI bytes per second, enforced fleet-wide as this times the AI byte window;
/// `0` (and import mode) disables it. A token may spend a whole window's budget at once.
#[envconfig(default = "0")]
pub ai_byte_limit_per_second: u64,
/// CSV list of `token=bytesPerSecond` pairs setting specific tokens' budgets.
/// Same unit as `ai_byte_limit_per_second`.
pub ai_byte_limit_overrides_csv: Option<String>,
/// When true, the AI byte limiter evaluates and reports but does not drop.
/// Separate from `global_rate_limit_dry_run` so this rollout and the
/// token+distinct_id limiter's can move independently.
#[envconfig(default = "false")]
pub ai_byte_limit_dry_run: bool,
/// Window the AI byte budget is enforced over. Falls back to
/// `GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS` when unset.
///
/// The AI byte budget is shared across capture deployments through one
/// Redis counter, and the epoch key derives from this window, so every
/// deployment must set the same value or the shared budget splits. Kept
/// separate so the token+distinct_id window can be tuned per deployment.
pub ai_byte_limit_window_interval_secs: Option<u64>,
/// Max local cache entries for the AI byte limiter. Keyed per token, so this
/// is bounded by the number of projects sending AI traffic — far smaller
/// than the per-(token, distinct_id) limiter's key space.
#[envconfig(default = "300000")]
pub ai_byte_limit_local_cache_max_entries: u64,
}
/// Mirrors Node.js ingestion's `INGESTION_OUTPUT_<OUTPUT>_{TOPIC,PRODUCER}`.
///
/// Topic defaults are the local dev and hobby topics. Those stacks pull
/// `capture:master` with compose files that can predate these variables.
#[derive(Envconfig, Clone)]
pub struct OutputsConfig {
#[envconfig(
from = "CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
default = "events_plugin_ingestion"
)]
pub analytics_main_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_ANALYTICS_MAIN_PRODUCER", default = "INGESTION")]
pub analytics_main_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_ANALYTICS_OVERFLOW_TOPIC",
default = "events_plugin_ingestion_overflow"
)]
pub analytics_overflow_topic: String,
#[envconfig(
from = "CAPTURE_OUTPUT_ANALYTICS_OVERFLOW_PRODUCER",
default = "INGESTION"
)]
pub analytics_overflow_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_ANALYTICS_HISTORICAL_TOPIC",
default = "events_plugin_ingestion_historical"
)]
pub analytics_historical_topic: String,
#[envconfig(
from = "CAPTURE_OUTPUT_ANALYTICS_HISTORICAL_PRODUCER",
default = "INGESTION"
)]
pub analytics_historical_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_SESSION_REPLAY_MAIN_TOPIC",
default = "session_recording_snapshot_item_events"
)]
pub session_replay_main_topic: String,
#[envconfig(
from = "CAPTURE_OUTPUT_SESSION_REPLAY_MAIN_PRODUCER",
default = "INGESTION"
)]
pub session_replay_main_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_SESSION_REPLAY_OVERFLOW_TOPIC",
default = "session_recording_snapshot_item_overflow"
)]
pub session_replay_overflow_topic: String,
#[envconfig(
from = "CAPTURE_OUTPUT_SESSION_REPLAY_OVERFLOW_PRODUCER",
default = "INGESTION"
)]
pub session_replay_overflow_producer: ProducerName,
#[envconfig(from = "CAPTURE_OUTPUT_HEATMAPS_TOPIC", default = "heatmaps_ingestion")]
pub heatmaps_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_HEATMAPS_PRODUCER", default = "INGESTION")]
pub heatmaps_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_CLIENT_WARNINGS_TOPIC",
default = "ingestion-clientwarnings-main-1"
)]
pub client_warnings_topic: String,
#[envconfig(
from = "CAPTURE_OUTPUT_CLIENT_WARNINGS_PRODUCER",
default = "INGESTION"
)]
pub client_warnings_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_ERROR_TRACKING_TOPIC",
default = "ingestion-errortracking-main"
)]
pub error_tracking_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_ERROR_TRACKING_PRODUCER", default = "INGESTION")]
pub error_tracking_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_DLQ_TOPIC",
default = "events_plugin_ingestion_dlq"
)]
pub dlq_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_DLQ_PRODUCER", default = "INGESTION")]
pub dlq_producer: ProducerName,
/// The v0 (`DataType::AiEvents`) and v1 (`Destination::AiMain`) pipelines
/// divert AI events here instead of the main topic on every deployment,
/// capture-ai included.
#[envconfig(
from = "CAPTURE_OUTPUT_AI_MAIN_TOPIC",
default = "events_plugin_ingestion_ai"
)]
pub ai_main_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_AI_MAIN_PRODUCER", default = "INGESTION")]
pub ai_main_producer: ProducerName,
#[envconfig(
from = "CAPTURE_OUTPUT_AI_OVERFLOW_TOPIC",
default = "events_plugin_ingestion_ai_overflow"
)]
pub ai_overflow_topic: String,
#[envconfig(from = "CAPTURE_OUTPUT_AI_OVERFLOW_PRODUCER", default = "INGESTION")]
pub ai_overflow_producer: ProducerName,
/// Carries event-restriction redirects to an admin-supplied topic.
#[envconfig(from = "CAPTURE_OUTPUT_CUSTOM_PRODUCER", default = "INGESTION")]
pub custom_producer: ProducerName,
}
#[derive(Envconfig, Clone)]
pub struct KafkaConfig {
#[envconfig(default = "20")]
pub kafka_producer_linger_ms: u32, // Maximum time between producer batches during low traffic
#[envconfig(default = "400")]
pub kafka_producer_queue_mib: u32, // Size of the in-memory producer queue in mebibytes
#[envconfig(default = "20000")]
pub kafka_message_timeout_ms: u32, // Time before we stop retrying producing a message: 20 seconds
#[envconfig(default = "1000000")]
pub kafka_producer_message_max_bytes: u32, // message.max.bytes - max kafka message size we will produce
#[envconfig(default = "none")]
pub kafka_compression_codec: String, // none, gzip, snappy, lz4, zstd
pub kafka_hosts: String,
#[envconfig(default = "events_plugin_ingestion")]
pub kafka_topic: String,
#[envconfig(default = "ingestion-traces")]
pub kafka_traces_topic: String,
#[envconfig(default = "ingestion-metrics")]
pub kafka_metrics_topic: String,
#[envconfig(default = "false")]
pub kafka_tls: bool,
#[envconfig(default = "")]
pub kafka_client_id: String,
#[envconfig(default = "2")]
pub kafka_producer_max_retries: u32,
#[envconfig(default = "all")]
pub kafka_producer_acks: String,
// interval between metadata refreshes from the Kafka brokers
#[envconfig(default = "20000")]
pub kafka_topic_metadata_refresh_interval_ms: u32,
// default is 3x metadata refresh interval so we maintain that here
#[envconfig(default = "60000")]
pub kafka_metadata_max_age_ms: u32,
// Traces-cluster overrides (consumed by capture-logs). When unset, the
// traces producer reuses the corresponding `kafka_*` value above.
pub kafka_traces_hosts: Option<String>,
pub kafka_traces_tls: Option<bool>,
pub kafka_traces_client_id: Option<String>,
pub kafka_traces_compression_codec: Option<String>,
pub kafka_traces_producer_acks: Option<String>,
pub kafka_traces_producer_linger_ms: Option<u32>,
pub kafka_traces_producer_queue_mib: Option<u32>,
pub kafka_traces_message_timeout_ms: Option<u32>,
pub kafka_traces_producer_message_max_bytes: Option<u32>,
pub kafka_traces_producer_max_retries: Option<u32>,
pub kafka_traces_topic_metadata_refresh_interval_ms: Option<u32>,
pub kafka_traces_metadata_max_age_ms: Option<u32>,
// Metrics-cluster overrides (consumed by capture-logs). When unset, the
// metrics producer reuses the corresponding `kafka_*` value above.
pub kafka_metrics_hosts: Option<String>,
pub kafka_metrics_tls: Option<bool>,
pub kafka_metrics_client_id: Option<String>,
pub kafka_metrics_compression_codec: Option<String>,
pub kafka_metrics_producer_acks: Option<String>,
pub kafka_metrics_producer_linger_ms: Option<u32>,
pub kafka_metrics_producer_queue_mib: Option<u32>,
pub kafka_metrics_message_timeout_ms: Option<u32>,
pub kafka_metrics_producer_message_max_bytes: Option<u32>,
pub kafka_metrics_producer_max_retries: Option<u32>,
pub kafka_metrics_topic_metadata_refresh_interval_ms: Option<u32>,
pub kafka_metrics_metadata_max_age_ms: Option<u32>,
}
#[cfg(test)]
mod tests {
use super::{CaptureMode, Config};
use crate::producers::ProducerName;
use std::collections::HashMap;
use std::str::FromStr;
fn required_config_env() -> HashMap<String, String> {
[("REDIS_URL", "redis://localhost:6379/")]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
#[test]
fn output_topic_defaults_match_the_local_stack() {
let config: Config =
envconfig::Envconfig::init_from_hashmap(&required_config_env()).unwrap();
assert_eq!(config.outputs.ai_main_topic, "events_plugin_ingestion_ai");
assert_eq!(
config.outputs.ai_overflow_topic,
"events_plugin_ingestion_ai_overflow"
);
assert!(!config.ai_events_overflow_enabled);
assert_eq!(
config.outputs.session_replay_main_topic,
"session_recording_snapshot_item_events"
);
assert_eq!(
config.outputs.error_tracking_topic,
"ingestion-errortracking-main"
);
}
#[rstest::rstest]
#[case("CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC", |c: &Config| c.outputs.analytics_main_topic.clone())]
#[case("CAPTURE_OUTPUT_ANALYTICS_OVERFLOW_TOPIC", |c: &Config| c.outputs.analytics_overflow_topic.clone())]
#[case("CAPTURE_OUTPUT_ANALYTICS_HISTORICAL_TOPIC", |c: &Config| c.outputs.analytics_historical_topic.clone())]
#[case("CAPTURE_OUTPUT_SESSION_REPLAY_MAIN_TOPIC", |c: &Config| c.outputs.session_replay_main_topic.clone())]
#[case("CAPTURE_OUTPUT_SESSION_REPLAY_OVERFLOW_TOPIC", |c: &Config| c.outputs.session_replay_overflow_topic.clone())]
#[case("CAPTURE_OUTPUT_HEATMAPS_TOPIC", |c: &Config| c.outputs.heatmaps_topic.clone())]
#[case("CAPTURE_OUTPUT_CLIENT_WARNINGS_TOPIC", |c: &Config| c.outputs.client_warnings_topic.clone())]
#[case("CAPTURE_OUTPUT_ERROR_TRACKING_TOPIC", |c: &Config| c.outputs.error_tracking_topic.clone())]
#[case("CAPTURE_OUTPUT_DLQ_TOPIC", |c: &Config| c.outputs.dlq_topic.clone())]
#[case("CAPTURE_OUTPUT_AI_MAIN_TOPIC", |c: &Config| c.outputs.ai_main_topic.clone())]
#[case("CAPTURE_OUTPUT_AI_OVERFLOW_TOPIC", |c: &Config| c.outputs.ai_overflow_topic.clone())]
fn topic_env_var_binds_to_its_field(
#[case] env_var: &str,
#[case] field: fn(&Config) -> String,
) {
let mut env = required_config_env();
env.insert(env_var.into(), "configured_topic".into());
let config: Config = envconfig::Envconfig::init_from_hashmap(&env).unwrap();
assert_eq!(field(&config), "configured_topic");
}
#[rstest::rstest]
#[case("KAFKA_TOPIC")]
#[case("KAFKA_OVERFLOW_TOPIC")]
#[case("KAFKA_HISTORICAL_TOPIC")]
#[case("KAFKA_CLIENT_INGESTION_WARNING_TOPIC")]
#[case("KAFKA_ERROR_TRACKING_TOPIC")]
#[case("KAFKA_HEATMAPS_TOPIC")]
#[case("KAFKA_REPLAY_OVERFLOW_TOPIC")]
#[case("KAFKA_DLQ_TOPIC")]
#[case("CAPTURE_ANALYTICS_AI_EVENTS_TOPIC")]
#[case("CAPTURE_ANALYTICS_AI_EVENTS_OVERFLOW_TOPIC")]
fn legacy_topic_env_var_is_not_read(#[case] env_var: &str) {
let mut env = required_config_env();
env.insert(env_var.into(), "legacy_topic".into());
let config: Config = envconfig::Envconfig::init_from_hashmap(&env).unwrap();
let outputs = &config.outputs;
let topics = [
outputs.analytics_main_topic.as_str(),
outputs.analytics_overflow_topic.as_str(),
outputs.analytics_historical_topic.as_str(),
outputs.session_replay_main_topic.as_str(),
outputs.session_replay_overflow_topic.as_str(),
outputs.heatmaps_topic.as_str(),
outputs.client_warnings_topic.as_str(),
outputs.error_tracking_topic.as_str(),
outputs.dlq_topic.as_str(),
outputs.ai_main_topic.as_str(),
outputs.ai_overflow_topic.as_str(),
];
assert!(!topics.contains(&"legacy_topic"), "{env_var} is still read");
}
#[test]
fn output_producers_default_to_ingestion_and_parse_their_env_var() {
let config: Config =
envconfig::Envconfig::init_from_hashmap(&required_config_env()).unwrap();
assert_eq!(
config.outputs.analytics_main_producer,
ProducerName::Ingestion
);
assert_eq!(config.outputs.custom_producer, ProducerName::Ingestion);
let mut env = required_config_env();
env.insert(
"CAPTURE_OUTPUT_HEATMAPS_PRODUCER".into(),
"WARPSTREAM".into(),
);
let bad: Result<Config, _> = envconfig::Envconfig::init_from_hashmap(&env);
assert!(bad.is_err(), "an undeclared producer must fail startup");
}
#[test]
fn moved_output_settings_keep_their_env_vars() {
let mut env = required_config_env();
env.insert("KAFKA_REPLAY_ENVELOPE_COMPRESSION".into(), "lz4".into());
env.insert(
"CAPTURE_OUTPUTS_COMPLETENESS_CHECK_ENABLED".into(),
"true".into(),
);
let config: Config = envconfig::Envconfig::init_from_hashmap(&env).unwrap();
assert_eq!(
config.replay_envelope_compression,
super::EnvelopeCompression::Lz4
);
assert!(config.outputs_completeness_check_enabled);
}
#[test]
fn capture_mode_from_str_and_tag_roundtrip() {
// Locks the CAPTURE_MODE env contract, including the new `import` mode
// and case/whitespace handling, plus the tag used as a metric label.
let ok = [
("events", CaptureMode::Events, "events"),
("Recordings", CaptureMode::Recordings, "recordings"),
(" ai ", CaptureMode::Ai, "ai"),
("import", CaptureMode::Import, "import"),
("IMPORT", CaptureMode::Import, "import"),
];
for (input, expected, tag) in ok {
let parsed = CaptureMode::from_str(input).unwrap();
assert_eq!(parsed, expected, "input={input}");
assert_eq!(parsed.as_tag(), tag, "input={input}");
}
for bad in ["", "imports", "backfill", "historical"] {
assert!(
CaptureMode::from_str(bad).is_err(),
"expected err for {bad:?}"
);
}
}
#[test]
fn capture_mode_import_policy_differs_from_events() {
// The whole point of Import mode: it skips the global rate limiter and
// drops non-historical batches, while every other mode does neither.
assert!(!CaptureMode::Import.applies_global_rate_limit());
assert!(CaptureMode::Import.requires_historical_migration());
for mode in [
CaptureMode::Events,
CaptureMode::Recordings,
CaptureMode::Ai,
] {
assert!(
mode.applies_global_rate_limit(),
"{mode:?} should apply GRL"
);
assert!(
!mode.requires_historical_migration(),
"{mode:?} should not require historical_migration"
);
}
}
}