:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::atomic::{AtomicUsize, Ordering};
use common_continuous_profiling::ContinuousProfilingConfig;
use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;
use crate::core::config::ResolverConfig;
pub static BATCH_APPLY_CONCURRENCY: AtomicUsize = AtomicUsize::new(64);
#[derive(Envconfig, Clone)]
pub struct ProcessingConfig {
#[envconfig(nested = true)]
pub continuous_profiling: ContinuousProfilingConfig,
/// Shared symbol-resolution config (Postgres, object storage, frame cache,
/// sourcemap fetching). Both modes read this slice; it lives in `core` so
/// resolution mode never pulls in the processing-only knobs below.
#[envconfig(nested = true)]
pub resolver: ResolverConfig,
#[envconfig(from = "BIND_HOST", default = "::")]
pub host: String,
#[envconfig(from = "BIND_PORT", default = "3305")]
pub port: u16,
pub posthog_api_key: Option<String>,
#[envconfig(default = "https://us.i.posthog.com/capture")]
pub posthog_endpoint: String,
#[envconfig(nested = true)]
pub kafka: KafkaConfig,
// Optional override for the brokers used to produce `cdp_internal_events`. When set,
// cymbal opens a second producer pointed at this host list (used for the warpstream-cyclotron
// VC, where the hog-functions consumer reads). When unset, internal events go through the
// primary `kafka` producer like everything else.
#[envconfig(from = "CYMBAL_CYCLOTRON_KAFKA_HOSTS")]
pub cyclotron_kafka_hosts: Option<String>,
// Optional TLS override for the cyclotron producer. When unset, the cyclotron producer
// inherits `KAFKA_TLS` from the primary kafka config; set this to flip TLS independently
// (e.g. primary on plaintext warpstream-shared, secondary on SSL MSK).
#[envconfig(from = "CYMBAL_CYCLOTRON_KAFKA_TLS")]
pub cyclotron_kafka_tls: Option<bool>,
// Optional override for the brokers used to produce `clickhouse_app_metrics2`. When set,
// cymbal opens a dedicated producer pointed at this host list (the warpstream-ingestion VC,
// where ClickHouse consumes app_metrics2). When unset, app metrics go through the primary
// `kafka` producer — which only carries that topic where the cluster has it (e.g. local dev).
#[envconfig(from = "CYMBAL_APP_METRICS_KAFKA_HOSTS")]
pub app_metrics_kafka_hosts: Option<String>,
// Optional TLS override for the app-metrics producer. When unset, it inherits `KAFKA_TLS`
// from the primary kafka config.
#[envconfig(from = "CYMBAL_APP_METRICS_KAFKA_TLS")]
pub app_metrics_kafka_tls: Option<bool>,
#[envconfig(default = "cdp_internal_events")]
pub internal_events_topic: String,
#[envconfig(default = "clickhouse_error_tracking_fingerprint_issue_state")]
pub fingerprint_issue_state_topic: String,
#[envconfig(default = "document_embeddings_input")]
pub embedding_worker_topic: String,
#[envconfig(default = "error_tracking_ingestion_notifications")]
pub ingestion_notifications_topic: String,
#[envconfig(default = "600")]
pub issue_cache_ttl_seconds: u64,
// Sized generously on purpose: entries are ~100 bytes, and versioned fingerprinting
// probes this cache once per registered fingerprint version per event, so the working
// set is several multiples of the distinct-issue count at current event volume.
#[envconfig(default = "100000")]
pub issue_cache_capacity: u64,
// Event-level release resolution runs once per exception event. A release row is immutable
// once the CLI creates it, so a positive hit never goes stale; the TTL exists to let a
// negative result (app metadata that matches no release yet) expire after a dSYM upload
// creates the release, without re-querying Postgres on every event in the meantime.
#[envconfig(default = "300")]
pub release_cache_ttl_seconds: u64,
// An entry-count bound is a real memory bound: cached records clamp metadata to
// MAX_RELEASE_METADATA_BYTES at fetch, so a full cache tops out around
// max_entries * 8 KiB per lookup kind.
#[envconfig(default = "10000")]
pub release_cache_max_entries: u64,
// Maximum number of in-flight futures for a single `Batch::apply_func` call.
// This is a per-call-site limit, not a global pipeline-wide concurrency cap.
#[envconfig(default = "64")]
pub batch_apply_concurrency: usize,
// Maximum number of in-flight /process requests accepted by the API.
// Requests above this limit are rejected with 429 to apply backpressure.
#[envconfig(default = "128")]
pub process_max_in_flight_requests: usize,
#[envconfig(default = "60000")]
pub process_slow_log_threshold_ms: u64,
#[envconfig(default = "300")]
pub team_cache_ttl_secs: u64,
#[envconfig(default = "10000")]
pub max_team_cache_size: u64,
#[envconfig(default = "300")]
pub assignment_rule_cache_ttl_secs: u64,
#[envconfig(default = "100000")]
// The maximum number of bytecode operations we'll store in the cache, across all rules, across all teams
pub max_assignment_rule_cache_size: u64,
#[envconfig(default = "300")]
pub severity_rule_cache_ttl_secs: u64,
#[envconfig(default = "100000")]
pub max_severity_rule_cache_size: u64,
#[envconfig(default = "300")]
pub grouping_rule_cache_ttl_secs: u64,
#[envconfig(default = "100000")]
// The maximum number of bytecode operations we'll store in the cache, across all rules, across all teams
pub max_grouping_rule_cache_size: u64,
#[envconfig(default = "300")]
pub suppression_rule_cache_ttl_secs: u64,
#[envconfig(default = "100000")]
// The maximum number of bytecode operations we'll store in the cache, across all rules, across all teams
pub max_suppression_rule_cache_size: u64,
#[envconfig(default = "300")]
pub bypass_rule_cache_ttl_secs: u64,
#[envconfig(default = "100000")]
// The maximum number of bytecode operations we'll store in the cache, across all rules, across all teams
pub max_bypass_rule_cache_size: u64,
#[envconfig(from = "ISSUE_BUCKETS_REDIS_URL", default = "redis://localhost:6379/")]
pub issue_buckets_redis_url: String,
#[envconfig(from = "ERROR_TRACKING_PROCESSING_SHADOW_REDIS_URL")]
pub shadow_redis_url: Option<String>,
#[envconfig(
from = "ERROR_TRACKING_EVENT_PROPERTIES_TTL_SECONDS",
default = "172800"
)]
pub event_properties_ttl_seconds: u64,
#[envconfig(
from = "ERROR_TRACKING_EVENT_PROPERTIES_MAX_BYTES",
default = "1048576"
)]
pub event_properties_max_bytes: usize,
#[envconfig(default = "100")]
pub redis_response_timeout_ms: u64,
#[envconfig(default = "5000")]
pub redis_connection_timeout_ms: u64,
#[envconfig(from = "ERROR_TRACKING_CYMBAL_RATE_LIMITER_ENABLED", default = "false")]
pub error_tracking_rate_limiter_enabled: bool,
#[envconfig(
from = "ERROR_TRACKING_CYMBAL_RATE_LIMITER_REDIS_URL",
default = "redis://localhost:6379/"
)]
pub error_tracking_rate_limiter_redis_url: String,
#[envconfig(
from = "ERROR_TRACKING_CYMBAL_RATE_LIMITER_KEY_PREFIX",
default = "@posthog/error-tracking-cymbal-rate-limiter"
)]
pub error_tracking_rate_limiter_key_prefix: String,
#[envconfig(
from = "ERROR_TRACKING_CYMBAL_RATE_LIMITER_BUCKET_TTL_SECONDS",
default = "86400"
)]
pub error_tracking_rate_limiter_bucket_ttl_seconds: u64,
// Comma separated list of team IDs the error-tracking rate limiter applies to.
// If empty, it applies to all teams (that have limits configured).
#[envconfig(
from = "ERROR_TRACKING_CYMBAL_RATE_LIMITER_ENABLED_TEAM_IDS",
default = ""
)]
pub error_tracking_rate_limiter_enabled_team_ids: String,
// Comma separated list of team IDs that can receive spike alerts.
// If empty, all teams can receive alerts
#[envconfig(default = "")]
pub spike_alert_enabled_team_ids: String,
// Teams whose frame code variables are dropped, for old SDK senders we cannot upgrade.
#[envconfig(from = "ERROR_TRACKING_DROP_CODE_VARIABLES_TEAM_IDS", default = "")]
pub drop_code_variables_team_ids: String,
// ----------------------------------------------------------------------
// Remote resolution (cymbal.resolution.v1).
// ----------------------------------------------------------------------
/// Hostname of the cymbal-resolution service. Resolved via DNS, then each
/// returned address gets its own gRPC channel in the endpoint pool.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_HOST", default = "")]
pub remote_resolution_host: String,
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_PORT", default = "50061")]
pub remote_resolution_port: u16,
/// How often to re-resolve the configured hostname and refresh the pool.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_DNS_REFRESH_SECS", default = "30")]
pub remote_resolution_dns_refresh_secs: u64,
/// Per-call deadline for a single Resolve RPC, in milliseconds. The
/// stage enforces this independently of any transport-level keepalive.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_DEADLINE_MS", default = "15000")]
pub remote_resolution_deadline_ms: u64,
/// Connection establishment timeout for a single endpoint, in
/// milliseconds. Endpoints that exceed this are skipped for routing.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_CONNECT_TIMEOUT_MS", default = "1000")]
pub remote_resolution_connect_timeout_ms: u64,
/// Maximum number of caller-side retries against the endpoint pool when
/// transport, load shedding, or explicit `retry` outcomes are observed.
/// `0` disables retries; the first attempt still runs.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_MAX_RETRIES", default = "2")]
pub remote_resolution_max_retries: u32,
/// Initial backoff applied between caller-side retries, in milliseconds.
/// Each subsequent attempt doubles the wait (capped at
/// `CYMBAL_REMOTE_RESOLUTION_RETRY_MAX_BACKOFF_MS`), plus up to ~50% random
/// jitter so a fleet of cymbal pods does not synchronize retries against a
/// briefly-overloaded upstream.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_RETRY_BACKOFF_MS", default = "50")]
pub remote_resolution_retry_backoff_ms: u64,
/// Upper bound on the retry backoff window, in milliseconds. The exponential
/// schedule never sleeps longer than this between attempts.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_RETRY_MAX_BACKOFF_MS",
default = "1000"
)]
pub remote_resolution_retry_max_backoff_ms: u64,
/// Initial duration to temporarily remove an endpoint from routing after it
/// returns a per-item overload outcome. `0` keeps the legacy per-item-only
/// reroute behavior.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_MS",
default = "100"
)]
pub remote_resolution_overload_ejection_ms: u64,
/// Maximum endpoint ejection duration after repeated overloads.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_MAX_MS",
default = "5000"
)]
pub remote_resolution_overload_ejection_max_ms: u64,
/// Quiet window after which an endpoint's overload ejection duration resets
/// to the initial value.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_DECAY_MS",
default = "30000"
)]
pub remote_resolution_overload_ejection_decay_ms: u64,
/// Flattens remote resolution routing across the rendezvous-ranked candidate
/// list. `0.0` sends all traffic to the top-ranked endpoint, `1.0` is
/// uniform across all candidates, and intermediate values decay by rank.
#[envconfig(from = "CYMBAL_REMOTE_RESOLUTION_ROUTING_JITTER", default = "0.0")]
pub remote_resolution_routing_jitter: f64,
/// Maximum number of remote resolution items that can concurrently wait
/// for a pod to accept routing ownership.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_ROUTING_ACCEPTANCE_CONCURRENCY",
default = "10"
)]
pub remote_resolution_routing_acceptance_concurrency: usize,
/// Tick cadence hint sent on `SubscribeRequest.tick_hint_ms` to the
/// cymbal-resolution freshness/draining stream. The server clamps to its own bounds
/// (see cymbal-resolution `SUBSCRIBE_MIN_TICK_MS`/`SUBSCRIBE_MAX_TICK_MS`),
/// so this is only a hint.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_SUBSCRIBE_TICK_HINT_MS",
default = "1000"
)]
pub remote_resolution_subscribe_tick_hint_ms: u64,
/// Backoff between reconnect attempts when a per-endpoint subscription
/// stream terminates, in milliseconds. Kept small so a transient blip
/// doesn't keep the pool blind for long.
#[envconfig(
from = "CYMBAL_REMOTE_RESOLUTION_SUBSCRIBE_RECONNECT_BACKOFF_MS",
default = "500"
)]
pub remote_resolution_subscribe_reconnect_backoff_ms: u64,
}
impl ProcessingConfig {
pub fn init_with_defaults() -> Result<Self, envconfig::Error> {
let res = Self::init_from_env()?;
init_global_state(&res);
Ok(res)
}
}
pub fn init_global_state(config: &ProcessingConfig) {
BATCH_APPLY_CONCURRENCY.store(config.batch_apply_concurrency.max(1), Ordering::Relaxed);
}