:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use common_kafka::kafka_producer::{create_kafka_producer, KafkaContext};
use common_redis::{Client as RedisClientTrait, RedisClient};
use health::HealthRegistry;
use moka::future::{Cache, CacheBuilder};
use rdkafka::producer::FutureProducer;
use sqlx::PgPool;
use std::{collections::HashSet, sync::Arc, time::Duration};
use tokio::{sync::Semaphore, task::JoinHandle};
use tracing::info;
use uuid::Uuid;
use crate::modes::{processing::redis_heal::HealGate, shadow_redis};
use crate::{
core::config::build_pg_pool,
error::UnhandledError,
modes::processing::config::{init_global_state, ProcessingConfig},
stages::rate_limiting::RedisRateLimiter,
stages::resolution::event_release::ReleaseCache,
stages::resolution::remote::{
dns::TokioDnsResolver, pool::EndpointPool, resolver::RemoteResolutionContext,
RemoteResolutionConfig,
},
teams::TeamManager,
types::operator::TeamId,
};
pub struct AppContext {
pub health_registry: HealthRegistry,
pub immediate_producer: FutureProducer<KafkaContext>,
// Dedicated producer for `cdp_internal_events`. Points at warpstream-cyclotron when
// `CYMBAL_CYCLOTRON_KAFKA_HOSTS` is set; otherwise falls back to `immediate_producer`.
pub cyclotron_producer: FutureProducer<KafkaContext>,
// Dedicated producer for `clickhouse_app_metrics2`. Points at warpstream-ingestion when
// `CYMBAL_APP_METRICS_KAFKA_HOSTS` is set; otherwise falls back to `immediate_producer`.
// The primary producer's cluster (warpstream-shared) does not carry that topic.
pub app_metrics_producer: FutureProducer<KafkaContext>,
pub posthog_pool: PgPool,
pub process_request_limiter: Arc<Semaphore>,
/// The `cymbal-resolution` service pool. Built once at startup; the
/// endpoint pool refreshes itself in the background.
pub remote_resolution: Option<RemoteResolutionContext>,
remote_resolution_refresh_task: Option<JoinHandle<()>>,
pub config: ProcessingConfig,
pub team_manager: TeamManager,
pub issue_buckets_redis_client: Arc<dyn RedisClientTrait + Send + Sync>,
pub shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>>,
// Debounces heal-task spawns for the issue-buckets client (see redis_heal).
pub issue_buckets_heal_gate: HealGate,
// Error-tracking rate limiter. `None` when disabled (the default), in which
// case `RateLimitingStage` is a pass-through no-op.
pub rate_limiter: Option<Arc<RedisRateLimiter>>,
// Team allowlist for the rate limiter: `None` = all teams, `Some(set)` = only
// these. Parsed from ERROR_TRACKING_RATE_LIMITER_ENABLED_TEAM_IDS.
pub rate_limiter_enabled_team_ids: Option<HashSet<i32>>,
pub drop_code_variables_team_ids: Arc<HashSet<i32>>,
// Shared `(team_id, fingerprint) -> issue_id` mapping cache. Lives on AppContext so
// it persists across requests — only the stable mapping is cached, never the Issue
// itself, so suppression / reopen always see current PG state (see `IssueLinker`).
// moka caches are cheap to clone (internally Arc'd).
pub issue_cache: Cache<(TeamId, String), Uuid>,
// Caches event-level release resolution (`$release_id` and the mobile app-metadata hash) so a
// per-event lookup doesn't re-hit Postgres for the same release, including the negative result
// for apps that never bound one. Lives here so it survives across batches.
pub release_cache: ReleaseCache,
}
impl Drop for AppContext {
fn drop(&mut self) {
if let Some(handle) = &self.remote_resolution_refresh_task {
handle.abort();
}
}
}
impl AppContext {
pub async fn from_config(config: &ProcessingConfig) -> Result<Self, UnhandledError> {
let posthog_pool = build_pg_pool(
&config.resolver.database_url,
config.resolver.max_pg_connections,
"cymbal_processing",
)?;
let issue_buckets_redis_client = RedisClient::with_config(
config.issue_buckets_redis_url.clone(),
common_redis::CompressionConfig::disabled(),
common_redis::RedisValueFormat::default(),
if config.redis_response_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(config.redis_response_timeout_ms))
},
if config.redis_connection_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(config.redis_connection_timeout_ms))
},
)
.await?;
let issue_buckets_redis_client: Arc<dyn RedisClientTrait + Send + Sync> =
Arc::new(issue_buckets_redis_client);
AppContext::new(config, posthog_pool, issue_buckets_redis_client).await
}
pub async fn new(
config: &ProcessingConfig,
posthog_pool: PgPool,
issue_buckets_redis_client: Arc<dyn RedisClientTrait + Send + Sync>,
) -> Result<Self, UnhandledError> {
if config.remote_resolution_host.trim().is_empty() {
return Err(UnhandledError::Other(
"CYMBAL_REMOTE_RESOLUTION_HOST is empty".to_string(),
));
}
init_global_state(config);
let health_registry = HealthRegistry::new("liveness");
let kafka_immediate_liveness = health_registry
.register("immediate_kafka".to_string(), Duration::from_secs(30))
.await;
let immediate_producer =
create_kafka_producer(&config.kafka, kafka_immediate_liveness).await?;
// Build the cyclotron producer if a separate broker list is configured; otherwise
// reuse the primary producer (so call sites can always target `cyclotron_producer`
// without branching).
let cyclotron_producer = match config.cyclotron_kafka_hosts.as_deref() {
Some(hosts) if !hosts.is_empty() => {
let mut cyclotron_config = config.kafka.clone();
cyclotron_config.kafka_hosts = hosts.to_string();
if let Some(tls) = config.cyclotron_kafka_tls {
cyclotron_config.kafka_tls = tls;
}
let kafka_cyclotron_liveness = health_registry
.register("cyclotron_kafka".to_string(), Duration::from_secs(30))
.await;
create_kafka_producer(&cyclotron_config, kafka_cyclotron_liveness).await?
}
_ => immediate_producer.clone(),
};
// Build the app-metrics producer if a separate broker list is configured; otherwise
// reuse the primary producer (so local dev, where one cluster carries every topic,
// works without extra config).
let app_metrics_producer = match config.app_metrics_kafka_hosts.as_deref() {
Some(hosts) if !hosts.is_empty() => {
let mut app_metrics_config = config.kafka.clone();
app_metrics_config.kafka_hosts = hosts.to_string();
if let Some(tls) = config.app_metrics_kafka_tls {
app_metrics_config.kafka_tls = tls;
}
let kafka_app_metrics_liveness = health_registry
.register("app_metrics_kafka".to_string(), Duration::from_secs(30))
.await;
create_kafka_producer(&app_metrics_config, kafka_app_metrics_liveness).await?
}
_ => immediate_producer.clone(),
};
info!("AppContext initialized");
let team_manager = TeamManager::new(config);
let process_request_limiter =
Arc::new(Semaphore::new(config.process_max_in_flight_requests.max(1)));
let issue_cache = CacheBuilder::new(config.issue_cache_capacity)
.time_to_live(Duration::from_secs(config.issue_cache_ttl_seconds))
.build();
let release_cache = ReleaseCache::new(
config.release_cache_max_entries,
Duration::from_secs(config.release_cache_ttl_seconds),
);
let (remote_resolution, remote_resolution_refresh_task) =
build_remote_resolution(config).await?;
let shadow = shadow_redis::connect(
config.shadow_redis_url.as_deref(),
&[
&config.issue_buckets_redis_url,
&config.error_tracking_rate_limiter_redis_url,
],
common_redis::RedisValueFormat::default(),
config.redis_response_timeout_ms,
config.redis_connection_timeout_ms,
)
.await;
let rate_limiter = build_rate_limiter(config, shadow.clone()).await?;
let shadow_redis_client: Option<Arc<dyn RedisClientTrait + Send + Sync>> =
shadow.map(|client| -> Arc<dyn RedisClientTrait + Send + Sync> { client });
let rate_limiter_enabled_team_ids =
parse_team_id_allowlist(&config.error_tracking_rate_limiter_enabled_team_ids);
let drop_code_variables_team_ids = Arc::new(parse_strict_team_id_list(
&config.drop_code_variables_team_ids,
)?);
Ok(Self {
health_registry,
immediate_producer,
cyclotron_producer,
app_metrics_producer,
posthog_pool,
config: config.clone(),
process_request_limiter,
team_manager,
issue_buckets_redis_client,
shadow_redis_client,
issue_buckets_heal_gate: HealGate::new(),
rate_limiter,
rate_limiter_enabled_team_ids,
drop_code_variables_team_ids,
issue_cache,
release_cache,
remote_resolution,
remote_resolution_refresh_task,
})
}
}
async fn build_rate_limiter(
config: &ProcessingConfig,
shadow: Option<Arc<RedisClient>>,
) -> Result<Option<Arc<RedisRateLimiter>>, UnhandledError> {
if !config.error_tracking_rate_limiter_enabled {
return Ok(None);
}
// Dedicated connection to the rate-limiter Redis. Defaults to localhost,
// which is shared with the issue-buckets Redis in local dev.
let client = RedisClient::with_config(
config.error_tracking_rate_limiter_redis_url.clone(),
common_redis::CompressionConfig::disabled(),
common_redis::RedisValueFormat::Utf8,
if config.redis_response_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(config.redis_response_timeout_ms))
},
if config.redis_connection_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(config.redis_connection_timeout_ms))
},
)
.await?;
info!("Error-tracking rate limiter enabled");
let shadow: Option<Arc<dyn common_redis::ScriptRunner>> =
shadow.map(|client| -> Arc<dyn common_redis::ScriptRunner> { client });
Ok(Some(Arc::new(
RedisRateLimiter::new(
Arc::new(client),
config.error_tracking_rate_limiter_key_prefix.clone(),
config.error_tracking_rate_limiter_bucket_ttl_seconds,
)
.with_shadow(shadow),
)))
}
/// Parse a comma-separated team-id allowlist. `None` (empty input) means the
/// rate limiter applies to all teams; `Some(set)` restricts it to those teams.
fn parse_team_id_allowlist(value: &str) -> Option<HashSet<i32>> {
if value.is_empty() {
return None;
}
Some(
value
.split(',')
.filter_map(|s| s.trim().parse::<i32>().ok())
.collect(),
)
}
/// A typo must stop startup, because a skipped entry would silently leave that team out.
fn parse_strict_team_id_list(value: &str) -> Result<HashSet<i32>, UnhandledError> {
value
.split(',')
.map(str::trim)
.filter(|entry| !entry.is_empty())
.map(|entry| {
entry.parse::<i32>().map_err(|_| {
UnhandledError::Other(format!("invalid team id {entry:?} in team id list"))
})
})
.collect()
}
async fn build_remote_resolution(
config: &ProcessingConfig,
) -> Result<(Option<RemoteResolutionContext>, Option<JoinHandle<()>>), UnhandledError> {
let remote_config = RemoteResolutionConfig::from_config(config)?;
info!(
host = %remote_config.host,
port = remote_config.port,
deadline_ms = remote_config.request_deadline.as_millis() as u64,
dns_refresh_secs = remote_config.dns_refresh.as_secs(),
max_retries = remote_config.max_retries,
"building remote resolution endpoint pool"
);
let resolver = Arc::new(TokioDnsResolver);
let pool = EndpointPool::new(remote_config.clone(), resolver)
.await
.map_err(|e| {
UnhandledError::Other(format!("failed to build remote resolution pool: {e}"))
})?;
let readiness_timeout = std::cmp::max(
remote_config.subscribe_tick_hint.saturating_mul(3),
remote_config.connect_timeout,
);
pool.wait_ready(readiness_timeout).await.map_err(|e| {
UnhandledError::Other(format!("remote resolution pool did not become ready: {e}"))
})?;
let refresh_task = crate::stages::resolution::remote::pool::spawn_refresh_task(pool.clone());
Ok((
Some(RemoteResolutionContext::new(pool, remote_config)),
Some(refresh_task),
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strict_team_id_list_parses_valid_input() {
for (input, expected) in [("", vec![]), ("2", vec![2]), (" 2, 3 ,", vec![2, 3])] {
assert_eq!(
parse_strict_team_id_list(input).unwrap(),
expected.into_iter().collect::<HashSet<i32>>(),
"input {input:?}"
);
}
}
#[test]
fn strict_team_id_list_rejects_a_typo() {
for input in ["2,x3", "2;3", "two"] {
assert!(parse_strict_team_id_list(input).is_err(), "input {input:?}");
}
}
}