:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::num::NonZeroUsize;
use aws_config::{BehaviorVersion, Region};
use common_database::{get_pool_with_config, PoolConfig};
use envconfig::Envconfig;
use sqlx::PgPool;
use tracing::{info, warn};
/// Configuration for the shared symbol-resolution stack. Both run modes parse
/// this slice; processing mode nests it on [`crate::modes::processing::ProcessingConfig`], and
/// resolution mode reads it alongside its own service config. Keeping it in
/// `core` means the resolver (and a future `cymbal-core` crate) never needs the
/// processing-only knobs (Kafka, Redis, rules, remote-resolution client).
#[derive(Envconfig, Clone)]
pub struct ResolverConfig {
#[envconfig(default = "postgres://posthog:posthog@localhost:5432/posthog")]
pub database_url: String,
// Rust services connect directly to postgres, not via pgbouncer, so we keep this low
#[envconfig(default = "4")]
pub max_pg_connections: u32,
// cymbal makes HTTP get requests to auto-resolve sourcemaps - and follows redirects. To protect against SSRF, we only allow requests to public URLs by default
#[envconfig(default = "false")]
pub allow_internal_ips: bool,
#[envconfig(default = "30")]
pub sourcemap_timeout_seconds: u64,
#[envconfig(default = "5")]
pub sourcemap_connect_timeout_seconds: u64,
#[envconfig(default = "25000000")]
pub sourcemap_max_response_bytes: usize,
#[envconfig(default = "100000000")] // 100MB - in prod, we should use closer to 1-10GB
pub symbol_store_cache_max_bytes: usize,
#[envconfig(default = "http://127.0.0.1:19000")] // minio
pub object_storage_endpoint: String,
#[envconfig(default = "symbol_sets")]
pub object_storage_bucket: String,
#[envconfig(default = "us-east-1")]
pub object_storage_region: String,
#[envconfig(default = "object_storage_root_user")]
pub object_storage_access_key_id: String,
#[envconfig(default = "object_storage_root_password")]
pub object_storage_secret_access_key: String,
#[envconfig(default = "false")] // Enable for MinIO compatibility
pub object_storage_force_path_style: bool,
#[envconfig(default = "symbolsets")]
pub ss_prefix: String,
#[envconfig(default = "100000")]
pub frame_cache_size: u64,
// Used by cymbal and cymbal-resolution through the shared symbol resolver config.
// Resolved frame results are relatively stable, while unresolved results can become
// resolvable after a user uploads missing symbols, so keep them shorter.
#[envconfig(default = "1800")]
pub frame_resolved_ttl_seconds: u64,
#[envconfig(default = "300")]
pub frame_unresolved_ttl_seconds: u64,
// When enabled, a re-resolution that produced an unchanged frame snapshot refreshes
// created_at with a timestamp-only UPDATE instead of rewriting every row's full
// contents, and skips the write entirely when no row needs a freshness refresh yet.
#[envconfig(default = "false")]
pub skip_unchanged_frame_rewrites: bool,
// TTL for the in-memory negative cache of symbol-set lookup failures in the `Saving` layer.
// Like `frame_unresolved_ttl_seconds`, this is kept short: a failure record can become
// resolvable the moment a user uploads the missing symbols, so a cached negative result
// must not outlive that upload by long.
#[envconfig(default = "300")]
pub symbol_set_negative_cache_ttl_seconds: u64,
// TTL for the (team, symbol-set refs) -> latest-release-id cache in the resolution
// service. Kept short because a release (re)bound to already-cached refs only becomes
// visible after expiry; new releases usually ship new refs, which miss naturally.
#[envconfig(default = "60")]
pub release_id_cache_ttl_seconds: u64,
// Byte budget for that cache, weighed by the ref bytes each entry holds. Refs are
// event-controlled (frame source URLs, chunk ids), so an entry-count bound would let a
// flood of unique long refs grow the cache without limit. 32 MiB comfortably holds the
// working set of a 60s TTL at typical ref-set sizes.
#[envconfig(default = "33554432")]
pub release_id_cache_max_bytes: u64,
// Maximum number of lines of pre and post context to get per frame
#[envconfig(default = "15")]
pub context_line_count: usize,
// Global maximum number of concurrent symbol resolution operations.
// This limiter is shared across frame and exception symbol resolution paths.
#[envconfig(default = "64")]
pub symbol_resolution_concurrency: usize,
// Parsing a symbol set can need many times its decompressed size in memory, and the symbol
// store cache does not count that memory. Parses of symbol sets at least this large
// (decompressed) reserve their estimated peak memory from a shared budget, so together
// they cannot run the pod out of memory.
#[envconfig(default = "10000000")]
pub symbol_set_large_parse_bytes: NonZeroUsize,
#[envconfig(default = "2")]
pub symbol_set_max_concurrent_large_parses: NonZeroUsize,
// Set this below the memory limit minus the symbol store cache budget and the steady-state usage.
#[envconfig(default = "1000000000")]
pub symbol_set_parse_memory_budget_bytes: NonZeroUsize,
// Estimated peak parse memory, as a multiple of the decompressed symbol set size.
#[envconfig(default = "8")]
pub symbol_set_parse_memory_multiplier: NonZeroUsize,
// A small compressed symbol set can decompress to many gigabytes, and the size its container
// declares is written by the uploader. A parse fails instead of decompressing a container
// payload past this many bytes.
#[envconfig(default = "536870912")] // 512 MiB
pub symbol_set_max_decompressed_bytes: NonZeroUsize,
// Native and Apple symbol sets are a ZIP of deflated entries inside the container, so they
// expand a second time. A parse reads at most this many bytes out of the ZIP, summed over
// every entry it reads.
#[envconfig(default = "1610612736")] // 1.5 GiB
pub symbol_set_max_extracted_bytes: NonZeroUsize,
// Shared secret authenticating the cymbal <-> cymbal-resolution gRPC seam.
#[envconfig(default = "")]
pub internal_api_secret: String,
// Comma-separated previous secrets still accepted for verification during zero-downtime rotation.
#[envconfig(default = "")]
pub internal_api_secret_fallbacks: String,
}
impl ResolverConfig {
pub fn init_with_defaults() -> Result<Self, envconfig::Error> {
Self::init_from_env()
}
}
/// Build the Postgres pool through the shared `common-database` helper so cymbal
/// gets `test_before_acquire` (severed connections are health-checked and
/// recycled instead of handed straight to a query), idle recycling, and the
/// disabled max-lifetime that avoids thundering-herd reconnects. Building the
/// pool directly with `PgPoolOptions` skips all of that, which is how a stale
/// connection turns into an `expected to read N bytes, got 0 bytes at EOF`
/// error mid-query. `pool_name` labels the connection-churn counter.
pub fn build_pg_pool(
database_url: &str,
max_connections: u32,
pool_name: &str,
) -> Result<PgPool, sqlx::Error> {
let pool_config = PoolConfig {
max_connections,
pool_name: Some(pool_name.to_string()),
..Default::default()
};
get_pool_with_config(database_url, pool_config)
}
pub async fn get_aws_config(config: &ResolverConfig) -> aws_sdk_s3::Config {
// If we have a role ARN and token file, which are added to the container due to the SA annotation we use in prod
if std::env::var("AWS_ROLE_ARN").is_ok() && std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE").is_ok()
{
info!("AWS role and token file detected, config loaded from environment variables");
// Use default aws config loading behaviour, which should pick up the role-based credentials. We
// assume region and endpoint will be properly set due to SA annotation. Behaviour version will
// be latest due to config crate feature flag
aws_sdk_s3::config::Builder::from(&aws_config::load_from_env().await)
.force_path_style(config.object_storage_force_path_style)
.build()
} else {
warn!("Falling back to building config from explicit environment variables");
// Fall back to building our config from the explicit environment variables we use in local dev
let env_credentials = aws_sdk_s3::config::Credentials::new(
&config.object_storage_access_key_id,
&config.object_storage_secret_access_key,
None,
None,
"environment",
);
aws_sdk_s3::config::Builder::new()
.region(Region::new(config.object_storage_region.clone()))
.endpoint_url(&config.object_storage_endpoint)
.credentials_provider(env_credentials)
.behavior_version(BehaviorVersion::latest())
.force_path_style(config.object_storage_force_path_style)
.build()
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
#[test]
fn parse_limiter_settings_must_be_positive() {
for name in [
"SYMBOL_SET_LARGE_PARSE_BYTES",
"SYMBOL_SET_MAX_CONCURRENT_LARGE_PARSES",
"SYMBOL_SET_PARSE_MEMORY_BUDGET_BYTES",
"SYMBOL_SET_PARSE_MEMORY_MULTIPLIER",
] {
let values = HashMap::from([(name.to_owned(), "0".to_owned())]);
assert!(matches!(
ResolverConfig::init_from_hashmap(&values),
Err(envconfig::Error::ParseError { name: field }) if field == name
));
}
}
}