:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::api::api_key_usage::ApiKeyKind;
use std::{
future::ready,
panic::{catch_unwind, AssertUnwindSafe},
sync::Arc,
time::Duration,
};
use crate::billing::{BillingAggregator, FeatureFlagsLimiter, SessionReplayLimiter};
use crate::database_pools::DatabasePools;
use axum::{
error_handling::HandleErrorLayer,
extract::{DefaultBodyLimit, Extension, Request},
http::{header, HeaderValue, Method, StatusCode},
middleware::Next,
response::Response,
routing::{any, get, post},
Router,
};
use common_cache::{NegativeCache, ReadThroughCacheWithMetrics};
use common_cookieless::CookielessManager;
use common_geoip::GeoIpClient;
use common_hypercache::HyperCacheReader;
use common_metrics::inc;
use common_metrics::setup_metrics_routes_for_product_with_overrides;
use common_redis::Client as RedisClient;
use governor::clock;
use lifecycle::{LivenessHandler, ReadinessHandler};
use metrics::gauge;
use sqlx::PgPool;
use tower::limit::ConcurrencyLimitLayer;
use tower::timeout::TimeoutLayer;
use tower::ServiceBuilder;
use tower_http::{
cors::{AllowHeaders, AllowOrigin, CorsLayer},
trace::TraceLayer,
};
use crate::{
api::{
batch_flag_evaluation,
body_read_metrics::{record_body_read, MAX_FLAGS_BODY_BYTES},
concurrency_metrics::{record_concurrency_enter, record_concurrency_wait},
endpoint, flag_definitions,
flag_definitions::DefinitionsBillableCache,
flag_definitions_rate_limiter::{FlagDefinitionsRateLimiter, RemoteConfigRateLimiter},
flags_rate_limiter::{FlagsRateLimiter, IpRateLimiter},
remote_config,
request_outcome::{precreate_request_outcome_counters, record_request_outcome},
},
cohorts::{cohort_cache_manager::CohortCacheManager, membership::CohortMembershipProvider},
config::{Config, ServiceMode, TeamIdCollection},
flags::{
flag_definitions_cache::FlagDefinitionsCache,
flag_group_type_mapping::GroupTypeCacheManager,
flag_payload_decryptor::{FlagPayloadDecryptor, FlagPayloadDecryptorError},
flag_service::FlagService,
},
handler::body_logger::BodyLogger,
metrics::{
buckets::bucket_overrides,
consts::{
FLAG_DEFINITIONS_RATE_LIMITED_COUNTER, FLAG_DEFINITIONS_RATE_LIMIT_BYPASSED_COUNTER,
FLAG_DEFINITIONS_REQUESTS_COUNTER, FLAG_REQUEST_TIMEOUT_COUNTER,
REMOTE_CONFIG_RATE_LIMITED_COUNTER, REMOTE_CONFIG_RATE_LIMIT_BYPASSED_COUNTER,
REMOTE_CONFIG_REQUESTS_COUNTER,
},
utils::team_id_label_filter,
},
rayon_dispatcher::RayonDispatcher,
utils::bot_detection,
};
/// `/flags` and its legacy `/decide` alias, with and without a trailing slash.
pub(crate) const FLAGS_ROUTES: [&str; 4] = ["/flags", "/flags/", "/decide", "/decide/"];
#[derive(Clone)]
pub struct FlagsEndpointRateLimiters<C = clock::DefaultClock>
where
C: clock::Clock,
{
pub(crate) token: FlagsRateLimiter<C>,
pub(crate) ip: IpRateLimiter<C>,
}
#[derive(Clone)]
pub struct State {
// Shared Redis for non-critical path (analytics counters, billing limits)
// ReadWriteClient automatically routes reads to replica and writes to primary
pub redis_client: Arc<dyn RedisClient + Send + Sync>,
// Dedicated Redis for flags cache (critical path isolation)
// None if not configured (falls back to shared Redis)
// ReadWriteClient automatically routes reads to replica and writes to primary
pub dedicated_redis_client: Option<Arc<dyn RedisClient + Send + Sync>>,
pub database_pools: Arc<DatabasePools>,
pub cohort_cache_manager: Arc<CohortCacheManager>,
pub group_type_cache_manager: Arc<GroupTypeCacheManager>,
pub geoip: Arc<GeoIpClient>,
pub team_ids_to_track: TeamIdCollection,
pub feature_flags_billing_limiter: FeatureFlagsLimiter,
pub session_replay_billing_limiter: SessionReplayLimiter,
pub cookieless_manager: Arc<CookielessManager>,
pub(crate) flag_definitions_full_limiter: FlagDefinitionsRateLimiter,
/// Per-team limiter for flag definitions requests with an ETag in If-None-Match.
/// Separate budget so ETag revalidation polls don't consume the full-response budget.
pub(crate) flag_definitions_conditional_limiter: FlagDefinitionsRateLimiter,
/// Per-credential limiter (keyed on the personal API key id) for the remote_config endpoint,
/// mirroring Django's RemoteConfigThrottle. Separate budget from flag definitions.
pub(crate) remote_config_limiter: RemoteConfigRateLimiter,
pub config: Config,
/// Pre-initialized HyperCacheReader for feature flags (flags.json)
/// Initialized once at startup to avoid per-request AWS SDK initialization
pub flags_hypercache_reader: Arc<HyperCacheReader>,
/// In-memory cache for deserialized + regex-compiled flag definitions
pub flag_definitions_cache: Arc<FlagDefinitionsCache>,
/// Pre-initialized HyperCacheReader for feature flags with cohorts (flags_with_cohorts.json)
/// Used by the /flags/definitions endpoint
pub flags_with_cohorts_hypercache_reader: Arc<HyperCacheReader>,
/// Billable status of each team's current flag definitions, keyed by ETag, so a 304 on
/// /flags/definitions for survey-only or product-tour-only definitions goes unbilled
/// without a payload read on every poll
pub definitions_billable_cache: DefinitionsBillableCache,
/// Pre-initialized HyperCacheReader for team metadata (full_metadata.json)
/// Uses token-based lookup instead of team_id
pub team_hypercache_reader: Arc<HyperCacheReader>,
/// Pre-initialized HyperCacheReader for remote config (array/config.json)
/// Reads pre-computed config from Python's RemoteConfig.build_config()
/// Uses token-based lookup (api_token)
pub config_hypercache_reader: Arc<HyperCacheReader>,
/// Bounds concurrent large-batch dispatches to the Rayon pool
pub rayon_dispatcher: RayonDispatcher,
/// In-memory negative cache for invalid API tokens, preventing repeated
/// Redis/S3/PG lookups for tokens that don't correspond to any team
pub team_negative_cache: NegativeCache,
/// Read-through cache for auth tokens (secret + personal API keys).
/// Handles Redis-backed positive caching and loader ordering.
pub auth_token_cache: Arc<ReadThroughCacheWithMetrics>,
/// Provider for realtime/behavioral cohort membership lookups
pub cohort_membership_provider: Arc<dyn CohortMembershipProvider>,
/// Authoritative writer for the production billing keyspace. See
/// `crate::billing` for the aggregation/flush contract.
pub billing_aggregator: Arc<BillingAggregator>,
/// Per-team request/response body logging for `/flags`. Refreshed
/// every ~60s from `posthog_instancesetting`.
pub body_logger: Arc<BodyLogger>,
/// Decrypts encrypted remote-config flag payloads. `None` when no keys are
/// configured (e.g. local dev without FLAGS_SECRET_KEYS/SECRET_KEY); in that
/// case encrypted payloads cannot be served and the handler errors.
pub flag_payload_decryptor: Option<FlagPayloadDecryptor>,
}
impl State {
/// The Redis cluster the flags namespace lives in: the dedicated flags cluster when its
/// client exists, and the shared cluster when it does not. `server.rs` repeats this
/// derivation inline for the flags.json, team_metadata, and remote-config readers and for
/// the auth token cache, because those are built before `State` exists.
///
/// The client is absent for two different reasons. `FLAGS_REDIS_URL` can be unset, and the
/// dedicated cluster can be unreachable at process start (`create_dedicated_readwrite_client`
/// in `server.rs`, which logs that failure at error level). Only the first reason keeps this
/// process and Django on one cluster. The second sends every caller here to the shared
/// cluster for the life of the process, while Django keeps using the dedicated one.
pub(crate) fn flags_namespace_redis_client(&self) -> Arc<dyn RedisClient + Send + Sync> {
self.dedicated_redis_client
.clone()
.unwrap_or_else(|| self.redis_client.clone())
}
/// Builds a `FlagService` from shared state. Centralized so every endpoint gets the
/// same caching/fallback config instead of copying the constructor per handler.
pub(crate) fn flag_service(&self) -> FlagService {
FlagService::new(
self.redis_client.clone(),
self.database_pools.non_persons_reader.clone(),
self.team_hypercache_reader.clone(),
self.flags_hypercache_reader.clone(),
self.flag_definitions_cache.clone(),
self.team_negative_cache.clone(),
*self.config.skip_pg_team_fallback,
)
}
/// Records API key usage (`last_used_at`), gated on `skip_writes`. Centralized so the
/// API key auth paths (`flag_definitions`, `remote_config`) share one set of gating and
/// client choices instead of copying them per handler. Advisory: uses the shared Redis client
/// (not the flags cache) and the non-persons writer, and the DB write only fires when the
/// Redis debounce key is newly set.
pub(crate) async fn record_api_key_last_used(&self, kind: ApiKeyKind, key_id: String) {
if *self.config.skip_writes {
return;
}
let redis = self.redis_client.clone();
let pg_writer: Arc<dyn common_database::Client + Send + Sync> =
self.database_pools.non_persons_writer.clone();
drop(
crate::api::api_key_usage::record_api_key_last_used(redis, pg_writer, kind, key_id)
.await,
);
}
/// Stamps a project secret API key when the `phs_` token resolved to one. A team-level secret
/// token carries no key id, so it records nothing.
pub(crate) async fn record_project_secret_key_usage(&self, key_id: Option<String>) {
if let Some(key_id) = key_id {
self.record_api_key_last_used(ApiKeyKind::ProjectSecret, key_id)
.await;
}
}
}
#[allow(clippy::too_many_arguments)]
pub fn router(
redis_client: Arc<dyn RedisClient + Send + Sync>,
dedicated_redis_client: Option<Arc<dyn RedisClient + Send + Sync>>,
database_pools: Arc<DatabasePools>,
cohort_cache: Arc<CohortCacheManager>,
group_type_cache: Arc<GroupTypeCacheManager>,
geoip: Arc<GeoIpClient>,
readiness: ReadinessHandler,
liveness: LivenessHandler,
feature_flags_billing_limiter: FeatureFlagsLimiter,
session_replay_billing_limiter: SessionReplayLimiter,
cookieless_manager: Arc<CookielessManager>,
flags_hypercache_reader: Arc<HyperCacheReader>,
flag_definitions_cache: Arc<FlagDefinitionsCache>,
flags_with_cohorts_hypercache_reader: Arc<HyperCacheReader>,
team_hypercache_reader: Arc<HyperCacheReader>,
config_hypercache_reader: Arc<HyperCacheReader>,
rayon_dispatcher: RayonDispatcher,
team_negative_cache: NegativeCache,
auth_token_cache: Arc<ReadThroughCacheWithMetrics>,
cohort_membership_provider: Arc<dyn CohortMembershipProvider>,
billing_aggregator: Arc<BillingAggregator>,
config: Config,
) -> Router {
router_with_rate_limiter_clock(
redis_client,
dedicated_redis_client,
database_pools,
cohort_cache,
group_type_cache,
geoip,
readiness,
liveness,
feature_flags_billing_limiter,
session_replay_billing_limiter,
cookieless_manager,
flags_hypercache_reader,
flag_definitions_cache,
flags_with_cohorts_hypercache_reader,
team_hypercache_reader,
config_hypercache_reader,
rayon_dispatcher,
team_negative_cache,
auth_token_cache,
cohort_membership_provider,
billing_aggregator,
config,
clock::DefaultClock::default(),
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn router_with_rate_limiter_clock<C>(
redis_client: Arc<dyn RedisClient + Send + Sync>,
dedicated_redis_client: Option<Arc<dyn RedisClient + Send + Sync>>,
database_pools: Arc<DatabasePools>,
cohort_cache: Arc<CohortCacheManager>,
group_type_cache: Arc<GroupTypeCacheManager>,
geoip: Arc<GeoIpClient>,
readiness: ReadinessHandler,
liveness: LivenessHandler,
feature_flags_billing_limiter: FeatureFlagsLimiter,
session_replay_billing_limiter: SessionReplayLimiter,
cookieless_manager: Arc<CookielessManager>,
flags_hypercache_reader: Arc<HyperCacheReader>,
flag_definitions_cache: Arc<FlagDefinitionsCache>,
flags_with_cohorts_hypercache_reader: Arc<HyperCacheReader>,
team_hypercache_reader: Arc<HyperCacheReader>,
config_hypercache_reader: Arc<HyperCacheReader>,
rayon_dispatcher: RayonDispatcher,
team_negative_cache: NegativeCache,
auth_token_cache: Arc<ReadThroughCacheWithMetrics>,
cohort_membership_provider: Arc<dyn CohortMembershipProvider>,
billing_aggregator: Arc<BillingAggregator>,
config: Config,
rate_limiter_clock: C,
) -> Router
where
C: clock::Clock + Clone + Send + Sync + 'static,
{
// Initialize flag definitions rate limiter with default and custom team rates
let flag_definitions_full_limiter = FlagDefinitionsRateLimiter::new(
config.flag_definitions_default_rate_per_minute,
config.flag_definitions_rate_limits.0.clone(),
config.rate_limiting_allow_list_teams.0.clone(),
FLAG_DEFINITIONS_REQUESTS_COUNTER,
FLAG_DEFINITIONS_RATE_LIMITED_COUNTER,
FLAG_DEFINITIONS_RATE_LIMIT_BYPASSED_COUNTER,
)
.expect("Failed to initialize flag definitions rate limiter")
.with_labels(&[("budget", "full")]);
// Both limiters share metric names, so dashboards that sum the counters still see every request.
let flag_definitions_conditional_limiter = FlagDefinitionsRateLimiter::new(
config.flag_definitions_conditional_rate_per_minute,
config.flag_definitions_conditional_rate_limits.0.clone(),
config.rate_limiting_allow_list_teams.0.clone(),
FLAG_DEFINITIONS_REQUESTS_COUNTER,
FLAG_DEFINITIONS_RATE_LIMITED_COUNTER,
FLAG_DEFINITIONS_RATE_LIMIT_BYPASSED_COUNTER,
)
.expect("Failed to initialize flag definitions conditional rate limiter")
.with_labels(&[("budget", "conditional")]);
// Per-credential limiter for the remote_config endpoint (mirrors Django's
// RemoteConfigThrottle, which buckets per hashed bearer token). The team allowlist is
// applied by the handler, not the limiter, so this is constructed with an empty allowlist
// and no per-key overrides.
let remote_config_limiter = RemoteConfigRateLimiter::new(
config.remote_config_default_rate_per_minute,
std::collections::HashMap::new(),
std::collections::HashSet::new(),
REMOTE_CONFIG_REQUESTS_COUNTER,
REMOTE_CONFIG_RATE_LIMITED_COUNTER,
REMOTE_CONFIG_RATE_LIMIT_BYPASSED_COUNTER,
)
.expect("Failed to initialize remote config rate limiter");
// Initialize token-based rate limiter.
// Both modes use the same thresholds (warn at ratio of enforce capacity).
// log_only=true sets warn_only, so requests are never blocked.
let (flags_warn_cap, flags_enforce_cap, flags_warn_only) = resolve_rate_limit_capacities(
config.flags_bucket_capacity,
config.flags_warn_capacity_ratio,
*config.flags_rate_limit_log_only,
);
let flags_rate_limiter = FlagsRateLimiter::new_with_clock(
*config.flags_rate_limit_enabled,
config.flags_bucket_replenish_rate,
flags_warn_cap,
flags_enforce_cap,
flags_warn_only,
config.flags_token_rate_limit_overrides.0.clone(),
rate_limiter_clock.clone(),
)
.unwrap_or_else(|e| {
panic!(
"Invalid token-based rate limit configuration: {e}. \
Check FLAGS_BUCKET_REPLENISH_RATE (must be > 0) and FLAGS_BUCKET_CAPACITY (must be > 0)"
)
});
// Initialize IP-based rate limiter with backwards-compatible configuration
let (ip_warn_cap, ip_enforce_cap, ip_warn_only) = resolve_rate_limit_capacities(
config.flags_ip_burst_size,
config.flags_warn_capacity_ratio,
*config.flags_ip_rate_limit_log_only,
);
let ip_rate_limiter = IpRateLimiter::new_with_clock(
*config.flags_ip_rate_limit_enabled,
config.flags_ip_replenish_rate,
ip_warn_cap,
ip_enforce_cap,
ip_warn_only,
rate_limiter_clock,
)
.unwrap_or_else(|e| {
panic!(
"Invalid IP-based rate limit configuration: {e}. \
Check FLAGS_IP_REPLENISH_RATE (must be > 0) and FLAGS_IP_BURST_SIZE (must be > 0)"
)
});
spawn_rate_limiter_cleanup_task(
flags_rate_limiter.clone(),
ip_rate_limiter.clone(),
flag_definitions_full_limiter.clone(),
flag_definitions_conditional_limiter.clone(),
remote_config_limiter.clone(),
config.rate_limiter_cleanup_interval_secs,
);
let rate_limiters = FlagsEndpointRateLimiters {
token: flags_rate_limiter,
ip: ip_rate_limiter,
};
// Force eager construction of the bot UA matcher and IP-range table so
// the first `/flags` request after a pod restart doesn't pay the
// (sub-ms) build cost on its hot path.
bot_detection::warm_caches();
let body_logger = Arc::new(BodyLogger::new(
config.flags_log_bodies_teams.clone(),
config.flags_log_bodies_request_max_bytes,
));
// 1 query per pod per interval regardless of /flags RPS.
body_logger
.clone()
.spawn_refresh_task(database_pools.non_persons_reader.clone());
// Build the remote-config payload decryptor from FLAGS_SECRET_KEYS (or SECRET_KEY
// fallback). Malformed keys fail boot loudly (the documented prep trap); a fully
// empty config is tolerated (local dev) — encrypted payloads just can't be served.
let flag_payload_decryptor = match FlagPayloadDecryptor::from_config(
&config.flags_secret_keys,
&config.secret_key,
) {
Ok(d) => Some(d),
Err(FlagPayloadDecryptorError::NoKeys) => {
tracing::warn!(
"No FLAGS_SECRET_KEYS or SECRET_KEY configured; encrypted remote config payloads cannot be decrypted"
);
None
}
Err(e) => panic!("Invalid FLAGS_SECRET_KEYS configuration: {e}"),
};
let state = State {
redis_client,
dedicated_redis_client,
database_pools,
cohort_cache_manager: cohort_cache,
group_type_cache_manager: group_type_cache,
geoip,
team_ids_to_track: config.team_ids_to_track.clone(),
feature_flags_billing_limiter,
session_replay_billing_limiter,
cookieless_manager,
flag_definitions_full_limiter,
flag_definitions_conditional_limiter,
remote_config_limiter,
config: config.clone(),
flags_hypercache_reader,
flag_definitions_cache,
flags_with_cohorts_hypercache_reader,
team_hypercache_reader,
config_hypercache_reader,
rayon_dispatcher,
team_negative_cache,
definitions_billable_cache: DefinitionsBillableCache::new(
config.definitions_billable_cache_capacity,
std::time::Duration::from_secs(config.definitions_billable_cache_ttl_seconds),
),
cohort_membership_provider,
auth_token_cache,
billing_aggregator,
body_logger,
flag_payload_decryptor,
};
// Very permissive CORS policy, as old SDK versions
// and reverse proxies might send funky headers.
let cors = CorsLayer::new()
.allow_methods([Method::GET, Method::POST, Method::OPTIONS, Method::HEAD])
.allow_headers(AllowHeaders::mirror_request())
.allow_credentials(true)
.allow_origin(AllowOrigin::mirror_request())
.expose_headers([axum::http::HeaderName::from_static(
"x-posthog-rate-limit-warning",
)]);
// Clone database_pools for the startup route
let db_pools_for_startup = state.database_pools.clone();
// liveness/readiness/startup checks
let status_router = Router::new()
.route("/", get(index))
.route(
"/_readiness",
get({
let r = readiness.clone();
move || {
let r = r.clone();
async move { r.check().await }
}
}),
)
.route(
"/_liveness",
get({
let l = liveness.clone();
move || ready(l.check())
}),
)
.route(
"/_startup",
get(move || startup(db_pools_for_startup.clone())),
);
// Layer ordering (outermost to innermost, last `.layer()` is outermost):
// record_request_outcome → HandleErrorLayer → TimeoutLayer →
// record_concurrency_enter → ConcurrencyLimitLayer →
// record_concurrency_wait → (per-sub-router).
//
// The body-read shim and `DefaultBodyLimit` are per-sub-router and only
// attached to `/flags|/decide`. `/flags/definitions` is GET-only and
// 405s non-GET before any body is read. `DefaultBodyLimit::max` is the
// marker the handler's `Bytes` extractor reads; the shim's `to_bytes`
// cap (see `MAX_FLAGS_BODY_BYTES`) handles the same boundary while the
// body is being buffered. Both are required.
let mut flags_endpoints: Router<State> = Router::new();
if matches!(config.service_mode, ServiceMode::All | ServiceMode::Flags) {
for route in FLAGS_ROUTES {
flags_endpoints = flags_endpoints.route(route, any(endpoint::flags::<C>));
}
flags_endpoints = flags_endpoints
.layer(axum::middleware::from_fn(record_body_read))
.layer(DefaultBodyLimit::max(MAX_FLAGS_BODY_BYTES))
.layer(Extension(rate_limiters));
}
// Internal-only batch flag evaluation for static cohort generation. Kept outside
// `flags_router` on purpose: it must not share the live path's concurrency limit or
// request timeout (a page evaluates up to BATCH_FLAG_EVAL_MAX_LIMIT persons), and it
// bypasses the /flags pipeline (no billing, no flag analytics). Auth is enforced in
// the handler before the body is read: the handler accepts a raw `Body` (not `Bytes`)
// so unauthenticated requests are rejected without buffering the payload.
let mut internal_endpoints: Router<State> = Router::new();
if matches!(config.service_mode, ServiceMode::All | ServiceMode::Flags) {
internal_endpoints = internal_endpoints
.route(
"/internal/batch_flag_evaluation",
post(batch_flag_evaluation::batch_flag_evaluation),
)
.layer(DefaultBodyLimit::max(
batch_flag_evaluation::MAX_BATCH_BODY_BYTES,
))
.layer(
ServiceBuilder::new()
.layer(HandleErrorLayer::new(|err: tower::BoxError| async move {
let is_timeout = err.is::<tower::timeout::error::Elapsed>();
tracing::warn!(error = %err, timeout = is_timeout, "Batch flag evaluation request aborted by tower layer");
let body = if is_timeout {
"Request timed out"
} else {
"Service unavailable"
};
(StatusCode::SERVICE_UNAVAILABLE, body)
}))
.layer(TimeoutLayer::new(Duration::from_millis(
config.batch_flag_eval_timeout_ms,
))),
);
}
let mut definitions_endpoints: Router<State> = Router::new();
if matches!(
config.service_mode,
ServiceMode::All | ServiceMode::Definitions
) {
definitions_endpoints = definitions_endpoints
.route(
"/flags/definitions",
any(flag_definitions::flags_definitions),
)
.route(
"/flags/definitions/",
any(flag_definitions::flags_definitions),
)
// Alias for Django's local_evaluation endpoint — allows Contour to route
// traffic to Rust without path rewriting (same pattern as /decide → /flags)
.route(
"/api/feature_flag/local_evaluation",
any(flag_definitions::flags_definitions),
)
.route(
"/api/feature_flag/local_evaluation/",
any(flag_definitions::flags_definitions),
)
// Alias for Django's remote_config endpoint — Contour routes to Rust
// without path rewriting (same pattern as local_evaluation).
.route(
"/api/projects/:project_id/feature_flags/:key/remote_config",
any(remote_config::remote_config),
)
.route(
"/api/projects/:project_id/feature_flags/:key/remote_config/",
any(remote_config::remote_config),
);
}
let flags_router = flags_endpoints
.merge(definitions_endpoints)
// After `ConcurrencyLimitLayer` releases a permit, so this measures
// permit wait excluding body buffering.
.layer(axum::middleware::from_fn(record_concurrency_wait))
.layer(ConcurrencyLimitLayer::new(config.max_concurrency))
// Stamps `Instant::now()` after timeout-deadline propagation but
// before permit acquisition. The flags handler caps the persons DB
// deadline relative to this instant. That cap holds only while this
// layer sits directly inside `TimeoutLayer`.
.layer(axum::middleware::from_fn(record_concurrency_enter))
.layer(
ServiceBuilder::new()
.layer(HandleErrorLayer::new(|err: tower::BoxError| async move {
let is_timeout = err.is::<tower::timeout::error::Elapsed>();
tracing::warn!(error = %err, timeout = is_timeout, "Request aborted by tower layer");
if is_timeout {
inc(FLAG_REQUEST_TIMEOUT_COUNTER, &[], 1);
}
let body = if is_timeout {
"Request timed out"
} else {
"Service unavailable"
};
(StatusCode::SERVICE_UNAVAILABLE, body)
}))
.layer(TimeoutLayer::new(Duration::from_millis(
config.request_timeout_ms,
))),
)
// Outside the timeout, so it also counts the requests that the timeout drops.
.layer(axum::middleware::from_fn(record_request_outcome));
let router = Router::new()
.merge(status_router)
.merge(flags_router)
.merge(internal_endpoints)
.layer(TraceLayer::new_for_http())
.layer(cors)
.layer(axum::middleware::from_fn_with_state(
readiness,
close_connection_unless_ready,
))
.with_state(state);
// Don't install metrics unless asked to
// Global metrics recorders can play poorly with e.g. tests
// In other words, only turn these on in production
if config.enable_metrics {
common_metrics::set_label_filter(team_id_label_filter(config.team_ids_to_track.clone()));
let router = setup_metrics_routes_for_product_with_overrides(
router,
"feature_flags",
&bucket_overrides(),
);
// After the recorder exists.
precreate_request_outcome_counters();
router
} else {
router
}
}
/// Resolves warn/enforce capacities with backwards-compat logic for the old `log_only` flag.
///
/// Returns `(warn_capacity, enforce_capacity, warn_only)`:
/// - Both modes use the same threshold calculation: warn at `warn_ratio` of enforce capacity.
/// - `log_only == true`: Sets `warn_only = true` so requests are never blocked, only warned.
/// Thresholds are identical to enforcing mode, giving operators visibility into what
/// _would_ happen when they flip `log_only` to `false`.
/// - `log_only == false`: Same thresholds, but enforce tier actually blocks with 429s.
/// - Set `warn_ratio` to 0.0 to disable the warn tier entirely.
fn resolve_rate_limit_capacities(
enforce_capacity: u32,
warn_ratio: f64,
log_only: bool,
) -> (Option<u32>, u32, bool) {
let derived_warn = (enforce_capacity as f64 * warn_ratio.clamp(0.0, 1.0)) as u32;
let warn_capacity = if derived_warn > 0 {
Some(derived_warn)
} else {
None
};
(warn_capacity, enforce_capacity, log_only)
}
/// Spawns a background task to periodically clean up stale rate limiter entries.
/// Without this, the rate limiters would accumulate entries for every unique
/// token/IP that makes a request, leading to unbounded memory growth.
/// See: https://docs.rs/governor/latest/governor/struct.RateLimiter.html#method.retain_recent
fn spawn_rate_limiter_cleanup_task<C>(
flags_rate_limiter: FlagsRateLimiter<C>,
ip_rate_limiter: IpRateLimiter<C>,
flag_definitions_full_limiter: FlagDefinitionsRateLimiter,
flag_definitions_conditional_limiter: FlagDefinitionsRateLimiter,
remote_config_limiter: RemoteConfigRateLimiter,
cleanup_interval_secs: u64,
) where
C: clock::Clock + Clone + Send + Sync + 'static,
{
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval_secs));
loop {
interval.tick().await;
let result = catch_unwind(AssertUnwindSafe(|| {
// Remove stale entries and reclaim memory
flags_rate_limiter.cleanup();
ip_rate_limiter.cleanup();
flag_definitions_full_limiter.cleanup();
flag_definitions_conditional_limiter.cleanup();
remote_config_limiter.cleanup();
// Report metrics for monitoring
gauge!("flags_rate_limiter_token_entries").set(flags_rate_limiter.len() as f64);
gauge!("flags_rate_limiter_ip_entries").set(ip_rate_limiter.len() as f64);
gauge!("flags_rate_limiter_definitions_entries", "budget" => "full")
.set(flag_definitions_full_limiter.len() as f64);
gauge!("flags_rate_limiter_definitions_entries", "budget" => "conditional")
.set(flag_definitions_conditional_limiter.len() as f64);
gauge!("flags_rate_limiter_remote_config_entries")
.set(remote_config_limiter.len() as f64);
tracing::debug!(
token_entries = flags_rate_limiter.len(),
ip_entries = ip_rate_limiter.len(),
definitions_full_entries = flag_definitions_full_limiter.len(),
definitions_conditional_entries = flag_definitions_conditional_limiter.len(),
remote_config_entries = remote_config_limiter.len(),
"Rate limiter cleanup completed"
);
}));
if let Err(e) = result {
tracing::error!(
?e,
"Rate limiter cleanup panicked, will retry next interval"
);
}
}
});
}
/// Tests a single database pool by acquiring a connection and optionally running a test query.
///
/// When `skip_query` is true (because `test_before_acquire` is enabled), only connection
/// acquisition is tested since sqlx already validates connections on acquire.
async fn test_pool_connection(pool: &PgPool, name: &str, skip_query: bool) -> Result<(), String> {
let mut conn = pool
.acquire()
.await
.map_err(|e| format!("{name} pool unavailable: {e}"))?;
if !skip_query {
sqlx::query("SELECT 1")
.execute(&mut *conn)
.await
.map_err(|e| format!("{name} connection test failed: {e}"))?;
}
Ok(())
}
/// Asks HTTP/1.1 clients to close the connection after this response once the pod is not
/// ready, so their next request opens a new connection through the Service to a pod that stays.
async fn close_connection_unless_ready(
axum::extract::State(readiness): axum::extract::State<ReadinessHandler>,
request: Request,
next: Next,
) -> Response {
let mut response = next.run(request).await;
if !readiness.is_ready() {
response
.headers_mut()
.insert(header::CONNECTION, HeaderValue::from_static("close"));
}
response
}
pub async fn index() -> &'static str {
"feature flags"
}
/// Startup probe for Kubernetes.
///
/// Warms up database connection pools by acquiring and testing connections.
/// This ensures the first user request doesn't pay connection establishment latency.
///
/// Always returns 200 OK - warmup failures are logged but don't block startup.
/// This preserves the resilience of lazy pool initialization: if the DB is temporarily
/// unavailable at startup, the pod still starts and will connect on first request.
pub async fn startup(database_pools: Arc<DatabasePools>) -> &'static str {
// Check if persons pools are aliased to non-persons pools
// (this happens when persons_db_routing is disabled)
let persons_reader_is_distinct = !Arc::ptr_eq(
&database_pools.non_persons_reader,
&database_pools.persons_reader,
);
let persons_writer_is_distinct = !Arc::ptr_eq(
&database_pools.non_persons_writer,
&database_pools.persons_writer,
);
// Best-effort warmup: try all pools in parallel, log failures, never block startup
let skip_query = database_pools.test_before_acquire;
let (reader_result, writer_result, persons_reader_result, persons_writer_result) = tokio::join!(
test_pool_connection(
&database_pools.non_persons_reader,
"non_persons_reader",
skip_query
),
test_pool_connection(
&database_pools.non_persons_writer,
"non_persons_writer",
skip_query
),
async {
if persons_reader_is_distinct {
test_pool_connection(&database_pools.persons_reader, "persons_reader", skip_query)
.await
} else {
Ok(()) // Already warmed via non_persons_reader
}
},
async {
if persons_writer_is_distinct {
test_pool_connection(&database_pools.persons_writer, "persons_writer", skip_query)
.await
} else {
Ok(()) // Already warmed via non_persons_writer
}
},
);
// Log results
log_warmup_result("non_persons_reader", reader_result);
log_warmup_result("non_persons_writer", writer_result);
if persons_reader_is_distinct {
log_warmup_result("persons_reader", persons_reader_result);
} else {
tracing::info!("persons_reader is aliased to non_persons_reader, already warmed");
}
if persons_writer_is_distinct {
log_warmup_result("persons_writer", persons_writer_result);
} else {
tracing::info!("persons_writer is aliased to non_persons_writer, already warmed");
}
"started"
}
fn log_warmup_result(name: &str, result: Result<(), String>) {
match result {
Ok(()) => tracing::info!("{name} pool warmed up successfully"),
Err(e) => tracing::warn!("{name} warmup failed (will connect on first use): {e}"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_pool_connection_success() {
let database_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://posthog:posthog@localhost:5432/test_database".to_string());
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.connect_lazy(&database_url)
.expect("Failed to create pool");
let result = test_pool_connection(&pool, "test_pool", false).await;
// If database is available, connection test should succeed
// If not, it will fail - both are acceptable in test environment
match result {
Ok(()) => {}
Err(e) => {
assert!(e.contains("test_pool"));
}
}
}
#[tokio::test]
async fn test_pool_connection_failure_includes_pool_name() {
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(std::time::Duration::from_millis(100))
.connect_lazy("postgres://invalid:invalid@localhost:54321/nonexistent")
.expect("Failed to create pool");
let result = test_pool_connection(&pool, "test_pool", false).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(
err.contains("test_pool"),
"Error should contain pool name: {err}"
);
}
#[tokio::test]
async fn test_startup_always_returns_started() {
let invalid_pool = Arc::new(
sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(std::time::Duration::from_millis(100))
.connect_lazy("postgres://invalid:invalid@localhost:54321/nonexistent")
.expect("Failed to create pool"),
);
let database_pools = Arc::new(DatabasePools {
non_persons_reader: invalid_pool.clone(),
non_persons_writer: invalid_pool.clone(),
persons_reader: invalid_pool.clone(),
persons_writer: invalid_pool,
behavioral_cohorts_reader: None,
test_before_acquire: false,
});
let result = startup(database_pools).await;
assert_eq!(result, "started");
}
#[tokio::test]
async fn test_startup_with_aliased_pools() {
let shared_pool = Arc::new(
sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(std::time::Duration::from_millis(100))
.connect_lazy("postgres://invalid:invalid@localhost:54321/nonexistent")
.expect("Failed to create pool"),
);
let database_pools = Arc::new(DatabasePools {
non_persons_reader: shared_pool.clone(),
non_persons_writer: shared_pool.clone(),
persons_reader: shared_pool.clone(),
persons_writer: shared_pool,
behavioral_cohorts_reader: None,
test_before_acquire: false,
});
let result = startup(database_pools).await;
assert_eq!(result, "started");
}
#[test]
fn test_resolve_rate_limit_default_ratio() {
// 80% of 200 = 160
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(200, 0.8, false);
assert_eq!(warn, Some(160));
assert_eq!(enforce, 200);
assert!(!warn_only);
}
#[test]
fn test_resolve_rate_limit_custom_ratio() {
// 40% of 500 = 200
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(500, 0.4, false);
assert_eq!(warn, Some(200));
assert_eq!(enforce, 500);
assert!(!warn_only);
}
#[test]
fn test_resolve_rate_limit_zero_ratio_disables_warn() {
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(500, 0.0, false);
assert_eq!(warn, None);
assert_eq!(enforce, 500);
assert!(!warn_only);
}
#[test]
fn test_resolve_rate_limit_ratio_clamped_to_1() {
// ratio > 1.0 is clamped to 1.0, so warn == enforce
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(200, 1.5, false);
assert_eq!(warn, Some(200));
assert_eq!(enforce, 200);
assert!(!warn_only);
}
#[test]
fn test_resolve_rate_limit_log_only_uses_real_thresholds() {
// log_only uses the same thresholds as enforcing mode — only warn_only differs
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(200, 0.8, true);
assert_eq!(warn, Some(160));
assert_eq!(enforce, 200);
assert!(warn_only);
}
#[test]
fn test_resolve_rate_limit_tiny_capacity_no_warn() {
// enforce=1 → 80% rounds to 0, too small for warn tier
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(1, 0.8, false);
assert_eq!(warn, None);
assert_eq!(enforce, 1);
assert!(!warn_only);
}
#[test]
fn test_resolve_rate_limit_zero_capacity() {
let (warn, enforce, warn_only) = resolve_rate_limit_capacities(0, 0.8, false);
assert_eq!(warn, None);
assert_eq!(enforce, 0);
assert!(!warn_only);
}
}