Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): skip the session statement_timeout SET on pooled database pools (#127843)

Gustavo H. Strassburgerc15dca0f8a94
diff --git a/docs/internal/feature-flags/database-interaction-patterns.md b/docs/internal/feature-flags/database-interaction-patterns.md
index 1560bc960eaa..58050b8e6049 100644
--- a/docs/internal/feature-flags/database-interaction-patterns.md
+++ b/docs/internal/feature-flags/database-interaction-patterns.md
@@ -73,6 +73,12 @@ Different pools can have different statement timeouts to match their workload:
Statement timeouts are set via `SET statement_timeout = {ms}` on each new connection using SQLx's `after_connect` hook.
+Behind a PgBouncer pooler in transaction mode, that session `SET` lands on whichever server connection the pooler assigns, and later clients on that connection inherit it.
+The pooler sets the timeout itself with `connect_query`.
+List those pools in `POOLER_STATEMENT_TIMEOUT_POOLS`, and they send no `SET`.
+Their statement timeouts still count in the `REQUEST_TIMEOUT_MS` check, so keep each one equal to the pooler's value. For `behavioral_cohorts` that is the fixed 1000ms.
+An aliased pool uses the setting of the pool it aliases. At startup the service logs the pools that send no `SET`, and warns about a listed pool that the service does not create, because it aliases another pool or has no database URL.
+
The same hook also increments the `db_connection_created_total` counter when `pool_name` is set, providing visibility into connection churn per pool.
### Persons DB deadline
@@ -436,6 +442,7 @@ Queries exceeding 500ms are logged at WARN level with timing information.
| `NON_PERSONS_READER_STATEMENT_TIMEOUT_MS` | 2000 | Statement timeout for non-persons reads |
| `PERSONS_READER_STATEMENT_TIMEOUT_MS` | 1000 | Statement timeout for persons reads |
| `WRITER_STATEMENT_TIMEOUT_MS` | 2000 | Statement timeout for writes |
+| `POOLER_STATEMENT_TIMEOUT_POOLS` | empty | Comma-separated pool names whose pooler sets statement_timeout |
| `BEHAVIORAL_COHORTS_READ_DATABASE_URL` | empty | Behavioral cohorts database (enables realtime cohort evaluation) |
| `BATCH_FLAG_EVAL_SCAN_STATEMENT_TIMEOUT_MS` | 10000 | Statement timeout for the batch evaluation person scan |
| `PERSONS_DB_DEADLINE_MS` | 2500 | Deadline shared by all persons DB calls in one evaluation |
diff --git a/rust/feature-flags/src/config.rs b/rust/feature-flags/src/config.rs
index 7aae29e3a45f..23b008e7b393 100644
--- a/rust/feature-flags/src/config.rs
+++ b/rust/feature-flags/src/config.rs
@@ -1,4 +1,5 @@
use crate::cohorts::cohort_models::MembershipStampPolicy;
+use crate::database::pool_names;
use common_continuous_profiling::ContinuousProfilingConfig;
use common_cookieless::CookielessConfig;
use common_types::TeamId;
@@ -240,6 +241,32 @@ impl FromStr for RateLimitingAllowList {
}
}
+/// Comma-separated names of the database pools whose PgBouncer pooler sets statement_timeout.
+/// An unknown name fails startup, because a misspelled pool would keep sending its `SET`.
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct PoolerStatementTimeoutPools(pub HashSet<&'static str>);
+
+impl FromStr for PoolerStatementTimeoutPools {
+ type Err = String;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ let mut pools = HashSet::new();
+ for part in s.split(',').map(|p| p.trim()).filter(|p| !p.is_empty()) {
+ let pool = pool_names::ALL
+ .iter()
+ .find(|name| **name == part)
+ .ok_or_else(|| {
+ format!(
+ "Unknown pool '{part}' in POOLER_STATEMENT_TIMEOUT_POOLS. Known pools: {}",
+ pool_names::ALL.join(", ")
+ )
+ })?;
+ pools.insert(*pool);
+ }
+ Ok(PoolerStatementTimeoutPools(pools))
+ }
+}
+
/// Per-team /flags request/response body logging config.
/// Parses JSON from FLAGS_LOG_BODIES_TEAMS environment variable, also refreshed
/// at runtime from posthog_instancesetting (key:
@@ -607,6 +634,13 @@ pub struct Config {
#[envconfig(default = "2000")]
pub writer_statement_timeout_ms: u64,
+ // Pools whose PgBouncer pooler sets statement_timeout with connect_query. These pools send
+ // no session `SET`, because in transaction mode it lands on whichever server connection the
+ // pooler assigns and the next client inherits it. Each pool's timeout still counts in the
+ // REQUEST_TIMEOUT_MS check, so keep it equal to the pooler's value.
+ #[envconfig(default = "")]
+ pub pooler_statement_timeout_pools: PoolerStatementTimeoutPools,
+
// How often to report database pool metrics (seconds)
// - Decrease for more granular monitoring (e.g., 10-15)
// - Increase to reduce metric volume (e.g., 60-120)
@@ -1338,6 +1372,7 @@ impl Config {
flag_definitions_conditional_rate_limits: FlagDefinitionsRateLimits::default(),
remote_config_default_rate_per_minute: 600,
rate_limiting_allow_list_teams: RateLimitingAllowList::default(),
+ pooler_statement_timeout_pools: PoolerStatementTimeoutPools::default(),
flags_log_bodies_teams: BodyLogTeams::default(),
flags_log_bodies_request_max_bytes: 65_536,
otel_url: None,
@@ -1771,6 +1806,39 @@ mod tests {
assert_eq!(limits.0.get(&123), Some(&"600/minute".to_string()));
}
+ #[rstest::rstest]
+ #[case::empty("", &[])]
+ #[case::whitespace_and_commas(
+ " persons_reader , ,behavioral_cohorts ",
+ &[pool_names::PERSONS_READER, pool_names::BEHAVIORAL_COHORTS]
+ )]
+ fn test_pooler_statement_timeout_pools_parses(#[case] input: &str, #[case] expected: &[&str]) {
+ let pools: PoolerStatementTimeoutPools = input.parse().unwrap();
+ assert_eq!(pools.0, expected.iter().copied().collect::<HashSet<_>>());
+ }
+
+ #[test]
+ fn test_pooler_statement_timeout_pools_reads_env_var() {
+ let env = HashMap::from([(
+ "POOLER_STATEMENT_TIMEOUT_POOLS".to_string(),
+ "persons_reader".to_string(),
+ )]);
+ let config = Config::init_from_hashmap(&env).unwrap();
+
+ assert!(config
+ .pooler_statement_timeout_pools
+ .0
+ .contains(pool_names::PERSONS_READER));
+ }
+
+ #[test]
+ fn test_pooler_statement_timeout_pools_rejects_unknown_pool() {
+ let error = "persons_reader,persons_readr"
+ .parse::<PoolerStatementTimeoutPools>()
+ .unwrap_err();
+ assert!(error.contains("'persons_readr'"), "{error}");
+ }
+
#[test]
fn test_rate_limiting_allow_list_empty() {
let list: RateLimitingAllowList = "".parse().unwrap();
diff --git a/rust/feature-flags/src/database/pool_names.rs b/rust/feature-flags/src/database/pool_names.rs
index a19ab6793aba..0dd10fdea2f8 100644
--- a/rust/feature-flags/src/database/pool_names.rs
+++ b/rust/feature-flags/src/database/pool_names.rs
@@ -12,3 +12,11 @@ pub const PERSONS_WRITER: &str = "persons_writer";
pub const NON_PERSONS_READER: &str = "non_persons_reader";
pub const NON_PERSONS_WRITER: &str = "non_persons_writer";
pub const BEHAVIORAL_COHORTS: &str = "behavioral_cohorts";
+
+pub const ALL: [&str; 5] = [
+ PERSONS_READER,
+ PERSONS_WRITER,
+ NON_PERSONS_READER,
+ NON_PERSONS_WRITER,
+ BEHAVIORAL_COHORTS,
+];
diff --git a/rust/feature-flags/src/database_pools.rs b/rust/feature-flags/src/database_pools.rs
index 57aabfb6feb1..6940fc67ed76 100644
--- a/rust/feature-flags/src/database_pools.rs
+++ b/rust/feature-flags/src/database_pools.rs
@@ -1,5 +1,5 @@
use crate::api::errors::FlagError;
-use crate::config::Config;
+use crate::config::{Config, PoolerStatementTimeoutPools};
use crate::database::pool_names;
use common_database::{get_pool_with_config, PoolConfig};
use sqlx::PgPool;
@@ -31,11 +31,14 @@ impl DatabasePools {
max_connections: Option<u32>,
statement_timeout_ms: u64,
pool_name: &str,
+ pooler_statement_timeout_pools: &PoolerStatementTimeoutPools,
) -> PoolConfig {
PoolConfig {
min_connections,
max_connections: max_connections.unwrap_or(base.max_connections),
- statement_timeout_ms: if statement_timeout_ms > 0 {
+ statement_timeout_ms: if statement_timeout_ms > 0
+ && !pooler_statement_timeout_pools.0.contains(pool_name)
+ {
Some(statement_timeout_ms)
} else {
None
@@ -45,16 +48,9 @@ impl DatabasePools {
}
}
- /// Lists each pool, with its statement timeout, where one call can outlast the request timeout.
- /// A statement timeout of 0 leaves the query unbounded.
- ///
- /// This bounds one call, not one request. A request that runs several calls in sequence can
- /// still outlast the request timeout.
- ///
- /// A pool that aliases another pool runs with that pool's statement timeout. The check skips
- /// an aliased pool, because its own setting has no effect.
- fn pools_over_request_timeout(config: &Config) -> Vec<(&'static str, u64)> {
- let acquire_ms = config.acquire_timeout_secs.saturating_mul(1000);
+ /// Lists each pool with its configured statement timeout and whether the service creates it.
+ /// The service does not create a pool that aliases another pool or has no database URL.
+ fn pool_statement_timeouts(config: &Config) -> [(&'static str, u64, bool); 5] {
let routing = config.is_persons_db_routing_enabled();
let writes = !*config.skip_writes;
[
@@ -84,14 +80,27 @@ impl DatabasePools {
config.is_behavioral_cohorts_db_configured(),
),
]
- .into_iter()
- .filter(|&(_, statement_timeout_ms, built)| {
- built
- && (statement_timeout_ms == 0
- || acquire_ms.saturating_add(statement_timeout_ms) >= config.request_timeout_ms)
- })
- .map(|(pool, statement_timeout_ms, _)| (pool, statement_timeout_ms))
- .collect()
+ }
+
+ /// Lists each pool, with its statement timeout, where one call can outlast the request timeout.
+ /// A statement timeout of 0 leaves the query unbounded.
+ ///
+ /// This bounds one call, not one request. A request that runs several calls in sequence can
+ /// still outlast the request timeout.
+ ///
+ /// The check skips an aliased pool, because its own setting has no effect.
+ fn pools_over_request_timeout(config: &Config) -> Vec<(&'static str, u64)> {
+ let acquire_ms = config.acquire_timeout_secs.saturating_mul(1000);
+ Self::pool_statement_timeouts(config)
+ .into_iter()
+ .filter(|&(_, statement_timeout_ms, built)| {
+ built
+ && (statement_timeout_ms == 0
+ || acquire_ms.saturating_add(statement_timeout_ms)
+ >= config.request_timeout_ms)
+ })
+ .map(|(pool, statement_timeout_ms, _)| (pool, statement_timeout_ms))
+ .collect()
}
pub async fn from_config(config: &Config) -> Result<Self, FlagError> {
@@ -173,6 +182,28 @@ impl DatabasePools {
);
}
+ let listed = &config.pooler_statement_timeout_pools.0;
+ if !listed.is_empty() {
+ let mut pools_without_set = Vec::new();
+ for (pool, _, built) in Self::pool_statement_timeouts(config) {
+ if !listed.contains(pool) {
+ continue;
+ }
+ if built {
+ pools_without_set.push(pool);
+ } else {
+ tracing::warn!(
+ pool,
+ "POOLER_STATEMENT_TIMEOUT_POOLS lists a pool that this service does not create, because it aliases another pool or has no database URL, so the listing has no effect. An aliased pool uses the setting of the pool it aliases"
+ );
+ }
+ }
+ info!(
+ pools = ?pools_without_set,
+ "The pooler sets statement_timeout on these pools, so they send no SET"
+ );
+ }
+
// Create base pool config (used for both readers and writers)
let base_pool_config = PoolConfig {
min_connections: 0, // Will be overridden per pool
@@ -194,7 +225,8 @@ impl DatabasePools {
min_non_persons_reader_connections,
None,
config.non_persons_reader_statement_timeout_ms,
- "non_persons_reader",
+ pool_names::NON_PERSONS_READER,
+ &config.pooler_statement_timeout_pools,
);
info!(
pool = "non_persons_reader",
@@ -208,7 +240,8 @@ impl DatabasePools {
min_persons_reader_connections,
None,
config.persons_reader_statement_timeout_ms,
- "persons_reader",
+ pool_names::PERSONS_READER,
+ &config.pooler_statement_timeout_pools,
);
info!(
pool = "persons_reader",
@@ -222,7 +255,8 @@ impl DatabasePools {
min_non_persons_writer_connections,
None,
config.writer_statement_timeout_ms,
- "non_persons_writer",
+ pool_names::NON_PERSONS_WRITER,
+ &config.pooler_statement_timeout_pools,
);
// Persons writer pool config (should be fast transactional operations)
@@ -231,7 +265,8 @@ impl DatabasePools {
min_persons_writer_connections,
None,
config.writer_statement_timeout_ms,
- "persons_writer",
+ pool_names::PERSONS_WRITER,
+ &config.pooler_statement_timeout_pools,
);
info!(
pool = "writer",
@@ -339,6 +374,7 @@ impl DatabasePools {
Some(5),
Self::BEHAVIORAL_COHORTS_STATEMENT_TIMEOUT_MS,
pool_names::BEHAVIORAL_COHORTS,
+ &config.pooler_statement_timeout_pools,
);
info!("Creating behavioral cohorts reader pool");
Some(Arc::new(
@@ -374,23 +410,32 @@ mod tests {
use std::collections::HashMap;
use std::sync::Arc;
- #[test]
- fn test_build_pool_config_sets_pool_name_and_timeout() {
+ #[rstest::rstest]
+ #[case::binary_sets_timeout(5000, "", Some(5000))]
+ #[case::zero_timeout(0, "", None)]
+ #[case::pooler_sets_timeout(5000, "persons_reader", None)]
+ #[case::pooler_sets_another_pool(5000, "persons_writer,behavioral_cohorts", Some(5000))]
+ fn test_build_pool_config_statement_timeout(
+ #[case] statement_timeout_ms: u64,
+ #[case] pooler_timeout_pools: &str,
+ #[case] expected: Option<u64>,
+ ) {
let base = PoolConfig::default();
- let config = DatabasePools::build_pool_config(&base, 3, None, 5000, "persons_reader");
+ let config = DatabasePools::build_pool_config(
+ &base,
+ 3,
+ None,
+ statement_timeout_ms,
+ pool_names::PERSONS_READER,
+ &pooler_timeout_pools.parse().unwrap(),
+ );
assert_eq!(config.min_connections, 3);
- assert_eq!(config.statement_timeout_ms, Some(5000));
- assert_eq!(config.pool_name, Some("persons_reader".to_string()));
- }
-
- #[test]
- fn test_build_pool_config_zero_timeout_is_none() {
- let base = PoolConfig::default();
- let config = DatabasePools::build_pool_config(&base, 0, None, 0, "non_persons_writer");
-
- assert_eq!(config.statement_timeout_ms, None);
- assert_eq!(config.pool_name, Some("non_persons_writer".to_string()));
+ assert_eq!(config.statement_timeout_ms, expected);
+ assert_eq!(
+ config.pool_name,
+ Some(pool_names::PERSONS_READER.to_string())
+ );
}
#[rstest::rstest]
@@ -411,6 +456,21 @@ mod tests {
(pool_names::NON_PERSONS_WRITER, 2000),
]
)]
+ #[case::pooler_sets_timeouts(
+ &[
+ ("ACQUIRE_TIMEOUT_SECS", "4"),
+ (
+ "POOLER_STATEMENT_TIMEOUT_POOLS",
+ "non_persons_reader,persons_reader,persons_writer,non_persons_writer",
+ ),
+ ],
+ &[
+ (pool_names::NON_PERSONS_READER, 2000),
+ (pool_names::PERSONS_READER, 1000),
+ (pool_names::PERSONS_WRITER, 2000),
+ (pool_names::NON_PERSONS_WRITER, 2000),
+ ]
+ )]
#[case::behavioral_cohorts_configured(
&[
("ACQUIRE_TIMEOUT_SECS", "4"),