:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): add a persons stall recovery test (#124525)
Matheus Batistadb364db61a00
diff --git a/.github/scripts/rust_test_shards.py b/.github/scripts/rust_test_shards.pyindex b27dcd990e62..e47f4476c920 100644--- a/.github/scripts/rust_test_shards.py+++ b/.github/scripts/rust_test_shards.py@@ -42,7 +42,7 @@ PACKAGE_WEIGHTS: dict[str, int] = {"deltalite-core": 5,"deltalite-python": 5,"embedding-worker": 90,- "feature-flags": 241,+ "feature-flags": 356,"flags-consumer": 23,"health": 5,"hogql_parser_rs": 7,diff --git a/.github/scripts/test_rust_test_shards.py b/.github/scripts/test_rust_test_shards.pyindex 30464e4ec064..828651cc6c55 100644--- a/.github/scripts/test_rust_test_shards.py+++ b/.github/scripts/test_rust_test_shards.py@@ -15,7 +15,7 @@ from rust_test_shards import PACKAGE_WEIGHTS, plan_shards({"capture"}, [["capture"]]),({"capture", "feature-flags", "cohort-stream-processor"},- [["cohort-stream-processor"], ["capture", "feature-flags"]],+ [["feature-flags"], ["capture", "cohort-stream-processor"]],),({"new-crate", "capture"}, [["capture", "new-crate"]]),],diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.ymlindex 580a1c32eef9..366950940b68 100644--- a/.github/workflows/ci-rust.yml+++ b/.github/workflows/ci-rust.yml@@ -50,6 +50,7 @@ jobs:outputs:rust: ${{ steps.filter.outputs.rust || 'true' }}all_crates: ${{ steps.filter.outputs.all_crates || 'false' }}+ flags_db: ${{ steps.filter.outputs.flags_db || 'true' }}steps:# For pull requests it's not necessary to checkout the code, but we# also want this to run on master so we need to checkout@@ -103,6 +104,14 @@ jobs:# Pins the service images that every crate's tests run against.# A determinator rule would also rebuild and deploy every Rust image.- 'docker-compose.base.yml'+ # What the feature-flags stall recovery tests exercise: the whole request path.+ flags_db:+ - 'rust/feature-flags/**'+ - 'rust/common/database/**'+ # An sqlx bump can change how a pool recovers.+ - 'rust/Cargo.lock'+ - 'rust/.config/nextest.toml'+ - '.github/workflows/ci-rust.yml'# Compute affected crates and test shards. Only runs when Rust changes# are detected, keeping the lightweight `changes` job fast for non-Rust PRs.@@ -382,6 +391,7 @@ jobs:# Set up dual database environment for feature-flags servicePERSONS_READ_DATABASE_URL: ${{ contains(format(' {0} ', matrix.packages), ' feature-flags ') && 'postgres://posthog:posthog@localhost:5432/posthog_persons' || '' }}PERSONS_WRITE_DATABASE_URL: ${{ contains(format(' {0} ', matrix.packages), ' feature-flags ') && 'postgres://posthog:posthog@localhost:5432/posthog_persons' || '' }}+ FLAGS_DB_CHANGED: ${{ needs.changes.outputs.flags_db }}run: |# nextest (instead of plain `cargo test`) emits JUnit XML for the Trunk quarantine# gate below. It does not run doctests; those run in their own step, outside the@@ -455,6 +465,12 @@ jobs:CHECKPOINT_S3_SECRET_ACCESS_KEY=object_storage_root_password \run_nextest cohort-stream-processor-ignored -p cohort-stream-processor -E 'kind(test)' --run-ignored only --test-threads=1 || overall_rc=1fi+ # feature-flags' persons DB stall tests are #[ignore]'d: they lock posthog_person,+ # so they run one at a time, and only on master or when flags_db matches. The file+ # guard keeps this a no-op on branches that predate them.+ if [ "$pkg" = "feature-flags" ] && [ "$FLAGS_DB_CHANGED" = "true" ] && [ -f feature-flags/tests/stall_recovery.rs ]; then+ run_nextest feature-flags-stall -p feature-flags -E 'binary(stall_recovery)' --run-ignored only --test-threads=1 || overall_rc=1+ fiecho "::endgroup::"doneexit $overall_rcdiff --git a/docs/internal/feature-flags/database-interaction-patterns.md b/docs/internal/feature-flags/database-interaction-patterns.mdindex 54c595894d06..1560bc960eaa 100644--- a/docs/internal/feature-flags/database-interaction-patterns.md+++ b/docs/internal/feature-flags/database-interaction-patterns.md@@ -135,6 +135,8 @@ If `statement_timeout` cancels the query, the ping fails and sqlx closes the conIf the database never answers, the connection stays checked out.Keep the persons statement timeouts below the deadline so that Postgres cancels ordinary slow queries before the deadline drops them.+`rust/feature-flags/tests/stall_recovery.rs` runs persons DB faults under load with the production pool and timeout settings, and checks that the server recovers without a restart. Only the lock-wait fault changes a setting: it raises the persons statement timeout above the persons DB deadline. The tests are ignored by default, because they lock `posthog_person`. Run them alone with `cargo nextest run -p feature-flags -E 'binary(stall_recovery)' --run-ignored only --test-threads=1`. CI runs them on master and on pull requests that change `rust/feature-flags`.+### Total connection count```text@@ -352,34 +354,35 @@ let retry_strategy = ExponentialBackoff::from_millis(100)### Prometheus metrics-| Metric | Labels | Purpose |-| ------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| `flags_db_connection_time` | `pool`, `operation` | Connection acquisition latency (sub-ms precision, bucket floor 0.05ms) |-| `flags_person_query_time` | - | Person lookup query duration |-| `flags_definition_query_time` | - | Flag definition query duration |-| `flags_pool_utilization_ratio` | `pool` | Pool utilization (0.0-1.0) |-| `flags_connection_hold_time_ms` | `pool`, `operation` | How long connections are held |-| `flags_hash_key_retries_total` | `team_id`, `operation` | Retries that started. A failed attempt counts only when its retry starts |-| `flags_flag_evaluation_error_total` | `error_type` | Error counter |-| `flags_database_error_total` | `error_type`, `operation`, `retried`, `timeout_type` | Database errors by operation. `timeout_type="persons_db_deadline"` counts persons DB calls stopped by `PERSONS_DB_DEADLINE_MS`, including calls skipped because it had passed |-| `db_connection_created_total` | `pool` | Connection creation events (physical TCP/TLS, not pool reuse) |-| `flags_db_connection_pool_size` | `pool` | Total pool size (should equal active + idle) |-| `flags_db_connection_pool_active_total` | `pool` | Active (in-use) connections |-| `flags_db_connection_pool_idle_total` | `pool` | Idle (available) connections |-| `flags_db_connection_pool_max_total` | `pool` | Configured maximum connections |-| `flags_queue_time_ms` | `team_id` | Request queue wait time (bucket ceiling 30000ms) |-| `flags_pre_handler_time_ms` | `team_id` | Pre-handler work timing (UA parse, rate limit checks, token extract) |-| `flags_rate_limit_check_ms` | `kind` | Rate limit check duration (`kind="ip"` or `kind="token"`) |-| `flags_token_extract_ms` | - | Token extraction timing |-| `flags_concurrency_limit_wait_ms` | - | Concurrency limit permit wait time (pod-level, no `team_id`) |-| `flags_realtime_cohort_query_time` | `team_id` | Realtime cohort lookup at the evaluation site, including cache hits (sub-ms precision) |-| `flags_realtime_cohort_query_error_total` | `team_id` | Realtime cohort lookups that failed and degraded to non-membership |-| `flags_realtime_cohort_db_query_time` | `outcome` | Behavioral cohorts DB query latency (`success`, `error`, `timeout`; sub-ms precision, 20ms SLO bucket) |-| `flags_db_cohort_membership_reads_total` | - | Successful behavioral cohorts DB reads |-| `flags_db_cohort_membership_errors_total` | - | Failed or timed-out behavioral cohorts DB reads |-| `flags_cohort_membership_cache_hit_total` | - | Membership lookups fully served from the Moka cache |-| `flags_cohort_membership_cache_miss_total` | - | Membership lookups that issued a behavioral cohorts DB query |-| `flags_cohort_membership_cache_entries` | - | Current entries in the membership cache (one per team + person pair) |+| Metric | Labels | Purpose |+| ------------------------------------------ | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |+| `flags_db_connection_time` | `pool`, `operation` | Connection acquisition latency (sub-ms precision, bucket floor 0.05ms) |+| `flags_person_query_time` | - | Person lookup query duration |+| `flags_definition_query_time` | - | Flag definition query duration |+| `flags_pool_utilization_ratio` | `pool` | Pool utilization (0.0-1.0) |+| `flags_connection_hold_time_ms` | `pool`, `operation` | How long connections are held |+| `flags_hash_key_retries_total` | `team_id`, `operation` | Retries that started. A failed attempt counts only when its retry starts |+| `flags_flag_evaluation_error_total` | `error_type` | Error counter |+| `flags_database_error_total` | `error_type`, `operation`, `retried`, `timeout_type` | Database errors by operation. `timeout_type="persons_db_deadline"` counts persons DB calls stopped by `PERSONS_DB_DEADLINE_MS`, including calls skipped because it had passed |+| `db_connection_created_total` | `pool` | Connection creation events (physical TCP/TLS, not pool reuse) |+| `flags_request_outcome_total` | `outcome`, `needs_persons` | One count per `/flags` request, including requests the request timeout or a client disconnect drops. `outcome` is `ok`, `degraded` (a 2xx where a flag failed on a database or a timeout), `error` (5xx or a handler panic), `rejected` (4xx) or `abandoned` (the client left first). `needs_persons` is `true` once evaluation reaches a step that needs persons DB data, such as person properties or group type mappings. A request that ends before that, for example while it waits for a concurrency permit, counts as `false` |+| `flags_db_connection_pool_size` | `pool` | Total pool size (should equal active + idle) |+| `flags_db_connection_pool_active_total` | `pool` | Active (in-use) connections |+| `flags_db_connection_pool_idle_total` | `pool` | Idle (available) connections |+| `flags_db_connection_pool_max_total` | `pool` | Configured maximum connections |+| `flags_queue_time_ms` | `team_id` | Request queue wait time (bucket ceiling 30000ms) |+| `flags_pre_handler_time_ms` | `team_id` | Pre-handler work timing (UA parse, rate limit checks, token extract) |+| `flags_rate_limit_check_ms` | `kind` | Rate limit check duration (`kind="ip"` or `kind="token"`) |+| `flags_token_extract_ms` | - | Token extraction timing |+| `flags_concurrency_limit_wait_ms` | - | Concurrency limit permit wait time (pod-level, no `team_id`) |+| `flags_realtime_cohort_query_time` | `team_id` | Realtime cohort lookup at the evaluation site, including cache hits (sub-ms precision) |+| `flags_realtime_cohort_query_error_total` | `team_id` | Realtime cohort lookups that failed and degraded to non-membership |+| `flags_realtime_cohort_db_query_time` | `outcome` | Behavioral cohorts DB query latency (`success`, `error`, `timeout`; sub-ms precision, 20ms SLO bucket) |+| `flags_db_cohort_membership_reads_total` | - | Successful behavioral cohorts DB reads |+| `flags_db_cohort_membership_errors_total` | - | Failed or timed-out behavioral cohorts DB reads |+| `flags_cohort_membership_cache_hit_total` | - | Membership lookups fully served from the Moka cache |+| `flags_cohort_membership_cache_miss_total` | - | Membership lookups that issued a behavioral cohorts DB query |+| `flags_cohort_membership_cache_entries` | - | Current entries in the membership cache (one per team + person pair) |### Example PromQL queriesdiff --git a/docs/internal/feature-flags/rust-service-overview.md b/docs/internal/feature-flags/rust-service-overview.mdindex 08ef90eb055e..0b47ba3f4f59 100644--- a/docs/internal/feature-flags/rust-service-overview.md+++ b/docs/internal/feature-flags/rust-service-overview.md@@ -292,12 +292,13 @@ Cache and query health are observable via the `flags_cohort_membership_cache_*`### S3 / HyperCache-| Variable | Default | Purpose |-| ------------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |-| `OBJECT_STORAGE_BUCKET` | `posthog` | S3 bucket name |-| `OBJECT_STORAGE_REGION` | `us-east-1` | AWS region |-| `OBJECT_STORAGE_ENDPOINT` | (empty) | Custom S3 endpoint for local dev |-| `HYPERCACHE_READ_REPAIR_TTL_SECONDS` | `600` | TTL for writing an S3 hit back into Redis after a confirmed miss. `0` disables, as does `SKIP_WRITES`. Also read by hypercache-server. See [read repair](hypercache-system.md#read-repair) |+| Variable | Default | Purpose |+| ------------------------------------ | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `OBJECT_STORAGE_BUCKET` | `posthog` | S3 bucket name |+| `OBJECT_STORAGE_REGION` | `us-east-1` | AWS region |+| `OBJECT_STORAGE_ENDPOINT` | (empty) | Custom S3 endpoint for local dev |+| `HYPERCACHE_READ_REPAIR_TTL_SECONDS` | `600` | TTL for writing an S3 hit back into Redis after a confirmed miss. `0` disables, as does `SKIP_WRITES`. Also read by hypercache-server. See [read repair](hypercache-system.md#read-repair) |+| `HYPERCACHE_S3_TIMEOUT_MS` | `3000` | Upper bound on one S3 read, for every HyperCache reader. Keep it below `REQUEST_TIMEOUT_MS`, so a slow read leaves time for the fallback. Startup replaces `0` or a value above `10000` with `3000` |### Team lookupdiff --git a/rust/.config/nextest.toml b/rust/.config/nextest.tomlindex 143acd3e2897..896a9c468384 100644--- a/rust/.config/nextest.toml+++ b/rust/.config/nextest.toml@@ -51,3 +51,12 @@ threads-required = "num-cpus"[[profile.ci.overrides]]filter = 'package(cohort-stream-processor) & kind(test)'slow-timeout = { period = "60s", terminate-after = 2 }++# The persons DB stall tests each run for up to a minute by design. A retry of every failing one+# would push the shard past its job timeout, so they get none. Fail a hung one at 2 minutes, and+# keep the measurements each one prints in the job log.+[[profile.ci.overrides]]+filter = 'package(feature-flags) & binary(stall_recovery)'+retries = 0+slow-timeout = { period = "60s", terminate-after = 2 }+success-output = "final"diff --git a/rust/feature-flags/src/api/endpoint.rs b/rust/feature-flags/src/api/endpoint.rsindex ef2c714eb5f6..48abc3d0668a 100644--- a/rust/feature-flags/src/api/endpoint.rs+++ b/rust/feature-flags/src/api/endpoint.rs@@ -4,6 +4,7 @@ use crate::{concurrency_metrics::{ConcurrencyEnterTime, ConcurrencyLimitWait},errors::{ClientFacingError, FlagError},flags_rate_limiter::RateLimitResult,+ request_outcome::OutcomeMarks,types::{FlagsQueryParams, FlagsResponse, FlagsResponseV4, LegacyFlagsResponse, ServiceResponse,},@@ -235,6 +236,8 @@ pub async fn flags<C>(// the shim being removed or absent on routes that bypass it (e.g. a// future `/flags`-only sub-router).body_read_duration: Option<Extension<BodyReadDuration>>,+ // Populated by the `record_request_outcome` middleware.+ outcome_marks: Option<Extension<OutcomeMarks>>,headers: HeaderMap,method: Method,path: MatchedPath,@@ -360,6 +363,9 @@ wherequeue_time_ms,concurrency_limit_wait_ms,body_read_ms,+ outcome_marks: outcome_marks+ .map(|Extension(marks)| marks)+ .unwrap_or_default(),..Default::default()};diff --git a/rust/feature-flags/src/api/errors.rs b/rust/feature-flags/src/api/errors.rsindex 34cc165cd0e4..c8a417307dfd 100644--- a/rust/feature-flags/src/api/errors.rs+++ b/rust/feature-flags/src/api/errors.rs@@ -121,6 +121,10 @@ pub enum FlagError {DependencyFailed(i64),#[error("Group type mapping lookup failed: {0}")]GroupTypeLookupFailed(Arc<FlagError>),+ /// A failed `posthog_cohort` query, wrapping a `FlagError::internal`. Clients see what the+ /// wrapped error shows them, but the request outcome counts it as a database failure.+ #[error(transparent)]+ CohortQueryFailed(Box<FlagError>),#[error("Failed to parse cohort filters")]CohortFiltersParsingError,#[error("Dependency cycle detected: {0} id {1} starts the cycle")]@@ -266,6 +270,7 @@ impl FlagError {FlagError::DependencyNotFound(_, _) => ("dependency_not_found", 500),FlagError::DependencyFailed(_) => (CODE_DEPENDENCY_FAILED, 500),FlagError::GroupTypeLookupFailed(cause) => cause.error_metadata(),+ FlagError::CohortQueryFailed(cause) => cause.error_metadata(),FlagError::CohortFiltersParsingError => ("cohort_filters_parsing_error", 500),FlagError::DependencyCycle(_, _) => ("dependency_cycle", 500),FlagError::HashKeyOverrideError => ("hash_key_override_error", 500),@@ -395,6 +400,41 @@ impl FlagError {pub fn is_5xx(&self) -> bool {self.status_code() >= 500}++ /// A flag failure caused by a database or a timeout, not by the flag's configuration.+ /// `DependencyFailed` is excluded, because the flag it depends on carries the cause.+ pub fn is_database_or_timeout_failure(&self) -> bool {+ match self {+ FlagError::DatabaseError(..)+ | FlagError::DatabaseUnavailable+ | FlagError::TimeoutError(_)+ | FlagError::HashKeyOverrideError+ | FlagError::CohortQueryFailed(_)+ // The lookup reports a failed query, a statement timeout included, as an internal error.+ | FlagError::GroupTypeLookupFailed(_) => true,+ FlagError::ClientFacing(_)+ | FlagError::InternalError { .. }+ | FlagError::Unavailable { .. }+ | FlagError::RequestDecodingError(_)+ | FlagError::PayloadTooLarge { .. }+ | FlagError::RequestParsingError(_)+ | FlagError::MissingDistinctId+ | FlagError::NoTokenError+ | FlagError::TokenValidationError+ | FlagError::PersonalApiKeyInvalid+ | FlagError::PersonalApiKeyInsufficientScopes+ | FlagError::SecretApiTokenInvalid+ | FlagError::NoAuthenticationProvided+ | FlagError::RowNotFound+ | FlagError::DependencyNotFound(..)+ | FlagError::DependencyFailed(_)+ | FlagError::CohortFiltersParsingError+ | FlagError::DependencyCycle(..)+ | FlagError::RayonSemaphoreTimeout(_)+ | FlagError::CookielessError(_)+ | FlagError::RemoteConfigDecryptFailed(_) => false,+ }+ }}impl IntoResponse for FlagError {@@ -576,6 +616,7 @@ impl IntoResponse for FlagError {"Group type mapping lookup failed".to_string(),)}+ FlagError::CohortQueryFailed(cause) => return cause.into_response(),FlagError::CohortFiltersParsingError => {tracing::error!("Failed to parse cohort filters: {:?}", self);(StatusCode::INTERNAL_SERVER_ERROR, "Failed to parse cohort filters. Please try again later or contact support if the problem persists.".to_string())@@ -810,6 +851,27 @@ mod tests {}}+ #[rstest]+ #[case::database_error(FlagError::DatabaseError(sqlx::Error::PoolClosed, None), true)]+ #[case::database_unavailable(FlagError::DatabaseUnavailable, true)]+ #[case::pool_timeout(FlagError::TimeoutError(Some("pool_timeout".to_string())), true)]+ #[case::persons_db_deadline(FlagError::persons_db_deadline(), true)]+ #[case::hash_key_override(FlagError::HashKeyOverrideError, true)]+ #[case::group_type_query_failed(+ FlagError::GroupTypeLookupFailed(Arc::new(FlagError::internal(anyhow::anyhow!(+ "canceling statement due to statement timeout"+ )))),+ true+ )]+ #[case::deleted_cohort(FlagError::DependencyNotFound(DependencyType::Cohort, 1), false)]+ #[case::failed_flag_dependency(FlagError::DependencyFailed(1), false)]+ #[case::dependency_cycle(FlagError::DependencyCycle(DependencyType::Flag, 1), false)]+ #[case::unsupported_filters(FlagError::flag_data_parsing("unsupported format"), false)]+ #[case::cohort_filters(FlagError::CohortFiltersParsingError, false)]+ fn test_is_database_or_timeout_failure(#[case] error: FlagError, #[case] expected: bool) {+ assert_eq!(error.is_database_or_timeout_failure(), expected);+ }+#[test]fn test_date_out_of_range_is_400() {// is_5xx() for this variant is covered by the SaltCacheError table in test_is_5xx.diff --git a/rust/feature-flags/src/api/mod.rs b/rust/feature-flags/src/api/mod.rsindex d08445454b39..ce59ff0a021d 100644--- a/rust/feature-flags/src/api/mod.rs+++ b/rust/feature-flags/src/api/mod.rs@@ -11,4 +11,5 @@ pub mod flags_rate_limiter;pub mod instance_setting;pub mod rate_parser;pub mod remote_config;+pub mod request_outcome;pub mod types;diff --git a/rust/feature-flags/src/api/request_outcome.rs b/rust/feature-flags/src/api/request_outcome.rsnew file mode 100644index 000000000000..983eeb24b3c3--- /dev/null+++ b/rust/feature-flags/src/api/request_outcome.rs@@ -0,0 +1,286 @@+//! One `flags_request_outcome_total` count per `/flags` request. The middleware wraps+//! `TimeoutLayer`, so requests that the timeout or a client disconnect drops still count.++use std::sync::{+ atomic::{AtomicBool, Ordering},+ Arc,+};++use axum::{+ extract::{MatchedPath, Request},+ http::StatusCode,+ middleware::Next,+ response::Response,+};+use strum::{EnumIter, IntoEnumIterator};++use crate::{+ api::errors::FlagError, metrics::consts::FLAG_REQUEST_OUTCOME_COUNTER, router::FLAGS_ROUTES,+};++#[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter)]+enum Outcome {+ /// 2xx, and no flag failed because a database or a timeout failed.+ Ok,+ /// 2xx, and at least one flag failed because a database or a timeout failed.+ Degraded,+ /// 5xx, including the 503 that the request timeout returns, or a panic in the handler.+ Error,+ /// 4xx: rate limits, auth failures and bad requests.+ Rejected,+ /// The client left before the server answered.+ Abandoned,+}++impl Outcome {+ fn of_response(status: StatusCode, marks: &OutcomeMarks) -> Self {+ if status.is_server_error() {+ Self::Error+ } else if status.is_client_error() {+ Self::Rejected+ } else if marks.degraded() {+ Self::Degraded+ } else {+ Self::Ok+ }+ }++ fn as_str(self) -> &'static str {+ match self {+ Self::Ok => "ok",+ Self::Degraded => "degraded",+ Self::Error => "error",+ Self::Rejected => "rejected",+ Self::Abandoned => "abandoned",+ }+ }+}++/// Clones share the marks: evaluation writes them, and [`record_request_outcome`] reads them.+#[derive(Clone, Debug, Default)]+pub struct OutcomeMarks(Arc<Marks>);++// Relaxed is enough: the middleware reads the marks on the same task, after the handler future+// completes or is dropped.+#[derive(Debug, Default)]+struct Marks {+ needs_persons: AtomicBool,+ degraded: AtomicBool,+}++impl OutcomeMarks {+ /// Call before the persons DB work, so a request that the timeout drops keeps the label.+ pub fn mark_needs_persons(&self) {+ self.0.needs_persons.store(true, Ordering::Relaxed);+ }++ pub fn mark_flag_failed(&self, error: &FlagError) {+ if error.is_database_or_timeout_failure() {+ self.0.degraded.store(true, Ordering::Relaxed);+ }+ }++ /// Whether the request needed person data, even when no persons DB call ran.+ pub(crate) fn needs_persons(&self) -> bool {+ self.0.needs_persons.load(Ordering::Relaxed)+ }++ pub(crate) fn degraded(&self) -> bool {+ self.0.degraded.load(Ordering::Relaxed)+ }+}++/// Records `Abandoned` when dropped before `finish`, or `Error` when a handler panic drops it.+struct PendingOutcome(Option<OutcomeMarks>);++impl PendingOutcome {+ fn finish(mut self, status: StatusCode) {+ if let Some(marks) = self.0.take() {+ counter(Outcome::of_response(status, &marks), marks.needs_persons()).increment(1);+ }+ }+}++impl Drop for PendingOutcome {+ fn drop(&mut self) {+ if let Some(marks) = self.0.take() {+ // Tokio drops a panicking task's future while it unwinds, so a handler panic gets here+ // with `panicking()` true.+ let outcome = if std::thread::panicking() {+ Outcome::Error+ } else {+ Outcome::Abandoned+ };+ counter(outcome, marks.needs_persons()).increment(1);+ }+ }+}++fn counter(outcome: Outcome, needs_persons: bool) -> metrics::Counter {+ metrics::counter!(+ FLAG_REQUEST_OUTCOME_COUNTER,+ "outcome" => outcome.as_str(),+ "needs_persons" => if needs_persons { "true" } else { "false" }+ )+}++/// A ratio alert over a series that never fired evaluates to nothing.+pub fn precreate_request_outcome_counters() {+ for outcome in Outcome::iter() {+ for needs_persons in [false, true] {+ counter(outcome, needs_persons).increment(0);+ }+ }+}++/// Skips the definitions routes, which share the timeout stack.+pub async fn record_request_outcome(mut req: Request, next: Next) -> Response {+ let is_flags_request = req+ .extensions()+ .get::<MatchedPath>()+ .is_some_and(|path| FLAGS_ROUTES.contains(&path.as_str()));+ if !is_flags_request {+ return next.run(req).await;+ }++ let marks = OutcomeMarks::default();+ req.extensions_mut().insert(marks.clone());+ let pending = PendingOutcome(Some(marks));+ let response = next.run(req).await;+ pending.finish(response.status());+ response+}++#[cfg(test)]+mod tests {+ use std::time::Duration;++ use axum::{+ body::Body, error_handling::HandleErrorLayer, extract::Extension, routing::get, Router,+ };+ use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};+ use rstest::rstest;+ use tower::{timeout::TimeoutLayer, ServiceBuilder, ServiceExt};++ use super::*;++ const REQUEST_TIMEOUT: Duration = Duration::from_secs(1);++ #[derive(Clone, Copy)]+ enum Evaluation {+ Healthy,+ PersonsDbFailed,+ RateLimited,+ /// Needs person data and never finishes.+ Stalls,+ Panics,+ }++ async fn evaluate(evaluation: Evaluation, marks: OutcomeMarks) -> StatusCode {+ match evaluation {+ Evaluation::Healthy => StatusCode::OK,+ Evaluation::PersonsDbFailed => {+ marks.mark_needs_persons();+ marks.mark_flag_failed(&FlagError::DatabaseUnavailable);+ StatusCode::OK+ }+ Evaluation::RateLimited => StatusCode::TOO_MANY_REQUESTS,+ Evaluation::Stalls => {+ marks.mark_needs_persons();+ std::future::pending().await+ }+ Evaluation::Panics => panic!("evaluation panicked"),+ }+ }++ /// Production layer order: the outcome middleware wraps the timeout.+ fn app(evaluation: Evaluation) -> Router {+ Router::new()+ .route(+ "/flags",+ get(move |Extension(marks): Extension<OutcomeMarks>| evaluate(evaluation, marks)),+ )+ .route("/flags/definitions", get(|| async { StatusCode::OK }))+ .layer(+ ServiceBuilder::new()+ .layer(HandleErrorLayer::new(|_: tower::BoxError| async {+ StatusCode::SERVICE_UNAVAILABLE+ }))+ .layer(TimeoutLayer::new(REQUEST_TIMEOUT)),+ )+ .layer(axum::middleware::from_fn(record_request_outcome))+ }++ fn request(path: &str) -> Request {+ axum::http::Request::builder()+ .uri(path)+ .body(Body::empty())+ .unwrap()+ }++ fn recorded_outcomes(snapshotter: &Snapshotter) -> Vec<(String, String, u64)> {+ snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .filter_map(|(key, _, _, value)| {+ let key = key.key();+ let DebugValue::Counter(count) = value else {+ return None;+ };+ let label = |name: &str| {+ key.labels()+ .find(|label| label.key() == name)+ .map(|label| label.value().to_string())+ .unwrap()+ };+ (key.name() == FLAG_REQUEST_OUTCOME_COUNTER)+ .then(|| (label("outcome"), label("needs_persons"), count))+ })+ .collect()+ }++ #[rstest]+ #[case::healthy("/flags", Evaluation::Healthy, Some(("ok", "false")))]+ #[case::persons_db_failed("/flags", Evaluation::PersonsDbFailed, Some(("degraded", "true")))]+ #[case::rate_limited("/flags", Evaluation::RateLimited, Some(("rejected", "false")))]+ #[case::timed_out("/flags", Evaluation::Stalls, Some(("error", "true")))]+ #[case::panicked("/flags", Evaluation::Panics, Some(("error", "false")))]+ #[case::not_a_flags_route("/flags/definitions", Evaluation::Healthy, None)]+ #[tokio::test(start_paused = true)]+ async fn test_records_one_outcome_per_request(+ #[case] path: &str,+ #[case] evaluation: Evaluation,+ #[case] expected: Option<(&str, &str)>,+ ) {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);++ // Spawned, so a panic unwinds through a tokio task as it does in production.+ let joined = tokio::spawn(app(evaluation).oneshot(request(path))).await;+ assert_eq!(joined.is_err(), matches!(evaluation, Evaluation::Panics));++ let expected: Vec<_> = expected+ .into_iter()+ .map(|(outcome, needs_persons)| (outcome.to_string(), needs_persons.to_string(), 1))+ .collect();+ assert_eq!(recorded_outcomes(&snapshotter), expected);+ }++ #[tokio::test(start_paused = true)]+ async fn test_records_abandoned_when_the_client_leaves_first() {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);++ let response = app(Evaluation::Stalls).oneshot(request("/flags"));+ let client_wait = REQUEST_TIMEOUT / 2;+ assert!(tokio::time::timeout(client_wait, response).await.is_err());++ assert_eq!(+ recorded_outcomes(&snapshotter),+ vec![("abandoned".to_string(), "true".to_string(), 1)]+ );+ }+}diff --git a/rust/feature-flags/src/cohorts/cohort_cache_manager.rs b/rust/feature-flags/src/cohorts/cohort_cache_manager.rsindex a1146b43b353..5ad3e544a258 100644--- a/rust/feature-flags/src/cohorts/cohort_cache_manager.rs+++ b/rust/feature-flags/src/cohorts/cohort_cache_manager.rs@@ -236,7 +236,9 @@ impl From<CohortFetchError> for FlagError {fn from(value: CohortFetchError) -> Self {match value {CohortFetchError::DatabaseUnavailable => FlagError::DatabaseUnavailable,- CohortFetchError::QueryFailed(msg) => FlagError::internal(anyhow::anyhow!(msg)),+ CohortFetchError::QueryFailed(msg) => {+ FlagError::CohortQueryFailed(Box::new(FlagError::internal(anyhow::anyhow!(msg))))+ }}}}@@ -1227,13 +1229,11 @@ mod tests {let query_failed = CohortFetchError::QueryFailed("test error".to_string());let flag_error: FlagError = query_failed.into();- assert!(- matches!(flag_error, FlagError::InternalError { ref cause, .. } if cause.to_string() == "Internal error: test error"),- "QueryFailed should render as `Internal error: <message>`"- );+ assert_eq!(flag_error.evaluation_error_code(), "internal_error");assert_eq!(flag_error.evaluation_error_description(),"Internal error: test error");+ assert!(flag_error.is_database_or_timeout_failure());}}diff --git a/rust/feature-flags/src/config.rs b/rust/feature-flags/src/config.rsindex e813e4906064..7aae29e3a45f 100644--- a/rust/feature-flags/src/config.rs+++ b/rust/feature-flags/src/config.rs@@ -538,6 +538,11 @@ pub struct Config {#[envconfig(default = "")]pub object_storage_endpoint: String,+ // Upper bound on one HyperCache S3 read. Keep it below REQUEST_TIMEOUT_MS, so that a slow read+ // leaves time for the Postgres fallback. Startup replaces 0, or a value above the max, with 3000.+ #[envconfig(from = "HYPERCACHE_S3_TIMEOUT_MS", default = "3000")]+ pub hypercache_s3_timeout_ms: u64,+// Redis timeout settings (in milliseconds)#[envconfig(default = "100")]pub redis_response_timeout_ms: u64,@@ -1130,6 +1135,10 @@ impl Config {/// from the persons DB budget.const MAX_PERSONS_DB_DEADLINE_RESERVE_MS: u64 = 2_000;+ const DEFAULT_HYPERCACHE_S3_TIMEOUT_MS: u64 = 3_000;+ /// The top bucket of the S3 read histogram.+ const MAX_HYPERCACHE_S3_TIMEOUT_MS: u64 = 10_000;+/// Validate and fix timeout configuration, logging warnings and applying defaults for invalid values////// This method checks timeout values and relationships, applying safe defaults when invalid@@ -1192,6 +1201,24 @@ impl Config {"{problem}");}++ if !(1..=Self::MAX_HYPERCACHE_S3_TIMEOUT_MS).contains(&self.hypercache_s3_timeout_ms) {+ tracing::warn!(+ configured = self.hypercache_s3_timeout_ms,+ using = Self::DEFAULT_HYPERCACHE_S3_TIMEOUT_MS,+ "HYPERCACHE_S3_TIMEOUT_MS is outside 1..={}ms, using the default",+ Self::MAX_HYPERCACHE_S3_TIMEOUT_MS+ );+ self.hypercache_s3_timeout_ms = Self::DEFAULT_HYPERCACHE_S3_TIMEOUT_MS;+ }++ if self.hypercache_s3_timeout_ms >= self.request_timeout_ms {+ tracing::warn!(+ hypercache_s3_timeout_ms = self.hypercache_s3_timeout_ms,+ request_timeout_ms = self.request_timeout_ms,+ "HYPERCACHE_S3_TIMEOUT_MS is at least REQUEST_TIMEOUT_MS, so a slow S3 read times out the request before any fallback runs"+ );+ }}/// Why the persons DB deadline settings defeat the /flags deadline, or `None` when they do not.@@ -1320,6 +1347,7 @@ impl Config {object_storage_bucket: "posthog".to_string(),object_storage_region: "us-east-1".to_string(),object_storage_endpoint: "".to_string(),+ hypercache_s3_timeout_ms: 3000,// `Enforced` so tests exercise the bot short-circuit envelope.// Tests wanting prod posture override to `LogOnly` explicitly.bot_filter_mode: BotFilterMode::Enforced,@@ -1851,6 +1879,21 @@ mod tests {assert_eq!(config.persons_db_deadline_reserve_ms, expected_ms);}+ #[rstest::rstest]+ #[case::zero(0, 3_000)]+ #[case::in_range(1_500, 1_500)]+ #[case::at_maximum(10_000, 10_000)]+ #[case::above_maximum(10_001, 3_000)]+ fn test_validate_and_fix_timeouts_resets_hypercache_s3_timeout(+ #[case] configured_ms: u64,+ #[case] expected_ms: u64,+ ) {+ let mut config = Config::default_test_config();+ config.hypercache_s3_timeout_ms = configured_ms;+ config.validate_and_fix_timeouts();+ assert_eq!(config.hypercache_s3_timeout_ms, expected_ms);+ }+#[rstest::rstest]#[case::fits(2500, 500, 4500, false)]#[case::budget_reaches_the_cap(2500, 2000, 4500, true)]diff --git a/rust/feature-flags/src/flags/flag_matching.rs b/rust/feature-flags/src/flags/flag_matching.rsindex aaf9a6a14567..13304474595b 100644--- a/rust/feature-flags/src/flags/flag_matching.rs+++ b/rust/feature-flags/src/flags/flag_matching.rs@@ -569,6 +569,12 @@ fn record_persons_db_deadline_exceeded<T>(operation: &'static str, result: &Resu});}+/// The one place a failed flag is built, so the request outcome sees every failure.+fn failed_flag_details(flag: &FeatureFlag, error: &FlagError) -> FlagDetails {+ with_canonical_log(|log| log.outcome_marks.mark_flag_failed(error));+ FlagDetails::create_error(flag, error, None)+}+impl FeatureFlagMatcher {#[allow(clippy::too_many_arguments)]pub fn new(@@ -1134,10 +1140,7 @@ impl FeatureFlagMatcher {continue;}if let Err(error) = flag.filters.require_supported() {- evaluated_flags_map.insert(- flag.key.clone(),- FlagDetails::create_error(flag, &error, None),- );+ evaluated_flags_map.insert(flag.key.clone(), failed_flag_details(flag, &error));unsupported_flag_ids.push(flag.id);}}@@ -1161,10 +1164,8 @@ impl FeatureFlagMatcher {&& flag.filters.is_v1()&& flag.ensure_experience_continuity.unwrap_or(false)}) {- evaluated_flags_map.insert(- flag.key.clone(),- FlagDetails::create_error(flag, &hash_key_error, None),- );+ evaluated_flags_map+ .insert(flag.key.clone(), failed_flag_details(flag, &hash_key_error));}}@@ -1250,6 +1251,7 @@ impl FeatureFlagMatcher {self.flag_evaluation_state.skip_person_properties();return false;}+ with_canonical_log(|log| log.outcome_marks.mark_needs_persons());match self.prepare_flag_evaluation_state(flags_requiring_db_preparation.as_slice())@@ -1274,12 +1276,11 @@ impl FeatureFlagMatcher {error: &FlagError,evaluated_flags_map: &mut HashMap<String, FlagDetails>,) {- evaluated_flags_map.extend(flags_requiring_db_preparation.iter().map(|flag| {- (- flag.key.clone(),- FlagDetails::create_error(flag, error, None),- )- }));+ evaluated_flags_map.extend(+ flags_requiring_db_preparation+ .iter()+ .map(|flag| (flag.key.clone(), failed_flag_details(flag, error))),+ );error!("Error preparing flag evaluation state for team {} distinct_id {}: {:?}",@@ -1486,8 +1487,7 @@ impl FeatureFlagMatcher {&[("reason".to_string(), reason)],1,);- level_evaluated_flags_map- .insert(flag.key.clone(), FlagDetails::create_error(flag, e, None));+ level_evaluated_flags_map.insert(flag.key.clone(), failed_flag_details(flag, e));}}}@@ -3138,6 +3138,7 @@ impl FeatureFlagMatcher {let (hash_key_overrides, flag_hash_key_override_error) =if flags_have_experience_continuity_enabled {common_metrics::inc(FLAG_EXPERIENCE_CONTINUITY_REQUESTS_COUNTER, &[], 1);+ with_canonical_log(|log| log.outcome_marks.mark_needs_persons());match hash_key_override {Some(hash_key) => {let target_distinct_ids = vec![self.distinct_id.clone(), hash_key.clone()];@@ -3242,6 +3243,7 @@ impl FeatureFlagMatcher {if !has_type_indexes {return false;}+ with_canonical_log(|log| log.outcome_marks.mark_needs_persons());let group_type_mapping_timer = common_metrics::timing_guard(FLAG_GROUP_DB_FETCH_TIME, &[]);let mut errors_while_computing_flags = false;diff --git a/rust/feature-flags/src/flags/test_flag_matching.rs b/rust/feature-flags/src/flags/test_flag_matching.rsindex 8f3dd761c931..1890d0331538 100644--- a/rust/feature-flags/src/flags/test_flag_matching.rs+++ b/rust/feature-flags/src/flags/test_flag_matching.rs@@ -8332,6 +8332,8 @@ mod tests {"a persons call that starts after the deadline must not take a connection");assert_eq!(log.persons_db_deadline_exceeded, Some(first_stopped_call));+ assert!(log.outcome_marks.needs_persons());+ assert!(log.outcome_marks.degraded());for operation in [first_stopped_call, "fetch_properties"] {assert_eq!(counter_total(@@ -8348,6 +8350,53 @@ mod tests {}}+ #[tokio::test(start_paused = true)]+ async fn test_db_free_flags_do_not_need_persons() {+ let stalled_db = Arc::new(StalledPgClient::default());+ let router = PostgresRouter::new(+ stalled_db.clone(),+ stalled_db.clone(),+ stalled_db.clone(),+ stalled_db.clone(),+ );+ let mut matcher = FeatureFlagMatcher::new(+ "db_free_user".to_string(),+ None,+ 1,+ router,+ Arc::new(CohortCacheManager::new(stalled_db.clone(), None, None)),+ Arc::new(GroupTypeCacheManager::new(stalled_db.clone(), None, None)),+ None,+ );+ let flags = flag_list_with_metadata(vec![+ mock!(FeatureFlag, id: 1, key: "rollout_flag".mock_into()),+ ]);++ let (response, log) = run_with_canonical_log(+ FlagsCanonicalLogLine::new(Uuid::new_v4(), "127.0.0.1".to_string()),+ tokio::time::timeout(+ Duration::from_secs(60),+ matcher.evaluate_all_feature_flags(+ flags,+ None,+ None,+ None,+ Uuid::new_v4(),+ None,+ false,+ ),+ ),+ )+ .await;+ response+ .expect("database-free flags must not wait on the database")+ .unwrap();++ assert_eq!(stalled_db.connection_requests.load(Ordering::SeqCst), 0);+ assert!(!log.outcome_marks.needs_persons());+ assert!(!log.outcome_marks.degraded());+ }+#[tokio::test]async fn test_hash_key_override_error_marks_continuity_flags_as_errors() {let context = TestContext::new(None).await;diff --git a/rust/feature-flags/src/handler/canonical_log.rs b/rust/feature-flags/src/handler/canonical_log.rsindex 2877fdb2a8bb..4fe1ff6f2160 100644--- a/rust/feature-flags/src/handler/canonical_log.rs+++ b/rust/feature-flags/src/handler/canonical_log.rs@@ -1,4 +1,5 @@use crate::api::errors::FlagError;+use crate::api::request_outcome::OutcomeMarks;use crate::config::PersonsDbDeadlineBound;use crate::flags::flag_matching::EvaluationType;use crate::handler::phases::PhaseDurations;@@ -263,6 +264,10 @@ pub struct FlagsCanonicalLogLine {/// Which limit set the persons DB deadline. With `RequestTimeout`, a/// `persons_db_deadline_exceeded` on the same line can occur on a healthy persons DB.pub persons_db_deadline_bound: Option<PersonsDbDeadlineBound>,+ /// Shared with the request outcome middleware. Mark it from the tokio task. A rayon thread's+ /// canonical log gets its own `OutcomeMarks`, and `merge_rayon_delta` does not copy it back,+ /// so a mark made there never reaches the middleware.+ pub outcome_marks: OutcomeMarks,/// Which evaluation strategy was used for this request./// Set to `Parallel` if any dependency level triggered parallel evaluation.@@ -371,6 +376,7 @@ impl Default for FlagsCanonicalLogLine {hash_key_override_status: None,persons_db_deadline_exceeded: None,persons_db_deadline_bound: None,+ outcome_marks: OutcomeMarks::default(),evaluation_type: None,rate_limited: false,rate_limit_warned: false,diff --git a/rust/feature-flags/src/metrics/buckets.rs b/rust/feature-flags/src/metrics/buckets.rsindex a8e130cec138..137242dd5e31 100644--- a/rust/feature-flags/src/metrics/buckets.rs+++ b/rust/feature-flags/src/metrics/buckets.rs@@ -64,8 +64,8 @@ const INMEM_LOAD_BUCKETS_MS: &[f64] = &[// band, so the sub-ms buckets that `INMEM_LOAD_BUCKETS_MS` carries for// Redis would be permanently empty here. 1ms floor still captures// immediate errors (short-circuit before any network call) without-// losing them in an unbounded `<le_inf>` bucket. 10s ceiling covers the-// 3s `s3_timeout` plus headroom for the post-timeout emission.+// losing them in an unbounded `<le_inf>` bucket. 10s ceiling matches the+// largest `HYPERCACHE_S3_TIMEOUT_MS` that startup accepts.const S3_OP_BUCKETS_MS: &[f64] = &[1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0,];diff --git a/rust/feature-flags/src/metrics/consts.rs b/rust/feature-flags/src/metrics/consts.rsindex 1ab75901a638..46ac7d9d080b 100644--- a/rust/feature-flags/src/metrics/consts.rs+++ b/rust/feature-flags/src/metrics/consts.rs@@ -425,6 +425,9 @@ pub const FLAG_REQUEST_BLOCKED_COUNTER: &str = "flags_request_blocked_total";// Request-level timeout (tower TimeoutLayer killed the request before completion)pub const FLAG_REQUEST_TIMEOUT_COUNTER: &str = "flags_request_timeout_total";+// One count per /flags request, labeled `outcome` and `needs_persons`.+pub const FLAG_REQUEST_OUTCOME_COUNTER: &str = "flags_request_outcome_total";+// Timeout tracking and classificationpub const FLAG_ACQUIRE_TIMEOUT_COUNTER: &str = "flags_acquire_timeout_total";diff --git a/rust/feature-flags/src/router.rs b/rust/feature-flags/src/router.rsindex a2d647bc31b2..4dfd9c8f6b40 100644--- a/rust/feature-flags/src/router.rs+++ b/rust/feature-flags/src/router.rs@@ -43,6 +43,7 @@ use crate::{flag_definitions_rate_limiter::{FlagDefinitionsRateLimiter, RemoteConfigRateLimiter},flags_rate_limiter::{FlagsRateLimiter, IpRateLimiter},remote_config,+ request_outcome::{precreate_request_outcome_counters, record_request_outcome},},cohorts::{cohort_cache_manager::CohortCacheManager, membership::CohortMembershipProvider},config::{Config, ServiceMode, TeamIdCollection},@@ -67,6 +68,9 @@ use crate::{utils::bot_detection,};+/// `/flags` and its legacy `/decide` alias, with and without a trailing slash.+pub(crate) const FLAGS_ROUTES: [&str; 4] = ["/flags", "/flags/", "/decide", "/decide/"];+#[derive(Clone)]pub struct FlagsEndpointRateLimiters<C = clock::DefaultClock>where@@ -472,8 +476,9 @@ where);// Layer ordering (outermost to innermost, last `.layer()` is outermost):- // HandleErrorLayer → TimeoutLayer → record_concurrency_enter →- // ConcurrencyLimitLayer → record_concurrency_wait → (per-sub-router).+ // record_request_outcome → HandleErrorLayer → TimeoutLayer →+ // record_concurrency_enter → ConcurrencyLimitLayer →+ // record_concurrency_wait → (per-sub-router).//// The body-read shim and `DefaultBodyLimit` are per-sub-router and only// attached to `/flags|/decide`. `/flags/definitions` is GET-only and@@ -483,11 +488,10 @@ where// body is being buffered. Both are required.let mut flags_endpoints: Router<State> = Router::new();if matches!(config.service_mode, ServiceMode::All | ServiceMode::Flags) {+ for route in FLAGS_ROUTES {+ flags_endpoints = flags_endpoints.route(route, any(endpoint::flags::<C>));+ }flags_endpoints = flags_endpoints- .route("/flags", any(endpoint::flags::<C>))- .route("/flags/", any(endpoint::flags::<C>))- .route("/decide", any(endpoint::flags::<C>))- .route("/decide/", any(endpoint::flags::<C>)).layer(axum::middleware::from_fn(record_body_read)).layer(DefaultBodyLimit::max(MAX_FLAGS_BODY_BYTES)).layer(Extension(rate_limiters));@@ -592,7 +596,9 @@ where.layer(TimeoutLayer::new(Duration::from_millis(config.request_timeout_ms,))),- );+ )+ // Outside the timeout, so it also counts the requests that the timeout drops.+ .layer(axum::middleware::from_fn(record_request_outcome));let router = Router::new().merge(status_router)@@ -607,11 +613,14 @@ where// In other words, only turn these on in productionif config.enable_metrics {common_metrics::set_label_filter(team_id_label_filter(config.team_ids_to_track.clone()));- setup_metrics_routes_for_product_with_overrides(+ let router = setup_metrics_routes_for_product_with_overrides(router,"feature_flags",&bucket_overrides(),- )+ );+ // After the recorder exists.+ precreate_request_outcome_counters();+ router} else {router}diff --git a/rust/feature-flags/src/server.rs b/rust/feature-flags/src/server.rsindex 23f728da272f..a6a044adf6a0 100644--- a/rust/feature-flags/src/server.rs+++ b/rust/feature-flags/src/server.rs@@ -329,12 +329,15 @@ pub async fn serve_with_rate_limiter_clock<C>(.clone().unwrap_or_else(|| redis_client.clone());+ let hypercache_s3_timeout = Duration::from_millis(config.hypercache_s3_timeout_ms);+let mut flags_hypercache_config = HyperCacheConfig::new("feature_flags".to_string(),"flags.json".to_string(),config.object_storage_region.clone(),config.object_storage_bucket.clone(),);+ flags_hypercache_config.s3_timeout = hypercache_s3_timeout;// Etag-paired on the Django writer side, which makes HyperCacheReader refuse read repair.flags_hypercache_config.enable_etag = true;@@ -380,6 +383,7 @@ pub async fn serve_with_rate_limiter_clock<C>(config.object_storage_bucket.clone(),);team_hypercache_config.token_based = true;+ team_hypercache_config.s3_timeout = hypercache_s3_timeout;// Left out of read repair: a hit here is trusted as proof of a valid token (see// get_team_from_cache_or_pg), with no Postgres re-check. Team deletion clears Redis// before S3, so a request landing in that gap reads the deleted team from S3; repairing@@ -413,6 +417,7 @@ pub async fn serve_with_rate_limiter_clock<C>();// Etag-paired, same as flags.json above.flags_with_cohorts_config.enable_etag = true;+ flags_with_cohorts_config.s3_timeout = hypercache_s3_timeout;if !config.object_storage_endpoint.is_empty() {flags_with_cohorts_config.s3_endpoint = Some(config.object_storage_endpoint.clone());@@ -458,6 +463,7 @@ pub async fn serve_with_rate_limiter_clock<C>(config.object_storage_bucket.clone(),);config_hypercache_config.token_based = true;+ config_hypercache_config.s3_timeout = hypercache_s3_timeout;config_hypercache_config.read_repair_ttl_seconds = read_repair_ttl_seconds;if !config.object_storage_endpoint.is_empty() {diff --git a/rust/feature-flags/tests/stall_recovery.rs b/rust/feature-flags/tests/stall_recovery.rsnew file mode 100644index 000000000000..25fe862e50cf--- /dev/null+++ b/rust/feature-flags/tests/stall_recovery.rs@@ -0,0 +1,635 @@+//! Persons DB faults under steady load, with production pool and timeout settings, except the+//! longer persons statement timeout that `Fault::LockWait` needs.+//! Ignored because the faults lock `posthog_person` and would stall other tests: run alone with+//! `cargo nextest run -p feature-flags -E 'binary(stall_recovery)' --run-ignored only --test-threads=1`.++pub mod common;+#[path = "stall_recovery/persons_db_proxy.rs"]+mod persons_db_proxy;++use std::net::SocketAddr;+use std::sync::{Arc, Mutex, OnceLock};+use std::time::Duration;++use feature_flags::api::types::{FlagDetails, FlagsResponse};+use feature_flags::config::{Config, FlexBool, DEFAULT_TEST_CONFIG};+use feature_flags::utils::test_utils::{+ insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client, TestContext,+};+use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};+use reqwest::{header::CONTENT_TYPE, StatusCode};+use serde_json::json;+use sqlx::{Connection, PgConnection};+use tokio::task::JoinHandle;+use tokio::time::Instant;+use tokio_util::sync::CancellationToken;++use common::ServerHandle;+use persons_db_proxy::PersonsDbProxy;++const DB_FREE_FLAG: &str = "db-free";+const PERSON_FLAG: &str = "person";+const DISTINCT_ID: &str = "stall_user";+const PERSON_EMAIL: &str = "[email protected]";++const BASELINE: Duration = Duration::from_secs(3);+const RECOVERY_LIMIT: Duration = Duration::from_secs(10);+/// Load continues this long after recovery, so connections reopened after the fault show.+const SETTLE: Duration = Duration::from_secs(5);+/// Longer than the 3 s acquire timeout.+const SLOW_CONNECT: Duration = Duration::from_secs(5);++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+#[ignore = "locks posthog_person, which stalls other tests"]+async fn test_cancels_heard_late_for_16s() {+ let report = Scenario::new(Fault::CancelsHeardLate, Duration::from_secs(16))+ .run()+ .await;+ report.assert_degrades_then_recovers();+ report.assert_connections_reopen();+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+#[ignore = "locks posthog_person, which stalls other tests"]+async fn test_cancels_heard_late_for_30s() {+ let report = Scenario::new(Fault::CancelsHeardLate, Duration::from_secs(30))+ .run()+ .await;+ report.assert_degrades_then_recovers();+ report.assert_connections_reopen();+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+#[ignore = "locks posthog_person, which stalls other tests"]+async fn test_lock_wait_for_16s() {+ let report = Scenario::new(Fault::LockWait, Duration::from_secs(16))+ .run()+ .await;+ report.assert_degrades_then_recovers();+ report.assert_connections_reopen();+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+#[ignore = "timing-sensitive, so it runs alone"]+async fn test_connects_slower_than_the_acquire_timeout_for_16s() {+ let report = Scenario::new(Fault::SlowConnects, Duration::from_secs(16))+ .run()+ .await;+ report.assert_degrades_then_recovers();+ assert!(+ report.connect_attempts_during() > 0,+ "the pool made no connect attempt during the fault"+ );+ assert_eq!(+ report.connections_opened_during(),+ 0,+ "today no connect succeeds during the fault; #122930 and #122931 change this"+ );+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+#[ignore = "timing-sensitive, so it runs alone"]+async fn test_request_timeout_counts_as_an_error_that_needed_persons() {+ let proxy = PersonsDbProxy::start(&DEFAULT_TEST_CONFIG.persons_read_database_url).await;+ let token = seed().await;+ let server = ServerHandle::for_config(Config {+ request_timeout_ms: 1_000,+ persons_db_deadline_ms: 0,+ persons_read_database_url: proxy.url().to_string(),+ persons_write_database_url: proxy.url().to_string(),+ ..Config::default_test_config()+ })+ .await;+ server.wait_until_ready().await;+ let errors_before = persons_requests("error");++ proxy.hold_replies();+ let response = server+ .send_flags_request(RequestKind::BothFlags.body(&token), Some("2"), None)+ .await;+ proxy.release_replies();++ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(persons_requests("error") - errors_before, 1);+}++#[derive(Clone, Copy, Debug)]+enum Fault {+ /// Postgres cancels person queries on time, and the proxy holds the cancels until the end.+ CancelsHeardLate,+ /// Person queries wait on a lock past the persons DB deadline, and Postgres cancels them on time.+ LockWait,+ /// Every new connection takes longer than the acquire timeout.+ SlowConnects,+}++impl Fault {+ fn persons_statement_timeout_ms(self) -> u64 {+ match self {+ // Between the 3.5 s persons DB deadline and the lock.+ Fault::LockWait => 5_000,+ Fault::CancelsHeardLate | Fault::SlowConnects => 1_000,+ }+ }++ /// Returns when the fault took effect.+ async fn inject(self, proxy: &PersonsDbProxy, admin: &mut PgConnection) -> Instant {+ match self {+ Fault::CancelsHeardLate => {+ lock_person_table(admin).await;+ let start = Instant::now();+ // Less than the statement timeout, so each pooled connection is waiting on the+ // lock when the replies stop, and parks on a held cancel.+ tokio::time::sleep(Duration::from_millis(500)).await;+ proxy.hold_replies();+ start+ }+ Fault::LockWait => {+ lock_person_table(admin).await;+ Instant::now()+ }+ Fault::SlowConnects => {+ proxy.delay_connects(SLOW_CONNECT);+ proxy.close_connections();+ Instant::now()+ }+ }+ }++ async fn lift(self, proxy: &PersonsDbProxy, admin: &mut PgConnection) {+ match self {+ Fault::CancelsHeardLate => {+ unlock_person_table(admin).await;+ proxy.release_replies();+ }+ Fault::LockWait => unlock_person_table(admin).await,+ Fault::SlowConnects => proxy.delay_connects(Duration::ZERO),+ }+ }+}++async fn lock_person_table(admin: &mut PgConnection) {+ sqlx::query("BEGIN").execute(&mut *admin).await.unwrap();+ sqlx::query("LOCK TABLE posthog_person IN ACCESS EXCLUSIVE MODE")+ .execute(&mut *admin)+ .await+ .unwrap();+}++async fn unlock_person_table(admin: &mut PgConnection) {+ sqlx::query("ROLLBACK").execute(admin).await.unwrap();+}++struct Scenario {+ fault: Fault,+ fault_length: Duration,+ clients: usize,+ max_concurrency: usize,+}++impl Scenario {+ fn new(fault: Fault, fault_length: Duration) -> Self {+ Self {+ fault,+ fault_length,+ clients: 20,+ max_concurrency: 100,+ }+ }++ fn server_config(&self, persons_url: &str) -> Config {+ Config {+ acquire_timeout_secs: 3,+ persons_db_deadline_ms: 3_500,+ persons_db_deadline_reserve_ms: 500,+ request_timeout_ms: 4_500,+ max_pg_connections: 8,+ max_concurrency: self.max_concurrency,+ persons_reader_statement_timeout_ms: self.fault.persons_statement_timeout_ms(),+ test_before_acquire: FlexBool(true),+ persons_read_database_url: persons_url.to_string(),+ persons_write_database_url: persons_url.to_string(),+ ..Config::default_test_config()+ }+ }++ async fn run(&self) -> Report {+ let persons_url = &DEFAULT_TEST_CONFIG.persons_read_database_url;+ let proxy = PersonsDbProxy::start(persons_url).await;+ let mut admin = PgConnection::connect(persons_url).await.unwrap();+ let token = seed().await;+ let server = ServerHandle::for_config(self.server_config(proxy.url())).await;+ server.wait_until_ready().await;++ let load = Load::start(server.addr, &token, self.clients);+ assert!(+ load.wait_until_healthy(Instant::now(), RECOVERY_LIMIT)+ .await,+ "no healthy answer before the fault"+ );+ tokio::time::sleep(BASELINE).await;++ let at_start = Counts::read(&proxy);+ let fault_start = self.fault.inject(&proxy, &mut admin).await;+ tokio::time::sleep_until(fault_start + self.fault_length).await;+ let at_end = Counts::read(&proxy);+ let fault_end = Instant::now();+ self.fault.lift(&proxy, &mut admin).await;++ load.wait_until_healthy(fault_end, RECOVERY_LIMIT).await;+ tokio::time::sleep(SETTLE).await;+ let at_finish = Counts::read(&proxy);++ let report = Report {+ fault: self.fault,+ fault_start,+ fault_end,+ samples: load.stop().await,+ at_start,+ at_end,+ at_finish,+ };+ report.print();+ report+ }+}++/// Returns the team token.+async fn seed() -> String {+ let redis = setup_redis_client(Some(DEFAULT_TEST_CONFIG.redis_url.clone())).await;+ let team = insert_new_team_in_redis(redis.clone()).await.unwrap();+ let context = TestContext::new(None).await;+ context.insert_new_team(Some(team.id)).await.unwrap();+ context+ .insert_person(+ team.id,+ DISTINCT_ID.to_string(),+ Some(json!({ "email": PERSON_EMAIL })),+ )+ .await+ .unwrap();+ let flags = json!([+ {+ "id": 1,+ "key": DB_FREE_FLAG,+ "active": true,+ "deleted": false,+ "team_id": team.id,+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]}+ },+ {+ "id": 2,+ "key": PERSON_FLAG,+ "active": true,+ "deleted": false,+ "team_id": team.id,+ "filters": {"groups": [{+ "properties": [{"key": "email", "value": PERSON_EMAIL, "type": "person", "operator": "exact"}],+ "rollout_percentage": 100+ }]}+ }+ ]);+ insert_flags_for_team_in_redis(redis, team.id, Some(flags.to_string()))+ .await+ .unwrap();+ team.api_token+}++/// One global recorder per process, so each test compares how much a counter grew between two+/// reads, never its absolute value.+fn metrics() -> &'static Snapshotter {+ static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();+ SNAPSHOTTER.get_or_init(|| {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ recorder+ .install()+ .expect("no other global metrics recorder");+ snapshotter+ })+}++struct Counts {+ persons_connections: u64,+ connect_attempts: usize,+ degraded_persons_requests: u64,+ ok_persons_requests: u64,+}++fn counter(name: &str, labels: &[(&str, &str)]) -> u64 {+ metrics()+ .snapshot()+ .into_vec()+ .into_iter()+ .filter(|(key, ..)| {+ let key = key.key();+ key.name() == name+ && labels.iter().all(|(label, value)| {+ key.labels()+ .any(|l| l.key() == *label && l.value() == *value)+ })+ })+ .map(|(.., value)| match value {+ DebugValue::Counter(count) => count,+ other => panic!("{name} is not a counter: {other:?}"),+ })+ .sum()+}++fn persons_requests(outcome: &str) -> u64 {+ counter(+ "flags_request_outcome_total",+ &[("outcome", outcome), ("needs_persons", "true")],+ )+}++impl Counts {+ fn read(proxy: &PersonsDbProxy) -> Self {+ Self {+ persons_connections: counter(+ "db_connection_created_total",+ &[("pool", "persons_reader")],+ ),+ connect_attempts: proxy.accepted(),+ degraded_persons_requests: persons_requests("degraded"),+ ok_persons_requests: persons_requests("ok"),+ }+ }+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum RequestKind {+ BothFlags,+ /// Only the database-free flag, through `flag_keys`, to time database-free work alone.+ DbFreeOnly,+}++impl RequestKind {+ fn body(self, token: &str) -> String {+ match self {+ RequestKind::BothFlags => json!({"token": token, "distinct_id": DISTINCT_ID}),+ RequestKind::DbFreeOnly => {+ json!({"token": token, "distinct_id": DISTINCT_ID, "flag_keys": [DB_FREE_FLAG]})+ }+ }+ .to_string()+ }++ /// What every request of this kind answers while the persons DB is down.+ fn answer_during_fault(self) -> Answer {+ match self {+ RequestKind::BothFlags => Answer::PersonFlagFailed,+ RequestKind::DbFreeOnly => Answer::Healthy,+ }+ }+}++#[derive(Clone, Debug, PartialEq, Eq)]+enum Answer {+ /// 200, and every requested flag evaluated.+ Healthy,+ /// 200, the database-free flag evaluated, and the person flag failed with+ /// `errorsWhileComputingFlags` set.+ PersonFlagFailed,+ Unexpected(String),+}++impl Answer {+ async fn read(kind: RequestKind, response: reqwest::Result<reqwest::Response>) -> Self {+ let response = match response {+ Ok(response) if response.status() == StatusCode::OK => response,+ Ok(response) => return Self::Unexpected(format!("HTTP {}", response.status())),+ Err(e) => return Self::Unexpected(e.to_string()),+ };+ let flags = match response.json::<FlagsResponse>().await {+ Ok(flags) => flags,+ Err(e) => return Self::Unexpected(e.to_string()),+ };+ let evaluated = |flag: &FlagDetails| flag.enabled && !flag.failed;+ let db_free = flags.flags.get(DB_FREE_FLAG);+ let person = flags.flags.get(PERSON_FLAG);+ let person_evaluated = match kind {+ RequestKind::BothFlags => person.is_some_and(evaluated),+ RequestKind::DbFreeOnly => person.is_none(),+ };+ match (db_free, person, flags.errors_while_computing_flags) {+ (Some(db_free), _, false) if evaluated(db_free) && person_evaluated => Self::Healthy,+ (Some(db_free), Some(person), true) if evaluated(db_free) && person.failed => {+ Self::PersonFlagFailed+ }+ (db_free, person, errors) => {+ Self::Unexpected(format!("{db_free:?} {person:?} errors={errors}"))+ }+ }+ }+}++struct Sample {+ kind: RequestKind,+ started: Instant,+ finished: Instant,+ answer: Answer,+}++/// When the first healthy request for both flags that started at or after `since` finished.+fn first_healthy_since(samples: &[Sample], since: Instant) -> Option<Instant> {+ samples+ .iter()+ .filter(|s| {+ s.kind == RequestKind::BothFlags && s.started >= since && s.answer == Answer::Healthy+ })+ .map(|s| s.finished)+ .min()+}++/// `clients` loop requests for both flags, and one more client loops database-free requests.+struct Load {+ samples: Arc<Mutex<Vec<Sample>>>,+ stop: CancellationToken,+ clients: Vec<JoinHandle<()>>,+}++impl Load {+ fn start(addr: SocketAddr, token: &str, clients: usize) -> Self {+ let http = reqwest::Client::builder()+ .timeout(Duration::from_secs(10))+ .build()+ .unwrap();+ let url = format!("http://{addr}/flags?v=2");+ let samples: Arc<Mutex<Vec<Sample>>> = Arc::default();+ let stop = CancellationToken::new();+ let clients = std::iter::repeat_n(RequestKind::BothFlags, clients)+ .chain([RequestKind::DbFreeOnly])+ .map(|kind| {+ let (http, url, body) = (http.clone(), url.clone(), kind.body(token));+ let (samples, stop) = (samples.clone(), stop.clone());+ tokio::spawn(async move {+ while !stop.is_cancelled() {+ let started = Instant::now();+ let response = http+ .post(&url)+ .header(CONTENT_TYPE, "application/json")+ .body(body.clone())+ .send()+ .await;+ let answer = Answer::read(kind, response).await;+ samples.lock().unwrap().push(Sample {+ kind,+ started,+ finished: Instant::now(),+ answer,+ });+ }+ })+ })+ .collect();+ Self {+ samples,+ stop,+ clients,+ }+ }++ async fn wait_until_healthy(&self, since: Instant, limit: Duration) -> bool {+ while Instant::now() < since + limit {+ if first_healthy_since(&self.samples.lock().unwrap(), since).is_some() {+ return true;+ }+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ false+ }++ async fn stop(self) -> Vec<Sample> {+ self.stop.cancel();+ for client in self.clients {+ client.await.unwrap();+ }+ std::mem::take(&mut *self.samples.lock().unwrap())+ }+}++struct Report {+ fault: Fault,+ fault_start: Instant,+ fault_end: Instant,+ samples: Vec<Sample>,+ at_start: Counts,+ at_end: Counts,+ at_finish: Counts,+}++impl Report {+ fn before(&self) -> impl Iterator<Item = &Sample> {+ self.samples+ .iter()+ .filter(|s| s.finished <= self.fault_start)+ }++ fn during(&self) -> impl Iterator<Item = &Sample> {+ self.samples+ .iter()+ .filter(|s| s.started >= self.fault_start && s.finished <= self.fault_end)+ }++ fn recovery(&self) -> Option<Duration> {+ first_healthy_since(&self.samples, self.fault_end).map(|finished| finished - self.fault_end)+ }++ fn connections_opened_during(&self) -> u64 {+ self.at_end.persons_connections - self.at_start.persons_connections+ }++ fn connections_opened_after(&self) -> u64 {+ self.at_finish.persons_connections - self.at_end.persons_connections+ }++ fn connect_attempts_during(&self) -> usize {+ self.at_end.connect_attempts - self.at_start.connect_attempts+ }++ fn print(&self) {+ let db_free = |s: &&Sample| s.kind == RequestKind::DbFreeOnly;+ let both_flags: Vec<_> = self+ .during()+ .filter(|s| s.kind == RequestKind::BothFlags)+ .collect();+ let failed = both_flags+ .iter()+ .filter(|s| s.answer == Answer::PersonFlagFailed)+ .count();+ println!(+ "{:?} for {:?}",+ self.fault,+ self.fault_end - self.fault_start+ );+ println!(+ " database-free latency: before {}, during {}",+ latency_summary(self.before().filter(db_free)),+ latency_summary(self.during().filter(db_free))+ );+ println!(+ " person flag failed in {failed} of {} requests during the fault",+ both_flags.len()+ );+ println!(" first success after the fault: {:?}", self.recovery());+ println!(+ " persons_reader connections opened: {} during, {} after",+ self.connections_opened_during(),+ self.connections_opened_after()+ );+ println!(+ " connect attempts at the proxy: {} during, {} after",+ self.connect_attempts_during(),+ self.at_finish.connect_attempts - self.at_end.connect_attempts+ );+ }++ /// Holds today and after every later change to the persons data path.+ fn assert_degrades_then_recovers(&self) {+ let mut during = self.during().peekable();+ assert!(during.peek().is_some(), "no request ran inside the fault");+ for sample in during {+ assert_eq!(+ sample.answer,+ sample.kind.answer_during_fault(),+ "{:?} request {:?} into the fault",+ sample.kind,+ sample.started - self.fault_start+ );+ }++ let recovery = self.recovery();+ assert!(+ recovery.is_some_and(|recovery| recovery <= RECOVERY_LIMIT),+ "no success within {RECOVERY_LIMIT:?} of the fault ending: {recovery:?}"+ );++ assert!(+ self.at_end.degraded_persons_requests > self.at_start.degraded_persons_requests,+ "no degraded persons request counted during the fault"+ );+ assert!(+ self.at_finish.ok_persons_requests > self.at_end.ok_persons_requests,+ "no ok persons request counted after the fault"+ );+ }++ fn assert_connections_reopen(&self) {+ assert!(+ self.connections_opened_during() + self.connections_opened_after() > 0,+ "today the pool reopens persons connections; #122930 flips this"+ );+ }+}++fn latency_summary<'a>(samples: impl Iterator<Item = &'a Sample>) -> String {+ let mut latencies: Vec<Duration> = samples.map(|s| s.finished - s.started).collect();+ latencies.sort();+ match (latencies.get(latencies.len() / 2), latencies.last()) {+ (Some(p50), Some(max)) => {+ format!("p50 {p50:?}, max {max:?} over {} requests", latencies.len())+ }+ _ => "no requests".to_string(),+ }+}diff --git a/rust/feature-flags/tests/stall_recovery/persons_db_proxy.rs b/rust/feature-flags/tests/stall_recovery/persons_db_proxy.rsnew file mode 100644index 000000000000..8d8bb18cf20f--- /dev/null+++ b/rust/feature-flags/tests/stall_recovery/persons_db_proxy.rs@@ -0,0 +1,167 @@+//! A TCP proxy between the persons pools and Postgres that injects the faults of a persons DB+//! stall.++use std::sync::{+ atomic::{AtomicUsize, Ordering},+ Arc, Mutex,+};+use std::time::Duration;++use tokio::io::{AsyncReadExt, AsyncWriteExt};+use tokio::net::{+ tcp::{OwnedReadHalf, OwnedWriteHalf},+ TcpListener, TcpStream,+};+use tokio::sync::watch;+use tokio::task::JoinHandle;+use tokio_util::sync::CancellationToken;+use url::Url;++#[derive(Clone, Copy, Default)]+struct Faults {+ hold_replies: bool,+ connect_delay: Duration,+}++pub struct PersonsDbProxy {+ url: String,+ faults: watch::Sender<Faults>,+ accepted: Arc<AtomicUsize>,+ open_connections: Arc<Mutex<CancellationToken>>,+ accept_loop: JoinHandle<()>,+}++impl PersonsDbProxy {+ /// Proxies the Postgres server of `database_url`. [`Self::url`] reaches the same database+ /// through the proxy.+ pub async fn start(database_url: &str) -> Self {+ let mut url = Url::parse(database_url).expect("valid database URL");+ let upstream = (+ url.host_str().expect("database URL has a host").to_string(),+ url.port().unwrap_or(5432),+ );+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ url.set_host(Some("127.0.0.1")).unwrap();+ url.set_port(Some(listener.local_addr().unwrap().port()))+ .unwrap();++ let (faults, faults_rx) = watch::channel(Faults::default());+ let accepted = Arc::new(AtomicUsize::new(0));+ let open_connections = Arc::new(Mutex::new(CancellationToken::new()));+ let accept_loop = tokio::spawn(accept(+ listener,+ upstream,+ faults_rx,+ accepted.clone(),+ open_connections.clone(),+ ));++ Self {+ url: url.to_string(),+ faults,+ accepted,+ open_connections,+ accept_loop,+ }+ }++ pub fn url(&self) -> &str {+ &self.url+ }++ /// Connections the proxy accepted, which counts connect attempts whether they succeed or not.+ pub fn accepted(&self) -> usize {+ self.accepted.load(Ordering::Relaxed)+ }++ /// Holds every server-to-client byte, on open and new connections alike, until+ /// [`Self::release_replies`].+ pub fn hold_replies(&self) {+ self.faults.send_modify(|faults| faults.hold_replies = true);+ }++ pub fn release_replies(&self) {+ self.faults+ .send_modify(|faults| faults.hold_replies = false);+ }++ /// Waits this long before connecting each newly accepted connection to Postgres.+ pub fn delay_connects(&self, delay: Duration) {+ self.faults+ .send_modify(|faults| faults.connect_delay = delay);+ }++ pub fn close_connections(&self) {+ let mut open = self.open_connections.lock().unwrap();+ open.cancel();+ *open = CancellationToken::new();+ }+}++impl Drop for PersonsDbProxy {+ fn drop(&mut self) {+ self.accept_loop.abort();+ self.close_connections();+ }+}++async fn accept(+ listener: TcpListener,+ upstream: (String, u16),+ faults: watch::Receiver<Faults>,+ accepted: Arc<AtomicUsize>,+ open_connections: Arc<Mutex<CancellationToken>>,+) {+ while let Ok((client, _)) = listener.accept().await {+ accepted.fetch_add(1, Ordering::Relaxed);+ let connect_delay = faults.borrow().connect_delay;+ let closed = open_connections.lock().unwrap().clone();+ let relay = relay(client, upstream.clone(), connect_delay, faults.clone());+ tokio::spawn(async move {+ tokio::select! {+ _ = closed.cancelled() => {}+ _ = relay => {}+ }+ });+ }+}++async fn relay(+ client: TcpStream,+ (host, port): (String, u16),+ connect_delay: Duration,+ faults: watch::Receiver<Faults>,+) {+ tokio::time::sleep(connect_delay).await;+ let Ok(server) = TcpStream::connect((host.as_str(), port)).await else {+ return;+ };+ let (mut client_read, client_write) = client.into_split();+ let (server_read, mut server_write) = server.into_split();+ tokio::select! {+ _ = tokio::io::copy(&mut client_read, &mut server_write) => {}+ _ = forward_replies(server_read, client_write, faults) => {}+ }+}++async fn forward_replies(+ mut server: OwnedReadHalf,+ mut client: OwnedWriteHalf,+ mut faults: watch::Receiver<Faults>,+) -> std::io::Result<()> {+ let mut buf = vec![0; 16 * 1024];+ loop {+ let read = server.read(&mut buf).await?;+ if read == 0 {+ return Ok(());+ }+ if faults+ .wait_for(|faults| !faults.hold_replies)+ .await+ .is_err()+ {+ return Ok(());+ }+ client.write_all(&buf[..read]).await?;+ }+}