Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.4 KB
Raw Blame
//! Persons DB faults under steady load, with production pool and timeout settings, except the
//! longer persons statement timeout that `Fault::LockWait` needs.
//! Ignored because the faults lock `posthog_person` and would stall other tests: run alone with
//! `cargo nextest run -p feature-flags -E 'binary(stall_recovery)' --run-ignored only --test-threads=1`.

pub mod common;
#[path = "stall_recovery/persons_db_proxy.rs"]
mod persons_db_proxy;

use std::net::SocketAddr;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;

use feature_flags::api::types::{FlagDetails, FlagsResponse};
use feature_flags::config::{Config, FlexBool, DEFAULT_TEST_CONFIG};
use feature_flags::utils::test_utils::{
    insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client, TestContext,
};
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
use reqwest::{header::CONTENT_TYPE, StatusCode};
use serde_json::json;
use sqlx::{Connection, PgConnection};
use tokio::task::JoinHandle;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;

use common::ServerHandle;
use persons_db_proxy::PersonsDbProxy;

const DB_FREE_FLAG: &str = "db-free";
const PERSON_FLAG: &str = "person";
const DISTINCT_ID: &str = "stall_user";
const PERSON_EMAIL: &str = "[email protected]";

const BASELINE: Duration = Duration::from_secs(3);
const RECOVERY_LIMIT: Duration = Duration::from_secs(10);
/// Load continues this long after recovery, so connections reopened after the fault show.
const SETTLE: Duration = Duration::from_secs(5);
/// Longer than the 3 s acquire timeout.
const SLOW_CONNECT: Duration = Duration::from_secs(5);

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "locks posthog_person, which stalls other tests"]
async fn test_cancels_heard_late_for_16s() {
    let report = Scenario::new(Fault::CancelsHeardLate, Duration::from_secs(16))
        .run()
        .await;
    report.assert_degrades_then_recovers();
    report.assert_connections_reopen();
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "locks posthog_person, which stalls other tests"]
async fn test_cancels_heard_late_for_30s() {
    let report = Scenario::new(Fault::CancelsHeardLate, Duration::from_secs(30))
        .run()
        .await;
    report.assert_degrades_then_recovers();
    report.assert_connections_reopen();
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "locks posthog_person, which stalls other tests"]
async fn test_lock_wait_for_16s() {
    let report = Scenario::new(Fault::LockWait, Duration::from_secs(16))
        .run()
        .await;
    report.assert_degrades_then_recovers();
    report.assert_connections_reopen();
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "timing-sensitive, so it runs alone"]
async fn test_connects_slower_than_the_acquire_timeout_for_16s() {
    let report = Scenario::new(Fault::SlowConnects, Duration::from_secs(16))
        .run()
        .await;
    report.assert_degrades_then_recovers();
    assert!(
        report.connect_attempts_during() > 0,
        "the pool made no connect attempt during the fault"
    );
    assert_eq!(
        report.connections_opened_during(),
        0,
        "today no connect succeeds during the fault; #122930 and #122931 change this"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "timing-sensitive, so it runs alone"]
async fn test_request_timeout_counts_as_an_error_that_needed_persons() {
    let proxy = PersonsDbProxy::start(&DEFAULT_TEST_CONFIG.persons_read_database_url).await;
    let token = seed().await;
    let server = ServerHandle::for_config(Config {
        request_timeout_ms: 1_000,
        persons_db_deadline_ms: 0,
        persons_read_database_url: proxy.url().to_string(),
        persons_write_database_url: proxy.url().to_string(),
        ..Config::default_test_config()
    })
    .await;
    server.wait_until_ready().await;
    let errors_before = persons_requests("error");

    proxy.hold_replies();
    let response = server
        .send_flags_request(RequestKind::BothFlags.body(&token), Some("2"), None)
        .await;
    proxy.release_replies();

    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
    assert_eq!(persons_requests("error") - errors_before, 1);
}

#[derive(Clone, Copy, Debug)]
enum Fault {
    /// Postgres cancels person queries on time, and the proxy holds the cancels until the end.
    CancelsHeardLate,
    /// Person queries wait on a lock past the persons DB deadline, and Postgres cancels them on time.
    LockWait,
    /// Every new connection takes longer than the acquire timeout.
    SlowConnects,
}

impl Fault {
    fn persons_statement_timeout_ms(self) -> u64 {
        match self {
            // Between the 3.5 s persons DB deadline and the lock.
            Fault::LockWait => 5_000,
            Fault::CancelsHeardLate | Fault::SlowConnects => 1_000,
        }
    }

    /// Returns when the fault took effect.
    async fn inject(self, proxy: &PersonsDbProxy, admin: &mut PgConnection) -> Instant {
        match self {
            Fault::CancelsHeardLate => {
                lock_person_table(admin).await;
                let start = Instant::now();
                // Less than the statement timeout, so each pooled connection is waiting on the
                // lock when the replies stop, and parks on a held cancel.
                tokio::time::sleep(Duration::from_millis(500)).await;
                proxy.hold_replies();
                start
            }
            Fault::LockWait => {
                lock_person_table(admin).await;
                Instant::now()
            }
            Fault::SlowConnects => {
                proxy.delay_connects(SLOW_CONNECT);
                proxy.close_connections();
                Instant::now()
            }
        }
    }

    async fn lift(self, proxy: &PersonsDbProxy, admin: &mut PgConnection) {
        match self {
            Fault::CancelsHeardLate => {
                unlock_person_table(admin).await;
                proxy.release_replies();
            }
            Fault::LockWait => unlock_person_table(admin).await,
            Fault::SlowConnects => proxy.delay_connects(Duration::ZERO),
        }
    }
}

async fn lock_person_table(admin: &mut PgConnection) {
    sqlx::query("BEGIN").execute(&mut *admin).await.unwrap();
    sqlx::query("LOCK TABLE posthog_person IN ACCESS EXCLUSIVE MODE")
        .execute(&mut *admin)
        .await
        .unwrap();
}

async fn unlock_person_table(admin: &mut PgConnection) {
    sqlx::query("ROLLBACK").execute(admin).await.unwrap();
}

struct Scenario {
    fault: Fault,
    fault_length: Duration,
    clients: usize,
    max_concurrency: usize,
}

impl Scenario {
    fn new(fault: Fault, fault_length: Duration) -> Self {
        Self {
            fault,
            fault_length,
            clients: 20,
            max_concurrency: 100,
        }
    }

    fn server_config(&self, persons_url: &str) -> Config {
        Config {
            acquire_timeout_secs: 3,
            persons_db_deadline_ms: 3_500,
            persons_db_deadline_reserve_ms: 500,
            request_timeout_ms: 4_500,
            max_pg_connections: 8,
            max_concurrency: self.max_concurrency,
            persons_reader_statement_timeout_ms: self.fault.persons_statement_timeout_ms(),
            test_before_acquire: FlexBool(true),
            persons_read_database_url: persons_url.to_string(),
            persons_write_database_url: persons_url.to_string(),
            ..Config::default_test_config()
        }
    }

    async fn run(&self) -> Report {
        let persons_url = &DEFAULT_TEST_CONFIG.persons_read_database_url;
        let proxy = PersonsDbProxy::start(persons_url).await;
        let mut admin = PgConnection::connect(persons_url).await.unwrap();
        let token = seed().await;
        let server = ServerHandle::for_config(self.server_config(proxy.url())).await;
        server.wait_until_ready().await;

        let load = Load::start(server.addr, &token, self.clients);
        assert!(
            load.wait_until_healthy(Instant::now(), RECOVERY_LIMIT)
                .await,
            "no healthy answer before the fault"
        );
        tokio::time::sleep(BASELINE).await;

        let at_start = Counts::read(&proxy);
        let fault_start = self.fault.inject(&proxy, &mut admin).await;
        tokio::time::sleep_until(fault_start + self.fault_length).await;
        let at_end = Counts::read(&proxy);
        let fault_end = Instant::now();
        self.fault.lift(&proxy, &mut admin).await;

        load.wait_until_healthy(fault_end, RECOVERY_LIMIT).await;
        tokio::time::sleep(SETTLE).await;
        let at_finish = Counts::read(&proxy);

        let report = Report {
            fault: self.fault,
            fault_start,
            fault_end,
            samples: load.stop().await,
            at_start,
            at_end,
            at_finish,
        };
        report.print();
        report
    }
}

/// Returns the team token.
async fn seed() -> String {
    let redis = setup_redis_client(Some(DEFAULT_TEST_CONFIG.redis_url.clone())).await;
    let team = insert_new_team_in_redis(redis.clone()).await.unwrap();
    let context = TestContext::new(None).await;
    context.insert_new_team(Some(team.id)).await.unwrap();
    context
        .insert_person(
            team.id,
            DISTINCT_ID.to_string(),
            Some(json!({ "email": PERSON_EMAIL })),
        )
        .await
        .unwrap();
    let flags = json!([
        {
            "id": 1,
            "key": DB_FREE_FLAG,
            "active": true,
            "deleted": false,
            "team_id": team.id,
            "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]}
        },
        {
            "id": 2,
            "key": PERSON_FLAG,
            "active": true,
            "deleted": false,
            "team_id": team.id,
            "filters": {"groups": [{
                "properties": [{"key": "email", "value": PERSON_EMAIL, "type": "person", "operator": "exact"}],
                "rollout_percentage": 100
            }]}
        }
    ]);
    insert_flags_for_team_in_redis(redis, team.id, Some(flags.to_string()))
        .await
        .unwrap();
    team.api_token
}

/// One global recorder per process, so each test compares how much a counter grew between two
/// reads, never its absolute value.
fn metrics() -> &'static Snapshotter {
    static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();
    SNAPSHOTTER.get_or_init(|| {
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        recorder
            .install()
            .expect("no other global metrics recorder");
        snapshotter
    })
}

struct Counts {
    persons_connections: u64,
    connect_attempts: usize,
    degraded_persons_requests: u64,
    ok_persons_requests: u64,
}

fn counter(name: &str, labels: &[(&str, &str)]) -> u64 {
    metrics()
        .snapshot()
        .into_vec()
        .into_iter()
        .filter(|(key, ..)| {
            let key = key.key();
            key.name() == name
                && labels.iter().all(|(label, value)| {
                    key.labels()
                        .any(|l| l.key() == *label && l.value() == *value)
                })
        })
        .map(|(.., value)| match value {
            DebugValue::Counter(count) => count,
            other => panic!("{name} is not a counter: {other:?}"),
        })
        .sum()
}

fn persons_requests(outcome: &str) -> u64 {
    counter(
        "flags_request_outcome_total",
        &[("outcome", outcome), ("needs_persons", "true")],
    )
}

impl Counts {
    fn read(proxy: &PersonsDbProxy) -> Self {
        Self {
            persons_connections: counter(
                "db_connection_created_total",
                &[("pool", "persons_reader")],
            ),
            connect_attempts: proxy.accepted(),
            degraded_persons_requests: persons_requests("degraded"),
            ok_persons_requests: persons_requests("ok"),
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum RequestKind {
    BothFlags,
    /// Only the database-free flag, through `flag_keys`, to time database-free work alone.
    DbFreeOnly,
}

impl RequestKind {
    fn body(self, token: &str) -> String {
        match self {
            RequestKind::BothFlags => json!({"token": token, "distinct_id": DISTINCT_ID}),
            RequestKind::DbFreeOnly => {
                json!({"token": token, "distinct_id": DISTINCT_ID, "flag_keys": [DB_FREE_FLAG]})
            }
        }
        .to_string()
    }

    /// What every request of this kind answers while the persons DB is down.
    fn answer_during_fault(self) -> Answer {
        match self {
            RequestKind::BothFlags => Answer::PersonFlagFailed,
            RequestKind::DbFreeOnly => Answer::Healthy,
        }
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
enum Answer {
    /// 200, and every requested flag evaluated.
    Healthy,
    /// 200, the database-free flag evaluated, and the person flag failed with
    /// `errorsWhileComputingFlags` set.
    PersonFlagFailed,
    Unexpected(String),
}

impl Answer {
    async fn read(kind: RequestKind, response: reqwest::Result<reqwest::Response>) -> Self {
        let response = match response {
            Ok(response) if response.status() == StatusCode::OK => response,
            Ok(response) => return Self::Unexpected(format!("HTTP {}", response.status())),
            Err(e) => return Self::Unexpected(e.to_string()),
        };
        let flags = match response.json::<FlagsResponse>().await {
            Ok(flags) => flags,
            Err(e) => return Self::Unexpected(e.to_string()),
        };
        let evaluated = |flag: &FlagDetails| flag.enabled && !flag.failed;
        let db_free = flags.flags.get(DB_FREE_FLAG);
        let person = flags.flags.get(PERSON_FLAG);
        let person_evaluated = match kind {
            RequestKind::BothFlags => person.is_some_and(evaluated),
            RequestKind::DbFreeOnly => person.is_none(),
        };
        match (db_free, person, flags.errors_while_computing_flags) {
            (Some(db_free), _, false) if evaluated(db_free) && person_evaluated => Self::Healthy,
            (Some(db_free), Some(person), true) if evaluated(db_free) && person.failed => {
                Self::PersonFlagFailed
            }
            (db_free, person, errors) => {
                Self::Unexpected(format!("{db_free:?} {person:?} errors={errors}"))
            }
        }
    }
}

struct Sample {
    kind: RequestKind,
    started: Instant,
    finished: Instant,
    answer: Answer,
}

/// When the first healthy request for both flags that started at or after `since` finished.
fn first_healthy_since(samples: &[Sample], since: Instant) -> Option<Instant> {
    samples
        .iter()
        .filter(|s| {
            s.kind == RequestKind::BothFlags && s.started >= since && s.answer == Answer::Healthy
        })
        .map(|s| s.finished)
        .min()
}

/// `clients` loop requests for both flags, and one more client loops database-free requests.
struct Load {
    samples: Arc<Mutex<Vec<Sample>>>,
    stop: CancellationToken,
    clients: Vec<JoinHandle<()>>,
}

impl Load {
    fn start(addr: SocketAddr, token: &str, clients: usize) -> Self {
        let http = reqwest::Client::builder()
            .timeout(Duration::from_secs(10))
            .build()
            .unwrap();
        let url = format!("http://{addr}/flags?v=2");
        let samples: Arc<Mutex<Vec<Sample>>> = Arc::default();
        let stop = CancellationToken::new();
        let clients = std::iter::repeat_n(RequestKind::BothFlags, clients)
            .chain([RequestKind::DbFreeOnly])
            .map(|kind| {
                let (http, url, body) = (http.clone(), url.clone(), kind.body(token));
                let (samples, stop) = (samples.clone(), stop.clone());
                tokio::spawn(async move {
                    while !stop.is_cancelled() {
                        let started = Instant::now();
                        let response = http
                            .post(&url)
                            .header(CONTENT_TYPE, "application/json")
                            .body(body.clone())
                            .send()
                            .await;
                        let answer = Answer::read(kind, response).await;
                        samples.lock().unwrap().push(Sample {
                            kind,
                            started,
                            finished: Instant::now(),
                            answer,
                        });
                    }
                })
            })
            .collect();
        Self {
            samples,
            stop,
            clients,
        }
    }

    async fn wait_until_healthy(&self, since: Instant, limit: Duration) -> bool {
        while Instant::now() < since + limit {
            if first_healthy_since(&self.samples.lock().unwrap(), since).is_some() {
                return true;
            }
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
        false
    }

    async fn stop(self) -> Vec<Sample> {
        self.stop.cancel();
        for client in self.clients {
            client.await.unwrap();
        }
        std::mem::take(&mut *self.samples.lock().unwrap())
    }
}

struct Report {
    fault: Fault,
    fault_start: Instant,
    fault_end: Instant,
    samples: Vec<Sample>,
    at_start: Counts,
    at_end: Counts,
    at_finish: Counts,
}

impl Report {
    fn before(&self) -> impl Iterator<Item = &Sample> {
        self.samples
            .iter()
            .filter(|s| s.finished <= self.fault_start)
    }

    fn during(&self) -> impl Iterator<Item = &Sample> {
        self.samples
            .iter()
            .filter(|s| s.started >= self.fault_start && s.finished <= self.fault_end)
    }

    fn recovery(&self) -> Option<Duration> {
        first_healthy_since(&self.samples, self.fault_end).map(|finished| finished - self.fault_end)
    }

    fn connections_opened_during(&self) -> u64 {
        self.at_end.persons_connections - self.at_start.persons_connections
    }

    fn connections_opened_after(&self) -> u64 {
        self.at_finish.persons_connections - self.at_end.persons_connections
    }

    fn connect_attempts_during(&self) -> usize {
        self.at_end.connect_attempts - self.at_start.connect_attempts
    }

    fn print(&self) {
        let db_free = |s: &&Sample| s.kind == RequestKind::DbFreeOnly;
        let both_flags: Vec<_> = self
            .during()
            .filter(|s| s.kind == RequestKind::BothFlags)
            .collect();
        let failed = both_flags
            .iter()
            .filter(|s| s.answer == Answer::PersonFlagFailed)
            .count();
        println!(
            "{:?} for {:?}",
            self.fault,
            self.fault_end - self.fault_start
        );
        println!(
            "  database-free latency: before {}, during {}",
            latency_summary(self.before().filter(db_free)),
            latency_summary(self.during().filter(db_free))
        );
        println!(
            "  person flag failed in {failed} of {} requests during the fault",
            both_flags.len()
        );
        println!("  first success after the fault: {:?}", self.recovery());
        println!(
            "  persons_reader connections opened: {} during, {} after",
            self.connections_opened_during(),
            self.connections_opened_after()
        );
        println!(
            "  connect attempts at the proxy: {} during, {} after",
            self.connect_attempts_during(),
            self.at_finish.connect_attempts - self.at_end.connect_attempts
        );
    }

    /// Holds today and after every later change to the persons data path.
    fn assert_degrades_then_recovers(&self) {
        let mut during = self.during().peekable();
        assert!(during.peek().is_some(), "no request ran inside the fault");
        for sample in during {
            assert_eq!(
                sample.answer,
                sample.kind.answer_during_fault(),
                "{:?} request {:?} into the fault",
                sample.kind,
                sample.started - self.fault_start
            );
        }

        let recovery = self.recovery();
        assert!(
            recovery.is_some_and(|recovery| recovery <= RECOVERY_LIMIT),
            "no success within {RECOVERY_LIMIT:?} of the fault ending: {recovery:?}"
        );

        assert!(
            self.at_end.degraded_persons_requests > self.at_start.degraded_persons_requests,
            "no degraded persons request counted during the fault"
        );
        assert!(
            self.at_finish.ok_persons_requests > self.at_end.ok_persons_requests,
            "no ok persons request counted after the fault"
        );
    }

    fn assert_connections_reopen(&self) {
        assert!(
            self.connections_opened_during() + self.connections_opened_after() > 0,
            "today the pool reopens persons connections; #122930 flips this"
        );
    }
}

fn latency_summary<'a>(samples: impl Iterator<Item = &'a Sample>) -> String {
    let mut latencies: Vec<Duration> = samples.map(|s| s.finished - s.started).collect();
    latencies.sort();
    match (latencies.get(latencies.len() / 2), latencies.last()) {
        (Some(p50), Some(max)) => {
            format!("p50 {p50:?}, max {max:?} over {} requests", latencies.len())
        }
        _ => "no requests".to_string(),
    }
}