Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.9 KB
Raw Blame
use std::net::SocketAddr;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;

use common_cache::{CacheConfig, NegativeCache, ReadThroughCache, ReadThroughCacheWithMetrics};
use common_database::get_pool;
use common_hypercache::{HyperCacheConfig, HyperCacheReader};
use common_redis::MockRedisClient;
use feature_flags::team::team_models::Team;
use feature_flags::utils::test_utils::team_token_hypercache_key;
use governor::clock;
use lifecycle::{Manager, ManagerBuilder, PrestopAction};
use limiters::redis::QUOTA_LIMITER_CACHE_KEY;
use reqwest::header::CONTENT_TYPE;
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;

use feature_flags::cohorts::membership::NoOpCohortMembershipProvider;
use feature_flags::config::Config;
use feature_flags::rayon_dispatcher::RayonDispatcher;
use feature_flags::server::{register_components, serve_with_rate_limiter_clock};

pub struct ServerHandle {
    pub addr: SocketAddr,
    shutdown: CancellationToken,
}

impl ServerHandle {
    /// Cancel the shutdown token. Drop also does this; use only for explicit
    /// early shutdown.
    #[allow(dead_code)]
    pub fn shutdown_now(&self) {
        self.shutdown.cancel();
    }
}

fn test_manager_builder() -> ManagerBuilder {
    Manager::builder("feature-flags-test")
        .with_trap_signals(false)
        .with_prestop_check(false)
        .with_global_shutdown_timeout(Duration::from_secs(10))
}

impl ServerHandle {
    pub async fn for_config(config: Config) -> ServerHandle {
        Self::for_config_with_s3(config, None).await
    }

    /// Like `for_config`, but injects an S3 client for the flags-with-cohorts reader.
    /// Pass a dummy that returns NotFound to make a cache miss classify as CacheMiss
    /// (the flags Rust test job has no object store, so real S3 reads error out).
    #[allow(dead_code)]
    pub async fn for_config_with_s3(
        config: Config,
        flags_with_cohorts_s3: Option<Arc<dyn common_hypercache::S3Client + Send + Sync>>,
    ) -> ServerHandle {
        Self::for_config_with_s3_and_rate_limiter_clock(
            config,
            test_manager_builder(),
            flags_with_cohorts_s3,
            clock::DefaultClock::default(),
        )
        .await
    }

    /// Like `for_config`, but the server drains once `prestop_path` exists, as in production.
    #[allow(dead_code)]
    pub async fn for_config_with_prestop_drain(
        config: Config,
        prestop_path: &Path,
    ) -> ServerHandle {
        let manager_builder = test_manager_builder()
            .with_prestop_path(prestop_path)
            .with_prestop_action(PrestopAction::Drain {
                // Longer than any test, so the drain limit never starts shutdown mid-test.
                max: Duration::from_secs(60),
            });
        Self::for_config_with_s3_and_rate_limiter_clock(
            config,
            manager_builder,
            None,
            clock::DefaultClock::default(),
        )
        .await
    }

    #[allow(dead_code)]
    pub async fn for_config_with_rate_limiter_clock<C>(
        config: Config,
        rate_limiter_clock: C,
    ) -> ServerHandle
    where
        C: clock::Clock + Clone + Send + Sync + 'static,
    {
        Self::for_config_with_s3_and_rate_limiter_clock(
            config,
            test_manager_builder(),
            None,
            rate_limiter_clock,
        )
        .await
    }

    async fn for_config_with_s3_and_rate_limiter_clock<C>(
        config: Config,
        manager_builder: ManagerBuilder,
        flags_with_cohorts_s3: Option<Arc<dyn common_hypercache::S3Client + Send + Sync>>,
        rate_limiter_clock: C,
    ) -> ServerHandle
    where
        C: clock::Clock + Clone + Send + Sync + 'static,
    {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        let shutdown = CancellationToken::new();
        let mut manager = manager_builder
            .with_shutdown_token(shutdown.clone())
            .build();
        let handles = register_components(&mut manager);
        let monitor_guard = manager.monitor_background();

        let rayon_dispatcher = RayonDispatcher::new(2, None);
        tokio::spawn(async move {
            serve_with_rate_limiter_clock(
                config,
                listener,
                rayon_dispatcher,
                handles,
                flags_with_cohorts_s3,
                rate_limiter_clock,
            )
            .await;
            // Drain the lifecycle monitor after serve returns so the supervisor
            // thread exits cleanly. Any error is logged — a failing shutdown
            // shouldn't fail the test unless the test explicitly asserts on it.
            if let Err(e) = monitor_guard.wait().await {
                tracing::warn!("test lifecycle monitor reported: {e}");
            }
        });
        ServerHandle { addr, shutdown }
    }

    /// Poll the server's readiness endpoint until it responds successfully.
    /// Panics if the server doesn't become ready within 5 seconds.
    #[allow(dead_code)]
    pub async fn wait_until_ready(&self) {
        let client = reqwest::Client::new();
        let url = format!("http://{}/_readiness", self.addr);
        for _ in 0..100 {
            match client.get(&url).send().await {
                Ok(resp) if resp.status().is_success() => return,
                _ => tokio::time::sleep(std::time::Duration::from_millis(50)).await,
            }
        }
        panic!("Server failed to become ready within 5 seconds");
    }

    #[allow(dead_code)]
    pub async fn for_config_with_mock_redis(
        config: Config,
        limited_tokens: Vec<String>,
        valid_tokens: Vec<(String, i32)>, // (token, team_id) pairs
    ) -> ServerHandle {
        Self::for_config_with_mock_redis_and_recordings(
            config,
            limited_tokens,
            vec![], // no recordings-limited tokens
            valid_tokens,
        )
        .await
    }

    /// Create a server with mock Redis that supports both feature flag and recordings quota limits.
    #[allow(dead_code)]
    pub async fn for_config_with_mock_redis_and_recordings(
        config: Config,
        limited_tokens: Vec<String>,            // feature flags limited
        recordings_limited_tokens: Vec<String>, // session recordings limited
        valid_tokens: Vec<(String, i32)>,       // (token, team_id) pairs
    ) -> ServerHandle {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        let shutdown = CancellationToken::new();
        let mut manager = test_manager_builder()
            .with_shutdown_token(shutdown.clone())
            .build();
        // `monitor_background()` is intentionally not called in this path:
        // ReadinessHandler reads the token directly and `trap_signals=false`
        // means no supervisor is required. `manager` is dropped at the end of
        // this function — before the spawned task runs — which closes the
        // receiver for the supervision channel. That's safe here because
        // shutdown is driven by the `CancellationToken` (via
        // `handles.http.shutdown_signal()`), not by the manager loop, and the
        // lifecycle crate intentionally ignores send errors from
        // `work_completed()` and handle drops. If that ever changes (e.g. a
        // channel-full panic, or handles relying on a manager ACK), this test
        // harness will need to keep `manager` alive for the spawn's lifetime.
        let handles = register_components(&mut manager);

        // Create a mock client that handles both quota limit checks and token verification
        let mut mock_client = MockRedisClient::new()
            .zrangebyscore_ret(
                "@posthog/quota-limits/feature_flag_requests",
                limited_tokens.clone(),
            )
            .zrangebyscore_ret(
                "@posthog/quota-limits/recordings",
                recordings_limited_tokens.clone(),
            );

        // Add handling for token verification using HyperCache format
        for (token, team_id) in valid_tokens {
            let cache_key = team_token_hypercache_key(&token);
            println!("Setting up mock for token: {token} with key: {cache_key}");

            // Create a minimal valid Team object
            let team = Team {
                id: team_id,
                name: "Test Team".to_string(),
                api_token: token.clone(),
                cookieless_server_hash_mode: Some(0),
                timezone: "UTC".to_string(),
                ..Default::default()
            };

            // Serialize to JSON, then Pickle-encode it (matching HyperCache format)
            let team_json = serde_json::to_string(&team).unwrap();
            println!("Team JSON for mock: {team_json}");
            let pickled_bytes = serde_pickle::ser::to_vec(&team_json, Default::default()).unwrap();

            mock_client = mock_client.get_raw_bytes_ret(&cache_key, Ok(pickled_bytes));
        }

        tokio::spawn(async move {
            let redis_reader_client: Arc<dyn common_redis::Client + Send + Sync> =
                Arc::new(mock_client);
            let redis_writer_client = redis_reader_client.clone();

            let (persons_reader, persons_writer, non_persons_reader, non_persons_writer) = if config
                .is_persons_db_routing_enabled()
            {
                // Separate persons and non-persons databases
                let persons_reader = match get_pool(
                    &config.get_persons_read_database_url(),
                    config.max_pg_connections,
                ) {
                    Ok(client) => Arc::new(client),
                    Err(e) => {
                        tracing::error!("Failed to create persons read Postgres client: {}", e);
                        return;
                    }
                };
                let persons_writer = match get_pool(
                    &config.get_persons_write_database_url(),
                    config.max_pg_connections,
                ) {
                    Ok(client) => Arc::new(client),
                    Err(e) => {
                        tracing::error!("Failed to create persons write Postgres client: {}", e);
                        return;
                    }
                };
                let non_persons_reader =
                    match get_pool(&config.read_database_url, config.max_pg_connections) {
                        Ok(client) => Arc::new(client),
                        Err(e) => {
                            tracing::error!(
                                "Failed to create non-persons read Postgres client: {}",
                                e
                            );
                            return;
                        }
                    };
                let non_persons_writer =
                    match get_pool(&config.write_database_url, config.max_pg_connections) {
                        Ok(client) => Arc::new(client),
                        Err(e) => {
                            tracing::error!(
                                "Failed to create non-persons write Postgres client: {}",
                                e
                            );
                            return;
                        }
                    };
                (
                    persons_reader,
                    persons_writer,
                    non_persons_reader,
                    non_persons_writer,
                )
            } else {
                // Same database for both persons and non-persons tables
                let reader = match get_pool(&config.read_database_url, config.max_pg_connections) {
                    Ok(client) => Arc::new(client),
                    Err(e) => {
                        tracing::error!("Failed to create read Postgres client: {}", e);
                        return;
                    }
                };
                let writer = match get_pool(&config.write_database_url, config.max_pg_connections) {
                    Ok(client) => Arc::new(client),
                    Err(e) => {
                        tracing::error!("Failed to create write Postgres client: {}", e);
                        return;
                    }
                };
                (
                    reader.clone(),
                    writer.clone(),
                    reader.clone(),
                    writer.clone(),
                )
            };

            let geoip_service = match common_geoip::GeoIpClient::new(config.get_maxmind_db_path()) {
                Ok(service) => Arc::new(service),
                Err(e) => {
                    tracing::error!("Failed to create GeoIP service: {}", e);
                    return;
                }
            };
            let cohort_cache = Arc::new(
                feature_flags::cohorts::cohort_cache_manager::CohortCacheManager::new(
                    non_persons_reader.clone(),
                    Some(config.cohort_cache_capacity_bytes),
                    Some(config.cache_ttl_seconds),
                ),
            );

            let feature_flags_billing_limiter = feature_flags::billing::FeatureFlagsLimiter::new(
                Duration::from_secs(5),
                redis_reader_client.clone(),
                QUOTA_LIMITER_CACHE_KEY.to_string(),
                None,
            )
            .unwrap();

            let session_replay_billing_limiter = feature_flags::billing::SessionReplayLimiter::new(
                Duration::from_secs(5),
                redis_reader_client.clone(),
                QUOTA_LIMITER_CACHE_KEY.to_string(),
                None,
            )
            .unwrap();
            let cookieless_manager = Arc::new(common_cookieless::CookielessManager::new(
                config.get_cookieless_config(),
                redis_reader_client.clone(),
            ));

            // Create DatabasePools for tests
            let database_pools = Arc::new(feature_flags::database_pools::DatabasePools {
                non_persons_reader: non_persons_reader.clone(),
                non_persons_writer: non_persons_writer.clone(),
                persons_reader: persons_reader.clone(),
                persons_writer: persons_writer.clone(),
                behavioral_cohorts_reader: None,
                test_before_acquire: *config.test_before_acquire,
            });

            // Create HyperCacheReader for flags
            let flags_hypercache_config = HyperCacheConfig::new(
                "feature_flags".to_string(),
                "flags.json".to_string(),
                config.object_storage_region.clone(),
                config.object_storage_bucket.clone(),
            );
            let flags_hypercache_reader =
                match HyperCacheReader::new(redis_reader_client.clone(), flags_hypercache_config)
                    .await
                {
                    Ok(reader) => Arc::new(reader),
                    Err(e) => {
                        tracing::error!("Failed to create flags HyperCacheReader: {:?}", e);
                        return;
                    }
                };

            // Create HyperCacheReader for flags with cohorts (used by /flags/definitions endpoint)
            let flags_with_cohorts_hypercache_config = HyperCacheConfig::new(
                "feature_flags".to_string(),
                "flags_with_cohorts.json".to_string(),
                config.object_storage_region.clone(),
                config.object_storage_bucket.clone(),
            );
            let flags_with_cohorts_hypercache_reader = match HyperCacheReader::new(
                redis_reader_client.clone(),
                flags_with_cohorts_hypercache_config,
            )
            .await
            {
                Ok(reader) => Arc::new(reader),
                Err(e) => {
                    tracing::error!(
                        "Failed to create flags_with_cohorts HyperCacheReader: {:?}",
                        e
                    );
                    return;
                }
            };

            // Create team metadata hypercache reader
            let mut team_hypercache_config = HyperCacheConfig::new(
                "team_metadata".to_string(),
                "full_metadata.json".to_string(),
                config.object_storage_region.clone(),
                config.object_storage_bucket.clone(),
            );
            team_hypercache_config.token_based = true;
            let team_hypercache_reader =
                match HyperCacheReader::new(redis_reader_client.clone(), team_hypercache_config)
                    .await
                {
                    Ok(reader) => Arc::new(reader),
                    Err(e) => {
                        tracing::error!("Failed to create team HyperCacheReader: {:?}", e);
                        return;
                    }
                };

            // Create config hypercache reader for remote config (array/config.json)
            let mut config_hypercache_config = HyperCacheConfig::new(
                "array".to_string(),
                "config.json".to_string(),
                config.object_storage_region.clone(),
                config.object_storage_bucket.clone(),
            );
            config_hypercache_config.token_based = true;
            let config_hypercache_reader =
                match HyperCacheReader::new(redis_reader_client.clone(), config_hypercache_config)
                    .await
                {
                    Ok(reader) => Arc::new(reader),
                    Err(e) => {
                        tracing::error!("Failed to create config HyperCacheReader: {:?}", e);
                        return;
                    }
                };

            let group_type_cache = Arc::new(
                feature_flags::flags::flag_group_type_mapping::GroupTypeCacheManager::new(
                    persons_reader.clone(),
                    Some(config.group_type_cache_max_entries),
                    Some(config.group_type_cache_ttl_seconds),
                ),
            );

            let flag_definitions_cache = Arc::new(
                feature_flags::flags::flag_definitions_cache::FlagDefinitionsCache::disabled(),
            );

            let billing_aggregator = feature_flags::billing::BillingAggregator::start(
                redis_writer_client.clone(),
                config.get_billing_aggregator_config(),
            );
            let shutdown_for_test = billing_aggregator.clone();

            let app = feature_flags::router::router(
                redis_writer_client.clone(), // Use writer client for both reads and writes in tests
                None,                        // No dedicated flags Redis in tests
                database_pools,
                cohort_cache,
                group_type_cache,
                geoip_service,
                handles.readiness,
                handles.liveness,
                feature_flags_billing_limiter,
                session_replay_billing_limiter,
                cookieless_manager,
                flags_hypercache_reader,
                flag_definitions_cache,
                flags_with_cohorts_hypercache_reader,
                team_hypercache_reader,
                config_hypercache_reader,
                RayonDispatcher::new(2, None),
                NegativeCache::new(10_000, 300),
                Arc::new(ReadThroughCacheWithMetrics::new(
                    Arc::new(ReadThroughCache::new(
                        redis_writer_client.clone(),
                        redis_writer_client.clone(),
                        CacheConfig::with_ttl(
                            feature_flags::api::auth::TOKEN_CACHE_PREFIX,
                            config.auth_token_cache_ttl_seconds,
                        ),
                        None,
                    )),
                    "auth",
                    "token",
                    &[],
                )),
                Arc::new(NoOpCohortMembershipProvider),
                billing_aggregator,
                config,
            );

            // Mirror `server::serve`: capture the serve result, then await the
            // aggregator shutdown *unconditionally* before propagating any
            // serve error. Otherwise a `.unwrap()` on a serve failure would
            // panic the spawned task and leak the flusher into the next test.
            let serve_result = axum::serve(
                listener,
                app.into_make_service_with_connect_info::<SocketAddr>(),
            )
            .with_graceful_shutdown(handles.http.shutdown_signal())
            .await;
            shutdown_for_test.shutdown().await;
            serve_result.unwrap();
            handles.http.work_completed();
        });

        tokio::time::sleep(std::time::Duration::from_millis(100)).await;

        ServerHandle { addr, shutdown }
    }

    #[allow(dead_code)]
    pub async fn send_flags_request<T: Into<reqwest::Body>>(
        &self,
        body: T,
        version: Option<&str>,
        config: Option<&str>,
    ) -> reqwest::Response {
        let client = reqwest::Client::new();
        let mut url = format!("http://{}/flags", self.addr);
        let mut params = vec![];
        if let Some(v) = version {
            params.push(format!("v={v}"));
        }
        if let Some(c) = config {
            params.push(format!("config={c}"));
        }
        if !params.is_empty() {
            url.push('?');
            url.push_str(&params.join("&"));
        }
        client
            .post(url)
            .body(body)
            .header(CONTENT_TYPE, "application/json")
            .send()
            .await
            .expect("failed to send request")
    }

    #[allow(dead_code)]
    pub async fn send_invalid_header_for_flags_request<T: Into<reqwest::Body>>(
        &self,
        body: T,
    ) -> reqwest::Response {
        let client = reqwest::Client::new();
        client
            .post(format!("http://{:?}/flags", self.addr))
            .body(body)
            .header(CONTENT_TYPE, "xyz")
            .send()
            .await
            .expect("failed to send request")
    }
}

impl Drop for ServerHandle {
    fn drop(&mut self) {
        self.shutdown.cancel();
    }
}

/// Poll for a billing counter across every bucket field in `first_bucket..=last_bucket`,
/// returning the first value found. Panics on timeout. Use for the positive ("the write
/// should land") case in billing aggregator tests — for the negative case, sleep one flush
/// window and check. The server captures the bucket inside `record()` while it handles the
/// request, so a request that straddles a 2-minute bucket boundary lands the increment in
/// either the bucket seen before or after the roundtrip. Pass both ends of the range to poll
/// for the counter without racing the boundary.
#[allow(dead_code)]
pub async fn poll_for_billing_counter_across_buckets(
    client: &Arc<dyn common_redis::Client + Send + Sync>,
    key: &str,
    first_bucket: u64,
    last_bucket: u64,
) -> String {
    for _ in 0..40 {
        for bucket in first_bucket..=last_bucket {
            if let Ok(v) = client.hget(key.to_string(), bucket.to_string()).await {
                return v;
            }
        }
        tokio::time::sleep(Duration::from_millis(25)).await;
    }
    panic!(
        "billing counter at {key} did not appear within 1s for buckets {first_bucket}..={last_bucket}"
    );
}

/// Poll until `last_used_at` is set for the given project secret API key, or panic after ~4s.
#[allow(dead_code)]
pub async fn poll_for_psak_last_used_at(
    context: &feature_flags::utils::test_utils::TestContext,
    key_id: &str,
    message: &str,
) {
    let mut conn = context.get_non_persons_connection().await.unwrap();
    for _ in 0..80 {
        let count: (i64,) = sqlx::query_as(
            "SELECT COUNT(*) FROM posthog_projectsecretapikey WHERE id = $1 AND last_used_at IS NOT NULL",
        )
        .bind(key_id)
        .fetch_one(&mut *conn)
        .await
        .unwrap();
        if count.0 > 0 {
            return;
        }
        tokio::time::sleep(Duration::from_millis(50)).await;
    }
    panic!("{message}");
}